sys/dev/gve/gve_adminq.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-3-Clause
   3  *
   4  * Copyright (c) 2023 Google LLC
   5  *
   6  * Redistribution and use in source and binary forms, with or without modification,
   7  * are permitted provided that the following conditions are met:
   8  *
   9  * 1. Redistributions of source code must retain the above copyright notice, this
  10  *    list of conditions and the following disclaimer.
  11  *
  12  * 2. Redistributions in binary form must reproduce the above copyright notice,
  13  *    this list of conditions and the following disclaimer in the documentation
  14  *    and/or other materials provided with the distribution.
  15  *
  16  * 3. Neither the name of the copyright holder nor the names of its contributors
  17  *    may be used to endorse or promote products derived from this software without
  18  *    specific prior written permission.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
  21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  22  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  23  * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
  24  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  25  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  26  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
  27  * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  29  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30  */
  31 #include <sys/endian.h>
  32 #include <sys/socket.h>
  33 #include <sys/time.h>
  34
  35 #include <net/ethernet.h>
  36 #include <net/if.h>
  37 #include <net/if_var.h>
  38
  39 #include "gve.h"
  40 #include "gve_adminq.h"
  41
  42 #define GVE_ADMINQ_SLEEP_LEN_MS 20
  43 #define GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK 10
  44 #define GVE_ADMINQ_DEVICE_DESCRIPTOR_VERSION 1
  45 #define GVE_REG_ADMINQ_ADDR 16
  46 #define ADMINQ_SLOTS (ADMINQ_SIZE / sizeof(struct gve_adminq_command))
  47
  48 #define GVE_DEVICE_OPTION_ERROR_FMT "%s option error:\n" \
  49     "Expected: length=%d, feature_mask=%x.\n" \
  50     "Actual: length=%d, feature_mask=%x.\n"
  51
  52 #define GVE_DEVICE_OPTION_TOO_BIG_FMT "Length of %s option larger than expected." \
  53     " Possible older version of guest driver.\n"
  54
  55 static
  56 void gve_parse_device_option(struct gve_priv *priv,
  57     struct gve_device_descriptor *device_descriptor,
  58     struct gve_device_option *option,
  59     struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,
  60     struct gve_device_option_jumbo_frames **dev_op_jumbo_frames)
  61 {
  62         uint32_t req_feat_mask = be32toh(option->required_features_mask);
  63         uint16_t option_length = be16toh(option->option_length);
  64         uint16_t option_id = be16toh(option->option_id);
  65
  66         /*
  67          * If the length or feature mask doesn't match, continue without
  68          * enabling the feature.
  69          */
  70         switch (option_id) {
  71         case GVE_DEV_OPT_ID_GQI_QPL:
  72                 if (option_length < sizeof(**dev_op_gqi_qpl) ||
  73                     req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL) {
  74                         device_printf(priv->dev, GVE_DEVICE_OPTION_ERROR_FMT,
  75                             "GQI QPL", (int)sizeof(**dev_op_gqi_qpl),
  76                             GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL,
  77                             option_length, req_feat_mask);
  78                         break;
  79                 }
  80
  81                 if (option_length > sizeof(**dev_op_gqi_qpl)) {
  82                         device_printf(priv->dev, GVE_DEVICE_OPTION_TOO_BIG_FMT,
  83                             "GQI QPL");
  84                 }
  85                 *dev_op_gqi_qpl = (void *)(option + 1);
  86                 break;
  87
  88         case GVE_DEV_OPT_ID_JUMBO_FRAMES:
  89                 if (option_length < sizeof(**dev_op_jumbo_frames) ||
  90                     req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES) {
  91                         device_printf(priv->dev, GVE_DEVICE_OPTION_ERROR_FMT,
  92                             "Jumbo Frames", (int)sizeof(**dev_op_jumbo_frames),
  93                             GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES,
  94                             option_length, req_feat_mask);
  95                         break;
  96                 }
  97
  98                 if (option_length > sizeof(**dev_op_jumbo_frames)) {
  99                         device_printf(priv->dev,
 100                             GVE_DEVICE_OPTION_TOO_BIG_FMT, "Jumbo Frames");
 101                 }
 102                 *dev_op_jumbo_frames = (void *)(option + 1);
 103                 break;
 104
 105         default:
 106                 /*
 107                  * If we don't recognize the option just continue
 108                  * without doing anything.
 109                  */
 110                 device_printf(priv->dev, "Unrecognized device option 0x%hx not enabled.\n",
 111                     option_id);
 112         }
 113 }
 114
 115 /* Process all device options for a given describe device call. */
 116 static int
 117 gve_process_device_options(struct gve_priv *priv,
 118     struct gve_device_descriptor *descriptor,
 119     struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,
 120     struct gve_device_option_jumbo_frames **dev_op_jumbo_frames)
 121 {
 122         char *desc_end = (char *)descriptor + be16toh(descriptor->total_length);
 123         const int num_options = be16toh(descriptor->num_device_options);
 124         struct gve_device_option *dev_opt;
 125         int i;
 126
 127         /* The options struct directly follows the device descriptor. */
 128         dev_opt = (void *)(descriptor + 1);
 129         for (i = 0; i < num_options; i++) {
 130                 if ((char *)(dev_opt + 1) > desc_end ||
 131                     (char *)(dev_opt + 1) + be16toh(dev_opt->option_length) > desc_end) {
 132                         device_printf(priv->dev,
 133                             "options exceed device_descriptor's total length.\n");
 134                         return (EINVAL);
 135                 }
 136
 137                 gve_parse_device_option(priv, descriptor, dev_opt,
 138                     dev_op_gqi_qpl, dev_op_jumbo_frames);
 139                 dev_opt = (void *)((char *)(dev_opt + 1) + be16toh(dev_opt->option_length));
 140         }
 141
 142         return (0);
 143 }
 144
 145 static int gve_adminq_execute_cmd(struct gve_priv *priv,
 146     struct gve_adminq_command *cmd);
 147
 148 static int
 149 gve_adminq_destroy_tx_queue(struct gve_priv *priv, uint32_t id)
 150 {
 151         struct gve_adminq_command cmd = (struct gve_adminq_command){};
 152
 153         cmd.opcode = htobe32(GVE_ADMINQ_DESTROY_TX_QUEUE);
 154         cmd.destroy_tx_queue.queue_id = htobe32(id);
 155
 156         return (gve_adminq_execute_cmd(priv, &cmd));
 157 }
 158
 159 static int
 160 gve_adminq_destroy_rx_queue(struct gve_priv *priv, uint32_t id)
 161 {
 162         struct gve_adminq_command cmd = (struct gve_adminq_command){};
 163
 164         cmd.opcode = htobe32(GVE_ADMINQ_DESTROY_RX_QUEUE);
 165         cmd.destroy_rx_queue.queue_id = htobe32(id);
 166
 167         return (gve_adminq_execute_cmd(priv, &cmd));
 168 }
 169
 170 int
 171 gve_adminq_destroy_rx_queues(struct gve_priv *priv, uint32_t num_queues)
 172 {
 173         int err;
 174         int i;
 175
 176         for (i = 0; i < num_queues; i++) {
 177                 err = gve_adminq_destroy_rx_queue(priv, i);
 178                 if (err != 0) {
 179                         device_printf(priv->dev, "Failed to destroy rxq %d, err: %d\n",
 180                             i, err);
 181                 }
 182         }
 183
 184         if (err != 0)
 185                 return (err);
 186
 187         device_printf(priv->dev, "Destroyed %d rx queues\n", num_queues);
 188         return (0);
 189 }
 190
 191 int
 192 gve_adminq_destroy_tx_queues(struct gve_priv *priv, uint32_t num_queues)
 193 {
 194         int err;
 195         int i;
 196
 197         for (i = 0; i < num_queues; i++) {
 198                 err = gve_adminq_destroy_tx_queue(priv, i);
 199                 if (err != 0) {
 200                         device_printf(priv->dev, "Failed to destroy txq %d, err: %d\n",
 201                             i, err);
 202                 }
 203         }
 204
 205         if (err != 0)
 206                 return (err);
 207
 208         device_printf(priv->dev, "Destroyed %d tx queues\n", num_queues);
 209         return (0);
 210 }
 211
 212 static int
 213 gve_adminq_create_rx_queue(struct gve_priv *priv, uint32_t queue_index)
 214 {
 215         struct gve_adminq_command cmd = (struct gve_adminq_command){};
 216         struct gve_rx_ring *rx = &priv->rx[queue_index];
 217         struct gve_dma_handle *qres_dma = &rx->com.q_resources_mem;
 218
 219         bus_dmamap_sync(qres_dma->tag, qres_dma->map, BUS_DMASYNC_PREREAD);
 220
 221         cmd.opcode = htobe32(GVE_ADMINQ_CREATE_RX_QUEUE);
 222         cmd.create_rx_queue = (struct gve_adminq_create_rx_queue) {
 223                 .queue_id = htobe32(queue_index),
 224                 .index = htobe32(queue_index),
 225                 .ntfy_id = htobe32(rx->com.ntfy_id),
 226                 .queue_resources_addr = htobe64(qres_dma->bus_addr),
 227                 .rx_desc_ring_addr = htobe64(rx->desc_ring_mem.bus_addr),
 228                 .rx_data_ring_addr = htobe64(rx->data_ring_mem.bus_addr),
 229                 .queue_page_list_id = htobe32((rx->com.qpl)->id),
 230                 .rx_ring_size = htobe16(priv->rx_desc_cnt),
 231                 .packet_buffer_size = htobe16(GVE_DEFAULT_RX_BUFFER_SIZE),
 232         };
 233
 234         return (gve_adminq_execute_cmd(priv, &cmd));
 235 }
 236
 237 int
 238 gve_adminq_create_rx_queues(struct gve_priv *priv, uint32_t num_queues)
 239 {
 240         int err;
 241         int i;
 242
 243         for (i = 0; i < num_queues; i++) {
 244                 err = gve_adminq_create_rx_queue(priv, i);
 245                 if (err != 0) {
 246                         device_printf(priv->dev, "Failed to create rxq %d, err: %d\n",
 247                             i, err);
 248                         goto abort;
 249                 }
 250         }
 251
 252         if (bootverbose)
 253                 device_printf(priv->dev, "Created %d rx queues\n", num_queues);
 254         return (0);
 255
 256 abort:
 257         gve_adminq_destroy_rx_queues(priv, i);
 258         return (err);
 259 }
 260
 261 static int
 262 gve_adminq_create_tx_queue(struct gve_priv *priv, uint32_t queue_index)
 263 {
 264         struct gve_adminq_command cmd = (struct gve_adminq_command){};
 265         struct gve_tx_ring *tx = &priv->tx[queue_index];
 266         struct gve_dma_handle *qres_dma = &tx->com.q_resources_mem;
 267
 268         bus_dmamap_sync(qres_dma->tag, qres_dma->map, BUS_DMASYNC_PREREAD);
 269
 270         cmd.opcode = htobe32(GVE_ADMINQ_CREATE_TX_QUEUE);
 271         cmd.create_tx_queue = (struct gve_adminq_create_tx_queue) {
 272                 .queue_id = htobe32(queue_index),
 273                 .queue_resources_addr = htobe64(qres_dma->bus_addr),
 274                 .tx_ring_addr = htobe64(tx->desc_ring_mem.bus_addr),
 275                 .queue_page_list_id = htobe32((tx->com.qpl)->id),
 276                 .ntfy_id = htobe32(tx->com.ntfy_id),
 277                 .tx_ring_size = htobe16(priv->tx_desc_cnt),
 278         };
 279
 280         return (gve_adminq_execute_cmd(priv, &cmd));
 281 }
 282
 283 int
 284 gve_adminq_create_tx_queues(struct gve_priv *priv, uint32_t num_queues)
 285 {
 286         int err;
 287         int i;
 288
 289         for (i = 0; i < num_queues; i++) {
 290                 err = gve_adminq_create_tx_queue(priv, i);
 291                 if (err != 0) {
 292                         device_printf(priv->dev, "Failed to create txq %d, err: %d\n",
 293                             i, err);
 294                         goto abort;
 295                 }
 296         }
 297
 298         if (bootverbose)
 299                 device_printf(priv->dev, "Created %d tx queues\n", num_queues);
 300         return (0);
 301
 302 abort:
 303         gve_adminq_destroy_tx_queues(priv, i);
 304         return (err);
 305 }
 306
 307 int
 308 gve_adminq_set_mtu(struct gve_priv *priv, uint32_t mtu) {
 309         struct gve_adminq_command cmd = (struct gve_adminq_command){};
 310
 311         cmd.opcode = htobe32(GVE_ADMINQ_SET_DRIVER_PARAMETER);
 312         cmd.set_driver_param = (struct gve_adminq_set_driver_parameter) {
 313                 .parameter_type = htobe32(GVE_SET_PARAM_MTU),
 314                 .parameter_value = htobe64(mtu),
 315         };
 316
 317         return (gve_adminq_execute_cmd(priv, &cmd));
 318 }
 319
 320 static void
 321 gve_enable_supported_features(struct gve_priv *priv,
 322     uint32_t supported_features_mask,
 323     const struct gve_device_option_jumbo_frames *dev_op_jumbo_frames)
 324 {
 325         if (dev_op_jumbo_frames &&
 326             (supported_features_mask & GVE_SUP_JUMBO_FRAMES_MASK)) {
 327                 if (bootverbose)
 328                         device_printf(priv->dev, "JUMBO FRAMES device option enabled: %u.\n",
 329                             be16toh(dev_op_jumbo_frames->max_mtu));
 330                 priv->max_mtu = be16toh(dev_op_jumbo_frames->max_mtu);
 331         }
 332 }
 333
 334 int
 335 gve_adminq_describe_device(struct gve_priv *priv)
 336 {
 337         struct gve_adminq_command aq_cmd = (struct gve_adminq_command){};
 338         struct gve_device_descriptor *desc;
 339         struct gve_dma_handle desc_mem;
 340         struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL;
 341         struct gve_device_option_jumbo_frames *dev_op_jumbo_frames = NULL;
 342         uint32_t supported_features_mask = 0;
 343         int rc;
 344         int i;
 345
 346         rc = gve_dma_alloc_coherent(priv, ADMINQ_SIZE, ADMINQ_SIZE, &desc_mem);
 347         if (rc != 0) {
 348                 device_printf(priv->dev, "Failed to alloc DMA mem for DescribeDevice.\n");
 349                 return (rc);
 350         }
 351
 352         desc = desc_mem.cpu_addr;
 353
 354         aq_cmd.opcode = htobe32(GVE_ADMINQ_DESCRIBE_DEVICE);
 355         aq_cmd.describe_device.device_descriptor_addr = htobe64(
 356             desc_mem.bus_addr);
 357         aq_cmd.describe_device.device_descriptor_version = htobe32(
 358             GVE_ADMINQ_DEVICE_DESCRIPTOR_VERSION);
 359         aq_cmd.describe_device.available_length = htobe32(ADMINQ_SIZE);
 360
 361         bus_dmamap_sync(desc_mem.tag, desc_mem.map, BUS_DMASYNC_PREWRITE);
 362
 363         rc = gve_adminq_execute_cmd(priv, &aq_cmd);
 364         if (rc != 0)
 365                 goto free_device_descriptor;
 366
 367         bus_dmamap_sync(desc_mem.tag, desc_mem.map, BUS_DMASYNC_POSTREAD);
 368
 369         rc = gve_process_device_options(priv, desc, &dev_op_gqi_qpl,
 370             &dev_op_jumbo_frames);
 371         if (rc != 0)
 372                 goto free_device_descriptor;
 373
 374         if (dev_op_gqi_qpl != NULL) {
 375                 priv->queue_format = GVE_GQI_QPL_FORMAT;
 376                 supported_features_mask = be32toh(
 377                     dev_op_gqi_qpl->supported_features_mask);
 378                 if (bootverbose)
 379                         device_printf(priv->dev,
 380                             "Driver is running with GQI QPL queue format.\n");
 381         } else {
 382                 device_printf(priv->dev, "No compatible queue formats\n");
 383                 rc = (EINVAL);
 384                 goto free_device_descriptor;
 385         }
 386
 387         priv->num_event_counters = be16toh(desc->counters);
 388         priv->default_num_queues = be16toh(desc->default_num_queues);
 389         priv->tx_desc_cnt = be16toh(desc->tx_queue_entries);
 390         priv->rx_desc_cnt = be16toh(desc->rx_queue_entries);
 391         priv->rx_pages_per_qpl = be16toh(desc->rx_pages_per_qpl);
 392         priv->max_registered_pages = be64toh(desc->max_registered_pages);
 393         priv->max_mtu = be16toh(desc->mtu);
 394         priv->default_num_queues = be16toh(desc->default_num_queues);
 395         priv->supported_features =  supported_features_mask;
 396
 397         gve_enable_supported_features(priv, supported_features_mask,
 398             dev_op_jumbo_frames);
 399
 400         for (i = 0; i < ETHER_ADDR_LEN; i++)
 401                 priv->mac[i] = desc->mac[i];
 402
 403 free_device_descriptor:
 404         gve_dma_free_coherent(&desc_mem);
 405
 406         return (rc);
 407 }
 408
 409 int
 410 gve_adminq_register_page_list(struct gve_priv *priv,
 411     struct gve_queue_page_list *qpl)
 412 {
 413         struct gve_adminq_command cmd = (struct gve_adminq_command){};
 414         uint32_t num_entries = qpl->num_pages;
 415         uint32_t size = num_entries * sizeof(qpl->dmas[0].bus_addr);
 416         __be64 *page_list;
 417         struct gve_dma_handle dma;
 418         int err;
 419         int i;
 420
 421         err = gve_dma_alloc_coherent(priv, size, PAGE_SIZE, &dma);
 422         if (err != 0)
 423                 return (ENOMEM);
 424
 425         page_list = dma.cpu_addr;
 426
 427         for (i = 0; i < num_entries; i++)
 428                 page_list[i] = htobe64(qpl->dmas[i].bus_addr);
 429
 430         bus_dmamap_sync(dma.tag, dma.map, BUS_DMASYNC_PREWRITE);
 431
 432         cmd.opcode = htobe32(GVE_ADMINQ_REGISTER_PAGE_LIST);
 433         cmd.reg_page_list = (struct gve_adminq_register_page_list) {
 434                 .page_list_id = htobe32(qpl->id),
 435                 .num_pages = htobe32(num_entries),
 436                 .page_address_list_addr = htobe64(dma.bus_addr),
 437                 .page_size = htobe64(PAGE_SIZE),
 438         };
 439
 440         err = gve_adminq_execute_cmd(priv, &cmd);
 441         gve_dma_free_coherent(&dma);
 442         return (err);
 443 }
 444
 445 int
 446 gve_adminq_unregister_page_list(struct gve_priv *priv, uint32_t page_list_id)
 447 {
 448         struct gve_adminq_command cmd = (struct gve_adminq_command){};
 449
 450         cmd.opcode = htobe32(GVE_ADMINQ_UNREGISTER_PAGE_LIST);
 451         cmd.unreg_page_list = (struct gve_adminq_unregister_page_list) {
 452                 .page_list_id = htobe32(page_list_id),
 453         };
 454
 455         return (gve_adminq_execute_cmd(priv, &cmd));
 456 }
 457
 458 #define GVE_NTFY_BLK_BASE_MSIX_IDX      0
 459 int
 460 gve_adminq_configure_device_resources(struct gve_priv *priv)
 461 {
 462         struct gve_adminq_command aq_cmd = (struct gve_adminq_command){};
 463
 464         bus_dmamap_sync(priv->irqs_db_mem.tag, priv->irqs_db_mem.map,
 465             BUS_DMASYNC_PREREAD);
 466         bus_dmamap_sync(priv->counter_array_mem.tag,
 467             priv->counter_array_mem.map, BUS_DMASYNC_PREREAD);
 468
 469         aq_cmd.opcode = htobe32(GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES);
 470         aq_cmd.configure_device_resources =
 471             (struct gve_adminq_configure_device_resources) {
 472                 .counter_array = htobe64(priv->counter_array_mem.bus_addr),
 473                 .irq_db_addr = htobe64(priv->irqs_db_mem.bus_addr),
 474                 .num_counters = htobe32(priv->num_event_counters),
 475                 .num_irq_dbs = htobe32(priv->num_queues),
 476                 .irq_db_stride = htobe32(sizeof(struct gve_irq_db)),
 477                 .ntfy_blk_msix_base_idx = htobe32(GVE_NTFY_BLK_BASE_MSIX_IDX),
 478                 .queue_format = priv->queue_format,
 479         };
 480
 481         return (gve_adminq_execute_cmd(priv, &aq_cmd));
 482 }
 483
 484 int
 485 gve_adminq_deconfigure_device_resources(struct gve_priv *priv)
 486 {
 487         struct gve_adminq_command aq_cmd = (struct gve_adminq_command){};
 488
 489         aq_cmd.opcode = htobe32(GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES);
 490         return (gve_adminq_execute_cmd(priv, &aq_cmd));
 491 }
 492
 493 int
 494 gve_adminq_verify_driver_compatibility(struct gve_priv *priv,
 495     uint64_t driver_info_len,
 496     vm_paddr_t driver_info_addr)
 497 {
 498         struct gve_adminq_command aq_cmd = (struct gve_adminq_command){};
 499
 500         aq_cmd.opcode = htobe32(GVE_ADMINQ_VERIFY_DRIVER_COMPATIBILITY);
 501         aq_cmd.verify_driver_compatibility = (struct gve_adminq_verify_driver_compatibility) {
 502                 .driver_info_len = htobe64(driver_info_len),
 503                 .driver_info_addr = htobe64(driver_info_addr),
 504         };
 505
 506         return (gve_adminq_execute_cmd(priv, &aq_cmd));
 507 }
 508
 509 int
 510 gve_adminq_alloc(struct gve_priv *priv)
 511 {
 512         int rc;
 513
 514         if (gve_get_state_flag(priv, GVE_STATE_FLAG_ADMINQ_OK))
 515                 return (0);
 516
 517         if (priv->aq_mem.cpu_addr == NULL) {
 518                 rc = gve_dma_alloc_coherent(priv, ADMINQ_SIZE, ADMINQ_SIZE,
 519                     &priv->aq_mem);
 520                 if (rc != 0) {
 521                         device_printf(priv->dev, "Failed to allocate admin queue mem\n");
 522                         return (rc);
 523                 }
 524         }
 525
 526         priv->adminq = priv->aq_mem.cpu_addr;
 527         priv->adminq_bus_addr = priv->aq_mem.bus_addr;
 528
 529         if (priv->adminq == NULL)
 530                 return (ENOMEM);
 531
 532         priv->adminq_mask = ADMINQ_SLOTS - 1;
 533         priv->adminq_prod_cnt = 0;
 534         priv->adminq_cmd_fail = 0;
 535         priv->adminq_timeouts = 0;
 536         priv->adminq_describe_device_cnt = 0;
 537         priv->adminq_cfg_device_resources_cnt = 0;
 538         priv->adminq_register_page_list_cnt = 0;
 539         priv->adminq_unregister_page_list_cnt = 0;
 540         priv->adminq_create_tx_queue_cnt = 0;
 541         priv->adminq_create_rx_queue_cnt = 0;
 542         priv->adminq_destroy_tx_queue_cnt = 0;
 543         priv->adminq_destroy_rx_queue_cnt = 0;
 544         priv->adminq_dcfg_device_resources_cnt = 0;
 545         priv->adminq_set_driver_parameter_cnt = 0;
 546
 547         gve_reg_bar_write_4(priv, GVE_REG_ADMINQ_ADDR,
 548             priv->adminq_bus_addr / ADMINQ_SIZE);
 549
 550         gve_set_state_flag(priv, GVE_STATE_FLAG_ADMINQ_OK);
 551         return (0);
 552 }
 553
 554 void
 555 gve_release_adminq(struct gve_priv *priv)
 556 {
 557         if (!gve_get_state_flag(priv, GVE_STATE_FLAG_ADMINQ_OK))
 558                 return;
 559
 560         gve_reg_bar_write_4(priv, GVE_REG_ADMINQ_ADDR, 0);
 561         while (gve_reg_bar_read_4(priv, GVE_REG_ADMINQ_ADDR)) {
 562                 device_printf(priv->dev, "Waiting until admin queue is released.\n");
 563                 pause("gve release adminq", GVE_ADMINQ_SLEEP_LEN_MS);
 564         }
 565
 566         gve_dma_free_coherent(&priv->aq_mem);
 567         priv->aq_mem = (struct gve_dma_handle){};
 568         priv->adminq = 0;
 569         priv->adminq_bus_addr = 0;
 570
 571         gve_clear_state_flag(priv, GVE_STATE_FLAG_ADMINQ_OK);
 572
 573         if (bootverbose)
 574                 device_printf(priv->dev, "Admin queue released\n");
 575 }
 576
 577 static int
 578 gve_adminq_parse_err(struct gve_priv *priv, uint32_t opcode, uint32_t status)
 579 {
 580         if (status != GVE_ADMINQ_COMMAND_PASSED &&
 581             status != GVE_ADMINQ_COMMAND_UNSET) {
 582                 device_printf(priv->dev, "AQ command(%u): failed with status %d\n", opcode, status);
 583                 priv->adminq_cmd_fail++;
 584         }
 585         switch (status) {
 586         case GVE_ADMINQ_COMMAND_PASSED:
 587                 return (0);
 588
 589         case GVE_ADMINQ_COMMAND_UNSET:
 590                 device_printf(priv->dev,
 591                     "AQ command(%u): err and status both unset, this should not be possible.\n",
 592                     opcode);
 593                 return (EINVAL);
 594
 595         case GVE_ADMINQ_COMMAND_ERROR_ABORTED:
 596         case GVE_ADMINQ_COMMAND_ERROR_CANCELLED:
 597         case GVE_ADMINQ_COMMAND_ERROR_DATALOSS:
 598         case GVE_ADMINQ_COMMAND_ERROR_FAILED_PRECONDITION:
 599         case GVE_ADMINQ_COMMAND_ERROR_UNAVAILABLE:
 600                 return (EAGAIN);
 601
 602         case GVE_ADMINQ_COMMAND_ERROR_ALREADY_EXISTS:
 603         case GVE_ADMINQ_COMMAND_ERROR_INTERNAL_ERROR:
 604         case GVE_ADMINQ_COMMAND_ERROR_INVALID_ARGUMENT:
 605         case GVE_ADMINQ_COMMAND_ERROR_NOT_FOUND:
 606         case GVE_ADMINQ_COMMAND_ERROR_OUT_OF_RANGE:
 607         case GVE_ADMINQ_COMMAND_ERROR_UNKNOWN_ERROR:
 608                 return (EINVAL);
 609
 610         case GVE_ADMINQ_COMMAND_ERROR_DEADLINE_EXCEEDED:
 611                 return (ETIMEDOUT);
 612
 613         case GVE_ADMINQ_COMMAND_ERROR_PERMISSION_DENIED:
 614         case GVE_ADMINQ_COMMAND_ERROR_UNAUTHENTICATED:
 615                 return (EACCES);
 616
 617         case GVE_ADMINQ_COMMAND_ERROR_RESOURCE_EXHAUSTED:
 618                 return (ENOMEM);
 619
 620         case GVE_ADMINQ_COMMAND_ERROR_UNIMPLEMENTED:
 621                 return (EOPNOTSUPP);
 622
 623         default:
 624                 device_printf(priv->dev, "AQ command(%u): unknown status code %d\n",
 625                     opcode, status);
 626                 return (EINVAL);
 627         }
 628 }
 629
 630 static void
 631 gve_adminq_kick_cmd(struct gve_priv *priv, uint32_t prod_cnt)
 632 {
 633         gve_reg_bar_write_4(priv, ADMINQ_DOORBELL, prod_cnt);
 634
 635 }
 636
 637 static bool
 638 gve_adminq_wait_for_cmd(struct gve_priv *priv, uint32_t prod_cnt)
 639 {
 640         int i;
 641
 642         for (i = 0; i < GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK; i++) {
 643                 if (gve_reg_bar_read_4(priv, ADMINQ_EVENT_COUNTER) == prod_cnt)
 644                         return (true);
 645                 pause("gve adminq cmd", GVE_ADMINQ_SLEEP_LEN_MS);
 646         }
 647
 648         return (false);
 649 }
 650
 651 /*
 652  * Flushes all AQ commands currently queued and waits for them to complete.
 653  * If there are failures, it will return the first error.
 654  */
 655 static int
 656 gve_adminq_kick_and_wait(struct gve_priv *priv)
 657 {
 658         struct gve_adminq_command *cmd;
 659         uint32_t status, err;
 660         uint32_t tail, head;
 661         uint32_t opcode;
 662         int i;
 663
 664         tail = gve_reg_bar_read_4(priv, ADMINQ_EVENT_COUNTER);
 665         head = priv->adminq_prod_cnt;
 666
 667         gve_adminq_kick_cmd(priv, head);
 668         if (!gve_adminq_wait_for_cmd(priv, head)) {
 669                 device_printf(priv->dev, "AQ commands timed out, need to reset AQ\n");
 670                 priv->adminq_timeouts++;
 671                 return (ENOTRECOVERABLE);
 672         }
 673         bus_dmamap_sync(
 674             priv->aq_mem.tag, priv->aq_mem.map, BUS_DMASYNC_POSTREAD);
 675
 676         for (i = tail; i < head; i++) {
 677                 cmd = &priv->adminq[i & priv->adminq_mask];
 678                 status = be32toh(cmd->status);
 679                 opcode = be32toh(cmd->opcode);
 680                 err = gve_adminq_parse_err(priv, opcode, status);
 681                 if (err != 0)
 682                         return (err);
 683         }
 684
 685         return (0);
 686 }
 687
 688 /*
 689  * This function is not threadsafe - the caller is responsible for any
 690  * necessary locks.
 691  */
 692 static int
 693 gve_adminq_issue_cmd(struct gve_priv *priv, struct gve_adminq_command *cmd_orig)
 694 {
 695         struct gve_adminq_command *cmd;
 696         uint32_t opcode;
 697         uint32_t tail;
 698         int err;
 699
 700         tail = gve_reg_bar_read_4(priv, ADMINQ_EVENT_COUNTER);
 701
 702         /* Check if next command will overflow the buffer. */
 703         if ((priv->adminq_prod_cnt - tail) > priv->adminq_mask) {
 704                 /* Flush existing commands to make room. */
 705                 err = gve_adminq_kick_and_wait(priv);
 706                 if (err != 0)
 707                         return (err);
 708
 709                 /* Retry. */
 710                 tail = gve_reg_bar_read_4(priv, ADMINQ_EVENT_COUNTER);
 711                 if ((priv->adminq_prod_cnt - tail) > priv->adminq_mask) {
 712                         /*
 713                          * This should never happen. We just flushed the
 714                          * command queue so there should be enough space.
 715                          */
 716                         return (ENOMEM);
 717                 }
 718         }
 719
 720         cmd = &priv->adminq[priv->adminq_prod_cnt & priv->adminq_mask];
 721         priv->adminq_prod_cnt++;
 722
 723         memcpy(cmd, cmd_orig, sizeof(*cmd_orig));
 724
 725         bus_dmamap_sync(
 726             priv->aq_mem.tag, priv->aq_mem.map, BUS_DMASYNC_PREWRITE);
 727
 728         opcode = be32toh(cmd->opcode);
 729
 730         switch (opcode) {
 731         case GVE_ADMINQ_DESCRIBE_DEVICE:
 732                 priv->adminq_describe_device_cnt++;
 733                 break;
 734
 735         case GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES:
 736                 priv->adminq_cfg_device_resources_cnt++;
 737                 break;
 738
 739         case GVE_ADMINQ_REGISTER_PAGE_LIST:
 740                 priv->adminq_register_page_list_cnt++;
 741                 break;
 742
 743         case GVE_ADMINQ_UNREGISTER_PAGE_LIST:
 744                 priv->adminq_unregister_page_list_cnt++;
 745                 break;
 746
 747         case GVE_ADMINQ_CREATE_TX_QUEUE:
 748                 priv->adminq_create_tx_queue_cnt++;
 749                 break;
 750
 751         case GVE_ADMINQ_CREATE_RX_QUEUE:
 752                 priv->adminq_create_rx_queue_cnt++;
 753                 break;
 754
 755         case GVE_ADMINQ_DESTROY_TX_QUEUE:
 756                 priv->adminq_destroy_tx_queue_cnt++;
 757                 break;
 758
 759         case GVE_ADMINQ_DESTROY_RX_QUEUE:
 760                 priv->adminq_destroy_rx_queue_cnt++;
 761                 break;
 762
 763         case GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES:
 764                 priv->adminq_dcfg_device_resources_cnt++;
 765                 break;
 766
 767         case GVE_ADMINQ_SET_DRIVER_PARAMETER:
 768                 priv->adminq_set_driver_parameter_cnt++;
 769                 break;
 770
 771         case GVE_ADMINQ_VERIFY_DRIVER_COMPATIBILITY:
 772                 priv->adminq_verify_driver_compatibility_cnt++;
 773                 break;
 774
 775         default:
 776                 device_printf(priv->dev, "Unknown AQ command opcode %d\n", opcode);
 777         }
 778
 779         return (0);
 780 }
 781
 782 /*
 783  * This function is not threadsafe - the caller is responsible for any
 784  * necessary locks.
 785  * The caller is also responsible for making sure there are no commands
 786  * waiting to be executed.
 787  */
 788 static int
 789 gve_adminq_execute_cmd(struct gve_priv *priv, struct gve_adminq_command *cmd_orig)
 790 {
 791         uint32_t tail, head;
 792         int err;
 793
 794         tail = gve_reg_bar_read_4(priv, ADMINQ_EVENT_COUNTER);
 795         head = priv->adminq_prod_cnt;
 796
 797         if (tail != head)
 798                 return (EINVAL);
 799         err = gve_adminq_issue_cmd(priv, cmd_orig);
 800         if (err != 0)
 801                 return (err);
 802         return (gve_adminq_kick_and_wait(priv));
 803 }