sys/dev/ena/ena.c

   1 /*-
   2  * BSD LICENSE
   3  *
   4  * Copyright (c) 2015-2019 Amazon.com, Inc. or its affiliates.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  *
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29  */
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34 #include <sys/systm.h>
  35 #include <sys/bus.h>
  36 #include <sys/endian.h>
  37 #include <sys/kernel.h>
  38 #include <sys/kthread.h>
  39 #include <sys/malloc.h>
  40 #include <sys/mbuf.h>
  41 #include <sys/module.h>
  42 #include <sys/rman.h>
  43 #include <sys/smp.h>
  44 #include <sys/socket.h>
  45 #include <sys/sockio.h>
  46 #include <sys/sysctl.h>
  47 #include <sys/taskqueue.h>
  48 #include <sys/time.h>
  49 #include <sys/eventhandler.h>
  50
  51 #include <machine/bus.h>
  52 #include <machine/resource.h>
  53 #include <machine/in_cksum.h>
  54
  55 #include <net/bpf.h>
  56 #include <net/ethernet.h>
  57 #include <net/if.h>
  58 #include <net/if_var.h>
  59 #include <net/if_arp.h>
  60 #include <net/if_dl.h>
  61 #include <net/if_media.h>
  62 #include <net/if_types.h>
  63 #include <net/if_vlan_var.h>
  64
  65 #include <netinet/in_systm.h>
  66 #include <netinet/in.h>
  67 #include <netinet/if_ether.h>
  68 #include <netinet/ip.h>
  69 #include <netinet/ip6.h>
  70 #include <netinet/tcp.h>
  71 #include <netinet/udp.h>
  72
  73 #include <dev/pci/pcivar.h>
  74 #include <dev/pci/pcireg.h>
  75
  76 #include <vm/vm.h>
  77 #include <vm/pmap.h>
  78
  79 #include "ena_datapath.h"
  80 #include "ena.h"
  81 #include "ena_sysctl.h"
  82
  83 #ifdef DEV_NETMAP
  84 #include "ena_netmap.h"
  85 #endif /* DEV_NETMAP */
  86
  87 /*********************************************************
  88  *  Function prototypes
  89  *********************************************************/
  90 static int      ena_probe(device_t);
  91 static void     ena_intr_msix_mgmnt(void *);
  92 static void     ena_free_pci_resources(struct ena_adapter *);
  93 static int      ena_change_mtu(if_t, int);
  94 static inline void ena_alloc_counters(counter_u64_t *, int);
  95 static inline void ena_free_counters(counter_u64_t *, int);
  96 static inline void ena_reset_counters(counter_u64_t *, int);
  97 static void     ena_init_io_rings_common(struct ena_adapter *,
  98     struct ena_ring *, uint16_t);
  99 static void     ena_init_io_rings(struct ena_adapter *);
 100 static void     ena_free_io_ring_resources(struct ena_adapter *, unsigned int);
 101 static void     ena_free_all_io_rings_resources(struct ena_adapter *);
 102 static int      ena_setup_tx_dma_tag(struct ena_adapter *);
 103 static int      ena_free_tx_dma_tag(struct ena_adapter *);
 104 static int      ena_setup_rx_dma_tag(struct ena_adapter *);
 105 static int      ena_free_rx_dma_tag(struct ena_adapter *);
 106 static void     ena_release_all_tx_dmamap(struct ena_ring *);
 107 static int      ena_setup_tx_resources(struct ena_adapter *, int);
 108 static void     ena_free_tx_resources(struct ena_adapter *, int);
 109 static int      ena_setup_all_tx_resources(struct ena_adapter *);
 110 static void     ena_free_all_tx_resources(struct ena_adapter *);
 111 static int      ena_setup_rx_resources(struct ena_adapter *, unsigned int);
 112 static void     ena_free_rx_resources(struct ena_adapter *, unsigned int);
 113 static int      ena_setup_all_rx_resources(struct ena_adapter *);
 114 static void     ena_free_all_rx_resources(struct ena_adapter *);
 115 static inline int ena_alloc_rx_mbuf(struct ena_adapter *, struct ena_ring *,
 116     struct ena_rx_buffer *);
 117 static void     ena_free_rx_mbuf(struct ena_adapter *, struct ena_ring *,
 118     struct ena_rx_buffer *);
 119 static void     ena_free_rx_bufs(struct ena_adapter *, unsigned int);
 120 static void     ena_refill_all_rx_bufs(struct ena_adapter *);
 121 static void     ena_free_all_rx_bufs(struct ena_adapter *);
 122 static void     ena_free_tx_bufs(struct ena_adapter *, unsigned int);
 123 static void     ena_free_all_tx_bufs(struct ena_adapter *);
 124 static void     ena_destroy_all_tx_queues(struct ena_adapter *);
 125 static void     ena_destroy_all_rx_queues(struct ena_adapter *);
 126 static void     ena_destroy_all_io_queues(struct ena_adapter *);
 127 static int      ena_create_io_queues(struct ena_adapter *);
 128 static int      ena_handle_msix(void *);
 129 static int      ena_enable_msix(struct ena_adapter *);
 130 static void     ena_setup_mgmnt_intr(struct ena_adapter *);
 131 static int      ena_setup_io_intr(struct ena_adapter *);
 132 static int      ena_request_mgmnt_irq(struct ena_adapter *);
 133 static int      ena_request_io_irq(struct ena_adapter *);
 134 static void     ena_free_mgmnt_irq(struct ena_adapter *);
 135 static void     ena_free_io_irq(struct ena_adapter *);
 136 static void     ena_free_irqs(struct ena_adapter*);
 137 static void     ena_disable_msix(struct ena_adapter *);
 138 static void     ena_unmask_all_io_irqs(struct ena_adapter *);
 139 static int      ena_rss_configure(struct ena_adapter *);
 140 static int      ena_up_complete(struct ena_adapter *);
 141 static uint64_t ena_get_counter(if_t, ift_counter);
 142 static int      ena_media_change(if_t);
 143 static void     ena_media_status(if_t, struct ifmediareq *);
 144 static void     ena_init(void *);
 145 static int      ena_ioctl(if_t, u_long, caddr_t);
 146 static int      ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *);
 147 static void     ena_update_host_info(struct ena_admin_host_info *, if_t);
 148 static void     ena_update_hwassist(struct ena_adapter *);
 149 static int      ena_setup_ifnet(device_t, struct ena_adapter *,
 150     struct ena_com_dev_get_features_ctx *);
 151 static int      ena_enable_wc(struct resource *);
 152 static int      ena_set_queues_placement_policy(device_t, struct ena_com_dev *,
 153     struct ena_admin_feature_llq_desc *, struct ena_llq_configurations *);
 154 static int      ena_calc_io_queue_num(struct ena_adapter *,
 155     struct ena_com_dev_get_features_ctx *);
 156 static int      ena_calc_queue_size(struct ena_adapter *,
 157     struct ena_calc_queue_size_ctx *);
 158 static int      ena_handle_updated_queues(struct ena_adapter *,
 159     struct ena_com_dev_get_features_ctx *);
 160 static int      ena_rss_init_default(struct ena_adapter *);
 161 static void     ena_rss_init_default_deferred(void *);
 162 static void     ena_config_host_info(struct ena_com_dev *, device_t);
 163 static int      ena_attach(device_t);
 164 static int      ena_detach(device_t);
 165 static int      ena_device_init(struct ena_adapter *, device_t,
 166     struct ena_com_dev_get_features_ctx *, int *);
 167 static int      ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *,
 168     int);
 169 static void ena_update_on_link_change(void *, struct ena_admin_aenq_entry *);
 170 static void     unimplemented_aenq_handler(void *,
 171     struct ena_admin_aenq_entry *);
 172 static void     ena_timer_service(void *);
 173
 174 static char ena_version[] = DEVICE_NAME DRV_MODULE_NAME " v" DRV_MODULE_VERSION;
 175
 176 static ena_vendor_info_t ena_vendor_info_array[] = {
 177     { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_PF, 0},
 178     { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_LLQ_PF, 0},
 179     { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_VF, 0},
 180     { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_LLQ_VF, 0},
 181     /* Last entry */
 182     { 0, 0, 0 }
 183 };
 184
 185 /*
 186  * Contains pointers to event handlers, e.g. link state chage.
 187  */
 188 static struct ena_aenq_handlers aenq_handlers;
 189
 190 void
 191 ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 192 {
 193         if (error != 0)
 194                 return;
 195         *(bus_addr_t *) arg = segs[0].ds_addr;
 196 }
 197
 198 int
 199 ena_dma_alloc(device_t dmadev, bus_size_t size,
 200     ena_mem_handle_t *dma , int mapflags)
 201 {
 202         struct ena_adapter* adapter = device_get_softc(dmadev);
 203         uint32_t maxsize;
 204         uint64_t dma_space_addr;
 205         int error;
 206
 207         maxsize = ((size - 1) / PAGE_SIZE + 1) * PAGE_SIZE;
 208
 209         dma_space_addr = ENA_DMA_BIT_MASK(adapter->dma_width);
 210         if (unlikely(dma_space_addr == 0))
 211                 dma_space_addr = BUS_SPACE_MAXADDR;
 212
 213         error = bus_dma_tag_create(bus_get_dma_tag(dmadev), /* parent */
 214             8, 0,             /* alignment, bounds              */
 215             dma_space_addr,   /* lowaddr of exclusion window    */
 216             BUS_SPACE_MAXADDR,/* highaddr of exclusion window   */
 217             NULL, NULL,       /* filter, filterarg              */
 218             maxsize,          /* maxsize                        */
 219             1,                /* nsegments                      */
 220             maxsize,          /* maxsegsize                     */
 221             BUS_DMA_ALLOCNOW, /* flags                          */
 222             NULL,             /* lockfunc                       */
 223             NULL,             /* lockarg                        */
 224             &dma->tag);
 225         if (unlikely(error != 0)) {
 226                 ena_trace(ENA_ALERT, "bus_dma_tag_create failed: %d\n", error);
 227                 goto fail_tag;
 228         }
 229
 230         error = bus_dmamem_alloc(dma->tag, (void**) &dma->vaddr,
 231             BUS_DMA_COHERENT | BUS_DMA_ZERO, &dma->map);
 232         if (unlikely(error != 0)) {
 233                 ena_trace(ENA_ALERT, "bus_dmamem_alloc(%ju) failed: %d\n",
 234                     (uintmax_t)size, error);
 235                 goto fail_map_create;
 236         }
 237
 238         dma->paddr = 0;
 239         error = bus_dmamap_load(dma->tag, dma->map, dma->vaddr,
 240             size, ena_dmamap_callback, &dma->paddr, mapflags);
 241         if (unlikely((error != 0) || (dma->paddr == 0))) {
 242                 ena_trace(ENA_ALERT, ": bus_dmamap_load failed: %d\n", error);
 243                 goto fail_map_load;
 244         }
 245
 246         bus_dmamap_sync(dma->tag, dma->map,
 247             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 248
 249         return (0);
 250
 251 fail_map_load:
 252         bus_dmamem_free(dma->tag, dma->vaddr, dma->map);
 253 fail_map_create:
 254         bus_dma_tag_destroy(dma->tag);
 255 fail_tag:
 256         dma->tag = NULL;
 257         dma->vaddr = NULL;
 258         dma->paddr = 0;
 259
 260         return (error);
 261 }
 262
 263 static void
 264 ena_free_pci_resources(struct ena_adapter *adapter)
 265 {
 266         device_t pdev = adapter->pdev;
 267
 268         if (adapter->memory != NULL) {
 269                 bus_release_resource(pdev, SYS_RES_MEMORY,
 270                     PCIR_BAR(ENA_MEM_BAR), adapter->memory);
 271         }
 272
 273         if (adapter->registers != NULL) {
 274                 bus_release_resource(pdev, SYS_RES_MEMORY,
 275                     PCIR_BAR(ENA_REG_BAR), adapter->registers);
 276         }
 277 }
 278
 279 static int
 280 ena_probe(device_t dev)
 281 {
 282         ena_vendor_info_t *ent;
 283         char            adapter_name[60];
 284         uint16_t        pci_vendor_id = 0;
 285         uint16_t        pci_device_id = 0;
 286
 287         pci_vendor_id = pci_get_vendor(dev);
 288         pci_device_id = pci_get_device(dev);
 289
 290         ent = ena_vendor_info_array;
 291         while (ent->vendor_id != 0) {
 292                 if ((pci_vendor_id == ent->vendor_id) &&
 293                     (pci_device_id == ent->device_id)) {
 294                         ena_trace(ENA_DBG, "vendor=%x device=%x\n",
 295                             pci_vendor_id, pci_device_id);
 296
 297                         sprintf(adapter_name, DEVICE_DESC);
 298                         device_set_desc_copy(dev, adapter_name);
 299                         return (BUS_PROBE_DEFAULT);
 300                 }
 301
 302                 ent++;
 303
 304         }
 305
 306         return (ENXIO);
 307 }
 308
 309 static int
 310 ena_change_mtu(if_t ifp, int new_mtu)
 311 {
 312         struct ena_adapter *adapter = if_getsoftc(ifp);
 313         int rc;
 314
 315         if ((new_mtu > adapter->max_mtu) || (new_mtu < ENA_MIN_MTU)) {
 316                 device_printf(adapter->pdev, "Invalid MTU setting. "
 317                     "new_mtu: %d max mtu: %d min mtu: %d\n",
 318                     new_mtu, adapter->max_mtu, ENA_MIN_MTU);
 319                 return (EINVAL);
 320         }
 321
 322         rc = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu);
 323         if (likely(rc == 0)) {
 324                 ena_trace(ENA_DBG, "set MTU to %d\n", new_mtu);
 325                 if_setmtu(ifp, new_mtu);
 326         } else {
 327                 device_printf(adapter->pdev, "Failed to set MTU to %d\n",
 328                     new_mtu);
 329         }
 330
 331         return (rc);
 332 }
 333
 334 static inline void
 335 ena_alloc_counters(counter_u64_t *begin, int size)
 336 {
 337         counter_u64_t *end = (counter_u64_t *)((char *)begin + size);
 338
 339         for (; begin < end; ++begin)
 340                 *begin = counter_u64_alloc(M_WAITOK);
 341 }
 342
 343 static inline void
 344 ena_free_counters(counter_u64_t *begin, int size)
 345 {
 346         counter_u64_t *end = (counter_u64_t *)((char *)begin + size);
 347
 348         for (; begin < end; ++begin)
 349                 counter_u64_free(*begin);
 350 }
 351
 352 static inline void
 353 ena_reset_counters(counter_u64_t *begin, int size)
 354 {
 355         counter_u64_t *end = (counter_u64_t *)((char *)begin + size);
 356
 357         for (; begin < end; ++begin)
 358                 counter_u64_zero(*begin);
 359 }
 360
 361 static void
 362 ena_init_io_rings_common(struct ena_adapter *adapter, struct ena_ring *ring,
 363     uint16_t qid)
 364 {
 365
 366         ring->qid = qid;
 367         ring->adapter = adapter;
 368         ring->ena_dev = adapter->ena_dev;
 369         ring->first_interrupt = false;
 370         ring->no_interrupt_event_cnt = 0;
 371 }
 372
 373 static void
 374 ena_init_io_rings(struct ena_adapter *adapter)
 375 {
 376         struct ena_com_dev *ena_dev;
 377         struct ena_ring *txr, *rxr;
 378         struct ena_que *que;
 379         int i;
 380
 381         ena_dev = adapter->ena_dev;
 382
 383         for (i = 0; i < adapter->num_queues; i++) {
 384                 txr = &adapter->tx_ring[i];
 385                 rxr = &adapter->rx_ring[i];
 386
 387                 /* TX/RX common ring state */
 388                 ena_init_io_rings_common(adapter, txr, i);
 389                 ena_init_io_rings_common(adapter, rxr, i);
 390
 391                 /* TX specific ring state */
 392                 txr->ring_size = adapter->tx_ring_size;
 393                 txr->tx_max_header_size = ena_dev->tx_max_header_size;
 394                 txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type;
 395                 txr->smoothed_interval =
 396                     ena_com_get_nonadaptive_moderation_interval_tx(ena_dev);
 397
 398                 /* Allocate a buf ring */
 399                 txr->buf_ring_size = adapter->buf_ring_size;
 400                 txr->br = buf_ring_alloc(txr->buf_ring_size, M_DEVBUF,
 401                     M_WAITOK, &txr->ring_mtx);
 402
 403                 /* Alloc TX statistics. */
 404                 ena_alloc_counters((counter_u64_t *)&txr->tx_stats,
 405                     sizeof(txr->tx_stats));
 406
 407                 /* RX specific ring state */
 408                 rxr->ring_size = adapter->rx_ring_size;
 409                 rxr->smoothed_interval =
 410                     ena_com_get_nonadaptive_moderation_interval_rx(ena_dev);
 411
 412                 /* Alloc RX statistics. */
 413                 ena_alloc_counters((counter_u64_t *)&rxr->rx_stats,
 414                     sizeof(rxr->rx_stats));
 415
 416                 /* Initialize locks */
 417                 snprintf(txr->mtx_name, nitems(txr->mtx_name), "%s:tx(%d)",
 418                     device_get_nameunit(adapter->pdev), i);
 419                 snprintf(rxr->mtx_name, nitems(rxr->mtx_name), "%s:rx(%d)",
 420                     device_get_nameunit(adapter->pdev), i);
 421
 422                 mtx_init(&txr->ring_mtx, txr->mtx_name, NULL, MTX_DEF);
 423
 424                 que = &adapter->que[i];
 425                 que->adapter = adapter;
 426                 que->id = i;
 427                 que->tx_ring = txr;
 428                 que->rx_ring = rxr;
 429
 430                 txr->que = que;
 431                 rxr->que = que;
 432
 433                 rxr->empty_rx_queue = 0;
 434         }
 435 }
 436
 437 static void
 438 ena_free_io_ring_resources(struct ena_adapter *adapter, unsigned int qid)
 439 {
 440         struct ena_ring *txr = &adapter->tx_ring[qid];
 441         struct ena_ring *rxr = &adapter->rx_ring[qid];
 442
 443         ena_free_counters((counter_u64_t *)&txr->tx_stats,
 444             sizeof(txr->tx_stats));
 445         ena_free_counters((counter_u64_t *)&rxr->rx_stats,
 446             sizeof(rxr->rx_stats));
 447
 448         ENA_RING_MTX_LOCK(txr);
 449         drbr_free(txr->br, M_DEVBUF);
 450         ENA_RING_MTX_UNLOCK(txr);
 451
 452         mtx_destroy(&txr->ring_mtx);
 453 }
 454
 455 static void
 456 ena_free_all_io_rings_resources(struct ena_adapter *adapter)
 457 {
 458         int i;
 459
 460         for (i = 0; i < adapter->num_queues; i++)
 461                 ena_free_io_ring_resources(adapter, i);
 462
 463 }
 464
 465 static int
 466 ena_setup_tx_dma_tag(struct ena_adapter *adapter)
 467 {
 468         int ret;
 469
 470         /* Create DMA tag for Tx buffers */
 471         ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev),
 472             1, 0,                                 /* alignment, bounds       */
 473             ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window  */
 474             BUS_SPACE_MAXADDR,                    /* highaddr of excl window */
 475             NULL, NULL,                           /* filter, filterarg       */
 476             ENA_TSO_MAXSIZE,                      /* maxsize                 */
 477             adapter->max_tx_sgl_size - 1,         /* nsegments               */
 478             ENA_TSO_MAXSIZE,                      /* maxsegsize              */
 479             0,                                    /* flags                   */
 480             NULL,                                 /* lockfunc                */
 481             NULL,                                 /* lockfuncarg             */
 482             &adapter->tx_buf_tag);
 483
 484         return (ret);
 485 }
 486
 487 static int
 488 ena_free_tx_dma_tag(struct ena_adapter *adapter)
 489 {
 490         int ret;
 491
 492         ret = bus_dma_tag_destroy(adapter->tx_buf_tag);
 493
 494         if (likely(ret == 0))
 495                 adapter->tx_buf_tag = NULL;
 496
 497         return (ret);
 498 }
 499
 500 static int
 501 ena_setup_rx_dma_tag(struct ena_adapter *adapter)
 502 {
 503         int ret;
 504
 505         /* Create DMA tag for Rx buffers*/
 506         ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), /* parent   */
 507             1, 0,                                 /* alignment, bounds       */
 508             ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window  */
 509             BUS_SPACE_MAXADDR,                    /* highaddr of excl window */
 510             NULL, NULL,                           /* filter, filterarg       */
 511             MJUM16BYTES,                          /* maxsize                 */
 512             adapter->max_rx_sgl_size,             /* nsegments               */
 513             MJUM16BYTES,                          /* maxsegsize              */
 514             0,                                    /* flags                   */
 515             NULL,                                 /* lockfunc                */
 516             NULL,                                 /* lockarg                 */
 517             &adapter->rx_buf_tag);
 518
 519         return (ret);
 520 }
 521
 522 static int
 523 ena_free_rx_dma_tag(struct ena_adapter *adapter)
 524 {
 525         int ret;
 526
 527         ret = bus_dma_tag_destroy(adapter->rx_buf_tag);
 528
 529         if (likely(ret == 0))
 530                 adapter->rx_buf_tag = NULL;
 531
 532         return (ret);
 533 }
 534
 535 static void
 536 ena_release_all_tx_dmamap(struct ena_ring *tx_ring)
 537 {
 538         struct ena_adapter *adapter = tx_ring->adapter;
 539         struct ena_tx_buffer *tx_info;
 540         bus_dma_tag_t tx_tag = adapter->tx_buf_tag;;
 541         int i;
 542 #ifdef DEV_NETMAP
 543         struct ena_netmap_tx_info *nm_info;
 544         int j;
 545 #endif /* DEV_NETMAP */
 546
 547         for (i = 0; i < tx_ring->ring_size; ++i) {
 548                 tx_info = &tx_ring->tx_buffer_info[i];
 549 #ifdef DEV_NETMAP
 550                 if (adapter->ifp->if_capenable & IFCAP_NETMAP) {
 551                         nm_info = &tx_info->nm_info;
 552                         for (j = 0; j < ENA_PKT_MAX_BUFS; ++j) {
 553                                 if (nm_info->map_seg[j] != NULL) {
 554                                         bus_dmamap_destroy(tx_tag,
 555                                             nm_info->map_seg[j]);
 556                                         nm_info->map_seg[j] = NULL;
 557                                 }
 558                         }
 559                 }
 560 #endif /* DEV_NETMAP */
 561                 if (tx_info->dmamap != NULL) {
 562                         bus_dmamap_destroy(tx_tag, tx_info->dmamap);
 563                         tx_info->dmamap = NULL;
 564                 }
 565         }
 566 }
 567
 568 /**
 569  * ena_setup_tx_resources - allocate Tx resources (Descriptors)
 570  * @adapter: network interface device structure
 571  * @qid: queue index
 572  *
 573  * Returns 0 on success, otherwise on failure.
 574  **/
 575 static int
 576 ena_setup_tx_resources(struct ena_adapter *adapter, int qid)
 577 {
 578         struct ena_que *que = &adapter->que[qid];
 579         struct ena_ring *tx_ring = que->tx_ring;
 580         int size, i, err;
 581 #ifdef DEV_NETMAP
 582         bus_dmamap_t *map;
 583         int j;
 584
 585         ena_netmap_reset_tx_ring(adapter, qid);
 586 #endif /* DEV_NETMAP */
 587
 588         size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size;
 589
 590         tx_ring->tx_buffer_info = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
 591         if (unlikely(tx_ring->tx_buffer_info == NULL))
 592                 return (ENOMEM);
 593
 594         size = sizeof(uint16_t) * tx_ring->ring_size;
 595         tx_ring->free_tx_ids = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
 596         if (unlikely(tx_ring->free_tx_ids == NULL))
 597                 goto err_buf_info_free;
 598
 599         size = tx_ring->tx_max_header_size;
 600         tx_ring->push_buf_intermediate_buf = malloc(size, M_DEVBUF,
 601             M_NOWAIT | M_ZERO);
 602         if (unlikely(tx_ring->push_buf_intermediate_buf == NULL))
 603                 goto err_tx_ids_free;
 604
 605         /* Req id stack for TX OOO completions */
 606         for (i = 0; i < tx_ring->ring_size; i++)
 607                 tx_ring->free_tx_ids[i] = i;
 608
 609         /* Reset TX statistics. */
 610         ena_reset_counters((counter_u64_t *)&tx_ring->tx_stats,
 611             sizeof(tx_ring->tx_stats));
 612
 613         tx_ring->next_to_use = 0;
 614         tx_ring->next_to_clean = 0;
 615         tx_ring->acum_pkts = 0;
 616
 617         /* Make sure that drbr is empty */
 618         ENA_RING_MTX_LOCK(tx_ring);
 619         drbr_flush(adapter->ifp, tx_ring->br);
 620         ENA_RING_MTX_UNLOCK(tx_ring);
 621
 622         /* ... and create the buffer DMA maps */
 623         for (i = 0; i < tx_ring->ring_size; i++) {
 624                 err = bus_dmamap_create(adapter->tx_buf_tag, 0,
 625                     &tx_ring->tx_buffer_info[i].dmamap);
 626                 if (unlikely(err != 0)) {
 627                         ena_trace(ENA_ALERT,
 628                             "Unable to create Tx DMA map for buffer %d\n",
 629                             i);
 630                         goto err_map_release;
 631                 }
 632
 633 #ifdef DEV_NETMAP
 634                 if (adapter->ifp->if_capenable & IFCAP_NETMAP) {
 635                         map = tx_ring->tx_buffer_info[i].nm_info.map_seg;
 636                         for (j = 0; j < ENA_PKT_MAX_BUFS; j++) {
 637                                 err = bus_dmamap_create(adapter->tx_buf_tag, 0,
 638                                     &map[j]);
 639                                 if (unlikely(err != 0)) {
 640                                         ena_trace(ENA_ALERT, "Unable to create "
 641                                             "Tx DMA for buffer %d %d\n", i, j);
 642                                         goto err_map_release;
 643                                 }
 644                         }
 645                 }
 646 #endif /* DEV_NETMAP */
 647         }
 648
 649         /* Allocate taskqueues */
 650         TASK_INIT(&tx_ring->enqueue_task, 0, ena_deferred_mq_start, tx_ring);
 651         tx_ring->enqueue_tq = taskqueue_create_fast("ena_tx_enque", M_NOWAIT,
 652             taskqueue_thread_enqueue, &tx_ring->enqueue_tq);
 653         if (unlikely(tx_ring->enqueue_tq == NULL)) {
 654                 ena_trace(ENA_ALERT,
 655                     "Unable to create taskqueue for enqueue task\n");
 656                 i = tx_ring->ring_size;
 657                 goto err_map_release;
 658         }
 659
 660         tx_ring->running = true;
 661
 662         taskqueue_start_threads(&tx_ring->enqueue_tq, 1, PI_NET,
 663             "%s txeq %d", device_get_nameunit(adapter->pdev), que->cpu);
 664
 665         return (0);
 666
 667 err_map_release:
 668         ena_release_all_tx_dmamap(tx_ring);
 669 err_tx_ids_free:
 670         free(tx_ring->free_tx_ids, M_DEVBUF);
 671         tx_ring->free_tx_ids = NULL;
 672 err_buf_info_free:
 673         free(tx_ring->tx_buffer_info, M_DEVBUF);
 674         tx_ring->tx_buffer_info = NULL;
 675
 676         return (ENOMEM);
 677 }
 678
 679 /**
 680  * ena_free_tx_resources - Free Tx Resources per Queue
 681  * @adapter: network interface device structure
 682  * @qid: queue index
 683  *
 684  * Free all transmit software resources
 685  **/
 686 static void
 687 ena_free_tx_resources(struct ena_adapter *adapter, int qid)
 688 {
 689         struct ena_ring *tx_ring = &adapter->tx_ring[qid];
 690 #ifdef DEV_NETMAP
 691         struct ena_netmap_tx_info *nm_info;
 692         int j;
 693 #endif /* DEV_NETMAP */
 694
 695         while (taskqueue_cancel(tx_ring->enqueue_tq, &tx_ring->enqueue_task,
 696             NULL))
 697                 taskqueue_drain(tx_ring->enqueue_tq, &tx_ring->enqueue_task);
 698
 699         taskqueue_free(tx_ring->enqueue_tq);
 700
 701         ENA_RING_MTX_LOCK(tx_ring);
 702         /* Flush buffer ring, */
 703         drbr_flush(adapter->ifp, tx_ring->br);
 704
 705         /* Free buffer DMA maps, */
 706         for (int i = 0; i < tx_ring->ring_size; i++) {
 707                 bus_dmamap_sync(adapter->tx_buf_tag,
 708                     tx_ring->tx_buffer_info[i].dmamap, BUS_DMASYNC_POSTWRITE);
 709                 bus_dmamap_unload(adapter->tx_buf_tag,
 710                     tx_ring->tx_buffer_info[i].dmamap);
 711                 bus_dmamap_destroy(adapter->tx_buf_tag,
 712                     tx_ring->tx_buffer_info[i].dmamap);
 713
 714 #ifdef DEV_NETMAP
 715                 if (adapter->ifp->if_capenable & IFCAP_NETMAP) {
 716                         nm_info = &tx_ring->tx_buffer_info[i].nm_info;
 717                         for (j = 0; j < ENA_PKT_MAX_BUFS; j++) {
 718                                 if (nm_info->socket_buf_idx[j] != 0) {
 719                                         bus_dmamap_sync(adapter->tx_buf_tag,
 720                                             nm_info->map_seg[j],
 721                                             BUS_DMASYNC_POSTWRITE);
 722                                         ena_netmap_unload(adapter,
 723                                             nm_info->map_seg[j]);
 724                                 }
 725                                 bus_dmamap_destroy(adapter->tx_buf_tag,
 726                                     nm_info->map_seg[j]);
 727                                 nm_info->socket_buf_idx[j] = 0;
 728                         }
 729                 }
 730 #endif /* DEV_NETMAP */
 731
 732                 m_freem(tx_ring->tx_buffer_info[i].mbuf);
 733                 tx_ring->tx_buffer_info[i].mbuf = NULL;
 734         }
 735         ENA_RING_MTX_UNLOCK(tx_ring);
 736
 737         /* And free allocated memory. */
 738         free(tx_ring->tx_buffer_info, M_DEVBUF);
 739         tx_ring->tx_buffer_info = NULL;
 740
 741         free(tx_ring->free_tx_ids, M_DEVBUF);
 742         tx_ring->free_tx_ids = NULL;
 743
 744         ENA_MEM_FREE(adapter->ena_dev->dmadev,
 745             tx_ring->push_buf_intermediate_buf);
 746         tx_ring->push_buf_intermediate_buf = NULL;
 747 }
 748
 749 /**
 750  * ena_setup_all_tx_resources - allocate all queues Tx resources
 751  * @adapter: network interface device structure
 752  *
 753  * Returns 0 on success, otherwise on failure.
 754  **/
 755 static int
 756 ena_setup_all_tx_resources(struct ena_adapter *adapter)
 757 {
 758         int i, rc;
 759
 760         for (i = 0; i < adapter->num_queues; i++) {
 761                 rc = ena_setup_tx_resources(adapter, i);
 762                 if (rc != 0) {
 763                         device_printf(adapter->pdev,
 764                             "Allocation for Tx Queue %u failed\n", i);
 765                         goto err_setup_tx;
 766                 }
 767         }
 768
 769         return (0);
 770
 771 err_setup_tx:
 772         /* Rewind the index freeing the rings as we go */
 773         while (i--)
 774                 ena_free_tx_resources(adapter, i);
 775         return (rc);
 776 }
 777
 778 /**
 779  * ena_free_all_tx_resources - Free Tx Resources for All Queues
 780  * @adapter: network interface device structure
 781  *
 782  * Free all transmit software resources
 783  **/
 784 static void
 785 ena_free_all_tx_resources(struct ena_adapter *adapter)
 786 {
 787         int i;
 788
 789         for (i = 0; i < adapter->num_queues; i++)
 790                 ena_free_tx_resources(adapter, i);
 791 }
 792
 793 /**
 794  * ena_setup_rx_resources - allocate Rx resources (Descriptors)
 795  * @adapter: network interface device structure
 796  * @qid: queue index
 797  *
 798  * Returns 0 on success, otherwise on failure.
 799  **/
 800 static int
 801 ena_setup_rx_resources(struct ena_adapter *adapter, unsigned int qid)
 802 {
 803         struct ena_que *que = &adapter->que[qid];
 804         struct ena_ring *rx_ring = que->rx_ring;
 805         int size, err, i;
 806
 807         size = sizeof(struct ena_rx_buffer) * rx_ring->ring_size;
 808
 809 #ifdef DEV_NETMAP
 810         ena_netmap_reset_rx_ring(adapter, qid);
 811         rx_ring->initialized = false;
 812 #endif /* DEV_NETMAP */
 813
 814         /*
 815          * Alloc extra element so in rx path
 816          * we can always prefetch rx_info + 1
 817          */
 818         size += sizeof(struct ena_rx_buffer);
 819
 820         rx_ring->rx_buffer_info = malloc(size, M_DEVBUF, M_WAITOK | M_ZERO);
 821
 822         size = sizeof(uint16_t) * rx_ring->ring_size;
 823         rx_ring->free_rx_ids = malloc(size, M_DEVBUF, M_WAITOK);
 824
 825         for (i = 0; i < rx_ring->ring_size; i++)
 826                 rx_ring->free_rx_ids[i] = i;
 827
 828         /* Reset RX statistics. */
 829         ena_reset_counters((counter_u64_t *)&rx_ring->rx_stats,
 830             sizeof(rx_ring->rx_stats));
 831
 832         rx_ring->next_to_clean = 0;
 833         rx_ring->next_to_use = 0;
 834
 835         /* ... and create the buffer DMA maps */
 836         for (i = 0; i < rx_ring->ring_size; i++) {
 837                 err = bus_dmamap_create(adapter->rx_buf_tag, 0,
 838                     &(rx_ring->rx_buffer_info[i].map));
 839                 if (err != 0) {
 840                         ena_trace(ENA_ALERT,
 841                             "Unable to create Rx DMA map for buffer %d\n", i);
 842                         goto err_buf_info_unmap;
 843                 }
 844         }
 845
 846         /* Create LRO for the ring */
 847         if ((adapter->ifp->if_capenable & IFCAP_LRO) != 0) {
 848                 int err = tcp_lro_init(&rx_ring->lro);
 849                 if (err != 0) {
 850                         device_printf(adapter->pdev,
 851                             "LRO[%d] Initialization failed!\n", qid);
 852                 } else {
 853                         ena_trace(ENA_INFO,
 854                             "RX Soft LRO[%d] Initialized\n", qid);
 855                         rx_ring->lro.ifp = adapter->ifp;
 856                 }
 857         }
 858
 859         return (0);
 860
 861 err_buf_info_unmap:
 862         while (i--) {
 863                 bus_dmamap_destroy(adapter->rx_buf_tag,
 864                     rx_ring->rx_buffer_info[i].map);
 865         }
 866
 867         free(rx_ring->free_rx_ids, M_DEVBUF);
 868         rx_ring->free_rx_ids = NULL;
 869         free(rx_ring->rx_buffer_info, M_DEVBUF);
 870         rx_ring->rx_buffer_info = NULL;
 871         return (ENOMEM);
 872 }
 873
 874 /**
 875  * ena_free_rx_resources - Free Rx Resources
 876  * @adapter: network interface device structure
 877  * @qid: queue index
 878  *
 879  * Free all receive software resources
 880  **/
 881 static void
 882 ena_free_rx_resources(struct ena_adapter *adapter, unsigned int qid)
 883 {
 884         struct ena_ring *rx_ring = &adapter->rx_ring[qid];
 885
 886         /* Free buffer DMA maps, */
 887         for (int i = 0; i < rx_ring->ring_size; i++) {
 888                 bus_dmamap_sync(adapter->rx_buf_tag,
 889                     rx_ring->rx_buffer_info[i].map, BUS_DMASYNC_POSTREAD);
 890                 m_freem(rx_ring->rx_buffer_info[i].mbuf);
 891                 rx_ring->rx_buffer_info[i].mbuf = NULL;
 892                 bus_dmamap_unload(adapter->rx_buf_tag,
 893                     rx_ring->rx_buffer_info[i].map);
 894                 bus_dmamap_destroy(adapter->rx_buf_tag,
 895                     rx_ring->rx_buffer_info[i].map);
 896         }
 897
 898         /* free LRO resources, */
 899         tcp_lro_free(&rx_ring->lro);
 900
 901         /* free allocated memory */
 902         free(rx_ring->rx_buffer_info, M_DEVBUF);
 903         rx_ring->rx_buffer_info = NULL;
 904
 905         free(rx_ring->free_rx_ids, M_DEVBUF);
 906         rx_ring->free_rx_ids = NULL;
 907 }
 908
 909 /**
 910  * ena_setup_all_rx_resources - allocate all queues Rx resources
 911  * @adapter: network interface device structure
 912  *
 913  * Returns 0 on success, otherwise on failure.
 914  **/
 915 static int
 916 ena_setup_all_rx_resources(struct ena_adapter *adapter)
 917 {
 918         int i, rc = 0;
 919
 920         for (i = 0; i < adapter->num_queues; i++) {
 921                 rc = ena_setup_rx_resources(adapter, i);
 922                 if (rc != 0) {
 923                         device_printf(adapter->pdev,
 924                             "Allocation for Rx Queue %u failed\n", i);
 925                         goto err_setup_rx;
 926                 }
 927         }
 928         return (0);
 929
 930 err_setup_rx:
 931         /* rewind the index freeing the rings as we go */
 932         while (i--)
 933                 ena_free_rx_resources(adapter, i);
 934         return (rc);
 935 }
 936
 937 /**
 938  * ena_free_all_rx_resources - Free Rx resources for all queues
 939  * @adapter: network interface device structure
 940  *
 941  * Free all receive software resources
 942  **/
 943 static void
 944 ena_free_all_rx_resources(struct ena_adapter *adapter)
 945 {
 946         int i;
 947
 948         for (i = 0; i < adapter->num_queues; i++)
 949                 ena_free_rx_resources(adapter, i);
 950 }
 951
 952 static inline int
 953 ena_alloc_rx_mbuf(struct ena_adapter *adapter,
 954     struct ena_ring *rx_ring, struct ena_rx_buffer *rx_info)
 955 {
 956         struct ena_com_buf *ena_buf;
 957         bus_dma_segment_t segs[1];
 958         int nsegs, error;
 959         int mlen;
 960
 961         /* if previous allocated frag is not used */
 962         if (unlikely(rx_info->mbuf != NULL))
 963                 return (0);
 964
 965         /* Get mbuf using UMA allocator */
 966         rx_info->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, MJUM16BYTES);
 967
 968         if (unlikely(rx_info->mbuf == NULL)) {
 969                 counter_u64_add(rx_ring->rx_stats.mjum_alloc_fail, 1);
 970                 rx_info->mbuf = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
 971                 if (unlikely(rx_info->mbuf == NULL)) {
 972                         counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1);
 973                         return (ENOMEM);
 974                 }
 975                 mlen = MCLBYTES;
 976         } else {
 977                 mlen = MJUM16BYTES;
 978         }
 979         /* Set mbuf length*/
 980         rx_info->mbuf->m_pkthdr.len = rx_info->mbuf->m_len = mlen;
 981
 982         /* Map packets for DMA */
 983         ena_trace(ENA_DBG | ENA_RSC | ENA_RXPTH,
 984             "Using tag %p for buffers' DMA mapping, mbuf %p len: %d\n",
 985             adapter->rx_buf_tag,rx_info->mbuf, rx_info->mbuf->m_len);
 986         error = bus_dmamap_load_mbuf_sg(adapter->rx_buf_tag, rx_info->map,
 987             rx_info->mbuf, segs, &nsegs, BUS_DMA_NOWAIT);
 988         if (unlikely((error != 0) || (nsegs != 1))) {
 989                 ena_trace(ENA_WARNING, "failed to map mbuf, error: %d, "
 990                     "nsegs: %d\n", error, nsegs);
 991                 counter_u64_add(rx_ring->rx_stats.dma_mapping_err, 1);
 992                 goto exit;
 993
 994         }
 995
 996         bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, BUS_DMASYNC_PREREAD);
 997
 998         ena_buf = &rx_info->ena_buf;
 999         ena_buf->paddr = segs[0].ds_addr;
1000         ena_buf->len = mlen;
1001
1002         ena_trace(ENA_DBG | ENA_RSC | ENA_RXPTH,
1003             "ALLOC RX BUF: mbuf %p, rx_info %p, len %d, paddr %#jx\n",
1004             rx_info->mbuf, rx_info,ena_buf->len, (uintmax_t)ena_buf->paddr);
1005
1006         return (0);
1007
1008 exit:
1009         m_freem(rx_info->mbuf);
1010         rx_info->mbuf = NULL;
1011         return (EFAULT);
1012 }
1013
1014 static void
1015 ena_free_rx_mbuf(struct ena_adapter *adapter, struct ena_ring *rx_ring,
1016     struct ena_rx_buffer *rx_info)
1017 {
1018
1019         if (rx_info->mbuf == NULL) {
1020                 ena_trace(ENA_WARNING, "Trying to free unallocated buffer\n");
1021                 return;
1022         }
1023
1024         bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map,
1025             BUS_DMASYNC_POSTREAD);
1026         bus_dmamap_unload(adapter->rx_buf_tag, rx_info->map);
1027         m_freem(rx_info->mbuf);
1028         rx_info->mbuf = NULL;
1029 }
1030
1031 /**
1032  * ena_refill_rx_bufs - Refills ring with descriptors
1033  * @rx_ring: the ring which we want to feed with free descriptors
1034  * @num: number of descriptors to refill
1035  * Refills the ring with newly allocated DMA-mapped mbufs for receiving
1036  **/
1037 int
1038 ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t num)
1039 {
1040         struct ena_adapter *adapter = rx_ring->adapter;
1041         uint16_t next_to_use, req_id;
1042         uint32_t i;
1043         int rc;
1044
1045         ena_trace(ENA_DBG | ENA_RXPTH | ENA_RSC, "refill qid: %d\n",
1046             rx_ring->qid);
1047
1048         next_to_use = rx_ring->next_to_use;
1049
1050         for (i = 0; i < num; i++) {
1051                 struct ena_rx_buffer *rx_info;
1052
1053                 ena_trace(ENA_DBG | ENA_RXPTH | ENA_RSC,
1054                     "RX buffer - next to use: %d\n", next_to_use);
1055
1056                 req_id = rx_ring->free_rx_ids[next_to_use];
1057                 rx_info = &rx_ring->rx_buffer_info[req_id];
1058 #ifdef DEV_NETMAP
1059                 if (ena_rx_ring_in_netmap(adapter, rx_ring->qid))
1060                         rc = ena_netmap_alloc_rx_slot(adapter, rx_ring, rx_info);
1061                 else
1062 #endif /* DEV_NETMAP */
1063                         rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info);
1064                 if (unlikely(rc != 0)) {
1065                         ena_trace(ENA_WARNING,
1066                             "failed to alloc buffer for rx queue %d\n",
1067                             rx_ring->qid);
1068                         break;
1069                 }
1070                 rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq,
1071                     &rx_info->ena_buf, req_id);
1072                 if (unlikely(rc != 0)) {
1073                         ena_trace(ENA_WARNING,
1074                             "failed to add buffer for rx queue %d\n",
1075                             rx_ring->qid);
1076                         break;
1077                 }
1078                 next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use,
1079                     rx_ring->ring_size);
1080         }
1081
1082         if (unlikely(i < num)) {
1083                 counter_u64_add(rx_ring->rx_stats.refil_partial, 1);
1084                 ena_trace(ENA_WARNING,
1085                      "refilled rx qid %d with only %d mbufs (from %d)\n",
1086                      rx_ring->qid, i, num);
1087         }
1088
1089         if (likely(i != 0)) {
1090                 wmb();
1091                 ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq);
1092         }
1093         rx_ring->next_to_use = next_to_use;
1094         return (i);
1095 }
1096
1097 static void
1098 ena_free_rx_bufs(struct ena_adapter *adapter, unsigned int qid)
1099 {
1100         struct ena_ring *rx_ring = &adapter->rx_ring[qid];
1101         unsigned int i;
1102
1103         for (i = 0; i < rx_ring->ring_size; i++) {
1104                 struct ena_rx_buffer *rx_info = &rx_ring->rx_buffer_info[i];
1105
1106                 if (rx_info->mbuf != NULL)
1107                         ena_free_rx_mbuf(adapter, rx_ring, rx_info);
1108 #ifdef DEV_NETMAP
1109                 if (((if_getflags(adapter->ifp) & IFF_DYING) == 0) &&
1110                     (adapter->ifp->if_capenable & IFCAP_NETMAP)) {
1111                         if (rx_info->netmap_buf_idx != 0)
1112                                 ena_netmap_free_rx_slot(adapter, rx_ring,
1113                                     rx_info);
1114                 }
1115 #endif /* DEV_NETMAP */
1116         }
1117 }
1118
1119 /**
1120  * ena_refill_all_rx_bufs - allocate all queues Rx buffers
1121  * @adapter: network interface device structure
1122  *
1123  */
1124 static void
1125 ena_refill_all_rx_bufs(struct ena_adapter *adapter)
1126 {
1127         struct ena_ring *rx_ring;
1128         int i, rc, bufs_num;
1129
1130         for (i = 0; i < adapter->num_queues; i++) {
1131                 rx_ring = &adapter->rx_ring[i];
1132                 bufs_num = rx_ring->ring_size - 1;
1133                 rc = ena_refill_rx_bufs(rx_ring, bufs_num);
1134                 if (unlikely(rc != bufs_num))
1135                         ena_trace(ENA_WARNING, "refilling Queue %d failed. "
1136                             "Allocated %d buffers from: %d\n", i, rc, bufs_num);
1137 #ifdef DEV_NETMAP
1138                 rx_ring->initialized = true;
1139 #endif /* DEV_NETMAP */
1140         }
1141 }
1142
1143 static void
1144 ena_free_all_rx_bufs(struct ena_adapter *adapter)
1145 {
1146         int i;
1147
1148         for (i = 0; i < adapter->num_queues; i++)
1149                 ena_free_rx_bufs(adapter, i);
1150 }
1151
1152 /**
1153  * ena_free_tx_bufs - Free Tx Buffers per Queue
1154  * @adapter: network interface device structure
1155  * @qid: queue index
1156  **/
1157 static void
1158 ena_free_tx_bufs(struct ena_adapter *adapter, unsigned int qid)
1159 {
1160         bool print_once = true;
1161         struct ena_ring *tx_ring = &adapter->tx_ring[qid];
1162
1163         ENA_RING_MTX_LOCK(tx_ring);
1164         for (int i = 0; i < tx_ring->ring_size; i++) {
1165                 struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i];
1166
1167                 if (tx_info->mbuf == NULL)
1168                         continue;
1169
1170                 if (print_once) {
1171                         device_printf(adapter->pdev,
1172                             "free uncompleted tx mbuf qid %d idx 0x%x\n",
1173                             qid, i);
1174                         print_once = false;
1175                 } else {
1176                         ena_trace(ENA_DBG,
1177                             "free uncompleted tx mbuf qid %d idx 0x%x\n",
1178                              qid, i);
1179                 }
1180
1181                 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->dmamap,
1182                     BUS_DMASYNC_POSTWRITE);
1183                 bus_dmamap_unload(adapter->tx_buf_tag, tx_info->dmamap);
1184
1185                 m_free(tx_info->mbuf);
1186                 tx_info->mbuf = NULL;
1187         }
1188         ENA_RING_MTX_UNLOCK(tx_ring);
1189 }
1190
1191 static void
1192 ena_free_all_tx_bufs(struct ena_adapter *adapter)
1193 {
1194
1195         for (int i = 0; i < adapter->num_queues; i++)
1196                 ena_free_tx_bufs(adapter, i);
1197 }
1198
1199 static void
1200 ena_destroy_all_tx_queues(struct ena_adapter *adapter)
1201 {
1202         uint16_t ena_qid;
1203         int i;
1204
1205         for (i = 0; i < adapter->num_queues; i++) {
1206                 ena_qid = ENA_IO_TXQ_IDX(i);
1207                 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
1208         }
1209 }
1210
1211 static void
1212 ena_destroy_all_rx_queues(struct ena_adapter *adapter)
1213 {
1214         uint16_t ena_qid;
1215         int i;
1216
1217         for (i = 0; i < adapter->num_queues; i++) {
1218                 ena_qid = ENA_IO_RXQ_IDX(i);
1219                 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
1220         }
1221 }
1222
1223 static void
1224 ena_destroy_all_io_queues(struct ena_adapter *adapter)
1225 {
1226         struct ena_que *queue;
1227         int i;
1228
1229         for (i = 0; i < adapter->num_queues; i++) {
1230                 queue = &adapter->que[i];
1231                 while (taskqueue_cancel(queue->cleanup_tq,
1232                     &queue->cleanup_task, NULL))
1233                         taskqueue_drain(queue->cleanup_tq,
1234                             &queue->cleanup_task);
1235                 taskqueue_free(queue->cleanup_tq);
1236         }
1237
1238         ena_destroy_all_tx_queues(adapter);
1239         ena_destroy_all_rx_queues(adapter);
1240 }
1241
1242 static int
1243 ena_create_io_queues(struct ena_adapter *adapter)
1244 {
1245         struct ena_com_dev *ena_dev = adapter->ena_dev;
1246         struct ena_com_create_io_ctx ctx;
1247         struct ena_ring *ring;
1248         struct ena_que *queue;
1249         uint16_t ena_qid;
1250         uint32_t msix_vector;
1251         int rc, i;
1252
1253         /* Create TX queues */
1254         for (i = 0; i < adapter->num_queues; i++) {
1255                 msix_vector = ENA_IO_IRQ_IDX(i);
1256                 ena_qid = ENA_IO_TXQ_IDX(i);
1257                 ctx.mem_queue_type = ena_dev->tx_mem_queue_type;
1258                 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX;
1259                 ctx.queue_size = adapter->tx_ring_size;
1260                 ctx.msix_vector = msix_vector;
1261                 ctx.qid = ena_qid;
1262                 rc = ena_com_create_io_queue(ena_dev, &ctx);
1263                 if (rc != 0) {
1264                         device_printf(adapter->pdev,
1265                             "Failed to create io TX queue #%d rc: %d\n", i, rc);
1266                         goto err_tx;
1267                 }
1268                 ring = &adapter->tx_ring[i];
1269                 rc = ena_com_get_io_handlers(ena_dev, ena_qid,
1270                     &ring->ena_com_io_sq,
1271                     &ring->ena_com_io_cq);
1272                 if (rc != 0) {
1273                         device_printf(adapter->pdev,
1274                             "Failed to get TX queue handlers. TX queue num"
1275                             " %d rc: %d\n", i, rc);
1276                         ena_com_destroy_io_queue(ena_dev, ena_qid);
1277                         goto err_tx;
1278                 }
1279         }
1280
1281         /* Create RX queues */
1282         for (i = 0; i < adapter->num_queues; i++) {
1283                 msix_vector = ENA_IO_IRQ_IDX(i);
1284                 ena_qid = ENA_IO_RXQ_IDX(i);
1285                 ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
1286                 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX;
1287                 ctx.queue_size = adapter->rx_ring_size;
1288                 ctx.msix_vector = msix_vector;
1289                 ctx.qid = ena_qid;
1290                 rc = ena_com_create_io_queue(ena_dev, &ctx);
1291                 if (unlikely(rc != 0)) {
1292                         device_printf(adapter->pdev,
1293                             "Failed to create io RX queue[%d] rc: %d\n", i, rc);
1294                         goto err_rx;
1295                 }
1296
1297                 ring = &adapter->rx_ring[i];
1298                 rc = ena_com_get_io_handlers(ena_dev, ena_qid,
1299                     &ring->ena_com_io_sq,
1300                     &ring->ena_com_io_cq);
1301                 if (unlikely(rc != 0)) {
1302                         device_printf(adapter->pdev,
1303                             "Failed to get RX queue handlers. RX queue num"
1304                             " %d rc: %d\n", i, rc);
1305                         ena_com_destroy_io_queue(ena_dev, ena_qid);
1306                         goto err_rx;
1307                 }
1308         }
1309
1310         for (i = 0; i < adapter->num_queues; i++) {
1311                 queue = &adapter->que[i];
1312
1313                 NET_TASK_INIT(&queue->cleanup_task, 0, ena_cleanup, queue);
1314                 queue->cleanup_tq = taskqueue_create_fast("ena cleanup",
1315                     M_WAITOK, taskqueue_thread_enqueue, &queue->cleanup_tq);
1316
1317                 taskqueue_start_threads(&queue->cleanup_tq, 1, PI_NET,
1318                     "%s queue %d cleanup",
1319                     device_get_nameunit(adapter->pdev), i);
1320         }
1321
1322         return (0);
1323
1324 err_rx:
1325         while (i--)
1326                 ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(i));
1327         i = adapter->num_queues;
1328 err_tx:
1329         while (i--)
1330                 ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(i));
1331
1332         return (ENXIO);
1333 }
1334
1335 /*********************************************************************
1336  *
1337  *  MSIX & Interrupt Service routine
1338  *
1339  **********************************************************************/
1340
1341 /**
1342  * ena_handle_msix - MSIX Interrupt Handler for admin/async queue
1343  * @arg: interrupt number
1344  **/
1345 static void
1346 ena_intr_msix_mgmnt(void *arg)
1347 {
1348         struct ena_adapter *adapter = (struct ena_adapter *)arg;
1349
1350         ena_com_admin_q_comp_intr_handler(adapter->ena_dev);
1351         if (likely(ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter)))
1352                 ena_com_aenq_intr_handler(adapter->ena_dev, arg);
1353 }
1354
1355 /**
1356  * ena_handle_msix - MSIX Interrupt Handler for Tx/Rx
1357  * @arg: queue
1358  **/
1359 static int
1360 ena_handle_msix(void *arg)
1361 {
1362         struct ena_que *queue = arg;
1363         struct ena_adapter *adapter = queue->adapter;
1364         if_t ifp = adapter->ifp;
1365
1366         if (unlikely((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0))
1367                 return (FILTER_STRAY);
1368
1369         taskqueue_enqueue(queue->cleanup_tq, &queue->cleanup_task);
1370
1371         return (FILTER_HANDLED);
1372 }
1373
1374 static int
1375 ena_enable_msix(struct ena_adapter *adapter)
1376 {
1377         device_t dev = adapter->pdev;
1378         int msix_vecs, msix_req;
1379         int i, rc = 0;
1380
1381         if (ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter)) {
1382                 device_printf(dev, "Error, MSI-X is already enabled\n");
1383                 return (EINVAL);
1384         }
1385
1386         /* Reserved the max msix vectors we might need */
1387         msix_vecs = ENA_MAX_MSIX_VEC(adapter->num_queues);
1388
1389         adapter->msix_entries = malloc(msix_vecs * sizeof(struct msix_entry),
1390             M_DEVBUF, M_WAITOK | M_ZERO);
1391
1392         ena_trace(ENA_DBG, "trying to enable MSI-X, vectors: %d\n", msix_vecs);
1393
1394         for (i = 0; i < msix_vecs; i++) {
1395                 adapter->msix_entries[i].entry = i;
1396                 /* Vectors must start from 1 */
1397                 adapter->msix_entries[i].vector = i + 1;
1398         }
1399
1400         msix_req = msix_vecs;
1401         rc = pci_alloc_msix(dev, &msix_vecs);
1402         if (unlikely(rc != 0)) {
1403                 device_printf(dev,
1404                     "Failed to enable MSIX, vectors %d rc %d\n", msix_vecs, rc);
1405
1406                 rc = ENOSPC;
1407                 goto err_msix_free;
1408         }
1409
1410         if (msix_vecs != msix_req) {
1411                 if (msix_vecs == ENA_ADMIN_MSIX_VEC) {
1412                         device_printf(dev,
1413                             "Not enough number of MSI-x allocated: %d\n",
1414                             msix_vecs);
1415                         pci_release_msi(dev);
1416                         rc = ENOSPC;
1417                         goto err_msix_free;
1418                 }
1419                 device_printf(dev, "Enable only %d MSI-x (out of %d), reduce "
1420                     "the number of queues\n", msix_vecs, msix_req);
1421                 adapter->num_queues = msix_vecs - ENA_ADMIN_MSIX_VEC;
1422         }
1423
1424         adapter->msix_vecs = msix_vecs;
1425         ENA_FLAG_SET_ATOMIC(ENA_FLAG_MSIX_ENABLED, adapter);
1426
1427         return (0);
1428
1429 err_msix_free:
1430         free(adapter->msix_entries, M_DEVBUF);
1431         adapter->msix_entries = NULL;
1432
1433         return (rc);
1434 }
1435
1436 static void
1437 ena_setup_mgmnt_intr(struct ena_adapter *adapter)
1438 {
1439
1440         snprintf(adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].name,
1441             ENA_IRQNAME_SIZE, "ena-mgmnt@pci:%s",
1442             device_get_nameunit(adapter->pdev));
1443         /*
1444          * Handler is NULL on purpose, it will be set
1445          * when mgmnt interrupt is acquired
1446          */
1447         adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].handler = NULL;
1448         adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter;
1449         adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector =
1450             adapter->msix_entries[ENA_MGMNT_IRQ_IDX].vector;
1451 }
1452
1453 static int
1454 ena_setup_io_intr(struct ena_adapter *adapter)
1455 {
1456         static int last_bind_cpu = -1;
1457         int irq_idx;
1458
1459         if (adapter->msix_entries == NULL)
1460                 return (EINVAL);
1461
1462         for (int i = 0; i < adapter->num_queues; i++) {
1463                 irq_idx = ENA_IO_IRQ_IDX(i);
1464
1465                 snprintf(adapter->irq_tbl[irq_idx].name, ENA_IRQNAME_SIZE,
1466                     "%s-TxRx-%d", device_get_nameunit(adapter->pdev), i);
1467                 adapter->irq_tbl[irq_idx].handler = ena_handle_msix;
1468                 adapter->irq_tbl[irq_idx].data = &adapter->que[i];
1469                 adapter->irq_tbl[irq_idx].vector =
1470                     adapter->msix_entries[irq_idx].vector;
1471                 ena_trace(ENA_INFO | ENA_IOQ, "ena_setup_io_intr vector: %d\n",
1472                     adapter->msix_entries[irq_idx].vector);
1473
1474                 /*
1475                  * We want to bind rings to the corresponding cpu
1476                  * using something similar to the RSS round-robin technique.
1477                  */
1478                 if (unlikely(last_bind_cpu < 0))
1479                         last_bind_cpu = CPU_FIRST();
1480                 adapter->que[i].cpu = adapter->irq_tbl[irq_idx].cpu =
1481                     last_bind_cpu;
1482                 last_bind_cpu = CPU_NEXT(last_bind_cpu);
1483         }
1484
1485         return (0);
1486 }
1487
1488 static int
1489 ena_request_mgmnt_irq(struct ena_adapter *adapter)
1490 {
1491         struct ena_irq *irq;
1492         unsigned long flags;
1493         int rc, rcc;
1494
1495         flags = RF_ACTIVE | RF_SHAREABLE;
1496
1497         irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
1498         irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ,
1499             &irq->vector, flags);
1500
1501         if (unlikely(irq->res == NULL)) {
1502                 device_printf(adapter->pdev, "could not allocate "
1503                     "irq vector: %d\n", irq->vector);
1504                 return (ENXIO);
1505         }
1506
1507         rc = bus_setup_intr(adapter->pdev, irq->res,
1508             INTR_TYPE_NET | INTR_MPSAFE, NULL, ena_intr_msix_mgmnt,
1509             irq->data, &irq->cookie);
1510         if (unlikely(rc != 0)) {
1511                 device_printf(adapter->pdev, "failed to register "
1512                     "interrupt handler for irq %ju: %d\n",
1513                     rman_get_start(irq->res), rc);
1514                 goto err_res_free;
1515         }
1516         irq->requested = true;
1517
1518         return (rc);
1519
1520 err_res_free:
1521         ena_trace(ENA_INFO | ENA_ADMQ, "releasing resource for irq %d\n",
1522             irq->vector);
1523         rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ,
1524             irq->vector, irq->res);
1525         if (unlikely(rcc != 0))
1526                 device_printf(adapter->pdev, "dev has no parent while "
1527                     "releasing res for irq: %d\n", irq->vector);
1528         irq->res = NULL;
1529
1530         return (rc);
1531 }
1532
1533 static int
1534 ena_request_io_irq(struct ena_adapter *adapter)
1535 {
1536         struct ena_irq *irq;
1537         unsigned long flags = 0;
1538         int rc = 0, i, rcc;
1539
1540         if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter))) {
1541                 device_printf(adapter->pdev,
1542                     "failed to request I/O IRQ: MSI-X is not enabled\n");
1543                 return (EINVAL);
1544         } else {
1545                 flags = RF_ACTIVE | RF_SHAREABLE;
1546         }
1547
1548         for (i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) {
1549                 irq = &adapter->irq_tbl[i];
1550
1551                 if (unlikely(irq->requested))
1552                         continue;
1553
1554                 irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ,
1555                     &irq->vector, flags);
1556                 if (unlikely(irq->res == NULL)) {
1557                         rc = ENOMEM;
1558                         device_printf(adapter->pdev, "could not allocate "
1559                             "irq vector: %d\n", irq->vector);
1560                         goto err;
1561                 }
1562
1563                 rc = bus_setup_intr(adapter->pdev, irq->res,
1564                     INTR_TYPE_NET | INTR_MPSAFE, irq->handler, NULL,
1565                     irq->data, &irq->cookie);
1566                  if (unlikely(rc != 0)) {
1567                         device_printf(adapter->pdev, "failed to register "
1568                             "interrupt handler for irq %ju: %d\n",
1569                             rman_get_start(irq->res), rc);
1570                         goto err;
1571                 }
1572                 irq->requested = true;
1573
1574                 ena_trace(ENA_INFO, "queue %d - cpu %d\n",
1575                     i - ENA_IO_IRQ_FIRST_IDX, irq->cpu);
1576         }
1577
1578         return (rc);
1579
1580 err:
1581
1582         for (; i >= ENA_IO_IRQ_FIRST_IDX; i--) {
1583                 irq = &adapter->irq_tbl[i];
1584                 rcc = 0;
1585
1586                 /* Once we entered err: section and irq->requested is true we
1587                    free both intr and resources */
1588                 if (irq->requested)
1589                         rcc = bus_teardown_intr(adapter->pdev, irq->res, irq->cookie);
1590                 if (unlikely(rcc != 0))
1591                         device_printf(adapter->pdev, "could not release"
1592                             " irq: %d, error: %d\n", irq->vector, rcc);
1593
1594                 /* If we entred err: section without irq->requested set we know
1595                    it was bus_alloc_resource_any() that needs cleanup, provided
1596                    res is not NULL. In case res is NULL no work in needed in
1597                    this iteration */
1598                 rcc = 0;
1599                 if (irq->res != NULL) {
1600                         rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ,
1601                             irq->vector, irq->res);
1602                 }
1603                 if (unlikely(rcc != 0))
1604                         device_printf(adapter->pdev, "dev has no parent while "
1605                             "releasing res for irq: %d\n", irq->vector);
1606                 irq->requested = false;
1607                 irq->res = NULL;
1608         }
1609
1610         return (rc);
1611 }
1612
1613 static void
1614 ena_free_mgmnt_irq(struct ena_adapter *adapter)
1615 {
1616         struct ena_irq *irq;
1617         int rc;
1618
1619         irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
1620         if (irq->requested) {
1621                 ena_trace(ENA_INFO | ENA_ADMQ, "tear down irq: %d\n",
1622                     irq->vector);
1623                 rc = bus_teardown_intr(adapter->pdev, irq->res, irq->cookie);
1624                 if (unlikely(rc != 0))
1625                         device_printf(adapter->pdev, "failed to tear "
1626                             "down irq: %d\n", irq->vector);
1627                 irq->requested = 0;
1628         }
1629
1630         if (irq->res != NULL) {
1631                 ena_trace(ENA_INFO | ENA_ADMQ, "release resource irq: %d\n",
1632                     irq->vector);
1633                 rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ,
1634                     irq->vector, irq->res);
1635                 irq->res = NULL;
1636                 if (unlikely(rc != 0))
1637                         device_printf(adapter->pdev, "dev has no parent while "
1638                             "releasing res for irq: %d\n", irq->vector);
1639         }
1640 }
1641
1642 static void
1643 ena_free_io_irq(struct ena_adapter *adapter)
1644 {
1645         struct ena_irq *irq;
1646         int rc;
1647
1648         for (int i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) {
1649                 irq = &adapter->irq_tbl[i];
1650                 if (irq->requested) {
1651                         ena_trace(ENA_INFO | ENA_IOQ, "tear down irq: %d\n",
1652                             irq->vector);
1653                         rc = bus_teardown_intr(adapter->pdev, irq->res,
1654                             irq->cookie);
1655                         if (unlikely(rc != 0)) {
1656                                 device_printf(adapter->pdev, "failed to tear "
1657                                     "down irq: %d\n", irq->vector);
1658                         }
1659                         irq->requested = 0;
1660                 }
1661
1662                 if (irq->res != NULL) {
1663                         ena_trace(ENA_INFO | ENA_IOQ, "release resource irq: %d\n",
1664                             irq->vector);
1665                         rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ,
1666                             irq->vector, irq->res);
1667                         irq->res = NULL;
1668                         if (unlikely(rc != 0)) {
1669                                 device_printf(adapter->pdev, "dev has no parent"
1670                                     " while releasing res for irq: %d\n",
1671                                     irq->vector);
1672                         }
1673                 }
1674         }
1675 }
1676
1677 static void
1678 ena_free_irqs(struct ena_adapter* adapter)
1679 {
1680
1681         ena_free_io_irq(adapter);
1682         ena_free_mgmnt_irq(adapter);
1683         ena_disable_msix(adapter);
1684 }
1685
1686 static void
1687 ena_disable_msix(struct ena_adapter *adapter)
1688 {
1689
1690         if (ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter)) {
1691                 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_MSIX_ENABLED, adapter);
1692                 pci_release_msi(adapter->pdev);
1693         }
1694
1695         adapter->msix_vecs = 0;
1696         if (adapter->msix_entries != NULL)
1697                 free(adapter->msix_entries, M_DEVBUF);
1698         adapter->msix_entries = NULL;
1699 }
1700
1701 static void
1702 ena_unmask_all_io_irqs(struct ena_adapter *adapter)
1703 {
1704         struct ena_com_io_cq* io_cq;
1705         struct ena_eth_io_intr_reg intr_reg;
1706         uint16_t ena_qid;
1707         int i;
1708
1709         /* Unmask interrupts for all queues */
1710         for (i = 0; i < adapter->num_queues; i++) {
1711                 ena_qid = ENA_IO_TXQ_IDX(i);
1712                 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
1713                 ena_com_update_intr_reg(&intr_reg, 0, 0, true);
1714                 ena_com_unmask_intr(io_cq, &intr_reg);
1715         }
1716 }
1717
1718 /* Configure the Rx forwarding */
1719 static int
1720 ena_rss_configure(struct ena_adapter *adapter)
1721 {
1722         struct ena_com_dev *ena_dev = adapter->ena_dev;
1723         int rc;
1724
1725         /* Set indirect table */
1726         rc = ena_com_indirect_table_set(ena_dev);
1727         if (unlikely((rc != 0) && (rc != EOPNOTSUPP)))
1728                 return (rc);
1729
1730         /* Configure hash function (if supported) */
1731         rc = ena_com_set_hash_function(ena_dev);
1732         if (unlikely((rc != 0) && (rc != EOPNOTSUPP)))
1733                 return (rc);
1734
1735         /* Configure hash inputs (if supported) */
1736         rc = ena_com_set_hash_ctrl(ena_dev);
1737         if (unlikely((rc != 0) && (rc != EOPNOTSUPP)))
1738                 return (rc);
1739
1740         return (0);
1741 }
1742
1743 static int
1744 ena_up_complete(struct ena_adapter *adapter)
1745 {
1746         int rc;
1747
1748         if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) {
1749                 rc = ena_rss_configure(adapter);
1750                 if (rc != 0)
1751                         return (rc);
1752         }
1753
1754         rc = ena_change_mtu(adapter->ifp, adapter->ifp->if_mtu);
1755         if (unlikely(rc != 0))
1756                 return (rc);
1757
1758         ena_refill_all_rx_bufs(adapter);
1759         ena_reset_counters((counter_u64_t *)&adapter->hw_stats,
1760             sizeof(adapter->hw_stats));
1761
1762         return (0);
1763 }
1764
1765 int
1766 ena_up(struct ena_adapter *adapter)
1767 {
1768         int rc = 0;
1769
1770         if (unlikely(device_is_attached(adapter->pdev) == 0)) {
1771                 device_printf(adapter->pdev, "device is not attached!\n");
1772                 return (ENXIO);
1773         }
1774
1775         if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) {
1776                 device_printf(adapter->pdev, "device is going UP\n");
1777
1778                 /* setup interrupts for IO queues */
1779                 rc = ena_setup_io_intr(adapter);
1780                 if (unlikely(rc != 0)) {
1781                         ena_trace(ENA_ALERT, "error setting up IO interrupt\n");
1782                         goto error;
1783                 }
1784                 rc = ena_request_io_irq(adapter);
1785                 if (unlikely(rc != 0)) {
1786                         ena_trace(ENA_ALERT, "err_req_irq\n");
1787                         goto error;
1788                 }
1789
1790                 /* allocate transmit descriptors */
1791                 rc = ena_setup_all_tx_resources(adapter);
1792                 if (unlikely(rc != 0)) {
1793                         ena_trace(ENA_ALERT, "err_setup_tx\n");
1794                         goto err_setup_tx;
1795                 }
1796
1797                 /* allocate receive descriptors */
1798                 rc = ena_setup_all_rx_resources(adapter);
1799                 if (unlikely(rc != 0)) {
1800                         ena_trace(ENA_ALERT, "err_setup_rx\n");
1801                         goto err_setup_rx;
1802                 }
1803
1804                 /* create IO queues for Rx & Tx */
1805                 rc = ena_create_io_queues(adapter);
1806                 if (unlikely(rc != 0)) {
1807                         ena_trace(ENA_ALERT,
1808                             "create IO queues failed\n");
1809                         goto err_io_que;
1810                 }
1811
1812                 if (ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter))
1813                         if_link_state_change(adapter->ifp, LINK_STATE_UP);
1814
1815                 rc = ena_up_complete(adapter);
1816                 if (unlikely(rc != 0))
1817                         goto err_up_complete;
1818
1819                 counter_u64_add(adapter->dev_stats.interface_up, 1);
1820
1821                 ena_update_hwassist(adapter);
1822
1823                 if_setdrvflagbits(adapter->ifp, IFF_DRV_RUNNING,
1824                     IFF_DRV_OACTIVE);
1825
1826                 /* Activate timer service only if the device is running.
1827                  * If this flag is not set, it means that the driver is being
1828                  * reset and timer service will be activated afterwards.
1829                  */
1830                 if (ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter)) {
1831                         callout_reset_sbt(&adapter->timer_service, SBT_1S,
1832                             SBT_1S, ena_timer_service, (void *)adapter, 0);
1833                 }
1834
1835                 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP, adapter);
1836
1837                 ena_unmask_all_io_irqs(adapter);
1838         }
1839
1840         return (0);
1841
1842 err_up_complete:
1843         ena_destroy_all_io_queues(adapter);
1844 err_io_que:
1845         ena_free_all_rx_resources(adapter);
1846 err_setup_rx:
1847         ena_free_all_tx_resources(adapter);
1848 err_setup_tx:
1849         ena_free_io_irq(adapter);
1850 error:
1851         return (rc);
1852 }
1853
1854 static uint64_t
1855 ena_get_counter(if_t ifp, ift_counter cnt)
1856 {
1857         struct ena_adapter *adapter;
1858         struct ena_hw_stats *stats;
1859
1860         adapter = if_getsoftc(ifp);
1861         stats = &adapter->hw_stats;
1862
1863         switch (cnt) {
1864         case IFCOUNTER_IPACKETS:
1865                 return (counter_u64_fetch(stats->rx_packets));
1866         case IFCOUNTER_OPACKETS:
1867                 return (counter_u64_fetch(stats->tx_packets));
1868         case IFCOUNTER_IBYTES:
1869                 return (counter_u64_fetch(stats->rx_bytes));
1870         case IFCOUNTER_OBYTES:
1871                 return (counter_u64_fetch(stats->tx_bytes));
1872         case IFCOUNTER_IQDROPS:
1873                 return (counter_u64_fetch(stats->rx_drops));
1874         default:
1875                 return (if_get_counter_default(ifp, cnt));
1876         }
1877 }
1878
1879 static int
1880 ena_media_change(if_t ifp)
1881 {
1882         /* Media Change is not supported by firmware */
1883         return (0);
1884 }
1885
1886 static void
1887 ena_media_status(if_t ifp, struct ifmediareq *ifmr)
1888 {
1889         struct ena_adapter *adapter = if_getsoftc(ifp);
1890         ena_trace(ENA_DBG, "enter\n");
1891
1892         mtx_lock(&adapter->global_mtx);
1893
1894         ifmr->ifm_status = IFM_AVALID;
1895         ifmr->ifm_active = IFM_ETHER;
1896
1897         if (!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)) {
1898                 mtx_unlock(&adapter->global_mtx);
1899                 ena_trace(ENA_INFO, "Link is down\n");
1900                 return;
1901         }
1902
1903         ifmr->ifm_status |= IFM_ACTIVE;
1904         ifmr->ifm_active |= IFM_UNKNOWN | IFM_FDX;
1905
1906         mtx_unlock(&adapter->global_mtx);
1907 }
1908
1909 static void
1910 ena_init(void *arg)
1911 {
1912         struct ena_adapter *adapter = (struct ena_adapter *)arg;
1913
1914         if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) {
1915                 sx_xlock(&adapter->ioctl_sx);
1916                 ena_up(adapter);
1917                 sx_unlock(&adapter->ioctl_sx);
1918         }
1919 }
1920
1921 static int
1922 ena_ioctl(if_t ifp, u_long command, caddr_t data)
1923 {
1924         struct ena_adapter *adapter;
1925         struct ifreq *ifr;
1926         int rc;
1927
1928         adapter = ifp->if_softc;
1929         ifr = (struct ifreq *)data;
1930
1931         /*
1932          * Acquiring lock to prevent from running up and down routines parallel.
1933          */
1934         rc = 0;
1935         switch (command) {
1936         case SIOCSIFMTU:
1937                 if (ifp->if_mtu == ifr->ifr_mtu)
1938                         break;
1939                 sx_xlock(&adapter->ioctl_sx);
1940                 ena_down(adapter);
1941
1942                 ena_change_mtu(ifp, ifr->ifr_mtu);
1943
1944                 rc = ena_up(adapter);
1945                 sx_unlock(&adapter->ioctl_sx);
1946                 break;
1947
1948         case SIOCSIFFLAGS:
1949                 if ((ifp->if_flags & IFF_UP) != 0) {
1950                         if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
1951                                 if ((ifp->if_flags & (IFF_PROMISC |
1952                                     IFF_ALLMULTI)) != 0) {
1953                                         device_printf(adapter->pdev,
1954                                             "ioctl promisc/allmulti\n");
1955                                 }
1956                         } else {
1957                                 sx_xlock(&adapter->ioctl_sx);
1958                                 rc = ena_up(adapter);
1959                                 sx_unlock(&adapter->ioctl_sx);
1960                         }
1961                 } else {
1962                         if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
1963                                 sx_xlock(&adapter->ioctl_sx);
1964                                 ena_down(adapter);
1965                                 sx_unlock(&adapter->ioctl_sx);
1966                         }
1967                 }
1968                 break;
1969
1970         case SIOCADDMULTI:
1971         case SIOCDELMULTI:
1972                 break;
1973
1974         case SIOCSIFMEDIA:
1975         case SIOCGIFMEDIA:
1976                 rc = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1977                 break;
1978
1979         case SIOCSIFCAP:
1980                 {
1981                         int reinit = 0;
1982
1983                         if (ifr->ifr_reqcap != ifp->if_capenable) {
1984                                 ifp->if_capenable = ifr->ifr_reqcap;
1985                                 reinit = 1;
1986                         }
1987
1988                         if ((reinit != 0) &&
1989                             ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0)) {
1990                                 sx_xlock(&adapter->ioctl_sx);
1991                                 ena_down(adapter);
1992                                 rc = ena_up(adapter);
1993                                 sx_unlock(&adapter->ioctl_sx);
1994                         }
1995                 }
1996
1997                 break;
1998         default:
1999                 rc = ether_ioctl(ifp, command, data);
2000                 break;
2001         }
2002
2003         return (rc);
2004 }
2005
2006 static int
2007 ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *feat)
2008 {
2009         int caps = 0;
2010
2011         if ((feat->offload.tx &
2012             (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK |
2013             ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK |
2014             ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK)) != 0)
2015                 caps |= IFCAP_TXCSUM;
2016
2017         if ((feat->offload.tx &
2018             (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK |
2019             ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK)) != 0)
2020                 caps |= IFCAP_TXCSUM_IPV6;
2021
2022         if ((feat->offload.tx &
2023             ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) != 0)
2024                 caps |= IFCAP_TSO4;
2025
2026         if ((feat->offload.tx &
2027             ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK) != 0)
2028                 caps |= IFCAP_TSO6;
2029
2030         if ((feat->offload.rx_supported &
2031             (ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK |
2032             ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK)) != 0)
2033                 caps |= IFCAP_RXCSUM;
2034
2035         if ((feat->offload.rx_supported &
2036             ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK) != 0)
2037                 caps |= IFCAP_RXCSUM_IPV6;
2038
2039         caps |= IFCAP_LRO | IFCAP_JUMBO_MTU;
2040
2041         return (caps);
2042 }
2043
2044 static void
2045 ena_update_host_info(struct ena_admin_host_info *host_info, if_t ifp)
2046 {
2047
2048         host_info->supported_network_features[0] =
2049             (uint32_t)if_getcapabilities(ifp);
2050 }
2051
2052 static void
2053 ena_update_hwassist(struct ena_adapter *adapter)
2054 {
2055         if_t ifp = adapter->ifp;
2056         uint32_t feat = adapter->tx_offload_cap;
2057         int cap = if_getcapenable(ifp);
2058         int flags = 0;
2059
2060         if_clearhwassist(ifp);
2061
2062         if ((cap & IFCAP_TXCSUM) != 0) {
2063                 if ((feat &
2064                     ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK) != 0)
2065                         flags |= CSUM_IP;
2066                 if ((feat &
2067                     (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK |
2068                     ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK)) != 0)
2069                         flags |= CSUM_IP_UDP | CSUM_IP_TCP;
2070         }
2071
2072         if ((cap & IFCAP_TXCSUM_IPV6) != 0)
2073                 flags |= CSUM_IP6_UDP | CSUM_IP6_TCP;
2074
2075         if ((cap & IFCAP_TSO4) != 0)
2076                 flags |= CSUM_IP_TSO;
2077
2078         if ((cap & IFCAP_TSO6) != 0)
2079                 flags |= CSUM_IP6_TSO;
2080
2081         if_sethwassistbits(ifp, flags, 0);
2082 }
2083
2084 static int
2085 ena_setup_ifnet(device_t pdev, struct ena_adapter *adapter,
2086     struct ena_com_dev_get_features_ctx *feat)
2087 {
2088         if_t ifp;
2089         int caps = 0;
2090
2091         ifp = adapter->ifp = if_gethandle(IFT_ETHER);
2092         if (unlikely(ifp == NULL)) {
2093                 ena_trace(ENA_ALERT, "can not allocate ifnet structure\n");
2094                 return (ENXIO);
2095         }
2096         if_initname(ifp, device_get_name(pdev), device_get_unit(pdev));
2097         if_setdev(ifp, pdev);
2098         if_setsoftc(ifp, adapter);
2099
2100         if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
2101         if_setinitfn(ifp, ena_init);
2102         if_settransmitfn(ifp, ena_mq_start);
2103         if_setqflushfn(ifp, ena_qflush);
2104         if_setioctlfn(ifp, ena_ioctl);
2105         if_setgetcounterfn(ifp, ena_get_counter);
2106
2107         if_setsendqlen(ifp, adapter->tx_ring_size);
2108         if_setsendqready(ifp);
2109         if_setmtu(ifp, ETHERMTU);
2110         if_setbaudrate(ifp, 0);
2111         /* Zeroize capabilities... */
2112         if_setcapabilities(ifp, 0);
2113         if_setcapenable(ifp, 0);
2114         /* check hardware support */
2115         caps = ena_get_dev_offloads(feat);
2116         /* ... and set them */
2117         if_setcapabilitiesbit(ifp, caps, 0);
2118
2119         /* TSO parameters */
2120         ifp->if_hw_tsomax = ENA_TSO_MAXSIZE -
2121             (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
2122         ifp->if_hw_tsomaxsegcount = adapter->max_tx_sgl_size - 1;
2123         ifp->if_hw_tsomaxsegsize = ENA_TSO_MAXSIZE;
2124
2125         if_setifheaderlen(ifp, sizeof(struct ether_vlan_header));
2126         if_setcapenable(ifp, if_getcapabilities(ifp));
2127
2128         /*
2129          * Specify the media types supported by this adapter and register
2130          * callbacks to update media and link information
2131          */
2132         ifmedia_init(&adapter->media, IFM_IMASK,
2133             ena_media_change, ena_media_status);
2134         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2135         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2136
2137         ether_ifattach(ifp, adapter->mac_addr);
2138
2139         return (0);
2140 }
2141
2142 void
2143 ena_down(struct ena_adapter *adapter)
2144 {
2145         int rc;
2146
2147         if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) {
2148                 device_printf(adapter->pdev, "device is going DOWN\n");
2149
2150                 callout_drain(&adapter->timer_service);
2151
2152                 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP, adapter);
2153                 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE,
2154                     IFF_DRV_RUNNING);
2155
2156                 ena_free_io_irq(adapter);
2157
2158                 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) {
2159                         rc = ena_com_dev_reset(adapter->ena_dev,
2160                             adapter->reset_reason);
2161                         if (unlikely(rc != 0))
2162                                 device_printf(adapter->pdev,
2163                                     "Device reset failed\n");
2164                 }
2165
2166                 ena_destroy_all_io_queues(adapter);
2167
2168                 ena_free_all_tx_bufs(adapter);
2169                 ena_free_all_rx_bufs(adapter);
2170                 ena_free_all_tx_resources(adapter);
2171                 ena_free_all_rx_resources(adapter);
2172
2173                 counter_u64_add(adapter->dev_stats.interface_down, 1);
2174         }
2175 }
2176
2177 static int
2178 ena_calc_io_queue_num(struct ena_adapter *adapter,
2179     struct ena_com_dev_get_features_ctx *get_feat_ctx)
2180 {
2181         struct ena_com_dev *ena_dev = adapter->ena_dev;
2182         int io_tx_sq_num, io_tx_cq_num, io_rx_num, io_queue_num;
2183
2184         /* Regular queues capabilities */
2185         if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
2186                 struct ena_admin_queue_ext_feature_fields *max_queue_ext =
2187                     &get_feat_ctx->max_queue_ext.max_queue_ext;
2188                 io_rx_num = min_t(int, max_queue_ext->max_rx_sq_num,
2189                         max_queue_ext->max_rx_cq_num);
2190
2191                 io_tx_sq_num = max_queue_ext->max_tx_sq_num;
2192                 io_tx_cq_num = max_queue_ext->max_tx_cq_num;
2193         } else {
2194                 struct ena_admin_queue_feature_desc *max_queues =
2195                     &get_feat_ctx->max_queues;
2196                 io_tx_sq_num = max_queues->max_sq_num;
2197                 io_tx_cq_num = max_queues->max_cq_num;
2198                 io_rx_num = min_t(int, io_tx_sq_num, io_tx_cq_num);
2199         }
2200
2201         /* In case of LLQ use the llq fields for the tx SQ/CQ */
2202         if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
2203                 io_tx_sq_num = get_feat_ctx->llq.max_llq_num;
2204
2205         io_queue_num = min_t(int, mp_ncpus, ENA_MAX_NUM_IO_QUEUES);
2206         io_queue_num = min_t(int, io_queue_num, io_rx_num);
2207         io_queue_num = min_t(int, io_queue_num, io_tx_sq_num);
2208         io_queue_num = min_t(int, io_queue_num, io_tx_cq_num);
2209         /* 1 IRQ for for mgmnt and 1 IRQ for each TX/RX pair */
2210         io_queue_num = min_t(int, io_queue_num,
2211             pci_msix_count(adapter->pdev) - 1);
2212
2213         return (io_queue_num);
2214 }
2215
2216 static int
2217 ena_enable_wc(struct resource *res)
2218 {
2219 #if defined(__i386) || defined(__amd64) || defined(__aarch64__)
2220         vm_offset_t va;
2221         vm_size_t len;
2222         int rc;
2223
2224         va = (vm_offset_t)rman_get_virtual(res);
2225         len = rman_get_size(res);
2226         /* Enable write combining */
2227         rc = pmap_change_attr(va, len, VM_MEMATTR_WRITE_COMBINING);
2228         if (unlikely(rc != 0)) {
2229                 ena_trace(ENA_ALERT, "pmap_change_attr failed, %d\n", rc);
2230                 return (rc);
2231         }
2232
2233         return (0);
2234 #endif
2235         return (EOPNOTSUPP);
2236 }
2237
2238 static int
2239 ena_set_queues_placement_policy(device_t pdev, struct ena_com_dev *ena_dev,
2240     struct ena_admin_feature_llq_desc *llq,
2241     struct ena_llq_configurations *llq_default_configurations)
2242 {
2243         struct ena_adapter *adapter = device_get_softc(pdev);
2244         int rc, rid;
2245         uint32_t llq_feature_mask;
2246
2247         llq_feature_mask = 1 << ENA_ADMIN_LLQ;
2248         if (!(ena_dev->supported_features & llq_feature_mask)) {
2249                 device_printf(pdev,
2250                     "LLQ is not supported. Fallback to host mode policy.\n");
2251                 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
2252                 return (0);
2253         }
2254
2255         rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations);
2256         if (unlikely(rc != 0)) {
2257                 device_printf(pdev, "Failed to configure the device mode. "
2258                     "Fallback to host mode policy.\n");
2259                 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
2260                 return (0);
2261         }
2262
2263         /* Nothing to config, exit */
2264         if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
2265                 return (0);
2266
2267         /* Try to allocate resources for LLQ bar */
2268         rid = PCIR_BAR(ENA_MEM_BAR);
2269         adapter->memory = bus_alloc_resource_any(pdev, SYS_RES_MEMORY,
2270             &rid, RF_ACTIVE);
2271         if (unlikely(adapter->memory == NULL)) {
2272                 device_printf(pdev, "unable to allocate LLQ bar resource. "
2273                     "Fallback to host mode policy.\n");
2274                 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
2275                 return (0);
2276         }
2277
2278         /* Enable write combining for better LLQ performance */
2279         rc = ena_enable_wc(adapter->memory);
2280         if (unlikely(rc != 0)) {
2281                 device_printf(pdev, "failed to enable write combining.\n");
2282                 return (rc);
2283         }
2284
2285         /*
2286          * Save virtual address of the device's memory region
2287          * for the ena_com layer.
2288          */
2289         ena_dev->mem_bar = rman_get_virtual(adapter->memory);
2290
2291         return (0);
2292 }
2293
2294 static inline
2295 void set_default_llq_configurations(struct ena_llq_configurations *llq_config)
2296 {
2297         llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER;
2298         llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_128B;
2299         llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY;
2300         llq_config->llq_num_decs_before_header =
2301             ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2;
2302         llq_config->llq_ring_entry_size_value = 128;
2303 }
2304
2305 static int
2306 ena_calc_queue_size(struct ena_adapter *adapter,
2307     struct ena_calc_queue_size_ctx *ctx)
2308 {
2309         struct ena_admin_feature_llq_desc *llq = &ctx->get_feat_ctx->llq;
2310         struct ena_com_dev *ena_dev = ctx->ena_dev;
2311         uint32_t tx_queue_size = ENA_DEFAULT_RING_SIZE;
2312         uint32_t rx_queue_size = adapter->rx_ring_size;
2313
2314         if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
2315                 struct ena_admin_queue_ext_feature_fields *max_queue_ext =
2316                     &ctx->get_feat_ctx->max_queue_ext.max_queue_ext;
2317                 rx_queue_size = min_t(uint32_t, rx_queue_size,
2318                     max_queue_ext->max_rx_cq_depth);
2319                 rx_queue_size = min_t(uint32_t, rx_queue_size,
2320                     max_queue_ext->max_rx_sq_depth);
2321                 tx_queue_size = min_t(uint32_t, tx_queue_size,
2322                     max_queue_ext->max_tx_cq_depth);
2323
2324                 if (ena_dev->tx_mem_queue_type ==
2325                     ENA_ADMIN_PLACEMENT_POLICY_DEV)
2326                         tx_queue_size = min_t(uint32_t, tx_queue_size,
2327                             llq->max_llq_depth);
2328                 else
2329                         tx_queue_size = min_t(uint32_t, tx_queue_size,
2330                             max_queue_ext->max_tx_sq_depth);
2331
2332                 ctx->max_rx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS,
2333                     max_queue_ext->max_per_packet_rx_descs);
2334                 ctx->max_tx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS,
2335                     max_queue_ext->max_per_packet_tx_descs);
2336         } else {
2337                 struct ena_admin_queue_feature_desc *max_queues =
2338                     &ctx->get_feat_ctx->max_queues;
2339                 rx_queue_size = min_t(uint32_t, rx_queue_size,
2340                     max_queues->max_cq_depth);
2341                 rx_queue_size = min_t(uint32_t, rx_queue_size,
2342                     max_queues->max_sq_depth);
2343                 tx_queue_size = min_t(uint32_t, tx_queue_size,
2344                     max_queues->max_cq_depth);
2345
2346                 if (ena_dev->tx_mem_queue_type ==
2347                     ENA_ADMIN_PLACEMENT_POLICY_DEV)
2348                         tx_queue_size = min_t(uint32_t, tx_queue_size,
2349                             llq->max_llq_depth);
2350                 else
2351                         tx_queue_size = min_t(uint32_t, tx_queue_size,
2352                             max_queues->max_sq_depth);
2353
2354                 ctx->max_rx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS,
2355                     max_queues->max_packet_tx_descs);
2356                 ctx->max_tx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS,
2357                     max_queues->max_packet_rx_descs);
2358         }
2359
2360         /* round down to the nearest power of 2 */
2361         rx_queue_size = 1 << (fls(rx_queue_size) - 1);
2362         tx_queue_size = 1 << (fls(tx_queue_size) - 1);
2363
2364         if (unlikely(rx_queue_size == 0 || tx_queue_size == 0)) {
2365                 device_printf(ctx->pdev, "Invalid queue size\n");
2366                 return (EFAULT);
2367         }
2368
2369         ctx->rx_queue_size = rx_queue_size;
2370         ctx->tx_queue_size = tx_queue_size;
2371
2372         return (0);
2373 }
2374
2375 static int
2376 ena_handle_updated_queues(struct ena_adapter *adapter,
2377     struct ena_com_dev_get_features_ctx *get_feat_ctx)
2378 {
2379         struct ena_com_dev *ena_dev = adapter->ena_dev;
2380         struct ena_calc_queue_size_ctx calc_queue_ctx = { 0 };
2381         device_t pdev = adapter->pdev;
2382         bool are_queues_changed = false;
2383         int io_queue_num, rc;
2384
2385         calc_queue_ctx.ena_dev = ena_dev;
2386         calc_queue_ctx.get_feat_ctx = get_feat_ctx;
2387         calc_queue_ctx.pdev = pdev;
2388
2389         io_queue_num = ena_calc_io_queue_num(adapter, get_feat_ctx);
2390         rc = ena_calc_queue_size(adapter, &calc_queue_ctx);
2391         if (unlikely(rc != 0 || io_queue_num <= 0))
2392                 return EFAULT;
2393
2394         if (adapter->tx_ring->buf_ring_size != adapter->buf_ring_size)
2395                 are_queues_changed = true;
2396
2397         if (unlikely(adapter->tx_ring_size > calc_queue_ctx.tx_queue_size ||
2398             adapter->rx_ring_size > calc_queue_ctx.rx_queue_size)) {
2399                 device_printf(pdev,
2400                     "Not enough resources to allocate requested queue sizes "
2401                     "(TX,RX)=(%d,%d), falling back to queue sizes "
2402                     "(TX,RX)=(%d,%d)\n",
2403                     adapter->tx_ring_size,
2404                     adapter->rx_ring_size,
2405                     calc_queue_ctx.tx_queue_size,
2406                     calc_queue_ctx.rx_queue_size);
2407                 adapter->tx_ring_size = calc_queue_ctx.tx_queue_size;
2408                 adapter->rx_ring_size = calc_queue_ctx.rx_queue_size;
2409                 adapter->max_tx_sgl_size = calc_queue_ctx.max_tx_sgl_size;
2410                 adapter->max_rx_sgl_size = calc_queue_ctx.max_rx_sgl_size;
2411                 are_queues_changed = true;
2412         }
2413
2414         if (unlikely(adapter->num_queues > io_queue_num)) {
2415                 device_printf(pdev,
2416                     "Not enough resources to allocate %d queues, "
2417                     "falling back to %d queues\n",
2418                     adapter->num_queues, io_queue_num);
2419                 adapter->num_queues = io_queue_num;
2420                 if (ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter)) {
2421                         ena_com_rss_destroy(ena_dev);
2422                         rc = ena_rss_init_default(adapter);
2423                         if (unlikely(rc != 0) && (rc != EOPNOTSUPP)) {
2424                                 device_printf(pdev, "Cannot init RSS rc: %d\n",
2425                                     rc);
2426                                 return (rc);
2427                         }
2428                 }
2429                 are_queues_changed = true;
2430         }
2431
2432         if (unlikely(are_queues_changed)) {
2433                 ena_free_all_io_rings_resources(adapter);
2434                 ena_init_io_rings(adapter);
2435         }
2436
2437         return (0);
2438 }
2439
2440 static int
2441 ena_rss_init_default(struct ena_adapter *adapter)
2442 {
2443         struct ena_com_dev *ena_dev = adapter->ena_dev;
2444         device_t dev = adapter->pdev;
2445         int qid, rc, i;
2446
2447         rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE);
2448         if (unlikely(rc != 0)) {
2449                 device_printf(dev, "Cannot init indirect table\n");
2450                 return (rc);
2451         }
2452
2453         for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) {
2454                 qid = i % adapter->num_queues;
2455                 rc = ena_com_indirect_table_fill_entry(ena_dev, i,
2456                     ENA_IO_RXQ_IDX(qid));
2457                 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
2458                         device_printf(dev, "Cannot fill indirect table\n");
2459                         goto err_rss_destroy;
2460                 }
2461         }
2462
2463         rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL,
2464             ENA_HASH_KEY_SIZE, 0xFFFFFFFF);
2465         if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
2466                 device_printf(dev, "Cannot fill hash function\n");
2467                 goto err_rss_destroy;
2468         }
2469
2470         rc = ena_com_set_default_hash_ctrl(ena_dev);
2471         if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
2472                 device_printf(dev, "Cannot fill hash control\n");
2473                 goto err_rss_destroy;
2474         }
2475
2476         return (0);
2477
2478 err_rss_destroy:
2479         ena_com_rss_destroy(ena_dev);
2480         return (rc);
2481 }
2482
2483 static void
2484 ena_rss_init_default_deferred(void *arg)
2485 {
2486         struct ena_adapter *adapter;
2487         devclass_t dc;
2488         int max;
2489         int rc;
2490
2491         dc = devclass_find("ena");
2492         if (unlikely(dc == NULL)) {
2493                 ena_trace(ENA_ALERT, "No devclass ena\n");
2494                 return;
2495         }
2496
2497         max = devclass_get_maxunit(dc);
2498         while (max-- >= 0) {
2499                 adapter = devclass_get_softc(dc, max);
2500                 if (adapter != NULL) {
2501                         rc = ena_rss_init_default(adapter);
2502                         ENA_FLAG_SET_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter);
2503                         if (unlikely(rc != 0)) {
2504                                 device_printf(adapter->pdev,
2505                                     "WARNING: RSS was not properly initialized,"
2506                                     " it will affect bandwidth\n");
2507                                 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter);
2508                         }
2509                 }
2510         }
2511 }
2512 SYSINIT(ena_rss_init, SI_SUB_KICK_SCHEDULER, SI_ORDER_SECOND, ena_rss_init_default_deferred, NULL);
2513
2514 static void
2515 ena_config_host_info(struct ena_com_dev *ena_dev, device_t dev)
2516 {
2517         struct ena_admin_host_info *host_info;
2518         uintptr_t rid;
2519         int rc;
2520
2521         /* Allocate only the host info */
2522         rc = ena_com_allocate_host_info(ena_dev);
2523         if (unlikely(rc != 0)) {
2524                 ena_trace(ENA_ALERT, "Cannot allocate host info\n");
2525                 return;
2526         }
2527
2528         host_info = ena_dev->host_attr.host_info;
2529
2530         if (pci_get_id(dev, PCI_ID_RID, &rid) == 0)
2531                 host_info->bdf = rid;
2532         host_info->os_type = ENA_ADMIN_OS_FREEBSD;
2533         host_info->kernel_ver = osreldate;
2534
2535         sprintf(host_info->kernel_ver_str, "%d", osreldate);
2536         host_info->os_dist = 0;
2537         strncpy(host_info->os_dist_str, osrelease,
2538             sizeof(host_info->os_dist_str) - 1);
2539
2540         host_info->driver_version =
2541                 (DRV_MODULE_VER_MAJOR) |
2542                 (DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) |
2543                 (DRV_MODULE_VER_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT);
2544         host_info->num_cpus = mp_ncpus;
2545
2546         rc = ena_com_set_host_attributes(ena_dev);
2547         if (unlikely(rc != 0)) {
2548                 if (rc == EOPNOTSUPP)
2549                         ena_trace(ENA_WARNING, "Cannot set host attributes\n");
2550                 else
2551                         ena_trace(ENA_ALERT, "Cannot set host attributes\n");
2552
2553                 goto err;
2554         }
2555
2556         return;
2557
2558 err:
2559         ena_com_delete_host_info(ena_dev);
2560 }
2561
2562 static int
2563 ena_device_init(struct ena_adapter *adapter, device_t pdev,
2564     struct ena_com_dev_get_features_ctx *get_feat_ctx, int *wd_active)
2565 {
2566         struct ena_com_dev* ena_dev = adapter->ena_dev;
2567         bool readless_supported;
2568         uint32_t aenq_groups;
2569         int dma_width;
2570         int rc;
2571
2572         rc = ena_com_mmio_reg_read_request_init(ena_dev);
2573         if (unlikely(rc != 0)) {
2574                 device_printf(pdev, "failed to init mmio read less\n");
2575                 return (rc);
2576         }
2577
2578         /*
2579          * The PCIe configuration space revision id indicate if mmio reg
2580          * read is disabled
2581          */
2582         readless_supported = !(pci_get_revid(pdev) & ENA_MMIO_DISABLE_REG_READ);
2583         ena_com_set_mmio_read_mode(ena_dev, readless_supported);
2584
2585         rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL);
2586         if (unlikely(rc != 0)) {
2587                 device_printf(pdev, "Can not reset device\n");
2588                 goto err_mmio_read_less;
2589         }
2590
2591         rc = ena_com_validate_version(ena_dev);
2592         if (unlikely(rc != 0)) {
2593                 device_printf(pdev, "device version is too low\n");
2594                 goto err_mmio_read_less;
2595         }
2596
2597         dma_width = ena_com_get_dma_width(ena_dev);
2598         if (unlikely(dma_width < 0)) {
2599                 device_printf(pdev, "Invalid dma width value %d", dma_width);
2600                 rc = dma_width;
2601                 goto err_mmio_read_less;
2602         }
2603         adapter->dma_width = dma_width;
2604
2605         /* ENA admin level init */
2606         rc = ena_com_admin_init(ena_dev, &aenq_handlers);
2607         if (unlikely(rc != 0)) {
2608                 device_printf(pdev,
2609                     "Can not initialize ena admin queue with device\n");
2610                 goto err_mmio_read_less;
2611         }
2612
2613         /*
2614          * To enable the msix interrupts the driver needs to know the number
2615          * of queues. So the driver uses polling mode to retrieve this
2616          * information
2617          */
2618         ena_com_set_admin_polling_mode(ena_dev, true);
2619
2620         ena_config_host_info(ena_dev, pdev);
2621
2622         /* Get Device Attributes */
2623         rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx);
2624         if (unlikely(rc != 0)) {
2625                 device_printf(pdev,
2626                     "Cannot get attribute for ena device rc: %d\n", rc);
2627                 goto err_admin_init;
2628         }
2629
2630         aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) |
2631             BIT(ENA_ADMIN_FATAL_ERROR) |
2632             BIT(ENA_ADMIN_WARNING) |
2633             BIT(ENA_ADMIN_NOTIFICATION) |
2634             BIT(ENA_ADMIN_KEEP_ALIVE);
2635
2636         aenq_groups &= get_feat_ctx->aenq.supported_groups;
2637         rc = ena_com_set_aenq_config(ena_dev, aenq_groups);
2638         if (unlikely(rc != 0)) {
2639                 device_printf(pdev, "Cannot configure aenq groups rc: %d\n", rc);
2640                 goto err_admin_init;
2641         }
2642
2643         *wd_active = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE));
2644
2645         return (0);
2646
2647 err_admin_init:
2648         ena_com_delete_host_info(ena_dev);
2649         ena_com_admin_destroy(ena_dev);
2650 err_mmio_read_less:
2651         ena_com_mmio_reg_read_request_destroy(ena_dev);
2652
2653         return (rc);
2654 }
2655
2656 static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter,
2657     int io_vectors)
2658 {
2659         struct ena_com_dev *ena_dev = adapter->ena_dev;
2660         int rc;
2661
2662         rc = ena_enable_msix(adapter);
2663         if (unlikely(rc != 0)) {
2664                 device_printf(adapter->pdev, "Error with MSI-X enablement\n");
2665                 return (rc);
2666         }
2667
2668         ena_setup_mgmnt_intr(adapter);
2669
2670         rc = ena_request_mgmnt_irq(adapter);
2671         if (unlikely(rc != 0)) {
2672                 device_printf(adapter->pdev, "Cannot setup mgmnt queue intr\n");
2673                 goto err_disable_msix;
2674         }
2675
2676         ena_com_set_admin_polling_mode(ena_dev, false);
2677
2678         ena_com_admin_aenq_enable(ena_dev);
2679
2680         return (0);
2681
2682 err_disable_msix:
2683         ena_disable_msix(adapter);
2684
2685         return (rc);
2686 }
2687
2688 /* Function called on ENA_ADMIN_KEEP_ALIVE event */
2689 static void ena_keep_alive_wd(void *adapter_data,
2690     struct ena_admin_aenq_entry *aenq_e)
2691 {
2692         struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
2693         struct ena_admin_aenq_keep_alive_desc *desc;
2694         sbintime_t stime;
2695         uint64_t rx_drops;
2696
2697         desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e;
2698
2699         rx_drops = ((uint64_t)desc->rx_drops_high << 32) | desc->rx_drops_low;
2700         counter_u64_zero(adapter->hw_stats.rx_drops);
2701         counter_u64_add(adapter->hw_stats.rx_drops, rx_drops);
2702
2703         stime = getsbinuptime();
2704         atomic_store_rel_64(&adapter->keep_alive_timestamp, stime);
2705 }
2706
2707 /* Check for keep alive expiration */
2708 static void check_for_missing_keep_alive(struct ena_adapter *adapter)
2709 {
2710         sbintime_t timestamp, time;
2711
2712         if (adapter->wd_active == 0)
2713                 return;
2714
2715         if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT)
2716                 return;
2717
2718         timestamp = atomic_load_acq_64(&adapter->keep_alive_timestamp);
2719         time = getsbinuptime() - timestamp;
2720         if (unlikely(time > adapter->keep_alive_timeout)) {
2721                 device_printf(adapter->pdev,
2722                     "Keep alive watchdog timeout.\n");
2723                 counter_u64_add(adapter->dev_stats.wd_expired, 1);
2724                 if (likely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) {
2725                         adapter->reset_reason = ENA_REGS_RESET_KEEP_ALIVE_TO;
2726                         ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
2727                 }
2728         }
2729 }
2730
2731 /* Check if admin queue is enabled */
2732 static void check_for_admin_com_state(struct ena_adapter *adapter)
2733 {
2734         if (unlikely(ena_com_get_admin_running_state(adapter->ena_dev) ==
2735             false)) {
2736                 device_printf(adapter->pdev,
2737                     "ENA admin queue is not in running state!\n");
2738                 counter_u64_add(adapter->dev_stats.admin_q_pause, 1);
2739                 if (likely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) {
2740                         adapter->reset_reason = ENA_REGS_RESET_ADMIN_TO;
2741                         ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
2742                 }
2743         }
2744 }
2745
2746 static int
2747 check_for_rx_interrupt_queue(struct ena_adapter *adapter,
2748     struct ena_ring *rx_ring)
2749 {
2750         if (likely(rx_ring->first_interrupt))
2751                 return (0);
2752
2753         if (ena_com_cq_empty(rx_ring->ena_com_io_cq))
2754                 return (0);
2755
2756         rx_ring->no_interrupt_event_cnt++;
2757
2758         if (rx_ring->no_interrupt_event_cnt == ENA_MAX_NO_INTERRUPT_ITERATIONS) {
2759                 device_printf(adapter->pdev, "Potential MSIX issue on Rx side "
2760                     "Queue = %d. Reset the device\n", rx_ring->qid);
2761                 if (likely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) {
2762                         adapter->reset_reason = ENA_REGS_RESET_MISS_INTERRUPT;
2763                         ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
2764                 }
2765                 return (EIO);
2766         }
2767
2768         return (0);
2769 }
2770
2771 static int
2772 check_missing_comp_in_tx_queue(struct ena_adapter *adapter,
2773     struct ena_ring *tx_ring)
2774 {
2775         struct bintime curtime, time;
2776         struct ena_tx_buffer *tx_buf;
2777         sbintime_t time_offset;
2778         uint32_t missed_tx = 0;
2779         int i, rc = 0;
2780
2781         getbinuptime(&curtime);
2782
2783         for (i = 0; i < tx_ring->ring_size; i++) {
2784                 tx_buf = &tx_ring->tx_buffer_info[i];
2785
2786                 if (bintime_isset(&tx_buf->timestamp) == 0)
2787                         continue;
2788
2789                 time = curtime;
2790                 bintime_sub(&time, &tx_buf->timestamp);
2791                 time_offset = bttosbt(time);
2792
2793                 if (unlikely(!tx_ring->first_interrupt &&
2794                     time_offset > 2 * adapter->missing_tx_timeout)) {
2795                         /*
2796                          * If after graceful period interrupt is still not
2797                          * received, we schedule a reset.
2798                          */
2799                         device_printf(adapter->pdev,
2800                             "Potential MSIX issue on Tx side Queue = %d. "
2801                             "Reset the device\n", tx_ring->qid);
2802                         if (likely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET,
2803                             adapter))) {
2804                                 adapter->reset_reason =
2805                                     ENA_REGS_RESET_MISS_INTERRUPT;
2806                                 ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET,
2807                                     adapter);
2808                         }
2809                         return (EIO);
2810                 }
2811
2812                 /* Check again if packet is still waiting */
2813                 if (unlikely(time_offset > adapter->missing_tx_timeout)) {
2814
2815                         if (!tx_buf->print_once)
2816                                 ena_trace(ENA_WARNING, "Found a Tx that wasn't "
2817                                     "completed on time, qid %d, index %d.\n",
2818                                     tx_ring->qid, i);
2819
2820                         tx_buf->print_once = true;
2821                         missed_tx++;
2822                 }
2823         }
2824
2825         if (unlikely(missed_tx > adapter->missing_tx_threshold)) {
2826                 device_printf(adapter->pdev,
2827                     "The number of lost tx completion is above the threshold "
2828                     "(%d > %d). Reset the device\n",
2829                     missed_tx, adapter->missing_tx_threshold);
2830                 if (likely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) {
2831                         adapter->reset_reason = ENA_REGS_RESET_MISS_TX_CMPL;
2832                         ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
2833                 }
2834                 rc = EIO;
2835         }
2836
2837         counter_u64_add(tx_ring->tx_stats.missing_tx_comp, missed_tx);
2838
2839         return (rc);
2840 }
2841
2842 /*
2843  * Check for TX which were not completed on time.
2844  * Timeout is defined by "missing_tx_timeout".
2845  * Reset will be performed if number of incompleted
2846  * transactions exceeds "missing_tx_threshold".
2847  */
2848 static void
2849 check_for_missing_completions(struct ena_adapter *adapter)
2850 {
2851         struct ena_ring *tx_ring;
2852         struct ena_ring *rx_ring;
2853         int i, budget, rc;
2854
2855         /* Make sure the driver doesn't turn the device in other process */
2856         rmb();
2857
2858         if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter))
2859                 return;
2860
2861         if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))
2862                 return;
2863
2864         if (adapter->missing_tx_timeout == ENA_HW_HINTS_NO_TIMEOUT)
2865                 return;
2866
2867         budget = adapter->missing_tx_max_queues;
2868
2869         for (i = adapter->next_monitored_tx_qid; i < adapter->num_queues; i++) {
2870                 tx_ring = &adapter->tx_ring[i];
2871                 rx_ring = &adapter->rx_ring[i];
2872
2873                 rc = check_missing_comp_in_tx_queue(adapter, tx_ring);
2874                 if (unlikely(rc != 0))
2875                         return;
2876
2877                 rc = check_for_rx_interrupt_queue(adapter, rx_ring);
2878                 if (unlikely(rc != 0))
2879                         return;
2880
2881                 budget--;
2882                 if (budget == 0) {
2883                         i++;
2884                         break;
2885                 }
2886         }
2887
2888         adapter->next_monitored_tx_qid = i % adapter->num_queues;
2889 }
2890
2891 /* trigger rx cleanup after 2 consecutive detections */
2892 #define EMPTY_RX_REFILL 2
2893 /* For the rare case where the device runs out of Rx descriptors and the
2894  * msix handler failed to refill new Rx descriptors (due to a lack of memory
2895  * for example).
2896  * This case will lead to a deadlock:
2897  * The device won't send interrupts since all the new Rx packets will be dropped
2898  * The msix handler won't allocate new Rx descriptors so the device won't be
2899  * able to send new packets.
2900  *
2901  * When such a situation is detected - execute rx cleanup task in another thread
2902  */
2903 static void
2904 check_for_empty_rx_ring(struct ena_adapter *adapter)
2905 {
2906         struct ena_ring *rx_ring;
2907         int i, refill_required;
2908
2909         if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter))
2910                 return;
2911
2912         if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))
2913                 return;
2914
2915         for (i = 0; i < adapter->num_queues; i++) {
2916                 rx_ring = &adapter->rx_ring[i];
2917
2918                 refill_required = ena_com_free_desc(rx_ring->ena_com_io_sq);
2919                 if (unlikely(refill_required == (rx_ring->ring_size - 1))) {
2920                         rx_ring->empty_rx_queue++;
2921
2922                         if (rx_ring->empty_rx_queue >= EMPTY_RX_REFILL) {
2923                                 counter_u64_add(rx_ring->rx_stats.empty_rx_ring,
2924                                     1);
2925
2926                                 device_printf(adapter->pdev,
2927                                     "trigger refill for ring %d\n", i);
2928
2929                                 taskqueue_enqueue(rx_ring->que->cleanup_tq,
2930                                     &rx_ring->que->cleanup_task);
2931                                 rx_ring->empty_rx_queue = 0;
2932                         }
2933                 } else {
2934                         rx_ring->empty_rx_queue = 0;
2935                 }
2936         }
2937 }
2938
2939 static void ena_update_hints(struct ena_adapter *adapter,
2940                              struct ena_admin_ena_hw_hints *hints)
2941 {
2942         struct ena_com_dev *ena_dev = adapter->ena_dev;
2943
2944         if (hints->admin_completion_tx_timeout)
2945                 ena_dev->admin_queue.completion_timeout =
2946                     hints->admin_completion_tx_timeout * 1000;
2947
2948         if (hints->mmio_read_timeout)
2949                 /* convert to usec */
2950                 ena_dev->mmio_read.reg_read_to =
2951                     hints->mmio_read_timeout * 1000;
2952
2953         if (hints->missed_tx_completion_count_threshold_to_reset)
2954                 adapter->missing_tx_threshold =
2955                     hints->missed_tx_completion_count_threshold_to_reset;
2956
2957         if (hints->missing_tx_completion_timeout) {
2958                 if (hints->missing_tx_completion_timeout ==
2959                      ENA_HW_HINTS_NO_TIMEOUT)
2960                         adapter->missing_tx_timeout = ENA_HW_HINTS_NO_TIMEOUT;
2961                 else
2962                         adapter->missing_tx_timeout =
2963                             SBT_1MS * hints->missing_tx_completion_timeout;
2964         }
2965
2966         if (hints->driver_watchdog_timeout) {
2967                 if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT)
2968                         adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT;
2969                 else
2970                         adapter->keep_alive_timeout =
2971                             SBT_1MS * hints->driver_watchdog_timeout;
2972         }
2973 }
2974
2975 static void
2976 ena_timer_service(void *data)
2977 {
2978         struct ena_adapter *adapter = (struct ena_adapter *)data;
2979         struct ena_admin_host_info *host_info =
2980             adapter->ena_dev->host_attr.host_info;
2981
2982         check_for_missing_keep_alive(adapter);
2983
2984         check_for_admin_com_state(adapter);
2985
2986         check_for_missing_completions(adapter);
2987
2988         check_for_empty_rx_ring(adapter);
2989
2990         if (host_info != NULL)
2991                 ena_update_host_info(host_info, adapter->ifp);
2992
2993         if (unlikely(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) {
2994                 device_printf(adapter->pdev, "Trigger reset is on\n");
2995                 taskqueue_enqueue(adapter->reset_tq, &adapter->reset_task);
2996                 return;
2997         }
2998
2999         /*
3000          * Schedule another timeout one second from now.
3001          */
3002         callout_schedule_sbt(&adapter->timer_service, SBT_1S, SBT_1S, 0);
3003 }
3004
3005 void
3006 ena_destroy_device(struct ena_adapter *adapter, bool graceful)
3007 {
3008         if_t ifp = adapter->ifp;
3009         struct ena_com_dev *ena_dev = adapter->ena_dev;
3010         bool dev_up;
3011
3012         if (!ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter))
3013                 return;
3014
3015         if_link_state_change(ifp, LINK_STATE_DOWN);
3016
3017         callout_drain(&adapter->timer_service);
3018
3019         dev_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter);
3020         if (dev_up)
3021                 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter);
3022         else
3023                 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter);
3024
3025         if (!graceful)
3026                 ena_com_set_admin_running_state(ena_dev, false);
3027
3028         if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter))
3029                 ena_down(adapter);
3030
3031         /*
3032          * Stop the device from sending AENQ events (if the device was up, and
3033          * the trigger reset was on, ena_down already performs device reset)
3034          */
3035         if (!(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter) && dev_up))
3036                 ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason);
3037
3038         ena_free_mgmnt_irq(adapter);
3039
3040         ena_disable_msix(adapter);
3041
3042         ena_com_abort_admin_commands(ena_dev);
3043
3044         ena_com_wait_for_abort_completion(ena_dev);
3045
3046         ena_com_admin_destroy(ena_dev);
3047
3048         ena_com_mmio_reg_read_request_destroy(ena_dev);
3049
3050         adapter->reset_reason = ENA_REGS_RESET_NORMAL;
3051
3052         ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
3053         ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter);
3054 }
3055
3056 static int
3057 ena_device_validate_params(struct ena_adapter *adapter,
3058     struct ena_com_dev_get_features_ctx *get_feat_ctx)
3059 {
3060
3061         if (memcmp(get_feat_ctx->dev_attr.mac_addr, adapter->mac_addr,
3062             ETHER_ADDR_LEN) != 0) {
3063                 device_printf(adapter->pdev,
3064                     "Error, mac address are different\n");
3065                 return (EINVAL);
3066         }
3067
3068         if (get_feat_ctx->dev_attr.max_mtu < if_getmtu(adapter->ifp)) {
3069                 device_printf(adapter->pdev,
3070                     "Error, device max mtu is smaller than ifp MTU\n");
3071                 return (EINVAL);
3072         }
3073
3074         return 0;
3075 }
3076
3077 int
3078 ena_restore_device(struct ena_adapter *adapter)
3079 {
3080         struct ena_com_dev_get_features_ctx get_feat_ctx;
3081         struct ena_com_dev *ena_dev = adapter->ena_dev;
3082         if_t ifp = adapter->ifp;
3083         device_t dev = adapter->pdev;
3084         int wd_active;
3085         int rc;
3086
3087         ENA_FLAG_SET_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter);
3088
3089         rc = ena_device_init(adapter, dev, &get_feat_ctx, &wd_active);
3090         if (rc != 0) {
3091                 device_printf(dev, "Cannot initialize device\n");
3092                 goto err;
3093         }
3094         /*
3095          * Only enable WD if it was enabled before reset, so it won't override
3096          * value set by the user by the sysctl.
3097          */
3098         if (adapter->wd_active != 0)
3099                 adapter->wd_active = wd_active;
3100
3101         rc = ena_device_validate_params(adapter, &get_feat_ctx);
3102         if (rc != 0) {
3103                 device_printf(dev, "Validation of device parameters failed\n");
3104                 goto err_device_destroy;
3105         }
3106
3107         rc = ena_handle_updated_queues(adapter, &get_feat_ctx);
3108         if (rc != 0)
3109                 goto err_device_destroy;
3110
3111         ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter);
3112         /* Make sure we don't have a race with AENQ Links state handler */
3113         if (ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter))
3114                 if_link_state_change(ifp, LINK_STATE_UP);
3115
3116         rc = ena_enable_msix_and_set_admin_interrupts(adapter,
3117             adapter->num_queues);
3118         if (rc != 0) {
3119                 device_printf(dev, "Enable MSI-X failed\n");
3120                 goto err_device_destroy;
3121         }
3122
3123         /* If the interface was up before the reset bring it up */
3124         if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter)) {
3125                 rc = ena_up(adapter);
3126                 if (rc != 0) {
3127                         device_printf(dev, "Failed to create I/O queues\n");
3128                         goto err_disable_msix;
3129                 }
3130         }
3131
3132         /* Indicate that device is running again and ready to work */
3133         ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter);
3134
3135         if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter)) {
3136                 /*
3137                  * As the AENQ handlers weren't executed during reset because
3138                  * the flag ENA_FLAG_DEVICE_RUNNING was turned off, the
3139                  * timestamp must be updated again That will prevent next reset
3140                  * caused by missing keep alive.
3141                  */
3142                 adapter->keep_alive_timestamp = getsbinuptime();
3143                 callout_reset_sbt(&adapter->timer_service, SBT_1S, SBT_1S,
3144                     ena_timer_service, (void *)adapter, 0);
3145         }
3146
3147         device_printf(dev,
3148             "Device reset completed successfully, Driver info: %s\n", ena_version);
3149
3150         return (rc);
3151
3152 err_disable_msix:
3153         ena_free_mgmnt_irq(adapter);
3154         ena_disable_msix(adapter);
3155 err_device_destroy:
3156         ena_com_abort_admin_commands(ena_dev);
3157         ena_com_wait_for_abort_completion(ena_dev);
3158         ena_com_admin_destroy(ena_dev);
3159         ena_com_dev_reset(ena_dev, ENA_REGS_RESET_DRIVER_INVALID_STATE);
3160         ena_com_mmio_reg_read_request_destroy(ena_dev);
3161 err:
3162         ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter);
3163         ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter);
3164         device_printf(dev, "Reset attempt failed. Can not reset the device\n");
3165
3166         return (rc);
3167 }
3168
3169 static void
3170 ena_reset_task(void *arg, int pending)
3171 {
3172         struct ena_adapter *adapter = (struct ena_adapter *)arg;
3173
3174         if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) {
3175                 device_printf(adapter->pdev,
3176                     "device reset scheduled but trigger_reset is off\n");
3177                 return;
3178         }
3179
3180         sx_xlock(&adapter->ioctl_sx);
3181         ena_destroy_device(adapter, false);
3182         ena_restore_device(adapter);
3183         sx_unlock(&adapter->ioctl_sx);
3184 }
3185
3186 /**
3187  * ena_attach - Device Initialization Routine
3188  * @pdev: device information struct
3189  *
3190  * Returns 0 on success, otherwise on failure.
3191  *
3192  * ena_attach initializes an adapter identified by a device structure.
3193  * The OS initialization, configuring of the adapter private structure,
3194  * and a hardware reset occur.
3195  **/
3196 static int
3197 ena_attach(device_t pdev)
3198 {
3199         struct ena_com_dev_get_features_ctx get_feat_ctx;
3200         struct ena_llq_configurations llq_config;
3201         struct ena_calc_queue_size_ctx calc_queue_ctx = { 0 };
3202         static int version_printed;
3203         struct ena_adapter *adapter;
3204         struct ena_com_dev *ena_dev = NULL;
3205         const char *queue_type_str;
3206         int io_queue_num;
3207         int rid, rc;
3208
3209         adapter = device_get_softc(pdev);
3210         adapter->pdev = pdev;
3211
3212         mtx_init(&adapter->global_mtx, "ENA global mtx", NULL, MTX_DEF);
3213         sx_init(&adapter->ioctl_sx, "ENA ioctl sx");
3214
3215         /* Set up the timer service */
3216         callout_init_mtx(&adapter->timer_service, &adapter->global_mtx, 0);
3217         adapter->keep_alive_timeout = DEFAULT_KEEP_ALIVE_TO;
3218         adapter->missing_tx_timeout = DEFAULT_TX_CMP_TO;
3219         adapter->missing_tx_max_queues = DEFAULT_TX_MONITORED_QUEUES;
3220         adapter->missing_tx_threshold = DEFAULT_TX_CMP_THRESHOLD;
3221
3222         if (version_printed++ == 0)
3223                 device_printf(pdev, "%s\n", ena_version);
3224
3225         /* Allocate memory for ena_dev structure */
3226         ena_dev = malloc(sizeof(struct ena_com_dev), M_DEVBUF,
3227             M_WAITOK | M_ZERO);
3228
3229         adapter->ena_dev = ena_dev;
3230         ena_dev->dmadev = pdev;
3231
3232         rid = PCIR_BAR(ENA_REG_BAR);
3233         adapter->memory = NULL;
3234         adapter->registers = bus_alloc_resource_any(pdev, SYS_RES_MEMORY,
3235             &rid, RF_ACTIVE);
3236         if (unlikely(adapter->registers == NULL)) {
3237                 device_printf(pdev,
3238                     "unable to allocate bus resource: registers!\n");
3239                 rc = ENOMEM;
3240                 goto err_dev_free;
3241         }
3242
3243         ena_dev->bus = malloc(sizeof(struct ena_bus), M_DEVBUF,
3244             M_WAITOK | M_ZERO);
3245
3246         /* Store register resources */
3247         ((struct ena_bus*)(ena_dev->bus))->reg_bar_t =
3248             rman_get_bustag(adapter->registers);
3249         ((struct ena_bus*)(ena_dev->bus))->reg_bar_h =
3250             rman_get_bushandle(adapter->registers);
3251
3252         if (unlikely(((struct ena_bus*)(ena_dev->bus))->reg_bar_h == 0)) {
3253                 device_printf(pdev, "failed to pmap registers bar\n");
3254                 rc = ENXIO;
3255                 goto err_bus_free;
3256         }
3257
3258         ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
3259
3260         /* Initially clear all the flags */
3261         ENA_FLAG_ZERO(adapter);
3262
3263         /* Device initialization */
3264         rc = ena_device_init(adapter, pdev, &get_feat_ctx, &adapter->wd_active);
3265         if (unlikely(rc != 0)) {
3266                 device_printf(pdev, "ENA device init failed! (err: %d)\n", rc);
3267                 rc = ENXIO;
3268                 goto err_bus_free;
3269         }
3270
3271         set_default_llq_configurations(&llq_config);
3272
3273         rc = ena_set_queues_placement_policy(pdev, ena_dev, &get_feat_ctx.llq,
3274              &llq_config);
3275         if (unlikely(rc != 0)) {
3276                 device_printf(pdev, "failed to set placement policy\n");
3277                 goto err_com_free;
3278         }
3279
3280         if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
3281                 queue_type_str = "Regular";
3282         else
3283                 queue_type_str = "Low Latency";
3284         device_printf(pdev, "Placement policy: %s\n", queue_type_str);
3285
3286         adapter->keep_alive_timestamp = getsbinuptime();
3287
3288         adapter->tx_offload_cap = get_feat_ctx.offload.tx;
3289
3290         memcpy(adapter->mac_addr, get_feat_ctx.dev_attr.mac_addr,
3291             ETHER_ADDR_LEN);
3292
3293         calc_queue_ctx.ena_dev = ena_dev;
3294         calc_queue_ctx.get_feat_ctx = &get_feat_ctx;
3295         calc_queue_ctx.pdev = pdev;
3296
3297         /* calculate IO queue number to create */
3298         io_queue_num = ena_calc_io_queue_num(adapter, &get_feat_ctx);
3299
3300         ENA_ASSERT(io_queue_num > 0, "Invalid queue number: %d\n",
3301             io_queue_num);
3302         adapter->num_queues = io_queue_num;
3303
3304         adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu;
3305         // Set the requested Rx ring size
3306         adapter->rx_ring_size = ENA_DEFAULT_RING_SIZE;
3307         /* calculatre ring sizes */
3308         rc = ena_calc_queue_size(adapter, &calc_queue_ctx);
3309         if (unlikely((rc != 0) || (io_queue_num <= 0))) {
3310                 rc = EFAULT;
3311                 goto err_com_free;
3312         }
3313
3314         adapter->reset_reason = ENA_REGS_RESET_NORMAL;
3315
3316         adapter->tx_ring_size = calc_queue_ctx.tx_queue_size;
3317         adapter->rx_ring_size = calc_queue_ctx.rx_queue_size;
3318
3319         adapter->max_tx_sgl_size = calc_queue_ctx.max_tx_sgl_size;
3320         adapter->max_rx_sgl_size = calc_queue_ctx.max_rx_sgl_size;
3321
3322         adapter->buf_ring_size = ENA_DEFAULT_BUF_RING_SIZE;
3323
3324         /* set up dma tags for rx and tx buffers */
3325         rc = ena_setup_tx_dma_tag(adapter);
3326         if (unlikely(rc != 0)) {
3327                 device_printf(pdev, "Failed to create TX DMA tag\n");
3328                 goto err_com_free;
3329         }
3330
3331         rc = ena_setup_rx_dma_tag(adapter);
3332         if (unlikely(rc != 0)) {
3333                 device_printf(pdev, "Failed to create RX DMA tag\n");
3334                 goto err_tx_tag_free;
3335         }
3336
3337         /* initialize rings basic information */
3338         device_printf(pdev,
3339             "Creating %d io queues. Rx queue size: %d, Tx queue size: %d\n",
3340             io_queue_num,
3341             calc_queue_ctx.rx_queue_size,
3342             calc_queue_ctx.tx_queue_size);
3343         ena_init_io_rings(adapter);
3344
3345         rc = ena_enable_msix_and_set_admin_interrupts(adapter, io_queue_num);
3346         if (unlikely(rc != 0)) {
3347                 device_printf(pdev,
3348                     "Failed to enable and set the admin interrupts\n");
3349                 goto err_io_free;
3350         }
3351
3352         /* setup network interface */
3353         rc = ena_setup_ifnet(pdev, adapter, &get_feat_ctx);
3354         if (unlikely(rc != 0)) {
3355                 device_printf(pdev, "Error with network interface setup\n");
3356                 goto err_msix_free;
3357         }
3358
3359         /* Initialize reset task queue */
3360         TASK_INIT(&adapter->reset_task, 0, ena_reset_task, adapter);
3361         adapter->reset_tq = taskqueue_create("ena_reset_enqueue",
3362             M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &adapter->reset_tq);
3363         taskqueue_start_threads(&adapter->reset_tq, 1, PI_NET,
3364             "%s rstq", device_get_nameunit(adapter->pdev));
3365
3366         /* Initialize statistics */
3367         ena_alloc_counters((counter_u64_t *)&adapter->dev_stats,
3368             sizeof(struct ena_stats_dev));
3369         ena_alloc_counters((counter_u64_t *)&adapter->hw_stats,
3370             sizeof(struct ena_hw_stats));
3371         ena_sysctl_add_nodes(adapter);
3372
3373 #ifdef DEV_NETMAP
3374         rc = ena_netmap_attach(adapter);
3375         if (rc != 0) {
3376                 device_printf(pdev, "netmap attach failed: %d\n", rc);
3377                 goto err_detach;
3378         }
3379 #endif /* DEV_NETMAP */
3380
3381         /* Tell the stack that the interface is not active */
3382         if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
3383         ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter);
3384
3385         return (0);
3386
3387 #ifdef DEV_NETMAP
3388 err_detach:
3389         ether_ifdetach(adapter->ifp);
3390 #endif /* DEV_NETMAP */
3391 err_msix_free:
3392         ena_com_dev_reset(adapter->ena_dev, ENA_REGS_RESET_INIT_ERR);
3393         ena_free_mgmnt_irq(adapter);
3394         ena_disable_msix(adapter);
3395 err_io_free:
3396         ena_free_all_io_rings_resources(adapter);
3397         ena_free_rx_dma_tag(adapter);
3398 err_tx_tag_free:
3399         ena_free_tx_dma_tag(adapter);
3400 err_com_free:
3401         ena_com_admin_destroy(ena_dev);
3402         ena_com_delete_host_info(ena_dev);
3403         ena_com_mmio_reg_read_request_destroy(ena_dev);
3404 err_bus_free:
3405         free(ena_dev->bus, M_DEVBUF);
3406         ena_free_pci_resources(adapter);
3407 err_dev_free:
3408         free(ena_dev, M_DEVBUF);
3409
3410         return (rc);
3411 }
3412
3413 /**
3414  * ena_detach - Device Removal Routine
3415  * @pdev: device information struct
3416  *
3417  * ena_detach is called by the device subsystem to alert the driver
3418  * that it should release a PCI device.
3419  **/
3420 static int
3421 ena_detach(device_t pdev)
3422 {
3423         struct ena_adapter *adapter = device_get_softc(pdev);
3424         struct ena_com_dev *ena_dev = adapter->ena_dev;
3425         int rc;
3426
3427         /* Make sure VLANS are not using driver */
3428         if (adapter->ifp->if_vlantrunk != NULL) {
3429                 device_printf(adapter->pdev ,"VLAN is in use, detach first\n");
3430                 return (EBUSY);
3431         }
3432
3433         ether_ifdetach(adapter->ifp);
3434
3435         /* Free reset task and callout */
3436         callout_drain(&adapter->timer_service);
3437         while (taskqueue_cancel(adapter->reset_tq, &adapter->reset_task, NULL))
3438                 taskqueue_drain(adapter->reset_tq, &adapter->reset_task);
3439         taskqueue_free(adapter->reset_tq);
3440
3441         sx_xlock(&adapter->ioctl_sx);
3442         ena_down(adapter);
3443         ena_destroy_device(adapter, true);
3444         sx_unlock(&adapter->ioctl_sx);
3445
3446 #ifdef DEV_NETMAP
3447         netmap_detach(adapter->ifp);
3448 #endif /* DEV_NETMAP */
3449
3450         ena_free_all_io_rings_resources(adapter);
3451
3452         ena_free_counters((counter_u64_t *)&adapter->hw_stats,
3453             sizeof(struct ena_hw_stats));
3454         ena_free_counters((counter_u64_t *)&adapter->dev_stats,
3455             sizeof(struct ena_stats_dev));
3456
3457         rc = ena_free_rx_dma_tag(adapter);
3458         if (unlikely(rc != 0))
3459                 device_printf(adapter->pdev,
3460                     "Unmapped RX DMA tag associations\n");
3461
3462         rc = ena_free_tx_dma_tag(adapter);
3463         if (unlikely(rc != 0))
3464                 device_printf(adapter->pdev,
3465                     "Unmapped TX DMA tag associations\n");
3466
3467         ena_free_irqs(adapter);
3468
3469         ena_free_pci_resources(adapter);
3470
3471         if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter)))
3472                 ena_com_rss_destroy(ena_dev);
3473
3474         ena_com_delete_host_info(ena_dev);
3475
3476         mtx_destroy(&adapter->global_mtx);
3477         sx_destroy(&adapter->ioctl_sx);
3478
3479         if_free(adapter->ifp);
3480
3481         if (ena_dev->bus != NULL)
3482                 free(ena_dev->bus, M_DEVBUF);
3483
3484         if (ena_dev != NULL)
3485                 free(ena_dev, M_DEVBUF);
3486
3487         return (bus_generic_detach(pdev));
3488 }
3489
3490 /******************************************************************************
3491  ******************************** AENQ Handlers *******************************
3492  *****************************************************************************/
3493 /**
3494  * ena_update_on_link_change:
3495  * Notify the network interface about the change in link status
3496  **/
3497 static void
3498 ena_update_on_link_change(void *adapter_data,
3499     struct ena_admin_aenq_entry *aenq_e)
3500 {
3501         struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
3502         struct ena_admin_aenq_link_change_desc *aenq_desc;
3503         int status;
3504         if_t ifp;
3505
3506         aenq_desc = (struct ena_admin_aenq_link_change_desc *)aenq_e;
3507         ifp = adapter->ifp;
3508         status = aenq_desc->flags &
3509             ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK;
3510
3511         if (status != 0) {
3512                 device_printf(adapter->pdev, "link is UP\n");
3513                 ENA_FLAG_SET_ATOMIC(ENA_FLAG_LINK_UP, adapter);
3514                 if (!ENA_FLAG_ISSET(ENA_FLAG_ONGOING_RESET, adapter))
3515                         if_link_state_change(ifp, LINK_STATE_UP);
3516         } else {
3517                 device_printf(adapter->pdev, "link is DOWN\n");
3518                 if_link_state_change(ifp, LINK_STATE_DOWN);
3519                 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_LINK_UP, adapter);
3520         }
3521 }
3522
3523 static void ena_notification(void *adapter_data,
3524     struct ena_admin_aenq_entry *aenq_e)
3525 {
3526         struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
3527         struct ena_admin_ena_hw_hints *hints;
3528
3529         ENA_WARN(aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION,
3530             "Invalid group(%x) expected %x\n",  aenq_e->aenq_common_desc.group,
3531             ENA_ADMIN_NOTIFICATION);
3532
3533         switch (aenq_e->aenq_common_desc.syndrom) {
3534         case ENA_ADMIN_UPDATE_HINTS:
3535                 hints =
3536                     (struct ena_admin_ena_hw_hints *)(&aenq_e->inline_data_w4);
3537                 ena_update_hints(adapter, hints);
3538                 break;
3539         default:
3540                 device_printf(adapter->pdev,
3541                     "Invalid aenq notification link state %d\n",
3542                     aenq_e->aenq_common_desc.syndrom);
3543         }
3544 }
3545
3546 /**
3547  * This handler will called for unknown event group or unimplemented handlers
3548  **/
3549 static void
3550 unimplemented_aenq_handler(void *adapter_data,
3551     struct ena_admin_aenq_entry *aenq_e)
3552 {
3553         struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
3554
3555         device_printf(adapter->pdev,
3556             "Unknown event was received or event with unimplemented handler\n");
3557 }
3558
3559 static struct ena_aenq_handlers aenq_handlers = {
3560     .handlers = {
3561             [ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change,
3562             [ENA_ADMIN_NOTIFICATION] = ena_notification,
3563             [ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive_wd,
3564     },
3565     .unimplemented_handler = unimplemented_aenq_handler
3566 };
3567
3568 /*********************************************************************
3569  *  FreeBSD Device Interface Entry Points
3570  *********************************************************************/
3571
3572 static device_method_t ena_methods[] = {
3573     /* Device interface */
3574     DEVMETHOD(device_probe, ena_probe),
3575     DEVMETHOD(device_attach, ena_attach),
3576     DEVMETHOD(device_detach, ena_detach),
3577     DEVMETHOD_END
3578 };
3579
3580 static driver_t ena_driver = {
3581     "ena", ena_methods, sizeof(struct ena_adapter),
3582 };
3583
3584 devclass_t ena_devclass;
3585 DRIVER_MODULE(ena, pci, ena_driver, ena_devclass, 0, 0);
3586 MODULE_PNP_INFO("U16:vendor;U16:device", pci, ena, ena_vendor_info_array,
3587     nitems(ena_vendor_info_array) - 1);
3588 MODULE_DEPEND(ena, pci, 1, 1, 1);
3589 MODULE_DEPEND(ena, ether, 1, 1, 1);
3590 #ifdef DEV_NETMAP
3591 MODULE_DEPEND(ena, netmap, 1, 1, 1);
3592 #endif /* DEV_NETMAP */
3593
3594 /*********************************************************************/