sys/dev/gve/gve_main.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-3-Clause
   3  *
   4  * Copyright (c) 2023 Google LLC
   5  *
   6  * Redistribution and use in source and binary forms, with or without modification,
   7  * are permitted provided that the following conditions are met:
   8  *
   9  * 1. Redistributions of source code must retain the above copyright notice, this
  10  *    list of conditions and the following disclaimer.
  11  *
  12  * 2. Redistributions in binary form must reproduce the above copyright notice,
  13  *    this list of conditions and the following disclaimer in the documentation
  14  *    and/or other materials provided with the distribution.
  15  *
  16  * 3. Neither the name of the copyright holder nor the names of its contributors
  17  *    may be used to endorse or promote products derived from this software without
  18  *    specific prior written permission.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
  21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  22  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  23  * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
  24  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  25  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  26  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
  27  * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  29  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30  */
  31 #include "gve.h"
  32 #include "gve_adminq.h"
  33
  34 #define GVE_DRIVER_VERSION "GVE-FBSD-1.0.0\n"
  35 #define GVE_VERSION_MAJOR 0
  36 #define GVE_VERSION_MINOR 9
  37 #define GVE_VERSION_SUB 0
  38
  39 #define GVE_DEFAULT_RX_COPYBREAK 256
  40
  41 /* Devices supported by this driver. */
  42 static struct gve_dev {
  43         uint16_t vendor_id;
  44         uint16_t device_id;
  45         const char *name;
  46 } gve_devs[] = {
  47         { PCI_VENDOR_ID_GOOGLE, PCI_DEV_ID_GVNIC, "gVNIC" }
  48 };
  49
  50 struct sx gve_global_lock;
  51
  52 static int
  53 gve_verify_driver_compatibility(struct gve_priv *priv)
  54 {
  55         int err;
  56         struct gve_driver_info *driver_info;
  57         struct gve_dma_handle driver_info_mem;
  58
  59         err = gve_dma_alloc_coherent(priv, sizeof(struct gve_driver_info),
  60             PAGE_SIZE, &driver_info_mem);
  61
  62         if (err != 0)
  63                 return (ENOMEM);
  64
  65         driver_info = driver_info_mem.cpu_addr;
  66
  67         *driver_info = (struct gve_driver_info) {
  68                 .os_type = 3, /* Freebsd */
  69                 .driver_major = GVE_VERSION_MAJOR,
  70                 .driver_minor = GVE_VERSION_MINOR,
  71                 .driver_sub = GVE_VERSION_SUB,
  72                 .os_version_major = htobe32(FBSD_VERSION_MAJOR),
  73                 .os_version_minor = htobe32(FBSD_VERSION_MINOR),
  74                 .os_version_sub = htobe32(FBSD_VERSION_PATCH),
  75                 .driver_capability_flags = {
  76                         htobe64(GVE_DRIVER_CAPABILITY_FLAGS1),
  77                         htobe64(GVE_DRIVER_CAPABILITY_FLAGS2),
  78                         htobe64(GVE_DRIVER_CAPABILITY_FLAGS3),
  79                         htobe64(GVE_DRIVER_CAPABILITY_FLAGS4),
  80                 },
  81         };
  82
  83         snprintf(driver_info->os_version_str1, sizeof(driver_info->os_version_str1),
  84             "FreeBSD %u", __FreeBSD_version);
  85
  86         bus_dmamap_sync(driver_info_mem.tag, driver_info_mem.map,
  87             BUS_DMASYNC_PREREAD);
  88
  89         err = gve_adminq_verify_driver_compatibility(priv,
  90             sizeof(struct gve_driver_info), driver_info_mem.bus_addr);
  91
  92         /* It's ok if the device doesn't support this */
  93         if (err == EOPNOTSUPP)
  94                 err = 0;
  95
  96         gve_dma_free_coherent(&driver_info_mem);
  97
  98         return (err);
  99 }
 100
 101 static int
 102 gve_up(struct gve_priv *priv)
 103 {
 104         if_t ifp = priv->ifp;
 105         int err;
 106
 107         GVE_IFACE_LOCK_ASSERT(priv->gve_iface_lock);
 108
 109         if (device_is_attached(priv->dev) == 0) {
 110                 device_printf(priv->dev, "Cannot bring the iface up when detached\n");
 111                 return (ENXIO);
 112         }
 113
 114         if (gve_get_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP))
 115                 return (0);
 116
 117         if_clearhwassist(ifp);
 118         if (if_getcapenable(ifp) & IFCAP_TXCSUM)
 119                 if_sethwassistbits(ifp, CSUM_TCP | CSUM_UDP, 0);
 120         if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6)
 121                 if_sethwassistbits(ifp, CSUM_IP6_TCP | CSUM_IP6_UDP, 0);
 122         if (if_getcapenable(ifp) & IFCAP_TSO4)
 123                 if_sethwassistbits(ifp, CSUM_IP_TSO, 0);
 124         if (if_getcapenable(ifp) & IFCAP_TSO6)
 125                 if_sethwassistbits(ifp, CSUM_IP6_TSO, 0);
 126
 127         err = gve_register_qpls(priv);
 128         if (err != 0)
 129                 goto reset;
 130
 131         err = gve_create_rx_rings(priv);
 132         if (err != 0)
 133                 goto reset;
 134
 135         err = gve_create_tx_rings(priv);
 136         if (err != 0)
 137                 goto reset;
 138
 139         if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
 140
 141         if (!gve_get_state_flag(priv, GVE_STATE_FLAG_LINK_UP)) {
 142                 if_link_state_change(ifp, LINK_STATE_UP);
 143                 gve_set_state_flag(priv, GVE_STATE_FLAG_LINK_UP);
 144         }
 145
 146         gve_unmask_all_queue_irqs(priv);
 147         gve_set_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP);
 148         priv->interface_up_cnt++;
 149         return (0);
 150
 151 reset:
 152         gve_schedule_reset(priv);
 153         return (err);
 154 }
 155
 156 static void
 157 gve_down(struct gve_priv *priv)
 158 {
 159         GVE_IFACE_LOCK_ASSERT(priv->gve_iface_lock);
 160
 161         if (!gve_get_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP))
 162                 return;
 163
 164         if (gve_get_state_flag(priv, GVE_STATE_FLAG_LINK_UP)) {
 165                 if_link_state_change(priv->ifp, LINK_STATE_DOWN);
 166                 gve_clear_state_flag(priv, GVE_STATE_FLAG_LINK_UP);
 167         }
 168
 169         if_setdrvflagbits(priv->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
 170
 171         if (gve_destroy_rx_rings(priv) != 0)
 172                 goto reset;
 173
 174         if (gve_destroy_tx_rings(priv) != 0)
 175                 goto reset;
 176
 177         if (gve_unregister_qpls(priv) != 0)
 178                 goto reset;
 179
 180         gve_mask_all_queue_irqs(priv);
 181         gve_clear_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP);
 182         priv->interface_down_cnt++;
 183         return;
 184
 185 reset:
 186         gve_schedule_reset(priv);
 187 }
 188
 189 static int
 190 gve_set_mtu(if_t ifp, uint32_t new_mtu)
 191 {
 192         struct gve_priv *priv = if_getsoftc(ifp);
 193         int err;
 194
 195         if ((new_mtu > priv->max_mtu) || (new_mtu < ETHERMIN)) {
 196                 device_printf(priv->dev, "Invalid new MTU setting. new mtu: %d max mtu: %d min mtu: %d\n",
 197                     new_mtu, priv->max_mtu, ETHERMIN);
 198                 return (EINVAL);
 199         }
 200
 201         err = gve_adminq_set_mtu(priv, new_mtu);
 202         if (err == 0) {
 203                 if (bootverbose)
 204                         device_printf(priv->dev, "MTU set to %d\n", new_mtu);
 205                 if_setmtu(ifp, new_mtu);
 206         } else {
 207                 device_printf(priv->dev, "Failed to set MTU to %d\n", new_mtu);
 208         }
 209
 210         return (err);
 211 }
 212
 213 static void
 214 gve_init(void *arg)
 215 {
 216         struct gve_priv *priv = (struct gve_priv *)arg;
 217
 218         if (!gve_get_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP)) {
 219                 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
 220                 gve_up(priv);
 221                 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
 222         }
 223 }
 224
 225 static int
 226 gve_ioctl(if_t ifp, u_long command, caddr_t data)
 227 {
 228         struct gve_priv *priv;
 229         struct ifreq *ifr;
 230         int rc = 0;
 231
 232         priv = if_getsoftc(ifp);
 233         ifr = (struct ifreq *)data;
 234
 235         switch (command) {
 236         case SIOCSIFMTU:
 237                 if (if_getmtu(ifp) == ifr->ifr_mtu)
 238                         break;
 239                 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
 240                 gve_down(priv);
 241                 gve_set_mtu(ifp, ifr->ifr_mtu);
 242                 rc = gve_up(priv);
 243                 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
 244                 break;
 245
 246         case SIOCSIFFLAGS:
 247                 if ((if_getflags(ifp) & IFF_UP) != 0) {
 248                         if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) {
 249                                 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
 250                                 rc = gve_up(priv);
 251                                 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
 252                         }
 253                 } else {
 254                         if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
 255                                 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
 256                                 gve_down(priv);
 257                                 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
 258                         }
 259                 }
 260                 break;
 261
 262         case SIOCSIFCAP:
 263                 if (ifr->ifr_reqcap == if_getcapenable(ifp))
 264                         break;
 265                 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
 266                 gve_down(priv);
 267                 if_setcapenable(ifp, ifr->ifr_reqcap);
 268                 rc = gve_up(priv);
 269                 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
 270                 break;
 271
 272         case SIOCSIFMEDIA:
 273                 /* FALLTHROUGH */
 274         case SIOCGIFMEDIA:
 275                 rc = ifmedia_ioctl(ifp, ifr, &priv->media, command);
 276                 break;
 277
 278         default:
 279                 rc = ether_ioctl(ifp, command, data);
 280                 break;
 281         }
 282
 283         return (rc);
 284 }
 285
 286 static int
 287 gve_media_change(if_t ifp)
 288 {
 289         struct gve_priv *priv = if_getsoftc(ifp);
 290
 291         device_printf(priv->dev, "Media change not supported\n");
 292         return (0);
 293 }
 294
 295 static void
 296 gve_media_status(if_t ifp, struct ifmediareq *ifmr)
 297 {
 298         struct gve_priv *priv = if_getsoftc(ifp);
 299
 300         GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
 301
 302         ifmr->ifm_status = IFM_AVALID;
 303         ifmr->ifm_active = IFM_ETHER;
 304
 305         if (gve_get_state_flag(priv, GVE_STATE_FLAG_LINK_UP)) {
 306                 ifmr->ifm_status |= IFM_ACTIVE;
 307                 ifmr->ifm_active |= IFM_AUTO;
 308         } else {
 309                 ifmr->ifm_active |= IFM_NONE;
 310         }
 311
 312         GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
 313 }
 314
 315 static uint64_t
 316 gve_get_counter(if_t ifp, ift_counter cnt)
 317 {
 318         struct gve_priv *priv;
 319         uint64_t rpackets = 0;
 320         uint64_t tpackets = 0;
 321         uint64_t rbytes = 0;
 322         uint64_t tbytes = 0;
 323         uint64_t rx_dropped_pkt = 0;
 324         uint64_t tx_dropped_pkt = 0;
 325
 326         priv = if_getsoftc(ifp);
 327
 328         gve_accum_stats(priv, &rpackets, &rbytes, &rx_dropped_pkt, &tpackets,
 329             &tbytes, &tx_dropped_pkt);
 330
 331         switch (cnt) {
 332         case IFCOUNTER_IPACKETS:
 333                 return (rpackets);
 334
 335         case IFCOUNTER_OPACKETS:
 336                 return (tpackets);
 337
 338         case IFCOUNTER_IBYTES:
 339                 return (rbytes);
 340
 341         case IFCOUNTER_OBYTES:
 342                 return (tbytes);
 343
 344         case IFCOUNTER_IQDROPS:
 345                 return (rx_dropped_pkt);
 346
 347         case IFCOUNTER_OQDROPS:
 348                 return (tx_dropped_pkt);
 349
 350         default:
 351                 return (if_get_counter_default(ifp, cnt));
 352         }
 353 }
 354
 355 static int
 356 gve_setup_ifnet(device_t dev, struct gve_priv *priv)
 357 {
 358         int caps = 0;
 359         if_t ifp;
 360
 361         ifp = priv->ifp = if_alloc(IFT_ETHER);
 362         if (ifp == NULL) {
 363                 device_printf(priv->dev, "Failed to allocate ifnet struct\n");
 364                 return (ENXIO);
 365         }
 366
 367         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 368         if_setsoftc(ifp, priv);
 369         if_setdev(ifp, dev);
 370         if_setinitfn(ifp, gve_init);
 371         if_setioctlfn(ifp, gve_ioctl);
 372         if_settransmitfn(ifp, gve_xmit_ifp);
 373         if_setqflushfn(ifp, gve_qflush);
 374
 375 #if __FreeBSD_version >= 1400086
 376         if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
 377 #else
 378         if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST | IFF_KNOWSEPOCH);
 379 #endif
 380
 381         ifmedia_init(&priv->media, IFM_IMASK, gve_media_change, gve_media_status);
 382         if_setgetcounterfn(ifp, gve_get_counter);
 383
 384         caps = IFCAP_RXCSUM |
 385                IFCAP_TXCSUM |
 386                IFCAP_TXCSUM_IPV6 |
 387                IFCAP_TSO |
 388                IFCAP_LRO;
 389
 390         if ((priv->supported_features & GVE_SUP_JUMBO_FRAMES_MASK) != 0)
 391                 caps |= IFCAP_JUMBO_MTU;
 392
 393         if_setcapabilities(ifp, caps);
 394         if_setcapenable(ifp, caps);
 395
 396         if (bootverbose)
 397                 device_printf(priv->dev, "Setting initial MTU to %d\n", priv->max_mtu);
 398         if_setmtu(ifp, priv->max_mtu);
 399
 400         ether_ifattach(ifp, priv->mac);
 401
 402         ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO, 0, NULL);
 403         ifmedia_set(&priv->media, IFM_ETHER | IFM_AUTO);
 404
 405         return (0);
 406 }
 407
 408 static int
 409 gve_alloc_counter_array(struct gve_priv *priv)
 410 {
 411         int err;
 412
 413         err = gve_dma_alloc_coherent(priv, sizeof(uint32_t) * priv->num_event_counters,
 414             PAGE_SIZE, &priv->counter_array_mem);
 415         if (err != 0)
 416                 return (err);
 417
 418         priv->counters = priv->counter_array_mem.cpu_addr;
 419         return (0);
 420 }
 421
 422 static void
 423 gve_free_counter_array(struct gve_priv *priv)
 424 {
 425         if (priv->counters != NULL)
 426                 gve_dma_free_coherent(&priv->counter_array_mem);
 427         priv->counter_array_mem = (struct gve_dma_handle){};
 428 }
 429
 430 static int
 431 gve_alloc_irq_db_array(struct gve_priv *priv)
 432 {
 433         int err;
 434
 435         err = gve_dma_alloc_coherent(priv,
 436             sizeof(struct gve_irq_db) * (priv->num_queues), PAGE_SIZE,
 437             &priv->irqs_db_mem);
 438         if (err != 0)
 439                 return (err);
 440
 441         priv->irq_db_indices = priv->irqs_db_mem.cpu_addr;
 442         return (0);
 443 }
 444
 445 static void
 446 gve_free_irq_db_array(struct gve_priv *priv)
 447 {
 448         if (priv->irq_db_indices != NULL)
 449                 gve_dma_free_coherent(&priv->irqs_db_mem);
 450         priv->irqs_db_mem = (struct gve_dma_handle){};
 451 }
 452
 453 static void
 454 gve_free_rings(struct gve_priv *priv)
 455 {
 456         gve_free_irqs(priv);
 457         gve_free_tx_rings(priv);
 458         gve_free_rx_rings(priv);
 459         gve_free_qpls(priv);
 460 }
 461
 462 static int
 463 gve_alloc_rings(struct gve_priv *priv)
 464 {
 465         int err;
 466
 467         err = gve_alloc_qpls(priv);
 468         if (err != 0)
 469                 goto abort;
 470
 471         err = gve_alloc_rx_rings(priv);
 472         if (err != 0)
 473                 goto abort;
 474
 475         err = gve_alloc_tx_rings(priv);
 476         if (err != 0)
 477                 goto abort;
 478
 479         err = gve_alloc_irqs(priv);
 480         if (err != 0)
 481                 goto abort;
 482
 483         return (0);
 484
 485 abort:
 486         gve_free_rings(priv);
 487         return (err);
 488 }
 489
 490 static void
 491 gve_deconfigure_resources(struct gve_priv *priv)
 492 {
 493         int err;
 494
 495         if (gve_get_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK)) {
 496                 err = gve_adminq_deconfigure_device_resources(priv);
 497                 if (err != 0) {
 498                         device_printf(priv->dev, "Failed to deconfigure device resources: err=%d\n",
 499                             err);
 500                         return;
 501                 }
 502                 if (bootverbose)
 503                         device_printf(priv->dev, "Deconfigured device resources\n");
 504                 gve_clear_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK);
 505         }
 506
 507         gve_free_irq_db_array(priv);
 508         gve_free_counter_array(priv);
 509 }
 510
 511 static int
 512 gve_configure_resources(struct gve_priv *priv)
 513 {
 514         int err;
 515
 516         if (gve_get_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK))
 517                 return (0);
 518
 519         err = gve_alloc_counter_array(priv);
 520         if (err != 0)
 521                 return (err);
 522
 523         err = gve_alloc_irq_db_array(priv);
 524         if (err != 0)
 525                 goto abort;
 526
 527         err = gve_adminq_configure_device_resources(priv);
 528         if (err != 0) {
 529                 device_printf(priv->dev, "Failed to configure device resources: err=%d\n",
 530                               err);
 531                 err = (ENXIO);
 532                 goto abort;
 533         }
 534
 535         gve_set_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK);
 536         if (bootverbose)
 537                 device_printf(priv->dev, "Configured device resources\n");
 538         return (0);
 539
 540 abort:
 541         gve_deconfigure_resources(priv);
 542         return (err);
 543 }
 544
 545 static void
 546 gve_set_queue_cnts(struct gve_priv *priv)
 547 {
 548         priv->tx_cfg.max_queues = gve_reg_bar_read_4(priv, MAX_TX_QUEUES);
 549         priv->rx_cfg.max_queues = gve_reg_bar_read_4(priv, MAX_RX_QUEUES);
 550         priv->tx_cfg.num_queues = priv->tx_cfg.max_queues;
 551         priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
 552
 553         if (priv->default_num_queues > 0) {
 554                 priv->tx_cfg.num_queues = MIN(priv->default_num_queues,
 555                     priv->tx_cfg.num_queues);
 556                 priv->rx_cfg.num_queues = MIN(priv->default_num_queues,
 557                     priv->rx_cfg.num_queues);
 558         }
 559
 560         priv->num_queues = priv->tx_cfg.num_queues + priv->rx_cfg.num_queues;
 561         priv->mgmt_msix_idx = priv->num_queues;
 562 }
 563
 564 static int
 565 gve_alloc_adminq_and_describe_device(struct gve_priv *priv)
 566 {
 567         int err;
 568
 569         if ((err = gve_adminq_alloc(priv)) != 0)
 570                 return (err);
 571
 572         if ((err = gve_verify_driver_compatibility(priv)) != 0) {
 573                 device_printf(priv->dev,
 574                     "Failed to verify driver compatibility: err=%d\n", err);
 575                 goto abort;
 576         }
 577
 578         if ((err = gve_adminq_describe_device(priv)) != 0)
 579                 goto abort;
 580
 581         gve_set_queue_cnts(priv);
 582
 583         priv->num_registered_pages = 0;
 584         return (0);
 585
 586 abort:
 587         gve_release_adminq(priv);
 588         return (err);
 589 }
 590
 591 void
 592 gve_schedule_reset(struct gve_priv *priv)
 593 {
 594         if (gve_get_state_flag(priv, GVE_STATE_FLAG_IN_RESET))
 595                 return;
 596
 597         device_printf(priv->dev, "Scheduling reset task!\n");
 598         gve_set_state_flag(priv, GVE_STATE_FLAG_DO_RESET);
 599         taskqueue_enqueue(priv->service_tq, &priv->service_task);
 600 }
 601
 602 static void
 603 gve_destroy(struct gve_priv *priv)
 604 {
 605         gve_down(priv);
 606         gve_deconfigure_resources(priv);
 607         gve_release_adminq(priv);
 608 }
 609
 610 static void
 611 gve_restore(struct gve_priv *priv)
 612 {
 613         int err;
 614
 615         err = gve_adminq_alloc(priv);
 616         if (err != 0)
 617                 goto abort;
 618
 619         err = gve_configure_resources(priv);
 620         if (err != 0)
 621                 goto abort;
 622
 623         err = gve_up(priv);
 624         if (err != 0)
 625                 goto abort;
 626
 627         return;
 628
 629 abort:
 630         device_printf(priv->dev, "Restore failed!\n");
 631         return;
 632 }
 633
 634 static void
 635 gve_handle_reset(struct gve_priv *priv)
 636 {
 637         if (!gve_get_state_flag(priv, GVE_STATE_FLAG_DO_RESET))
 638                 return;
 639
 640         gve_clear_state_flag(priv, GVE_STATE_FLAG_DO_RESET);
 641         gve_set_state_flag(priv, GVE_STATE_FLAG_IN_RESET);
 642
 643         GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
 644
 645         if_setdrvflagbits(priv->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
 646         if_link_state_change(priv->ifp, LINK_STATE_DOWN);
 647         gve_clear_state_flag(priv, GVE_STATE_FLAG_LINK_UP);
 648
 649         /*
 650          * Releasing the adminq causes the NIC to destroy all resources
 651          * registered with it, so by clearing the flags beneath we cause
 652          * the subsequent gve_down call below to not attempt to tell the
 653          * NIC to destroy these resources again.
 654          *
 655          * The call to gve_down is needed in the first place to refresh
 656          * the state and the DMA-able memory within each driver ring.
 657          */
 658         gve_release_adminq(priv);
 659         gve_clear_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK);
 660         gve_clear_state_flag(priv, GVE_STATE_FLAG_QPLREG_OK);
 661         gve_clear_state_flag(priv, GVE_STATE_FLAG_RX_RINGS_OK);
 662         gve_clear_state_flag(priv, GVE_STATE_FLAG_TX_RINGS_OK);
 663
 664         gve_down(priv);
 665         gve_restore(priv);
 666
 667         GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
 668
 669         priv->reset_cnt++;
 670         gve_clear_state_flag(priv, GVE_STATE_FLAG_IN_RESET);
 671 }
 672
 673 static void
 674 gve_handle_link_status(struct gve_priv *priv)
 675 {
 676         uint32_t status = gve_reg_bar_read_4(priv, DEVICE_STATUS);
 677         bool link_up = status & GVE_DEVICE_STATUS_LINK_STATUS;
 678
 679         if (link_up == gve_get_state_flag(priv, GVE_STATE_FLAG_LINK_UP))
 680                 return;
 681
 682         if (link_up) {
 683                 if (bootverbose)
 684                         device_printf(priv->dev, "Device link is up.\n");
 685                 if_link_state_change(priv->ifp, LINK_STATE_UP);
 686                 gve_set_state_flag(priv, GVE_STATE_FLAG_LINK_UP);
 687         } else {
 688                 device_printf(priv->dev, "Device link is down.\n");
 689                 if_link_state_change(priv->ifp, LINK_STATE_DOWN);
 690                 gve_clear_state_flag(priv, GVE_STATE_FLAG_LINK_UP);
 691         }
 692 }
 693
 694 static void
 695 gve_service_task(void *arg, int pending)
 696 {
 697         struct gve_priv *priv = (struct gve_priv *)arg;
 698         uint32_t status = gve_reg_bar_read_4(priv, DEVICE_STATUS);
 699
 700         if (((GVE_DEVICE_STATUS_RESET_MASK & status) != 0) &&
 701             !gve_get_state_flag(priv, GVE_STATE_FLAG_IN_RESET)) {
 702                 device_printf(priv->dev, "Device requested reset\n");
 703                 gve_set_state_flag(priv, GVE_STATE_FLAG_DO_RESET);
 704         }
 705
 706         gve_handle_reset(priv);
 707         gve_handle_link_status(priv);
 708 }
 709
 710 static int
 711 gve_probe(device_t dev)
 712 {
 713         uint16_t deviceid, vendorid;
 714         int i;
 715
 716         vendorid = pci_get_vendor(dev);
 717         deviceid = pci_get_device(dev);
 718
 719         for (i = 0; i < nitems(gve_devs); i++) {
 720                 if (vendorid == gve_devs[i].vendor_id &&
 721                     deviceid == gve_devs[i].device_id) {
 722                         device_set_desc(dev, gve_devs[i].name);
 723                         return (BUS_PROBE_DEFAULT);
 724                 }
 725         }
 726         return (ENXIO);
 727 }
 728
 729 static void
 730 gve_free_sys_res_mem(struct gve_priv *priv)
 731 {
 732         if (priv->msix_table != NULL)
 733                 bus_release_resource(priv->dev, SYS_RES_MEMORY,
 734                     rman_get_rid(priv->msix_table), priv->msix_table);
 735
 736         if (priv->db_bar != NULL)
 737                 bus_release_resource(priv->dev, SYS_RES_MEMORY,
 738                     rman_get_rid(priv->db_bar), priv->db_bar);
 739
 740         if (priv->reg_bar != NULL)
 741                 bus_release_resource(priv->dev, SYS_RES_MEMORY,
 742                     rman_get_rid(priv->reg_bar), priv->reg_bar);
 743 }
 744
 745 static int
 746 gve_attach(device_t dev)
 747 {
 748         struct gve_priv *priv;
 749         int rid;
 750         int err;
 751
 752         priv = device_get_softc(dev);
 753         priv->dev = dev;
 754         GVE_IFACE_LOCK_INIT(priv->gve_iface_lock);
 755
 756         pci_enable_busmaster(dev);
 757
 758         rid = PCIR_BAR(GVE_REGISTER_BAR);
 759         priv->reg_bar = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 760             &rid, RF_ACTIVE);
 761         if (priv->reg_bar == NULL) {
 762                 device_printf(dev, "Failed to allocate BAR0\n");
 763                 err = ENXIO;
 764                 goto abort;
 765         }
 766
 767         rid = PCIR_BAR(GVE_DOORBELL_BAR);
 768         priv->db_bar = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 769             &rid, RF_ACTIVE);
 770         if (priv->db_bar == NULL) {
 771                 device_printf(dev, "Failed to allocate BAR2\n");
 772                 err = ENXIO;
 773                 goto abort;
 774         }
 775
 776         rid = pci_msix_table_bar(priv->dev);
 777         priv->msix_table = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 778             &rid, RF_ACTIVE);
 779         if (priv->msix_table == NULL) {
 780                 device_printf(dev, "Failed to allocate msix table\n");
 781                 err = ENXIO;
 782                 goto abort;
 783         }
 784
 785         err = gve_alloc_adminq_and_describe_device(priv);
 786         if (err != 0)
 787                 goto abort;
 788
 789         err = gve_configure_resources(priv);
 790         if (err != 0)
 791                 goto abort;
 792
 793         err = gve_alloc_rings(priv);
 794         if (err != 0)
 795                 goto abort;
 796
 797         err = gve_setup_ifnet(dev, priv);
 798         if (err != 0)
 799                 goto abort;
 800
 801         priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK;
 802
 803         bus_write_multi_1(priv->reg_bar, DRIVER_VERSION, GVE_DRIVER_VERSION,
 804             sizeof(GVE_DRIVER_VERSION) - 1);
 805
 806         TASK_INIT(&priv->service_task, 0, gve_service_task, priv);
 807         priv->service_tq = taskqueue_create("gve service", M_WAITOK | M_ZERO,
 808             taskqueue_thread_enqueue, &priv->service_tq);
 809         taskqueue_start_threads(&priv->service_tq, 1, PI_NET, "%s service tq",
 810             device_get_nameunit(priv->dev));
 811
 812         gve_setup_sysctl(priv);
 813
 814         if (bootverbose)
 815                 device_printf(priv->dev, "Successfully attached %s", GVE_DRIVER_VERSION);
 816         return (0);
 817
 818 abort:
 819         gve_free_rings(priv);
 820         gve_deconfigure_resources(priv);
 821         gve_release_adminq(priv);
 822         gve_free_sys_res_mem(priv);
 823         GVE_IFACE_LOCK_DESTROY(priv->gve_iface_lock);
 824         return (err);
 825 }
 826
 827 static int
 828 gve_detach(device_t dev)
 829 {
 830         struct gve_priv *priv = device_get_softc(dev);
 831         if_t ifp = priv->ifp;
 832
 833         ether_ifdetach(ifp);
 834
 835         GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
 836         gve_destroy(priv);
 837         GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
 838
 839         gve_free_rings(priv);
 840         gve_free_sys_res_mem(priv);
 841         GVE_IFACE_LOCK_DESTROY(priv->gve_iface_lock);
 842
 843         while (taskqueue_cancel(priv->service_tq, &priv->service_task, NULL))
 844                 taskqueue_drain(priv->service_tq, &priv->service_task);
 845         taskqueue_free(priv->service_tq);
 846
 847         if_free(ifp);
 848         return (bus_generic_detach(dev));
 849 }
 850
 851 static device_method_t gve_methods[] = {
 852         DEVMETHOD(device_probe, gve_probe),
 853         DEVMETHOD(device_attach, gve_attach),
 854         DEVMETHOD(device_detach, gve_detach),
 855         DEVMETHOD_END
 856 };
 857
 858 static driver_t gve_driver = {
 859         "gve",
 860         gve_methods,
 861         sizeof(struct gve_priv)
 862 };
 863
 864 #if __FreeBSD_version < 1301503
 865 static devclass_t gve_devclass;
 866
 867 DRIVER_MODULE(gve, pci, gve_driver, gve_devclass, 0, 0);
 868 #else
 869 DRIVER_MODULE(gve, pci, gve_driver, 0, 0);
 870 #endif
 871 MODULE_PNP_INFO("U16:vendor;U16:device;D:#", pci, gve, gve_devs,
 872     nitems(gve_devs));