sys/dev/mlx5/mlx5_core/mlx5_main.c

   1 /*-
   2  * Copyright (c) 2013-2019, Mellanox Technologies, Ltd.  All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
  14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  16  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
  17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  23  * SUCH DAMAGE.
  24  *
  25  * $FreeBSD$
  26  */
  27
  28 #include <linux/kmod.h>
  29 #include <linux/module.h>
  30 #include <linux/errno.h>
  31 #include <linux/pci.h>
  32 #include <linux/dma-mapping.h>
  33 #include <linux/slab.h>
  34 #include <linux/io-mapping.h>
  35 #include <linux/interrupt.h>
  36 #include <linux/hardirq.h>
  37 #include <dev/mlx5/driver.h>
  38 #include <dev/mlx5/cq.h>
  39 #include <dev/mlx5/qp.h>
  40 #include <dev/mlx5/srq.h>
  41 #include <dev/mlx5/mpfs.h>
  42 #include <dev/mlx5/vport.h>
  43 #include <linux/delay.h>
  44 #include <dev/mlx5/mlx5_ifc.h>
  45 #include <dev/mlx5/mlx5_fpga/core.h>
  46 #include <dev/mlx5/mlx5_lib/mlx5.h>
  47 #include "mlx5_core.h"
  48 #include "eswitch.h"
  49 #include "fs_core.h"
  50 #ifdef PCI_IOV
  51 #include <sys/nv.h>
  52 #include <dev/pci/pci_iov.h>
  53 #include <sys/iov_schema.h>
  54 #endif
  55
  56 static const char mlx5_version[] = "Mellanox Core driver "
  57         DRIVER_VERSION " (" DRIVER_RELDATE ")";
  58 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
  59 MODULE_DESCRIPTION("Mellanox Connect-IB, ConnectX-4 core driver");
  60 MODULE_LICENSE("Dual BSD/GPL");
  61 MODULE_DEPEND(mlx5, linuxkpi, 1, 1, 1);
  62 MODULE_DEPEND(mlx5, mlxfw, 1, 1, 1);
  63 MODULE_DEPEND(mlx5, firmware, 1, 1, 1);
  64 MODULE_VERSION(mlx5, 1);
  65
  66 SYSCTL_NODE(_hw, OID_AUTO, mlx5, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
  67     "mlx5 hardware controls");
  68
  69 int mlx5_core_debug_mask;
  70 SYSCTL_INT(_hw_mlx5, OID_AUTO, debug_mask, CTLFLAG_RWTUN,
  71     &mlx5_core_debug_mask, 0,
  72     "debug mask: 1 = dump cmd data, 2 = dump cmd exec time, 3 = both. Default=0");
  73
  74 #define MLX5_DEFAULT_PROF       2
  75 static int mlx5_prof_sel = MLX5_DEFAULT_PROF;
  76 SYSCTL_INT(_hw_mlx5, OID_AUTO, prof_sel, CTLFLAG_RWTUN,
  77     &mlx5_prof_sel, 0,
  78     "profile selector. Valid range 0 - 2");
  79
  80 static int mlx5_fast_unload_enabled = 1;
  81 SYSCTL_INT(_hw_mlx5, OID_AUTO, fast_unload_enabled, CTLFLAG_RWTUN,
  82     &mlx5_fast_unload_enabled, 0,
  83     "Set to enable fast unload. Clear to disable.");
  84
  85 #define NUMA_NO_NODE       -1
  86
  87 static LIST_HEAD(intf_list);
  88 static LIST_HEAD(dev_list);
  89 static DEFINE_MUTEX(intf_mutex);
  90
  91 struct mlx5_device_context {
  92         struct list_head        list;
  93         struct mlx5_interface  *intf;
  94         void                   *context;
  95 };
  96
  97 enum {
  98         MLX5_ATOMIC_REQ_MODE_BE = 0x0,
  99         MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS = 0x1,
 100 };
 101
 102 static struct mlx5_profile profiles[] = {
 103         [0] = {
 104                 .mask           = 0,
 105         },
 106         [1] = {
 107                 .mask           = MLX5_PROF_MASK_QP_SIZE,
 108                 .log_max_qp     = 12,
 109         },
 110         [2] = {
 111                 .mask           = MLX5_PROF_MASK_QP_SIZE |
 112                                   MLX5_PROF_MASK_MR_CACHE,
 113                 .log_max_qp     = 17,
 114                 .mr_cache[0]    = {
 115                         .size   = 500,
 116                         .limit  = 250
 117                 },
 118                 .mr_cache[1]    = {
 119                         .size   = 500,
 120                         .limit  = 250
 121                 },
 122                 .mr_cache[2]    = {
 123                         .size   = 500,
 124                         .limit  = 250
 125                 },
 126                 .mr_cache[3]    = {
 127                         .size   = 500,
 128                         .limit  = 250
 129                 },
 130                 .mr_cache[4]    = {
 131                         .size   = 500,
 132                         .limit  = 250
 133                 },
 134                 .mr_cache[5]    = {
 135                         .size   = 500,
 136                         .limit  = 250
 137                 },
 138                 .mr_cache[6]    = {
 139                         .size   = 500,
 140                         .limit  = 250
 141                 },
 142                 .mr_cache[7]    = {
 143                         .size   = 500,
 144                         .limit  = 250
 145                 },
 146                 .mr_cache[8]    = {
 147                         .size   = 500,
 148                         .limit  = 250
 149                 },
 150                 .mr_cache[9]    = {
 151                         .size   = 500,
 152                         .limit  = 250
 153                 },
 154                 .mr_cache[10]   = {
 155                         .size   = 500,
 156                         .limit  = 250
 157                 },
 158                 .mr_cache[11]   = {
 159                         .size   = 500,
 160                         .limit  = 250
 161                 },
 162                 .mr_cache[12]   = {
 163                         .size   = 64,
 164                         .limit  = 32
 165                 },
 166                 .mr_cache[13]   = {
 167                         .size   = 32,
 168                         .limit  = 16
 169                 },
 170                 .mr_cache[14]   = {
 171                         .size   = 16,
 172                         .limit  = 8
 173                 },
 174         },
 175         [3] = {
 176                 .mask           = MLX5_PROF_MASK_QP_SIZE,
 177                 .log_max_qp     = 17,
 178         },
 179 };
 180
 181 #ifdef PCI_IOV
 182 static const char iov_mac_addr_name[] = "mac-addr";
 183 #endif
 184
 185 static int set_dma_caps(struct pci_dev *pdev)
 186 {
 187         struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
 188         int err;
 189
 190         err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
 191         if (err) {
 192                 mlx5_core_warn(dev, "couldn't set 64-bit PCI DMA mask\n");
 193                 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
 194                 if (err) {
 195                         mlx5_core_err(dev, "Can't set PCI DMA mask, aborting\n");
 196                         return err;
 197                 }
 198         }
 199
 200         err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
 201         if (err) {
 202                 mlx5_core_warn(dev, "couldn't set 64-bit consistent PCI DMA mask\n");
 203                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
 204                 if (err) {
 205                         mlx5_core_err(dev, "Can't set consistent PCI DMA mask, aborting\n");
 206                         return err;
 207                 }
 208         }
 209
 210         dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024);
 211         return err;
 212 }
 213
 214 int mlx5_pci_read_power_status(struct mlx5_core_dev *dev,
 215                                u16 *p_power, u8 *p_status)
 216 {
 217         u32 in[MLX5_ST_SZ_DW(mpein_reg)] = {};
 218         u32 out[MLX5_ST_SZ_DW(mpein_reg)] = {};
 219         int err;
 220
 221         err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
 222             MLX5_ACCESS_REG_SUMMARY_CTRL_ID_MPEIN, 0, 0);
 223
 224         *p_status = MLX5_GET(mpein_reg, out, pwr_status);
 225         *p_power = MLX5_GET(mpein_reg, out, pci_power);
 226         return err;
 227 }
 228
 229 static int mlx5_pci_enable_device(struct mlx5_core_dev *dev)
 230 {
 231         struct pci_dev *pdev = dev->pdev;
 232         int err = 0;
 233
 234         mutex_lock(&dev->pci_status_mutex);
 235         if (dev->pci_status == MLX5_PCI_STATUS_DISABLED) {
 236                 err = pci_enable_device(pdev);
 237                 if (!err)
 238                         dev->pci_status = MLX5_PCI_STATUS_ENABLED;
 239         }
 240         mutex_unlock(&dev->pci_status_mutex);
 241
 242         return err;
 243 }
 244
 245 static void mlx5_pci_disable_device(struct mlx5_core_dev *dev)
 246 {
 247         struct pci_dev *pdev = dev->pdev;
 248
 249         mutex_lock(&dev->pci_status_mutex);
 250         if (dev->pci_status == MLX5_PCI_STATUS_ENABLED) {
 251                 pci_disable_device(pdev);
 252                 dev->pci_status = MLX5_PCI_STATUS_DISABLED;
 253         }
 254         mutex_unlock(&dev->pci_status_mutex);
 255 }
 256
 257 static int request_bar(struct pci_dev *pdev)
 258 {
 259         struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
 260         int err = 0;
 261
 262         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
 263                 mlx5_core_err(dev, "Missing registers BAR, aborting\n");
 264                 return -ENODEV;
 265         }
 266
 267         err = pci_request_regions(pdev, DRIVER_NAME);
 268         if (err)
 269                 mlx5_core_err(dev, "Couldn't get PCI resources, aborting\n");
 270
 271         return err;
 272 }
 273
 274 static void release_bar(struct pci_dev *pdev)
 275 {
 276         pci_release_regions(pdev);
 277 }
 278
 279 static int mlx5_enable_msix(struct mlx5_core_dev *dev)
 280 {
 281         struct mlx5_priv *priv = &dev->priv;
 282         struct mlx5_eq_table *table = &priv->eq_table;
 283         int num_eqs = 1 << MLX5_CAP_GEN(dev, log_max_eq);
 284         int limit = dev->msix_eqvec;
 285         int nvec = MLX5_EQ_VEC_COMP_BASE;
 286         int i;
 287
 288         if (limit > 0)
 289                 nvec += limit;
 290         else
 291                 nvec += MLX5_CAP_GEN(dev, num_ports) * num_online_cpus();
 292
 293         if (nvec > num_eqs)
 294                 nvec = num_eqs;
 295         if (nvec > 256)
 296                 nvec = 256;     /* limit of firmware API */
 297         if (nvec <= MLX5_EQ_VEC_COMP_BASE)
 298                 return -ENOMEM;
 299
 300         priv->msix_arr = kzalloc(nvec * sizeof(*priv->msix_arr), GFP_KERNEL);
 301
 302         for (i = 0; i < nvec; i++)
 303                 priv->msix_arr[i].entry = i;
 304
 305         nvec = pci_enable_msix_range(dev->pdev, priv->msix_arr,
 306                                      MLX5_EQ_VEC_COMP_BASE + 1, nvec);
 307         if (nvec < 0)
 308                 return nvec;
 309
 310         table->num_comp_vectors = nvec - MLX5_EQ_VEC_COMP_BASE;
 311         return 0;
 312 }
 313
 314 static void mlx5_disable_msix(struct mlx5_core_dev *dev)
 315 {
 316         struct mlx5_priv *priv = &dev->priv;
 317
 318         pci_disable_msix(dev->pdev);
 319         kfree(priv->msix_arr);
 320 }
 321
 322 struct mlx5_reg_host_endianess {
 323         u8      he;
 324         u8      rsvd[15];
 325 };
 326
 327
 328 #define CAP_MASK(pos, size) ((u64)((1 << (size)) - 1) << (pos))
 329
 330 enum {
 331         MLX5_CAP_BITS_RW_MASK = CAP_MASK(MLX5_CAP_OFF_CMDIF_CSUM, 2) |
 332                                 MLX5_DEV_CAP_FLAG_DCT |
 333                                 MLX5_DEV_CAP_FLAG_DRAIN_SIGERR,
 334 };
 335
 336 static u16 to_fw_pkey_sz(struct mlx5_core_dev *dev, u32 size)
 337 {
 338         switch (size) {
 339         case 128:
 340                 return 0;
 341         case 256:
 342                 return 1;
 343         case 512:
 344                 return 2;
 345         case 1024:
 346                 return 3;
 347         case 2048:
 348                 return 4;
 349         case 4096:
 350                 return 5;
 351         default:
 352                 mlx5_core_warn(dev, "invalid pkey table size %d\n", size);
 353                 return 0;
 354         }
 355 }
 356
 357 static int mlx5_core_get_caps_mode(struct mlx5_core_dev *dev,
 358                                    enum mlx5_cap_type cap_type,
 359                                    enum mlx5_cap_mode cap_mode)
 360 {
 361         u8 in[MLX5_ST_SZ_BYTES(query_hca_cap_in)];
 362         int out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
 363         void *out, *hca_caps;
 364         u16 opmod = (cap_type << 1) | (cap_mode & 0x01);
 365         int err;
 366
 367         memset(in, 0, sizeof(in));
 368         out = kzalloc(out_sz, GFP_KERNEL);
 369
 370         MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
 371         MLX5_SET(query_hca_cap_in, in, op_mod, opmod);
 372         err = mlx5_cmd_exec(dev, in, sizeof(in), out, out_sz);
 373         if (err) {
 374                 mlx5_core_warn(dev,
 375                                "QUERY_HCA_CAP : type(%x) opmode(%x) Failed(%d)\n",
 376                                cap_type, cap_mode, err);
 377                 goto query_ex;
 378         }
 379
 380         hca_caps =  MLX5_ADDR_OF(query_hca_cap_out, out, capability);
 381
 382         switch (cap_mode) {
 383         case HCA_CAP_OPMOD_GET_MAX:
 384                 memcpy(dev->hca_caps_max[cap_type], hca_caps,
 385                        MLX5_UN_SZ_BYTES(hca_cap_union));
 386                 break;
 387         case HCA_CAP_OPMOD_GET_CUR:
 388                 memcpy(dev->hca_caps_cur[cap_type], hca_caps,
 389                        MLX5_UN_SZ_BYTES(hca_cap_union));
 390                 break;
 391         default:
 392                 mlx5_core_warn(dev,
 393                                "Tried to query dev cap type(%x) with wrong opmode(%x)\n",
 394                                cap_type, cap_mode);
 395                 err = -EINVAL;
 396                 break;
 397         }
 398 query_ex:
 399         kfree(out);
 400         return err;
 401 }
 402
 403 int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type)
 404 {
 405         int ret;
 406
 407         ret = mlx5_core_get_caps_mode(dev, cap_type, HCA_CAP_OPMOD_GET_CUR);
 408         if (ret)
 409                 return ret;
 410
 411         return mlx5_core_get_caps_mode(dev, cap_type, HCA_CAP_OPMOD_GET_MAX);
 412 }
 413
 414 static int set_caps(struct mlx5_core_dev *dev, void *in, int in_sz)
 415 {
 416         u32 out[MLX5_ST_SZ_DW(set_hca_cap_out)] = {0};
 417
 418         MLX5_SET(set_hca_cap_in, in, opcode, MLX5_CMD_OP_SET_HCA_CAP);
 419
 420         return mlx5_cmd_exec(dev, in, in_sz, out, sizeof(out));
 421 }
 422
 423 static int handle_hca_cap(struct mlx5_core_dev *dev)
 424 {
 425         void *set_ctx = NULL;
 426         struct mlx5_profile *prof = dev->profile;
 427         int err = -ENOMEM;
 428         int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in);
 429         void *set_hca_cap;
 430
 431         set_ctx = kzalloc(set_sz, GFP_KERNEL);
 432
 433         err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL);
 434         if (err)
 435                 goto query_ex;
 436
 437         set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx,
 438                                    capability);
 439         memcpy(set_hca_cap, dev->hca_caps_cur[MLX5_CAP_GENERAL],
 440                MLX5_ST_SZ_BYTES(cmd_hca_cap));
 441
 442         mlx5_core_dbg(dev, "Current Pkey table size %d Setting new size %d\n",
 443                       mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(dev, pkey_table_size)),
 444                       128);
 445         /* we limit the size of the pkey table to 128 entries for now */
 446         MLX5_SET(cmd_hca_cap, set_hca_cap, pkey_table_size,
 447                  to_fw_pkey_sz(dev, 128));
 448
 449         if (prof->mask & MLX5_PROF_MASK_QP_SIZE)
 450                 MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_qp,
 451                          prof->log_max_qp);
 452
 453         /* disable cmdif checksum */
 454         MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, 0);
 455
 456         /* enable drain sigerr */
 457         MLX5_SET(cmd_hca_cap, set_hca_cap, drain_sigerr, 1);
 458
 459         MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, PAGE_SHIFT - 12);
 460
 461         err = set_caps(dev, set_ctx, set_sz);
 462
 463 query_ex:
 464         kfree(set_ctx);
 465         return err;
 466 }
 467
 468 static int handle_hca_cap_atomic(struct mlx5_core_dev *dev)
 469 {
 470         void *set_ctx;
 471         void *set_hca_cap;
 472         int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in);
 473         int req_endianness;
 474         int err;
 475
 476         if (MLX5_CAP_GEN(dev, atomic)) {
 477                 err = mlx5_core_get_caps(dev, MLX5_CAP_ATOMIC);
 478                 if (err)
 479                         return err;
 480         } else {
 481                 return 0;
 482         }
 483
 484         req_endianness =
 485                 MLX5_CAP_ATOMIC(dev,
 486                                 supported_atomic_req_8B_endianess_mode_1);
 487
 488         if (req_endianness != MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS)
 489                 return 0;
 490
 491         set_ctx = kzalloc(set_sz, GFP_KERNEL);
 492         if (!set_ctx)
 493                 return -ENOMEM;
 494
 495         MLX5_SET(set_hca_cap_in, set_ctx, op_mod,
 496                  MLX5_SET_HCA_CAP_OP_MOD_ATOMIC << 1);
 497         set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability);
 498
 499         /* Set requestor to host endianness */
 500         MLX5_SET(atomic_caps, set_hca_cap, atomic_req_8B_endianess_mode,
 501                  MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS);
 502
 503         err = set_caps(dev, set_ctx, set_sz);
 504
 505         kfree(set_ctx);
 506         return err;
 507 }
 508
 509 static int set_hca_ctrl(struct mlx5_core_dev *dev)
 510 {
 511         struct mlx5_reg_host_endianess he_in;
 512         struct mlx5_reg_host_endianess he_out;
 513         int err;
 514
 515         if (MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH &&
 516             !MLX5_CAP_GEN(dev, roce))
 517                 return 0;
 518
 519         memset(&he_in, 0, sizeof(he_in));
 520         he_in.he = MLX5_SET_HOST_ENDIANNESS;
 521         err = mlx5_core_access_reg(dev, &he_in,  sizeof(he_in),
 522                                         &he_out, sizeof(he_out),
 523                                         MLX5_REG_HOST_ENDIANNESS, 0, 1);
 524         return err;
 525 }
 526
 527 static int mlx5_core_enable_hca(struct mlx5_core_dev *dev, u16 func_id)
 528 {
 529         u32 out[MLX5_ST_SZ_DW(enable_hca_out)] = {0};
 530         u32 in[MLX5_ST_SZ_DW(enable_hca_in)] = {0};
 531
 532         MLX5_SET(enable_hca_in, in, opcode, MLX5_CMD_OP_ENABLE_HCA);
 533         MLX5_SET(enable_hca_in, in, function_id, func_id);
 534         return mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
 535 }
 536
 537 static int mlx5_core_disable_hca(struct mlx5_core_dev *dev)
 538 {
 539         u32 out[MLX5_ST_SZ_DW(disable_hca_out)] = {0};
 540         u32 in[MLX5_ST_SZ_DW(disable_hca_in)] = {0};
 541
 542         MLX5_SET(disable_hca_in, in, opcode, MLX5_CMD_OP_DISABLE_HCA);
 543         return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 544 }
 545
 546 static int mlx5_core_set_issi(struct mlx5_core_dev *dev)
 547 {
 548         u32 query_in[MLX5_ST_SZ_DW(query_issi_in)] = {0};
 549         u32 query_out[MLX5_ST_SZ_DW(query_issi_out)] = {0};
 550         u32 sup_issi;
 551         int err;
 552
 553         MLX5_SET(query_issi_in, query_in, opcode, MLX5_CMD_OP_QUERY_ISSI);
 554
 555         err = mlx5_cmd_exec(dev, query_in, sizeof(query_in), query_out, sizeof(query_out));
 556         if (err) {
 557                 u32 syndrome;
 558                 u8 status;
 559
 560                 mlx5_cmd_mbox_status(query_out, &status, &syndrome);
 561                 if (status == MLX5_CMD_STAT_BAD_OP_ERR) {
 562                         mlx5_core_dbg(dev, "Only ISSI 0 is supported\n");
 563                         return 0;
 564                 }
 565
 566                 mlx5_core_err(dev, "failed to query ISSI\n");
 567                 return err;
 568         }
 569
 570         sup_issi = MLX5_GET(query_issi_out, query_out, supported_issi_dw0);
 571
 572         if (sup_issi & (1 << 1)) {
 573                 u32 set_in[MLX5_ST_SZ_DW(set_issi_in)]   = {0};
 574                 u32 set_out[MLX5_ST_SZ_DW(set_issi_out)] = {0};
 575
 576                 MLX5_SET(set_issi_in, set_in, opcode, MLX5_CMD_OP_SET_ISSI);
 577                 MLX5_SET(set_issi_in, set_in, current_issi, 1);
 578
 579                 err = mlx5_cmd_exec(dev, set_in, sizeof(set_in), set_out, sizeof(set_out));
 580                 if (err) {
 581                         mlx5_core_err(dev, "failed to set ISSI=1 err(%d)\n", err);
 582                         return err;
 583                 }
 584
 585                 dev->issi = 1;
 586
 587                 return 0;
 588         } else if (sup_issi & (1 << 0)) {
 589                 return 0;
 590         }
 591
 592         return -ENOTSUPP;
 593 }
 594
 595
 596 int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn, int *irqn)
 597 {
 598         struct mlx5_eq_table *table = &dev->priv.eq_table;
 599         struct mlx5_eq *eq;
 600         int err = -ENOENT;
 601
 602         spin_lock(&table->lock);
 603         list_for_each_entry(eq, &table->comp_eqs_list, list) {
 604                 if (eq->index == vector) {
 605                         *eqn = eq->eqn;
 606                         *irqn = eq->irqn;
 607                         err = 0;
 608                         break;
 609                 }
 610         }
 611         spin_unlock(&table->lock);
 612
 613         return err;
 614 }
 615 EXPORT_SYMBOL(mlx5_vector2eqn);
 616
 617 static void free_comp_eqs(struct mlx5_core_dev *dev)
 618 {
 619         struct mlx5_eq_table *table = &dev->priv.eq_table;
 620         struct mlx5_eq *eq, *n;
 621
 622         spin_lock(&table->lock);
 623         list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) {
 624                 list_del(&eq->list);
 625                 spin_unlock(&table->lock);
 626                 if (mlx5_destroy_unmap_eq(dev, eq))
 627                         mlx5_core_warn(dev, "failed to destroy EQ 0x%x\n",
 628                                        eq->eqn);
 629                 kfree(eq);
 630                 spin_lock(&table->lock);
 631         }
 632         spin_unlock(&table->lock);
 633 }
 634
 635 static int alloc_comp_eqs(struct mlx5_core_dev *dev)
 636 {
 637         struct mlx5_eq_table *table = &dev->priv.eq_table;
 638         struct mlx5_eq *eq;
 639         int ncomp_vec;
 640         int nent;
 641         int err;
 642         int i;
 643
 644         INIT_LIST_HEAD(&table->comp_eqs_list);
 645         ncomp_vec = table->num_comp_vectors;
 646         nent = MLX5_COMP_EQ_SIZE;
 647         for (i = 0; i < ncomp_vec; i++) {
 648                 eq = kzalloc(sizeof(*eq), GFP_KERNEL);
 649
 650                 err = mlx5_create_map_eq(dev, eq,
 651                                          i + MLX5_EQ_VEC_COMP_BASE, nent, 0,
 652                                          &dev->priv.uuari.uars[0]);
 653                 if (err) {
 654                         kfree(eq);
 655                         goto clean;
 656                 }
 657                 mlx5_core_dbg(dev, "allocated completion EQN %d\n", eq->eqn);
 658                 eq->index = i;
 659                 spin_lock(&table->lock);
 660                 list_add_tail(&eq->list, &table->comp_eqs_list);
 661                 spin_unlock(&table->lock);
 662         }
 663
 664         return 0;
 665
 666 clean:
 667         free_comp_eqs(dev);
 668         return err;
 669 }
 670
 671 static int map_bf_area(struct mlx5_core_dev *dev)
 672 {
 673         resource_size_t bf_start = pci_resource_start(dev->pdev, 0);
 674         resource_size_t bf_len = pci_resource_len(dev->pdev, 0);
 675
 676         dev->priv.bf_mapping = io_mapping_create_wc(bf_start, bf_len);
 677
 678         return dev->priv.bf_mapping ? 0 : -ENOMEM;
 679 }
 680
 681 static void unmap_bf_area(struct mlx5_core_dev *dev)
 682 {
 683         if (dev->priv.bf_mapping)
 684                 io_mapping_free(dev->priv.bf_mapping);
 685 }
 686
 687 static inline int fw_initializing(struct mlx5_core_dev *dev)
 688 {
 689         return ioread32be(&dev->iseg->initializing) >> 31;
 690 }
 691
 692 static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili,
 693                         u32 warn_time_mili)
 694 {
 695         int warn = jiffies + msecs_to_jiffies(warn_time_mili);
 696         int end = jiffies + msecs_to_jiffies(max_wait_mili);
 697         int err = 0;
 698
 699         MPASS(max_wait_mili > warn_time_mili);
 700
 701         while (fw_initializing(dev) == 1) {
 702                 if (time_after(jiffies, end)) {
 703                         err = -EBUSY;
 704                         break;
 705                 }
 706                 if (warn_time_mili && time_after(jiffies, warn)) {
 707                         mlx5_core_warn(dev,
 708                             "Waiting for FW initialization, timeout abort in %u s\n",
 709                             (unsigned int)(jiffies_to_msecs(end - warn) / 1000));
 710                         warn = jiffies + msecs_to_jiffies(warn_time_mili);
 711                 }
 712                 msleep(FW_INIT_WAIT_MS);
 713         }
 714
 715         if (err != 0)
 716                 mlx5_core_dbg(dev, "Full initializing bit dword = 0x%x\n",
 717                     ioread32be(&dev->iseg->initializing));
 718
 719         return err;
 720 }
 721
 722 static void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
 723 {
 724         struct mlx5_device_context *dev_ctx;
 725         struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv);
 726
 727         dev_ctx = kzalloc(sizeof(*dev_ctx), GFP_KERNEL);
 728         if (!dev_ctx)
 729                 return;
 730
 731         dev_ctx->intf    = intf;
 732         CURVNET_SET_QUIET(vnet0);
 733         dev_ctx->context = intf->add(dev);
 734         CURVNET_RESTORE();
 735
 736         if (dev_ctx->context) {
 737                 spin_lock_irq(&priv->ctx_lock);
 738                 list_add_tail(&dev_ctx->list, &priv->ctx_list);
 739                 spin_unlock_irq(&priv->ctx_lock);
 740         } else {
 741                 kfree(dev_ctx);
 742         }
 743 }
 744
 745 static void mlx5_remove_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
 746 {
 747         struct mlx5_device_context *dev_ctx;
 748         struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv);
 749
 750         list_for_each_entry(dev_ctx, &priv->ctx_list, list)
 751                 if (dev_ctx->intf == intf) {
 752                         spin_lock_irq(&priv->ctx_lock);
 753                         list_del(&dev_ctx->list);
 754                         spin_unlock_irq(&priv->ctx_lock);
 755
 756                         intf->remove(dev, dev_ctx->context);
 757                         kfree(dev_ctx);
 758                         return;
 759                 }
 760 }
 761
 762 int
 763 mlx5_register_device(struct mlx5_core_dev *dev)
 764 {
 765         struct mlx5_priv *priv = &dev->priv;
 766         struct mlx5_interface *intf;
 767
 768         mutex_lock(&intf_mutex);
 769         list_add_tail(&priv->dev_list, &dev_list);
 770         list_for_each_entry(intf, &intf_list, list)
 771                 mlx5_add_device(intf, priv);
 772         mutex_unlock(&intf_mutex);
 773
 774         return 0;
 775 }
 776
 777 void
 778 mlx5_unregister_device(struct mlx5_core_dev *dev)
 779 {
 780         struct mlx5_priv *priv = &dev->priv;
 781         struct mlx5_interface *intf;
 782
 783         mutex_lock(&intf_mutex);
 784         list_for_each_entry(intf, &intf_list, list)
 785                 mlx5_remove_device(intf, priv);
 786         list_del(&priv->dev_list);
 787         mutex_unlock(&intf_mutex);
 788 }
 789
 790 int mlx5_register_interface(struct mlx5_interface *intf)
 791 {
 792         struct mlx5_priv *priv;
 793
 794         if (!intf->add || !intf->remove)
 795                 return -EINVAL;
 796
 797         mutex_lock(&intf_mutex);
 798         list_add_tail(&intf->list, &intf_list);
 799         list_for_each_entry(priv, &dev_list, dev_list)
 800                 mlx5_add_device(intf, priv);
 801         mutex_unlock(&intf_mutex);
 802
 803         return 0;
 804 }
 805 EXPORT_SYMBOL(mlx5_register_interface);
 806
 807 void mlx5_unregister_interface(struct mlx5_interface *intf)
 808 {
 809         struct mlx5_priv *priv;
 810
 811         mutex_lock(&intf_mutex);
 812         list_for_each_entry(priv, &dev_list, dev_list)
 813                 mlx5_remove_device(intf, priv);
 814         list_del(&intf->list);
 815         mutex_unlock(&intf_mutex);
 816 }
 817 EXPORT_SYMBOL(mlx5_unregister_interface);
 818
 819 void *mlx5_get_protocol_dev(struct mlx5_core_dev *mdev, int protocol)
 820 {
 821         struct mlx5_priv *priv = &mdev->priv;
 822         struct mlx5_device_context *dev_ctx;
 823         unsigned long flags;
 824         void *result = NULL;
 825
 826         spin_lock_irqsave(&priv->ctx_lock, flags);
 827
 828         list_for_each_entry(dev_ctx, &mdev->priv.ctx_list, list)
 829                 if ((dev_ctx->intf->protocol == protocol) &&
 830                     dev_ctx->intf->get_dev) {
 831                         result = dev_ctx->intf->get_dev(dev_ctx->context);
 832                         break;
 833                 }
 834
 835         spin_unlock_irqrestore(&priv->ctx_lock, flags);
 836
 837         return result;
 838 }
 839 EXPORT_SYMBOL(mlx5_get_protocol_dev);
 840
 841 static int mlx5_auto_fw_update;
 842 SYSCTL_INT(_hw_mlx5, OID_AUTO, auto_fw_update, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
 843     &mlx5_auto_fw_update, 0,
 844     "Allow automatic firmware update on driver start");
 845 static int
 846 mlx5_firmware_update(struct mlx5_core_dev *dev)
 847 {
 848         const struct firmware *fw;
 849         int err;
 850
 851         TUNABLE_INT_FETCH("hw.mlx5.auto_fw_update", &mlx5_auto_fw_update);
 852         if (!mlx5_auto_fw_update)
 853                 return (0);
 854         fw = firmware_get("mlx5fw_mfa");
 855         if (fw) {
 856                 err = mlx5_firmware_flash(dev, fw);
 857                 firmware_put(fw, FIRMWARE_UNLOAD);
 858         }
 859         else
 860                 return (-ENOENT);
 861
 862         return err;
 863 }
 864
 865 static int mlx5_pci_init(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
 866 {
 867         struct pci_dev *pdev = dev->pdev;
 868         device_t bsddev;
 869         int err;
 870
 871         pdev = dev->pdev;
 872         bsddev = pdev->dev.bsddev;
 873         pci_set_drvdata(dev->pdev, dev);
 874         strncpy(priv->name, dev_name(&pdev->dev), MLX5_MAX_NAME_LEN);
 875         priv->name[MLX5_MAX_NAME_LEN - 1] = 0;
 876
 877         mutex_init(&priv->pgdir_mutex);
 878         INIT_LIST_HEAD(&priv->pgdir_list);
 879         spin_lock_init(&priv->mkey_lock);
 880
 881         priv->numa_node = NUMA_NO_NODE;
 882
 883         err = mlx5_pci_enable_device(dev);
 884         if (err) {
 885                 mlx5_core_err(dev, "Cannot enable PCI device, aborting\n");
 886                 goto err_dbg;
 887         }
 888
 889         err = request_bar(pdev);
 890         if (err) {
 891                 mlx5_core_err(dev, "error requesting BARs, aborting\n");
 892                 goto err_disable;
 893         }
 894
 895         pci_set_master(pdev);
 896
 897         err = set_dma_caps(pdev);
 898         if (err) {
 899                 mlx5_core_err(dev, "Failed setting DMA capabilities mask, aborting\n");
 900                 goto err_clr_master;
 901         }
 902
 903         dev->iseg_base = pci_resource_start(dev->pdev, 0);
 904         dev->iseg = ioremap(dev->iseg_base, sizeof(*dev->iseg));
 905         if (!dev->iseg) {
 906                 err = -ENOMEM;
 907                 mlx5_core_err(dev, "Failed mapping initialization segment, aborting\n");
 908                 goto err_clr_master;
 909         }
 910
 911         return 0;
 912
 913 err_clr_master:
 914         release_bar(dev->pdev);
 915 err_disable:
 916         mlx5_pci_disable_device(dev);
 917 err_dbg:
 918         return err;
 919 }
 920
 921 static void mlx5_pci_close(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
 922 {
 923 #ifdef PCI_IOV
 924         if (MLX5_CAP_GEN(dev, eswitch_flow_table))
 925                 pci_iov_detach(dev->pdev->dev.bsddev);
 926 #endif
 927         iounmap(dev->iseg);
 928         release_bar(dev->pdev);
 929         mlx5_pci_disable_device(dev);
 930 }
 931
 932 static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
 933 {
 934         int err;
 935
 936         err = mlx5_vsc_find_cap(dev);
 937         if (err)
 938                 mlx5_core_err(dev, "Unable to find vendor specific capabilities\n");
 939
 940         err = mlx5_query_hca_caps(dev);
 941         if (err) {
 942                 mlx5_core_err(dev, "query hca failed\n");
 943                 goto out;
 944         }
 945
 946         err = mlx5_query_board_id(dev);
 947         if (err) {
 948                 mlx5_core_err(dev, "query board id failed\n");
 949                 goto out;
 950         }
 951
 952         err = mlx5_eq_init(dev);
 953         if (err) {
 954                 mlx5_core_err(dev, "failed to initialize eq\n");
 955                 goto out;
 956         }
 957
 958         MLX5_INIT_DOORBELL_LOCK(&priv->cq_uar_lock);
 959
 960         err = mlx5_init_cq_table(dev);
 961         if (err) {
 962                 mlx5_core_err(dev, "failed to initialize cq table\n");
 963                 goto err_eq_cleanup;
 964         }
 965
 966         mlx5_init_qp_table(dev);
 967         mlx5_init_srq_table(dev);
 968         mlx5_init_mr_table(dev);
 969
 970         mlx5_init_reserved_gids(dev);
 971         mlx5_fpga_init(dev);
 972
 973 #ifdef RATELIMIT
 974         err = mlx5_init_rl_table(dev);
 975         if (err) {
 976                 mlx5_core_err(dev, "Failed to init rate limiting\n");
 977                 goto err_tables_cleanup;
 978         }
 979 #endif
 980         return 0;
 981
 982 #ifdef RATELIMIT
 983 err_tables_cleanup:
 984         mlx5_cleanup_mr_table(dev);
 985         mlx5_cleanup_srq_table(dev);
 986         mlx5_cleanup_qp_table(dev);
 987         mlx5_cleanup_cq_table(dev);
 988 #endif
 989
 990 err_eq_cleanup:
 991         mlx5_eq_cleanup(dev);
 992
 993 out:
 994         return err;
 995 }
 996
 997 static void mlx5_cleanup_once(struct mlx5_core_dev *dev)
 998 {
 999 #ifdef RATELIMIT
1000         mlx5_cleanup_rl_table(dev);
1001 #endif
1002         mlx5_fpga_cleanup(dev);
1003         mlx5_cleanup_reserved_gids(dev);
1004         mlx5_cleanup_mr_table(dev);
1005         mlx5_cleanup_srq_table(dev);
1006         mlx5_cleanup_qp_table(dev);
1007         mlx5_cleanup_cq_table(dev);
1008         mlx5_eq_cleanup(dev);
1009 }
1010
1011 static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
1012                          bool boot)
1013 {
1014         int err;
1015
1016         mutex_lock(&dev->intf_state_mutex);
1017         if (test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) {
1018                 mlx5_core_warn(dev, "interface is up, NOP\n");
1019                 goto out;
1020         }
1021
1022         mlx5_core_dbg(dev, "firmware version: %d.%d.%d\n",
1023             fw_rev_maj(dev), fw_rev_min(dev), fw_rev_sub(dev));
1024
1025         /*
1026          * On load removing any previous indication of internal error,
1027          * device is up
1028          */
1029         dev->state = MLX5_DEVICE_STATE_UP;
1030
1031         /* wait for firmware to accept initialization segments configurations
1032         */
1033         err = wait_fw_init(dev, FW_PRE_INIT_TIMEOUT_MILI,
1034             FW_INIT_WARN_MESSAGE_INTERVAL);
1035         if (err) {
1036                 dev_err(&dev->pdev->dev,
1037                     "Firmware over %d MS in pre-initializing state, aborting\n",
1038                     FW_PRE_INIT_TIMEOUT_MILI);
1039                 goto out_err;
1040         }
1041
1042         err = mlx5_cmd_init(dev);
1043         if (err) {
1044                 mlx5_core_err(dev,
1045                     "Failed initializing command interface, aborting\n");
1046                 goto out_err;
1047         }
1048
1049         err = wait_fw_init(dev, FW_INIT_TIMEOUT_MILI, 0);
1050         if (err) {
1051                 mlx5_core_err(dev,
1052                     "Firmware over %d MS in initializing state, aborting\n",
1053                     FW_INIT_TIMEOUT_MILI);
1054                 goto err_cmd_cleanup;
1055         }
1056
1057         err = mlx5_core_enable_hca(dev, 0);
1058         if (err) {
1059                 mlx5_core_err(dev, "enable hca failed\n");
1060                 goto err_cmd_cleanup;
1061         }
1062
1063         err = mlx5_core_set_issi(dev);
1064         if (err) {
1065                 mlx5_core_err(dev, "failed to set issi\n");
1066                 goto err_disable_hca;
1067         }
1068
1069         err = mlx5_pagealloc_start(dev);
1070         if (err) {
1071                 mlx5_core_err(dev, "mlx5_pagealloc_start failed\n");
1072                 goto err_disable_hca;
1073         }
1074
1075         err = mlx5_satisfy_startup_pages(dev, 1);
1076         if (err) {
1077                 mlx5_core_err(dev, "failed to allocate boot pages\n");
1078                 goto err_pagealloc_stop;
1079         }
1080
1081         err = set_hca_ctrl(dev);
1082         if (err) {
1083                 mlx5_core_err(dev, "set_hca_ctrl failed\n");
1084                 goto reclaim_boot_pages;
1085         }
1086
1087         err = handle_hca_cap(dev);
1088         if (err) {
1089                 mlx5_core_err(dev, "handle_hca_cap failed\n");
1090                 goto reclaim_boot_pages;
1091         }
1092
1093         err = handle_hca_cap_atomic(dev);
1094         if (err) {
1095                 mlx5_core_err(dev, "handle_hca_cap_atomic failed\n");
1096                 goto reclaim_boot_pages;
1097         }
1098
1099         err = mlx5_satisfy_startup_pages(dev, 0);
1100         if (err) {
1101                 mlx5_core_err(dev, "failed to allocate init pages\n");
1102                 goto reclaim_boot_pages;
1103         }
1104
1105         err = mlx5_cmd_init_hca(dev);
1106         if (err) {
1107                 mlx5_core_err(dev, "init hca failed\n");
1108                 goto reclaim_boot_pages;
1109         }
1110
1111         mlx5_start_health_poll(dev);
1112
1113         if (boot && mlx5_init_once(dev, priv)) {
1114                 mlx5_core_err(dev, "sw objs init failed\n");
1115                 goto err_stop_poll;
1116         }
1117
1118         err = mlx5_enable_msix(dev);
1119         if (err) {
1120                 mlx5_core_err(dev, "enable msix failed\n");
1121                 goto err_cleanup_once;
1122         }
1123
1124         err = mlx5_alloc_uuars(dev, &priv->uuari);
1125         if (err) {
1126                 mlx5_core_err(dev, "Failed allocating uar, aborting\n");
1127                 goto err_disable_msix;
1128         }
1129
1130         err = mlx5_start_eqs(dev);
1131         if (err) {
1132                 mlx5_core_err(dev, "Failed to start pages and async EQs\n");
1133                 goto err_free_uar;
1134         }
1135
1136         err = alloc_comp_eqs(dev);
1137         if (err) {
1138                 mlx5_core_err(dev, "Failed to alloc completion EQs\n");
1139                 goto err_stop_eqs;
1140         }
1141
1142         if (map_bf_area(dev))
1143                 mlx5_core_err(dev, "Failed to map blue flame area\n");
1144
1145         err = mlx5_init_fs(dev);
1146         if (err) {
1147                 mlx5_core_err(dev, "flow steering init %d\n", err);
1148                 goto err_free_comp_eqs;
1149         }
1150
1151         err = mlx5_mpfs_init(dev);
1152         if (err) {
1153                 mlx5_core_err(dev, "mpfs init failed %d\n", err);
1154                 goto err_fs;
1155         }
1156
1157         err = mlx5_fpga_device_start(dev);
1158         if (err) {
1159                 mlx5_core_err(dev, "fpga device start failed %d\n", err);
1160                 goto err_mpfs;
1161         }
1162
1163         err = mlx5_register_device(dev);
1164         if (err) {
1165                 mlx5_core_err(dev, "mlx5_register_device failed %d\n", err);
1166                 goto err_fpga;
1167         }
1168
1169         set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
1170
1171 out:
1172         mutex_unlock(&dev->intf_state_mutex);
1173         return 0;
1174
1175 err_fpga:
1176         mlx5_fpga_device_stop(dev);
1177
1178 err_mpfs:
1179         mlx5_mpfs_destroy(dev);
1180
1181 err_fs:
1182         mlx5_cleanup_fs(dev);
1183
1184 err_free_comp_eqs:
1185         free_comp_eqs(dev);
1186         unmap_bf_area(dev);
1187
1188 err_stop_eqs:
1189         mlx5_stop_eqs(dev);
1190
1191 err_free_uar:
1192         mlx5_free_uuars(dev, &priv->uuari);
1193
1194 err_disable_msix:
1195         mlx5_disable_msix(dev);
1196
1197 err_cleanup_once:
1198         if (boot)
1199                 mlx5_cleanup_once(dev);
1200
1201 err_stop_poll:
1202         mlx5_stop_health_poll(dev, boot);
1203         if (mlx5_cmd_teardown_hca(dev)) {
1204                 mlx5_core_err(dev, "tear_down_hca failed, skip cleanup\n");
1205                 goto out_err;
1206         }
1207
1208 reclaim_boot_pages:
1209         mlx5_reclaim_startup_pages(dev);
1210
1211 err_pagealloc_stop:
1212         mlx5_pagealloc_stop(dev);
1213
1214 err_disable_hca:
1215         mlx5_core_disable_hca(dev);
1216
1217 err_cmd_cleanup:
1218         mlx5_cmd_cleanup(dev);
1219
1220 out_err:
1221         dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
1222         mutex_unlock(&dev->intf_state_mutex);
1223
1224         return err;
1225 }
1226
1227 static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
1228                            bool cleanup)
1229 {
1230         int err = 0;
1231
1232         if (cleanup)
1233                 mlx5_drain_health_recovery(dev);
1234
1235         mutex_lock(&dev->intf_state_mutex);
1236         if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) {
1237                 mlx5_core_warn(dev, "%s: interface is down, NOP\n", __func__);
1238                 if (cleanup)
1239                         mlx5_cleanup_once(dev);
1240                 goto out;
1241         }
1242
1243         mlx5_unregister_device(dev);
1244
1245         mlx5_eswitch_cleanup(dev->priv.eswitch);
1246         mlx5_fpga_device_stop(dev);
1247         mlx5_mpfs_destroy(dev);
1248         mlx5_cleanup_fs(dev);
1249         unmap_bf_area(dev);
1250         mlx5_wait_for_reclaim_vfs_pages(dev);
1251         free_comp_eqs(dev);
1252         mlx5_stop_eqs(dev);
1253         mlx5_free_uuars(dev, &priv->uuari);
1254         mlx5_disable_msix(dev);
1255         if (cleanup)
1256                 mlx5_cleanup_once(dev);
1257         mlx5_stop_health_poll(dev, cleanup);
1258         err = mlx5_cmd_teardown_hca(dev);
1259         if (err) {
1260                 mlx5_core_err(dev, "tear_down_hca failed, skip cleanup\n");
1261                 goto out;
1262         }
1263         mlx5_pagealloc_stop(dev);
1264         mlx5_reclaim_startup_pages(dev);
1265         mlx5_core_disable_hca(dev);
1266         mlx5_cmd_cleanup(dev);
1267
1268 out:
1269         clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
1270         mutex_unlock(&dev->intf_state_mutex);
1271         return err;
1272 }
1273
1274 void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event,
1275                      unsigned long param)
1276 {
1277         struct mlx5_priv *priv = &dev->priv;
1278         struct mlx5_device_context *dev_ctx;
1279         unsigned long flags;
1280
1281         spin_lock_irqsave(&priv->ctx_lock, flags);
1282
1283         list_for_each_entry(dev_ctx, &priv->ctx_list, list)
1284                 if (dev_ctx->intf->event)
1285                         dev_ctx->intf->event(dev, dev_ctx->context, event, param);
1286
1287         spin_unlock_irqrestore(&priv->ctx_lock, flags);
1288 }
1289
1290 struct mlx5_core_event_handler {
1291         void (*event)(struct mlx5_core_dev *dev,
1292                       enum mlx5_dev_event event,
1293                       void *data);
1294 };
1295
1296 #define MLX5_STATS_DESC(a, b, c, d, e, ...) d, e,
1297
1298 #define MLX5_PORT_MODULE_ERROR_STATS(m)                         \
1299 m(+1, u64, power_budget_exceeded, "power_budget", "Module Power Budget Exceeded") \
1300 m(+1, u64, long_range, "long_range", "Module Long Range for non MLNX cable/module") \
1301 m(+1, u64, bus_stuck, "bus_stuck", "Module Bus stuck(I2C or data shorted)") \
1302 m(+1, u64, no_eeprom, "no_eeprom", "No EEPROM/retry timeout") \
1303 m(+1, u64, enforce_part_number, "enforce_part_number", "Module Enforce part number list") \
1304 m(+1, u64, unknown_id, "unknown_id", "Module Unknown identifier") \
1305 m(+1, u64, high_temp, "high_temp", "Module High Temperature") \
1306 m(+1, u64, cable_shorted, "cable_shorted", "Module Cable is shorted") \
1307 m(+1, u64, pmd_type_not_enabled, "pmd_type_not_enabled", "PMD type is not enabled") \
1308 m(+1, u64, laster_tec_failure, "laster_tec_failure", "Laster TEC failure") \
1309 m(+1, u64, high_current, "high_current", "High current") \
1310 m(+1, u64, high_voltage, "high_voltage", "High voltage") \
1311 m(+1, u64, pcie_sys_power_slot_exceeded, "pcie_sys_power_slot_exceeded", "PCIe system power slot Exceeded") \
1312 m(+1, u64, high_power, "high_power", "High power")                      \
1313 m(+1, u64, module_state_machine_fault, "module_state_machine_fault", "Module State Machine fault")
1314
1315 static const char *mlx5_pme_err_desc[] = {
1316         MLX5_PORT_MODULE_ERROR_STATS(MLX5_STATS_DESC)
1317 };
1318
1319 static int init_one(struct pci_dev *pdev,
1320                     const struct pci_device_id *id)
1321 {
1322         struct mlx5_core_dev *dev;
1323         struct mlx5_priv *priv;
1324         device_t bsddev = pdev->dev.bsddev;
1325 #ifdef PCI_IOV
1326         nvlist_t *pf_schema, *vf_schema;
1327         int num_vfs, sriov_pos;
1328 #endif
1329         int i,err;
1330         struct sysctl_oid *pme_sysctl_node;
1331         struct sysctl_oid *pme_err_sysctl_node;
1332         struct sysctl_oid *cap_sysctl_node;
1333         struct sysctl_oid *current_cap_sysctl_node;
1334         struct sysctl_oid *max_cap_sysctl_node;
1335
1336         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
1337         priv = &dev->priv;
1338         if (id)
1339                 priv->pci_dev_data = id->driver_data;
1340
1341         if (mlx5_prof_sel < 0 || mlx5_prof_sel >= ARRAY_SIZE(profiles)) {
1342                 device_printf(bsddev,
1343                     "WARN: selected profile out of range, selecting default (%d)\n",
1344                     MLX5_DEFAULT_PROF);
1345                 mlx5_prof_sel = MLX5_DEFAULT_PROF;
1346         }
1347         dev->profile = &profiles[mlx5_prof_sel];
1348         dev->pdev = pdev;
1349         dev->event = mlx5_core_event;
1350
1351         /* Set desc */
1352         device_set_desc(bsddev, mlx5_version);
1353
1354         sysctl_ctx_init(&dev->sysctl_ctx);
1355         SYSCTL_ADD_INT(&dev->sysctl_ctx,
1356             SYSCTL_CHILDREN(device_get_sysctl_tree(bsddev)),
1357             OID_AUTO, "msix_eqvec", CTLFLAG_RDTUN, &dev->msix_eqvec, 0,
1358             "Maximum number of MSIX event queue vectors, if set");
1359         SYSCTL_ADD_INT(&dev->sysctl_ctx,
1360             SYSCTL_CHILDREN(device_get_sysctl_tree(bsddev)),
1361             OID_AUTO, "power_status", CTLFLAG_RD, &dev->pwr_status, 0,
1362             "0:Invalid 1:Sufficient 2:Insufficient");
1363         SYSCTL_ADD_INT(&dev->sysctl_ctx,
1364             SYSCTL_CHILDREN(device_get_sysctl_tree(bsddev)),
1365             OID_AUTO, "power_value", CTLFLAG_RD, &dev->pwr_value, 0,
1366             "Current power value in Watts");
1367
1368         pme_sysctl_node = SYSCTL_ADD_NODE(&dev->sysctl_ctx,
1369             SYSCTL_CHILDREN(device_get_sysctl_tree(bsddev)),
1370             OID_AUTO, "pme_stats", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
1371             "Port module event statistics");
1372         if (pme_sysctl_node == NULL) {
1373                 err = -ENOMEM;
1374                 goto clean_sysctl_ctx;
1375         }
1376         pme_err_sysctl_node = SYSCTL_ADD_NODE(&dev->sysctl_ctx,
1377             SYSCTL_CHILDREN(pme_sysctl_node),
1378             OID_AUTO, "errors", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
1379             "Port module event error statistics");
1380         if (pme_err_sysctl_node == NULL) {
1381                 err = -ENOMEM;
1382                 goto clean_sysctl_ctx;
1383         }
1384         SYSCTL_ADD_U64(&dev->sysctl_ctx,
1385             SYSCTL_CHILDREN(pme_sysctl_node), OID_AUTO,
1386             "module_plug", CTLFLAG_RD | CTLFLAG_MPSAFE,
1387             &dev->priv.pme_stats.status_counters[MLX5_MODULE_STATUS_PLUGGED_ENABLED],
1388             0, "Number of time module plugged");
1389         SYSCTL_ADD_U64(&dev->sysctl_ctx,
1390             SYSCTL_CHILDREN(pme_sysctl_node), OID_AUTO,
1391             "module_unplug", CTLFLAG_RD | CTLFLAG_MPSAFE,
1392             &dev->priv.pme_stats.status_counters[MLX5_MODULE_STATUS_UNPLUGGED],
1393             0, "Number of time module unplugged");
1394         for (i = 0 ; i < MLX5_MODULE_EVENT_ERROR_NUM; i++) {
1395                 SYSCTL_ADD_U64(&dev->sysctl_ctx,
1396                     SYSCTL_CHILDREN(pme_err_sysctl_node), OID_AUTO,
1397                     mlx5_pme_err_desc[2 * i], CTLFLAG_RD | CTLFLAG_MPSAFE,
1398                     &dev->priv.pme_stats.error_counters[i],
1399                     0, mlx5_pme_err_desc[2 * i + 1]);
1400         }
1401
1402         cap_sysctl_node = SYSCTL_ADD_NODE(&dev->sysctl_ctx,
1403             SYSCTL_CHILDREN(device_get_sysctl_tree(bsddev)),
1404             OID_AUTO, "caps", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
1405             "hardware capabilities raw bitstrings");
1406         if (cap_sysctl_node == NULL) {
1407                 err = -ENOMEM;
1408                 goto clean_sysctl_ctx;
1409         }
1410         current_cap_sysctl_node = SYSCTL_ADD_NODE(&dev->sysctl_ctx,
1411             SYSCTL_CHILDREN(cap_sysctl_node),
1412             OID_AUTO, "current", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
1413             "");
1414         if (current_cap_sysctl_node == NULL) {
1415                 err = -ENOMEM;
1416                 goto clean_sysctl_ctx;
1417         }
1418         max_cap_sysctl_node = SYSCTL_ADD_NODE(&dev->sysctl_ctx,
1419             SYSCTL_CHILDREN(cap_sysctl_node),
1420             OID_AUTO, "max", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
1421             "");
1422         if (max_cap_sysctl_node == NULL) {
1423                 err = -ENOMEM;
1424                 goto clean_sysctl_ctx;
1425         }
1426         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1427             SYSCTL_CHILDREN(current_cap_sysctl_node),
1428             OID_AUTO, "general", CTLFLAG_RD | CTLFLAG_MPSAFE,
1429             &dev->hca_caps_cur[MLX5_CAP_GENERAL],
1430             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1431         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1432             SYSCTL_CHILDREN(max_cap_sysctl_node),
1433             OID_AUTO, "general", CTLFLAG_RD | CTLFLAG_MPSAFE,
1434             &dev->hca_caps_max[MLX5_CAP_GENERAL],
1435             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1436         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1437             SYSCTL_CHILDREN(current_cap_sysctl_node),
1438             OID_AUTO, "ether", CTLFLAG_RD | CTLFLAG_MPSAFE,
1439             &dev->hca_caps_cur[MLX5_CAP_ETHERNET_OFFLOADS],
1440             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1441         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1442             SYSCTL_CHILDREN(max_cap_sysctl_node),
1443             OID_AUTO, "ether", CTLFLAG_RD | CTLFLAG_MPSAFE,
1444             &dev->hca_caps_max[MLX5_CAP_ETHERNET_OFFLOADS],
1445             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1446         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1447             SYSCTL_CHILDREN(current_cap_sysctl_node),
1448             OID_AUTO, "odp", CTLFLAG_RD | CTLFLAG_MPSAFE,
1449             &dev->hca_caps_cur[MLX5_CAP_ODP],
1450             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1451         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1452             SYSCTL_CHILDREN(max_cap_sysctl_node),
1453             OID_AUTO, "odp", CTLFLAG_RD | CTLFLAG_MPSAFE,
1454             &dev->hca_caps_max[MLX5_CAP_ODP],
1455             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1456         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1457             SYSCTL_CHILDREN(current_cap_sysctl_node),
1458             OID_AUTO, "atomic", CTLFLAG_RD | CTLFLAG_MPSAFE,
1459             &dev->hca_caps_cur[MLX5_CAP_ATOMIC],
1460             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1461         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1462             SYSCTL_CHILDREN(max_cap_sysctl_node),
1463             OID_AUTO, "atomic", CTLFLAG_RD | CTLFLAG_MPSAFE,
1464             &dev->hca_caps_max[MLX5_CAP_ATOMIC],
1465             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1466         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1467             SYSCTL_CHILDREN(current_cap_sysctl_node),
1468             OID_AUTO, "roce", CTLFLAG_RD | CTLFLAG_MPSAFE,
1469             &dev->hca_caps_cur[MLX5_CAP_ROCE],
1470             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1471         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1472             SYSCTL_CHILDREN(max_cap_sysctl_node),
1473             OID_AUTO, "roce", CTLFLAG_RD | CTLFLAG_MPSAFE,
1474             &dev->hca_caps_max[MLX5_CAP_ROCE],
1475             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1476         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1477             SYSCTL_CHILDREN(current_cap_sysctl_node),
1478             OID_AUTO, "ipoib", CTLFLAG_RD | CTLFLAG_MPSAFE,
1479             &dev->hca_caps_cur[MLX5_CAP_IPOIB_OFFLOADS],
1480             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1481         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1482             SYSCTL_CHILDREN(max_cap_sysctl_node),
1483             OID_AUTO, "ipoib", CTLFLAG_RD | CTLFLAG_MPSAFE,
1484             &dev->hca_caps_max[MLX5_CAP_IPOIB_OFFLOADS],
1485             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1486         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1487             SYSCTL_CHILDREN(current_cap_sysctl_node),
1488             OID_AUTO, "eoib", CTLFLAG_RD | CTLFLAG_MPSAFE,
1489             &dev->hca_caps_cur[MLX5_CAP_EOIB_OFFLOADS],
1490             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1491         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1492             SYSCTL_CHILDREN(max_cap_sysctl_node),
1493             OID_AUTO, "eoib", CTLFLAG_RD | CTLFLAG_MPSAFE,
1494             &dev->hca_caps_max[MLX5_CAP_EOIB_OFFLOADS],
1495             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1496         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1497             SYSCTL_CHILDREN(current_cap_sysctl_node),
1498             OID_AUTO, "flow_table", CTLFLAG_RD | CTLFLAG_MPSAFE,
1499             &dev->hca_caps_cur[MLX5_CAP_FLOW_TABLE],
1500             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1501         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1502             SYSCTL_CHILDREN(max_cap_sysctl_node),
1503             OID_AUTO, "flow_table", CTLFLAG_RD | CTLFLAG_MPSAFE,
1504             &dev->hca_caps_max[MLX5_CAP_FLOW_TABLE],
1505             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1506         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1507             SYSCTL_CHILDREN(current_cap_sysctl_node),
1508             OID_AUTO, "eswitch_flow_table", CTLFLAG_RD | CTLFLAG_MPSAFE,
1509             &dev->hca_caps_cur[MLX5_CAP_ESWITCH_FLOW_TABLE],
1510             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1511         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1512             SYSCTL_CHILDREN(max_cap_sysctl_node),
1513             OID_AUTO, "eswitch_flow_table", CTLFLAG_RD | CTLFLAG_MPSAFE,
1514             &dev->hca_caps_max[MLX5_CAP_ESWITCH_FLOW_TABLE],
1515             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1516         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1517             SYSCTL_CHILDREN(current_cap_sysctl_node),
1518             OID_AUTO, "eswitch", CTLFLAG_RD | CTLFLAG_MPSAFE,
1519             &dev->hca_caps_cur[MLX5_CAP_ESWITCH],
1520             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1521         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1522             SYSCTL_CHILDREN(max_cap_sysctl_node),
1523             OID_AUTO, "eswitch", CTLFLAG_RD | CTLFLAG_MPSAFE,
1524             &dev->hca_caps_max[MLX5_CAP_ESWITCH],
1525             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1526         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1527             SYSCTL_CHILDREN(current_cap_sysctl_node),
1528             OID_AUTO, "snapshot", CTLFLAG_RD | CTLFLAG_MPSAFE,
1529             &dev->hca_caps_cur[MLX5_CAP_SNAPSHOT],
1530             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1531         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1532             SYSCTL_CHILDREN(max_cap_sysctl_node),
1533             OID_AUTO, "snapshot", CTLFLAG_RD | CTLFLAG_MPSAFE,
1534             &dev->hca_caps_max[MLX5_CAP_SNAPSHOT],
1535             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1536         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1537             SYSCTL_CHILDREN(current_cap_sysctl_node),
1538             OID_AUTO, "vector_calc", CTLFLAG_RD | CTLFLAG_MPSAFE,
1539             &dev->hca_caps_cur[MLX5_CAP_VECTOR_CALC],
1540             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1541         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1542             SYSCTL_CHILDREN(max_cap_sysctl_node),
1543             OID_AUTO, "vector_calc", CTLFLAG_RD | CTLFLAG_MPSAFE,
1544             &dev->hca_caps_max[MLX5_CAP_VECTOR_CALC],
1545             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1546         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1547             SYSCTL_CHILDREN(current_cap_sysctl_node),
1548             OID_AUTO, "qos", CTLFLAG_RD | CTLFLAG_MPSAFE,
1549             &dev->hca_caps_cur[MLX5_CAP_QOS],
1550             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1551         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1552             SYSCTL_CHILDREN(max_cap_sysctl_node),
1553             OID_AUTO, "qos", CTLFLAG_RD | CTLFLAG_MPSAFE,
1554             &dev->hca_caps_max[MLX5_CAP_QOS],
1555             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1556         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1557             SYSCTL_CHILDREN(current_cap_sysctl_node),
1558             OID_AUTO, "debug", CTLFLAG_RD | CTLFLAG_MPSAFE,
1559             &dev->hca_caps_cur[MLX5_CAP_DEBUG],
1560             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1561         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1562             SYSCTL_CHILDREN(max_cap_sysctl_node),
1563             OID_AUTO, "debug", CTLFLAG_RD | CTLFLAG_MPSAFE,
1564             &dev->hca_caps_max[MLX5_CAP_DEBUG],
1565             MLX5_UN_SZ_DW(hca_cap_union) * sizeof(u32), "IU", "");
1566         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1567             SYSCTL_CHILDREN(cap_sysctl_node),
1568             OID_AUTO, "pcam", CTLFLAG_RD | CTLFLAG_MPSAFE,
1569             &dev->caps.pcam, sizeof(dev->caps.pcam), "IU", "");
1570         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1571             SYSCTL_CHILDREN(cap_sysctl_node),
1572             OID_AUTO, "mcam", CTLFLAG_RD | CTLFLAG_MPSAFE,
1573             &dev->caps.mcam, sizeof(dev->caps.mcam), "IU", "");
1574         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1575             SYSCTL_CHILDREN(cap_sysctl_node),
1576             OID_AUTO, "qcam", CTLFLAG_RD | CTLFLAG_MPSAFE,
1577             &dev->caps.qcam, sizeof(dev->caps.qcam), "IU", "");
1578         SYSCTL_ADD_OPAQUE(&dev->sysctl_ctx,
1579             SYSCTL_CHILDREN(cap_sysctl_node),
1580             OID_AUTO, "fpga", CTLFLAG_RD | CTLFLAG_MPSAFE,
1581             &dev->caps.fpga, sizeof(dev->caps.fpga), "IU", "");
1582
1583         INIT_LIST_HEAD(&priv->ctx_list);
1584         spin_lock_init(&priv->ctx_lock);
1585         mutex_init(&dev->pci_status_mutex);
1586         mutex_init(&dev->intf_state_mutex);
1587         mtx_init(&dev->dump_lock, "mlx5dmp", NULL, MTX_DEF | MTX_NEW);
1588         err = mlx5_pci_init(dev, priv);
1589         if (err) {
1590                 mlx5_core_err(dev, "mlx5_pci_init failed %d\n", err);
1591                 goto clean_dev;
1592         }
1593
1594         err = mlx5_health_init(dev);
1595         if (err) {
1596                 mlx5_core_err(dev, "mlx5_health_init failed %d\n", err);
1597                 goto close_pci;
1598         }
1599
1600         mlx5_pagealloc_init(dev);
1601
1602         err = mlx5_load_one(dev, priv, true);
1603         if (err) {
1604                 mlx5_core_err(dev, "mlx5_load_one failed %d\n", err);
1605                 goto clean_health;
1606         }
1607
1608         mlx5_fwdump_prep(dev);
1609
1610         mlx5_firmware_update(dev);
1611
1612 #ifdef PCI_IOV
1613         if (MLX5_CAP_GEN(dev, vport_group_manager)) {
1614                 if (pci_find_extcap(bsddev, PCIZ_SRIOV, &sriov_pos) == 0) {
1615                         num_vfs = pci_read_config(bsddev, sriov_pos +
1616                             PCIR_SRIOV_TOTAL_VFS, 2);
1617                 } else {
1618                         mlx5_core_info(dev, "cannot find SR-IOV PCIe cap\n");
1619                         num_vfs = 0;
1620                 }
1621                 err = mlx5_eswitch_init(dev, 1 + num_vfs);
1622                 if (err == 0) {
1623                         pf_schema = pci_iov_schema_alloc_node();
1624                         vf_schema = pci_iov_schema_alloc_node();
1625                         pci_iov_schema_add_unicast_mac(vf_schema,
1626                             iov_mac_addr_name, 0, NULL);
1627                         err = pci_iov_attach(bsddev, pf_schema, vf_schema);
1628                         if (err != 0) {
1629                                 device_printf(bsddev,
1630                             "Failed to initialize SR-IOV support, error %d\n",
1631                                     err);
1632                         }
1633                 } else {
1634                         mlx5_core_err(dev, "eswitch init failed, error %d\n",
1635                             err);
1636                 }
1637         }
1638 #endif
1639
1640         pci_save_state(bsddev);
1641         return 0;
1642
1643 clean_health:
1644         mlx5_pagealloc_cleanup(dev);
1645         mlx5_health_cleanup(dev);
1646 close_pci:
1647         mlx5_pci_close(dev, priv);
1648 clean_dev:
1649         mtx_destroy(&dev->dump_lock);
1650 clean_sysctl_ctx:
1651         sysctl_ctx_free(&dev->sysctl_ctx);
1652         kfree(dev);
1653         return err;
1654 }
1655
1656 static void remove_one(struct pci_dev *pdev)
1657 {
1658         struct mlx5_core_dev *dev  = pci_get_drvdata(pdev);
1659         struct mlx5_priv *priv = &dev->priv;
1660
1661         if (mlx5_unload_one(dev, priv, true)) {
1662                 mlx5_core_err(dev, "mlx5_unload_one failed\n");
1663                 mlx5_health_cleanup(dev);
1664                 return;
1665         }
1666
1667         mlx5_pagealloc_cleanup(dev);
1668         mlx5_health_cleanup(dev);
1669         mlx5_fwdump_clean(dev);
1670         mlx5_pci_close(dev, priv);
1671         mtx_destroy(&dev->dump_lock);
1672         pci_set_drvdata(pdev, NULL);
1673         sysctl_ctx_free(&dev->sysctl_ctx);
1674         kfree(dev);
1675 }
1676
1677 static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev,
1678                                               pci_channel_state_t state)
1679 {
1680         struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
1681         struct mlx5_priv *priv = &dev->priv;
1682
1683         mlx5_core_info(dev, "%s was called\n", __func__);
1684         mlx5_enter_error_state(dev, false);
1685         mlx5_unload_one(dev, priv, false);
1686
1687         if (state) {
1688                 mlx5_drain_health_wq(dev);
1689                 mlx5_pci_disable_device(dev);
1690         }
1691
1692         return state == pci_channel_io_perm_failure ?
1693                 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
1694 }
1695
1696 static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev)
1697 {
1698         struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
1699         int err = 0;
1700
1701         mlx5_core_info(dev,"%s was called\n", __func__);
1702
1703         err = mlx5_pci_enable_device(dev);
1704         if (err) {
1705                 mlx5_core_err(dev, "mlx5_pci_enable_device failed with error code: %d\n"
1706                         ,err);
1707                 return PCI_ERS_RESULT_DISCONNECT;
1708         }
1709         pci_set_master(pdev);
1710         pci_set_powerstate(pdev->dev.bsddev, PCI_POWERSTATE_D0);
1711         pci_restore_state(pdev->dev.bsddev);
1712         pci_save_state(pdev->dev.bsddev);
1713
1714         return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
1715 }
1716
1717 /* wait for the device to show vital signs. For now we check
1718  * that we can read the device ID and that the health buffer
1719  * shows a non zero value which is different than 0xffffffff
1720  */
1721 static void wait_vital(struct pci_dev *pdev)
1722 {
1723         struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
1724         struct mlx5_core_health *health = &dev->priv.health;
1725         const int niter = 100;
1726         u32 count;
1727         u16 did;
1728         int i;
1729
1730         /* Wait for firmware to be ready after reset */
1731         msleep(1000);
1732         for (i = 0; i < niter; i++) {
1733                 if (pci_read_config_word(pdev, 2, &did)) {
1734                         mlx5_core_warn(dev, "failed reading config word\n");
1735                         break;
1736                 }
1737                 if (did == pdev->device) {
1738                         mlx5_core_info(dev,
1739                             "device ID correctly read after %d iterations\n", i);
1740                         break;
1741                 }
1742                 msleep(50);
1743         }
1744         if (i == niter)
1745                 mlx5_core_warn(dev, "could not read device ID\n");
1746
1747         for (i = 0; i < niter; i++) {
1748                 count = ioread32be(health->health_counter);
1749                 if (count && count != 0xffffffff) {
1750                         mlx5_core_info(dev,
1751                         "Counter value 0x%x after %d iterations\n", count, i);
1752                         break;
1753                 }
1754                 msleep(50);
1755         }
1756
1757         if (i == niter)
1758                 mlx5_core_warn(dev, "could not read device ID\n");
1759 }
1760
1761 static void mlx5_pci_resume(struct pci_dev *pdev)
1762 {
1763         struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
1764         struct mlx5_priv *priv = &dev->priv;
1765         int err;
1766
1767         mlx5_core_info(dev,"%s was called\n", __func__);
1768
1769         wait_vital(pdev);
1770
1771         err = mlx5_load_one(dev, priv, false);
1772         if (err)
1773                 mlx5_core_err(dev,
1774                     "mlx5_load_one failed with error code: %d\n" ,err);
1775         else
1776                 mlx5_core_info(dev,"device recovered\n");
1777 }
1778
1779 static const struct pci_error_handlers mlx5_err_handler = {
1780         .error_detected = mlx5_pci_err_detected,
1781         .slot_reset     = mlx5_pci_slot_reset,
1782         .resume         = mlx5_pci_resume
1783 };
1784
1785 #ifdef PCI_IOV
1786 static int
1787 mlx5_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *pf_config)
1788 {
1789         struct pci_dev *pdev;
1790         struct mlx5_core_dev *core_dev;
1791         struct mlx5_priv *priv;
1792         int err;
1793
1794         pdev = device_get_softc(dev);
1795         core_dev = pci_get_drvdata(pdev);
1796         priv = &core_dev->priv;
1797
1798         if (priv->eswitch == NULL)
1799                 return (ENXIO);
1800         if (priv->eswitch->total_vports < num_vfs + 1)
1801                 num_vfs = priv->eswitch->total_vports - 1;
1802         err = mlx5_eswitch_enable_sriov(priv->eswitch, num_vfs);
1803         return (-err);
1804 }
1805
1806 static void
1807 mlx5_iov_uninit(device_t dev)
1808 {
1809         struct pci_dev *pdev;
1810         struct mlx5_core_dev *core_dev;
1811         struct mlx5_priv *priv;
1812
1813         pdev = device_get_softc(dev);
1814         core_dev = pci_get_drvdata(pdev);
1815         priv = &core_dev->priv;
1816
1817         mlx5_eswitch_disable_sriov(priv->eswitch);
1818 }
1819
1820 static int
1821 mlx5_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *vf_config)
1822 {
1823         struct pci_dev *pdev;
1824         struct mlx5_core_dev *core_dev;
1825         struct mlx5_priv *priv;
1826         const void *mac;
1827         size_t mac_size;
1828         int error;
1829
1830         pdev = device_get_softc(dev);
1831         core_dev = pci_get_drvdata(pdev);
1832         priv = &core_dev->priv;
1833
1834         if (vfnum + 1 >= priv->eswitch->total_vports)
1835                 return (ENXIO);
1836
1837         if (nvlist_exists_binary(vf_config, iov_mac_addr_name)) {
1838                 mac = nvlist_get_binary(vf_config, iov_mac_addr_name,
1839                     &mac_size);
1840                 error = -mlx5_eswitch_set_vport_mac(priv->eswitch,
1841                     vfnum + 1, __DECONST(u8 *, mac));
1842         }
1843
1844         error = -mlx5_eswitch_set_vport_state(priv->eswitch, vfnum + 1,
1845             VPORT_STATE_FOLLOW);
1846         if (error != 0) {
1847                 mlx5_core_err(core_dev,
1848                     "upping vport for VF %d failed, error %d\n",
1849                     vfnum + 1, error);
1850         }
1851         error = -mlx5_core_enable_hca(core_dev, vfnum + 1);
1852         if (error != 0) {
1853                 mlx5_core_err(core_dev, "enabling VF %d failed, error %d\n",
1854                     vfnum + 1, error);
1855         }
1856         return (error);
1857 }
1858 #endif
1859
1860 static int mlx5_try_fast_unload(struct mlx5_core_dev *dev)
1861 {
1862         bool fast_teardown, force_teardown;
1863         int err;
1864
1865         if (!mlx5_fast_unload_enabled) {
1866                 mlx5_core_dbg(dev, "fast unload is disabled by user\n");
1867                 return -EOPNOTSUPP;
1868         }
1869
1870         fast_teardown = MLX5_CAP_GEN(dev, fast_teardown);
1871         force_teardown = MLX5_CAP_GEN(dev, force_teardown);
1872
1873         mlx5_core_dbg(dev, "force teardown firmware support=%d\n", force_teardown);
1874         mlx5_core_dbg(dev, "fast teardown firmware support=%d\n", fast_teardown);
1875
1876         if (!fast_teardown && !force_teardown)
1877                 return -EOPNOTSUPP;
1878
1879         if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
1880                 mlx5_core_dbg(dev, "Device in internal error state, giving up\n");
1881                 return -EAGAIN;
1882         }
1883
1884         /* Panic tear down fw command will stop the PCI bus communication
1885          * with the HCA, so the health polll is no longer needed.
1886          */
1887         mlx5_drain_health_wq(dev);
1888         mlx5_stop_health_poll(dev, false);
1889
1890         err = mlx5_cmd_fast_teardown_hca(dev);
1891         if (!err)
1892                 goto done;
1893
1894         err = mlx5_cmd_force_teardown_hca(dev);
1895         if (!err)
1896                 goto done;
1897
1898         mlx5_core_dbg(dev, "Firmware couldn't do fast unload error: %d\n", err);
1899         mlx5_start_health_poll(dev);
1900         return err;
1901 done:
1902         mlx5_enter_error_state(dev, true);
1903         return 0;
1904 }
1905
1906 static void mlx5_shutdown_disable_interrupts(struct mlx5_core_dev *mdev)
1907 {
1908         int nvec = mdev->priv.eq_table.num_comp_vectors + MLX5_EQ_VEC_COMP_BASE;
1909         int x;
1910
1911         mdev->priv.disable_irqs = 1;
1912
1913         /* wait for all IRQ handlers to finish processing */
1914         for (x = 0; x != nvec; x++)
1915                 synchronize_irq(mdev->priv.msix_arr[x].vector);
1916 }
1917
1918 static void shutdown_one(struct pci_dev *pdev)
1919 {
1920         struct mlx5_core_dev *dev  = pci_get_drvdata(pdev);
1921         struct mlx5_priv *priv = &dev->priv;
1922         int err;
1923
1924         /* enter polling mode */
1925         mlx5_cmd_use_polling(dev);
1926
1927         set_bit(MLX5_INTERFACE_STATE_TEARDOWN, &dev->intf_state);
1928
1929         /* disable all interrupts */
1930         mlx5_shutdown_disable_interrupts(dev);
1931
1932         err = mlx5_try_fast_unload(dev);
1933         if (err)
1934                 mlx5_unload_one(dev, priv, false);
1935         mlx5_pci_disable_device(dev);
1936 }
1937
1938 static const struct pci_device_id mlx5_core_pci_table[] = {
1939         { PCI_VDEVICE(MELLANOX, 4113) }, /* Connect-IB */
1940         { PCI_VDEVICE(MELLANOX, 4114) }, /* Connect-IB VF */
1941         { PCI_VDEVICE(MELLANOX, 4115) }, /* ConnectX-4 */
1942         { PCI_VDEVICE(MELLANOX, 4116) }, /* ConnectX-4 VF */
1943         { PCI_VDEVICE(MELLANOX, 4117) }, /* ConnectX-4LX */
1944         { PCI_VDEVICE(MELLANOX, 4118) }, /* ConnectX-4LX VF */
1945         { PCI_VDEVICE(MELLANOX, 4119) }, /* ConnectX-5 */
1946         { PCI_VDEVICE(MELLANOX, 4120) }, /* ConnectX-5 VF */
1947         { PCI_VDEVICE(MELLANOX, 4121) },
1948         { PCI_VDEVICE(MELLANOX, 4122) },
1949         { PCI_VDEVICE(MELLANOX, 4123) },
1950         { PCI_VDEVICE(MELLANOX, 4124) },
1951         { PCI_VDEVICE(MELLANOX, 4125) },
1952         { PCI_VDEVICE(MELLANOX, 4126) },
1953         { PCI_VDEVICE(MELLANOX, 4127) },
1954         { PCI_VDEVICE(MELLANOX, 4128) },
1955         { PCI_VDEVICE(MELLANOX, 4129) },
1956         { PCI_VDEVICE(MELLANOX, 4130) },
1957         { PCI_VDEVICE(MELLANOX, 4131) },
1958         { PCI_VDEVICE(MELLANOX, 4132) },
1959         { PCI_VDEVICE(MELLANOX, 4133) },
1960         { PCI_VDEVICE(MELLANOX, 4134) },
1961         { PCI_VDEVICE(MELLANOX, 4135) },
1962         { PCI_VDEVICE(MELLANOX, 4136) },
1963         { PCI_VDEVICE(MELLANOX, 4137) },
1964         { PCI_VDEVICE(MELLANOX, 4138) },
1965         { PCI_VDEVICE(MELLANOX, 4139) },
1966         { PCI_VDEVICE(MELLANOX, 4140) },
1967         { PCI_VDEVICE(MELLANOX, 4141) },
1968         { PCI_VDEVICE(MELLANOX, 4142) },
1969         { PCI_VDEVICE(MELLANOX, 4143) },
1970         { PCI_VDEVICE(MELLANOX, 4144) },
1971         { 0, }
1972 };
1973
1974 MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table);
1975
1976 void mlx5_disable_device(struct mlx5_core_dev *dev)
1977 {
1978         mlx5_pci_err_detected(dev->pdev, 0);
1979 }
1980
1981 void mlx5_recover_device(struct mlx5_core_dev *dev)
1982 {
1983         mlx5_pci_disable_device(dev);
1984         if (mlx5_pci_slot_reset(dev->pdev) == PCI_ERS_RESULT_RECOVERED)
1985                 mlx5_pci_resume(dev->pdev);
1986 }
1987
1988 struct pci_driver mlx5_core_driver = {
1989         .name           = DRIVER_NAME,
1990         .id_table       = mlx5_core_pci_table,
1991         .shutdown       = shutdown_one,
1992         .probe          = init_one,
1993         .remove         = remove_one,
1994         .err_handler    = &mlx5_err_handler,
1995 #ifdef PCI_IOV
1996         .bsd_iov_init   = mlx5_iov_init,
1997         .bsd_iov_uninit = mlx5_iov_uninit,
1998         .bsd_iov_add_vf = mlx5_iov_add_vf,
1999 #endif
2000 };
2001
2002 static int __init init(void)
2003 {
2004         int err;
2005
2006         err = pci_register_driver(&mlx5_core_driver);
2007         if (err)
2008                 goto err_debug;
2009
2010         err = mlx5_ctl_init();
2011         if (err)
2012                 goto err_ctl;
2013
2014         return 0;
2015
2016 err_ctl:
2017         pci_unregister_driver(&mlx5_core_driver);
2018
2019 err_debug:
2020         return err;
2021 }
2022
2023 static void __exit cleanup(void)
2024 {
2025         mlx5_ctl_fini();
2026         pci_unregister_driver(&mlx5_core_driver);
2027 }
2028
2029 module_init(init);
2030 module_exit(cleanup);