sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c

   1 /*-
   2  * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
  14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  16  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
  17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  23  * SUCH DAMAGE.
  24  *
  25  * $FreeBSD$
  26  */
  27
  28 #include <linux/errno.h>
  29 #include <linux/pci.h>
  30 #include <linux/dma-mapping.h>
  31 #include <linux/slab.h>
  32 #include <linux/io-mapping.h>
  33 #include <linux/sched.h>
  34 #include <linux/netdevice.h>
  35 #include <linux/etherdevice.h>
  36 #include <linux/list.h>
  37 #include <dev/mlx5/driver.h>
  38 #include <dev/mlx5/vport.h>
  39 #include <asm/pgtable.h>
  40 #include <linux/fs.h>
  41 #undef inode
  42
  43 #include <rdma/ib_user_verbs.h>
  44 #include <rdma/ib_smi.h>
  45 #include <rdma/ib_umem.h>
  46 #include "user.h"
  47 #include "mlx5_ib.h"
  48
  49 #include <sys/unistd.h>
  50 #include <sys/kthread.h>
  51
  52 #define DRIVER_NAME "mlx5_ib"
  53 #define DRIVER_VERSION "3.2-rc1"
  54 #define DRIVER_RELDATE  "May 2016"
  55
  56 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
  57 MODULE_DESCRIPTION("Mellanox Connect-IB HCA IB driver");
  58 MODULE_LICENSE("Dual BSD/GPL");
  59 MODULE_DEPEND(mlx5ib, linuxkpi, 1, 1, 1);
  60 MODULE_DEPEND(mlx5ib, mlx5, 1, 1, 1);
  61 MODULE_DEPEND(mlx5ib, ibcore, 1, 1, 1);
  62 MODULE_VERSION(mlx5ib, 1);
  63
  64 static int deprecated_prof_sel = 2;
  65 module_param_named(prof_sel, deprecated_prof_sel, int, 0444);
  66 MODULE_PARM_DESC(prof_sel, "profile selector. Deprecated here. Moved to module mlx5_core");
  67
  68 enum {
  69         MLX5_STANDARD_ATOMIC_SIZE = 0x8,
  70 };
  71
  72 struct workqueue_struct *mlx5_ib_wq;
  73
  74 static char mlx5_version[] =
  75         DRIVER_NAME ": Mellanox Connect-IB Infiniband driver v"
  76         DRIVER_VERSION " (" DRIVER_RELDATE ")\n";
  77
  78 static void get_atomic_caps(struct mlx5_ib_dev *dev,
  79                             struct ib_device_attr *props)
  80 {
  81         int tmp;
  82         u8 atomic_operations;
  83         u8 atomic_size_qp;
  84         u8 atomic_req_endianess;
  85
  86         atomic_operations = MLX5_CAP_ATOMIC(dev->mdev, atomic_operations);
  87         atomic_size_qp = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp);
  88         atomic_req_endianess = MLX5_CAP_ATOMIC(dev->mdev,
  89                                                atomic_req_8B_endianess_mode) ||
  90                                !mlx5_host_is_le();
  91
  92         tmp = MLX5_ATOMIC_OPS_CMP_SWAP | MLX5_ATOMIC_OPS_FETCH_ADD;
  93         if (((atomic_operations & tmp) == tmp)
  94             && (atomic_size_qp & 8)) {
  95                 if (atomic_req_endianess) {
  96                         props->atomic_cap = IB_ATOMIC_HCA;
  97                 } else {
  98                         props->atomic_cap = IB_ATOMIC_NONE;
  99                 }
 100         } else {
 101                 props->atomic_cap = IB_ATOMIC_NONE;
 102         }
 103
 104         tmp = MLX5_ATOMIC_OPS_MASKED_CMP_SWAP | MLX5_ATOMIC_OPS_MASKED_FETCH_ADD;
 105         if (((atomic_operations & tmp) == tmp)
 106             &&(atomic_size_qp & 8)) {
 107                 if (atomic_req_endianess)
 108                         props->masked_atomic_cap = IB_ATOMIC_HCA;
 109                 else {
 110                         props->masked_atomic_cap = IB_ATOMIC_NONE;
 111                 }
 112         } else {
 113                 props->masked_atomic_cap = IB_ATOMIC_NONE;
 114         }
 115 }
 116
 117 static enum rdma_link_layer
 118 mlx5_ib_port_link_layer(struct ib_device *device, u8 port_num)
 119 {
 120         struct mlx5_ib_dev *dev = to_mdev(device);
 121
 122         switch (MLX5_CAP_GEN(dev->mdev, port_type)) {
 123         case MLX5_CAP_PORT_TYPE_IB:
 124                 return IB_LINK_LAYER_INFINIBAND;
 125         case MLX5_CAP_PORT_TYPE_ETH:
 126                 return IB_LINK_LAYER_ETHERNET;
 127         default:
 128                 return IB_LINK_LAYER_UNSPECIFIED;
 129         }
 130 }
 131
 132 static int mlx5_use_mad_ifc(struct mlx5_ib_dev *dev)
 133 {
 134         return !dev->mdev->issi;
 135 }
 136
 137 enum {
 138         MLX5_VPORT_ACCESS_METHOD_MAD,
 139         MLX5_VPORT_ACCESS_METHOD_HCA,
 140         MLX5_VPORT_ACCESS_METHOD_NIC,
 141 };
 142
 143 static int mlx5_get_vport_access_method(struct ib_device *ibdev)
 144 {
 145         if (mlx5_use_mad_ifc(to_mdev(ibdev)))
 146                 return MLX5_VPORT_ACCESS_METHOD_MAD;
 147
 148         if (mlx5_ib_port_link_layer(ibdev, 1) ==
 149             IB_LINK_LAYER_ETHERNET)
 150                 return MLX5_VPORT_ACCESS_METHOD_NIC;
 151
 152         return MLX5_VPORT_ACCESS_METHOD_HCA;
 153 }
 154
 155 static int mlx5_query_system_image_guid(struct ib_device *ibdev,
 156                                         __be64 *sys_image_guid)
 157 {
 158         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 159         struct mlx5_core_dev *mdev = dev->mdev;
 160         u64 tmp;
 161         int err;
 162
 163         switch (mlx5_get_vport_access_method(ibdev)) {
 164         case MLX5_VPORT_ACCESS_METHOD_MAD:
 165                 return mlx5_query_system_image_guid_mad_ifc(ibdev,
 166                                                             sys_image_guid);
 167
 168         case MLX5_VPORT_ACCESS_METHOD_HCA:
 169                 err = mlx5_query_hca_vport_system_image_guid(mdev, &tmp);
 170                 if (!err)
 171                         *sys_image_guid = cpu_to_be64(tmp);
 172                 return err;
 173
 174         case MLX5_VPORT_ACCESS_METHOD_NIC:
 175                 err = mlx5_query_nic_vport_system_image_guid(mdev, &tmp);
 176                 if (!err)
 177                         *sys_image_guid = cpu_to_be64(tmp);
 178                 return err;
 179
 180         default:
 181                 return -EINVAL;
 182         }
 183 }
 184
 185 static int mlx5_query_max_pkeys(struct ib_device *ibdev,
 186                                 u16 *max_pkeys)
 187 {
 188         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 189         struct mlx5_core_dev *mdev = dev->mdev;
 190
 191         switch (mlx5_get_vport_access_method(ibdev)) {
 192         case MLX5_VPORT_ACCESS_METHOD_MAD:
 193                 return mlx5_query_max_pkeys_mad_ifc(ibdev, max_pkeys);
 194
 195         case MLX5_VPORT_ACCESS_METHOD_HCA:
 196         case MLX5_VPORT_ACCESS_METHOD_NIC:
 197                 *max_pkeys = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev,
 198                                                 pkey_table_size));
 199                 return 0;
 200
 201         default:
 202                 return -EINVAL;
 203         }
 204 }
 205
 206 static int mlx5_query_vendor_id(struct ib_device *ibdev,
 207                                 u32 *vendor_id)
 208 {
 209         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 210
 211         switch (mlx5_get_vport_access_method(ibdev)) {
 212         case MLX5_VPORT_ACCESS_METHOD_MAD:
 213                 return mlx5_query_vendor_id_mad_ifc(ibdev, vendor_id);
 214
 215         case MLX5_VPORT_ACCESS_METHOD_HCA:
 216         case MLX5_VPORT_ACCESS_METHOD_NIC:
 217                 return mlx5_core_query_vendor_id(dev->mdev, vendor_id);
 218
 219         default:
 220                 return -EINVAL;
 221         }
 222 }
 223
 224 static int mlx5_query_node_guid(struct mlx5_ib_dev *dev,
 225                                 __be64 *node_guid)
 226 {
 227         u64 tmp;
 228         int err;
 229
 230         switch (mlx5_get_vport_access_method(&dev->ib_dev)) {
 231         case MLX5_VPORT_ACCESS_METHOD_MAD:
 232                 return mlx5_query_node_guid_mad_ifc(dev, node_guid);
 233
 234         case MLX5_VPORT_ACCESS_METHOD_HCA:
 235                 err = mlx5_query_hca_vport_node_guid(dev->mdev, &tmp);
 236                 if (!err)
 237                         *node_guid = cpu_to_be64(tmp);
 238                 return err;
 239
 240         case MLX5_VPORT_ACCESS_METHOD_NIC:
 241                 err = mlx5_query_nic_vport_node_guid(dev->mdev, &tmp);
 242                 if (!err)
 243                         *node_guid = cpu_to_be64(tmp);
 244                 return err;
 245
 246         default:
 247                 return -EINVAL;
 248         }
 249 }
 250
 251 struct mlx5_reg_node_desc {
 252         u8      desc[64];
 253 };
 254
 255 static int mlx5_query_node_desc(struct mlx5_ib_dev *dev, char *node_desc)
 256 {
 257         struct mlx5_reg_node_desc in;
 258
 259         if (mlx5_use_mad_ifc(dev))
 260                 return mlx5_query_node_desc_mad_ifc(dev, node_desc);
 261
 262         memset(&in, 0, sizeof(in));
 263
 264         return mlx5_core_access_reg(dev->mdev, &in, sizeof(in), node_desc,
 265                                     sizeof(struct mlx5_reg_node_desc),
 266                                     MLX5_REG_NODE_DESC, 0, 0);
 267 }
 268
 269 static int mlx5_ib_query_device(struct ib_device *ibdev,
 270                                 struct ib_device_attr *props)
 271 {
 272         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 273         struct mlx5_core_dev *mdev = dev->mdev;
 274         int max_sq_desc;
 275         int max_rq_sg;
 276         int max_sq_sg;
 277         int err;
 278
 279
 280         memset(props, 0, sizeof(*props));
 281
 282         err = mlx5_query_system_image_guid(ibdev,
 283                                            &props->sys_image_guid);
 284         if (err)
 285                 return err;
 286
 287         err = mlx5_query_max_pkeys(ibdev, &props->max_pkeys);
 288         if (err)
 289                 return err;
 290
 291         err = mlx5_query_vendor_id(ibdev, &props->vendor_id);
 292         if (err)
 293                 return err;
 294
 295         props->fw_ver = ((u64)fw_rev_maj(dev->mdev) << 32) |
 296                 ((u64)fw_rev_min(dev->mdev) << 16) |
 297                 fw_rev_sub(dev->mdev);
 298         props->device_cap_flags    = IB_DEVICE_CHANGE_PHY_PORT |
 299                 IB_DEVICE_PORT_ACTIVE_EVENT             |
 300                 IB_DEVICE_SYS_IMAGE_GUID                |
 301                 IB_DEVICE_RC_RNR_NAK_GEN;
 302
 303         if (MLX5_CAP_GEN(mdev, pkv))
 304                 props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
 305         if (MLX5_CAP_GEN(mdev, qkv))
 306                 props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
 307         if (MLX5_CAP_GEN(mdev, apm))
 308                 props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
 309         props->device_cap_flags |= IB_DEVICE_LOCAL_DMA_LKEY;
 310         if (MLX5_CAP_GEN(mdev, xrc))
 311                 props->device_cap_flags |= IB_DEVICE_XRC;
 312         props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
 313         if (MLX5_CAP_GEN(mdev, block_lb_mc))
 314                 props->device_cap_flags |= IB_DEVICE_BLOCK_MULTICAST_LOOPBACK;
 315
 316         props->vendor_part_id      = mdev->pdev->device;
 317         props->hw_ver              = mdev->pdev->revision;
 318
 319         props->max_mr_size         = ~0ull;
 320         props->page_size_cap       = ~(u32)((1ull << MLX5_CAP_GEN(mdev, log_pg_sz)) -1);
 321         props->max_qp              = 1 << MLX5_CAP_GEN(mdev, log_max_qp);
 322         props->max_qp_wr           = 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
 323         max_rq_sg =  MLX5_CAP_GEN(mdev, max_wqe_sz_rq) /
 324                      sizeof(struct mlx5_wqe_data_seg);
 325         max_sq_desc = min((int)MLX5_CAP_GEN(mdev, max_wqe_sz_sq), 512);
 326         max_sq_sg = (max_sq_desc -
 327                      sizeof(struct mlx5_wqe_ctrl_seg) -
 328                      sizeof(struct mlx5_wqe_raddr_seg)) / sizeof(struct mlx5_wqe_data_seg);
 329         props->max_sge = min(max_rq_sg, max_sq_sg);
 330         props->max_cq              = 1 << MLX5_CAP_GEN(mdev, log_max_cq);
 331         props->max_cqe = (1 << MLX5_CAP_GEN(mdev, log_max_cq_sz)) - 1;
 332         props->max_mr              = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
 333         props->max_pd              = 1 << MLX5_CAP_GEN(mdev, log_max_pd);
 334         props->max_qp_rd_atom      = 1 << MLX5_CAP_GEN(mdev, log_max_ra_req_qp);
 335         props->max_qp_init_rd_atom = 1 << MLX5_CAP_GEN(mdev, log_max_ra_res_qp);
 336         props->max_srq             = 1 << MLX5_CAP_GEN(mdev, log_max_srq);
 337         props->max_srq_wr = (1 << MLX5_CAP_GEN(mdev, log_max_srq_sz)) - 1;
 338         props->local_ca_ack_delay  = MLX5_CAP_GEN(mdev, local_ca_ack_delay);
 339         props->max_res_rd_atom     = props->max_qp_rd_atom * props->max_qp;
 340         props->max_srq_sge         = max_rq_sg - 1;
 341         props->max_fast_reg_page_list_len = (unsigned int)-1;
 342         get_atomic_caps(dev, props);
 343         props->max_mcast_grp       = 1 << MLX5_CAP_GEN(mdev, log_max_mcg);
 344         props->max_mcast_qp_attach = MLX5_CAP_GEN(mdev, max_qp_mcg);
 345         props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
 346                                            props->max_mcast_grp;
 347         props->max_map_per_fmr = INT_MAX; /* no limit in ConnectIB */
 348         props->max_ah           = INT_MAX;
 349
 350         return 0;
 351 }
 352
 353 enum mlx5_ib_width {
 354         MLX5_IB_WIDTH_1X        = 1 << 0,
 355         MLX5_IB_WIDTH_2X        = 1 << 1,
 356         MLX5_IB_WIDTH_4X        = 1 << 2,
 357         MLX5_IB_WIDTH_8X        = 1 << 3,
 358         MLX5_IB_WIDTH_12X       = 1 << 4
 359 };
 360
 361 static int translate_active_width(struct ib_device *ibdev, u8 active_width,
 362                                   u8 *ib_width)
 363 {
 364         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 365         int err = 0;
 366
 367         if (active_width & MLX5_IB_WIDTH_1X) {
 368                 *ib_width = IB_WIDTH_1X;
 369         } else if (active_width & MLX5_IB_WIDTH_2X) {
 370                 mlx5_ib_warn(dev, "active_width %d is not supported by IB spec\n",
 371                              (int)active_width);
 372                 err = -EINVAL;
 373         } else if (active_width & MLX5_IB_WIDTH_4X) {
 374                 *ib_width = IB_WIDTH_4X;
 375         } else if (active_width & MLX5_IB_WIDTH_8X) {
 376                 *ib_width = IB_WIDTH_8X;
 377         } else if (active_width & MLX5_IB_WIDTH_12X) {
 378                 *ib_width = IB_WIDTH_12X;
 379         } else {
 380                 mlx5_ib_dbg(dev, "Invalid active_width %d\n",
 381                             (int)active_width);
 382                 err = -EINVAL;
 383         }
 384
 385         return err;
 386 }
 387
 388 /*
 389  * TODO: Move to IB core
 390  */
 391 enum ib_max_vl_num {
 392         __IB_MAX_VL_0           = 1,
 393         __IB_MAX_VL_0_1         = 2,
 394         __IB_MAX_VL_0_3         = 3,
 395         __IB_MAX_VL_0_7         = 4,
 396         __IB_MAX_VL_0_14        = 5,
 397 };
 398
 399 enum mlx5_vl_hw_cap {
 400         MLX5_VL_HW_0    = 1,
 401         MLX5_VL_HW_0_1  = 2,
 402         MLX5_VL_HW_0_2  = 3,
 403         MLX5_VL_HW_0_3  = 4,
 404         MLX5_VL_HW_0_4  = 5,
 405         MLX5_VL_HW_0_5  = 6,
 406         MLX5_VL_HW_0_6  = 7,
 407         MLX5_VL_HW_0_7  = 8,
 408         MLX5_VL_HW_0_14 = 15
 409 };
 410
 411 static int translate_max_vl_num(struct ib_device *ibdev, u8 vl_hw_cap,
 412                                 u8 *max_vl_num)
 413 {
 414         switch (vl_hw_cap) {
 415         case MLX5_VL_HW_0:
 416                 *max_vl_num = __IB_MAX_VL_0;
 417                 break;
 418         case MLX5_VL_HW_0_1:
 419                 *max_vl_num = __IB_MAX_VL_0_1;
 420                 break;
 421         case MLX5_VL_HW_0_3:
 422                 *max_vl_num = __IB_MAX_VL_0_3;
 423                 break;
 424         case MLX5_VL_HW_0_7:
 425                 *max_vl_num = __IB_MAX_VL_0_7;
 426                 break;
 427         case MLX5_VL_HW_0_14:
 428                 *max_vl_num = __IB_MAX_VL_0_14;
 429                 break;
 430
 431         default:
 432                 return -EINVAL;
 433         }
 434
 435         return 0;
 436 }
 437
 438 static int mlx5_query_port_ib(struct ib_device *ibdev, u8 port,
 439                               struct ib_port_attr *props)
 440 {
 441         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 442         struct mlx5_core_dev *mdev = dev->mdev;
 443         u32 *rep;
 444         int outlen = MLX5_ST_SZ_BYTES(query_hca_vport_context_out);
 445         struct mlx5_ptys_reg *ptys;
 446         struct mlx5_pmtu_reg *pmtu;
 447         struct mlx5_pvlc_reg pvlc;
 448         void *ctx;
 449         int err;
 450
 451         rep = mlx5_vzalloc(outlen);
 452         ptys = kzalloc(sizeof(*ptys), GFP_KERNEL);
 453         pmtu = kzalloc(sizeof(*pmtu), GFP_KERNEL);
 454         if (!rep || !ptys || !pmtu) {
 455                 err = -ENOMEM;
 456                 goto out;
 457         }
 458
 459         memset(props, 0, sizeof(*props));
 460
 461         /* what if I am pf with dual port */
 462         err = mlx5_query_hca_vport_context(mdev, port, 0, rep, outlen);
 463         if (err)
 464                 goto out;
 465
 466         ctx = MLX5_ADDR_OF(query_hca_vport_context_out, rep, hca_vport_context);
 467
 468         props->lid              = MLX5_GET(hca_vport_context, ctx, lid);
 469         props->lmc              = MLX5_GET(hca_vport_context, ctx, lmc);
 470         props->sm_lid           = MLX5_GET(hca_vport_context, ctx, sm_lid);
 471         props->sm_sl            = MLX5_GET(hca_vport_context, ctx, sm_sl);
 472         props->state            = MLX5_GET(hca_vport_context, ctx, vport_state);
 473         props->phys_state       = MLX5_GET(hca_vport_context, ctx,
 474                                         port_physical_state);
 475         props->port_cap_flags   = MLX5_GET(hca_vport_context, ctx, cap_mask1);
 476         props->gid_tbl_len      = mlx5_get_gid_table_len(MLX5_CAP_GEN(mdev, gid_table_size));
 477         props->max_msg_sz       = 1 << MLX5_CAP_GEN(mdev, log_max_msg);
 478         props->pkey_tbl_len     = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev, pkey_table_size));
 479         props->bad_pkey_cntr    = MLX5_GET(hca_vport_context, ctx,
 480                                               pkey_violation_counter);
 481         props->qkey_viol_cntr   = MLX5_GET(hca_vport_context, ctx,
 482                                               qkey_violation_counter);
 483         props->subnet_timeout   = MLX5_GET(hca_vport_context, ctx,
 484                                               subnet_timeout);
 485         props->init_type_reply  = MLX5_GET(hca_vport_context, ctx,
 486                                            init_type_reply);
 487
 488         ptys->proto_mask |= MLX5_PTYS_IB;
 489         ptys->local_port = port;
 490         err = mlx5_core_access_ptys(mdev, ptys, 0);
 491         if (err)
 492                 goto out;
 493
 494         err = translate_active_width(ibdev, ptys->ib_link_width_oper,
 495                                      &props->active_width);
 496         if (err)
 497                 goto out;
 498
 499         props->active_speed     = (u8)ptys->ib_proto_oper;
 500
 501         pmtu->local_port = port;
 502         err = mlx5_core_access_pmtu(mdev, pmtu, 0);
 503         if (err)
 504                 goto out;
 505
 506         props->max_mtu          = pmtu->max_mtu;
 507         props->active_mtu       = pmtu->oper_mtu;
 508
 509         memset(&pvlc, 0, sizeof(pvlc));
 510         pvlc.local_port = port;
 511         err = mlx5_core_access_pvlc(mdev, &pvlc, 0);
 512         if (err)
 513                 goto out;
 514
 515         err = translate_max_vl_num(ibdev, pvlc.vl_hw_cap,
 516                                    &props->max_vl_num);
 517 out:
 518         kvfree(rep);
 519         kfree(ptys);
 520         kfree(pmtu);
 521         return err;
 522 }
 523
 524 int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,
 525                        struct ib_port_attr *props)
 526 {
 527         switch (mlx5_get_vport_access_method(ibdev)) {
 528         case MLX5_VPORT_ACCESS_METHOD_MAD:
 529                 return mlx5_query_port_mad_ifc(ibdev, port, props);
 530
 531         case MLX5_VPORT_ACCESS_METHOD_HCA:
 532                 return mlx5_query_port_ib(ibdev, port, props);
 533
 534         case MLX5_VPORT_ACCESS_METHOD_NIC:
 535                 return mlx5_query_port_roce(ibdev, port, props);
 536
 537         default:
 538                 return -EINVAL;
 539         }
 540 }
 541
 542 static inline int
 543 mlx5_addrconf_ifid_eui48(u8 *eui, struct net_device *dev)
 544 {
 545         if (dev->if_addrlen != ETH_ALEN)
 546                 return -1;
 547         memcpy(eui, IF_LLADDR(dev), 3);
 548         memcpy(eui + 5, IF_LLADDR(dev) + 3, 3);
 549
 550         /* NOTE: The scope ID is added by the GID to IP conversion */
 551
 552         eui[3] = 0xFF;
 553         eui[4] = 0xFE;
 554         eui[0] ^= 2;
 555         return 0;
 556 }
 557
 558 static void
 559 mlx5_make_default_gid(struct net_device *dev, union ib_gid *gid)
 560 {
 561         gid->global.subnet_prefix = cpu_to_be64(0xfe80000000000000LL);
 562         mlx5_addrconf_ifid_eui48(&gid->raw[8], dev);
 563 }
 564
 565 static void
 566 mlx5_ib_roce_port_update(void *arg)
 567 {
 568         struct mlx5_ib_port *port = (struct mlx5_ib_port *)arg;
 569         struct mlx5_ib_dev *dev = port->dev;
 570         struct mlx5_core_dev *mdev = dev->mdev;
 571         struct net_device *xdev[MLX5_IB_GID_MAX];
 572         struct net_device *idev;
 573         struct net_device *ndev;
 574         struct ifaddr *ifa;
 575         union ib_gid gid_temp;
 576
 577         while (port->port_gone == 0) {
 578                 int update = 0;
 579                 int gid_index = 0;
 580                 int j;
 581                 int error;
 582
 583                 ndev = mlx5_get_protocol_dev(mdev, MLX5_INTERFACE_PROTOCOL_ETH);
 584                 if (ndev == NULL) {
 585                         pause("W", hz);
 586                         continue;
 587                 }
 588
 589                 CURVNET_SET_QUIET(ndev->if_vnet);
 590
 591                 memset(&gid_temp, 0, sizeof(gid_temp));
 592                 mlx5_make_default_gid(ndev, &gid_temp);
 593                 if (bcmp(&gid_temp, &port->gid_table[gid_index], sizeof(gid_temp))) {
 594                         port->gid_table[gid_index] = gid_temp;
 595                         update = 1;
 596                 }
 597                 xdev[gid_index] = ndev;
 598                 gid_index++;
 599
 600                 IFNET_RLOCK();
 601                 TAILQ_FOREACH(idev, &V_ifnet, if_link) {
 602                         if (idev == ndev)
 603                                 break;
 604                 }
 605                 if (idev != NULL) {
 606                     TAILQ_FOREACH(idev, &V_ifnet, if_link) {
 607                         if (idev != ndev) {
 608                                 if (idev->if_type != IFT_L2VLAN)
 609                                         continue;
 610                                 if (ndev != rdma_vlan_dev_real_dev(idev))
 611                                         continue;
 612                         }
 613                         /* clone address information for IPv4 and IPv6 */
 614                         IF_ADDR_RLOCK(idev);
 615                         TAILQ_FOREACH(ifa, &idev->if_addrhead, ifa_link) {
 616                                 if (ifa->ifa_addr == NULL ||
 617                                     (ifa->ifa_addr->sa_family != AF_INET &&
 618                                      ifa->ifa_addr->sa_family != AF_INET6) ||
 619                                     gid_index >= MLX5_IB_GID_MAX)
 620                                         continue;
 621                                 memset(&gid_temp, 0, sizeof(gid_temp));
 622                                 rdma_ip2gid(ifa->ifa_addr, &gid_temp);
 623                                 /* check for existing entry */
 624                                 for (j = 0; j != gid_index; j++) {
 625                                         if (bcmp(&gid_temp, &port->gid_table[j], sizeof(gid_temp)) == 0)
 626                                                 break;
 627                                 }
 628                                 /* check if new entry must be added */
 629                                 if (j == gid_index) {
 630                                         if (bcmp(&gid_temp, &port->gid_table[gid_index], sizeof(gid_temp))) {
 631                                                 port->gid_table[gid_index] = gid_temp;
 632                                                 update = 1;
 633                                         }
 634                                         xdev[gid_index] = idev;
 635                                         gid_index++;
 636                                 }
 637                         }
 638                         IF_ADDR_RUNLOCK(idev);
 639                     }
 640                 }
 641                 IFNET_RUNLOCK();
 642                 CURVNET_RESTORE();
 643
 644                 if (update != 0 &&
 645                     mlx5_ib_port_link_layer(&dev->ib_dev, 1) == IB_LINK_LAYER_ETHERNET) {
 646                         struct ib_event event = {
 647                             .device = &dev->ib_dev,
 648                             .element.port_num = port->port_num + 1,
 649                             .event = IB_EVENT_GID_CHANGE,
 650                         };
 651
 652                         /* add new entries, if any */
 653                         for (j = 0; j != gid_index; j++) {
 654                                 error = modify_gid_roce(&dev->ib_dev, port->port_num, j,
 655                                     port->gid_table + j, xdev[j]);
 656                                 if (error != 0)
 657                                         printf("mlx5_ib: Failed to update ROCE GID table: %d\n", error);
 658                         }
 659                         memset(&gid_temp, 0, sizeof(gid_temp));
 660
 661                         /* clear old entries, if any */
 662                         for (; j != MLX5_IB_GID_MAX; j++) {
 663                                 if (bcmp(&gid_temp, port->gid_table + j, sizeof(gid_temp)) == 0)
 664                                         continue;
 665                                 port->gid_table[j] = gid_temp;
 666                                 (void) modify_gid_roce(&dev->ib_dev, port->port_num, j,
 667                                     port->gid_table + j, ndev);
 668                         }
 669
 670                         /* make sure ibcore gets updated */
 671                         ib_dispatch_event(&event);
 672                 }
 673                 pause("W", hz);
 674         }
 675         do {
 676                 struct ib_event event = {
 677                         .device = &dev->ib_dev,
 678                         .element.port_num = port->port_num + 1,
 679                         .event = IB_EVENT_GID_CHANGE,
 680                 };
 681                 /* make sure ibcore gets updated */
 682                 ib_dispatch_event(&event);
 683
 684                 /* wait a bit */
 685                 pause("W", hz);
 686         } while (0);
 687         port->port_gone = 2;
 688         kthread_exit();
 689 }
 690
 691 static int mlx5_ib_query_gid(struct ib_device *ibdev, u8 port, int index,
 692                              union ib_gid *gid)
 693 {
 694         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 695         struct mlx5_core_dev *mdev = dev->mdev;
 696
 697         switch (mlx5_get_vport_access_method(ibdev)) {
 698         case MLX5_VPORT_ACCESS_METHOD_MAD:
 699                 return mlx5_query_gids_mad_ifc(ibdev, port, index, gid);
 700
 701         case MLX5_VPORT_ACCESS_METHOD_HCA:
 702                 return mlx5_query_hca_vport_gid(mdev, port, 0, index, gid);
 703
 704         case MLX5_VPORT_ACCESS_METHOD_NIC:
 705                 if (port == 0 || port > MLX5_CAP_GEN(mdev, num_ports) ||
 706                     index < 0 || index >= MLX5_IB_GID_MAX ||
 707                     dev->port[port - 1].port_gone != 0)
 708                         memset(gid, 0, sizeof(*gid));
 709                 else
 710                         *gid = dev->port[port - 1].gid_table[index];
 711                 return 0;
 712
 713         default:
 714                 return -EINVAL;
 715         }
 716 }
 717
 718 static int mlx5_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
 719                               u16 *pkey)
 720 {
 721         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 722         struct mlx5_core_dev *mdev = dev->mdev;
 723
 724         switch (mlx5_get_vport_access_method(ibdev)) {
 725         case MLX5_VPORT_ACCESS_METHOD_MAD:
 726                 return mlx5_query_pkey_mad_ifc(ibdev, port, index, pkey);
 727
 728         case MLX5_VPORT_ACCESS_METHOD_HCA:
 729         case MLX5_VPORT_ACCESS_METHOD_NIC:
 730                 return mlx5_query_hca_vport_pkey(mdev, 0, port, 0, index,
 731                                                  pkey);
 732
 733         default:
 734                 return -EINVAL;
 735         }
 736 }
 737
 738 static int mlx5_ib_modify_device(struct ib_device *ibdev, int mask,
 739                                  struct ib_device_modify *props)
 740 {
 741         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 742         struct mlx5_reg_node_desc in;
 743         struct mlx5_reg_node_desc out;
 744         int err;
 745
 746         if (mask & ~IB_DEVICE_MODIFY_NODE_DESC)
 747                 return -EOPNOTSUPP;
 748
 749         if (!(mask & IB_DEVICE_MODIFY_NODE_DESC))
 750                 return 0;
 751
 752         /*
 753          * If possible, pass node desc to FW, so it can generate
 754          * a 144 trap.  If cmd fails, just ignore.
 755          */
 756         memcpy(&in, props->node_desc, 64);
 757         err = mlx5_core_access_reg(dev->mdev, &in, sizeof(in), &out,
 758                                    sizeof(out), MLX5_REG_NODE_DESC, 0, 1);
 759         if (err)
 760                 return err;
 761
 762         memcpy(ibdev->node_desc, props->node_desc, 64);
 763
 764         return err;
 765 }
 766
 767 static int mlx5_ib_modify_port(struct ib_device *ibdev, u8 port, int mask,
 768                                struct ib_port_modify *props)
 769 {
 770         u8 is_eth = (mlx5_ib_port_link_layer(ibdev, port) ==
 771                      IB_LINK_LAYER_ETHERNET);
 772         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 773         struct ib_port_attr attr;
 774         u32 tmp;
 775         int err;
 776
 777         /* return OK if this is RoCE. CM calls ib_modify_port() regardless
 778          * of whether port link layer is ETH or IB. For ETH ports, qkey
 779          * violations and port capabilities are not valid.
 780          */
 781         if (is_eth)
 782                 return 0;
 783
 784         mutex_lock(&dev->cap_mask_mutex);
 785
 786         err = mlx5_ib_query_port(ibdev, port, &attr);
 787         if (err)
 788                 goto out;
 789
 790         tmp = (attr.port_cap_flags | props->set_port_cap_mask) &
 791                 ~props->clr_port_cap_mask;
 792
 793         err = mlx5_set_port_caps(dev->mdev, port, tmp);
 794
 795 out:
 796         mutex_unlock(&dev->cap_mask_mutex);
 797         return err;
 798 }
 799
 800 enum mlx5_cap_flags {
 801         MLX5_CAP_COMPACT_AV = 1 << 0,
 802 };
 803
 804 static void set_mlx5_flags(u32 *flags, struct mlx5_core_dev *dev)
 805 {
 806         *flags |= MLX5_CAP_GEN(dev, compact_address_vector) ?
 807                   MLX5_CAP_COMPACT_AV : 0;
 808 }
 809
 810 static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
 811                                                   struct ib_udata *udata)
 812 {
 813         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 814         struct mlx5_ib_alloc_ucontext_req_v2 req;
 815         struct mlx5_ib_alloc_ucontext_resp resp;
 816         struct mlx5_ib_ucontext *context;
 817         struct mlx5_uuar_info *uuari;
 818         struct mlx5_uar *uars;
 819         int gross_uuars;
 820         int num_uars;
 821         int ver;
 822         int uuarn;
 823         int err;
 824         int i;
 825         size_t reqlen;
 826
 827         if (!dev->ib_active)
 828                 return ERR_PTR(-EAGAIN);
 829
 830         memset(&req, 0, sizeof(req));
 831         memset(&resp, 0, sizeof(resp));
 832
 833         reqlen = udata->inlen - sizeof(struct ib_uverbs_cmd_hdr);
 834         if (reqlen == sizeof(struct mlx5_ib_alloc_ucontext_req))
 835                 ver = 0;
 836         else if (reqlen == sizeof(struct mlx5_ib_alloc_ucontext_req_v2))
 837                 ver = 2;
 838         else {
 839                 mlx5_ib_err(dev, "request malformed, reqlen: %ld\n", (long)reqlen);
 840                 return ERR_PTR(-EINVAL);
 841         }
 842
 843         err = ib_copy_from_udata(&req, udata, reqlen);
 844         if (err) {
 845                 mlx5_ib_err(dev, "copy failed\n");
 846                 return ERR_PTR(err);
 847         }
 848
 849         if (req.reserved) {
 850                 mlx5_ib_err(dev, "request corrupted\n");
 851                 return ERR_PTR(-EINVAL);
 852         }
 853
 854         if (req.total_num_uuars == 0 || req.total_num_uuars > MLX5_MAX_UUARS) {
 855                 mlx5_ib_warn(dev, "wrong num_uuars: %d\n", req.total_num_uuars);
 856                 return ERR_PTR(-ENOMEM);
 857         }
 858
 859         req.total_num_uuars = ALIGN(req.total_num_uuars,
 860                                     MLX5_NON_FP_BF_REGS_PER_PAGE);
 861         if (req.num_low_latency_uuars > req.total_num_uuars - 1) {
 862                 mlx5_ib_warn(dev, "wrong num_low_latency_uuars: %d ( > %d)\n",
 863                              req.total_num_uuars, req.total_num_uuars);
 864                 return ERR_PTR(-EINVAL);
 865         }
 866
 867         num_uars = req.total_num_uuars / MLX5_NON_FP_BF_REGS_PER_PAGE;
 868         gross_uuars = num_uars * MLX5_BF_REGS_PER_PAGE;
 869         resp.qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp);
 870         if (mlx5_core_is_pf(dev->mdev) && MLX5_CAP_GEN(dev->mdev, bf))
 871                 resp.bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size);
 872         resp.cache_line_size = L1_CACHE_BYTES;
 873         resp.max_sq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq);
 874         resp.max_rq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq);
 875         resp.max_send_wqebb = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
 876         resp.max_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
 877         resp.max_srq_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
 878         set_mlx5_flags(&resp.flags, dev->mdev);
 879
 880         if (offsetof(struct mlx5_ib_alloc_ucontext_resp, max_desc_sz_sq_dc) < udata->outlen)
 881                 resp.max_desc_sz_sq_dc = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq_dc);
 882
 883         if (offsetof(struct mlx5_ib_alloc_ucontext_resp, atomic_arg_sizes_dc) < udata->outlen)
 884                 resp.atomic_arg_sizes_dc = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_dc);
 885
 886         context = kzalloc(sizeof(*context), GFP_KERNEL);
 887         if (!context)
 888                 return ERR_PTR(-ENOMEM);
 889
 890         uuari = &context->uuari;
 891         mutex_init(&uuari->lock);
 892         uars = kcalloc(num_uars, sizeof(*uars), GFP_KERNEL);
 893         if (!uars) {
 894                 err = -ENOMEM;
 895                 goto out_ctx;
 896         }
 897
 898         uuari->bitmap = kcalloc(BITS_TO_LONGS(gross_uuars),
 899                                 sizeof(*uuari->bitmap),
 900                                 GFP_KERNEL);
 901         if (!uuari->bitmap) {
 902                 err = -ENOMEM;
 903                 goto out_uar_ctx;
 904         }
 905         /*
 906          * clear all fast path uuars
 907          */
 908         for (i = 0; i < gross_uuars; i++) {
 909                 uuarn = i & 3;
 910                 if (uuarn == 2 || uuarn == 3)
 911                         set_bit(i, uuari->bitmap);
 912         }
 913
 914         uuari->count = kcalloc(gross_uuars, sizeof(*uuari->count), GFP_KERNEL);
 915         if (!uuari->count) {
 916                 err = -ENOMEM;
 917                 goto out_bitmap;
 918         }
 919
 920         for (i = 0; i < num_uars; i++) {
 921                 err = mlx5_cmd_alloc_uar(dev->mdev, &uars[i].index);
 922                 if (err) {
 923                         mlx5_ib_err(dev, "uar alloc failed at %d\n", i);
 924                         goto out_uars;
 925                 }
 926         }
 927         for (i = 0; i < MLX5_IB_MAX_CTX_DYNAMIC_UARS; i++)
 928                 context->dynamic_wc_uar_index[i] = MLX5_IB_INVALID_UAR_INDEX;
 929
 930         INIT_LIST_HEAD(&context->db_page_list);
 931         mutex_init(&context->db_page_mutex);
 932
 933         resp.tot_uuars = req.total_num_uuars;
 934         resp.num_ports = MLX5_CAP_GEN(dev->mdev, num_ports);
 935         err = ib_copy_to_udata(udata, &resp,
 936                                min_t(size_t, udata->outlen, sizeof(resp)));
 937         if (err)
 938                 goto out_uars;
 939
 940         uuari->ver = ver;
 941         uuari->num_low_latency_uuars = req.num_low_latency_uuars;
 942         uuari->uars = uars;
 943         uuari->num_uars = num_uars;
 944
 945         if (mlx5_ib_port_link_layer(&dev->ib_dev, 1) ==
 946             IB_LINK_LAYER_ETHERNET) {
 947                 err = mlx5_alloc_transport_domain(dev->mdev, &context->tdn);
 948                 if (err)
 949                         goto out_uars;
 950         }
 951
 952         return &context->ibucontext;
 953
 954 out_uars:
 955         for (i--; i >= 0; i--)
 956                 mlx5_cmd_free_uar(dev->mdev, uars[i].index);
 957         kfree(uuari->count);
 958
 959 out_bitmap:
 960         kfree(uuari->bitmap);
 961
 962 out_uar_ctx:
 963         kfree(uars);
 964
 965 out_ctx:
 966         kfree(context);
 967         return ERR_PTR(err);
 968 }
 969
 970 static int mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
 971 {
 972         struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
 973         struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
 974         struct mlx5_uuar_info *uuari = &context->uuari;
 975         int i;
 976
 977         if (mlx5_ib_port_link_layer(&dev->ib_dev, 1) ==
 978             IB_LINK_LAYER_ETHERNET)
 979                 mlx5_dealloc_transport_domain(dev->mdev, context->tdn);
 980
 981         for (i = 0; i < uuari->num_uars; i++) {
 982                 if (mlx5_cmd_free_uar(dev->mdev, uuari->uars[i].index))
 983                         mlx5_ib_warn(dev, "failed to free UAR 0x%x\n", uuari->uars[i].index);
 984         }
 985         for (i = 0; i < MLX5_IB_MAX_CTX_DYNAMIC_UARS; i++) {
 986                 if (context->dynamic_wc_uar_index[i] != MLX5_IB_INVALID_UAR_INDEX)
 987                         mlx5_cmd_free_uar(dev->mdev, context->dynamic_wc_uar_index[i]);
 988         }
 989
 990         kfree(uuari->count);
 991         kfree(uuari->bitmap);
 992         kfree(uuari->uars);
 993         kfree(context);
 994
 995         return 0;
 996 }
 997
 998 static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev, int index)
 999 {
1000         return (pci_resource_start(dev->mdev->pdev, 0) >> PAGE_SHIFT) + index;
1001 }
1002
1003 static int get_command(unsigned long offset)
1004 {
1005         return (offset >> MLX5_IB_MMAP_CMD_SHIFT) & MLX5_IB_MMAP_CMD_MASK;
1006 }
1007
1008 static int get_arg(unsigned long offset)
1009 {
1010         return offset & ((1 << MLX5_IB_MMAP_CMD_SHIFT) - 1);
1011 }
1012
1013 static int get_index(unsigned long offset)
1014 {
1015         return get_arg(offset);
1016 }
1017
1018 static int uar_mmap(struct vm_area_struct *vma, pgprot_t prot, bool is_wc,
1019                     struct mlx5_uuar_info *uuari, struct mlx5_ib_dev *dev,
1020                     struct mlx5_ib_ucontext *context)
1021 {
1022         unsigned long idx;
1023         phys_addr_t pfn;
1024
1025         if (vma->vm_end - vma->vm_start != PAGE_SIZE) {
1026                 mlx5_ib_warn(dev, "wrong size, expected PAGE_SIZE(%ld) got %ld\n",
1027                              (long)PAGE_SIZE, (long)(vma->vm_end - vma->vm_start));
1028                 return -EINVAL;
1029         }
1030
1031         idx = get_index(vma->vm_pgoff);
1032         if (idx >= uuari->num_uars) {
1033                 mlx5_ib_warn(dev, "wrong offset, idx:%ld num_uars:%d\n",
1034                              idx, uuari->num_uars);
1035                 return -EINVAL;
1036         }
1037
1038         pfn = uar_index2pfn(dev, uuari->uars[idx].index);
1039         mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn 0x%llx\n", idx,
1040                     (unsigned long long)pfn);
1041
1042         vma->vm_page_prot = prot;
1043         if (io_remap_pfn_range(vma, vma->vm_start, pfn,
1044                                PAGE_SIZE, vma->vm_page_prot)) {
1045                 mlx5_ib_err(dev, "io remap failed\n");
1046                 return -EAGAIN;
1047         }
1048
1049         mlx5_ib_dbg(dev, "mapped %s at 0x%lx, PA 0x%llx\n", is_wc ? "WC" : "NC",
1050                     (long)vma->vm_start, (unsigned long long)pfn << PAGE_SHIFT);
1051
1052         return 0;
1053 }
1054
1055 static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
1056 {
1057         struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
1058         struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
1059         struct mlx5_uuar_info *uuari = &context->uuari;
1060         unsigned long command;
1061
1062         command = get_command(vma->vm_pgoff);
1063         switch (command) {
1064         case MLX5_IB_MMAP_REGULAR_PAGE:
1065                 return uar_mmap(vma, pgprot_writecombine(vma->vm_page_prot),
1066                                 true,
1067                                 uuari, dev, context);
1068
1069                 break;
1070
1071         case MLX5_IB_MMAP_WC_PAGE:
1072                 return uar_mmap(vma, pgprot_writecombine(vma->vm_page_prot),
1073                                 true, uuari, dev, context);
1074                 break;
1075
1076         case MLX5_IB_MMAP_NC_PAGE:
1077                 return uar_mmap(vma, pgprot_noncached(vma->vm_page_prot),
1078                                 false, uuari, dev, context);
1079                 break;
1080
1081         default:
1082                 return -EINVAL;
1083         }
1084
1085         return 0;
1086 }
1087
1088 static int alloc_pa_mkey(struct mlx5_ib_dev *dev, u32 *key, u32 pdn)
1089 {
1090         struct mlx5_create_mkey_mbox_in *in;
1091         struct mlx5_mkey_seg *seg;
1092         struct mlx5_core_mr mr;
1093         int err;
1094
1095         in = kzalloc(sizeof(*in), GFP_KERNEL);
1096         if (!in)
1097                 return -ENOMEM;
1098
1099         seg = &in->seg;
1100         seg->flags = MLX5_PERM_LOCAL_READ | MLX5_ACCESS_MODE_PA;
1101         seg->flags_pd = cpu_to_be32(pdn | MLX5_MKEY_LEN64);
1102         seg->qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
1103         seg->start_addr = 0;
1104
1105         err = mlx5_core_create_mkey(dev->mdev, &mr, in, sizeof(*in),
1106                                     NULL, NULL, NULL);
1107         if (err) {
1108                 mlx5_ib_warn(dev, "failed to create mkey, %d\n", err);
1109                 goto err_in;
1110         }
1111
1112         kfree(in);
1113         *key = mr.key;
1114
1115         return 0;
1116
1117 err_in:
1118         kfree(in);
1119
1120         return err;
1121 }
1122
1123 static void free_pa_mkey(struct mlx5_ib_dev *dev, u32 key)
1124 {
1125         struct mlx5_core_mr mr;
1126         int err;
1127
1128         memset(&mr, 0, sizeof(mr));
1129         mr.key = key;
1130         err = mlx5_core_destroy_mkey(dev->mdev, &mr);
1131         if (err)
1132                 mlx5_ib_warn(dev, "failed to destroy mkey 0x%x\n", key);
1133 }
1134
1135 static struct ib_pd *mlx5_ib_alloc_pd(struct ib_device *ibdev,
1136                                       struct ib_ucontext *context,
1137                                       struct ib_udata *udata)
1138 {
1139         struct mlx5_ib_dev *dev = to_mdev(ibdev);
1140         struct mlx5_ib_alloc_pd_resp resp;
1141         struct mlx5_ib_pd *pd;
1142         int err;
1143
1144         pd = kmalloc(sizeof(*pd), GFP_KERNEL);
1145         if (!pd)
1146                 return ERR_PTR(-ENOMEM);
1147
1148         err = mlx5_core_alloc_pd(to_mdev(ibdev)->mdev, &pd->pdn);
1149         if (err) {
1150                 mlx5_ib_warn(dev, "pd alloc failed\n");
1151                 kfree(pd);
1152                 return ERR_PTR(err);
1153         }
1154
1155         if (context) {
1156                 resp.pdn = pd->pdn;
1157                 if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
1158                         mlx5_ib_err(dev, "copy failed\n");
1159                         mlx5_core_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn);
1160                         kfree(pd);
1161                         return ERR_PTR(-EFAULT);
1162                 }
1163         } else {
1164                 err = alloc_pa_mkey(to_mdev(ibdev), &pd->pa_lkey, pd->pdn);
1165                 if (err) {
1166                         mlx5_ib_err(dev, "alloc mkey failed\n");
1167                         mlx5_core_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn);
1168                         kfree(pd);
1169                         return ERR_PTR(err);
1170                 }
1171         }
1172
1173         return &pd->ibpd;
1174 }
1175
1176 static int mlx5_ib_dealloc_pd(struct ib_pd *pd)
1177 {
1178         struct mlx5_ib_dev *mdev = to_mdev(pd->device);
1179         struct mlx5_ib_pd *mpd = to_mpd(pd);
1180
1181         if (!pd->uobject)
1182                 free_pa_mkey(mdev, mpd->pa_lkey);
1183
1184         mlx5_core_dealloc_pd(mdev->mdev, mpd->pdn);
1185         kfree(mpd);
1186
1187         return 0;
1188 }
1189
1190 static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
1191 {
1192         struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
1193         int err;
1194
1195         if (ibqp->qp_type == IB_QPT_RAW_PACKET)
1196                 err = -EOPNOTSUPP;
1197         else
1198                 err = mlx5_core_attach_mcg(dev->mdev, gid, ibqp->qp_num);
1199         if (err)
1200                 mlx5_ib_warn(dev, "failed attaching QPN 0x%x, MGID %pI6\n",
1201                              ibqp->qp_num, gid->raw);
1202
1203         return err;
1204 }
1205
1206 static int mlx5_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
1207 {
1208         struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
1209         int err;
1210
1211         if (ibqp->qp_type == IB_QPT_RAW_PACKET)
1212                 err = -EOPNOTSUPP;
1213         else
1214                 err = mlx5_core_detach_mcg(dev->mdev, gid, ibqp->qp_num);
1215         if (err)
1216                 mlx5_ib_warn(dev, "failed detaching QPN 0x%x, MGID %pI6\n",
1217                              ibqp->qp_num, gid->raw);
1218
1219         return err;
1220 }
1221
1222 static int init_node_data(struct mlx5_ib_dev *dev)
1223 {
1224         int err;
1225
1226         err = mlx5_query_node_desc(dev, dev->ib_dev.node_desc);
1227         if (err)
1228                 return err;
1229
1230         return mlx5_query_node_guid(dev, &dev->ib_dev.node_guid);
1231 }
1232
1233 static ssize_t show_fw_pages(struct device *device, struct device_attribute *attr,
1234                              char *buf)
1235 {
1236         struct mlx5_ib_dev *dev =
1237                 container_of(device, struct mlx5_ib_dev, ib_dev.dev);
1238
1239         return sprintf(buf, "%lld\n", (long long)dev->mdev->priv.fw_pages);
1240 }
1241
1242 static ssize_t show_reg_pages(struct device *device,
1243                               struct device_attribute *attr, char *buf)
1244 {
1245         struct mlx5_ib_dev *dev =
1246                 container_of(device, struct mlx5_ib_dev, ib_dev.dev);
1247
1248         return sprintf(buf, "%d\n", atomic_read(&dev->mdev->priv.reg_pages));
1249 }
1250
1251 static ssize_t show_hca(struct device *device, struct device_attribute *attr,
1252                         char *buf)
1253 {
1254         struct mlx5_ib_dev *dev =
1255                 container_of(device, struct mlx5_ib_dev, ib_dev.dev);
1256         return sprintf(buf, "MT%d\n", dev->mdev->pdev->device);
1257 }
1258
1259 static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr,
1260                            char *buf)
1261 {
1262         struct mlx5_ib_dev *dev =
1263                 container_of(device, struct mlx5_ib_dev, ib_dev.dev);
1264         return sprintf(buf, "%d.%d.%04d\n", fw_rev_maj(dev->mdev),
1265                        fw_rev_min(dev->mdev), fw_rev_sub(dev->mdev));
1266 }
1267
1268 static ssize_t show_rev(struct device *device, struct device_attribute *attr,
1269                         char *buf)
1270 {
1271         struct mlx5_ib_dev *dev =
1272                 container_of(device, struct mlx5_ib_dev, ib_dev.dev);
1273         return sprintf(buf, "%x\n", (unsigned)dev->mdev->pdev->revision);
1274 }
1275
1276 static ssize_t show_board(struct device *device, struct device_attribute *attr,
1277                           char *buf)
1278 {
1279         struct mlx5_ib_dev *dev =
1280                 container_of(device, struct mlx5_ib_dev, ib_dev.dev);
1281         return sprintf(buf, "%.*s\n", MLX5_BOARD_ID_LEN,
1282                        dev->mdev->board_id);
1283 }
1284
1285 static DEVICE_ATTR(hw_rev,   S_IRUGO, show_rev,    NULL);
1286 static DEVICE_ATTR(fw_ver,   S_IRUGO, show_fw_ver, NULL);
1287 static DEVICE_ATTR(hca_type, S_IRUGO, show_hca,    NULL);
1288 static DEVICE_ATTR(board_id, S_IRUGO, show_board,  NULL);
1289 static DEVICE_ATTR(fw_pages, S_IRUGO, show_fw_pages, NULL);
1290 static DEVICE_ATTR(reg_pages, S_IRUGO, show_reg_pages, NULL);
1291
1292 static struct device_attribute *mlx5_class_attributes[] = {
1293         &dev_attr_hw_rev,
1294         &dev_attr_fw_ver,
1295         &dev_attr_hca_type,
1296         &dev_attr_board_id,
1297         &dev_attr_fw_pages,
1298         &dev_attr_reg_pages,
1299 };
1300
1301 static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)
1302 {
1303         struct mlx5_ib_qp *mqp;
1304         struct mlx5_ib_cq *send_mcq, *recv_mcq;
1305         struct mlx5_core_cq *mcq;
1306         struct list_head cq_armed_list;
1307         unsigned long flags_qp;
1308         unsigned long flags_cq;
1309         unsigned long flags;
1310
1311         mlx5_ib_warn(ibdev, " started\n");
1312         INIT_LIST_HEAD(&cq_armed_list);
1313
1314         /* Go over qp list reside on that ibdev, sync with create/destroy qp.*/
1315         spin_lock_irqsave(&ibdev->reset_flow_resource_lock, flags);
1316         list_for_each_entry(mqp, &ibdev->qp_list, qps_list) {
1317                 spin_lock_irqsave(&mqp->sq.lock, flags_qp);
1318                 if (mqp->sq.tail != mqp->sq.head) {
1319                         send_mcq = to_mcq(mqp->ibqp.send_cq);
1320                         spin_lock_irqsave(&send_mcq->lock, flags_cq);
1321                         if (send_mcq->mcq.comp &&
1322                             mqp->ibqp.send_cq->comp_handler) {
1323                                 if (!send_mcq->mcq.reset_notify_added) {
1324                                         send_mcq->mcq.reset_notify_added = 1;
1325                                         list_add_tail(&send_mcq->mcq.reset_notify,
1326                                                       &cq_armed_list);
1327                                 }
1328                         }
1329                         spin_unlock_irqrestore(&send_mcq->lock, flags_cq);
1330                 }
1331                 spin_unlock_irqrestore(&mqp->sq.lock, flags_qp);
1332                 spin_lock_irqsave(&mqp->rq.lock, flags_qp);
1333                 /* no handling is needed for SRQ */
1334                 if (!mqp->ibqp.srq) {
1335                         if (mqp->rq.tail != mqp->rq.head) {
1336                                 recv_mcq = to_mcq(mqp->ibqp.recv_cq);
1337                                 spin_lock_irqsave(&recv_mcq->lock, flags_cq);
1338                                 if (recv_mcq->mcq.comp &&
1339                                     mqp->ibqp.recv_cq->comp_handler) {
1340                                         if (!recv_mcq->mcq.reset_notify_added) {
1341                                                 recv_mcq->mcq.reset_notify_added = 1;
1342                                                 list_add_tail(&recv_mcq->mcq.reset_notify,
1343                                                               &cq_armed_list);
1344                                         }
1345                                 }
1346                                 spin_unlock_irqrestore(&recv_mcq->lock,
1347                                                        flags_cq);
1348                         }
1349                 }
1350                 spin_unlock_irqrestore(&mqp->rq.lock, flags_qp);
1351         }
1352         /*At that point all inflight post send were put to be executed as of we
1353          * lock/unlock above locks Now need to arm all involved CQs.
1354          */
1355         list_for_each_entry(mcq, &cq_armed_list, reset_notify) {
1356                 mcq->comp(mcq);
1357         }
1358         spin_unlock_irqrestore(&ibdev->reset_flow_resource_lock, flags);
1359         mlx5_ib_warn(ibdev, " ended\n");
1360         return;
1361 }
1362
1363 static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context,
1364                           enum mlx5_dev_event event, unsigned long param)
1365 {
1366         struct mlx5_ib_dev *ibdev = (struct mlx5_ib_dev *)context;
1367         struct ib_event ibev;
1368
1369         u8 port = 0;
1370
1371         switch (event) {
1372         case MLX5_DEV_EVENT_SYS_ERROR:
1373                 ibdev->ib_active = false;
1374                 ibev.event = IB_EVENT_DEVICE_FATAL;
1375                 mlx5_ib_handle_internal_error(ibdev);
1376                 break;
1377
1378         case MLX5_DEV_EVENT_PORT_UP:
1379                 ibev.event = IB_EVENT_PORT_ACTIVE;
1380                 port = (u8)param;
1381                 break;
1382
1383         case MLX5_DEV_EVENT_PORT_DOWN:
1384         case MLX5_DEV_EVENT_PORT_INITIALIZED:
1385                 ibev.event = IB_EVENT_PORT_ERR;
1386                 port = (u8)param;
1387                 break;
1388
1389         case MLX5_DEV_EVENT_LID_CHANGE:
1390                 ibev.event = IB_EVENT_LID_CHANGE;
1391                 port = (u8)param;
1392                 break;
1393
1394         case MLX5_DEV_EVENT_PKEY_CHANGE:
1395                 ibev.event = IB_EVENT_PKEY_CHANGE;
1396                 port = (u8)param;
1397                 break;
1398
1399         case MLX5_DEV_EVENT_GUID_CHANGE:
1400                 ibev.event = IB_EVENT_GID_CHANGE;
1401                 port = (u8)param;
1402                 break;
1403
1404         case MLX5_DEV_EVENT_CLIENT_REREG:
1405                 ibev.event = IB_EVENT_CLIENT_REREGISTER;
1406                 port = (u8)param;
1407                 break;
1408
1409         default:
1410                 break;
1411         }
1412
1413         ibev.device           = &ibdev->ib_dev;
1414         ibev.element.port_num = port;
1415
1416         if ((event != MLX5_DEV_EVENT_SYS_ERROR) &&
1417             (port < 1 || port > ibdev->num_ports)) {
1418                 mlx5_ib_warn(ibdev, "warning: event on port %d\n", port);
1419                 return;
1420         }
1421
1422         if (ibdev->ib_active)
1423                 ib_dispatch_event(&ibev);
1424 }
1425
1426 static void get_ext_port_caps(struct mlx5_ib_dev *dev)
1427 {
1428         int port;
1429
1430         for (port = 1; port <= MLX5_CAP_GEN(dev->mdev, num_ports); port++)
1431                 mlx5_query_ext_port_caps(dev, port);
1432 }
1433
1434 static void config_atomic_responder(struct mlx5_ib_dev *dev,
1435                                     struct ib_device_attr *props)
1436 {
1437         enum ib_atomic_cap cap = props->atomic_cap;
1438
1439 #if 0
1440         if (cap == IB_ATOMIC_HCA ||
1441             cap == IB_ATOMIC_GLOB)
1442 #endif
1443                 dev->enable_atomic_resp = 1;
1444
1445         dev->atomic_cap = cap;
1446 }
1447
1448 enum mlx5_addr_align {
1449         MLX5_ADDR_ALIGN_0       = 0,
1450         MLX5_ADDR_ALIGN_64      = 64,
1451         MLX5_ADDR_ALIGN_128     = 128,
1452 };
1453
1454 static int get_port_caps(struct mlx5_ib_dev *dev)
1455 {
1456         struct ib_device_attr *dprops = NULL;
1457         struct ib_port_attr *pprops = NULL;
1458         int err = -ENOMEM;
1459         int port;
1460
1461         pprops = kmalloc(sizeof(*pprops), GFP_KERNEL);
1462         if (!pprops)
1463                 goto out;
1464
1465         dprops = kmalloc(sizeof(*dprops), GFP_KERNEL);
1466         if (!dprops)
1467                 goto out;
1468
1469         err = mlx5_ib_query_device(&dev->ib_dev, dprops);
1470         if (err) {
1471                 mlx5_ib_warn(dev, "query_device failed %d\n", err);
1472                 goto out;
1473         }
1474         config_atomic_responder(dev, dprops);
1475
1476         for (port = 1; port <= MLX5_CAP_GEN(dev->mdev, num_ports); port++) {
1477                 err = mlx5_ib_query_port(&dev->ib_dev, port, pprops);
1478                 if (err) {
1479                         mlx5_ib_warn(dev, "query_port %d failed %d\n",
1480                                      port, err);
1481                         break;
1482                 }
1483                 dev->mdev->port_caps[port - 1].pkey_table_len = dprops->max_pkeys;
1484                 dev->mdev->port_caps[port - 1].gid_table_len = pprops->gid_tbl_len;
1485                 mlx5_ib_dbg(dev, "pkey_table_len %d, gid_table_len %d\n",
1486                             dprops->max_pkeys, pprops->gid_tbl_len);
1487         }
1488
1489 out:
1490         kfree(pprops);
1491         kfree(dprops);
1492
1493         return err;
1494 }
1495
1496 static void destroy_umrc_res(struct mlx5_ib_dev *dev)
1497 {
1498         int err;
1499
1500         err = mlx5_mr_cache_cleanup(dev);
1501         if (err)
1502                 mlx5_ib_warn(dev, "mr cache cleanup failed\n");
1503
1504         ib_dereg_mr(dev->umrc.mr);
1505         ib_dealloc_pd(dev->umrc.pd);
1506 }
1507
1508 enum {
1509         MAX_UMR_WR = 128,
1510 };
1511
1512 static int create_umr_res(struct mlx5_ib_dev *dev)
1513 {
1514         struct ib_pd *pd;
1515         struct ib_mr *mr;
1516         int ret;
1517
1518         pd = ib_alloc_pd(&dev->ib_dev);
1519         if (IS_ERR(pd)) {
1520                 mlx5_ib_dbg(dev, "Couldn't create PD for sync UMR QP\n");
1521                 ret = PTR_ERR(pd);
1522                 goto error_0;
1523         }
1524
1525         mr = ib_get_dma_mr(pd,  IB_ACCESS_LOCAL_WRITE);
1526         if (IS_ERR(mr)) {
1527                 mlx5_ib_dbg(dev, "Couldn't create DMA MR for sync UMR QP\n");
1528                 ret = PTR_ERR(mr);
1529                 goto error_1;
1530         }
1531
1532         dev->umrc.mr = mr;
1533         dev->umrc.pd = pd;
1534
1535         ret = mlx5_mr_cache_init(dev);
1536         if (ret) {
1537                 mlx5_ib_warn(dev, "mr cache init failed %d\n", ret);
1538                 goto error_4;
1539         }
1540
1541         return 0;
1542
1543 error_4:
1544         ib_dereg_mr(mr);
1545 error_1:
1546         ib_dealloc_pd(pd);
1547 error_0:
1548         return ret;
1549 }
1550
1551 static int create_dev_resources(struct mlx5_ib_resources *devr)
1552 {
1553         struct ib_srq_init_attr attr;
1554         struct mlx5_ib_dev *dev;
1555         int ret = 0;
1556         struct ib_cq_init_attr cq_attr = { .cqe = 1 };
1557
1558         dev = container_of(devr, struct mlx5_ib_dev, devr);
1559
1560         devr->p0 = mlx5_ib_alloc_pd(&dev->ib_dev, NULL, NULL);
1561         if (IS_ERR(devr->p0)) {
1562                 ret = PTR_ERR(devr->p0);
1563                 goto error0;
1564         }
1565         devr->p0->device  = &dev->ib_dev;
1566         devr->p0->uobject = NULL;
1567         atomic_set(&devr->p0->usecnt, 0);
1568
1569         devr->c0 = mlx5_ib_create_cq(&dev->ib_dev, &cq_attr, NULL, NULL);
1570         if (IS_ERR(devr->c0)) {
1571                 ret = PTR_ERR(devr->c0);
1572                 goto error1;
1573         }
1574         devr->c0->device        = &dev->ib_dev;
1575         devr->c0->uobject       = NULL;
1576         devr->c0->comp_handler  = NULL;
1577         devr->c0->event_handler = NULL;
1578         devr->c0->cq_context    = NULL;
1579         atomic_set(&devr->c0->usecnt, 0);
1580
1581         devr->x0 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL, NULL);
1582         if (IS_ERR(devr->x0)) {
1583                 ret = PTR_ERR(devr->x0);
1584                 goto error2;
1585         }
1586         devr->x0->device = &dev->ib_dev;
1587         devr->x0->inode = NULL;
1588         atomic_set(&devr->x0->usecnt, 0);
1589         mutex_init(&devr->x0->tgt_qp_mutex);
1590         INIT_LIST_HEAD(&devr->x0->tgt_qp_list);
1591
1592         devr->x1 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL, NULL);
1593         if (IS_ERR(devr->x1)) {
1594                 ret = PTR_ERR(devr->x1);
1595                 goto error3;
1596         }
1597         devr->x1->device = &dev->ib_dev;
1598         devr->x1->inode = NULL;
1599         atomic_set(&devr->x1->usecnt, 0);
1600         mutex_init(&devr->x1->tgt_qp_mutex);
1601         INIT_LIST_HEAD(&devr->x1->tgt_qp_list);
1602
1603         memset(&attr, 0, sizeof(attr));
1604         attr.attr.max_sge = 1;
1605         attr.attr.max_wr = 1;
1606         attr.srq_type = IB_SRQT_XRC;
1607         attr.ext.xrc.cq = devr->c0;
1608         attr.ext.xrc.xrcd = devr->x0;
1609
1610         devr->s0 = mlx5_ib_create_srq(devr->p0, &attr, NULL);
1611         if (IS_ERR(devr->s0)) {
1612                 ret = PTR_ERR(devr->s0);
1613                 goto error4;
1614         }
1615         devr->s0->device        = &dev->ib_dev;
1616         devr->s0->pd            = devr->p0;
1617         devr->s0->uobject       = NULL;
1618         devr->s0->event_handler = NULL;
1619         devr->s0->srq_context   = NULL;
1620         devr->s0->srq_type      = IB_SRQT_XRC;
1621         devr->s0->ext.xrc.xrcd  = devr->x0;
1622         devr->s0->ext.xrc.cq    = devr->c0;
1623         atomic_inc(&devr->s0->ext.xrc.xrcd->usecnt);
1624         atomic_inc(&devr->s0->ext.xrc.cq->usecnt);
1625         atomic_inc(&devr->p0->usecnt);
1626         atomic_set(&devr->s0->usecnt, 0);
1627
1628         memset(&attr, 0, sizeof(attr));
1629         attr.attr.max_sge = 1;
1630         attr.attr.max_wr = 1;
1631         attr.srq_type = IB_SRQT_BASIC;
1632         devr->s1 = mlx5_ib_create_srq(devr->p0, &attr, NULL);
1633         if (IS_ERR(devr->s1)) {
1634                 ret = PTR_ERR(devr->s1);
1635                 goto error5;
1636         }
1637         devr->s1->device        = &dev->ib_dev;
1638         devr->s1->pd            = devr->p0;
1639         devr->s1->uobject       = NULL;
1640         devr->s1->event_handler = NULL;
1641         devr->s1->srq_context   = NULL;
1642         devr->s1->srq_type      = IB_SRQT_BASIC;
1643         devr->s1->ext.xrc.cq    = devr->c0;
1644         atomic_inc(&devr->p0->usecnt);
1645         atomic_set(&devr->s1->usecnt, 0);
1646
1647         return 0;
1648
1649 error5:
1650         mlx5_ib_destroy_srq(devr->s0);
1651 error4:
1652         mlx5_ib_dealloc_xrcd(devr->x1);
1653 error3:
1654         mlx5_ib_dealloc_xrcd(devr->x0);
1655 error2:
1656         mlx5_ib_destroy_cq(devr->c0);
1657 error1:
1658         mlx5_ib_dealloc_pd(devr->p0);
1659 error0:
1660         return ret;
1661 }
1662
1663 static void destroy_dev_resources(struct mlx5_ib_resources *devr)
1664 {
1665         mlx5_ib_destroy_srq(devr->s1);
1666         mlx5_ib_destroy_srq(devr->s0);
1667         mlx5_ib_dealloc_xrcd(devr->x0);
1668         mlx5_ib_dealloc_xrcd(devr->x1);
1669         mlx5_ib_destroy_cq(devr->c0);
1670         mlx5_ib_dealloc_pd(devr->p0);
1671 }
1672
1673 static u32 get_core_cap_flags(struct ib_device *ibdev)
1674 {
1675         struct mlx5_ib_dev *dev = to_mdev(ibdev);
1676         enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, 1);
1677         u8 l3_type_cap = MLX5_CAP_ROCE(dev->mdev, l3_type);
1678         u8 roce_version_cap = MLX5_CAP_ROCE(dev->mdev, roce_version);
1679         u32 ret = 0;
1680
1681         if (ll == IB_LINK_LAYER_INFINIBAND)
1682                 return RDMA_CORE_PORT_IBA_IB;
1683
1684         ret = RDMA_CORE_PORT_RAW_PACKET;
1685
1686         if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV4_CAP))
1687                 return ret;
1688
1689         if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV6_CAP))
1690                 return ret;
1691
1692         if (roce_version_cap & MLX5_ROCE_VERSION_1_CAP)
1693                 ret |= RDMA_CORE_PORT_IBA_ROCE;
1694
1695         if (roce_version_cap & MLX5_ROCE_VERSION_2_CAP)
1696                 ret |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
1697
1698         return ret;
1699 }
1700
1701 static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num,
1702                                struct ib_port_immutable *immutable)
1703 {
1704         struct ib_port_attr attr;
1705         struct mlx5_ib_dev *dev = to_mdev(ibdev);
1706         enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, port_num);
1707         int err;
1708
1709         immutable->core_cap_flags = get_core_cap_flags(ibdev);
1710
1711         err = ib_query_port(ibdev, port_num, &attr);
1712         if (err)
1713                 return err;
1714
1715         immutable->pkey_tbl_len = attr.pkey_tbl_len;
1716         immutable->gid_tbl_len = attr.gid_tbl_len;
1717         immutable->core_cap_flags = get_core_cap_flags(ibdev);
1718         if ((ll == IB_LINK_LAYER_INFINIBAND) || MLX5_CAP_GEN(dev->mdev, roce))
1719                 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
1720
1721         return 0;
1722 }
1723
1724 static void enable_dc_tracer(struct mlx5_ib_dev *dev)
1725 {
1726         struct device *device = dev->ib_dev.dma_device;
1727         struct mlx5_dc_tracer *dct = &dev->dctr;
1728         int order;
1729         void *tmp;
1730         int size;
1731         int err;
1732
1733         size = MLX5_CAP_GEN(dev->mdev, num_ports) * 4096;
1734         if (size <= PAGE_SIZE)
1735                 order = 0;
1736         else
1737                 order = 1;
1738
1739         dct->pg = alloc_pages(GFP_KERNEL, order);
1740         if (!dct->pg) {
1741                 mlx5_ib_err(dev, "failed to allocate %d pages\n", order);
1742                 return;
1743         }
1744
1745         tmp = page_address(dct->pg);
1746         memset(tmp, 0xff, size);
1747
1748         dct->size = size;
1749         dct->order = order;
1750         dct->dma = dma_map_page(device, dct->pg, 0, size, DMA_FROM_DEVICE);
1751         if (dma_mapping_error(device, dct->dma)) {
1752                 mlx5_ib_err(dev, "dma mapping error\n");
1753                 goto map_err;
1754         }
1755
1756         err = mlx5_core_set_dc_cnak_trace(dev->mdev, 1, dct->dma);
1757         if (err) {
1758                 mlx5_ib_warn(dev, "failed to enable DC tracer\n");
1759                 goto cmd_err;
1760         }
1761
1762         return;
1763
1764 cmd_err:
1765         dma_unmap_page(device, dct->dma, size, DMA_FROM_DEVICE);
1766 map_err:
1767         __free_pages(dct->pg, dct->order);
1768         dct->pg = NULL;
1769 }
1770
1771 static void disable_dc_tracer(struct mlx5_ib_dev *dev)
1772 {
1773         struct device *device = dev->ib_dev.dma_device;
1774         struct mlx5_dc_tracer *dct = &dev->dctr;
1775         int err;
1776
1777         if (!dct->pg)
1778                 return;
1779
1780         err = mlx5_core_set_dc_cnak_trace(dev->mdev, 0, dct->dma);
1781         if (err) {
1782                 mlx5_ib_warn(dev, "failed to disable DC tracer\n");
1783                 return;
1784         }
1785
1786         dma_unmap_page(device, dct->dma, dct->size, DMA_FROM_DEVICE);
1787         __free_pages(dct->pg, dct->order);
1788         dct->pg = NULL;
1789 }
1790
1791 enum {
1792         MLX5_DC_CNAK_SIZE               = 128,
1793         MLX5_NUM_BUF_IN_PAGE            = PAGE_SIZE / MLX5_DC_CNAK_SIZE,
1794         MLX5_CNAK_TX_CQ_SIGNAL_FACTOR   = 128,
1795         MLX5_DC_CNAK_SL                 = 0,
1796         MLX5_DC_CNAK_VL                 = 0,
1797 };
1798
1799 static int init_dc_improvements(struct mlx5_ib_dev *dev)
1800 {
1801         if (!mlx5_core_is_pf(dev->mdev))
1802                 return 0;
1803
1804         if (!(MLX5_CAP_GEN(dev->mdev, dc_cnak_trace)))
1805                 return 0;
1806
1807         enable_dc_tracer(dev);
1808
1809         return 0;
1810 }
1811
1812 static void cleanup_dc_improvements(struct mlx5_ib_dev *dev)
1813 {
1814
1815         disable_dc_tracer(dev);
1816 }
1817
1818 static void mlx5_ib_dealloc_q_port_counter(struct mlx5_ib_dev *dev, u8 port_num)
1819 {
1820         mlx5_vport_dealloc_q_counter(dev->mdev,
1821                                      MLX5_INTERFACE_PROTOCOL_IB,
1822                                      dev->port[port_num].q_cnt_id);
1823         dev->port[port_num].q_cnt_id = 0;
1824 }
1825
1826 static void mlx5_ib_dealloc_q_counters(struct mlx5_ib_dev *dev)
1827 {
1828         unsigned int i;
1829
1830         for (i = 0; i < dev->num_ports; i++)
1831                 mlx5_ib_dealloc_q_port_counter(dev, i);
1832 }
1833
1834 static int mlx5_ib_alloc_q_counters(struct mlx5_ib_dev *dev)
1835 {
1836         int i;
1837         int ret;
1838
1839         for (i = 0; i < dev->num_ports; i++) {
1840                 ret = mlx5_vport_alloc_q_counter(dev->mdev,
1841                                                  MLX5_INTERFACE_PROTOCOL_IB,
1842                                                  &dev->port[i].q_cnt_id);
1843                 if (ret) {
1844                         mlx5_ib_warn(dev,
1845                                      "couldn't allocate queue counter for port %d\n",
1846                                      i + 1);
1847                         goto dealloc_counters;
1848                 }
1849         }
1850
1851         return 0;
1852
1853 dealloc_counters:
1854         while (--i >= 0)
1855                 mlx5_ib_dealloc_q_port_counter(dev, i);
1856
1857         return ret;
1858 }
1859
1860 struct port_attribute {
1861         struct attribute attr;
1862         ssize_t (*show)(struct mlx5_ib_port *,
1863                         struct port_attribute *, char *buf);
1864         ssize_t (*store)(struct mlx5_ib_port *,
1865                          struct port_attribute *,
1866                          const char *buf, size_t count);
1867 };
1868
1869 struct port_counter_attribute {
1870         struct port_attribute   attr;
1871         size_t                  offset;
1872 };
1873
1874 static ssize_t port_attr_show(struct kobject *kobj,
1875                               struct attribute *attr, char *buf)
1876 {
1877         struct port_attribute *port_attr =
1878                 container_of(attr, struct port_attribute, attr);
1879         struct mlx5_ib_port_sysfs_group *p =
1880                 container_of(kobj, struct mlx5_ib_port_sysfs_group,
1881                              kobj);
1882         struct mlx5_ib_port *mibport = container_of(p, struct mlx5_ib_port,
1883                                                     group);
1884
1885         if (!port_attr->show)
1886                 return -EIO;
1887
1888         return port_attr->show(mibport, port_attr, buf);
1889 }
1890
1891 static ssize_t show_port_counter(struct mlx5_ib_port *p,
1892                                  struct port_attribute *port_attr,
1893                                  char *buf)
1894 {
1895         int outlen = MLX5_ST_SZ_BYTES(query_q_counter_out);
1896         struct port_counter_attribute *counter_attr =
1897                 container_of(port_attr, struct port_counter_attribute, attr);
1898         void *out;
1899         int ret;
1900
1901         out = mlx5_vzalloc(outlen);
1902         if (!out)
1903                 return -ENOMEM;
1904
1905         ret = mlx5_vport_query_q_counter(p->dev->mdev,
1906                                          p->q_cnt_id, 0,
1907                                          out, outlen);
1908         if (ret)
1909                 goto free;
1910
1911         ret = sprintf(buf, "%d\n",
1912                       be32_to_cpu(*(__be32 *)(out + counter_attr->offset)));
1913
1914 free:
1915         kfree(out);
1916         return ret;
1917 }
1918
1919 #define PORT_COUNTER_ATTR(_name)                                        \
1920 struct port_counter_attribute port_counter_attr_##_name = {             \
1921         .attr  = __ATTR(_name, S_IRUGO, show_port_counter, NULL),       \
1922         .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)             \
1923 }
1924
1925 static PORT_COUNTER_ATTR(rx_write_requests);
1926 static PORT_COUNTER_ATTR(rx_read_requests);
1927 static PORT_COUNTER_ATTR(rx_atomic_requests);
1928 static PORT_COUNTER_ATTR(rx_dct_connect);
1929 static PORT_COUNTER_ATTR(out_of_buffer);
1930 static PORT_COUNTER_ATTR(out_of_sequence);
1931 static PORT_COUNTER_ATTR(duplicate_request);
1932 static PORT_COUNTER_ATTR(rnr_nak_retry_err);
1933 static PORT_COUNTER_ATTR(packet_seq_err);
1934 static PORT_COUNTER_ATTR(implied_nak_seq_err);
1935 static PORT_COUNTER_ATTR(local_ack_timeout_err);
1936
1937 static struct attribute *counter_attrs[] = {
1938         &port_counter_attr_rx_write_requests.attr.attr,
1939         &port_counter_attr_rx_read_requests.attr.attr,
1940         &port_counter_attr_rx_atomic_requests.attr.attr,
1941         &port_counter_attr_rx_dct_connect.attr.attr,
1942         &port_counter_attr_out_of_buffer.attr.attr,
1943         &port_counter_attr_out_of_sequence.attr.attr,
1944         &port_counter_attr_duplicate_request.attr.attr,
1945         &port_counter_attr_rnr_nak_retry_err.attr.attr,
1946         &port_counter_attr_packet_seq_err.attr.attr,
1947         &port_counter_attr_implied_nak_seq_err.attr.attr,
1948         &port_counter_attr_local_ack_timeout_err.attr.attr,
1949         NULL
1950 };
1951
1952 static struct attribute_group port_counters_group = {
1953         .name  = "counters",
1954         .attrs  = counter_attrs
1955 };
1956
1957 static const struct sysfs_ops port_sysfs_ops = {
1958         .show = port_attr_show
1959 };
1960
1961 static struct kobj_type port_type = {
1962         .sysfs_ops     = &port_sysfs_ops,
1963 };
1964
1965 static int add_port_attrs(struct mlx5_ib_dev *dev,
1966                           struct kobject *parent,
1967                           struct mlx5_ib_port_sysfs_group *port,
1968                           u8 port_num)
1969 {
1970         int ret;
1971
1972         ret = kobject_init_and_add(&port->kobj, &port_type,
1973                                    parent,
1974                                    "%d", port_num);
1975         if (ret)
1976                 return ret;
1977
1978         if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt) &&
1979             MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) {
1980                 ret = sysfs_create_group(&port->kobj, &port_counters_group);
1981                 if (ret)
1982                         goto put_kobj;
1983         }
1984
1985         port->enabled = true;
1986         return ret;
1987
1988 put_kobj:
1989         kobject_put(&port->kobj);
1990         return ret;
1991 }
1992
1993 static void destroy_ports_attrs(struct mlx5_ib_dev *dev,
1994                                 unsigned int num_ports)
1995 {
1996         unsigned int i;
1997
1998         for (i = 0; i < num_ports; i++) {
1999                 struct mlx5_ib_port_sysfs_group *port =
2000                         &dev->port[i].group;
2001
2002                 if (!port->enabled)
2003                         continue;
2004
2005                 if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt) &&
2006                     MLX5_CAP_GEN(dev->mdev, retransmission_q_counters))
2007                         sysfs_remove_group(&port->kobj,
2008                                            &port_counters_group);
2009                 kobject_put(&port->kobj);
2010                 port->enabled = false;
2011         }
2012
2013         if (dev->ports_parent) {
2014                 kobject_put(dev->ports_parent);
2015                 dev->ports_parent = NULL;
2016         }
2017 }
2018
2019 static int create_port_attrs(struct mlx5_ib_dev *dev)
2020 {
2021         int ret = 0;
2022         unsigned int i = 0;
2023         struct device *device = &dev->ib_dev.dev;
2024
2025         dev->ports_parent = kobject_create_and_add("mlx5_ports",
2026                                                    &device->kobj);
2027         if (!dev->ports_parent)
2028                 return -ENOMEM;
2029
2030         for (i = 0; i < dev->num_ports; i++) {
2031                 ret = add_port_attrs(dev,
2032                                      dev->ports_parent,
2033                                      &dev->port[i].group,
2034                                      i + 1);
2035
2036                 if (ret)
2037                         goto _destroy_ports_attrs;
2038         }
2039
2040         return 0;
2041
2042 _destroy_ports_attrs:
2043         destroy_ports_attrs(dev, i);
2044         return ret;
2045 }
2046
2047 static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
2048 {
2049         struct mlx5_ib_dev *dev;
2050         int err;
2051         int i;
2052
2053         printk_once(KERN_INFO "%s", mlx5_version);
2054
2055         dev = (struct mlx5_ib_dev *)ib_alloc_device(sizeof(*dev));
2056         if (!dev)
2057                 return NULL;
2058
2059         dev->mdev = mdev;
2060
2061         dev->port = kcalloc(MLX5_CAP_GEN(mdev, num_ports), sizeof(*dev->port),
2062                              GFP_KERNEL);
2063         if (!dev->port)
2064                 goto err_dealloc;
2065
2066         for (i = 0; i < MLX5_CAP_GEN(mdev, num_ports); i++) {
2067                 dev->port[i].dev = dev;
2068                 dev->port[i].port_num = i;
2069                 dev->port[i].port_gone = 0;
2070                 memset(dev->port[i].gid_table, 0, sizeof(dev->port[i].gid_table));
2071         }
2072
2073         err = get_port_caps(dev);
2074         if (err)
2075                 goto err_free_port;
2076
2077         if (mlx5_use_mad_ifc(dev))
2078                 get_ext_port_caps(dev);
2079
2080         if (mlx5_ib_port_link_layer(&dev->ib_dev, 1) ==
2081             IB_LINK_LAYER_ETHERNET) {
2082                 if (MLX5_CAP_GEN(mdev, roce)) {
2083                         err = mlx5_nic_vport_enable_roce(mdev);
2084                         if (err)
2085                                 goto err_free_port;
2086                 } else {
2087                         goto err_free_port;
2088                 }
2089         }
2090
2091         MLX5_INIT_DOORBELL_LOCK(&dev->uar_lock);
2092
2093         strlcpy(dev->ib_dev.name, "mlx5_%d", IB_DEVICE_NAME_MAX);
2094         dev->ib_dev.owner               = THIS_MODULE;
2095         dev->ib_dev.node_type           = RDMA_NODE_IB_CA;
2096         dev->ib_dev.local_dma_lkey      = mdev->special_contexts.resd_lkey;
2097         dev->num_ports          = MLX5_CAP_GEN(mdev, num_ports);
2098         dev->ib_dev.phys_port_cnt     = dev->num_ports;
2099         dev->ib_dev.num_comp_vectors    =
2100                 dev->mdev->priv.eq_table.num_comp_vectors;
2101         dev->ib_dev.dma_device  = &mdev->pdev->dev;
2102
2103         dev->ib_dev.uverbs_abi_ver      = MLX5_IB_UVERBS_ABI_VERSION;
2104         dev->ib_dev.uverbs_cmd_mask     =
2105                 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT)         |
2106                 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE)        |
2107                 (1ull << IB_USER_VERBS_CMD_QUERY_PORT)          |
2108                 (1ull << IB_USER_VERBS_CMD_ALLOC_PD)            |
2109                 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD)          |
2110                 (1ull << IB_USER_VERBS_CMD_REG_MR)              |
2111                 (1ull << IB_USER_VERBS_CMD_DEREG_MR)            |
2112                 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
2113                 (1ull << IB_USER_VERBS_CMD_CREATE_CQ)           |
2114                 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ)           |
2115                 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ)          |
2116                 (1ull << IB_USER_VERBS_CMD_CREATE_QP)           |
2117                 (1ull << IB_USER_VERBS_CMD_MODIFY_QP)           |
2118                 (1ull << IB_USER_VERBS_CMD_QUERY_QP)            |
2119                 (1ull << IB_USER_VERBS_CMD_DESTROY_QP)          |
2120                 (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST)        |
2121                 (1ull << IB_USER_VERBS_CMD_DETACH_MCAST)        |
2122                 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ)          |
2123                 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ)          |
2124                 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ)           |
2125                 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ)         |
2126                 (1ull << IB_USER_VERBS_CMD_CREATE_XSRQ)         |
2127                 (1ull << IB_USER_VERBS_CMD_OPEN_QP);
2128
2129         dev->ib_dev.query_device        = mlx5_ib_query_device;
2130         dev->ib_dev.query_port          = mlx5_ib_query_port;
2131         dev->ib_dev.get_link_layer      = mlx5_ib_port_link_layer;
2132         dev->ib_dev.get_netdev          = mlx5_ib_get_netdev;
2133         dev->ib_dev.query_gid           = mlx5_ib_query_gid;
2134         dev->ib_dev.query_pkey          = mlx5_ib_query_pkey;
2135         dev->ib_dev.modify_device       = mlx5_ib_modify_device;
2136         dev->ib_dev.modify_port         = mlx5_ib_modify_port;
2137         dev->ib_dev.alloc_ucontext      = mlx5_ib_alloc_ucontext;
2138         dev->ib_dev.dealloc_ucontext    = mlx5_ib_dealloc_ucontext;
2139         dev->ib_dev.mmap                = mlx5_ib_mmap;
2140         dev->ib_dev.alloc_pd            = mlx5_ib_alloc_pd;
2141         dev->ib_dev.dealloc_pd          = mlx5_ib_dealloc_pd;
2142         dev->ib_dev.create_ah           = mlx5_ib_create_ah;
2143         dev->ib_dev.query_ah            = mlx5_ib_query_ah;
2144         dev->ib_dev.destroy_ah          = mlx5_ib_destroy_ah;
2145         dev->ib_dev.create_srq          = mlx5_ib_create_srq;
2146         dev->ib_dev.modify_srq          = mlx5_ib_modify_srq;
2147         dev->ib_dev.query_srq           = mlx5_ib_query_srq;
2148         dev->ib_dev.destroy_srq         = mlx5_ib_destroy_srq;
2149         dev->ib_dev.post_srq_recv       = mlx5_ib_post_srq_recv;
2150         dev->ib_dev.create_qp           = mlx5_ib_create_qp;
2151         dev->ib_dev.modify_qp           = mlx5_ib_modify_qp;
2152         dev->ib_dev.query_qp            = mlx5_ib_query_qp;
2153         dev->ib_dev.destroy_qp          = mlx5_ib_destroy_qp;
2154         dev->ib_dev.post_send           = mlx5_ib_post_send;
2155         dev->ib_dev.post_recv           = mlx5_ib_post_recv;
2156         dev->ib_dev.create_cq           = mlx5_ib_create_cq;
2157         dev->ib_dev.modify_cq           = mlx5_ib_modify_cq;
2158         dev->ib_dev.resize_cq           = mlx5_ib_resize_cq;
2159         dev->ib_dev.destroy_cq          = mlx5_ib_destroy_cq;
2160         dev->ib_dev.poll_cq             = mlx5_ib_poll_cq;
2161         dev->ib_dev.req_notify_cq       = mlx5_ib_arm_cq;
2162         dev->ib_dev.get_dma_mr          = mlx5_ib_get_dma_mr;
2163         dev->ib_dev.reg_user_mr         = mlx5_ib_reg_user_mr;
2164         dev->ib_dev.reg_phys_mr         = mlx5_ib_reg_phys_mr;
2165         dev->ib_dev.dereg_mr            = mlx5_ib_dereg_mr;
2166         dev->ib_dev.attach_mcast        = mlx5_ib_mcg_attach;
2167         dev->ib_dev.detach_mcast        = mlx5_ib_mcg_detach;
2168         dev->ib_dev.process_mad         = mlx5_ib_process_mad;
2169         dev->ib_dev.get_port_immutable  = mlx5_port_immutable;
2170         dev->ib_dev.alloc_fast_reg_mr   = mlx5_ib_alloc_fast_reg_mr;
2171         dev->ib_dev.alloc_fast_reg_page_list = mlx5_ib_alloc_fast_reg_page_list;
2172         dev->ib_dev.free_fast_reg_page_list  = mlx5_ib_free_fast_reg_page_list;
2173
2174         if (MLX5_CAP_GEN(mdev, xrc)) {
2175                 dev->ib_dev.alloc_xrcd = mlx5_ib_alloc_xrcd;
2176                 dev->ib_dev.dealloc_xrcd = mlx5_ib_dealloc_xrcd;
2177                 dev->ib_dev.uverbs_cmd_mask |=
2178                         (1ull << IB_USER_VERBS_CMD_OPEN_XRCD) |
2179                         (1ull << IB_USER_VERBS_CMD_CLOSE_XRCD);
2180         }
2181
2182         err = init_node_data(dev);
2183         if (err)
2184                 goto err_disable_roce;
2185
2186         mutex_init(&dev->cap_mask_mutex);
2187         INIT_LIST_HEAD(&dev->qp_list);
2188         spin_lock_init(&dev->reset_flow_resource_lock);
2189
2190         err = create_dev_resources(&dev->devr);
2191         if (err)
2192                 goto err_disable_roce;
2193
2194
2195         err = mlx5_ib_alloc_q_counters(dev);
2196         if (err)
2197                 goto err_odp;
2198
2199         err = ib_register_device(&dev->ib_dev, NULL);
2200         if (err)
2201                 goto err_q_cnt;
2202
2203         err = create_umr_res(dev);
2204         if (err)
2205                 goto err_dev;
2206
2207         if (MLX5_CAP_GEN(dev->mdev, port_type) ==
2208             MLX5_CAP_PORT_TYPE_IB) {
2209                 if (init_dc_improvements(dev))
2210                         mlx5_ib_dbg(dev, "init_dc_improvements - continuing\n");
2211         }
2212
2213         err = create_port_attrs(dev);
2214         if (err)
2215                 goto err_dc;
2216
2217         for (i = 0; i < ARRAY_SIZE(mlx5_class_attributes); i++) {
2218                 err = device_create_file(&dev->ib_dev.dev,
2219                                          mlx5_class_attributes[i]);
2220                 if (err)
2221                         goto err_port_attrs;
2222         }
2223
2224         if (1) {
2225                 struct thread *rl_thread = NULL;
2226                 struct proc *rl_proc = NULL;
2227
2228                 for (i = 0; i < MLX5_CAP_GEN(mdev, num_ports); i++) {
2229                         (void) kproc_kthread_add(mlx5_ib_roce_port_update, dev->port + i, &rl_proc, &rl_thread,
2230                             RFHIGHPID, 0, "mlx5-ib-roce-port", "mlx5-ib-roce_port-%d", i);
2231                 }
2232         }
2233
2234         dev->ib_active = true;
2235
2236         return dev;
2237
2238 err_port_attrs:
2239         destroy_ports_attrs(dev, dev->num_ports);
2240
2241 err_dc:
2242         if (MLX5_CAP_GEN(dev->mdev, port_type) ==
2243             MLX5_CAP_PORT_TYPE_IB)
2244                 cleanup_dc_improvements(dev);
2245         destroy_umrc_res(dev);
2246
2247 err_dev:
2248         ib_unregister_device(&dev->ib_dev);
2249
2250 err_q_cnt:
2251         mlx5_ib_dealloc_q_counters(dev);
2252
2253 err_odp:
2254         destroy_dev_resources(&dev->devr);
2255
2256 err_disable_roce:
2257         if (mlx5_ib_port_link_layer(&dev->ib_dev, 1) ==
2258             IB_LINK_LAYER_ETHERNET && MLX5_CAP_GEN(mdev, roce))
2259                 mlx5_nic_vport_disable_roce(mdev);
2260 err_free_port:
2261         kfree(dev->port);
2262
2263 err_dealloc:
2264         ib_dealloc_device((struct ib_device *)dev);
2265
2266         return NULL;
2267 }
2268
2269 static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context)
2270 {
2271         struct mlx5_ib_dev *dev = context;
2272         int i;
2273
2274         for (i = 0; i < MLX5_CAP_GEN(mdev, num_ports); i++) {
2275                 dev->port[i].port_gone = 1;
2276                 while (dev->port[i].port_gone != 2)
2277                         pause("W", hz);
2278         }
2279
2280         for (i = 0; i < ARRAY_SIZE(mlx5_class_attributes); i++) {
2281                 device_remove_file(&dev->ib_dev.dev,
2282                     mlx5_class_attributes[i]);
2283         }
2284
2285         destroy_ports_attrs(dev, dev->num_ports);
2286         if (MLX5_CAP_GEN(dev->mdev, port_type) ==
2287             MLX5_CAP_PORT_TYPE_IB)
2288                 cleanup_dc_improvements(dev);
2289         mlx5_ib_dealloc_q_counters(dev);
2290         ib_unregister_device(&dev->ib_dev);
2291         destroy_umrc_res(dev);
2292         destroy_dev_resources(&dev->devr);
2293
2294         if (mlx5_ib_port_link_layer(&dev->ib_dev, 1) ==
2295             IB_LINK_LAYER_ETHERNET && MLX5_CAP_GEN(mdev, roce))
2296                 mlx5_nic_vport_disable_roce(mdev);
2297
2298         kfree(dev->port);
2299         ib_dealloc_device(&dev->ib_dev);
2300 }
2301
2302 static struct mlx5_interface mlx5_ib_interface = {
2303         .add            = mlx5_ib_add,
2304         .remove         = mlx5_ib_remove,
2305         .event          = mlx5_ib_event,
2306         .protocol       = MLX5_INTERFACE_PROTOCOL_IB,
2307 };
2308
2309 static int __init mlx5_ib_init(void)
2310 {
2311         int err;
2312
2313         if (deprecated_prof_sel != 2)
2314                 printf("mlx5_ib: WARN: ""prof_sel is deprecated for mlx5_ib, set it for mlx5_core\n");
2315
2316         err = mlx5_register_interface(&mlx5_ib_interface);
2317         if (err)
2318                 goto clean_odp;
2319
2320         mlx5_ib_wq = create_singlethread_workqueue("mlx5_ib_wq");
2321         if (!mlx5_ib_wq) {
2322                 printf("mlx5_ib: ERR: ""%s: failed to create mlx5_ib_wq\n", __func__);
2323                 goto err_unreg;
2324         }
2325
2326         return err;
2327
2328 err_unreg:
2329         mlx5_unregister_interface(&mlx5_ib_interface);
2330
2331 clean_odp:
2332         return err;
2333 }
2334
2335 static void __exit mlx5_ib_cleanup(void)
2336 {
2337         destroy_workqueue(mlx5_ib_wq);
2338         mlx5_unregister_interface(&mlx5_ib_interface);
2339 }
2340
2341 module_init_order(mlx5_ib_init, SI_ORDER_THIRD);
2342 module_exit_order(mlx5_ib_cleanup, SI_ORDER_THIRD);