sys/dev/mlx5/mlx5_en/mlx5_en_main.c

   1 /*-
   2  * Copyright (c) 2015-2021 Mellanox Technologies. All rights reserved.
   3  * Copyright (c) 2022 NVIDIA corporation & affiliates.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include "opt_kern_tls.h"
  28 #include "opt_rss.h"
  29 #include "opt_ratelimit.h"
  30
  31 #include <dev/mlx5/mlx5_en/en.h>
  32
  33 #include <sys/eventhandler.h>
  34 #include <sys/sockio.h>
  35 #include <machine/atomic.h>
  36
  37 #include <net/debugnet.h>
  38
  39 static int mlx5e_get_wqe_sz(struct mlx5e_priv *priv, u32 *wqe_sz, u32 *nsegs);
  40 static if_snd_tag_query_t mlx5e_ul_snd_tag_query;
  41 static if_snd_tag_free_t mlx5e_ul_snd_tag_free;
  42
  43 struct mlx5e_channel_param {
  44         struct mlx5e_rq_param rq;
  45         struct mlx5e_sq_param sq;
  46         struct mlx5e_cq_param rx_cq;
  47         struct mlx5e_cq_param tx_cq;
  48 };
  49
  50 struct media {
  51         u32     subtype;
  52         u64     baudrate;
  53 };
  54
  55 static const struct media mlx5e_mode_table[MLX5E_LINK_SPEEDS_NUMBER] =
  56 {
  57         [MLX5E_1000BASE_CX_SGMII] = {
  58                 .subtype = IFM_1000_CX_SGMII,
  59                 .baudrate = IF_Mbps(1000ULL),
  60         },
  61         [MLX5E_1000BASE_KX] = {
  62                 .subtype = IFM_1000_KX,
  63                 .baudrate = IF_Mbps(1000ULL),
  64         },
  65         [MLX5E_10GBASE_CX4] = {
  66                 .subtype = IFM_10G_CX4,
  67                 .baudrate = IF_Gbps(10ULL),
  68         },
  69         [MLX5E_10GBASE_KX4] = {
  70                 .subtype = IFM_10G_KX4,
  71                 .baudrate = IF_Gbps(10ULL),
  72         },
  73         [MLX5E_10GBASE_KR] = {
  74                 .subtype = IFM_10G_KR,
  75                 .baudrate = IF_Gbps(10ULL),
  76         },
  77         [MLX5E_20GBASE_KR2] = {
  78                 .subtype = IFM_20G_KR2,
  79                 .baudrate = IF_Gbps(20ULL),
  80         },
  81         [MLX5E_40GBASE_CR4] = {
  82                 .subtype = IFM_40G_CR4,
  83                 .baudrate = IF_Gbps(40ULL),
  84         },
  85         [MLX5E_40GBASE_KR4] = {
  86                 .subtype = IFM_40G_KR4,
  87                 .baudrate = IF_Gbps(40ULL),
  88         },
  89         [MLX5E_56GBASE_R4] = {
  90                 .subtype = IFM_56G_R4,
  91                 .baudrate = IF_Gbps(56ULL),
  92         },
  93         [MLX5E_10GBASE_CR] = {
  94                 .subtype = IFM_10G_CR1,
  95                 .baudrate = IF_Gbps(10ULL),
  96         },
  97         [MLX5E_10GBASE_SR] = {
  98                 .subtype = IFM_10G_SR,
  99                 .baudrate = IF_Gbps(10ULL),
 100         },
 101         [MLX5E_10GBASE_ER_LR] = {
 102                 .subtype = IFM_10G_ER,
 103                 .baudrate = IF_Gbps(10ULL),
 104         },
 105         [MLX5E_40GBASE_SR4] = {
 106                 .subtype = IFM_40G_SR4,
 107                 .baudrate = IF_Gbps(40ULL),
 108         },
 109         [MLX5E_40GBASE_LR4_ER4] = {
 110                 .subtype = IFM_40G_LR4,
 111                 .baudrate = IF_Gbps(40ULL),
 112         },
 113         [MLX5E_100GBASE_CR4] = {
 114                 .subtype = IFM_100G_CR4,
 115                 .baudrate = IF_Gbps(100ULL),
 116         },
 117         [MLX5E_100GBASE_SR4] = {
 118                 .subtype = IFM_100G_SR4,
 119                 .baudrate = IF_Gbps(100ULL),
 120         },
 121         [MLX5E_100GBASE_KR4] = {
 122                 .subtype = IFM_100G_KR4,
 123                 .baudrate = IF_Gbps(100ULL),
 124         },
 125         [MLX5E_100GBASE_LR4] = {
 126                 .subtype = IFM_100G_LR4,
 127                 .baudrate = IF_Gbps(100ULL),
 128         },
 129         [MLX5E_100BASE_TX] = {
 130                 .subtype = IFM_100_TX,
 131                 .baudrate = IF_Mbps(100ULL),
 132         },
 133         [MLX5E_1000BASE_T] = {
 134                 .subtype = IFM_1000_T,
 135                 .baudrate = IF_Mbps(1000ULL),
 136         },
 137         [MLX5E_10GBASE_T] = {
 138                 .subtype = IFM_10G_T,
 139                 .baudrate = IF_Gbps(10ULL),
 140         },
 141         [MLX5E_25GBASE_CR] = {
 142                 .subtype = IFM_25G_CR,
 143                 .baudrate = IF_Gbps(25ULL),
 144         },
 145         [MLX5E_25GBASE_KR] = {
 146                 .subtype = IFM_25G_KR,
 147                 .baudrate = IF_Gbps(25ULL),
 148         },
 149         [MLX5E_25GBASE_SR] = {
 150                 .subtype = IFM_25G_SR,
 151                 .baudrate = IF_Gbps(25ULL),
 152         },
 153         [MLX5E_50GBASE_CR2] = {
 154                 .subtype = IFM_50G_CR2,
 155                 .baudrate = IF_Gbps(50ULL),
 156         },
 157         [MLX5E_50GBASE_KR2] = {
 158                 .subtype = IFM_50G_KR2,
 159                 .baudrate = IF_Gbps(50ULL),
 160         },
 161         [MLX5E_50GBASE_KR4] = {
 162                 .subtype = IFM_50G_KR4,
 163                 .baudrate = IF_Gbps(50ULL),
 164         },
 165 };
 166
 167 static const struct media mlx5e_ext_mode_table[MLX5E_EXT_LINK_SPEEDS_NUMBER][MLX5E_CABLE_TYPE_NUMBER] =
 168 {
 169         /**/
 170         [MLX5E_SGMII_100M][MLX5E_CABLE_TYPE_UNKNOWN] = {
 171                 .subtype = IFM_100_SGMII,
 172                 .baudrate = IF_Mbps(100),
 173         },
 174
 175         /**/
 176         [MLX5E_1000BASE_X_SGMII][MLX5E_CABLE_TYPE_UNKNOWN] = {
 177                 .subtype = IFM_1000_CX,
 178                 .baudrate = IF_Mbps(1000),
 179         },
 180         [MLX5E_1000BASE_X_SGMII][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
 181                 .subtype = IFM_1000_SX,
 182                 .baudrate = IF_Mbps(1000),
 183         },
 184
 185         /**/
 186         [MLX5E_5GBASE_R][MLX5E_CABLE_TYPE_UNKNOWN] = {
 187                 .subtype = IFM_5000_KR,
 188                 .baudrate = IF_Mbps(5000),
 189         },
 190         [MLX5E_5GBASE_R][MLX5E_CABLE_TYPE_TWISTED_PAIR] = {
 191                 .subtype = IFM_5000_T,
 192                 .baudrate = IF_Mbps(5000),
 193         },
 194
 195         /**/
 196         [MLX5E_10GBASE_XFI_XAUI_1][MLX5E_CABLE_TYPE_UNKNOWN] = {
 197                 .subtype = IFM_10G_KR,
 198                 .baudrate = IF_Gbps(10ULL),
 199         },
 200         [MLX5E_10GBASE_XFI_XAUI_1][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
 201                 .subtype = IFM_10G_CR1,
 202                 .baudrate = IF_Gbps(10ULL),
 203         },
 204         [MLX5E_10GBASE_XFI_XAUI_1][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
 205                 .subtype = IFM_10G_SR,
 206                 .baudrate = IF_Gbps(10ULL),
 207         },
 208
 209         /**/
 210         [MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_CABLE_TYPE_UNKNOWN] = {
 211                 .subtype = IFM_40G_KR4,
 212                 .baudrate = IF_Gbps(40ULL),
 213         },
 214         [MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
 215                 .subtype = IFM_40G_CR4,
 216                 .baudrate = IF_Gbps(40ULL),
 217         },
 218         [MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
 219                 .subtype = IFM_40G_SR4,
 220                 .baudrate = IF_Gbps(40ULL),
 221         },
 222
 223         /**/
 224         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CABLE_TYPE_UNKNOWN] = {
 225                 .subtype = IFM_25G_KR,
 226                 .baudrate = IF_Gbps(25ULL),
 227         },
 228         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
 229                 .subtype = IFM_25G_CR,
 230                 .baudrate = IF_Gbps(25ULL),
 231         },
 232         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
 233                 .subtype = IFM_25G_SR,
 234                 .baudrate = IF_Gbps(25ULL),
 235         },
 236         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CABLE_TYPE_TWISTED_PAIR] = {
 237                 .subtype = IFM_25G_T,
 238                 .baudrate = IF_Gbps(25ULL),
 239         },
 240
 241         /**/
 242         [MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_CABLE_TYPE_UNKNOWN] = {
 243                 .subtype = IFM_50G_KR2,
 244                 .baudrate = IF_Gbps(50ULL),
 245         },
 246         [MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
 247                 .subtype = IFM_50G_CR2,
 248                 .baudrate = IF_Gbps(50ULL),
 249         },
 250         [MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
 251                 .subtype = IFM_50G_SR2,
 252                 .baudrate = IF_Gbps(50ULL),
 253         },
 254
 255         /**/
 256         [MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_CABLE_TYPE_UNKNOWN] = {
 257                 .subtype = IFM_50G_KR_PAM4,
 258                 .baudrate = IF_Gbps(50ULL),
 259         },
 260         [MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
 261                 .subtype = IFM_50G_CP,
 262                 .baudrate = IF_Gbps(50ULL),
 263         },
 264         [MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
 265                 .subtype = IFM_50G_SR,
 266                 .baudrate = IF_Gbps(50ULL),
 267         },
 268
 269         /**/
 270         [MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_CABLE_TYPE_UNKNOWN] = {
 271                 .subtype = IFM_100G_KR4,
 272                 .baudrate = IF_Gbps(100ULL),
 273         },
 274         [MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
 275                 .subtype = IFM_100G_CR4,
 276                 .baudrate = IF_Gbps(100ULL),
 277         },
 278         [MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
 279                 .subtype = IFM_100G_SR4,
 280                 .baudrate = IF_Gbps(100ULL),
 281         },
 282
 283         /**/
 284         [MLX5E_100GAUI_1_100GBASE_CR_KR][MLX5E_CABLE_TYPE_UNKNOWN] = {
 285                 .subtype = IFM_100G_KR_PAM4,
 286                 .baudrate = IF_Gbps(100ULL),
 287         },
 288         [MLX5E_100GAUI_1_100GBASE_CR_KR][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
 289                 .subtype = IFM_100G_CR_PAM4,
 290                 .baudrate = IF_Gbps(100ULL),
 291         },
 292         [MLX5E_100GAUI_1_100GBASE_CR_KR][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
 293                 .subtype = IFM_100G_SR2,        /* XXX */
 294                 .baudrate = IF_Gbps(100ULL),
 295         },
 296
 297         /**/
 298         [MLX5E_100GAUI_2_100GBASE_CR2_KR2][MLX5E_CABLE_TYPE_UNKNOWN] = {
 299                 .subtype = IFM_100G_KR4,
 300                 .baudrate = IF_Gbps(100ULL),
 301         },
 302         [MLX5E_100GAUI_2_100GBASE_CR2_KR2][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
 303                 .subtype = IFM_100G_CP2,
 304                 .baudrate = IF_Gbps(100ULL),
 305         },
 306         [MLX5E_100GAUI_2_100GBASE_CR2_KR2][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
 307                 .subtype = IFM_100G_SR2,
 308                 .baudrate = IF_Gbps(100ULL),
 309         },
 310
 311         /**/
 312         [MLX5E_200GAUI_2_200GBASE_CR2_KR2][MLX5E_CABLE_TYPE_UNKNOWN] = {
 313                 .subtype = IFM_200G_KR4_PAM4,   /* XXX */
 314                 .baudrate = IF_Gbps(200ULL),
 315         },
 316         [MLX5E_200GAUI_2_200GBASE_CR2_KR2][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
 317                 .subtype = IFM_200G_CR4_PAM4,   /* XXX */
 318                 .baudrate = IF_Gbps(200ULL),
 319         },
 320         [MLX5E_200GAUI_2_200GBASE_CR2_KR2][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
 321                 .subtype = IFM_200G_SR4,        /* XXX */
 322                 .baudrate = IF_Gbps(200ULL),
 323         },
 324
 325         /**/
 326         [MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_CABLE_TYPE_UNKNOWN] = {
 327                 .subtype = IFM_200G_KR4_PAM4,
 328                 .baudrate = IF_Gbps(200ULL),
 329         },
 330         [MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
 331                 .subtype = IFM_200G_CR4_PAM4,
 332                 .baudrate = IF_Gbps(200ULL),
 333         },
 334         [MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
 335                 .subtype = IFM_200G_SR4,
 336                 .baudrate = IF_Gbps(200ULL),
 337         },
 338
 339         /**/
 340         [MLX5E_400GAUI_8][MLX5E_CABLE_TYPE_UNKNOWN] = {
 341                 .subtype = IFM_400G_LR8,        /* XXX */
 342                 .baudrate = IF_Gbps(400ULL),
 343         },
 344
 345         /**/
 346         [MLX5E_400GAUI_4_400GBASE_CR4_KR4][MLX5E_CABLE_TYPE_UNKNOWN] = {
 347                 .subtype = IFM_400G_LR8,        /* XXX */
 348                 .baudrate = IF_Gbps(400ULL),
 349         },
 350 };
 351
 352 static const struct if_snd_tag_sw mlx5e_ul_snd_tag_sw = {
 353         .snd_tag_query = mlx5e_ul_snd_tag_query,
 354         .snd_tag_free = mlx5e_ul_snd_tag_free,
 355         .type = IF_SND_TAG_TYPE_UNLIMITED
 356 };
 357
 358 DEBUGNET_DEFINE(mlx5_en);
 359
 360 MALLOC_DEFINE(M_MLX5EN, "MLX5EN", "MLX5 Ethernet");
 361
 362 static void
 363 mlx5e_update_carrier(struct mlx5e_priv *priv)
 364 {
 365         struct mlx5_core_dev *mdev = priv->mdev;
 366         u32 out[MLX5_ST_SZ_DW(ptys_reg)];
 367         u32 eth_proto_oper;
 368         int error;
 369         u8 i;
 370         u8 cable_type;
 371         u8 port_state;
 372         u8 is_er_type;
 373         bool ext;
 374         struct media media_entry = {};
 375
 376         port_state = mlx5_query_vport_state(mdev,
 377             MLX5_QUERY_VPORT_STATE_IN_OP_MOD_VNIC_VPORT, 0);
 378
 379         if (port_state == VPORT_STATE_UP) {
 380                 priv->media_status_last |= IFM_ACTIVE;
 381         } else {
 382                 priv->media_status_last &= ~IFM_ACTIVE;
 383                 priv->media_active_last = IFM_ETHER;
 384                 if_link_state_change(priv->ifp, LINK_STATE_DOWN);
 385                 return;
 386         }
 387
 388         error = mlx5_query_port_ptys(mdev, out, sizeof(out),
 389             MLX5_PTYS_EN, 1);
 390         if (error) {
 391                 priv->media_active_last = IFM_ETHER;
 392                 if_setbaudrate(priv->ifp, 1);
 393                 mlx5_en_err(priv->ifp, "query port ptys failed: 0x%x\n",
 394                     error);
 395                 return;
 396         }
 397
 398         ext = MLX5_CAP_PCAM_FEATURE(mdev, ptys_extended_ethernet);
 399         eth_proto_oper = MLX5_GET_ETH_PROTO(ptys_reg, out, ext,
 400             eth_proto_oper);
 401
 402         i = ilog2(eth_proto_oper);
 403
 404         if (ext) {
 405                 error = mlx5_query_pddr_cable_type(mdev, 1, &cable_type);
 406                 if (error != 0) {
 407                         /* use fallback entry */
 408                         media_entry = mlx5e_ext_mode_table[i][MLX5E_CABLE_TYPE_UNKNOWN];
 409
 410                         mlx5_en_err(priv->ifp,
 411                             "query port pddr failed: %d\n", error);
 412                 } else {
 413                         media_entry = mlx5e_ext_mode_table[i][cable_type];
 414
 415                         /* check if we should use fallback entry */
 416                         if (media_entry.subtype == 0)
 417                                 media_entry = mlx5e_ext_mode_table[i][MLX5E_CABLE_TYPE_UNKNOWN];
 418                 }
 419         } else {
 420                 media_entry = mlx5e_mode_table[i];
 421         }
 422
 423         if (media_entry.subtype == 0) {
 424                 mlx5_en_err(priv->ifp,
 425                     "Could not find operational media subtype\n");
 426                 return;
 427         }
 428
 429         switch (media_entry.subtype) {
 430         case IFM_10G_ER:
 431                 error = mlx5_query_pddr_range_info(mdev, 1, &is_er_type);
 432                 if (error != 0) {
 433                         mlx5_en_err(priv->ifp,
 434                             "query port pddr failed: %d\n", error);
 435                 }
 436                 if (error != 0 || is_er_type == 0)
 437                         media_entry.subtype = IFM_10G_LR;
 438                 break;
 439         case IFM_40G_LR4:
 440                 error = mlx5_query_pddr_range_info(mdev, 1, &is_er_type);
 441                 if (error != 0) {
 442                         mlx5_en_err(priv->ifp,
 443                             "query port pddr failed: %d\n", error);
 444                 }
 445                 if (error == 0 && is_er_type != 0)
 446                         media_entry.subtype = IFM_40G_ER4;
 447                 break;
 448         }
 449         priv->media_active_last = media_entry.subtype | IFM_ETHER | IFM_FDX;
 450         if_setbaudrate(priv->ifp, media_entry.baudrate);
 451
 452         if_link_state_change(priv->ifp, LINK_STATE_UP);
 453 }
 454
 455 static void
 456 mlx5e_media_status(if_t dev, struct ifmediareq *ifmr)
 457 {
 458         struct mlx5e_priv *priv = if_getsoftc(dev);
 459
 460         ifmr->ifm_status = priv->media_status_last;
 461         ifmr->ifm_current = ifmr->ifm_active = priv->media_active_last |
 462             (priv->params.rx_pauseframe_control ? IFM_ETH_RXPAUSE : 0) |
 463             (priv->params.tx_pauseframe_control ? IFM_ETH_TXPAUSE : 0);
 464
 465 }
 466
 467 static u32
 468 mlx5e_find_link_mode(u32 subtype, bool ext)
 469 {
 470         u32 link_mode = 0;
 471
 472         switch (subtype) {
 473         case 0:
 474                 goto done;
 475         case IFM_10G_LR:
 476                 subtype = IFM_10G_ER;
 477                 break;
 478         case IFM_40G_ER4:
 479                 subtype = IFM_40G_LR4;
 480                 break;
 481         default:
 482                 break;
 483         }
 484
 485         if (ext) {
 486                 for (unsigned i = 0; i != MLX5E_EXT_LINK_SPEEDS_NUMBER; i++) {
 487                         for (unsigned j = 0; j != MLX5E_CABLE_TYPE_NUMBER; j++) {
 488                                 if (mlx5e_ext_mode_table[i][j].subtype == subtype)
 489                                         link_mode |= MLX5E_PROT_MASK(i);
 490                         }
 491                 }
 492         } else {
 493                 for (unsigned i = 0; i != MLX5E_LINK_SPEEDS_NUMBER; i++) {
 494                         if (mlx5e_mode_table[i].subtype == subtype)
 495                                 link_mode |= MLX5E_PROT_MASK(i);
 496                 }
 497         }
 498 done:
 499         return (link_mode);
 500 }
 501
 502 static int
 503 mlx5e_set_port_pause_and_pfc(struct mlx5e_priv *priv)
 504 {
 505         return (mlx5_set_port_pause_and_pfc(priv->mdev, 1,
 506             priv->params.rx_pauseframe_control,
 507             priv->params.tx_pauseframe_control,
 508             priv->params.rx_priority_flow_control,
 509             priv->params.tx_priority_flow_control));
 510 }
 511
 512 static int
 513 mlx5e_set_port_pfc(struct mlx5e_priv *priv)
 514 {
 515         int error;
 516
 517         if (priv->gone != 0) {
 518                 error = -ENXIO;
 519         } else if (priv->params.rx_pauseframe_control ||
 520             priv->params.tx_pauseframe_control) {
 521                 mlx5_en_err(priv->ifp,
 522                     "Global pauseframes must be disabled before enabling PFC.\n");
 523                 error = -EINVAL;
 524         } else {
 525                 error = mlx5e_set_port_pause_and_pfc(priv);
 526         }
 527         return (error);
 528 }
 529
 530 static int
 531 mlx5e_media_change(if_t dev)
 532 {
 533         struct mlx5e_priv *priv = if_getsoftc(dev);
 534         struct mlx5_core_dev *mdev = priv->mdev;
 535         u32 eth_proto_cap;
 536         u32 link_mode;
 537         u32 out[MLX5_ST_SZ_DW(ptys_reg)];
 538         int was_opened;
 539         int locked;
 540         int error;
 541         bool ext;
 542
 543         locked = PRIV_LOCKED(priv);
 544         if (!locked)
 545                 PRIV_LOCK(priv);
 546
 547         if (IFM_TYPE(priv->media.ifm_media) != IFM_ETHER) {
 548                 error = EINVAL;
 549                 goto done;
 550         }
 551
 552         error = mlx5_query_port_ptys(mdev, out, sizeof(out),
 553             MLX5_PTYS_EN, 1);
 554         if (error != 0) {
 555                 mlx5_en_err(dev, "Query port media capability failed\n");
 556                 goto done;
 557         }
 558
 559         ext = MLX5_CAP_PCAM_FEATURE(mdev, ptys_extended_ethernet);
 560         link_mode = mlx5e_find_link_mode(IFM_SUBTYPE(priv->media.ifm_media), ext);
 561
 562         /* query supported capabilities */
 563         eth_proto_cap = MLX5_GET_ETH_PROTO(ptys_reg, out, ext,
 564             eth_proto_capability);
 565
 566         /* check for autoselect */
 567         if (IFM_SUBTYPE(priv->media.ifm_media) == IFM_AUTO) {
 568                 link_mode = eth_proto_cap;
 569                 if (link_mode == 0) {
 570                         mlx5_en_err(dev, "Port media capability is zero\n");
 571                         error = EINVAL;
 572                         goto done;
 573                 }
 574         } else {
 575                 link_mode = link_mode & eth_proto_cap;
 576                 if (link_mode == 0) {
 577                         mlx5_en_err(dev, "Not supported link mode requested\n");
 578                         error = EINVAL;
 579                         goto done;
 580                 }
 581         }
 582         if (priv->media.ifm_media & (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE)) {
 583                 /* check if PFC is enabled */
 584                 if (priv->params.rx_priority_flow_control ||
 585                     priv->params.tx_priority_flow_control) {
 586                         mlx5_en_err(dev, "PFC must be disabled before enabling global pauseframes.\n");
 587                         error = EINVAL;
 588                         goto done;
 589                 }
 590         }
 591         /* update pauseframe control bits */
 592         priv->params.rx_pauseframe_control =
 593             (priv->media.ifm_media & IFM_ETH_RXPAUSE) ? 1 : 0;
 594         priv->params.tx_pauseframe_control =
 595             (priv->media.ifm_media & IFM_ETH_TXPAUSE) ? 1 : 0;
 596
 597         /* check if device is opened */
 598         was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
 599
 600         /* reconfigure the hardware */
 601         mlx5_set_port_status(mdev, MLX5_PORT_DOWN);
 602         mlx5_set_port_proto(mdev, link_mode, MLX5_PTYS_EN, ext);
 603         error = -mlx5e_set_port_pause_and_pfc(priv);
 604         if (was_opened)
 605                 mlx5_set_port_status(mdev, MLX5_PORT_UP);
 606
 607 done:
 608         if (!locked)
 609                 PRIV_UNLOCK(priv);
 610         return (error);
 611 }
 612
 613 static void
 614 mlx5e_update_carrier_work(struct work_struct *work)
 615 {
 616         struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
 617             update_carrier_work);
 618
 619         PRIV_LOCK(priv);
 620         if (test_bit(MLX5E_STATE_OPENED, &priv->state))
 621                 mlx5e_update_carrier(priv);
 622         PRIV_UNLOCK(priv);
 623 }
 624
 625 #define MLX5E_PCIE_PERF_GET_64(a,b,c,d,e,f)    \
 626         s_debug->c = MLX5_GET64(mpcnt_reg, out, counter_set.f.c);
 627
 628 #define MLX5E_PCIE_PERF_GET_32(a,b,c,d,e,f)    \
 629         s_debug->c = MLX5_GET(mpcnt_reg, out, counter_set.f.c);
 630
 631 static void
 632 mlx5e_update_pcie_counters(struct mlx5e_priv *priv)
 633 {
 634         struct mlx5_core_dev *mdev = priv->mdev;
 635         struct mlx5e_port_stats_debug *s_debug = &priv->stats.port_stats_debug;
 636         const unsigned sz = MLX5_ST_SZ_BYTES(mpcnt_reg);
 637         void *out;
 638         void *in;
 639         int err;
 640
 641         /* allocate firmware request structures */
 642         in = mlx5_vzalloc(sz);
 643         out = mlx5_vzalloc(sz);
 644         if (in == NULL || out == NULL)
 645                 goto free_out;
 646
 647         MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_PERFORMANCE_COUNTERS_GROUP);
 648         err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
 649         if (err != 0)
 650                 goto free_out;
 651
 652         MLX5E_PCIE_PERFORMANCE_COUNTERS_64(MLX5E_PCIE_PERF_GET_64)
 653         MLX5E_PCIE_PERFORMANCE_COUNTERS_32(MLX5E_PCIE_PERF_GET_32)
 654
 655         MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_TIMERS_AND_STATES_COUNTERS_GROUP);
 656         err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
 657         if (err != 0)
 658                 goto free_out;
 659
 660         MLX5E_PCIE_TIMERS_AND_STATES_COUNTERS_32(MLX5E_PCIE_PERF_GET_32)
 661
 662         MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_LANE_COUNTERS_GROUP);
 663         err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
 664         if (err != 0)
 665                 goto free_out;
 666
 667         MLX5E_PCIE_LANE_COUNTERS_32(MLX5E_PCIE_PERF_GET_32)
 668
 669 free_out:
 670         /* free firmware request structures */
 671         kvfree(in);
 672         kvfree(out);
 673 }
 674
 675 /*
 676  * This function reads the physical port counters from the firmware
 677  * using a pre-defined layout defined by various MLX5E_PPORT_XXX()
 678  * macros. The output is converted from big-endian 64-bit values into
 679  * host endian ones and stored in the "priv->stats.pport" structure.
 680  */
 681 static void
 682 mlx5e_update_pport_counters(struct mlx5e_priv *priv)
 683 {
 684         struct mlx5_core_dev *mdev = priv->mdev;
 685         struct mlx5e_pport_stats *s = &priv->stats.pport;
 686         struct mlx5e_port_stats_debug *s_debug = &priv->stats.port_stats_debug;
 687         u32 *in;
 688         u32 *out;
 689         const u64 *ptr;
 690         unsigned sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
 691         unsigned x;
 692         unsigned y;
 693         unsigned z;
 694
 695         /* allocate firmware request structures */
 696         in = mlx5_vzalloc(sz);
 697         out = mlx5_vzalloc(sz);
 698         if (in == NULL || out == NULL)
 699                 goto free_out;
 700
 701         /*
 702          * Get pointer to the 64-bit counter set which is located at a
 703          * fixed offset in the output firmware request structure:
 704          */
 705         ptr = (const uint64_t *)MLX5_ADDR_OF(ppcnt_reg, out, counter_set);
 706
 707         MLX5_SET(ppcnt_reg, in, local_port, 1);
 708
 709         /* read IEEE802_3 counter group using predefined counter layout */
 710         MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
 711         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 712         for (x = 0, y = MLX5E_PPORT_PER_PRIO_STATS_NUM;
 713              x != MLX5E_PPORT_IEEE802_3_STATS_NUM; x++, y++)
 714                 s->arg[y] = be64toh(ptr[x]);
 715
 716         /* read RFC2819 counter group using predefined counter layout */
 717         MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2819_COUNTERS_GROUP);
 718         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 719         for (x = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM; x++, y++)
 720                 s->arg[y] = be64toh(ptr[x]);
 721
 722         for (y = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM +
 723             MLX5E_PPORT_RFC2819_STATS_DEBUG_NUM; x++, y++)
 724                 s_debug->arg[y] = be64toh(ptr[x]);
 725
 726         /* read RFC2863 counter group using predefined counter layout */
 727         MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2863_COUNTERS_GROUP);
 728         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 729         for (x = 0; x != MLX5E_PPORT_RFC2863_STATS_DEBUG_NUM; x++, y++)
 730                 s_debug->arg[y] = be64toh(ptr[x]);
 731
 732         /* read physical layer stats counter group using predefined counter layout */
 733         MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP);
 734         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 735         for (x = 0; x != MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG_NUM; x++, y++)
 736                 s_debug->arg[y] = be64toh(ptr[x]);
 737
 738         /* read Extended Ethernet counter group using predefined counter layout */
 739         MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);
 740         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 741         for (x = 0; x != MLX5E_PPORT_ETHERNET_EXTENDED_STATS_DEBUG_NUM; x++, y++)
 742                 s_debug->arg[y] = be64toh(ptr[x]);
 743
 744         /* read Extended Statistical Group */
 745         if (MLX5_CAP_GEN(mdev, pcam_reg) &&
 746             MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group) &&
 747             MLX5_CAP_PCAM_FEATURE(mdev, per_lane_error_counters)) {
 748                 /* read Extended Statistical counter group using predefined counter layout */
 749                 MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_STATISTICAL_GROUP);
 750                 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 751
 752                 for (x = 0; x != MLX5E_PPORT_STATISTICAL_DEBUG_NUM; x++, y++)
 753                         s_debug->arg[y] = be64toh(ptr[x]);
 754         }
 755
 756         /* read PCIE counters */
 757         mlx5e_update_pcie_counters(priv);
 758
 759         /* read per-priority counters */
 760         MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_PRIORITY_COUNTERS_GROUP);
 761
 762         /* iterate all the priorities */
 763         for (y = z = 0; z != MLX5E_PPORT_PER_PRIO_STATS_NUM_PRIO; z++) {
 764                 MLX5_SET(ppcnt_reg, in, prio_tc, z);
 765                 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 766
 767                 /* read per priority stats counter group using predefined counter layout */
 768                 for (x = 0; x != (MLX5E_PPORT_PER_PRIO_STATS_NUM /
 769                     MLX5E_PPORT_PER_PRIO_STATS_NUM_PRIO); x++, y++)
 770                         s->arg[y] = be64toh(ptr[x]);
 771         }
 772
 773 free_out:
 774         /* free firmware request structures */
 775         kvfree(in);
 776         kvfree(out);
 777 }
 778
 779 static void
 780 mlx5e_grp_vnic_env_update_stats(struct mlx5e_priv *priv)
 781 {
 782         u32 out[MLX5_ST_SZ_DW(query_vnic_env_out)] = {};
 783         u32 in[MLX5_ST_SZ_DW(query_vnic_env_in)] = {};
 784
 785         if (!MLX5_CAP_GEN(priv->mdev, nic_receive_steering_discard))
 786                 return;
 787
 788         MLX5_SET(query_vnic_env_in, in, opcode,
 789             MLX5_CMD_OP_QUERY_VNIC_ENV);
 790         MLX5_SET(query_vnic_env_in, in, op_mod, 0);
 791         MLX5_SET(query_vnic_env_in, in, other_vport, 0);
 792
 793         if (mlx5_cmd_exec(priv->mdev, in, sizeof(in), out, sizeof(out)) != 0)
 794                 return;
 795
 796         priv->stats.vport.rx_steer_missed_packets =
 797             MLX5_GET64(query_vnic_env_out, out,
 798             vport_env.nic_receive_steering_discard);
 799 }
 800
 801 /*
 802  * This function is called regularly to collect all statistics
 803  * counters from the firmware. The values can be viewed through the
 804  * sysctl interface. Execution is serialized using the priv's global
 805  * configuration lock.
 806  */
 807 static void
 808 mlx5e_update_stats_locked(struct mlx5e_priv *priv)
 809 {
 810         struct mlx5_core_dev *mdev = priv->mdev;
 811         struct mlx5e_vport_stats *s = &priv->stats.vport;
 812         struct mlx5e_sq_stats *sq_stats;
 813         u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)];
 814         u32 *out;
 815         int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
 816         u64 tso_packets = 0;
 817         u64 tso_bytes = 0;
 818         u64 tx_queue_dropped = 0;
 819         u64 tx_defragged = 0;
 820         u64 tx_offload_none = 0;
 821         u64 lro_packets = 0;
 822         u64 lro_bytes = 0;
 823         u64 sw_lro_queued = 0;
 824         u64 sw_lro_flushed = 0;
 825         u64 rx_csum_none = 0;
 826         u64 rx_wqe_err = 0;
 827         u64 rx_packets = 0;
 828         u64 rx_bytes = 0;
 829         u64 rx_decrypted_error = 0;
 830         u64 rx_decrypted_ok = 0;
 831         u32 rx_out_of_buffer = 0;
 832         int error;
 833         int i;
 834         int j;
 835
 836         out = mlx5_vzalloc(outlen);
 837         if (out == NULL)
 838                 goto free_out;
 839
 840         /* Collect firts the SW counters and then HW for consistency */
 841         for (i = 0; i < priv->params.num_channels; i++) {
 842                 struct mlx5e_channel *pch = priv->channel + i;
 843                 struct mlx5e_rq *rq = &pch->rq;
 844                 struct mlx5e_rq_stats *rq_stats = &pch->rq.stats;
 845
 846                 /* collect stats from LRO */
 847                 rq_stats->sw_lro_queued = rq->lro.lro_queued;
 848                 rq_stats->sw_lro_flushed = rq->lro.lro_flushed;
 849                 sw_lro_queued += rq_stats->sw_lro_queued;
 850                 sw_lro_flushed += rq_stats->sw_lro_flushed;
 851                 lro_packets += rq_stats->lro_packets;
 852                 lro_bytes += rq_stats->lro_bytes;
 853                 rx_csum_none += rq_stats->csum_none;
 854                 rx_wqe_err += rq_stats->wqe_err;
 855                 rx_packets += rq_stats->packets;
 856                 rx_bytes += rq_stats->bytes;
 857                 rx_decrypted_error += rq_stats->decrypted_error_packets;
 858                 rx_decrypted_ok += rq_stats->decrypted_ok_packets;
 859
 860                 for (j = 0; j < priv->num_tc; j++) {
 861                         sq_stats = &pch->sq[j].stats;
 862
 863                         tso_packets += sq_stats->tso_packets;
 864                         tso_bytes += sq_stats->tso_bytes;
 865                         tx_queue_dropped += sq_stats->dropped;
 866                         tx_queue_dropped += sq_stats->enobuf;
 867                         tx_defragged += sq_stats->defragged;
 868                         tx_offload_none += sq_stats->csum_offload_none;
 869                 }
 870         }
 871
 872 #ifdef RATELIMIT
 873         /* Collect statistics from all rate-limit queues */
 874         for (j = 0; j < priv->rl.param.tx_worker_threads_def; j++) {
 875                 struct mlx5e_rl_worker *rlw = priv->rl.workers + j;
 876
 877                 for (i = 0; i < priv->rl.param.tx_channels_per_worker_def; i++) {
 878                         struct mlx5e_rl_channel *channel = rlw->channels + i;
 879                         struct mlx5e_sq *sq = channel->sq;
 880
 881                         if (sq == NULL)
 882                                 continue;
 883
 884                         sq_stats = &sq->stats;
 885
 886                         tso_packets += sq_stats->tso_packets;
 887                         tso_bytes += sq_stats->tso_bytes;
 888                         tx_queue_dropped += sq_stats->dropped;
 889                         tx_queue_dropped += sq_stats->enobuf;
 890                         tx_defragged += sq_stats->defragged;
 891                         tx_offload_none += sq_stats->csum_offload_none;
 892                 }
 893         }
 894 #endif
 895
 896         /* update counters */
 897         s->tso_packets = tso_packets;
 898         s->tso_bytes = tso_bytes;
 899         s->tx_queue_dropped = tx_queue_dropped;
 900         s->tx_defragged = tx_defragged;
 901         s->lro_packets = lro_packets;
 902         s->lro_bytes = lro_bytes;
 903         s->sw_lro_queued = sw_lro_queued;
 904         s->sw_lro_flushed = sw_lro_flushed;
 905         s->rx_csum_none = rx_csum_none;
 906         s->rx_wqe_err = rx_wqe_err;
 907         s->rx_packets = rx_packets;
 908         s->rx_bytes = rx_bytes;
 909         s->rx_decrypted_error_packets = rx_decrypted_error;
 910         s->rx_decrypted_ok_packets = rx_decrypted_ok;
 911
 912         mlx5e_grp_vnic_env_update_stats(priv);
 913
 914         /* HW counters */
 915         memset(in, 0, sizeof(in));
 916
 917         MLX5_SET(query_vport_counter_in, in, opcode,
 918             MLX5_CMD_OP_QUERY_VPORT_COUNTER);
 919         MLX5_SET(query_vport_counter_in, in, op_mod, 0);
 920         MLX5_SET(query_vport_counter_in, in, other_vport, 0);
 921
 922         memset(out, 0, outlen);
 923
 924         /* get number of out-of-buffer drops first */
 925         if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0 &&
 926             mlx5_vport_query_out_of_rx_buffer(mdev, priv->counter_set_id,
 927             &rx_out_of_buffer) == 0) {
 928                 s->rx_out_of_buffer = rx_out_of_buffer;
 929         }
 930
 931         /* get port statistics */
 932         if (mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen) == 0) {
 933 #define MLX5_GET_CTR(out, x) \
 934         MLX5_GET64(query_vport_counter_out, out, x)
 935
 936                 s->rx_error_packets =
 937                     MLX5_GET_CTR(out, received_errors.packets);
 938                 s->rx_error_bytes =
 939                     MLX5_GET_CTR(out, received_errors.octets);
 940                 s->tx_error_packets =
 941                     MLX5_GET_CTR(out, transmit_errors.packets);
 942                 s->tx_error_bytes =
 943                     MLX5_GET_CTR(out, transmit_errors.octets);
 944
 945                 s->rx_unicast_packets =
 946                     MLX5_GET_CTR(out, received_eth_unicast.packets);
 947                 s->rx_unicast_bytes =
 948                     MLX5_GET_CTR(out, received_eth_unicast.octets);
 949                 s->tx_unicast_packets =
 950                     MLX5_GET_CTR(out, transmitted_eth_unicast.packets);
 951                 s->tx_unicast_bytes =
 952                     MLX5_GET_CTR(out, transmitted_eth_unicast.octets);
 953
 954                 s->rx_multicast_packets =
 955                     MLX5_GET_CTR(out, received_eth_multicast.packets);
 956                 s->rx_multicast_bytes =
 957                     MLX5_GET_CTR(out, received_eth_multicast.octets);
 958                 s->tx_multicast_packets =
 959                     MLX5_GET_CTR(out, transmitted_eth_multicast.packets);
 960                 s->tx_multicast_bytes =
 961                     MLX5_GET_CTR(out, transmitted_eth_multicast.octets);
 962
 963                 s->rx_broadcast_packets =
 964                     MLX5_GET_CTR(out, received_eth_broadcast.packets);
 965                 s->rx_broadcast_bytes =
 966                     MLX5_GET_CTR(out, received_eth_broadcast.octets);
 967                 s->tx_broadcast_packets =
 968                     MLX5_GET_CTR(out, transmitted_eth_broadcast.packets);
 969                 s->tx_broadcast_bytes =
 970                     MLX5_GET_CTR(out, transmitted_eth_broadcast.octets);
 971
 972                 s->tx_packets = s->tx_unicast_packets +
 973                     s->tx_multicast_packets + s->tx_broadcast_packets;
 974                 s->tx_bytes = s->tx_unicast_bytes + s->tx_multicast_bytes +
 975                     s->tx_broadcast_bytes;
 976
 977                 /* Update calculated offload counters */
 978                 s->tx_csum_offload = s->tx_packets - tx_offload_none;
 979                 s->rx_csum_good = s->rx_packets - s->rx_csum_none;
 980         }
 981
 982         /* Get physical port counters */
 983         mlx5e_update_pport_counters(priv);
 984
 985         s->tx_jumbo_packets =
 986             priv->stats.port_stats_debug.tx_stat_p1519to2047octets +
 987             priv->stats.port_stats_debug.tx_stat_p2048to4095octets +
 988             priv->stats.port_stats_debug.tx_stat_p4096to8191octets +
 989             priv->stats.port_stats_debug.tx_stat_p8192to10239octets;
 990
 991 free_out:
 992         kvfree(out);
 993
 994         /* Update diagnostics, if any */
 995         if (priv->params_ethtool.diag_pci_enable ||
 996             priv->params_ethtool.diag_general_enable) {
 997                 error = mlx5_core_get_diagnostics_full(mdev,
 998                     priv->params_ethtool.diag_pci_enable ? &priv->params_pci : NULL,
 999                     priv->params_ethtool.diag_general_enable ? &priv->params_general : NULL);
1000                 if (error != 0)
1001                         mlx5_en_err(priv->ifp,
1002                             "Failed reading diagnostics: %d\n", error);
1003         }
1004
1005         /* Update FEC, if any */
1006         error = mlx5e_fec_update(priv);
1007         if (error != 0 && error != EOPNOTSUPP) {
1008                 mlx5_en_err(priv->ifp,
1009                     "Updating FEC failed: %d\n", error);
1010         }
1011
1012         /* Update temperature, if any */
1013         if (priv->params_ethtool.hw_num_temp != 0) {
1014                 error = mlx5e_hw_temperature_update(priv);
1015                 if (error != 0 && error != EOPNOTSUPP) {
1016                         mlx5_en_err(priv->ifp,
1017                             "Updating temperature failed: %d\n", error);
1018                 }
1019         }
1020 }
1021
1022 static void
1023 mlx5e_update_stats_work(struct work_struct *work)
1024 {
1025         struct mlx5e_priv *priv;
1026
1027         priv = container_of(work, struct mlx5e_priv, update_stats_work);
1028         PRIV_LOCK(priv);
1029         if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0 &&
1030             !test_bit(MLX5_INTERFACE_STATE_TEARDOWN, &priv->mdev->intf_state))
1031                 mlx5e_update_stats_locked(priv);
1032         PRIV_UNLOCK(priv);
1033 }
1034
1035 static void
1036 mlx5e_update_stats(void *arg)
1037 {
1038         struct mlx5e_priv *priv = arg;
1039
1040         queue_work(priv->wq, &priv->update_stats_work);
1041
1042         callout_reset(&priv->watchdog, hz / 4, &mlx5e_update_stats, priv);
1043 }
1044
1045 static void
1046 mlx5e_async_event_sub(struct mlx5e_priv *priv,
1047     enum mlx5_dev_event event)
1048 {
1049         switch (event) {
1050         case MLX5_DEV_EVENT_PORT_UP:
1051         case MLX5_DEV_EVENT_PORT_DOWN:
1052                 queue_work(priv->wq, &priv->update_carrier_work);
1053                 break;
1054
1055         default:
1056                 break;
1057         }
1058 }
1059
1060 static void
1061 mlx5e_async_event(struct mlx5_core_dev *mdev, void *vpriv,
1062     enum mlx5_dev_event event, unsigned long param)
1063 {
1064         struct mlx5e_priv *priv = vpriv;
1065
1066         mtx_lock(&priv->async_events_mtx);
1067         if (test_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state))
1068                 mlx5e_async_event_sub(priv, event);
1069         mtx_unlock(&priv->async_events_mtx);
1070 }
1071
1072 static void
1073 mlx5e_enable_async_events(struct mlx5e_priv *priv)
1074 {
1075         set_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state);
1076 }
1077
1078 static void
1079 mlx5e_disable_async_events(struct mlx5e_priv *priv)
1080 {
1081         mtx_lock(&priv->async_events_mtx);
1082         clear_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state);
1083         mtx_unlock(&priv->async_events_mtx);
1084 }
1085
1086 static void mlx5e_calibration_callout(void *arg);
1087 static int mlx5e_calibration_duration = 20;
1088 static int mlx5e_fast_calibration = 1;
1089 static int mlx5e_normal_calibration = 30;
1090
1091 static SYSCTL_NODE(_hw_mlx5, OID_AUTO, calibr, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1092     "MLX5 timestamp calibration parameters");
1093
1094 SYSCTL_INT(_hw_mlx5_calibr, OID_AUTO, duration, CTLFLAG_RWTUN,
1095     &mlx5e_calibration_duration, 0,
1096     "Duration of initial calibration");
1097 SYSCTL_INT(_hw_mlx5_calibr, OID_AUTO, fast, CTLFLAG_RWTUN,
1098     &mlx5e_fast_calibration, 0,
1099     "Recalibration interval during initial calibration");
1100 SYSCTL_INT(_hw_mlx5_calibr, OID_AUTO, normal, CTLFLAG_RWTUN,
1101     &mlx5e_normal_calibration, 0,
1102     "Recalibration interval during normal operations");
1103
1104 /*
1105  * Ignites the calibration process.
1106  */
1107 static void
1108 mlx5e_reset_calibration_callout(struct mlx5e_priv *priv)
1109 {
1110
1111         if (priv->clbr_done == 0)
1112                 mlx5e_calibration_callout(priv);
1113         else
1114                 callout_reset_sbt_curcpu(&priv->tstmp_clbr, (priv->clbr_done <
1115                     mlx5e_calibration_duration ? mlx5e_fast_calibration :
1116                     mlx5e_normal_calibration) * SBT_1S, 0,
1117                     mlx5e_calibration_callout, priv, C_DIRECT_EXEC);
1118 }
1119
1120 static uint64_t
1121 mlx5e_timespec2usec(const struct timespec *ts)
1122 {
1123
1124         return ((uint64_t)ts->tv_sec * 1000000000 + ts->tv_nsec);
1125 }
1126
1127 static uint64_t
1128 mlx5e_hw_clock(struct mlx5e_priv *priv)
1129 {
1130         struct mlx5_init_seg *iseg;
1131         uint32_t hw_h, hw_h1, hw_l;
1132
1133         iseg = priv->mdev->iseg;
1134         do {
1135                 hw_h = ioread32be(&iseg->internal_timer_h);
1136                 hw_l = ioread32be(&iseg->internal_timer_l);
1137                 hw_h1 = ioread32be(&iseg->internal_timer_h);
1138         } while (hw_h1 != hw_h);
1139         return (((uint64_t)hw_h << 32) | hw_l);
1140 }
1141
1142 /*
1143  * The calibration callout, it runs either in the context of the
1144  * thread which enables calibration, or in callout.  It takes the
1145  * snapshot of system and adapter clocks, then advances the pointers to
1146  * the calibration point to allow rx path to read the consistent data
1147  * lockless.
1148  */
1149 static void
1150 mlx5e_calibration_callout(void *arg)
1151 {
1152         struct mlx5e_priv *priv;
1153         struct mlx5e_clbr_point *next, *curr;
1154         struct timespec ts;
1155         int clbr_curr_next;
1156
1157         priv = arg;
1158         curr = &priv->clbr_points[priv->clbr_curr];
1159         clbr_curr_next = priv->clbr_curr + 1;
1160         if (clbr_curr_next >= nitems(priv->clbr_points))
1161                 clbr_curr_next = 0;
1162         next = &priv->clbr_points[clbr_curr_next];
1163
1164         next->base_prev = curr->base_curr;
1165         next->clbr_hw_prev = curr->clbr_hw_curr;
1166
1167         next->clbr_hw_curr = mlx5e_hw_clock(priv);
1168         if (((next->clbr_hw_curr - curr->clbr_hw_curr) >> MLX5E_TSTMP_PREC) ==
1169             0) {
1170                 if (priv->clbr_done != 0) {
1171                         mlx5_en_err(priv->ifp,
1172                             "HW failed tstmp frozen %#jx %#jx, disabling\n",
1173                              next->clbr_hw_curr, curr->clbr_hw_prev);
1174                         priv->clbr_done = 0;
1175                 }
1176                 atomic_store_rel_int(&curr->clbr_gen, 0);
1177                 return;
1178         }
1179
1180         nanouptime(&ts);
1181         next->base_curr = mlx5e_timespec2usec(&ts);
1182
1183         curr->clbr_gen = 0;
1184         atomic_thread_fence_rel();
1185         priv->clbr_curr = clbr_curr_next;
1186         atomic_store_rel_int(&next->clbr_gen, ++(priv->clbr_gen));
1187
1188         if (priv->clbr_done < mlx5e_calibration_duration)
1189                 priv->clbr_done++;
1190         mlx5e_reset_calibration_callout(priv);
1191 }
1192
1193 static const char *mlx5e_rq_stats_desc[] = {
1194         MLX5E_RQ_STATS(MLX5E_STATS_DESC)
1195 };
1196
1197 static int
1198 mlx5e_create_rq(struct mlx5e_channel *c,
1199     struct mlx5e_rq_param *param,
1200     struct mlx5e_rq *rq)
1201 {
1202         struct mlx5e_priv *priv = c->priv;
1203         struct mlx5_core_dev *mdev = priv->mdev;
1204         char buffer[16];
1205         void *rqc = param->rqc;
1206         void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
1207         int wq_sz;
1208         int err;
1209         int i;
1210         u32 nsegs, wqe_sz;
1211
1212         err = mlx5e_get_wqe_sz(priv, &wqe_sz, &nsegs);
1213         if (err != 0)
1214                 goto done;
1215
1216         /* Create DMA descriptor TAG */
1217         if ((err = -bus_dma_tag_create(
1218             bus_get_dma_tag(mdev->pdev->dev.bsddev),
1219             1,                          /* any alignment */
1220             0,                          /* no boundary */
1221             BUS_SPACE_MAXADDR,          /* lowaddr */
1222             BUS_SPACE_MAXADDR,          /* highaddr */
1223             NULL, NULL,                 /* filter, filterarg */
1224             nsegs * MLX5E_MAX_RX_BYTES, /* maxsize */
1225             nsegs,                      /* nsegments */
1226             nsegs * MLX5E_MAX_RX_BYTES, /* maxsegsize */
1227             0,                          /* flags */
1228             NULL, NULL,                 /* lockfunc, lockfuncarg */
1229             &rq->dma_tag)))
1230                 goto done;
1231
1232         err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
1233             &rq->wq_ctrl);
1234         if (err)
1235                 goto err_free_dma_tag;
1236
1237         rq->wq.db = &rq->wq.db[MLX5_RCV_DBR];
1238
1239         err = mlx5e_get_wqe_sz(priv, &rq->wqe_sz, &rq->nsegs);
1240         if (err != 0)
1241                 goto err_rq_wq_destroy;
1242
1243         wq_sz = mlx5_wq_ll_get_size(&rq->wq);
1244
1245         err = -tcp_lro_init_args(&rq->lro, priv->ifp, TCP_LRO_ENTRIES, wq_sz);
1246         if (err)
1247                 goto err_rq_wq_destroy;
1248
1249         rq->mbuf = malloc_domainset(wq_sz * sizeof(rq->mbuf[0]), M_MLX5EN,
1250             mlx5_dev_domainset(mdev), M_WAITOK | M_ZERO);
1251         for (i = 0; i != wq_sz; i++) {
1252                 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
1253                 int j;
1254
1255                 err = -bus_dmamap_create(rq->dma_tag, 0, &rq->mbuf[i].dma_map);
1256                 if (err != 0) {
1257                         while (i--)
1258                                 bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map);
1259                         goto err_rq_mbuf_free;
1260                 }
1261
1262                 /* set value for constant fields */
1263                 for (j = 0; j < rq->nsegs; j++)
1264                         wqe->data[j].lkey = cpu_to_be32(priv->mr.key);
1265         }
1266
1267         INIT_WORK(&rq->dim.work, mlx5e_dim_work);
1268         if (priv->params.rx_cq_moderation_mode < 2) {
1269                 rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
1270         } else {
1271                 void *cqc = container_of(param,
1272                     struct mlx5e_channel_param, rq)->rx_cq.cqc;
1273
1274                 switch (MLX5_GET(cqc, cqc, cq_period_mode)) {
1275                 case MLX5_CQ_PERIOD_MODE_START_FROM_EQE:
1276                         rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
1277                         break;
1278                 case MLX5_CQ_PERIOD_MODE_START_FROM_CQE:
1279                         rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE;
1280                         break;
1281                 default:
1282                         rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
1283                         break;
1284                 }
1285         }
1286
1287         rq->ifp = priv->ifp;
1288         rq->channel = c;
1289         rq->ix = c->ix;
1290
1291         snprintf(buffer, sizeof(buffer), "rxstat%d", c->ix);
1292         mlx5e_create_stats(&rq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
1293             buffer, mlx5e_rq_stats_desc, MLX5E_RQ_STATS_NUM,
1294             rq->stats.arg);
1295         return (0);
1296
1297 err_rq_mbuf_free:
1298         free(rq->mbuf, M_MLX5EN);
1299         tcp_lro_free(&rq->lro);
1300 err_rq_wq_destroy:
1301         mlx5_wq_destroy(&rq->wq_ctrl);
1302 err_free_dma_tag:
1303         bus_dma_tag_destroy(rq->dma_tag);
1304 done:
1305         return (err);
1306 }
1307
1308 static void
1309 mlx5e_destroy_rq(struct mlx5e_rq *rq)
1310 {
1311         int wq_sz;
1312         int i;
1313
1314         /* destroy all sysctl nodes */
1315         sysctl_ctx_free(&rq->stats.ctx);
1316
1317         /* free leftover LRO packets, if any */
1318         tcp_lro_free(&rq->lro);
1319
1320         wq_sz = mlx5_wq_ll_get_size(&rq->wq);
1321         for (i = 0; i != wq_sz; i++) {
1322                 if (rq->mbuf[i].mbuf != NULL) {
1323                         bus_dmamap_unload(rq->dma_tag, rq->mbuf[i].dma_map);
1324                         m_freem(rq->mbuf[i].mbuf);
1325                 }
1326                 bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map);
1327         }
1328         free(rq->mbuf, M_MLX5EN);
1329         mlx5_wq_destroy(&rq->wq_ctrl);
1330         bus_dma_tag_destroy(rq->dma_tag);
1331 }
1332
1333 static int
1334 mlx5e_enable_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
1335 {
1336         struct mlx5e_channel *c = rq->channel;
1337         struct mlx5e_priv *priv = c->priv;
1338         struct mlx5_core_dev *mdev = priv->mdev;
1339         void *in;
1340         void *rqc;
1341         void *wq;
1342         int inlen;
1343         int err;
1344         u8 ts_format;
1345
1346         inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
1347             sizeof(u64) * rq->wq_ctrl.buf.npages;
1348         in = mlx5_vzalloc(inlen);
1349         if (in == NULL)
1350                 return (-ENOMEM);
1351
1352         ts_format = mlx5_get_rq_default_ts(mdev);
1353         rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
1354         wq = MLX5_ADDR_OF(rqc, rqc, wq);
1355
1356         memcpy(rqc, param->rqc, sizeof(param->rqc));
1357
1358         MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
1359         MLX5_SET(rqc, rqc, ts_format, ts_format);
1360         MLX5_SET(rqc, rqc, flush_in_error_en, 1);
1361         if (priv->counter_set_id >= 0)
1362                 MLX5_SET(rqc, rqc, counter_set_id, priv->counter_set_id);
1363         MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift -
1364             MLX5_ADAPTER_PAGE_SHIFT);
1365         MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma);
1366
1367         mlx5_fill_page_array(&rq->wq_ctrl.buf,
1368             (__be64 *) MLX5_ADDR_OF(wq, wq, pas));
1369
1370         err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
1371
1372         kvfree(in);
1373
1374         return (err);
1375 }
1376
1377 static int
1378 mlx5e_modify_rq(struct mlx5e_rq *rq, int curr_state, int next_state)
1379 {
1380         struct mlx5e_channel *c = rq->channel;
1381         struct mlx5e_priv *priv = c->priv;
1382         struct mlx5_core_dev *mdev = priv->mdev;
1383
1384         void *in;
1385         void *rqc;
1386         int inlen;
1387         int err;
1388
1389         inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1390         in = mlx5_vzalloc(inlen);
1391         if (in == NULL)
1392                 return (-ENOMEM);
1393
1394         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
1395
1396         MLX5_SET(modify_rq_in, in, rqn, rq->rqn);
1397         MLX5_SET(modify_rq_in, in, rq_state, curr_state);
1398         MLX5_SET(rqc, rqc, state, next_state);
1399
1400         err = mlx5_core_modify_rq(mdev, in, inlen);
1401
1402         kvfree(in);
1403
1404         return (err);
1405 }
1406
1407 static void
1408 mlx5e_disable_rq(struct mlx5e_rq *rq)
1409 {
1410         struct mlx5e_channel *c = rq->channel;
1411         struct mlx5e_priv *priv = c->priv;
1412         struct mlx5_core_dev *mdev = priv->mdev;
1413
1414         mlx5_core_destroy_rq(mdev, rq->rqn);
1415 }
1416
1417 static int
1418 mlx5e_open_rq(struct mlx5e_channel *c,
1419     struct mlx5e_rq_param *param,
1420     struct mlx5e_rq *rq)
1421 {
1422         int err;
1423
1424         err = mlx5e_create_rq(c, param, rq);
1425         if (err)
1426                 return (err);
1427
1428         /* set CQN in RQ parameters */
1429         MLX5_SET(rqc, param->rqc, cqn, c->rq.cq.mcq.cqn);
1430
1431         err = mlx5e_enable_rq(rq, param);
1432         if (err)
1433                 goto err_destroy_rq;
1434
1435         err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
1436         if (err)
1437                 goto err_disable_rq;
1438
1439         c->rq.enabled = 1;
1440
1441         return (0);
1442
1443 err_disable_rq:
1444         mlx5e_disable_rq(rq);
1445 err_destroy_rq:
1446         mlx5e_destroy_rq(rq);
1447
1448         return (err);
1449 }
1450
1451 static void
1452 mlx5e_close_rq(struct mlx5e_rq *rq)
1453 {
1454         mtx_lock(&rq->mtx);
1455         rq->enabled = 0;
1456         callout_stop(&rq->watchdog);
1457         mtx_unlock(&rq->mtx);
1458
1459         mlx5e_modify_rq(rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
1460 }
1461
1462 static void
1463 mlx5e_close_rq_wait(struct mlx5e_rq *rq)
1464 {
1465
1466         mlx5e_disable_rq(rq);
1467         mlx5e_close_cq(&rq->cq);
1468         cancel_work_sync(&rq->dim.work);
1469         mlx5e_destroy_rq(rq);
1470 }
1471
1472 /*
1473  * What is a drop RQ and why is it needed?
1474  *
1475  * The RSS indirection table, also called the RQT, selects the
1476  * destination RQ based on the receive queue number, RQN. The RQT is
1477  * frequently referred to by flow steering rules to distribute traffic
1478  * among multiple RQs. The problem is that the RQs cannot be destroyed
1479  * before the RQT referring them is destroyed too. Further, TLS RX
1480  * rules may still be referring to the RQT even if the link went
1481  * down. Because there is no magic RQN for dropping packets, we create
1482  * a dummy RQ, also called drop RQ, which sole purpose is to drop all
1483  * received packets. When the link goes down this RQN is filled in all
1484  * RQT entries, of the main RQT, so the real RQs which are about to be
1485  * destroyed can be released and the TLS RX rules can be sustained.
1486  */
1487 static void
1488 mlx5e_open_drop_rq_comp(struct mlx5_core_cq *mcq __unused, struct mlx5_eqe *eqe __unused)
1489 {
1490 }
1491
1492 static int
1493 mlx5e_open_drop_rq(struct mlx5e_priv *priv,
1494     struct mlx5e_rq *drop_rq)
1495 {
1496         struct mlx5e_cq_param param_cq = {};
1497         struct mlx5e_rq_param param_rq = {};
1498         void *rqc_wq = MLX5_ADDR_OF(rqc, param_rq.rqc, wq);
1499         int err;
1500
1501         /* set channel pointer */
1502         drop_rq->channel = priv->channel;
1503
1504         /* set basic CQ parameters needed */
1505         MLX5_SET(cqc, param_cq.cqc, log_cq_size, 0);
1506         MLX5_SET(cqc, param_cq.cqc, uar_page, priv->mdev->priv.uar->index);
1507
1508         /* open receive completion queue */
1509         err = mlx5e_open_cq(priv, &param_cq, &drop_rq->cq,
1510             &mlx5e_open_drop_rq_comp, 0);
1511         if (err)
1512                 goto err_done;
1513
1514         /* set basic WQ parameters needed */
1515         MLX5_SET(wq, rqc_wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
1516         MLX5_SET(wq, rqc_wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
1517         MLX5_SET(wq, rqc_wq, log_wq_stride, ilog2(sizeof(struct mlx5e_rx_wqe) + sizeof(struct mlx5_wqe_data_seg)));
1518         MLX5_SET(wq, rqc_wq, log_wq_sz, 0);
1519         MLX5_SET(wq, rqc_wq, pd, priv->pdn);
1520
1521         param_rq.wq.linear = 1;
1522
1523         err = mlx5_wq_ll_create(priv->mdev, &param_rq.wq, rqc_wq, &drop_rq->wq,
1524             &drop_rq->wq_ctrl);
1525         if (err)
1526                 goto err_close_cq;
1527
1528         /* set CQN in RQ parameters */
1529         MLX5_SET(rqc, param_rq.rqc, cqn, drop_rq->cq.mcq.cqn);
1530
1531         err = mlx5e_enable_rq(drop_rq, &param_rq);
1532         if (err)
1533                 goto err_wq_destroy;
1534
1535         err = mlx5e_modify_rq(drop_rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
1536         if (err)
1537                 goto err_disable_rq;
1538
1539         return (err);
1540
1541 err_disable_rq:
1542         mlx5e_disable_rq(drop_rq);
1543 err_wq_destroy:
1544         mlx5_wq_destroy(&drop_rq->wq_ctrl);
1545 err_close_cq:
1546         mlx5e_close_cq(&drop_rq->cq);
1547 err_done:
1548         return (err);
1549 }
1550
1551 static void
1552 mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq)
1553 {
1554         mlx5e_modify_rq(drop_rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
1555         mlx5e_disable_rq(drop_rq);
1556         mlx5_wq_destroy(&drop_rq->wq_ctrl);
1557         mlx5e_close_cq(&drop_rq->cq);
1558 }
1559
1560 void
1561 mlx5e_free_sq_db(struct mlx5e_sq *sq)
1562 {
1563         int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1564         int x;
1565
1566         for (x = 0; x != wq_sz; x++) {
1567                 if (sq->mbuf[x].mbuf != NULL) {
1568                         bus_dmamap_unload(sq->dma_tag, sq->mbuf[x].dma_map);
1569                         m_freem(sq->mbuf[x].mbuf);
1570                 }
1571                 if (sq->mbuf[x].mst != NULL) {
1572                         m_snd_tag_rele(sq->mbuf[x].mst);
1573                         sq->mbuf[x].mst = NULL;
1574                 }
1575                 bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map);
1576         }
1577         free(sq->mbuf, M_MLX5EN);
1578 }
1579
1580 int
1581 mlx5e_alloc_sq_db(struct mlx5e_sq *sq)
1582 {
1583         int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1584         int err;
1585         int x;
1586
1587         sq->mbuf = malloc_domainset(wq_sz * sizeof(sq->mbuf[0]), M_MLX5EN,
1588             mlx5_dev_domainset(sq->priv->mdev), M_WAITOK | M_ZERO);
1589
1590         /* Create DMA descriptor MAPs */
1591         for (x = 0; x != wq_sz; x++) {
1592                 err = -bus_dmamap_create(sq->dma_tag, 0, &sq->mbuf[x].dma_map);
1593                 if (err != 0) {
1594                         while (x--)
1595                                 bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map);
1596                         free(sq->mbuf, M_MLX5EN);
1597                         return (err);
1598                 }
1599         }
1600         return (0);
1601 }
1602
1603 static const char *mlx5e_sq_stats_desc[] = {
1604         MLX5E_SQ_STATS(MLX5E_STATS_DESC)
1605 };
1606
1607 void
1608 mlx5e_update_sq_inline(struct mlx5e_sq *sq)
1609 {
1610         sq->max_inline = sq->priv->params.tx_max_inline;
1611         sq->min_inline_mode = sq->priv->params.tx_min_inline_mode;
1612
1613         /*
1614          * Check if trust state is DSCP or if inline mode is NONE which
1615          * indicates CX-5 or newer hardware.
1616          */
1617         if (sq->priv->params_ethtool.trust_state != MLX5_QPTS_TRUST_PCP ||
1618             sq->min_inline_mode == MLX5_INLINE_MODE_NONE) {
1619                 if (MLX5_CAP_ETH(sq->priv->mdev, wqe_vlan_insert))
1620                         sq->min_insert_caps = MLX5E_INSERT_VLAN | MLX5E_INSERT_NON_VLAN;
1621                 else
1622                         sq->min_insert_caps = MLX5E_INSERT_NON_VLAN;
1623         } else {
1624                 sq->min_insert_caps = 0;
1625         }
1626 }
1627
1628 static void
1629 mlx5e_refresh_sq_inline_sub(struct mlx5e_priv *priv, struct mlx5e_channel *c)
1630 {
1631         int i;
1632
1633         for (i = 0; i != priv->num_tc; i++) {
1634                 mtx_lock(&c->sq[i].lock);
1635                 mlx5e_update_sq_inline(&c->sq[i]);
1636                 mtx_unlock(&c->sq[i].lock);
1637         }
1638 }
1639
1640 void
1641 mlx5e_refresh_sq_inline(struct mlx5e_priv *priv)
1642 {
1643         int i;
1644
1645         /* check if channels are closed */
1646         if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
1647                 return;
1648
1649         for (i = 0; i < priv->params.num_channels; i++)
1650                 mlx5e_refresh_sq_inline_sub(priv, &priv->channel[i]);
1651 }
1652
1653 static int
1654 mlx5e_create_sq(struct mlx5e_channel *c,
1655     int tc,
1656     struct mlx5e_sq_param *param,
1657     struct mlx5e_sq *sq)
1658 {
1659         struct mlx5e_priv *priv = c->priv;
1660         struct mlx5_core_dev *mdev = priv->mdev;
1661         char buffer[16];
1662         void *sqc = param->sqc;
1663         void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq);
1664         int err;
1665
1666         /* Create DMA descriptor TAG */
1667         if ((err = -bus_dma_tag_create(
1668             bus_get_dma_tag(mdev->pdev->dev.bsddev),
1669             1,                          /* any alignment */
1670             0,                          /* no boundary */
1671             BUS_SPACE_MAXADDR,          /* lowaddr */
1672             BUS_SPACE_MAXADDR,          /* highaddr */
1673             NULL, NULL,                 /* filter, filterarg */
1674             MLX5E_MAX_TX_PAYLOAD_SIZE,  /* maxsize */
1675             MLX5E_MAX_TX_MBUF_FRAGS,    /* nsegments */
1676             MLX5E_MAX_TX_MBUF_SIZE,     /* maxsegsize */
1677             0,                          /* flags */
1678             NULL, NULL,                 /* lockfunc, lockfuncarg */
1679             &sq->dma_tag)))
1680                 goto done;
1681
1682         sq->mkey_be = cpu_to_be32(priv->mr.key);
1683         sq->ifp = priv->ifp;
1684         sq->priv = priv;
1685         sq->tc = tc;
1686
1687         err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq,
1688             &sq->wq_ctrl);
1689         if (err)
1690                 goto err_free_dma_tag;
1691
1692         sq->wq.db = &sq->wq.db[MLX5_SND_DBR];
1693
1694         err = mlx5e_alloc_sq_db(sq);
1695         if (err)
1696                 goto err_sq_wq_destroy;
1697
1698         mlx5e_update_sq_inline(sq);
1699
1700         snprintf(buffer, sizeof(buffer), "txstat%dtc%d", c->ix, tc);
1701         mlx5e_create_stats(&sq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
1702             buffer, mlx5e_sq_stats_desc, MLX5E_SQ_STATS_NUM,
1703             sq->stats.arg);
1704
1705         return (0);
1706
1707 err_sq_wq_destroy:
1708         mlx5_wq_destroy(&sq->wq_ctrl);
1709
1710 err_free_dma_tag:
1711         bus_dma_tag_destroy(sq->dma_tag);
1712 done:
1713         return (err);
1714 }
1715
1716 static void
1717 mlx5e_destroy_sq(struct mlx5e_sq *sq)
1718 {
1719         /* destroy all sysctl nodes */
1720         sysctl_ctx_free(&sq->stats.ctx);
1721
1722         mlx5e_free_sq_db(sq);
1723         mlx5_wq_destroy(&sq->wq_ctrl);
1724         bus_dma_tag_destroy(sq->dma_tag);
1725 }
1726
1727 int
1728 mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param,
1729     const struct mlx5_sq_bfreg *bfreg, int tis_num)
1730 {
1731         void *in;
1732         void *sqc;
1733         void *wq;
1734         int inlen;
1735         int err;
1736         u8 ts_format;
1737
1738         inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
1739             sizeof(u64) * sq->wq_ctrl.buf.npages;
1740         in = mlx5_vzalloc(inlen);
1741         if (in == NULL)
1742                 return (-ENOMEM);
1743
1744         sq->uar_map = bfreg->map;
1745
1746         ts_format = mlx5_get_sq_default_ts(sq->priv->mdev);
1747         sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1748         wq = MLX5_ADDR_OF(sqc, sqc, wq);
1749
1750         memcpy(sqc, param->sqc, sizeof(param->sqc));
1751
1752         MLX5_SET(sqc, sqc, tis_num_0, tis_num);
1753         MLX5_SET(sqc, sqc, cqn, sq->cq.mcq.cqn);
1754         MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
1755         MLX5_SET(sqc, sqc, ts_format, ts_format);
1756         MLX5_SET(sqc, sqc, tis_lst_sz, 1);
1757         MLX5_SET(sqc, sqc, flush_in_error_en, 1);
1758         MLX5_SET(sqc, sqc, allow_swp, 1);
1759
1760         MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
1761         MLX5_SET(wq, wq, uar_page, bfreg->index);
1762         MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift -
1763             MLX5_ADAPTER_PAGE_SHIFT);
1764         MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma);
1765
1766         mlx5_fill_page_array(&sq->wq_ctrl.buf,
1767             (__be64 *) MLX5_ADDR_OF(wq, wq, pas));
1768
1769         err = mlx5_core_create_sq(sq->priv->mdev, in, inlen, &sq->sqn);
1770
1771         kvfree(in);
1772
1773         return (err);
1774 }
1775
1776 int
1777 mlx5e_modify_sq(struct mlx5e_sq *sq, int curr_state, int next_state)
1778 {
1779         void *in;
1780         void *sqc;
1781         int inlen;
1782         int err;
1783
1784         inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1785         in = mlx5_vzalloc(inlen);
1786         if (in == NULL)
1787                 return (-ENOMEM);
1788
1789         sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
1790
1791         MLX5_SET(modify_sq_in, in, sqn, sq->sqn);
1792         MLX5_SET(modify_sq_in, in, sq_state, curr_state);
1793         MLX5_SET(sqc, sqc, state, next_state);
1794
1795         err = mlx5_core_modify_sq(sq->priv->mdev, in, inlen);
1796
1797         kvfree(in);
1798
1799         return (err);
1800 }
1801
1802 void
1803 mlx5e_disable_sq(struct mlx5e_sq *sq)
1804 {
1805
1806         mlx5_core_destroy_sq(sq->priv->mdev, sq->sqn);
1807 }
1808
1809 static int
1810 mlx5e_open_sq(struct mlx5e_channel *c,
1811     int tc,
1812     struct mlx5e_sq_param *param,
1813     struct mlx5e_sq *sq)
1814 {
1815         int err;
1816
1817         sq->cev_factor = c->priv->params_ethtool.tx_completion_fact;
1818
1819         /* ensure the TX completion event factor is not zero */
1820         if (sq->cev_factor == 0)
1821                 sq->cev_factor = 1;
1822
1823         err = mlx5e_create_sq(c, tc, param, sq);
1824         if (err)
1825                 return (err);
1826
1827         err = mlx5e_enable_sq(sq, param, &c->bfreg, c->priv->tisn[tc]);
1828         if (err)
1829                 goto err_destroy_sq;
1830
1831         err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY);
1832         if (err)
1833                 goto err_disable_sq;
1834
1835         WRITE_ONCE(sq->running, 1);
1836
1837         return (0);
1838
1839 err_disable_sq:
1840         mlx5e_disable_sq(sq);
1841 err_destroy_sq:
1842         mlx5e_destroy_sq(sq);
1843
1844         return (err);
1845 }
1846
1847 static void
1848 mlx5e_sq_send_nops_locked(struct mlx5e_sq *sq, int can_sleep)
1849 {
1850         /* fill up remainder with NOPs */
1851         while (sq->cev_counter != 0) {
1852                 while (!mlx5e_sq_has_room_for(sq, 1)) {
1853                         if (can_sleep != 0) {
1854                                 mtx_unlock(&sq->lock);
1855                                 msleep(4);
1856                                 mtx_lock(&sq->lock);
1857                         } else {
1858                                 goto done;
1859                         }
1860                 }
1861                 /* send a single NOP */
1862                 mlx5e_send_nop(sq, 1);
1863                 atomic_thread_fence_rel();
1864         }
1865 done:
1866         mlx5e_tx_notify_hw(sq, false);
1867 }
1868
1869 void
1870 mlx5e_sq_cev_timeout(void *arg)
1871 {
1872         struct mlx5e_sq *sq = arg;
1873
1874         mtx_assert(&sq->lock, MA_OWNED);
1875
1876         /* check next state */
1877         switch (sq->cev_next_state) {
1878         case MLX5E_CEV_STATE_SEND_NOPS:
1879                 /* fill TX ring with NOPs, if any */
1880                 mlx5e_sq_send_nops_locked(sq, 0);
1881
1882                 /* check if completed */
1883                 if (sq->cev_counter == 0) {
1884                         sq->cev_next_state = MLX5E_CEV_STATE_INITIAL;
1885                         return;
1886                 }
1887                 break;
1888         default:
1889                 /* send NOPs on next timeout */
1890                 sq->cev_next_state = MLX5E_CEV_STATE_SEND_NOPS;
1891                 break;
1892         }
1893
1894         /* restart timer */
1895         callout_reset_curcpu(&sq->cev_callout, hz, mlx5e_sq_cev_timeout, sq);
1896 }
1897
1898 void
1899 mlx5e_drain_sq(struct mlx5e_sq *sq)
1900 {
1901         int error;
1902         struct mlx5_core_dev *mdev= sq->priv->mdev;
1903
1904         /*
1905          * Check if already stopped.
1906          *
1907          * NOTE: Serialization of this function is managed by the
1908          * caller ensuring the priv's state lock is locked or in case
1909          * of rate limit support, a single thread manages drain and
1910          * resume of SQs. The "running" variable can therefore safely
1911          * be read without any locks.
1912          */
1913         if (READ_ONCE(sq->running) == 0)
1914                 return;
1915
1916         /* don't put more packets into the SQ */
1917         WRITE_ONCE(sq->running, 0);
1918
1919         /* serialize access to DMA rings */
1920         mtx_lock(&sq->lock);
1921
1922         /* teardown event factor timer, if any */
1923         sq->cev_next_state = MLX5E_CEV_STATE_HOLD_NOPS;
1924         callout_stop(&sq->cev_callout);
1925
1926         /* send dummy NOPs in order to flush the transmit ring */
1927         mlx5e_sq_send_nops_locked(sq, 1);
1928         mtx_unlock(&sq->lock);
1929
1930         /* wait till SQ is empty or link is down */
1931         mtx_lock(&sq->lock);
1932         while (sq->cc != sq->pc &&
1933             (sq->priv->media_status_last & IFM_ACTIVE) != 0 &&
1934             mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR &&
1935             pci_channel_offline(mdev->pdev) == 0) {
1936                 mtx_unlock(&sq->lock);
1937                 msleep(1);
1938                 sq->cq.mcq.comp(&sq->cq.mcq, NULL);
1939                 mtx_lock(&sq->lock);
1940         }
1941         mtx_unlock(&sq->lock);
1942
1943         /* error out remaining requests */
1944         error = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY, MLX5_SQC_STATE_ERR);
1945         if (error != 0) {
1946                 mlx5_en_err(sq->ifp,
1947                     "mlx5e_modify_sq() from RDY to ERR failed: %d\n", error);
1948         }
1949
1950         /* wait till SQ is empty */
1951         mtx_lock(&sq->lock);
1952         while (sq->cc != sq->pc &&
1953                mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR &&
1954                pci_channel_offline(mdev->pdev) == 0) {
1955                 mtx_unlock(&sq->lock);
1956                 msleep(1);
1957                 sq->cq.mcq.comp(&sq->cq.mcq, NULL);
1958                 mtx_lock(&sq->lock);
1959         }
1960         mtx_unlock(&sq->lock);
1961 }
1962
1963 static void
1964 mlx5e_close_sq_wait(struct mlx5e_sq *sq)
1965 {
1966
1967         mlx5e_drain_sq(sq);
1968         mlx5e_disable_sq(sq);
1969         mlx5e_destroy_sq(sq);
1970 }
1971
1972 static int
1973 mlx5e_create_cq(struct mlx5e_priv *priv,
1974     struct mlx5e_cq_param *param,
1975     struct mlx5e_cq *cq,
1976     mlx5e_cq_comp_t *comp,
1977     int eq_ix)
1978 {
1979         struct mlx5_core_dev *mdev = priv->mdev;
1980         struct mlx5_core_cq *mcq = &cq->mcq;
1981         int eqn_not_used;
1982         int irqn;
1983         int err;
1984         u32 i;
1985
1986         err = mlx5_vector2eqn(mdev, eq_ix, &eqn_not_used, &irqn);
1987         if (err)
1988                 return (err);
1989
1990         err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
1991             &cq->wq_ctrl);
1992         if (err)
1993                 return (err);
1994
1995         mcq->cqe_sz = 64;
1996         mcq->set_ci_db = cq->wq_ctrl.db.db;
1997         mcq->arm_db = cq->wq_ctrl.db.db + 1;
1998         *mcq->set_ci_db = 0;
1999         *mcq->arm_db = 0;
2000         mcq->vector = eq_ix;
2001         mcq->comp = comp;
2002         mcq->event = mlx5e_cq_error_event;
2003         mcq->irqn = irqn;
2004
2005         for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
2006                 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
2007
2008                 cqe->op_own = 0xf1;
2009         }
2010
2011         cq->priv = priv;
2012
2013         return (0);
2014 }
2015
2016 static void
2017 mlx5e_destroy_cq(struct mlx5e_cq *cq)
2018 {
2019         mlx5_wq_destroy(&cq->wq_ctrl);
2020 }
2021
2022 static int
2023 mlx5e_enable_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param, int eq_ix)
2024 {
2025         struct mlx5_core_cq *mcq = &cq->mcq;
2026         u32 out[MLX5_ST_SZ_DW(create_cq_out)];
2027         void *in;
2028         void *cqc;
2029         int inlen;
2030         int irqn_not_used;
2031         int eqn;
2032         int err;
2033
2034         inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
2035             sizeof(u64) * cq->wq_ctrl.buf.npages;
2036         in = mlx5_vzalloc(inlen);
2037         if (in == NULL)
2038                 return (-ENOMEM);
2039
2040         cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
2041
2042         memcpy(cqc, param->cqc, sizeof(param->cqc));
2043
2044         mlx5_fill_page_array(&cq->wq_ctrl.buf,
2045             (__be64 *) MLX5_ADDR_OF(create_cq_in, in, pas));
2046
2047         mlx5_vector2eqn(cq->priv->mdev, eq_ix, &eqn, &irqn_not_used);
2048
2049         MLX5_SET(cqc, cqc, c_eqn, eqn);
2050         MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
2051             MLX5_ADAPTER_PAGE_SHIFT);
2052         MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma);
2053
2054         err = mlx5_core_create_cq(cq->priv->mdev, mcq, in, inlen, out, sizeof(out));
2055
2056         kvfree(in);
2057
2058         if (err)
2059                 return (err);
2060
2061         mlx5e_cq_arm(cq, MLX5_GET_DOORBELL_LOCK(&cq->priv->doorbell_lock));
2062
2063         return (0);
2064 }
2065
2066 static void
2067 mlx5e_disable_cq(struct mlx5e_cq *cq)
2068 {
2069
2070         mlx5_core_destroy_cq(cq->priv->mdev, &cq->mcq);
2071 }
2072
2073 int
2074 mlx5e_open_cq(struct mlx5e_priv *priv,
2075     struct mlx5e_cq_param *param,
2076     struct mlx5e_cq *cq,
2077     mlx5e_cq_comp_t *comp,
2078     int eq_ix)
2079 {
2080         int err;
2081
2082         err = mlx5e_create_cq(priv, param, cq, comp, eq_ix);
2083         if (err)
2084                 return (err);
2085
2086         err = mlx5e_enable_cq(cq, param, eq_ix);
2087         if (err)
2088                 goto err_destroy_cq;
2089
2090         return (0);
2091
2092 err_destroy_cq:
2093         mlx5e_destroy_cq(cq);
2094
2095         return (err);
2096 }
2097
2098 void
2099 mlx5e_close_cq(struct mlx5e_cq *cq)
2100 {
2101         mlx5e_disable_cq(cq);
2102         mlx5e_destroy_cq(cq);
2103 }
2104
2105 static int
2106 mlx5e_open_tx_cqs(struct mlx5e_channel *c,
2107     struct mlx5e_channel_param *cparam)
2108 {
2109         int err;
2110         int tc;
2111
2112         for (tc = 0; tc < c->priv->num_tc; tc++) {
2113                 /* open completion queue */
2114                 err = mlx5e_open_cq(c->priv, &cparam->tx_cq, &c->sq[tc].cq,
2115                     &mlx5e_tx_cq_comp, c->ix);
2116                 if (err)
2117                         goto err_close_tx_cqs;
2118         }
2119         return (0);
2120
2121 err_close_tx_cqs:
2122         for (tc--; tc >= 0; tc--)
2123                 mlx5e_close_cq(&c->sq[tc].cq);
2124
2125         return (err);
2126 }
2127
2128 static void
2129 mlx5e_close_tx_cqs(struct mlx5e_channel *c)
2130 {
2131         int tc;
2132
2133         for (tc = 0; tc < c->priv->num_tc; tc++)
2134                 mlx5e_close_cq(&c->sq[tc].cq);
2135 }
2136
2137 static int
2138 mlx5e_open_sqs(struct mlx5e_channel *c,
2139     struct mlx5e_channel_param *cparam)
2140 {
2141         int err;
2142         int tc;
2143
2144         for (tc = 0; tc < c->priv->num_tc; tc++) {
2145                 err = mlx5e_open_sq(c, tc, &cparam->sq, &c->sq[tc]);
2146                 if (err)
2147                         goto err_close_sqs;
2148         }
2149
2150         return (0);
2151
2152 err_close_sqs:
2153         for (tc--; tc >= 0; tc--)
2154                 mlx5e_close_sq_wait(&c->sq[tc]);
2155
2156         return (err);
2157 }
2158
2159 static void
2160 mlx5e_close_sqs_wait(struct mlx5e_channel *c)
2161 {
2162         int tc;
2163
2164         for (tc = 0; tc < c->priv->num_tc; tc++)
2165                 mlx5e_close_sq_wait(&c->sq[tc]);
2166 }
2167
2168 static void
2169 mlx5e_chan_static_init(struct mlx5e_priv *priv, struct mlx5e_channel *c, int ix)
2170 {
2171         int tc;
2172
2173         /* setup priv and channel number */
2174         c->priv = priv;
2175         c->ix = ix;
2176
2177         /* setup send tag */
2178         m_snd_tag_init(&c->tag, c->priv->ifp, &mlx5e_ul_snd_tag_sw);
2179
2180         init_completion(&c->completion);
2181
2182         mtx_init(&c->rq.mtx, "mlx5rx", MTX_NETWORK_LOCK, MTX_DEF);
2183
2184         callout_init_mtx(&c->rq.watchdog, &c->rq.mtx, 0);
2185
2186         for (tc = 0; tc != MLX5E_MAX_TX_NUM_TC; tc++) {
2187                 struct mlx5e_sq *sq = c->sq + tc;
2188
2189                 mtx_init(&sq->lock, "mlx5tx",
2190                     MTX_NETWORK_LOCK " TX", MTX_DEF);
2191                 mtx_init(&sq->comp_lock, "mlx5comp",
2192                     MTX_NETWORK_LOCK " TX", MTX_DEF);
2193
2194                 callout_init_mtx(&sq->cev_callout, &sq->lock, 0);
2195         }
2196
2197         mlx5e_iq_static_init(&c->iq);
2198 }
2199
2200 static void
2201 mlx5e_chan_wait_for_completion(struct mlx5e_channel *c)
2202 {
2203
2204         m_snd_tag_rele(&c->tag);
2205         wait_for_completion(&c->completion);
2206 }
2207
2208 static void
2209 mlx5e_priv_wait_for_completion(struct mlx5e_priv *priv, const uint32_t channels)
2210 {
2211         uint32_t x;
2212
2213         for (x = 0; x != channels; x++)
2214                 mlx5e_chan_wait_for_completion(&priv->channel[x]);
2215 }
2216
2217 static void
2218 mlx5e_chan_static_destroy(struct mlx5e_channel *c)
2219 {
2220         int tc;
2221
2222         callout_drain(&c->rq.watchdog);
2223
2224         mtx_destroy(&c->rq.mtx);
2225
2226         for (tc = 0; tc != MLX5E_MAX_TX_NUM_TC; tc++) {
2227                 callout_drain(&c->sq[tc].cev_callout);
2228                 mtx_destroy(&c->sq[tc].lock);
2229                 mtx_destroy(&c->sq[tc].comp_lock);
2230         }
2231
2232         mlx5e_iq_static_destroy(&c->iq);
2233 }
2234
2235 static int
2236 mlx5e_open_channel(struct mlx5e_priv *priv,
2237     struct mlx5e_channel_param *cparam,
2238     struct mlx5e_channel *c)
2239 {
2240         struct epoch_tracker et;
2241         int i, err;
2242
2243         /* zero non-persistent data */
2244         MLX5E_ZERO(&c->rq, mlx5e_rq_zero_start);
2245         for (i = 0; i != priv->num_tc; i++)
2246                 MLX5E_ZERO(&c->sq[i], mlx5e_sq_zero_start);
2247         MLX5E_ZERO(&c->iq, mlx5e_iq_zero_start);
2248
2249         /* open transmit completion queue */
2250         err = mlx5e_open_tx_cqs(c, cparam);
2251         if (err)
2252                 goto err_free;
2253
2254         /* open receive completion queue */
2255         err = mlx5e_open_cq(c->priv, &cparam->rx_cq, &c->rq.cq,
2256             &mlx5e_rx_cq_comp, c->ix);
2257         if (err)
2258                 goto err_close_tx_cqs;
2259
2260         err = mlx5e_open_sqs(c, cparam);
2261         if (err)
2262                 goto err_close_rx_cq;
2263
2264         err = mlx5e_iq_open(c, &cparam->sq, &cparam->tx_cq, &c->iq);
2265         if (err)
2266                 goto err_close_sqs;
2267
2268         err = mlx5e_open_rq(c, &cparam->rq, &c->rq);
2269         if (err)
2270                 goto err_close_iq;
2271
2272         /* poll receive queue initially */
2273         NET_EPOCH_ENTER(et);
2274         c->rq.cq.mcq.comp(&c->rq.cq.mcq, NULL);
2275         NET_EPOCH_EXIT(et);
2276
2277         return (0);
2278
2279 err_close_iq:
2280         mlx5e_iq_close(&c->iq);
2281
2282 err_close_sqs:
2283         mlx5e_close_sqs_wait(c);
2284
2285 err_close_rx_cq:
2286         mlx5e_close_cq(&c->rq.cq);
2287
2288 err_close_tx_cqs:
2289         mlx5e_close_tx_cqs(c);
2290
2291 err_free:
2292         return (err);
2293 }
2294
2295 static void
2296 mlx5e_close_channel(struct mlx5e_channel *c)
2297 {
2298         mlx5e_close_rq(&c->rq);
2299 }
2300
2301 static void
2302 mlx5e_close_channel_wait(struct mlx5e_channel *c)
2303 {
2304         mlx5e_close_rq_wait(&c->rq);
2305         mlx5e_iq_close(&c->iq);
2306         mlx5e_close_sqs_wait(c);
2307         mlx5e_close_tx_cqs(c);
2308 }
2309
2310 static int
2311 mlx5e_get_wqe_sz(struct mlx5e_priv *priv, u32 *wqe_sz, u32 *nsegs)
2312 {
2313         u32 r, n;
2314
2315         r = priv->params.hw_lro_en ? priv->params.lro_wqe_sz :
2316             MLX5E_SW2MB_MTU(if_getmtu(priv->ifp));
2317         if (r > MJUM16BYTES)
2318                 return (-ENOMEM);
2319
2320         if (r > MJUM9BYTES)
2321                 r = MJUM16BYTES;
2322         else if (r > MJUMPAGESIZE)
2323                 r = MJUM9BYTES;
2324         else if (r > MCLBYTES)
2325                 r = MJUMPAGESIZE;
2326         else
2327                 r = MCLBYTES;
2328
2329         /*
2330          * n + 1 must be a power of two, because stride size must be.
2331          * Stride size is 16 * (n + 1), as the first segment is
2332          * control.
2333          */
2334         for (n = howmany(r, MLX5E_MAX_RX_BYTES); !powerof2(n + 1); n++)
2335                 ;
2336
2337         if (n > MLX5E_MAX_BUSDMA_RX_SEGS)
2338                 return (-ENOMEM);
2339
2340         *wqe_sz = r;
2341         *nsegs = n;
2342         return (0);
2343 }
2344
2345 static void
2346 mlx5e_build_rq_param(struct mlx5e_priv *priv,
2347     struct mlx5e_rq_param *param)
2348 {
2349         void *rqc = param->rqc;
2350         void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
2351         u32 wqe_sz, nsegs;
2352
2353         mlx5e_get_wqe_sz(priv, &wqe_sz, &nsegs);
2354         MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
2355         MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
2356         MLX5_SET(wq, wq, log_wq_stride, ilog2(sizeof(struct mlx5e_rx_wqe) +
2357             nsegs * sizeof(struct mlx5_wqe_data_seg)));
2358         MLX5_SET(wq, wq, log_wq_sz, priv->params.log_rq_size);
2359         MLX5_SET(wq, wq, pd, priv->pdn);
2360
2361         param->wq.linear = 1;
2362 }
2363
2364 static void
2365 mlx5e_build_sq_param(struct mlx5e_priv *priv,
2366     struct mlx5e_sq_param *param)
2367 {
2368         void *sqc = param->sqc;
2369         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
2370
2371         MLX5_SET(wq, wq, log_wq_sz, priv->params.log_sq_size);
2372         MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
2373         MLX5_SET(wq, wq, pd, priv->pdn);
2374
2375         param->wq.linear = 1;
2376 }
2377
2378 static void
2379 mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
2380     struct mlx5e_cq_param *param)
2381 {
2382         void *cqc = param->cqc;
2383
2384         MLX5_SET(cqc, cqc, uar_page, priv->mdev->priv.uar->index);
2385 }
2386
2387 static void
2388 mlx5e_get_default_profile(struct mlx5e_priv *priv, int mode, struct net_dim_cq_moder *ptr)
2389 {
2390
2391         *ptr = net_dim_get_profile(mode, MLX5E_DIM_DEFAULT_PROFILE);
2392
2393         /* apply LRO restrictions */
2394         if (priv->params.hw_lro_en &&
2395             ptr->pkts > MLX5E_DIM_MAX_RX_CQ_MODERATION_PKTS_WITH_LRO) {
2396                 ptr->pkts = MLX5E_DIM_MAX_RX_CQ_MODERATION_PKTS_WITH_LRO;
2397         }
2398 }
2399
2400 static void
2401 mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
2402     struct mlx5e_cq_param *param)
2403 {
2404         struct net_dim_cq_moder curr;
2405         void *cqc = param->cqc;
2406
2407         /*
2408          * We use MLX5_CQE_FORMAT_HASH because the RX hash mini CQE
2409          * format is more beneficial for FreeBSD use case.
2410          *
2411          * Adding support for MLX5_CQE_FORMAT_CSUM will require changes
2412          * in mlx5e_decompress_cqe.
2413          */
2414         if (priv->params.cqe_zipping_en) {
2415                 MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_HASH);
2416                 MLX5_SET(cqc, cqc, cqe_compression_en, 1);
2417         }
2418
2419         MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_rq_size);
2420
2421         switch (priv->params.rx_cq_moderation_mode) {
2422         case 0:
2423                 MLX5_SET(cqc, cqc, cq_period, priv->params.rx_cq_moderation_usec);
2424                 MLX5_SET(cqc, cqc, cq_max_count, priv->params.rx_cq_moderation_pkts);
2425                 MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2426                 break;
2427         case 1:
2428                 MLX5_SET(cqc, cqc, cq_period, priv->params.rx_cq_moderation_usec);
2429                 MLX5_SET(cqc, cqc, cq_max_count, priv->params.rx_cq_moderation_pkts);
2430                 if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
2431                         MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
2432                 else
2433                         MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2434                 break;
2435         case 2:
2436                 mlx5e_get_default_profile(priv, NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE, &curr);
2437                 MLX5_SET(cqc, cqc, cq_period, curr.usec);
2438                 MLX5_SET(cqc, cqc, cq_max_count, curr.pkts);
2439                 MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2440                 break;
2441         case 3:
2442                 mlx5e_get_default_profile(priv, NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE, &curr);
2443                 MLX5_SET(cqc, cqc, cq_period, curr.usec);
2444                 MLX5_SET(cqc, cqc, cq_max_count, curr.pkts);
2445                 if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
2446                         MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
2447                 else
2448                         MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2449                 break;
2450         default:
2451                 break;
2452         }
2453
2454         mlx5e_dim_build_cq_param(priv, param);
2455
2456         mlx5e_build_common_cq_param(priv, param);
2457 }
2458
2459 static void
2460 mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
2461     struct mlx5e_cq_param *param)
2462 {
2463         void *cqc = param->cqc;
2464
2465         MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_sq_size);
2466         MLX5_SET(cqc, cqc, cq_period, priv->params.tx_cq_moderation_usec);
2467         MLX5_SET(cqc, cqc, cq_max_count, priv->params.tx_cq_moderation_pkts);
2468
2469         switch (priv->params.tx_cq_moderation_mode) {
2470         case 0:
2471                 MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2472                 break;
2473         default:
2474                 if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
2475                         MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
2476                 else
2477                         MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2478                 break;
2479         }
2480
2481         mlx5e_build_common_cq_param(priv, param);
2482 }
2483
2484 static void
2485 mlx5e_build_channel_param(struct mlx5e_priv *priv,
2486     struct mlx5e_channel_param *cparam)
2487 {
2488         memset(cparam, 0, sizeof(*cparam));
2489
2490         mlx5e_build_rq_param(priv, &cparam->rq);
2491         mlx5e_build_sq_param(priv, &cparam->sq);
2492         mlx5e_build_rx_cq_param(priv, &cparam->rx_cq);
2493         mlx5e_build_tx_cq_param(priv, &cparam->tx_cq);
2494 }
2495
2496 static int
2497 mlx5e_open_channels(struct mlx5e_priv *priv)
2498 {
2499         struct mlx5e_channel_param *cparam;
2500         int err;
2501         int i;
2502
2503         cparam = malloc(sizeof(*cparam), M_MLX5EN, M_WAITOK);
2504
2505         mlx5e_build_channel_param(priv, cparam);
2506         for (i = 0; i < priv->params.num_channels; i++) {
2507                 err = mlx5e_open_channel(priv, cparam, &priv->channel[i]);
2508                 if (err)
2509                         goto err_close_channels;
2510
2511                 /* Bind interrupt vectors, if any. */
2512                 if (priv->params_ethtool.irq_cpu_base > -1) {
2513                         cpuset_t cpuset;
2514                         int cpu;
2515                         int irq;
2516                         int eqn;
2517                         int nirq;
2518
2519                         err = mlx5_vector2eqn(priv->mdev, i,
2520                             &eqn, &nirq);
2521
2522                         /* error here is non-fatal */
2523                         if (err != 0)
2524                                 continue;
2525
2526                         irq = priv->mdev->priv.msix_arr[nirq].vector;
2527                         cpu = (unsigned)(priv->params_ethtool.irq_cpu_base +
2528                             i * priv->params_ethtool.irq_cpu_stride) % (unsigned)mp_ncpus;
2529
2530                         CPU_ZERO(&cpuset);
2531                         CPU_SET(cpu, &cpuset);
2532                         intr_setaffinity(irq, CPU_WHICH_INTRHANDLER, &cpuset);
2533                 }
2534         }
2535         free(cparam, M_MLX5EN);
2536         return (0);
2537
2538 err_close_channels:
2539         while (i--) {
2540                 mlx5e_close_channel(&priv->channel[i]);
2541                 mlx5e_close_channel_wait(&priv->channel[i]);
2542         }
2543         free(cparam, M_MLX5EN);
2544         return (err);
2545 }
2546
2547 static void
2548 mlx5e_close_channels(struct mlx5e_priv *priv)
2549 {
2550         int i;
2551
2552         for (i = 0; i < priv->params.num_channels; i++)
2553                 mlx5e_close_channel(&priv->channel[i]);
2554         for (i = 0; i < priv->params.num_channels; i++)
2555                 mlx5e_close_channel_wait(&priv->channel[i]);
2556 }
2557
2558 static int
2559 mlx5e_refresh_sq_params(struct mlx5e_priv *priv, struct mlx5e_sq *sq)
2560 {
2561
2562         if (MLX5_CAP_GEN(priv->mdev, cq_period_mode_modify)) {
2563                 uint8_t cq_mode;
2564
2565                 switch (priv->params.tx_cq_moderation_mode) {
2566                 case 0:
2567                 case 2:
2568                         cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
2569                         break;
2570                 default:
2571                         cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_CQE;
2572                         break;
2573                 }
2574
2575                 return (mlx5_core_modify_cq_moderation_mode(priv->mdev, &sq->cq.mcq,
2576                     priv->params.tx_cq_moderation_usec,
2577                     priv->params.tx_cq_moderation_pkts,
2578                     cq_mode));
2579         }
2580
2581         return (mlx5_core_modify_cq_moderation(priv->mdev, &sq->cq.mcq,
2582             priv->params.tx_cq_moderation_usec,
2583             priv->params.tx_cq_moderation_pkts));
2584 }
2585
2586 static int
2587 mlx5e_refresh_rq_params(struct mlx5e_priv *priv, struct mlx5e_rq *rq)
2588 {
2589
2590         if (MLX5_CAP_GEN(priv->mdev, cq_period_mode_modify)) {
2591                 uint8_t cq_mode;
2592                 uint8_t dim_mode;
2593                 int retval;
2594
2595                 switch (priv->params.rx_cq_moderation_mode) {
2596                 case 0:
2597                 case 2:
2598                         cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
2599                         dim_mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
2600                         break;
2601                 default:
2602                         cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_CQE;
2603                         dim_mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE;
2604                         break;
2605                 }
2606
2607                 /* tear down dynamic interrupt moderation */
2608                 mtx_lock(&rq->mtx);
2609                 rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
2610                 mtx_unlock(&rq->mtx);
2611
2612                 /* wait for dynamic interrupt moderation work task, if any */
2613                 cancel_work_sync(&rq->dim.work);
2614
2615                 if (priv->params.rx_cq_moderation_mode >= 2) {
2616                         struct net_dim_cq_moder curr;
2617
2618                         mlx5e_get_default_profile(priv, dim_mode, &curr);
2619
2620                         retval = mlx5_core_modify_cq_moderation_mode(priv->mdev, &rq->cq.mcq,
2621                             curr.usec, curr.pkts, cq_mode);
2622
2623                         /* set dynamic interrupt moderation mode and zero defaults */
2624                         mtx_lock(&rq->mtx);
2625                         rq->dim.mode = dim_mode;
2626                         rq->dim.state = 0;
2627                         rq->dim.profile_ix = MLX5E_DIM_DEFAULT_PROFILE;
2628                         mtx_unlock(&rq->mtx);
2629                 } else {
2630                         retval = mlx5_core_modify_cq_moderation_mode(priv->mdev, &rq->cq.mcq,
2631                             priv->params.rx_cq_moderation_usec,
2632                             priv->params.rx_cq_moderation_pkts,
2633                             cq_mode);
2634                 }
2635                 return (retval);
2636         }
2637
2638         return (mlx5_core_modify_cq_moderation(priv->mdev, &rq->cq.mcq,
2639             priv->params.rx_cq_moderation_usec,
2640             priv->params.rx_cq_moderation_pkts));
2641 }
2642
2643 static int
2644 mlx5e_refresh_channel_params_sub(struct mlx5e_priv *priv, struct mlx5e_channel *c)
2645 {
2646         int err;
2647         int i;
2648
2649         err = mlx5e_refresh_rq_params(priv, &c->rq);
2650         if (err)
2651                 goto done;
2652
2653         for (i = 0; i != priv->num_tc; i++) {
2654                 err = mlx5e_refresh_sq_params(priv, &c->sq[i]);
2655                 if (err)
2656                         goto done;
2657         }
2658 done:
2659         return (err);
2660 }
2661
2662 int
2663 mlx5e_refresh_channel_params(struct mlx5e_priv *priv)
2664 {
2665         int i;
2666
2667         /* check if channels are closed */
2668         if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
2669                 return (EINVAL);
2670
2671         for (i = 0; i < priv->params.num_channels; i++) {
2672                 int err;
2673
2674                 err = mlx5e_refresh_channel_params_sub(priv, &priv->channel[i]);
2675                 if (err)
2676                         return (err);
2677         }
2678         return (0);
2679 }
2680
2681 static int
2682 mlx5e_open_tis(struct mlx5e_priv *priv, int tc)
2683 {
2684         struct mlx5_core_dev *mdev = priv->mdev;
2685         u32 in[MLX5_ST_SZ_DW(create_tis_in)];
2686         void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
2687
2688         memset(in, 0, sizeof(in));
2689
2690         MLX5_SET(tisc, tisc, prio, tc);
2691         MLX5_SET(tisc, tisc, transport_domain, priv->tdn);
2692
2693         return (mlx5_core_create_tis(mdev, in, sizeof(in), &priv->tisn[tc]));
2694 }
2695
2696 static void
2697 mlx5e_close_tis(struct mlx5e_priv *priv, int tc)
2698 {
2699         mlx5_core_destroy_tis(priv->mdev, priv->tisn[tc], 0);
2700 }
2701
2702 static int
2703 mlx5e_open_tises(struct mlx5e_priv *priv)
2704 {
2705         int num_tc = priv->num_tc;
2706         int err;
2707         int tc;
2708
2709         for (tc = 0; tc < num_tc; tc++) {
2710                 err = mlx5e_open_tis(priv, tc);
2711                 if (err)
2712                         goto err_close_tises;
2713         }
2714
2715         return (0);
2716
2717 err_close_tises:
2718         for (tc--; tc >= 0; tc--)
2719                 mlx5e_close_tis(priv, tc);
2720
2721         return (err);
2722 }
2723
2724 static void
2725 mlx5e_close_tises(struct mlx5e_priv *priv)
2726 {
2727         int num_tc = priv->num_tc;
2728         int tc;
2729
2730         for (tc = 0; tc < num_tc; tc++)
2731                 mlx5e_close_tis(priv, tc);
2732 }
2733
2734 static int
2735 mlx5e_open_default_rqt(struct mlx5e_priv *priv, u32 *prqtn, int sz)
2736 {
2737         u32 *in;
2738         void *rqtc;
2739         int inlen;
2740         int err;
2741         int i;
2742
2743         inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
2744         in = mlx5_vzalloc(inlen);
2745         if (in == NULL)
2746                 return (-ENOMEM);
2747         rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
2748
2749         MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
2750         MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
2751
2752         for (i = 0; i != sz; i++)
2753                 MLX5_SET(rqtc, rqtc, rq_num[i], priv->drop_rq.rqn);
2754
2755         err = mlx5_core_create_rqt(priv->mdev, in, inlen, prqtn);
2756         kvfree(in);
2757
2758         return (err);
2759 }
2760
2761 static int
2762 mlx5e_open_rqts(struct mlx5e_priv *priv)
2763 {
2764         int err;
2765         int i;
2766
2767         err = mlx5e_open_default_rqt(priv, &priv->rqtn,
2768             1 << priv->params.rx_hash_log_tbl_sz);
2769         if (err)
2770                 goto err_default;
2771
2772         for (i = 0; i != priv->mdev->priv.eq_table.num_comp_vectors; i++) {
2773                 err = mlx5e_open_default_rqt(priv, &priv->channel[i].rqtn, 1);
2774                 if (err)
2775                         goto err_channel;
2776         }
2777         return (0);
2778
2779 err_channel:
2780         while (i--)
2781                 mlx5_core_destroy_rqt(priv->mdev, priv->channel[i].rqtn, 0);
2782
2783         mlx5_core_destroy_rqt(priv->mdev, priv->rqtn, 0);
2784
2785 err_default:
2786         return (err);
2787 }
2788
2789 static void
2790 mlx5e_close_rqts(struct mlx5e_priv *priv)
2791 {
2792         int i;
2793
2794         for (i = 0; i != priv->mdev->priv.eq_table.num_comp_vectors; i++)
2795                 mlx5_core_destroy_rqt(priv->mdev, priv->channel[i].rqtn, 0);
2796
2797         mlx5_core_destroy_rqt(priv->mdev, priv->rqtn, 0);
2798 }
2799
2800 static int
2801 mlx5e_activate_rqt(struct mlx5e_priv *priv)
2802 {
2803         u32 *in;
2804         void *rqtc;
2805         int inlen;
2806         int err;
2807         int sz;
2808         int i;
2809
2810         sz = 1 << priv->params.rx_hash_log_tbl_sz;
2811
2812         inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32) * sz;
2813         in = mlx5_vzalloc(inlen);
2814         if (in == NULL)
2815                 return (-ENOMEM);
2816
2817         rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx);
2818
2819         MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
2820         MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1);
2821
2822         for (i = 0; i != sz; i++) {
2823                 int ix;
2824 #ifdef RSS
2825                 ix = rss_get_indirection_to_bucket(i);
2826 #else
2827                 ix = i;
2828 #endif
2829                 /* ensure we don't overflow */
2830                 ix %= priv->params.num_channels;
2831
2832                 /* apply receive side scaling stride, if any */
2833                 ix -= ix % (int)priv->params.channels_rsss;
2834
2835                 MLX5_SET(rqtc, rqtc, rq_num[i], priv->channel[ix].rq.rqn);
2836         }
2837
2838         err = mlx5_core_modify_rqt(priv->mdev, priv->rqtn, in, inlen);
2839         if (err)
2840                 goto err_modify;
2841
2842         inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32);
2843
2844         MLX5_SET(rqtc, rqtc, rqt_actual_size, 1);
2845
2846         for (i = 0; i != priv->mdev->priv.eq_table.num_comp_vectors; i++) {
2847                 int ix;
2848 #ifdef RSS
2849                 ix = rss_get_indirection_to_bucket(i);
2850 #else
2851                 ix = i;
2852 #endif
2853                 /* ensure we don't overflow */
2854                 ix %= priv->params.num_channels;
2855
2856                 /* apply receive side scaling stride, if any */
2857                 ix -= ix % (int)priv->params.channels_rsss;
2858
2859                 MLX5_SET(rqtc, rqtc, rq_num[0], priv->channel[ix].rq.rqn);
2860
2861                 err = mlx5_core_modify_rqt(priv->mdev, priv->channel[i].rqtn, in, inlen);
2862                 if (err)
2863                         goto err_modify;
2864         }
2865
2866 err_modify:
2867         kvfree(in);
2868         return (err);
2869 }
2870
2871 static int
2872 mlx5e_deactivate_rqt(struct mlx5e_priv *priv)
2873 {
2874         u32 *in;
2875         void *rqtc;
2876         int inlen;
2877         int err;
2878         int sz;
2879         int i;
2880
2881         sz = 1 << priv->params.rx_hash_log_tbl_sz;
2882
2883         inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32) * sz;
2884         in = mlx5_vzalloc(inlen);
2885         if (in == NULL)
2886                 return (-ENOMEM);
2887
2888         rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx);
2889
2890         MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
2891         MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1);
2892
2893         for (i = 0; i != sz; i++)
2894                 MLX5_SET(rqtc, rqtc, rq_num[i], priv->drop_rq.rqn);
2895
2896         err = mlx5_core_modify_rqt(priv->mdev, priv->rqtn, in, inlen);
2897         if (err)
2898                 goto err_modify;
2899
2900         inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32);
2901
2902         MLX5_SET(rqtc, rqtc, rqt_actual_size, 1);
2903
2904         for (i = 0; i != priv->mdev->priv.eq_table.num_comp_vectors; i++) {
2905                 MLX5_SET(rqtc, rqtc, rq_num[0], priv->drop_rq.rqn);
2906
2907                 err = mlx5_core_modify_rqt(priv->mdev, priv->channel[i].rqtn, in, inlen);
2908                 if (err)
2909                         goto err_modify;
2910         }
2911
2912 err_modify:
2913         kvfree(in);
2914         return (err);
2915 }
2916
2917 #define MLX5E_RSS_KEY_SIZE (10 * 4)     /* bytes */
2918
2919 static void
2920 mlx5e_get_rss_key(void *key_ptr)
2921 {
2922 #ifdef RSS
2923         rss_getkey(key_ptr);
2924 #else
2925         static const u32 rsskey[] = {
2926             cpu_to_be32(0xD181C62C),
2927             cpu_to_be32(0xF7F4DB5B),
2928             cpu_to_be32(0x1983A2FC),
2929             cpu_to_be32(0x943E1ADB),
2930             cpu_to_be32(0xD9389E6B),
2931             cpu_to_be32(0xD1039C2C),
2932             cpu_to_be32(0xA74499AD),
2933             cpu_to_be32(0x593D56D9),
2934             cpu_to_be32(0xF3253C06),
2935             cpu_to_be32(0x2ADC1FFC),
2936         };
2937         CTASSERT(sizeof(rsskey) == MLX5E_RSS_KEY_SIZE);
2938         memcpy(key_ptr, rsskey, MLX5E_RSS_KEY_SIZE);
2939 #endif
2940 }
2941
2942 static void
2943 mlx5e_build_tir_ctx(struct mlx5e_priv *priv, u32 * tirc, int tt, bool inner_vxlan)
2944 {
2945         void *hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
2946         void *hfsi = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner);
2947         void *hfs = inner_vxlan ? hfsi : hfso;
2948         __be32 *hkey;
2949
2950         MLX5_SET(tirc, tirc, transport_domain, priv->tdn);
2951
2952 #define ROUGH_MAX_L2_L3_HDR_SZ 256
2953
2954 #define MLX5_HASH_IP     (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2955                           MLX5_HASH_FIELD_SEL_DST_IP)
2956
2957 #define MLX5_HASH_ALL    (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2958                           MLX5_HASH_FIELD_SEL_DST_IP   |\
2959                           MLX5_HASH_FIELD_SEL_L4_SPORT |\
2960                           MLX5_HASH_FIELD_SEL_L4_DPORT)
2961
2962 #define MLX5_HASH_IP_IPSEC_SPI  (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2963                                  MLX5_HASH_FIELD_SEL_DST_IP   |\
2964                                  MLX5_HASH_FIELD_SEL_IPSEC_SPI)
2965
2966         if (priv->params.hw_lro_en) {
2967                 MLX5_SET(tirc, tirc, lro_enable_mask,
2968                     MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
2969                     MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO);
2970                 MLX5_SET(tirc, tirc, lro_max_msg_sz,
2971                     (priv->params.lro_wqe_sz -
2972                     ROUGH_MAX_L2_L3_HDR_SZ) >> 8);
2973                 /* TODO: add the option to choose timer value dynamically */
2974                 MLX5_SET(tirc, tirc, lro_timeout_period_usecs,
2975                     MLX5_CAP_ETH(priv->mdev,
2976                     lro_timer_supported_periods[2]));
2977         }
2978
2979         if (inner_vxlan)
2980                 MLX5_SET(tirc, tirc, tunneled_offload_en, 1);
2981
2982         /*
2983          * All packets must go through the indirection table, RQT,
2984          * because it is not possible to modify the RQN of the TIR
2985          * for direct dispatchment after it is created, typically
2986          * when the link goes up and down.
2987          */
2988         MLX5_SET(tirc, tirc, disp_type,
2989             MLX5_TIRC_DISP_TYPE_INDIRECT);
2990         MLX5_SET(tirc, tirc, indirect_table,
2991             priv->rqtn);
2992         MLX5_SET(tirc, tirc, rx_hash_fn,
2993                  MLX5_TIRC_RX_HASH_FN_HASH_TOEPLITZ);
2994         hkey = (__be32 *) MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
2995
2996         CTASSERT(MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key) >=
2997                  MLX5E_RSS_KEY_SIZE);
2998 #ifdef RSS
2999         /*
3000          * The FreeBSD RSS implementation does currently not
3001          * support symmetric Toeplitz hashes:
3002          */
3003         MLX5_SET(tirc, tirc, rx_hash_symmetric, 0);
3004 #else
3005         MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
3006 #endif
3007         mlx5e_get_rss_key(hkey);
3008
3009         switch (tt) {
3010         case MLX5E_TT_IPV4_TCP:
3011                 MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3012                     MLX5_L3_PROT_TYPE_IPV4);
3013                 MLX5_SET(rx_hash_field_select, hfs, l4_prot_type,
3014                     MLX5_L4_PROT_TYPE_TCP);
3015 #ifdef RSS
3016                 if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4)) {
3017                         MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3018                             MLX5_HASH_IP);
3019                 } else
3020 #endif
3021                 MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3022                     MLX5_HASH_ALL);
3023                 break;
3024
3025         case MLX5E_TT_IPV6_TCP:
3026                 MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3027                     MLX5_L3_PROT_TYPE_IPV6);
3028                 MLX5_SET(rx_hash_field_select, hfs, l4_prot_type,
3029                     MLX5_L4_PROT_TYPE_TCP);
3030 #ifdef RSS
3031                 if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV6)) {
3032                         MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3033                             MLX5_HASH_IP);
3034                 } else
3035 #endif
3036                 MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3037                     MLX5_HASH_ALL);
3038                 break;
3039
3040         case MLX5E_TT_IPV4_UDP:
3041                 MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3042                     MLX5_L3_PROT_TYPE_IPV4);
3043                 MLX5_SET(rx_hash_field_select, hfs, l4_prot_type,
3044                     MLX5_L4_PROT_TYPE_UDP);
3045 #ifdef RSS
3046                 if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4)) {
3047                         MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3048                             MLX5_HASH_IP);
3049                 } else
3050 #endif
3051                 MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3052                     MLX5_HASH_ALL);
3053                 break;
3054
3055         case MLX5E_TT_IPV6_UDP:
3056                 MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3057                     MLX5_L3_PROT_TYPE_IPV6);
3058                 MLX5_SET(rx_hash_field_select, hfs, l4_prot_type,
3059                     MLX5_L4_PROT_TYPE_UDP);
3060 #ifdef RSS
3061                 if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV6)) {
3062                         MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3063                             MLX5_HASH_IP);
3064                 } else
3065 #endif
3066                 MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3067                     MLX5_HASH_ALL);
3068                 break;
3069
3070         case MLX5E_TT_IPV4_IPSEC_AH:
3071                 MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3072                     MLX5_L3_PROT_TYPE_IPV4);
3073                 MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3074                     MLX5_HASH_IP_IPSEC_SPI);
3075                 break;
3076
3077         case MLX5E_TT_IPV6_IPSEC_AH:
3078                 MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3079                     MLX5_L3_PROT_TYPE_IPV6);
3080                 MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3081                     MLX5_HASH_IP_IPSEC_SPI);
3082                 break;
3083
3084         case MLX5E_TT_IPV4_IPSEC_ESP:
3085                 MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3086                     MLX5_L3_PROT_TYPE_IPV4);
3087                 MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3088                     MLX5_HASH_IP_IPSEC_SPI);
3089                 break;
3090
3091         case MLX5E_TT_IPV6_IPSEC_ESP:
3092                 MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3093                     MLX5_L3_PROT_TYPE_IPV6);
3094                 MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3095                     MLX5_HASH_IP_IPSEC_SPI);
3096                 break;
3097
3098         case MLX5E_TT_IPV4:
3099                 MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3100                     MLX5_L3_PROT_TYPE_IPV4);
3101                 MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3102                     MLX5_HASH_IP);
3103                 break;
3104
3105         case MLX5E_TT_IPV6:
3106                 MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3107                     MLX5_L3_PROT_TYPE_IPV6);
3108                 MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3109                     MLX5_HASH_IP);
3110                 break;
3111
3112         default:
3113                 break;
3114         }
3115 }
3116
3117 static int
3118 mlx5e_open_tir(struct mlx5e_priv *priv, int tt, bool inner_vxlan)
3119 {
3120         struct mlx5_core_dev *mdev = priv->mdev;
3121         u32 *in;
3122         void *tirc;
3123         int inlen;
3124         int err;
3125
3126         inlen = MLX5_ST_SZ_BYTES(create_tir_in);
3127         in = mlx5_vzalloc(inlen);
3128         if (in == NULL)
3129                 return (-ENOMEM);
3130         tirc = MLX5_ADDR_OF(create_tir_in, in, tir_context);
3131
3132         mlx5e_build_tir_ctx(priv, tirc, tt, inner_vxlan);
3133
3134         err = mlx5_core_create_tir(mdev, in, inlen, inner_vxlan ?
3135             &priv->tirn_inner_vxlan[tt] : &priv->tirn[tt]);
3136
3137         kvfree(in);
3138
3139         return (err);
3140 }
3141
3142 static void
3143 mlx5e_close_tir(struct mlx5e_priv *priv, int tt, bool inner_vxlan)
3144 {
3145         mlx5_core_destroy_tir(priv->mdev, inner_vxlan ?
3146             priv->tirn_inner_vxlan[tt] : priv->tirn[tt], 0);
3147 }
3148
3149 static int
3150 mlx5e_open_tirs(struct mlx5e_priv *priv)
3151 {
3152         int err;
3153         int i;
3154
3155         for (i = 0; i != 2 * MLX5E_NUM_TT; i++) {
3156                 err = mlx5e_open_tir(priv, i / 2, (i % 2) ? true : false);
3157                 if (err)
3158                         goto err_close_tirs;
3159         }
3160
3161         return (0);
3162
3163 err_close_tirs:
3164         for (i--; i >= 0; i--)
3165                 mlx5e_close_tir(priv, i / 2, (i % 2) ? true : false);
3166
3167         return (err);
3168 }
3169
3170 static void
3171 mlx5e_close_tirs(struct mlx5e_priv *priv)
3172 {
3173         int i;
3174
3175         for (i = 0; i != 2 * MLX5E_NUM_TT; i++)
3176                 mlx5e_close_tir(priv, i / 2, (i % 2) ? true : false);
3177 }
3178
3179 /*
3180  * SW MTU does not include headers,
3181  * HW MTU includes all headers and checksums.
3182  */
3183 static int
3184 mlx5e_set_dev_port_mtu(if_t ifp, int sw_mtu)
3185 {
3186         struct mlx5e_priv *priv = if_getsoftc(ifp);
3187         struct mlx5_core_dev *mdev = priv->mdev;
3188         int hw_mtu;
3189         int err;
3190
3191         hw_mtu = MLX5E_SW2HW_MTU(sw_mtu);
3192
3193         err = mlx5_set_port_mtu(mdev, hw_mtu);
3194         if (err) {
3195                 mlx5_en_err(ifp, "mlx5_set_port_mtu failed setting %d, err=%d\n",
3196                     sw_mtu, err);
3197                 return (err);
3198         }
3199
3200         /* Update vport context MTU */
3201         err = mlx5_set_vport_mtu(mdev, hw_mtu);
3202         if (err) {
3203                 mlx5_en_err(ifp,
3204                     "Failed updating vport context with MTU size, err=%d\n",
3205                     err);
3206         }
3207
3208         if_setmtu(ifp, sw_mtu);
3209
3210         err = mlx5_query_vport_mtu(mdev, &hw_mtu);
3211         if (err || !hw_mtu) {
3212                 /* fallback to port oper mtu */
3213                 err = mlx5_query_port_oper_mtu(mdev, &hw_mtu);
3214         }
3215         if (err) {
3216                 mlx5_en_err(ifp,
3217                     "Query port MTU, after setting new MTU value, failed\n");
3218                 return (err);
3219         } else if (MLX5E_HW2SW_MTU(hw_mtu) < sw_mtu) {
3220                 err = -E2BIG,
3221                 mlx5_en_err(ifp,
3222                     "Port MTU %d is smaller than ifp mtu %d\n",
3223                     hw_mtu, sw_mtu);
3224         } else if (MLX5E_HW2SW_MTU(hw_mtu) > sw_mtu) {
3225                 err = -EINVAL;
3226                 mlx5_en_err(ifp,
3227                     "Port MTU %d is bigger than ifp mtu %d\n",
3228                     hw_mtu, sw_mtu);
3229         }
3230         priv->params_ethtool.hw_mtu = hw_mtu;
3231
3232         /* compute MSB */
3233         while (hw_mtu & (hw_mtu - 1))
3234                 hw_mtu &= (hw_mtu - 1);
3235         priv->params_ethtool.hw_mtu_msb = hw_mtu;
3236
3237         return (err);
3238 }
3239
3240 int
3241 mlx5e_open_locked(if_t ifp)
3242 {
3243         struct mlx5e_priv *priv = if_getsoftc(ifp);
3244         int err;
3245         u16 set_id;
3246
3247         /* check if already opened */
3248         if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
3249                 return (0);
3250
3251 #ifdef RSS
3252         if (rss_getnumbuckets() > priv->params.num_channels) {
3253                 mlx5_en_info(ifp,
3254                     "NOTE: There are more RSS buckets(%u) than channels(%u) available\n",
3255                     rss_getnumbuckets(), priv->params.num_channels);
3256         }
3257 #endif
3258         err = mlx5e_open_tises(priv);
3259         if (err) {
3260                 mlx5_en_err(ifp, "mlx5e_open_tises failed, %d\n", err);
3261                 return (err);
3262         }
3263         err = mlx5_vport_alloc_q_counter(priv->mdev,
3264             MLX5_INTERFACE_PROTOCOL_ETH, &set_id);
3265         if (err) {
3266                 mlx5_en_err(priv->ifp,
3267                     "mlx5_vport_alloc_q_counter failed: %d\n", err);
3268                 goto err_close_tises;
3269         }
3270         /* store counter set ID */
3271         priv->counter_set_id = set_id;
3272
3273         err = mlx5e_open_channels(priv);
3274         if (err) {
3275                 mlx5_en_err(ifp,
3276                     "mlx5e_open_channels failed, %d\n", err);
3277                 goto err_dalloc_q_counter;
3278         }
3279         err = mlx5e_activate_rqt(priv);
3280         if (err) {
3281                 mlx5_en_err(ifp, "mlx5e_activate_rqt failed, %d\n", err);
3282                 goto err_close_channels;
3283         }
3284
3285         set_bit(MLX5E_STATE_OPENED, &priv->state);
3286
3287         mlx5e_update_carrier(priv);
3288
3289         return (0);
3290
3291 err_close_channels:
3292         mlx5e_close_channels(priv);
3293
3294 err_dalloc_q_counter:
3295         mlx5_vport_dealloc_q_counter(priv->mdev,
3296             MLX5_INTERFACE_PROTOCOL_ETH, priv->counter_set_id);
3297
3298 err_close_tises:
3299         mlx5e_close_tises(priv);
3300
3301         return (err);
3302 }
3303
3304 static void
3305 mlx5e_open(void *arg)
3306 {
3307         struct mlx5e_priv *priv = arg;
3308
3309         PRIV_LOCK(priv);
3310         if (mlx5_set_port_status(priv->mdev, MLX5_PORT_UP))
3311                 mlx5_en_err(priv->ifp,
3312                     "Setting port status to up failed\n");
3313
3314         mlx5e_open_locked(priv->ifp);
3315         if_setdrvflagbits(priv->ifp, IFF_DRV_RUNNING, 0);
3316         PRIV_UNLOCK(priv);
3317 }
3318
3319 int
3320 mlx5e_close_locked(if_t ifp)
3321 {
3322         struct mlx5e_priv *priv = if_getsoftc(ifp);
3323
3324         /* check if already closed */
3325         if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
3326                 return (0);
3327
3328         clear_bit(MLX5E_STATE_OPENED, &priv->state);
3329
3330         if_link_state_change(priv->ifp, LINK_STATE_DOWN);
3331
3332         mlx5e_deactivate_rqt(priv);
3333         mlx5e_close_channels(priv);
3334         mlx5_vport_dealloc_q_counter(priv->mdev,
3335             MLX5_INTERFACE_PROTOCOL_ETH, priv->counter_set_id);
3336         mlx5e_close_tises(priv);
3337
3338         return (0);
3339 }
3340
3341 static uint64_t
3342 mlx5e_get_counter(if_t ifp, ift_counter cnt)
3343 {
3344         struct mlx5e_priv *priv = if_getsoftc(ifp);
3345         u64 retval;
3346
3347         /* PRIV_LOCK(priv); XXX not allowed */
3348         switch (cnt) {
3349         case IFCOUNTER_IPACKETS:
3350                 retval = priv->stats.vport.rx_packets;
3351                 break;
3352         case IFCOUNTER_IERRORS:
3353                 retval = priv->stats.pport.in_range_len_errors +
3354                     priv->stats.pport.out_of_range_len +
3355                     priv->stats.pport.too_long_errors +
3356                     priv->stats.pport.check_seq_err +
3357                     priv->stats.pport.alignment_err;
3358                 break;
3359         case IFCOUNTER_IQDROPS:
3360                 retval = priv->stats.vport.rx_out_of_buffer;
3361                 break;
3362         case IFCOUNTER_OPACKETS:
3363                 retval = priv->stats.vport.tx_packets;
3364                 break;
3365         case IFCOUNTER_OERRORS:
3366                 retval = priv->stats.port_stats_debug.out_discards;
3367                 break;
3368         case IFCOUNTER_IBYTES:
3369                 retval = priv->stats.vport.rx_bytes;
3370                 break;
3371         case IFCOUNTER_OBYTES:
3372                 retval = priv->stats.vport.tx_bytes;
3373                 break;
3374         case IFCOUNTER_IMCASTS:
3375                 retval = priv->stats.vport.rx_multicast_packets;
3376                 break;
3377         case IFCOUNTER_OMCASTS:
3378                 retval = priv->stats.vport.tx_multicast_packets;
3379                 break;
3380         case IFCOUNTER_OQDROPS:
3381                 retval = priv->stats.vport.tx_queue_dropped;
3382                 break;
3383         case IFCOUNTER_COLLISIONS:
3384                 retval = priv->stats.pport.collisions;
3385                 break;
3386         default:
3387                 retval = if_get_counter_default(ifp, cnt);
3388                 break;
3389         }
3390         /* PRIV_UNLOCK(priv); XXX not allowed */
3391         return (retval);
3392 }
3393
3394 static void
3395 mlx5e_set_rx_mode(if_t ifp)
3396 {
3397         struct mlx5e_priv *priv = if_getsoftc(ifp);
3398
3399         queue_work(priv->wq, &priv->set_rx_mode_work);
3400 }
3401
3402 static int
3403 mlx5e_ioctl(if_t ifp, u_long command, caddr_t data)
3404 {
3405         struct mlx5e_priv *priv;
3406         struct ifreq *ifr;
3407         struct ifdownreason *ifdr;
3408         struct ifi2creq i2c;
3409         struct ifrsskey *ifrk;
3410         struct ifrsshash *ifrh;
3411         struct siocsifcapnv_driver_data *drv_ioctl_data, drv_ioctl_data_d;
3412         int error = 0;
3413         int mask;
3414         int size_read = 0;
3415         int module_status;
3416         int module_num;
3417         int max_mtu;
3418         uint8_t read_addr;
3419
3420         priv = if_getsoftc(ifp);
3421
3422         /* check if detaching */
3423         if (priv == NULL || priv->gone != 0)
3424                 return (ENXIO);
3425
3426         switch (command) {
3427         case SIOCSIFMTU:
3428                 ifr = (struct ifreq *)data;
3429
3430                 PRIV_LOCK(priv);
3431                 mlx5_query_port_max_mtu(priv->mdev, &max_mtu);
3432
3433                 if (ifr->ifr_mtu >= MLX5E_MTU_MIN &&
3434                     ifr->ifr_mtu <= MIN(MLX5E_MTU_MAX, max_mtu)) {
3435                         int was_opened;
3436
3437                         was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
3438                         if (was_opened)
3439                                 mlx5e_close_locked(ifp);
3440
3441                         /* set new MTU */
3442                         mlx5e_set_dev_port_mtu(ifp, ifr->ifr_mtu);
3443
3444                         if (was_opened)
3445                                 mlx5e_open_locked(ifp);
3446                 } else {
3447                         error = EINVAL;
3448                         mlx5_en_err(ifp,
3449                             "Invalid MTU value. Min val: %d, Max val: %d\n",
3450                             MLX5E_MTU_MIN, MIN(MLX5E_MTU_MAX, max_mtu));
3451                 }
3452                 PRIV_UNLOCK(priv);
3453                 break;
3454         case SIOCSIFFLAGS:
3455                 if ((if_getflags(ifp) & IFF_UP) &&
3456                     (if_getdrvflags(ifp) & IFF_DRV_RUNNING)) {
3457                         mlx5e_set_rx_mode(ifp);
3458                         break;
3459                 }
3460                 PRIV_LOCK(priv);
3461                 if (if_getflags(ifp) & IFF_UP) {
3462                         if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) {
3463                                 if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
3464                                         mlx5e_open_locked(ifp);
3465                                 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0);
3466                                 mlx5_set_port_status(priv->mdev, MLX5_PORT_UP);
3467                         }
3468                 } else {
3469                         if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
3470                                 mlx5_set_port_status(priv->mdev,
3471                                     MLX5_PORT_DOWN);
3472                                 if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
3473                                         mlx5e_close_locked(ifp);
3474                                 mlx5e_update_carrier(priv);
3475                                 if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
3476                         }
3477                 }
3478                 PRIV_UNLOCK(priv);
3479                 break;
3480         case SIOCADDMULTI:
3481         case SIOCDELMULTI:
3482                 mlx5e_set_rx_mode(ifp);
3483                 break;
3484         case SIOCSIFMEDIA:
3485         case SIOCGIFMEDIA:
3486         case SIOCGIFXMEDIA:
3487                 ifr = (struct ifreq *)data;
3488                 error = ifmedia_ioctl(ifp, ifr, &priv->media, command);
3489                 break;
3490         case SIOCGIFCAPNV:
3491                 error = 0;
3492                 break;
3493         case SIOCSIFCAP:
3494                 ifr = (struct ifreq *)data;
3495                 drv_ioctl_data = &drv_ioctl_data_d;
3496                 drv_ioctl_data->reqcap = ifr->ifr_reqcap;
3497                 PRIV_LOCK(priv);
3498                 drv_ioctl_data->reqcap2 = if_getcapenable2(ifp);
3499                 drv_ioctl_data->nvcap = NULL;
3500                 goto siocsifcap_driver;
3501         case SIOCSIFCAPNV:
3502                 drv_ioctl_data = (struct siocsifcapnv_driver_data *)data;
3503                 PRIV_LOCK(priv);
3504 siocsifcap_driver:
3505                 mask = drv_ioctl_data->reqcap ^ if_getcapenable(ifp);
3506
3507                 if (mask & IFCAP_TXCSUM) {
3508                         if_togglecapenable(ifp, IFCAP_TXCSUM);
3509                         if_togglehwassist(ifp, (CSUM_TCP | CSUM_UDP | CSUM_IP));
3510
3511                         if (IFCAP_TSO4 & if_getcapenable(ifp) &&
3512                             !(IFCAP_TXCSUM & if_getcapenable(ifp))) {
3513                                 mask &= ~IFCAP_TSO4;
3514                                 if_setcapenablebit(ifp, 0, IFCAP_TSO4);
3515                                 if_sethwassistbits(ifp, 0, CSUM_IP_TSO);
3516                                 mlx5_en_err(ifp,
3517                                     "tso4 disabled due to -txcsum.\n");
3518                         }
3519                 }
3520                 if (mask & IFCAP_TXCSUM_IPV6) {
3521                         if_togglecapenable(ifp, IFCAP_TXCSUM_IPV6);
3522                         if_togglehwassist(ifp, (CSUM_UDP_IPV6 | CSUM_TCP_IPV6));
3523
3524                         if (IFCAP_TSO6 & if_getcapenable(ifp) &&
3525                             !(IFCAP_TXCSUM_IPV6 & if_getcapenable(ifp))) {
3526                                 mask &= ~IFCAP_TSO6;
3527                                 if_setcapenablebit(ifp, 0, IFCAP_TSO6);
3528                                 if_sethwassistbits(ifp, 0, CSUM_IP6_TSO);
3529                                 mlx5_en_err(ifp,
3530                                     "tso6 disabled due to -txcsum6.\n");
3531                         }
3532                 }
3533                 if (mask & IFCAP_MEXTPG)
3534                         if_togglecapenable(ifp, IFCAP_MEXTPG);
3535                 if (mask & IFCAP_TXTLS4)
3536                         if_togglecapenable(ifp, IFCAP_TXTLS4);
3537                 if (mask & IFCAP_TXTLS6)
3538                         if_togglecapenable(ifp, IFCAP_TXTLS6);
3539 #ifdef RATELIMIT
3540                 if (mask & IFCAP_TXTLS_RTLMT)
3541                         if_togglecapenable(ifp, IFCAP_TXTLS_RTLMT);
3542 #endif
3543                 if (mask & IFCAP_RXCSUM)
3544                         if_togglecapenable(ifp, IFCAP_RXCSUM);
3545                 if (mask & IFCAP_RXCSUM_IPV6)
3546                         if_togglecapenable(ifp, IFCAP_RXCSUM_IPV6);
3547                 if (mask & IFCAP_TSO4) {
3548                         if (!(IFCAP_TSO4 & if_getcapenable(ifp)) &&
3549                             !(IFCAP_TXCSUM & if_getcapenable(ifp))) {
3550                                 mlx5_en_err(ifp, "enable txcsum first.\n");
3551                                 error = EAGAIN;
3552                                 goto out;
3553                         }
3554                         if_togglecapenable(ifp, IFCAP_TSO4);
3555                         if_togglehwassist(ifp, CSUM_IP_TSO);
3556                 }
3557                 if (mask & IFCAP_TSO6) {
3558                         if (!(IFCAP_TSO6 & if_getcapenable(ifp)) &&
3559                             !(IFCAP_TXCSUM_IPV6 & if_getcapenable(ifp))) {
3560                                 mlx5_en_err(ifp, "enable txcsum6 first.\n");
3561                                 error = EAGAIN;
3562                                 goto out;
3563                         }
3564                         if_togglecapenable(ifp, IFCAP_TSO6);
3565                         if_togglehwassist(ifp, CSUM_IP6_TSO);
3566                 }
3567                 if (mask & IFCAP_VLAN_HWTSO)
3568                         if_togglecapenable(ifp, IFCAP_VLAN_HWTSO);
3569                 if (mask & IFCAP_VLAN_HWFILTER) {
3570                         if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
3571                                 mlx5e_disable_vlan_filter(priv);
3572                         else
3573                                 mlx5e_enable_vlan_filter(priv);
3574
3575                         if_togglecapenable(ifp, IFCAP_VLAN_HWFILTER);
3576                 }
3577                 if (mask & IFCAP_VLAN_HWTAGGING)
3578                         if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING);
3579                 if (mask & IFCAP_WOL_MAGIC)
3580                         if_togglecapenable(ifp, IFCAP_WOL_MAGIC);
3581                 if (mask & IFCAP_VXLAN_HWCSUM) {
3582                         const bool was_enabled =
3583                             (if_getcapenable(ifp) & IFCAP_VXLAN_HWCSUM) != 0;
3584                         if (was_enabled)
3585                                 mlx5e_del_all_vxlan_rules(priv);
3586                         if_togglecapenable(ifp, IFCAP_VXLAN_HWCSUM);
3587                         if_togglehwassist(ifp, CSUM_INNER_IP | CSUM_INNER_IP_UDP |
3588                             CSUM_INNER_IP_TCP | CSUM_INNER_IP6_UDP |
3589                             CSUM_INNER_IP6_TCP);
3590                         if (!was_enabled) {
3591                                 int err = mlx5e_add_all_vxlan_rules(priv);
3592                                 if (err != 0) {
3593                                         mlx5_en_err(ifp,
3594                                             "mlx5e_add_all_vxlan_rules() failed, %d (ignored)\n", err);
3595                                 }
3596                         }
3597                 }
3598                 if (mask & IFCAP_VXLAN_HWTSO) {
3599                         if_togglecapenable(ifp, IFCAP_VXLAN_HWTSO);
3600                         if_togglehwassist(ifp, CSUM_INNER_IP_TSO |
3601                             CSUM_INNER_IP6_TSO);
3602                 }
3603
3604                 VLAN_CAPABILITIES(ifp);
3605                 /* turn off LRO means also turn of HW LRO - if it's on */
3606                 if (mask & IFCAP_LRO) {
3607                         int was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
3608                         bool need_restart = false;
3609
3610                         if_togglecapenable(ifp, IFCAP_LRO);
3611
3612                         /* figure out if updating HW LRO is needed */
3613                         if (!(if_getcapenable(ifp) & IFCAP_LRO)) {
3614                                 if (priv->params.hw_lro_en) {
3615                                         priv->params.hw_lro_en = false;
3616                                         need_restart = true;
3617                                 }
3618                         } else {
3619                                 if (priv->params.hw_lro_en == false &&
3620                                     priv->params_ethtool.hw_lro != 0) {
3621                                         priv->params.hw_lro_en = true;
3622                                         need_restart = true;
3623                                 }
3624                         }
3625                         if (was_opened && need_restart) {
3626                                 mlx5e_close_locked(ifp);
3627                                 mlx5e_open_locked(ifp);
3628                         }
3629                 }
3630                 if (mask & IFCAP_HWRXTSTMP) {
3631                         if_togglecapenable(ifp, IFCAP_HWRXTSTMP);
3632                         if (if_getcapenable(ifp) & IFCAP_HWRXTSTMP) {
3633                                 if (priv->clbr_done == 0)
3634                                         mlx5e_reset_calibration_callout(priv);
3635                         } else {
3636                                 callout_drain(&priv->tstmp_clbr);
3637                                 priv->clbr_done = 0;
3638                         }
3639                 }
3640                 mask = drv_ioctl_data->reqcap2 ^ if_getcapenable2(ifp);
3641                 if ((mask & IFCAP2_BIT(IFCAP2_RXTLS4)) != 0)
3642                         if_togglecapenable2(ifp, IFCAP2_BIT(IFCAP2_RXTLS4));
3643                 if ((mask & IFCAP2_BIT(IFCAP2_RXTLS6)) != 0)
3644                         if_togglecapenable2(ifp, IFCAP2_BIT(IFCAP2_RXTLS6));
3645 out:
3646                 PRIV_UNLOCK(priv);
3647                 break;
3648
3649         case SIOCGI2C:
3650                 ifr = (struct ifreq *)data;
3651
3652                 /*
3653                  * Copy from the user-space address ifr_data to the
3654                  * kernel-space address i2c
3655                  */
3656                 error = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c));
3657                 if (error)
3658                         break;
3659
3660                 if (i2c.len > sizeof(i2c.data)) {
3661                         error = EINVAL;
3662                         break;
3663                 }
3664
3665                 PRIV_LOCK(priv);
3666                 /* Get module_num which is required for the query_eeprom */
3667                 error = mlx5_query_module_num(priv->mdev, &module_num);
3668                 if (error) {
3669                         mlx5_en_err(ifp,
3670                             "Query module num failed, eeprom reading is not supported\n");
3671                         error = EINVAL;
3672                         goto err_i2c;
3673                 }
3674                 /* Check if module is present before doing an access */
3675                 module_status = mlx5_query_module_status(priv->mdev, module_num);
3676                 if (module_status != MLX5_MODULE_STATUS_PLUGGED_ENABLED) {
3677                         mlx5_en_err(ifp,
3678                             "Query module %d status: not plugged (%d), eeprom reading is not supported\n",
3679                             module_num, module_status);
3680                         error = EINVAL;
3681                         goto err_i2c;
3682                 }
3683                 /*
3684                  * Currently 0XA0 and 0xA2 are the only addresses permitted.
3685                  * The internal conversion is as follows:
3686                  */
3687                 if (i2c.dev_addr == 0xA0)
3688                         read_addr = MLX5_I2C_ADDR_LOW;
3689                 else if (i2c.dev_addr == 0xA2)
3690                         read_addr = MLX5_I2C_ADDR_HIGH;
3691                 else {
3692                         mlx5_en_err(ifp,
3693                             "Query eeprom failed, Invalid Address: %X\n",
3694                             i2c.dev_addr);
3695                         error = EINVAL;
3696                         goto err_i2c;
3697                 }
3698                 error = mlx5_query_eeprom(priv->mdev,
3699                     read_addr, MLX5_EEPROM_LOW_PAGE,
3700                     (uint32_t)i2c.offset, (uint32_t)i2c.len, module_num,
3701                     (uint32_t *)i2c.data, &size_read);
3702                 if (error) {
3703                         mlx5_en_err(ifp,
3704                             "Query eeprom failed, eeprom reading is not supported\n");
3705                         error = EINVAL;
3706                         goto err_i2c;
3707                 }
3708
3709                 if (i2c.len > MLX5_EEPROM_MAX_BYTES) {
3710                         error = mlx5_query_eeprom(priv->mdev,
3711                             read_addr, MLX5_EEPROM_LOW_PAGE,
3712                             (uint32_t)(i2c.offset + size_read),
3713                             (uint32_t)(i2c.len - size_read), module_num,
3714                             (uint32_t *)(i2c.data + size_read), &size_read);
3715                 }
3716                 if (error) {
3717                         mlx5_en_err(ifp,
3718                             "Query eeprom failed, eeprom reading is not supported\n");
3719                         error = EINVAL;
3720                         goto err_i2c;
3721                 }
3722
3723                 error = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c));
3724 err_i2c:
3725                 PRIV_UNLOCK(priv);
3726                 break;
3727         case SIOCGIFDOWNREASON:
3728                 ifdr = (struct ifdownreason *)data;
3729                 bzero(ifdr->ifdr_msg, sizeof(ifdr->ifdr_msg));
3730                 PRIV_LOCK(priv);
3731                 error = -mlx5_query_pddr_troubleshooting_info(priv->mdev, NULL,
3732                     ifdr->ifdr_msg, sizeof(ifdr->ifdr_msg));
3733                 PRIV_UNLOCK(priv);
3734                 if (error == 0)
3735                         ifdr->ifdr_reason = IFDR_REASON_MSG;
3736                 break;
3737
3738         case SIOCGIFRSSKEY:
3739                 ifrk = (struct ifrsskey *)data;
3740                 ifrk->ifrk_func = RSS_FUNC_TOEPLITZ;
3741                 ifrk->ifrk_keylen = MLX5E_RSS_KEY_SIZE;
3742                 CTASSERT(sizeof(ifrk->ifrk_key) >= MLX5E_RSS_KEY_SIZE);
3743                 mlx5e_get_rss_key(ifrk->ifrk_key);
3744                 break;
3745
3746         case SIOCGIFRSSHASH:
3747                 ifrh = (struct ifrsshash *)data;
3748                 ifrh->ifrh_func = RSS_FUNC_TOEPLITZ;
3749                 ifrh->ifrh_types =
3750                     RSS_TYPE_IPV4 |
3751                     RSS_TYPE_TCP_IPV4 |
3752                     RSS_TYPE_UDP_IPV4 |
3753                     RSS_TYPE_IPV6 |
3754                     RSS_TYPE_TCP_IPV6 |
3755                     RSS_TYPE_UDP_IPV6;
3756                 break;
3757
3758         default:
3759                 error = ether_ioctl(ifp, command, data);
3760                 break;
3761         }
3762         return (error);
3763 }
3764
3765 static int
3766 mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev)
3767 {
3768         /*
3769          * TODO: uncoment once FW really sets all these bits if
3770          * (!mdev->caps.eth.rss_ind_tbl_cap || !mdev->caps.eth.csum_cap ||
3771          * !mdev->caps.eth.max_lso_cap || !mdev->caps.eth.vlan_cap ||
3772          * !(mdev->caps.gen.flags & MLX5_DEV_CAP_FLAG_SCQE_BRK_MOD)) return
3773          * -ENOTSUPP;
3774          */
3775
3776         /* TODO: add more must-to-have features */
3777
3778         if (MLX5_CAP_GEN(mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
3779                 return (-ENODEV);
3780
3781         return (0);
3782 }
3783
3784 static u16
3785 mlx5e_get_max_inline_cap(struct mlx5_core_dev *mdev)
3786 {
3787         const int min_size = ETHER_VLAN_ENCAP_LEN + ETHER_HDR_LEN;
3788         const int max_size = MLX5E_MAX_TX_INLINE;
3789         const int bf_buf_size =
3790             ((1U << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2U) -
3791             (sizeof(struct mlx5e_tx_wqe) - 2);
3792
3793         /* verify against driver limits */
3794         if (bf_buf_size > max_size)
3795                 return (max_size);
3796         else if (bf_buf_size < min_size)
3797                 return (min_size);
3798         else
3799                 return (bf_buf_size);
3800 }
3801
3802 static int
3803 mlx5e_build_ifp_priv(struct mlx5_core_dev *mdev,
3804     struct mlx5e_priv *priv,
3805     int num_comp_vectors)
3806 {
3807         int err;
3808
3809         /*
3810          * TODO: Consider link speed for setting "log_sq_size",
3811          * "log_rq_size" and "cq_moderation_xxx":
3812          */
3813         priv->params.log_sq_size =
3814             MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
3815         priv->params.log_rq_size =
3816             MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
3817         priv->params.rx_cq_moderation_usec =
3818             MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
3819             MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE :
3820             MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
3821         priv->params.rx_cq_moderation_mode =
3822             MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? 1 : 0;
3823         priv->params.rx_cq_moderation_pkts =
3824             MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
3825         priv->params.tx_cq_moderation_usec =
3826             MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
3827         priv->params.tx_cq_moderation_pkts =
3828             MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
3829         priv->params.rx_hash_log_tbl_sz =
3830             (order_base_2(num_comp_vectors) >
3831             MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ) ?
3832             order_base_2(num_comp_vectors) :
3833             MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ;
3834         priv->params.num_tc = 1;
3835         priv->params.default_vlan_prio = 0;
3836         priv->counter_set_id = -1;
3837         priv->params.tx_max_inline = mlx5e_get_max_inline_cap(mdev);
3838
3839         err = mlx5_query_min_inline(mdev, &priv->params.tx_min_inline_mode);
3840         if (err)
3841                 return (err);
3842
3843         /*
3844          * hw lro is currently defaulted to off. when it won't anymore we
3845          * will consider the HW capability: "!!MLX5_CAP_ETH(mdev, lro_cap)"
3846          */
3847         priv->params.hw_lro_en = false;
3848         priv->params.lro_wqe_sz = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ;
3849
3850         /*
3851          * CQE zipping is off, because the per-packet 32-bit Toeplitz hash
3852          * is then not supported. The 32-bit Toeplitz hash is needed to
3853          * correctly demultiplex incoming traffic into the expected
3854          * network queues.
3855          */
3856         priv->params.cqe_zipping_en = false;
3857
3858         priv->mdev = mdev;
3859         priv->params.num_channels = num_comp_vectors;
3860         priv->params.channels_rsss = 1;
3861         priv->order_base_2_num_channels = order_base_2(num_comp_vectors);
3862         priv->queue_mapping_channel_mask =
3863             roundup_pow_of_two(num_comp_vectors) - 1;
3864         priv->num_tc = priv->params.num_tc;
3865         priv->default_vlan_prio = priv->params.default_vlan_prio;
3866
3867         INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
3868         INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
3869         INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
3870
3871         return (0);
3872 }
3873
3874 static void
3875 mlx5e_mkey_set_relaxed_ordering(struct mlx5_core_dev *mdev, void *mkc)
3876 {
3877         bool ro_pci_enable =
3878             pci_get_relaxed_ordering_enabled(mdev->pdev->dev.bsddev);
3879         bool ro_write = MLX5_CAP_GEN(mdev, relaxed_ordering_write);
3880         bool ro_read = MLX5_CAP_GEN(mdev, relaxed_ordering_read);
3881
3882         MLX5_SET(mkc, mkc, relaxed_ordering_read, ro_pci_enable && ro_read);
3883         MLX5_SET(mkc, mkc, relaxed_ordering_write, ro_pci_enable && ro_write);
3884 }
3885
3886 static int
3887 mlx5e_create_mkey(struct mlx5e_priv *priv, u32 pdn,
3888                   struct mlx5_core_mkey *mkey)
3889 {
3890         if_t ifp = priv->ifp;
3891         struct mlx5_core_dev *mdev = priv->mdev;
3892         int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
3893         void *mkc;
3894         u32 *in;
3895         int err;
3896
3897         in = mlx5_vzalloc(inlen);
3898         if (in == NULL) {
3899                 mlx5_en_err(ifp, "failed to allocate inbox\n");
3900                 return (-ENOMEM);
3901         }
3902
3903         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
3904         MLX5_SET(mkc, mkc, access_mode, MLX5_ACCESS_MODE_PA);
3905         MLX5_SET(mkc, mkc, umr_en, 1);  /* used by HW TLS */
3906         MLX5_SET(mkc, mkc, lw, 1);
3907         MLX5_SET(mkc, mkc, lr, 1);
3908         mlx5e_mkey_set_relaxed_ordering(mdev, mkc);
3909         MLX5_SET(mkc, mkc, pd, pdn);
3910         MLX5_SET(mkc, mkc, length64, 1);
3911         MLX5_SET(mkc, mkc, qpn, 0xffffff);
3912
3913         err = mlx5_core_create_mkey(mdev, mkey, in, inlen);
3914         if (err)
3915                 mlx5_en_err(ifp, "mlx5_core_create_mkey failed, %d\n",
3916                     err);
3917
3918         kvfree(in);
3919         return (err);
3920 }
3921
3922 static const char *mlx5e_vport_stats_desc[] = {
3923         MLX5E_VPORT_STATS(MLX5E_STATS_DESC)
3924 };
3925
3926 static const char *mlx5e_pport_stats_desc[] = {
3927         MLX5E_PPORT_STATS(MLX5E_STATS_DESC)
3928 };
3929
3930 static int
3931 mlx5e_priv_static_init(struct mlx5e_priv *priv, struct mlx5_core_dev *mdev,
3932     const uint32_t channels)
3933 {
3934         uint32_t x;
3935         int err;
3936
3937         mtx_init(&priv->async_events_mtx, "mlx5async", MTX_NETWORK_LOCK, MTX_DEF);
3938         sx_init(&priv->state_lock, "mlx5state");
3939         callout_init_mtx(&priv->watchdog, &priv->async_events_mtx, 0);
3940         MLX5_INIT_DOORBELL_LOCK(&priv->doorbell_lock);
3941         for (x = 0; x != channels; x++)
3942                 mlx5e_chan_static_init(priv, &priv->channel[x], x);
3943
3944         for (x = 0; x != channels; x++) {
3945                 err = mlx5_alloc_bfreg(mdev, &priv->channel[x].bfreg, false, false);
3946                 if (err)
3947                         goto err_alloc_bfreg;
3948         }
3949         return (0);
3950
3951 err_alloc_bfreg:
3952         while (x--)
3953                 mlx5_free_bfreg(mdev, &priv->channel[x].bfreg);
3954
3955         for (x = 0; x != channels; x++)
3956                 mlx5e_chan_static_destroy(&priv->channel[x]);
3957         callout_drain(&priv->watchdog);
3958         mtx_destroy(&priv->async_events_mtx);
3959         sx_destroy(&priv->state_lock);
3960         return (err);
3961 }
3962
3963 static void
3964 mlx5e_priv_static_destroy(struct mlx5e_priv *priv, struct mlx5_core_dev *mdev,
3965     const uint32_t channels)
3966 {
3967         uint32_t x;
3968
3969         for (x = 0; x != channels; x++)
3970                 mlx5_free_bfreg(mdev, &priv->channel[x].bfreg);
3971         for (x = 0; x != channels; x++)
3972                 mlx5e_chan_static_destroy(&priv->channel[x]);
3973         callout_drain(&priv->watchdog);
3974         mtx_destroy(&priv->async_events_mtx);
3975         sx_destroy(&priv->state_lock);
3976 }
3977
3978 static int
3979 sysctl_firmware(SYSCTL_HANDLER_ARGS)
3980 {
3981         /*
3982          * %d.%d%.d the string format.
3983          * fw_rev_{maj,min,sub} return u16, 2^16 = 65536.
3984          * We need at most 5 chars to store that.
3985          * It also has: two "." and NULL at the end, which means we need 18
3986          * (5*3 + 3) chars at most.
3987          */
3988         char fw[18];
3989         struct mlx5e_priv *priv = arg1;
3990         int error;
3991
3992         snprintf(fw, sizeof(fw), "%d.%d.%d", fw_rev_maj(priv->mdev), fw_rev_min(priv->mdev),
3993             fw_rev_sub(priv->mdev));
3994         error = sysctl_handle_string(oidp, fw, sizeof(fw), req);
3995         return (error);
3996 }
3997
3998 static void
3999 mlx5e_disable_tx_dma(struct mlx5e_channel *ch)
4000 {
4001         int i;
4002
4003         for (i = 0; i < ch->priv->num_tc; i++)
4004                 mlx5e_drain_sq(&ch->sq[i]);
4005 }
4006
4007 static void
4008 mlx5e_reset_sq_doorbell_record(struct mlx5e_sq *sq)
4009 {
4010
4011         sq->doorbell.d32[0] = cpu_to_be32(MLX5_OPCODE_NOP);
4012         sq->doorbell.d32[1] = cpu_to_be32(sq->sqn << 8);
4013         mlx5e_tx_notify_hw(sq, true);
4014 }
4015
4016 void
4017 mlx5e_resume_sq(struct mlx5e_sq *sq)
4018 {
4019         int err;
4020
4021         /* check if already enabled */
4022         if (READ_ONCE(sq->running) != 0)
4023                 return;
4024
4025         err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_ERR,
4026             MLX5_SQC_STATE_RST);
4027         if (err != 0) {
4028                 mlx5_en_err(sq->ifp,
4029                     "mlx5e_modify_sq() from ERR to RST failed: %d\n", err);
4030         }
4031
4032         sq->cc = 0;
4033         sq->pc = 0;
4034
4035         /* reset doorbell prior to moving from RST to RDY */
4036         mlx5e_reset_sq_doorbell_record(sq);
4037
4038         err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST,
4039             MLX5_SQC_STATE_RDY);
4040         if (err != 0) {
4041                 mlx5_en_err(sq->ifp,
4042                     "mlx5e_modify_sq() from RST to RDY failed: %d\n", err);
4043         }
4044
4045         sq->cev_next_state = MLX5E_CEV_STATE_INITIAL;
4046         WRITE_ONCE(sq->running, 1);
4047 }
4048
4049 static void
4050 mlx5e_enable_tx_dma(struct mlx5e_channel *ch)
4051 {
4052         int i;
4053
4054         for (i = 0; i < ch->priv->num_tc; i++)
4055                 mlx5e_resume_sq(&ch->sq[i]);
4056 }
4057
4058 static void
4059 mlx5e_disable_rx_dma(struct mlx5e_channel *ch)
4060 {
4061         struct mlx5e_rq *rq = &ch->rq;
4062         struct epoch_tracker et;
4063         int err;
4064
4065         mtx_lock(&rq->mtx);
4066         rq->enabled = 0;
4067         callout_stop(&rq->watchdog);
4068         mtx_unlock(&rq->mtx);
4069
4070         err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
4071         if (err != 0) {
4072                 mlx5_en_err(rq->ifp,
4073                     "mlx5e_modify_rq() from RDY to RST failed: %d\n", err);
4074         }
4075
4076         while (!mlx5_wq_ll_is_empty(&rq->wq)) {
4077                 msleep(1);
4078                 NET_EPOCH_ENTER(et);
4079                 rq->cq.mcq.comp(&rq->cq.mcq, NULL);
4080                 NET_EPOCH_EXIT(et);
4081         }
4082
4083         /*
4084          * Transitioning into RST state will allow the FW to track less ERR state queues,
4085          * thus reducing the recv queue flushing time
4086          */
4087         err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_ERR, MLX5_RQC_STATE_RST);
4088         if (err != 0) {
4089                 mlx5_en_err(rq->ifp,
4090                     "mlx5e_modify_rq() from ERR to RST failed: %d\n", err);
4091         }
4092 }
4093
4094 static void
4095 mlx5e_enable_rx_dma(struct mlx5e_channel *ch)
4096 {
4097         struct mlx5e_rq *rq = &ch->rq;
4098         struct epoch_tracker et;
4099         int err;
4100
4101         rq->wq.wqe_ctr = 0;
4102         mlx5_wq_ll_update_db_record(&rq->wq);
4103         err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
4104         if (err != 0) {
4105                 mlx5_en_err(rq->ifp,
4106                     "mlx5e_modify_rq() from RST to RDY failed: %d\n", err);
4107         }
4108
4109         rq->enabled = 1;
4110
4111         NET_EPOCH_ENTER(et);
4112         rq->cq.mcq.comp(&rq->cq.mcq, NULL);
4113         NET_EPOCH_EXIT(et);
4114 }
4115
4116 void
4117 mlx5e_modify_tx_dma(struct mlx5e_priv *priv, uint8_t value)
4118 {
4119         int i;
4120
4121         if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
4122                 return;
4123
4124         for (i = 0; i < priv->params.num_channels; i++) {
4125                 if (value)
4126                         mlx5e_disable_tx_dma(&priv->channel[i]);
4127                 else
4128                         mlx5e_enable_tx_dma(&priv->channel[i]);
4129         }
4130 }
4131
4132 void
4133 mlx5e_modify_rx_dma(struct mlx5e_priv *priv, uint8_t value)
4134 {
4135         int i;
4136
4137         if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
4138                 return;
4139
4140         for (i = 0; i < priv->params.num_channels; i++) {
4141                 if (value)
4142                         mlx5e_disable_rx_dma(&priv->channel[i]);
4143                 else
4144                         mlx5e_enable_rx_dma(&priv->channel[i]);
4145         }
4146 }
4147
4148 static void
4149 mlx5e_add_hw_stats(struct mlx5e_priv *priv)
4150 {
4151         SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw),
4152             OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
4153             priv, 0, sysctl_firmware, "A", "HCA firmware version");
4154
4155         SYSCTL_ADD_STRING(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw),
4156             OID_AUTO, "board_id", CTLFLAG_RD, priv->mdev->board_id, 0,
4157             "Board ID");
4158 }
4159
4160 static int
4161 mlx5e_sysctl_tx_priority_flow_control(SYSCTL_HANDLER_ARGS)
4162 {
4163         struct mlx5e_priv *priv = arg1;
4164         uint8_t temp[MLX5E_MAX_PRIORITY];
4165         uint32_t tx_pfc;
4166         int err;
4167         int i;
4168
4169         PRIV_LOCK(priv);
4170
4171         tx_pfc = priv->params.tx_priority_flow_control;
4172
4173         for (i = 0; i != MLX5E_MAX_PRIORITY; i++)
4174                 temp[i] = (tx_pfc >> i) & 1;
4175
4176         err = SYSCTL_OUT(req, temp, MLX5E_MAX_PRIORITY);
4177         if (err || !req->newptr)
4178                 goto done;
4179         err = SYSCTL_IN(req, temp, MLX5E_MAX_PRIORITY);
4180         if (err)
4181                 goto done;
4182
4183         priv->params.tx_priority_flow_control = 0;
4184
4185         /* range check input value */
4186         for (i = 0; i != MLX5E_MAX_PRIORITY; i++) {
4187                 if (temp[i] > 1) {
4188                         err = ERANGE;
4189                         goto done;
4190                 }
4191                 priv->params.tx_priority_flow_control |= (temp[i] << i);
4192         }
4193
4194         /* check if update is required */
4195         if (tx_pfc != priv->params.tx_priority_flow_control)
4196                 err = -mlx5e_set_port_pfc(priv);
4197 done:
4198         if (err != 0)
4199                 priv->params.tx_priority_flow_control= tx_pfc;
4200         PRIV_UNLOCK(priv);
4201
4202         return (err);
4203 }
4204
4205 static int
4206 mlx5e_sysctl_rx_priority_flow_control(SYSCTL_HANDLER_ARGS)
4207 {
4208         struct mlx5e_priv *priv = arg1;
4209         uint8_t temp[MLX5E_MAX_PRIORITY];
4210         uint32_t rx_pfc;
4211         int err;
4212         int i;
4213
4214         PRIV_LOCK(priv);
4215
4216         rx_pfc = priv->params.rx_priority_flow_control;
4217
4218         for (i = 0; i != MLX5E_MAX_PRIORITY; i++)
4219                 temp[i] = (rx_pfc >> i) & 1;
4220
4221         err = SYSCTL_OUT(req, temp, MLX5E_MAX_PRIORITY);
4222         if (err || !req->newptr)
4223                 goto done;
4224         err = SYSCTL_IN(req, temp, MLX5E_MAX_PRIORITY);
4225         if (err)
4226                 goto done;
4227
4228         priv->params.rx_priority_flow_control = 0;
4229
4230         /* range check input value */
4231         for (i = 0; i != MLX5E_MAX_PRIORITY; i++) {
4232                 if (temp[i] > 1) {
4233                         err = ERANGE;
4234                         goto done;
4235                 }
4236                 priv->params.rx_priority_flow_control |= (temp[i] << i);
4237         }
4238
4239         /* check if update is required */
4240         if (rx_pfc != priv->params.rx_priority_flow_control) {
4241                 err = -mlx5e_set_port_pfc(priv);
4242                 if (err == 0 && priv->sw_is_port_buf_owner)
4243                         err = mlx5e_update_buf_lossy(priv);
4244         }
4245 done:
4246         if (err != 0)
4247                 priv->params.rx_priority_flow_control= rx_pfc;
4248         PRIV_UNLOCK(priv);
4249
4250         return (err);
4251 }
4252
4253 static void
4254 mlx5e_setup_pauseframes(struct mlx5e_priv *priv)
4255 {
4256         int error;
4257
4258         /* enable pauseframes by default */
4259         priv->params.tx_pauseframe_control = 1;
4260         priv->params.rx_pauseframe_control = 1;
4261
4262         /* disable ports flow control, PFC, by default */
4263         priv->params.tx_priority_flow_control = 0;
4264         priv->params.rx_priority_flow_control = 0;
4265
4266         /* register pauseframe SYSCTLs */
4267         SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4268             OID_AUTO, "tx_pauseframe_control", CTLFLAG_RDTUN,
4269             &priv->params.tx_pauseframe_control, 0,
4270             "Set to enable TX pause frames. Clear to disable.");
4271
4272         SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4273             OID_AUTO, "rx_pauseframe_control", CTLFLAG_RDTUN,
4274             &priv->params.rx_pauseframe_control, 0,
4275             "Set to enable RX pause frames. Clear to disable.");
4276
4277         /* register priority flow control, PFC, SYSCTLs */
4278         SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4279             OID_AUTO, "tx_priority_flow_control", CTLTYPE_U8 | CTLFLAG_RWTUN |
4280             CTLFLAG_MPSAFE, priv, 0, &mlx5e_sysctl_tx_priority_flow_control, "CU",
4281             "Set to enable TX ports flow control frames for priorities 0..7. Clear to disable.");
4282
4283         SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4284             OID_AUTO, "rx_priority_flow_control", CTLTYPE_U8 | CTLFLAG_RWTUN |
4285             CTLFLAG_MPSAFE, priv, 0, &mlx5e_sysctl_rx_priority_flow_control, "CU",
4286             "Set to enable RX ports flow control frames for priorities 0..7. Clear to disable.");
4287
4288         PRIV_LOCK(priv);
4289
4290         /* range check */
4291         priv->params.tx_pauseframe_control =
4292             priv->params.tx_pauseframe_control ? 1 : 0;
4293         priv->params.rx_pauseframe_control =
4294             priv->params.rx_pauseframe_control ? 1 : 0;
4295
4296         /* update firmware */
4297         error = mlx5e_set_port_pause_and_pfc(priv);
4298         if (error == -EINVAL) {
4299                 mlx5_en_err(priv->ifp,
4300                     "Global pauseframes must be disabled before enabling PFC.\n");
4301                 priv->params.rx_priority_flow_control = 0;
4302                 priv->params.tx_priority_flow_control = 0;
4303
4304                 /* update firmware */
4305                 (void) mlx5e_set_port_pause_and_pfc(priv);
4306         }
4307         PRIV_UNLOCK(priv);
4308 }
4309
4310 static int
4311 mlx5e_ul_snd_tag_alloc(if_t ifp,
4312     union if_snd_tag_alloc_params *params,
4313     struct m_snd_tag **ppmt)
4314 {
4315         struct mlx5e_priv *priv;
4316         struct mlx5e_channel *pch;
4317
4318         priv = if_getsoftc(ifp);
4319
4320         if (unlikely(priv->gone || params->hdr.flowtype == M_HASHTYPE_NONE)) {
4321                 return (EOPNOTSUPP);
4322         } else {
4323                 /* keep this code synced with mlx5e_select_queue() */
4324                 u32 ch = priv->params.num_channels;
4325 #ifdef RSS
4326                 u32 temp;
4327
4328                 if (rss_hash2bucket(params->hdr.flowid,
4329                     params->hdr.flowtype, &temp) == 0)
4330                         ch = temp % ch;
4331                 else
4332 #endif
4333                         ch = (params->hdr.flowid % 128) % ch;
4334
4335                 /*
4336                  * NOTE: The channels array is only freed at detach
4337                  * and it safe to return a pointer to the send tag
4338                  * inside the channels structure as long as we
4339                  * reference the priv.
4340                  */
4341                 pch = priv->channel + ch;
4342
4343                 /* check if send queue is not running */
4344                 if (unlikely(pch->sq[0].running == 0))
4345                         return (ENXIO);
4346                 m_snd_tag_ref(&pch->tag);
4347                 *ppmt = &pch->tag;
4348                 return (0);
4349         }
4350 }
4351
4352 static int
4353 mlx5e_ul_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *params)
4354 {
4355         struct mlx5e_channel *pch =
4356             container_of(pmt, struct mlx5e_channel, tag);
4357
4358         params->unlimited.max_rate = -1ULL;
4359         params->unlimited.queue_level = mlx5e_sq_queue_level(&pch->sq[0]);
4360         return (0);
4361 }
4362
4363 static void
4364 mlx5e_ul_snd_tag_free(struct m_snd_tag *pmt)
4365 {
4366         struct mlx5e_channel *pch =
4367             container_of(pmt, struct mlx5e_channel, tag);
4368
4369         complete(&pch->completion);
4370 }
4371
4372 static int
4373 mlx5e_snd_tag_alloc(if_t ifp,
4374     union if_snd_tag_alloc_params *params,
4375     struct m_snd_tag **ppmt)
4376 {
4377
4378         switch (params->hdr.type) {
4379 #ifdef RATELIMIT
4380         case IF_SND_TAG_TYPE_RATE_LIMIT:
4381                 return (mlx5e_rl_snd_tag_alloc(ifp, params, ppmt));
4382 #ifdef KERN_TLS
4383         case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
4384                 return (mlx5e_tls_snd_tag_alloc(ifp, params, ppmt));
4385 #endif
4386 #endif
4387         case IF_SND_TAG_TYPE_UNLIMITED:
4388                 return (mlx5e_ul_snd_tag_alloc(ifp, params, ppmt));
4389 #ifdef KERN_TLS
4390         case IF_SND_TAG_TYPE_TLS:
4391                 return (mlx5e_tls_snd_tag_alloc(ifp, params, ppmt));
4392         case IF_SND_TAG_TYPE_TLS_RX:
4393                 return (mlx5e_tls_rx_snd_tag_alloc(ifp, params, ppmt));
4394 #endif
4395         default:
4396                 return (EOPNOTSUPP);
4397         }
4398 }
4399
4400 #ifdef RATELIMIT
4401 #define NUM_HDWR_RATES_MLX 13
4402 static const uint64_t adapter_rates_mlx[NUM_HDWR_RATES_MLX] = {
4403         135375,                 /* 1,083,000 */
4404         180500,                 /* 1,444,000 */
4405         270750,                 /* 2,166,000 */
4406         361000,                 /* 2,888,000 */
4407         541500,                 /* 4,332,000 */
4408         721875,                 /* 5,775,000 */
4409         1082875,                /* 8,663,000 */
4410         1443875,                /* 11,551,000 */
4411         2165750,                /* 17,326,000 */
4412         2887750,                /* 23,102,000 */
4413         4331625,                /* 34,653,000 */
4414         5775500,                /* 46,204,000 */
4415         8663125                 /* 69,305,000 */
4416 };
4417
4418 static void
4419 mlx5e_ratelimit_query(if_t ifp __unused, struct if_ratelimit_query_results *q)
4420 {
4421         /*
4422          * This function needs updating by the driver maintainer!
4423          * For the MLX card there are currently (ConectX-4?) 13
4424          * pre-set rates and others i.e. ConnectX-5, 6, 7??
4425          *
4426          * This will change based on later adapters
4427          * and this code should be updated to look at ifp
4428          * and figure out the specific adapter type
4429          * settings i.e. how many rates as well
4430          * as if they are fixed (as is shown here) or
4431          * if they are dynamic (example chelsio t4). Also if there
4432          * is a maximum number of flows that the adapter
4433          * can handle that too needs to be updated in
4434          * the max_flows field.
4435          */
4436         q->rate_table = adapter_rates_mlx;
4437         q->flags = RT_IS_FIXED_TABLE;
4438         q->max_flows = 0;       /* mlx has no limit */
4439         q->number_of_rates = NUM_HDWR_RATES_MLX;
4440         q->min_segment_burst = 1;
4441 }
4442 #endif
4443
4444 static void
4445 mlx5e_ifm_add(struct mlx5e_priv *priv, int type)
4446 {
4447         ifmedia_add(&priv->media, type | IFM_ETHER, 0, NULL);
4448         ifmedia_add(&priv->media, type | IFM_ETHER |
4449             IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE, 0, NULL);
4450         ifmedia_add(&priv->media, type | IFM_ETHER | IFM_ETH_RXPAUSE, 0, NULL);
4451         ifmedia_add(&priv->media, type | IFM_ETHER | IFM_ETH_TXPAUSE, 0, NULL);
4452         ifmedia_add(&priv->media, type | IFM_ETHER | IFM_FDX, 0, NULL);
4453         ifmedia_add(&priv->media, type | IFM_ETHER | IFM_FDX |
4454             IFM_ETH_RXPAUSE, 0, NULL);
4455         ifmedia_add(&priv->media, type | IFM_ETHER | IFM_FDX |
4456             IFM_ETH_TXPAUSE, 0, NULL);
4457         ifmedia_add(&priv->media, type | IFM_ETHER | IFM_FDX |
4458             IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE, 0, NULL);
4459 }
4460
4461 static void *
4462 mlx5e_create_ifp(struct mlx5_core_dev *mdev)
4463 {
4464         if_t ifp;
4465         struct mlx5e_priv *priv;
4466         u8 dev_addr[ETHER_ADDR_LEN] __aligned(4);
4467         struct sysctl_oid_list *child;
4468         int ncv = mdev->priv.eq_table.num_comp_vectors;
4469         char unit[16];
4470         struct pfil_head_args pa;
4471         int err;
4472         u32 eth_proto_cap;
4473         u32 out[MLX5_ST_SZ_DW(ptys_reg)];
4474         bool ext;
4475         struct media media_entry = {};
4476
4477         if (mlx5e_check_required_hca_cap(mdev)) {
4478                 mlx5_core_dbg(mdev, "mlx5e_check_required_hca_cap() failed\n");
4479                 return (NULL);
4480         }
4481
4482         /*
4483          * Try to allocate the priv and make room for worst-case
4484          * number of channel structures:
4485          */
4486         priv = malloc_domainset(sizeof(*priv) +
4487             (sizeof(priv->channel[0]) * mdev->priv.eq_table.num_comp_vectors),
4488             M_MLX5EN, mlx5_dev_domainset(mdev), M_WAITOK | M_ZERO);
4489
4490         ifp = priv->ifp = if_alloc_dev(IFT_ETHER, mdev->pdev->dev.bsddev);
4491         if (ifp == NULL) {
4492                 mlx5_core_err(mdev, "if_alloc() failed\n");
4493                 goto err_free_priv;
4494         }
4495         /* setup all static fields */
4496         if (mlx5e_priv_static_init(priv, mdev, mdev->priv.eq_table.num_comp_vectors)) {
4497                 mlx5_core_err(mdev, "mlx5e_priv_static_init() failed\n");
4498                 goto err_free_ifp;
4499         }
4500
4501         if_setsoftc(ifp, priv);
4502         if_initname(ifp, "mce", device_get_unit(mdev->pdev->dev.bsddev));
4503         if_setmtu(ifp, ETHERMTU);
4504         if_setinitfn(ifp, mlx5e_open);
4505         if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
4506         if_setioctlfn(ifp, mlx5e_ioctl);
4507         if_settransmitfn(ifp, mlx5e_xmit);
4508         if_setqflushfn(ifp, if_qflush);
4509         if_setgetcounterfn(ifp, mlx5e_get_counter);
4510         if_setsendqlen(ifp, ifqmaxlen);
4511         /*
4512          * Set driver features
4513          */
4514         if_setcapabilities(ifp, IFCAP_NV);
4515         if_setcapabilitiesbit(ifp, IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6, 0);
4516         if_setcapabilitiesbit(ifp, IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING, 0);
4517         if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWFILTER, 0);
4518         if_setcapabilitiesbit(ifp, IFCAP_LINKSTATE | IFCAP_JUMBO_MTU, 0);
4519         if_setcapabilitiesbit(ifp, IFCAP_LRO, 0);
4520         if_setcapabilitiesbit(ifp, IFCAP_TSO | IFCAP_VLAN_HWTSO, 0);
4521         if_setcapabilitiesbit(ifp, IFCAP_HWSTATS | IFCAP_HWRXTSTMP, 0);
4522         if_setcapabilitiesbit(ifp, IFCAP_MEXTPG, 0);
4523         if_setcapabilitiesbit(ifp, IFCAP_TXTLS4 | IFCAP_TXTLS6, 0);
4524 #ifdef RATELIMIT
4525         if_setcapabilitiesbit(ifp, IFCAP_TXRTLMT | IFCAP_TXTLS_RTLMT, 0);
4526 #endif
4527         if_setcapabilitiesbit(ifp, IFCAP_VXLAN_HWCSUM | IFCAP_VXLAN_HWTSO, 0);
4528         if_setcapabilities2bit(ifp, IFCAP2_BIT(IFCAP2_RXTLS4) |
4529             IFCAP2_BIT(IFCAP2_RXTLS6), 0);
4530         if_setsndtagallocfn(ifp, mlx5e_snd_tag_alloc);
4531 #ifdef RATELIMIT
4532         if_setratelimitqueryfn(ifp, mlx5e_ratelimit_query);
4533 #endif
4534         /* set TSO limits so that we don't have to drop TX packets */
4535         if_sethwtsomax(ifp, MLX5E_MAX_TX_PAYLOAD_SIZE - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN));
4536         if_sethwtsomaxsegcount(ifp, MLX5E_MAX_TX_MBUF_FRAGS - 1 /* hdr */);
4537         if_sethwtsomaxsegsize(ifp, MLX5E_MAX_TX_MBUF_SIZE);
4538
4539         if_setcapenable(ifp, if_getcapabilities(ifp));
4540         if_setcapenable2(ifp, if_getcapabilities2(ifp));
4541         if_sethwassist(ifp, 0);
4542         if (if_getcapenable(ifp) & IFCAP_TSO)
4543                 if_sethwassistbits(ifp, CSUM_TSO, 0);
4544         if (if_getcapenable(ifp) & IFCAP_TXCSUM)
4545                 if_sethwassistbits(ifp, (CSUM_TCP | CSUM_UDP | CSUM_IP), 0);
4546         if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6)
4547                 if_sethwassistbits(ifp, (CSUM_UDP_IPV6 | CSUM_TCP_IPV6), 0);
4548         if (if_getcapabilities(ifp) & IFCAP_VXLAN_HWCSUM)
4549                 if_sethwassistbits(ifp, CSUM_INNER_IP6_UDP | CSUM_INNER_IP6_TCP |
4550                     CSUM_INNER_IP | CSUM_INNER_IP_UDP | CSUM_INNER_IP_TCP |
4551                     CSUM_ENCAP_VXLAN, 0);
4552         if (if_getcapabilities(ifp) & IFCAP_VXLAN_HWTSO)
4553                 if_sethwassistbits(ifp, CSUM_INNER_IP6_TSO | CSUM_INNER_IP_TSO, 0);
4554
4555         /* ifnet sysctl tree */
4556         sysctl_ctx_init(&priv->sysctl_ctx);
4557         priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_STATIC_CHILDREN(_dev),
4558             OID_AUTO, if_getdname(ifp), CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
4559             "MLX5 ethernet - interface name");
4560         if (priv->sysctl_ifnet == NULL) {
4561                 mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
4562                 goto err_free_sysctl;
4563         }
4564         snprintf(unit, sizeof(unit), "%d", if_getdunit(ifp));
4565         priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4566             OID_AUTO, unit, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
4567             "MLX5 ethernet - interface unit");
4568         if (priv->sysctl_ifnet == NULL) {
4569                 mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
4570                 goto err_free_sysctl;
4571         }
4572
4573         /* HW sysctl tree */
4574         child = SYSCTL_CHILDREN(device_get_sysctl_tree(mdev->pdev->dev.bsddev));
4575         priv->sysctl_hw = SYSCTL_ADD_NODE(&priv->sysctl_ctx, child,
4576             OID_AUTO, "hw", CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
4577             "MLX5 ethernet dev hw");
4578         if (priv->sysctl_hw == NULL) {
4579                 mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
4580                 goto err_free_sysctl;
4581         }
4582
4583         err = mlx5e_build_ifp_priv(mdev, priv, ncv);
4584         if (err) {
4585                 mlx5_core_err(mdev, "mlx5e_build_ifp_priv() failed (%d)\n", err);
4586                 goto err_free_sysctl;
4587         }
4588
4589         /* reuse mlx5core's watchdog workqueue */
4590         priv->wq = mdev->priv.health.wq_watchdog;
4591
4592         err = mlx5_core_alloc_pd(mdev, &priv->pdn, 0);
4593         if (err) {
4594                 mlx5_en_err(ifp, "mlx5_core_alloc_pd failed, %d\n", err);
4595                 goto err_free_wq;
4596         }
4597         err = mlx5_alloc_transport_domain(mdev, &priv->tdn, 0);
4598         if (err) {
4599                 mlx5_en_err(ifp,
4600                     "mlx5_alloc_transport_domain failed, %d\n", err);
4601                 goto err_dealloc_pd;
4602         }
4603         err = mlx5e_create_mkey(priv, priv->pdn, &priv->mr);
4604         if (err) {
4605                 mlx5_en_err(ifp, "mlx5e_create_mkey failed, %d\n", err);
4606                 goto err_dealloc_transport_domain;
4607         }
4608         mlx5_query_nic_vport_mac_address(priv->mdev, 0, dev_addr);
4609
4610         /* check if we should generate a random MAC address */
4611         if (MLX5_CAP_GEN(priv->mdev, vport_group_manager) == 0 &&
4612             is_zero_ether_addr(dev_addr)) {
4613                 random_ether_addr(dev_addr);
4614                 mlx5_en_err(ifp, "Assigned random MAC address\n");
4615         }
4616
4617         err = mlx5e_rl_init(priv);
4618         if (err) {
4619                 mlx5_en_err(ifp, "mlx5e_rl_init failed, %d\n", err);
4620                 goto err_create_mkey;
4621         }
4622
4623         err = mlx5e_tls_init(priv);
4624         if (err) {
4625                 if_printf(ifp, "%s: mlx5e_tls_init failed\n", __func__);
4626                 goto err_rl_init;
4627         }
4628
4629         err = mlx5e_open_drop_rq(priv, &priv->drop_rq);
4630         if (err) {
4631                 if_printf(ifp, "%s: mlx5e_open_drop_rq failed (%d)\n", __func__, err);
4632                 goto err_tls_init;
4633         }
4634
4635         err = mlx5e_open_rqts(priv);
4636         if (err) {
4637                 if_printf(ifp, "%s: mlx5e_open_rqts failed (%d)\n", __func__, err);
4638                 goto err_open_drop_rq;
4639         }
4640
4641         err = mlx5e_open_tirs(priv);
4642         if (err) {
4643                 mlx5_en_err(ifp, "mlx5e_open_tirs() failed, %d\n", err);
4644                 goto err_open_rqts;
4645         }
4646
4647         err = mlx5e_open_flow_tables(priv);
4648         if (err) {
4649                 if_printf(ifp, "%s: mlx5e_open_flow_tables failed (%d)\n", __func__, err);
4650                 goto err_open_tirs;
4651         }
4652
4653         err = mlx5e_tls_rx_init(priv);
4654         if (err) {
4655                 if_printf(ifp, "%s: mlx5e_tls_rx_init() failed, %d\n", __func__, err);
4656                 goto err_open_flow_tables;
4657         }
4658
4659         /* set default MTU */
4660         mlx5e_set_dev_port_mtu(ifp, if_getmtu(ifp));
4661
4662         /* Set default media status */
4663         priv->media_status_last = IFM_AVALID;
4664         priv->media_active_last = IFM_ETHER | IFM_AUTO | IFM_FDX;
4665
4666         /* setup default pauseframes configuration */
4667         mlx5e_setup_pauseframes(priv);
4668
4669         /* Setup supported medias */
4670         if (!mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN, 1)) {
4671                 ext = MLX5_CAP_PCAM_FEATURE(mdev,
4672                     ptys_extended_ethernet);
4673                 eth_proto_cap = MLX5_GET_ETH_PROTO(ptys_reg, out, ext,
4674                     eth_proto_capability);
4675         } else {
4676                 ext = false;
4677                 eth_proto_cap = 0;
4678                 mlx5_en_err(ifp, "Query port media capability failed, %d\n", err);
4679         }
4680
4681         ifmedia_init(&priv->media, IFM_IMASK,
4682             mlx5e_media_change, mlx5e_media_status);
4683
4684         if (ext) {
4685                 for (unsigned i = 0; i != MLX5E_EXT_LINK_SPEEDS_NUMBER; i++) {
4686                         /* check if hardware has the right capability */
4687                         if (MLX5E_PROT_MASK(i) & ~eth_proto_cap)
4688                                 continue;
4689                         for (unsigned j = 0; j != MLX5E_CABLE_TYPE_NUMBER; j++) {
4690                                 media_entry = mlx5e_ext_mode_table[i][j];
4691                                 if (media_entry.subtype == 0)
4692                                         continue;
4693                                 /* check if this subtype was already added */
4694                                 for (unsigned k = 0; k != i; k++) {
4695                                         /* check if hardware has the right capability */
4696                                         if (MLX5E_PROT_MASK(k) & ~eth_proto_cap)
4697                                                 continue;
4698                                         for (unsigned m = 0; m != MLX5E_CABLE_TYPE_NUMBER; m++) {
4699                                                 if (media_entry.subtype == mlx5e_ext_mode_table[k][m].subtype)
4700                                                         goto skip_ext_media;
4701                                         }
4702                                 }
4703                                 mlx5e_ifm_add(priv, media_entry.subtype);
4704                         skip_ext_media:;
4705                         }
4706                 }
4707         } else {
4708                 for (unsigned i = 0; i != MLX5E_LINK_SPEEDS_NUMBER; i++) {
4709                         media_entry = mlx5e_mode_table[i];
4710                         if (media_entry.subtype == 0)
4711                                 continue;
4712                         if (MLX5E_PROT_MASK(i) & ~eth_proto_cap)
4713                                 continue;
4714                         /* check if this subtype was already added */
4715                         for (unsigned k = 0; k != i; k++) {
4716                                 if (media_entry.subtype == mlx5e_mode_table[k].subtype)
4717                                         goto skip_media;
4718                         }
4719                         mlx5e_ifm_add(priv, media_entry.subtype);
4720
4721                         /* NOTE: 10G ER and LR shares the same entry */
4722                         if (media_entry.subtype == IFM_10G_ER)
4723                                 mlx5e_ifm_add(priv, IFM_10G_LR);
4724                 skip_media:;
4725                 }
4726         }
4727
4728         mlx5e_ifm_add(priv, IFM_AUTO);
4729
4730         /* Set autoselect by default */
4731         ifmedia_set(&priv->media, IFM_ETHER | IFM_AUTO | IFM_FDX |
4732             IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE);
4733
4734         DEBUGNET_SET(ifp, mlx5_en);
4735
4736         ether_ifattach(ifp, dev_addr);
4737
4738         /* Register for VLAN events */
4739         priv->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
4740             mlx5e_vlan_rx_add_vid, priv, EVENTHANDLER_PRI_FIRST);
4741         priv->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
4742             mlx5e_vlan_rx_kill_vid, priv, EVENTHANDLER_PRI_FIRST);
4743
4744         /* Register for VxLAN events */
4745         priv->vxlan_start = EVENTHANDLER_REGISTER(vxlan_start,
4746             mlx5e_vxlan_start, priv, EVENTHANDLER_PRI_ANY);
4747         priv->vxlan_stop = EVENTHANDLER_REGISTER(vxlan_stop,
4748             mlx5e_vxlan_stop, priv, EVENTHANDLER_PRI_ANY);
4749
4750         /* Link is down by default */
4751         if_link_state_change(ifp, LINK_STATE_DOWN);
4752
4753         mlx5e_enable_async_events(priv);
4754
4755         mlx5e_add_hw_stats(priv);
4756
4757         mlx5e_create_stats(&priv->stats.vport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4758             "vstats", mlx5e_vport_stats_desc, MLX5E_VPORT_STATS_NUM,
4759             priv->stats.vport.arg);
4760
4761         mlx5e_create_stats(&priv->stats.pport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4762             "pstats", mlx5e_pport_stats_desc, MLX5E_PPORT_STATS_NUM,
4763             priv->stats.pport.arg);
4764
4765         mlx5e_create_ethtool(priv);
4766
4767         mtx_lock(&priv->async_events_mtx);
4768         mlx5e_update_stats(priv);
4769         mtx_unlock(&priv->async_events_mtx);
4770
4771         SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4772             OID_AUTO, "rx_clbr_done", CTLFLAG_RD,
4773             &priv->clbr_done, 0,
4774             "RX timestamps calibration state");
4775         callout_init(&priv->tstmp_clbr, 1);
4776         /* Pull out the frequency of the clock in hz */
4777         priv->cclk = (uint64_t)MLX5_CAP_GEN(mdev, device_frequency_khz) * 1000ULL;
4778         mlx5e_reset_calibration_callout(priv);
4779
4780         pa.pa_version = PFIL_VERSION;
4781         pa.pa_flags = PFIL_IN;
4782         pa.pa_type = PFIL_TYPE_ETHERNET;
4783         pa.pa_headname = if_name(ifp);
4784         priv->pfil = pfil_head_register(&pa);
4785
4786         PRIV_LOCK(priv);
4787         err = mlx5e_open_flow_rules(priv);
4788         if (err) {
4789                 mlx5_en_err(ifp,
4790                     "mlx5e_open_flow_rules() failed, %d (ignored)\n", err);
4791         }
4792         PRIV_UNLOCK(priv);
4793
4794         return (priv);
4795
4796 err_open_flow_tables:
4797         mlx5e_close_flow_tables(priv);
4798
4799 err_open_tirs:
4800         mlx5e_close_tirs(priv);
4801
4802 err_open_rqts:
4803         mlx5e_close_rqts(priv);
4804
4805 err_open_drop_rq:
4806         mlx5e_close_drop_rq(&priv->drop_rq);
4807
4808 err_tls_init:
4809         mlx5e_tls_cleanup(priv);
4810
4811 err_rl_init:
4812         mlx5e_rl_cleanup(priv);
4813
4814 err_create_mkey:
4815         mlx5_core_destroy_mkey(priv->mdev, &priv->mr);
4816
4817 err_dealloc_transport_domain:
4818         mlx5_dealloc_transport_domain(mdev, priv->tdn, 0);
4819
4820 err_dealloc_pd:
4821         mlx5_core_dealloc_pd(mdev, priv->pdn, 0);
4822
4823 err_free_wq:
4824         flush_workqueue(priv->wq);
4825
4826 err_free_sysctl:
4827         sysctl_ctx_free(&priv->sysctl_ctx);
4828         if (priv->sysctl_debug)
4829                 sysctl_ctx_free(&priv->stats.port_stats_debug.ctx);
4830         mlx5e_priv_static_destroy(priv, mdev, mdev->priv.eq_table.num_comp_vectors);
4831
4832 err_free_ifp:
4833         if_free(ifp);
4834
4835 err_free_priv:
4836         free(priv, M_MLX5EN);
4837         return (NULL);
4838 }
4839
4840 static void
4841 mlx5e_destroy_ifp(struct mlx5_core_dev *mdev, void *vpriv)
4842 {
4843         struct mlx5e_priv *priv = vpriv;
4844         if_t ifp = priv->ifp;
4845
4846         /* don't allow more IOCTLs */
4847         priv->gone = 1;
4848
4849         /* XXX wait a bit to allow IOCTL handlers to complete */
4850         pause("W", hz);
4851
4852 #ifdef RATELIMIT
4853         /*
4854          * The kernel can have reference(s) via the m_snd_tag's into
4855          * the ratelimit channels, and these must go away before
4856          * detaching:
4857          */
4858         while (READ_ONCE(priv->rl.stats.tx_active_connections) != 0) {
4859                 mlx5_en_err(priv->ifp,
4860                     "Waiting for all ratelimit connections to terminate\n");
4861                 pause("W", hz);
4862         }
4863 #endif
4864
4865 #ifdef KERN_TLS
4866         /* wait for all TLS tags to get freed */
4867         while (priv->tls.init != 0 &&
4868             uma_zone_get_cur(priv->tls.zone) != 0)  {
4869                 mlx5_en_err(priv->ifp,
4870                     "Waiting for all TLS connections to terminate\n");
4871                 pause("W", hz);
4872         }
4873
4874         /* wait for all TLS RX tags to get freed */
4875         while (priv->tls_rx.init != 0 &&
4876             uma_zone_get_cur(priv->tls_rx.zone) != 0)  {
4877                 mlx5_en_err(priv->ifp,
4878                     "Waiting for all TLS RX connections to terminate\n");
4879                 pause("W", hz);
4880         }
4881 #endif
4882         /* wait for all unlimited send tags to complete */
4883         mlx5e_priv_wait_for_completion(priv, mdev->priv.eq_table.num_comp_vectors);
4884
4885         /* stop watchdog timer */
4886         callout_drain(&priv->watchdog);
4887
4888         callout_drain(&priv->tstmp_clbr);
4889
4890         if (priv->vlan_attach != NULL)
4891                 EVENTHANDLER_DEREGISTER(vlan_config, priv->vlan_attach);
4892         if (priv->vlan_detach != NULL)
4893                 EVENTHANDLER_DEREGISTER(vlan_unconfig, priv->vlan_detach);
4894         if (priv->vxlan_start != NULL)
4895                 EVENTHANDLER_DEREGISTER(vxlan_start, priv->vxlan_start);
4896         if (priv->vxlan_stop != NULL)
4897                 EVENTHANDLER_DEREGISTER(vxlan_stop, priv->vxlan_stop);
4898
4899         /* make sure device gets closed */
4900         PRIV_LOCK(priv);
4901         mlx5e_close_locked(ifp);
4902         mlx5e_close_flow_rules(priv);
4903         PRIV_UNLOCK(priv);
4904
4905         /* deregister pfil */
4906         if (priv->pfil != NULL) {
4907                 pfil_head_unregister(priv->pfil);
4908                 priv->pfil = NULL;
4909         }
4910
4911         /* unregister device */
4912         ifmedia_removeall(&priv->media);
4913         ether_ifdetach(ifp);
4914
4915         mlx5e_tls_rx_cleanup(priv);
4916         mlx5e_close_flow_tables(priv);
4917         mlx5e_close_tirs(priv);
4918         mlx5e_close_rqts(priv);
4919         mlx5e_close_drop_rq(&priv->drop_rq);
4920         mlx5e_tls_cleanup(priv);
4921         mlx5e_rl_cleanup(priv);
4922
4923         /* destroy all remaining sysctl nodes */
4924         sysctl_ctx_free(&priv->stats.vport.ctx);
4925         sysctl_ctx_free(&priv->stats.pport.ctx);
4926         if (priv->sysctl_debug)
4927                 sysctl_ctx_free(&priv->stats.port_stats_debug.ctx);
4928         sysctl_ctx_free(&priv->sysctl_ctx);
4929
4930         mlx5_core_destroy_mkey(priv->mdev, &priv->mr);
4931         mlx5_dealloc_transport_domain(priv->mdev, priv->tdn, 0);
4932         mlx5_core_dealloc_pd(priv->mdev, priv->pdn, 0);
4933         mlx5e_disable_async_events(priv);
4934         flush_workqueue(priv->wq);
4935         mlx5e_priv_static_destroy(priv, mdev, mdev->priv.eq_table.num_comp_vectors);
4936         if_free(ifp);
4937         free(priv, M_MLX5EN);
4938 }
4939
4940 #ifdef DEBUGNET
4941 static void
4942 mlx5_en_debugnet_init(if_t dev, int *nrxr, int *ncl, int *clsize)
4943 {
4944         struct mlx5e_priv *priv = if_getsoftc(dev);
4945
4946         PRIV_LOCK(priv);
4947         *nrxr = priv->params.num_channels;
4948         *ncl = DEBUGNET_MAX_IN_FLIGHT;
4949         *clsize = MLX5E_MAX_RX_BYTES;
4950         PRIV_UNLOCK(priv);
4951 }
4952
4953 static void
4954 mlx5_en_debugnet_event(if_t dev, enum debugnet_ev event)
4955 {
4956 }
4957
4958 static int
4959 mlx5_en_debugnet_transmit(if_t dev, struct mbuf *m)
4960 {
4961         struct mlx5e_priv *priv = if_getsoftc(dev);
4962         struct mlx5e_sq *sq;
4963         int err;
4964
4965         if ((if_getdrvflags(dev) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
4966             IFF_DRV_RUNNING || (priv->media_status_last & IFM_ACTIVE) == 0)
4967                 return (ENOENT);
4968
4969         sq = &priv->channel[0].sq[0];
4970
4971         if (sq->running == 0) {
4972                 m_freem(m);
4973                 return (ENOENT);
4974         }
4975
4976         if (mlx5e_sq_xmit(sq, &m) != 0) {
4977                 m_freem(m);
4978                 err = ENOBUFS;
4979         } else {
4980                 err = 0;
4981         }
4982
4983         mlx5e_tx_notify_hw(sq, true);
4984
4985         return (err);
4986 }
4987
4988 static int
4989 mlx5_en_debugnet_poll(if_t dev, int count)
4990 {
4991         struct mlx5e_priv *priv = if_getsoftc(dev);
4992
4993         if ((if_getdrvflags(dev) & IFF_DRV_RUNNING) == 0 ||
4994             (priv->media_status_last & IFM_ACTIVE) == 0)
4995                 return (ENOENT);
4996
4997         mlx5_poll_interrupts(priv->mdev);
4998
4999         return (0);
5000 }
5001 #endif /* DEBUGNET */
5002
5003 static void *
5004 mlx5e_get_ifp(void *vpriv)
5005 {
5006         struct mlx5e_priv *priv = vpriv;
5007
5008         return (priv->ifp);
5009 }
5010
5011 static struct mlx5_interface mlx5e_interface = {
5012         .add = mlx5e_create_ifp,
5013         .remove = mlx5e_destroy_ifp,
5014         .event = mlx5e_async_event,
5015         .protocol = MLX5_INTERFACE_PROTOCOL_ETH,
5016         .get_dev = mlx5e_get_ifp,
5017 };
5018
5019 void
5020 mlx5e_init(void)
5021 {
5022         mlx5_register_interface(&mlx5e_interface);
5023 }
5024
5025 void
5026 mlx5e_cleanup(void)
5027 {
5028         mlx5_unregister_interface(&mlx5e_interface);
5029 }
5030
5031 module_init_order(mlx5e_init, SI_ORDER_SIXTH);
5032 module_exit_order(mlx5e_cleanup, SI_ORDER_SIXTH);
5033
5034 MODULE_DEPEND(mlx5en, linuxkpi, 1, 1, 1);
5035 MODULE_DEPEND(mlx5en, mlx5, 1, 1, 1);
5036 MODULE_VERSION(mlx5en, 1);