sys/dev/mlx5/mlx5_en/mlx5_en_main.c

   1 /*-
   2  * Copyright (c) 2015-2018 Mellanox Technologies. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
  14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  16  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
  17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  23  * SUCH DAMAGE.
  24  *
  25  * $FreeBSD$
  26  */
  27
  28 #include "en.h"
  29
  30 #include <sys/sockio.h>
  31 #include <machine/atomic.h>
  32
  33 #ifndef ETH_DRIVER_VERSION
  34 #define ETH_DRIVER_VERSION      "3.5.2"
  35 #endif
  36 #define DRIVER_RELDATE  "September 2019"
  37
  38 static const char mlx5e_version[] = "mlx5en: Mellanox Ethernet driver "
  39         ETH_DRIVER_VERSION " (" DRIVER_RELDATE ")\n";
  40
  41 static int mlx5e_get_wqe_sz(struct mlx5e_priv *priv, u32 *wqe_sz, u32 *nsegs);
  42
  43 struct mlx5e_channel_param {
  44         struct mlx5e_rq_param rq;
  45         struct mlx5e_sq_param sq;
  46         struct mlx5e_cq_param rx_cq;
  47         struct mlx5e_cq_param tx_cq;
  48 };
  49
  50 struct media {
  51         u32     subtype;
  52         u64     baudrate;
  53 };
  54
  55 static const struct media mlx5e_mode_table[MLX5E_LINK_SPEEDS_NUMBER][MLX5E_LINK_MODES_NUMBER] = {
  56
  57         [MLX5E_1000BASE_CX_SGMII][MLX5E_SGMII] = {
  58                 .subtype = IFM_1000_CX_SGMII,
  59                 .baudrate = IF_Mbps(1000ULL),
  60         },
  61         [MLX5E_1000BASE_KX][MLX5E_KX] = {
  62                 .subtype = IFM_1000_KX,
  63                 .baudrate = IF_Mbps(1000ULL),
  64         },
  65         [MLX5E_10GBASE_CX4][MLX5E_CX4] = {
  66                 .subtype = IFM_10G_CX4,
  67                 .baudrate = IF_Gbps(10ULL),
  68         },
  69         [MLX5E_10GBASE_KX4][MLX5E_KX4] = {
  70                 .subtype = IFM_10G_KX4,
  71                 .baudrate = IF_Gbps(10ULL),
  72         },
  73         [MLX5E_10GBASE_KR][MLX5E_KR] = {
  74                 .subtype = IFM_10G_KR,
  75                 .baudrate = IF_Gbps(10ULL),
  76         },
  77         [MLX5E_20GBASE_KR2][MLX5E_KR2] = {
  78                 .subtype = IFM_20G_KR2,
  79                 .baudrate = IF_Gbps(20ULL),
  80         },
  81         [MLX5E_40GBASE_CR4][MLX5E_CR4] = {
  82                 .subtype = IFM_40G_CR4,
  83                 .baudrate = IF_Gbps(40ULL),
  84         },
  85         [MLX5E_40GBASE_KR4][MLX5E_KR4] = {
  86                 .subtype = IFM_40G_KR4,
  87                 .baudrate = IF_Gbps(40ULL),
  88         },
  89         [MLX5E_56GBASE_R4][MLX5E_R] = {
  90                 .subtype = IFM_56G_R4,
  91                 .baudrate = IF_Gbps(56ULL),
  92         },
  93         [MLX5E_10GBASE_CR][MLX5E_CR1] = {
  94                 .subtype = IFM_10G_CR1,
  95                 .baudrate = IF_Gbps(10ULL),
  96         },
  97         [MLX5E_10GBASE_SR][MLX5E_SR] = {
  98                 .subtype = IFM_10G_SR,
  99                 .baudrate = IF_Gbps(10ULL),
 100         },
 101         [MLX5E_10GBASE_ER_LR][MLX5E_ER] = {
 102                 .subtype = IFM_10G_ER,
 103                 .baudrate = IF_Gbps(10ULL),
 104         },
 105         [MLX5E_10GBASE_ER_LR][MLX5E_LR] = {
 106                 .subtype = IFM_10G_LR,
 107                 .baudrate = IF_Gbps(10ULL),
 108         },
 109         [MLX5E_40GBASE_SR4][MLX5E_SR4] = {
 110                 .subtype = IFM_40G_SR4,
 111                 .baudrate = IF_Gbps(40ULL),
 112         },
 113         [MLX5E_40GBASE_LR4_ER4][MLX5E_LR4] = {
 114                 .subtype = IFM_40G_LR4,
 115                 .baudrate = IF_Gbps(40ULL),
 116         },
 117         [MLX5E_40GBASE_LR4_ER4][MLX5E_ER4] = {
 118                 .subtype = IFM_40G_ER4,
 119                 .baudrate = IF_Gbps(40ULL),
 120         },
 121         [MLX5E_100GBASE_CR4][MLX5E_CR4] = {
 122                 .subtype = IFM_100G_CR4,
 123                 .baudrate = IF_Gbps(100ULL),
 124         },
 125         [MLX5E_100GBASE_SR4][MLX5E_SR4] = {
 126                 .subtype = IFM_100G_SR4,
 127                 .baudrate = IF_Gbps(100ULL),
 128         },
 129         [MLX5E_100GBASE_KR4][MLX5E_KR4] = {
 130                 .subtype = IFM_100G_KR4,
 131                 .baudrate = IF_Gbps(100ULL),
 132         },
 133         [MLX5E_100GBASE_LR4][MLX5E_LR4] = {
 134                 .subtype = IFM_100G_LR4,
 135                 .baudrate = IF_Gbps(100ULL),
 136         },
 137         [MLX5E_100BASE_TX][MLX5E_TX] = {
 138                 .subtype = IFM_100_TX,
 139                 .baudrate = IF_Mbps(100ULL),
 140         },
 141         [MLX5E_1000BASE_T][MLX5E_T] = {
 142                 .subtype = IFM_1000_T,
 143                 .baudrate = IF_Mbps(1000ULL),
 144         },
 145         [MLX5E_10GBASE_T][MLX5E_T] = {
 146                 .subtype = IFM_10G_T,
 147                 .baudrate = IF_Gbps(10ULL),
 148         },
 149         [MLX5E_25GBASE_CR][MLX5E_CR] = {
 150                 .subtype = IFM_25G_CR,
 151                 .baudrate = IF_Gbps(25ULL),
 152         },
 153         [MLX5E_25GBASE_KR][MLX5E_KR] = {
 154                 .subtype = IFM_25G_KR,
 155                 .baudrate = IF_Gbps(25ULL),
 156         },
 157         [MLX5E_25GBASE_SR][MLX5E_SR] = {
 158                 .subtype = IFM_25G_SR,
 159                 .baudrate = IF_Gbps(25ULL),
 160         },
 161         [MLX5E_50GBASE_CR2][MLX5E_CR2] = {
 162                 .subtype = IFM_50G_CR2,
 163                 .baudrate = IF_Gbps(50ULL),
 164         },
 165         [MLX5E_50GBASE_KR2][MLX5E_KR2] = {
 166                 .subtype = IFM_50G_KR2,
 167                 .baudrate = IF_Gbps(50ULL),
 168         },
 169 };
 170
 171 static const struct media mlx5e_ext_mode_table[MLX5E_EXT_LINK_SPEEDS_NUMBER][MLX5E_LINK_MODES_NUMBER] = {
 172         [MLX5E_SGMII_100M][MLX5E_SGMII] = {
 173                 .subtype = IFM_100_SGMII,
 174                 .baudrate = IF_Mbps(100),
 175         },
 176         [MLX5E_1000BASE_X_SGMII][MLX5E_KX] = {
 177                 .subtype = IFM_1000_KX,
 178                 .baudrate = IF_Mbps(1000),
 179         },
 180         [MLX5E_1000BASE_X_SGMII][MLX5E_CX_SGMII] = {
 181                 .subtype = IFM_1000_CX_SGMII,
 182                 .baudrate = IF_Mbps(1000),
 183         },
 184         [MLX5E_1000BASE_X_SGMII][MLX5E_CX] = {
 185                 .subtype = IFM_1000_CX,
 186                 .baudrate = IF_Mbps(1000),
 187         },
 188         [MLX5E_1000BASE_X_SGMII][MLX5E_LX] = {
 189                 .subtype = IFM_1000_LX,
 190                 .baudrate = IF_Mbps(1000),
 191         },
 192         [MLX5E_1000BASE_X_SGMII][MLX5E_SX] = {
 193                 .subtype = IFM_1000_SX,
 194                 .baudrate = IF_Mbps(1000),
 195         },
 196         [MLX5E_1000BASE_X_SGMII][MLX5E_T] = {
 197                 .subtype = IFM_1000_T,
 198                 .baudrate = IF_Mbps(1000),
 199         },
 200         [MLX5E_5GBASE_R][MLX5E_T] = {
 201                 .subtype = IFM_5000_T,
 202                 .baudrate = IF_Mbps(5000),
 203         },
 204         [MLX5E_5GBASE_R][MLX5E_KR] = {
 205                 .subtype = IFM_5000_KR,
 206                 .baudrate = IF_Mbps(5000),
 207         },
 208         [MLX5E_5GBASE_R][MLX5E_KR1] = {
 209                 .subtype = IFM_5000_KR1,
 210                 .baudrate = IF_Mbps(5000),
 211         },
 212         [MLX5E_5GBASE_R][MLX5E_KR_S] = {
 213                 .subtype = IFM_5000_KR_S,
 214                 .baudrate = IF_Mbps(5000),
 215         },
 216         [MLX5E_10GBASE_XFI_XAUI_1][MLX5E_ER] = {
 217                 .subtype = IFM_10G_ER,
 218                 .baudrate = IF_Gbps(10ULL),
 219         },
 220         [MLX5E_10GBASE_XFI_XAUI_1][MLX5E_KR] = {
 221                 .subtype = IFM_10G_KR,
 222                 .baudrate = IF_Gbps(10ULL),
 223         },
 224         [MLX5E_10GBASE_XFI_XAUI_1][MLX5E_LR] = {
 225                 .subtype = IFM_10G_LR,
 226                 .baudrate = IF_Gbps(10ULL),
 227         },
 228         [MLX5E_10GBASE_XFI_XAUI_1][MLX5E_SR] = {
 229                 .subtype = IFM_10G_SR,
 230                 .baudrate = IF_Gbps(10ULL),
 231         },
 232         [MLX5E_10GBASE_XFI_XAUI_1][MLX5E_T] = {
 233                 .subtype = IFM_10G_T,
 234                 .baudrate = IF_Gbps(10ULL),
 235         },
 236         [MLX5E_10GBASE_XFI_XAUI_1][MLX5E_AOC] = {
 237                 .subtype = IFM_10G_AOC,
 238                 .baudrate = IF_Gbps(10ULL),
 239         },
 240         [MLX5E_10GBASE_XFI_XAUI_1][MLX5E_CR1] = {
 241                 .subtype = IFM_10G_CR1,
 242                 .baudrate = IF_Gbps(10ULL),
 243         },
 244         [MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_CR4] = {
 245                 .subtype = IFM_40G_CR4,
 246                 .baudrate = IF_Gbps(40ULL),
 247         },
 248         [MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_KR4] = {
 249                 .subtype = IFM_40G_KR4,
 250                 .baudrate = IF_Gbps(40ULL),
 251         },
 252         [MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_LR4] = {
 253                 .subtype = IFM_40G_LR4,
 254                 .baudrate = IF_Gbps(40ULL),
 255         },
 256         [MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_SR4] = {
 257                 .subtype = IFM_40G_SR4,
 258                 .baudrate = IF_Gbps(40ULL),
 259         },
 260         [MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_ER4] = {
 261                 .subtype = IFM_40G_ER4,
 262                 .baudrate = IF_Gbps(40ULL),
 263         },
 264
 265         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CR] = {
 266                 .subtype = IFM_25G_CR,
 267                 .baudrate = IF_Gbps(25ULL),
 268         },
 269         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_KR] = {
 270                 .subtype = IFM_25G_KR,
 271                 .baudrate = IF_Gbps(25ULL),
 272         },
 273         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_SR] = {
 274                 .subtype = IFM_25G_SR,
 275                 .baudrate = IF_Gbps(25ULL),
 276         },
 277         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_ACC] = {
 278                 .subtype = IFM_25G_ACC,
 279                 .baudrate = IF_Gbps(25ULL),
 280         },
 281         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_AOC] = {
 282                 .subtype = IFM_25G_AOC,
 283                 .baudrate = IF_Gbps(25ULL),
 284         },
 285         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CR1] = {
 286                 .subtype = IFM_25G_CR1,
 287                 .baudrate = IF_Gbps(25ULL),
 288         },
 289         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CR_S] = {
 290                 .subtype = IFM_25G_CR_S,
 291                 .baudrate = IF_Gbps(25ULL),
 292         },
 293         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_KR1] = {
 294                 .subtype = IFM_5000_KR1,
 295                 .baudrate = IF_Gbps(25ULL),
 296         },
 297         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_KR_S] = {
 298                 .subtype = IFM_25G_KR_S,
 299                 .baudrate = IF_Gbps(25ULL),
 300         },
 301         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_LR] = {
 302                 .subtype = IFM_25G_LR,
 303                 .baudrate = IF_Gbps(25ULL),
 304         },
 305         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_T] = {
 306                 .subtype = IFM_25G_T,
 307                 .baudrate = IF_Gbps(25ULL),
 308         },
 309         [MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_CR2] = {
 310                 .subtype = IFM_50G_CR2,
 311                 .baudrate = IF_Gbps(50ULL),
 312         },
 313         [MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_KR2] = {
 314                 .subtype = IFM_50G_KR2,
 315                 .baudrate = IF_Gbps(50ULL),
 316         },
 317         [MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_SR2] = {
 318                 .subtype = IFM_50G_SR2,
 319                 .baudrate = IF_Gbps(50ULL),
 320         },
 321         [MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_LR2] = {
 322                 .subtype = IFM_50G_LR2,
 323                 .baudrate = IF_Gbps(50ULL),
 324         },
 325         [MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_LR] = {
 326                 .subtype = IFM_50G_LR,
 327                 .baudrate = IF_Gbps(50ULL),
 328         },
 329         [MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_SR] = {
 330                 .subtype = IFM_50G_SR,
 331                 .baudrate = IF_Gbps(50ULL),
 332         },
 333         [MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_CP] = {
 334                 .subtype = IFM_50G_CP,
 335                 .baudrate = IF_Gbps(50ULL),
 336         },
 337         [MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_FR] = {
 338                 .subtype = IFM_50G_FR,
 339                 .baudrate = IF_Gbps(50ULL),
 340         },
 341         [MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_KR_PAM4] = {
 342                 .subtype = IFM_50G_KR_PAM4,
 343                 .baudrate = IF_Gbps(50ULL),
 344         },
 345         [MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_CR4] = {
 346                 .subtype = IFM_100G_CR4,
 347                 .baudrate = IF_Gbps(100ULL),
 348         },
 349         [MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_KR4] = {
 350                 .subtype = IFM_100G_KR4,
 351                 .baudrate = IF_Gbps(100ULL),
 352         },
 353         [MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_LR4] = {
 354                 .subtype = IFM_100G_LR4,
 355                 .baudrate = IF_Gbps(100ULL),
 356         },
 357         [MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_SR4] = {
 358                 .subtype = IFM_100G_SR4,
 359                 .baudrate = IF_Gbps(100ULL),
 360         },
 361         [MLX5E_100GAUI_2_100GBASE_CR2_KR2][MLX5E_SR2] = {
 362                 .subtype = IFM_100G_SR2,
 363                 .baudrate = IF_Gbps(100ULL),
 364         },
 365         [MLX5E_100GAUI_2_100GBASE_CR2_KR2][MLX5E_CP2] = {
 366                 .subtype = IFM_100G_CP2,
 367                 .baudrate = IF_Gbps(100ULL),
 368         },
 369         [MLX5E_100GAUI_2_100GBASE_CR2_KR2][MLX5E_KR2_PAM4] = {
 370                 .subtype = IFM_100G_KR2_PAM4,
 371                 .baudrate = IF_Gbps(100ULL),
 372         },
 373         [MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_DR4] = {
 374                 .subtype = IFM_200G_DR4,
 375                 .baudrate = IF_Gbps(200ULL),
 376         },
 377         [MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_LR4] = {
 378                 .subtype = IFM_200G_LR4,
 379                 .baudrate = IF_Gbps(200ULL),
 380         },
 381         [MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_SR4] = {
 382                 .subtype = IFM_200G_SR4,
 383                 .baudrate = IF_Gbps(200ULL),
 384         },
 385         [MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_FR4] = {
 386                 .subtype = IFM_200G_FR4,
 387                 .baudrate = IF_Gbps(200ULL),
 388         },
 389         [MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_CR4_PAM4] = {
 390                 .subtype = IFM_200G_CR4_PAM4,
 391                 .baudrate = IF_Gbps(200ULL),
 392         },
 393         [MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_KR4_PAM4] = {
 394                 .subtype = IFM_200G_KR4_PAM4,
 395                 .baudrate = IF_Gbps(200ULL),
 396         },
 397 };
 398
 399 MALLOC_DEFINE(M_MLX5EN, "MLX5EN", "MLX5 Ethernet");
 400
 401 static void
 402 mlx5e_update_carrier(struct mlx5e_priv *priv)
 403 {
 404         struct mlx5_core_dev *mdev = priv->mdev;
 405         u32 out[MLX5_ST_SZ_DW(ptys_reg)];
 406         u32 eth_proto_oper;
 407         int error;
 408         u8 port_state;
 409         u8 is_er_type;
 410         u8 i, j;
 411         bool ext;
 412         struct media media_entry = {};
 413
 414         port_state = mlx5_query_vport_state(mdev,
 415             MLX5_QUERY_VPORT_STATE_IN_OP_MOD_VNIC_VPORT, 0);
 416
 417         if (port_state == VPORT_STATE_UP) {
 418                 priv->media_status_last |= IFM_ACTIVE;
 419         } else {
 420                 priv->media_status_last &= ~IFM_ACTIVE;
 421                 priv->media_active_last = IFM_ETHER;
 422                 if_link_state_change(priv->ifp, LINK_STATE_DOWN);
 423                 return;
 424         }
 425
 426         error = mlx5_query_port_ptys(mdev, out, sizeof(out),
 427             MLX5_PTYS_EN, 1);
 428         if (error) {
 429                 priv->media_active_last = IFM_ETHER;
 430                 priv->ifp->if_baudrate = 1;
 431                 mlx5_en_err(priv->ifp, "query port ptys failed: 0x%x\n",
 432                     error);
 433                 return;
 434         }
 435
 436         ext = MLX5_CAP_PCAM_FEATURE(mdev, ptys_extended_ethernet);
 437         eth_proto_oper = MLX5_GET_ETH_PROTO(ptys_reg, out, ext,
 438             eth_proto_oper);
 439
 440         i = ilog2(eth_proto_oper);
 441
 442         for (j = 0; j != MLX5E_LINK_MODES_NUMBER; j++) {
 443                 media_entry = ext ? mlx5e_ext_mode_table[i][j] :
 444                     mlx5e_mode_table[i][j];
 445                 if (media_entry.baudrate != 0)
 446                         break;
 447         }
 448
 449         if (media_entry.subtype == 0) {
 450                 mlx5_en_err(priv->ifp,
 451                     "Could not find operational media subtype\n");
 452                 return;
 453         }
 454
 455         switch (media_entry.subtype) {
 456         case IFM_10G_ER:
 457                 error = mlx5_query_pddr_range_info(mdev, 1, &is_er_type);
 458                 if (error != 0) {
 459                         mlx5_en_err(priv->ifp,
 460                             "query port pddr failed: %d\n", error);
 461                 }
 462                 if (error != 0 || is_er_type == 0)
 463                         media_entry.subtype = IFM_10G_LR;
 464                 break;
 465         case IFM_40G_LR4:
 466                 error = mlx5_query_pddr_range_info(mdev, 1, &is_er_type);
 467                 if (error != 0) {
 468                         mlx5_en_err(priv->ifp,
 469                             "query port pddr failed: %d\n", error);
 470                 }
 471                 if (error == 0 && is_er_type != 0)
 472                         media_entry.subtype = IFM_40G_ER4;
 473                 break;
 474         }
 475         priv->media_active_last = media_entry.subtype | IFM_ETHER | IFM_FDX;
 476         priv->ifp->if_baudrate = media_entry.baudrate;
 477
 478         if_link_state_change(priv->ifp, LINK_STATE_UP);
 479 }
 480
 481 static void
 482 mlx5e_media_status(struct ifnet *dev, struct ifmediareq *ifmr)
 483 {
 484         struct mlx5e_priv *priv = dev->if_softc;
 485
 486         ifmr->ifm_status = priv->media_status_last;
 487         ifmr->ifm_active = priv->media_active_last |
 488             (priv->params.rx_pauseframe_control ? IFM_ETH_RXPAUSE : 0) |
 489             (priv->params.tx_pauseframe_control ? IFM_ETH_TXPAUSE : 0);
 490
 491 }
 492
 493 static u32
 494 mlx5e_find_link_mode(u32 subtype, bool ext)
 495 {
 496         u32 i;
 497         u32 j;
 498         u32 link_mode = 0;
 499         u32 speeds_num = 0;
 500         struct media media_entry = {};
 501
 502         switch (subtype) {
 503         case IFM_10G_LR:
 504                 subtype = IFM_10G_ER;
 505                 break;
 506         case IFM_40G_ER4:
 507                 subtype = IFM_40G_LR4;
 508                 break;
 509         }
 510
 511         speeds_num = ext ? MLX5E_EXT_LINK_SPEEDS_NUMBER :
 512             MLX5E_LINK_SPEEDS_NUMBER;
 513
 514         for (i = 0; i != speeds_num; i++) {
 515                 for (j = 0; j < MLX5E_LINK_MODES_NUMBER ; ++j) {
 516                         media_entry = ext ? mlx5e_ext_mode_table[i][j] :
 517                             mlx5e_mode_table[i][j];
 518                         if (media_entry.baudrate == 0)
 519                                 continue;
 520                         if (media_entry.subtype == subtype) {
 521                                 link_mode |= MLX5E_PROT_MASK(i);
 522                         }
 523                 }
 524         }
 525
 526         return (link_mode);
 527 }
 528
 529 static int
 530 mlx5e_set_port_pause_and_pfc(struct mlx5e_priv *priv)
 531 {
 532         return (mlx5_set_port_pause_and_pfc(priv->mdev, 1,
 533             priv->params.rx_pauseframe_control,
 534             priv->params.tx_pauseframe_control,
 535             priv->params.rx_priority_flow_control,
 536             priv->params.tx_priority_flow_control));
 537 }
 538
 539 static int
 540 mlx5e_set_port_pfc(struct mlx5e_priv *priv)
 541 {
 542         int error;
 543
 544         if (priv->gone != 0) {
 545                 error = -ENXIO;
 546         } else if (priv->params.rx_pauseframe_control ||
 547             priv->params.tx_pauseframe_control) {
 548                 mlx5_en_err(priv->ifp,
 549                     "Global pauseframes must be disabled before enabling PFC.\n");
 550                 error = -EINVAL;
 551         } else {
 552                 error = mlx5e_set_port_pause_and_pfc(priv);
 553         }
 554         return (error);
 555 }
 556
 557 static int
 558 mlx5e_media_change(struct ifnet *dev)
 559 {
 560         struct mlx5e_priv *priv = dev->if_softc;
 561         struct mlx5_core_dev *mdev = priv->mdev;
 562         u32 eth_proto_cap;
 563         u32 link_mode;
 564         u32 out[MLX5_ST_SZ_DW(ptys_reg)];
 565         int was_opened;
 566         int locked;
 567         int error;
 568         bool ext;
 569
 570         locked = PRIV_LOCKED(priv);
 571         if (!locked)
 572                 PRIV_LOCK(priv);
 573
 574         if (IFM_TYPE(priv->media.ifm_media) != IFM_ETHER) {
 575                 error = EINVAL;
 576                 goto done;
 577         }
 578
 579         error = mlx5_query_port_ptys(mdev, out, sizeof(out),
 580             MLX5_PTYS_EN, 1);
 581         if (error != 0) {
 582                 mlx5_en_err(dev, "Query port media capability failed\n");
 583                 goto done;
 584         }
 585
 586         ext = MLX5_CAP_PCAM_FEATURE(mdev, ptys_extended_ethernet);
 587         link_mode = mlx5e_find_link_mode(IFM_SUBTYPE(priv->media.ifm_media), ext);
 588
 589         /* query supported capabilities */
 590         eth_proto_cap = MLX5_GET_ETH_PROTO(ptys_reg, out, ext,
 591             eth_proto_capability);
 592
 593         /* check for autoselect */
 594         if (IFM_SUBTYPE(priv->media.ifm_media) == IFM_AUTO) {
 595                 link_mode = eth_proto_cap;
 596                 if (link_mode == 0) {
 597                         mlx5_en_err(dev, "Port media capability is zero\n");
 598                         error = EINVAL;
 599                         goto done;
 600                 }
 601         } else {
 602                 link_mode = link_mode & eth_proto_cap;
 603                 if (link_mode == 0) {
 604                         mlx5_en_err(dev, "Not supported link mode requested\n");
 605                         error = EINVAL;
 606                         goto done;
 607                 }
 608         }
 609         if (priv->media.ifm_media & (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE)) {
 610                 /* check if PFC is enabled */
 611                 if (priv->params.rx_priority_flow_control ||
 612                     priv->params.tx_priority_flow_control) {
 613                         mlx5_en_err(dev, "PFC must be disabled before enabling global pauseframes.\n");
 614                         error = EINVAL;
 615                         goto done;
 616                 }
 617         }
 618         /* update pauseframe control bits */
 619         priv->params.rx_pauseframe_control =
 620             (priv->media.ifm_media & IFM_ETH_RXPAUSE) ? 1 : 0;
 621         priv->params.tx_pauseframe_control =
 622             (priv->media.ifm_media & IFM_ETH_TXPAUSE) ? 1 : 0;
 623
 624         /* check if device is opened */
 625         was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
 626
 627         /* reconfigure the hardware */
 628         mlx5_set_port_status(mdev, MLX5_PORT_DOWN);
 629         mlx5_set_port_proto(mdev, link_mode, MLX5_PTYS_EN, ext);
 630         error = -mlx5e_set_port_pause_and_pfc(priv);
 631         if (was_opened)
 632                 mlx5_set_port_status(mdev, MLX5_PORT_UP);
 633
 634 done:
 635         if (!locked)
 636                 PRIV_UNLOCK(priv);
 637         return (error);
 638 }
 639
 640 static void
 641 mlx5e_update_carrier_work(struct work_struct *work)
 642 {
 643         struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
 644             update_carrier_work);
 645
 646         PRIV_LOCK(priv);
 647         if (test_bit(MLX5E_STATE_OPENED, &priv->state))
 648                 mlx5e_update_carrier(priv);
 649         PRIV_UNLOCK(priv);
 650 }
 651
 652 #define MLX5E_PCIE_PERF_GET_64(a,b,c,d,e,f)    \
 653         s_debug->c = MLX5_GET64(mpcnt_reg, out, counter_set.f.c);
 654
 655 #define MLX5E_PCIE_PERF_GET_32(a,b,c,d,e,f)    \
 656         s_debug->c = MLX5_GET(mpcnt_reg, out, counter_set.f.c);
 657
 658 static void
 659 mlx5e_update_pcie_counters(struct mlx5e_priv *priv)
 660 {
 661         struct mlx5_core_dev *mdev = priv->mdev;
 662         struct mlx5e_port_stats_debug *s_debug = &priv->stats.port_stats_debug;
 663         const unsigned sz = MLX5_ST_SZ_BYTES(mpcnt_reg);
 664         void *out;
 665         void *in;
 666         int err;
 667
 668         /* allocate firmware request structures */
 669         in = mlx5_vzalloc(sz);
 670         out = mlx5_vzalloc(sz);
 671         if (in == NULL || out == NULL)
 672                 goto free_out;
 673
 674         MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_PERFORMANCE_COUNTERS_GROUP);
 675         err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
 676         if (err != 0)
 677                 goto free_out;
 678
 679         MLX5E_PCIE_PERFORMANCE_COUNTERS_64(MLX5E_PCIE_PERF_GET_64)
 680         MLX5E_PCIE_PERFORMANCE_COUNTERS_32(MLX5E_PCIE_PERF_GET_32)
 681
 682         MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_TIMERS_AND_STATES_COUNTERS_GROUP);
 683         err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
 684         if (err != 0)
 685                 goto free_out;
 686
 687         MLX5E_PCIE_TIMERS_AND_STATES_COUNTERS_32(MLX5E_PCIE_PERF_GET_32)
 688
 689         MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_LANE_COUNTERS_GROUP);
 690         err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
 691         if (err != 0)
 692                 goto free_out;
 693
 694         MLX5E_PCIE_LANE_COUNTERS_32(MLX5E_PCIE_PERF_GET_32)
 695
 696 free_out:
 697         /* free firmware request structures */
 698         kvfree(in);
 699         kvfree(out);
 700 }
 701
 702 /*
 703  * This function reads the physical port counters from the firmware
 704  * using a pre-defined layout defined by various MLX5E_PPORT_XXX()
 705  * macros. The output is converted from big-endian 64-bit values into
 706  * host endian ones and stored in the "priv->stats.pport" structure.
 707  */
 708 static void
 709 mlx5e_update_pport_counters(struct mlx5e_priv *priv)
 710 {
 711         struct mlx5_core_dev *mdev = priv->mdev;
 712         struct mlx5e_pport_stats *s = &priv->stats.pport;
 713         struct mlx5e_port_stats_debug *s_debug = &priv->stats.port_stats_debug;
 714         u32 *in;
 715         u32 *out;
 716         const u64 *ptr;
 717         unsigned sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
 718         unsigned x;
 719         unsigned y;
 720         unsigned z;
 721
 722         /* allocate firmware request structures */
 723         in = mlx5_vzalloc(sz);
 724         out = mlx5_vzalloc(sz);
 725         if (in == NULL || out == NULL)
 726                 goto free_out;
 727
 728         /*
 729          * Get pointer to the 64-bit counter set which is located at a
 730          * fixed offset in the output firmware request structure:
 731          */
 732         ptr = (const uint64_t *)MLX5_ADDR_OF(ppcnt_reg, out, counter_set);
 733
 734         MLX5_SET(ppcnt_reg, in, local_port, 1);
 735
 736         /* read IEEE802_3 counter group using predefined counter layout */
 737         MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
 738         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 739         for (x = 0, y = MLX5E_PPORT_PER_PRIO_STATS_NUM;
 740              x != MLX5E_PPORT_IEEE802_3_STATS_NUM; x++, y++)
 741                 s->arg[y] = be64toh(ptr[x]);
 742
 743         /* read RFC2819 counter group using predefined counter layout */
 744         MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2819_COUNTERS_GROUP);
 745         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 746         for (x = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM; x++, y++)
 747                 s->arg[y] = be64toh(ptr[x]);
 748
 749         for (y = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM +
 750             MLX5E_PPORT_RFC2819_STATS_DEBUG_NUM; x++, y++)
 751                 s_debug->arg[y] = be64toh(ptr[x]);
 752
 753         /* read RFC2863 counter group using predefined counter layout */
 754         MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2863_COUNTERS_GROUP);
 755         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 756         for (x = 0; x != MLX5E_PPORT_RFC2863_STATS_DEBUG_NUM; x++, y++)
 757                 s_debug->arg[y] = be64toh(ptr[x]);
 758
 759         /* read physical layer stats counter group using predefined counter layout */
 760         MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP);
 761         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 762         for (x = 0; x != MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG_NUM; x++, y++)
 763                 s_debug->arg[y] = be64toh(ptr[x]);
 764
 765         /* read Extended Ethernet counter group using predefined counter layout */
 766         MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);
 767         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 768         for (x = 0; x != MLX5E_PPORT_ETHERNET_EXTENDED_STATS_DEBUG_NUM; x++, y++)
 769                 s_debug->arg[y] = be64toh(ptr[x]);
 770
 771         /* read Extended Statistical Group */
 772         if (MLX5_CAP_GEN(mdev, pcam_reg) &&
 773             MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group) &&
 774             MLX5_CAP_PCAM_FEATURE(mdev, per_lane_error_counters)) {
 775                 /* read Extended Statistical counter group using predefined counter layout */
 776                 MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_STATISTICAL_GROUP);
 777                 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 778
 779                 for (x = 0; x != MLX5E_PPORT_STATISTICAL_DEBUG_NUM; x++, y++)
 780                         s_debug->arg[y] = be64toh(ptr[x]);
 781         }
 782
 783         /* read PCIE counters */
 784         mlx5e_update_pcie_counters(priv);
 785
 786         /* read per-priority counters */
 787         MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_PRIORITY_COUNTERS_GROUP);
 788
 789         /* iterate all the priorities */
 790         for (y = z = 0; z != MLX5E_PPORT_PER_PRIO_STATS_NUM_PRIO; z++) {
 791                 MLX5_SET(ppcnt_reg, in, prio_tc, z);
 792                 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 793
 794                 /* read per priority stats counter group using predefined counter layout */
 795                 for (x = 0; x != (MLX5E_PPORT_PER_PRIO_STATS_NUM /
 796                     MLX5E_PPORT_PER_PRIO_STATS_NUM_PRIO); x++, y++)
 797                         s->arg[y] = be64toh(ptr[x]);
 798         }
 799
 800 free_out:
 801         /* free firmware request structures */
 802         kvfree(in);
 803         kvfree(out);
 804 }
 805
 806 static void
 807 mlx5e_grp_vnic_env_update_stats(struct mlx5e_priv *priv)
 808 {
 809         u32 out[MLX5_ST_SZ_DW(query_vnic_env_out)] = {};
 810         u32 in[MLX5_ST_SZ_DW(query_vnic_env_in)] = {};
 811
 812         if (!MLX5_CAP_GEN(priv->mdev, nic_receive_steering_discard))
 813                 return;
 814
 815         MLX5_SET(query_vnic_env_in, in, opcode,
 816             MLX5_CMD_OP_QUERY_VNIC_ENV);
 817         MLX5_SET(query_vnic_env_in, in, op_mod, 0);
 818         MLX5_SET(query_vnic_env_in, in, other_vport, 0);
 819
 820         if (mlx5_cmd_exec(priv->mdev, in, sizeof(in), out, sizeof(out)) != 0)
 821                 return;
 822
 823         priv->stats.vport.rx_steer_missed_packets =
 824             MLX5_GET64(query_vnic_env_out, out,
 825             vport_env.nic_receive_steering_discard);
 826 }
 827
 828 /*
 829  * This function is called regularly to collect all statistics
 830  * counters from the firmware. The values can be viewed through the
 831  * sysctl interface. Execution is serialized using the priv's global
 832  * configuration lock.
 833  */
 834 static void
 835 mlx5e_update_stats_locked(struct mlx5e_priv *priv)
 836 {
 837         struct mlx5_core_dev *mdev = priv->mdev;
 838         struct mlx5e_vport_stats *s = &priv->stats.vport;
 839         struct mlx5e_sq_stats *sq_stats;
 840         struct buf_ring *sq_br;
 841 #if (__FreeBSD_version < 1100000)
 842         struct ifnet *ifp = priv->ifp;
 843 #endif
 844
 845         u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)];
 846         u32 *out;
 847         int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
 848         u64 tso_packets = 0;
 849         u64 tso_bytes = 0;
 850         u64 tx_queue_dropped = 0;
 851         u64 tx_defragged = 0;
 852         u64 tx_offload_none = 0;
 853         u64 lro_packets = 0;
 854         u64 lro_bytes = 0;
 855         u64 sw_lro_queued = 0;
 856         u64 sw_lro_flushed = 0;
 857         u64 rx_csum_none = 0;
 858         u64 rx_wqe_err = 0;
 859         u64 rx_packets = 0;
 860         u64 rx_bytes = 0;
 861         u32 rx_out_of_buffer = 0;
 862         int error;
 863         int i;
 864         int j;
 865
 866         out = mlx5_vzalloc(outlen);
 867         if (out == NULL)
 868                 goto free_out;
 869
 870         /* Collect firts the SW counters and then HW for consistency */
 871         for (i = 0; i < priv->params.num_channels; i++) {
 872                 struct mlx5e_channel *pch = priv->channel + i;
 873                 struct mlx5e_rq *rq = &pch->rq;
 874                 struct mlx5e_rq_stats *rq_stats = &pch->rq.stats;
 875
 876                 /* collect stats from LRO */
 877                 rq_stats->sw_lro_queued = rq->lro.lro_queued;
 878                 rq_stats->sw_lro_flushed = rq->lro.lro_flushed;
 879                 sw_lro_queued += rq_stats->sw_lro_queued;
 880                 sw_lro_flushed += rq_stats->sw_lro_flushed;
 881                 lro_packets += rq_stats->lro_packets;
 882                 lro_bytes += rq_stats->lro_bytes;
 883                 rx_csum_none += rq_stats->csum_none;
 884                 rx_wqe_err += rq_stats->wqe_err;
 885                 rx_packets += rq_stats->packets;
 886                 rx_bytes += rq_stats->bytes;
 887
 888                 for (j = 0; j < priv->num_tc; j++) {
 889                         sq_stats = &pch->sq[j].stats;
 890                         sq_br = pch->sq[j].br;
 891
 892                         tso_packets += sq_stats->tso_packets;
 893                         tso_bytes += sq_stats->tso_bytes;
 894                         tx_queue_dropped += sq_stats->dropped;
 895                         if (sq_br != NULL)
 896                                 tx_queue_dropped += sq_br->br_drops;
 897                         tx_defragged += sq_stats->defragged;
 898                         tx_offload_none += sq_stats->csum_offload_none;
 899                 }
 900         }
 901
 902         /* update counters */
 903         s->tso_packets = tso_packets;
 904         s->tso_bytes = tso_bytes;
 905         s->tx_queue_dropped = tx_queue_dropped;
 906         s->tx_defragged = tx_defragged;
 907         s->lro_packets = lro_packets;
 908         s->lro_bytes = lro_bytes;
 909         s->sw_lro_queued = sw_lro_queued;
 910         s->sw_lro_flushed = sw_lro_flushed;
 911         s->rx_csum_none = rx_csum_none;
 912         s->rx_wqe_err = rx_wqe_err;
 913         s->rx_packets = rx_packets;
 914         s->rx_bytes = rx_bytes;
 915
 916         mlx5e_grp_vnic_env_update_stats(priv);
 917
 918         /* HW counters */
 919         memset(in, 0, sizeof(in));
 920
 921         MLX5_SET(query_vport_counter_in, in, opcode,
 922             MLX5_CMD_OP_QUERY_VPORT_COUNTER);
 923         MLX5_SET(query_vport_counter_in, in, op_mod, 0);
 924         MLX5_SET(query_vport_counter_in, in, other_vport, 0);
 925
 926         memset(out, 0, outlen);
 927
 928         /* get number of out-of-buffer drops first */
 929         if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0 &&
 930             mlx5_vport_query_out_of_rx_buffer(mdev, priv->counter_set_id,
 931             &rx_out_of_buffer) == 0) {
 932                 s->rx_out_of_buffer = rx_out_of_buffer;
 933         }
 934
 935         /* get port statistics */
 936         if (mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen) == 0) {
 937 #define MLX5_GET_CTR(out, x) \
 938         MLX5_GET64(query_vport_counter_out, out, x)
 939
 940                 s->rx_error_packets =
 941                     MLX5_GET_CTR(out, received_errors.packets);
 942                 s->rx_error_bytes =
 943                     MLX5_GET_CTR(out, received_errors.octets);
 944                 s->tx_error_packets =
 945                     MLX5_GET_CTR(out, transmit_errors.packets);
 946                 s->tx_error_bytes =
 947                     MLX5_GET_CTR(out, transmit_errors.octets);
 948
 949                 s->rx_unicast_packets =
 950                     MLX5_GET_CTR(out, received_eth_unicast.packets);
 951                 s->rx_unicast_bytes =
 952                     MLX5_GET_CTR(out, received_eth_unicast.octets);
 953                 s->tx_unicast_packets =
 954                     MLX5_GET_CTR(out, transmitted_eth_unicast.packets);
 955                 s->tx_unicast_bytes =
 956                     MLX5_GET_CTR(out, transmitted_eth_unicast.octets);
 957
 958                 s->rx_multicast_packets =
 959                     MLX5_GET_CTR(out, received_eth_multicast.packets);
 960                 s->rx_multicast_bytes =
 961                     MLX5_GET_CTR(out, received_eth_multicast.octets);
 962                 s->tx_multicast_packets =
 963                     MLX5_GET_CTR(out, transmitted_eth_multicast.packets);
 964                 s->tx_multicast_bytes =
 965                     MLX5_GET_CTR(out, transmitted_eth_multicast.octets);
 966
 967                 s->rx_broadcast_packets =
 968                     MLX5_GET_CTR(out, received_eth_broadcast.packets);
 969                 s->rx_broadcast_bytes =
 970                     MLX5_GET_CTR(out, received_eth_broadcast.octets);
 971                 s->tx_broadcast_packets =
 972                     MLX5_GET_CTR(out, transmitted_eth_broadcast.packets);
 973                 s->tx_broadcast_bytes =
 974                     MLX5_GET_CTR(out, transmitted_eth_broadcast.octets);
 975
 976                 s->tx_packets = s->tx_unicast_packets +
 977                     s->tx_multicast_packets + s->tx_broadcast_packets;
 978                 s->tx_bytes = s->tx_unicast_bytes + s->tx_multicast_bytes +
 979                     s->tx_broadcast_bytes;
 980
 981                 /* Update calculated offload counters */
 982                 s->tx_csum_offload = s->tx_packets - tx_offload_none;
 983                 s->rx_csum_good = s->rx_packets - s->rx_csum_none;
 984         }
 985
 986         /* Get physical port counters */
 987         mlx5e_update_pport_counters(priv);
 988
 989         s->tx_jumbo_packets =
 990             priv->stats.port_stats_debug.tx_stat_p1519to2047octets +
 991             priv->stats.port_stats_debug.tx_stat_p2048to4095octets +
 992             priv->stats.port_stats_debug.tx_stat_p4096to8191octets +
 993             priv->stats.port_stats_debug.tx_stat_p8192to10239octets;
 994
 995 #if (__FreeBSD_version < 1100000)
 996         /* no get_counters interface in fbsd 10 */
 997         ifp->if_ipackets = s->rx_packets;
 998         ifp->if_ierrors = priv->stats.pport.in_range_len_errors +
 999             priv->stats.pport.out_of_range_len +
1000             priv->stats.pport.too_long_errors +
1001             priv->stats.pport.check_seq_err +
1002             priv->stats.pport.alignment_err;
1003         ifp->if_iqdrops = s->rx_out_of_buffer;
1004         ifp->if_opackets = s->tx_packets;
1005         ifp->if_oerrors = priv->stats.port_stats_debug.out_discards;
1006         ifp->if_snd.ifq_drops = s->tx_queue_dropped;
1007         ifp->if_ibytes = s->rx_bytes;
1008         ifp->if_obytes = s->tx_bytes;
1009         ifp->if_collisions =
1010             priv->stats.pport.collisions;
1011 #endif
1012
1013 free_out:
1014         kvfree(out);
1015
1016         /* Update diagnostics, if any */
1017         if (priv->params_ethtool.diag_pci_enable ||
1018             priv->params_ethtool.diag_general_enable) {
1019                 error = mlx5_core_get_diagnostics_full(mdev,
1020                     priv->params_ethtool.diag_pci_enable ? &priv->params_pci : NULL,
1021                     priv->params_ethtool.diag_general_enable ? &priv->params_general : NULL);
1022                 if (error != 0)
1023                         mlx5_en_err(priv->ifp,
1024                             "Failed reading diagnostics: %d\n", error);
1025         }
1026
1027         /* Update FEC, if any */
1028         error = mlx5e_fec_update(priv);
1029         if (error != 0 && error != EOPNOTSUPP) {
1030                 mlx5_en_err(priv->ifp,
1031                     "Updating FEC failed: %d\n", error);
1032         }
1033 }
1034
1035 static void
1036 mlx5e_update_stats_work(struct work_struct *work)
1037 {
1038         struct mlx5e_priv *priv;
1039
1040         priv  = container_of(work, struct mlx5e_priv, update_stats_work);
1041         PRIV_LOCK(priv);
1042         if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
1043                 mlx5e_update_stats_locked(priv);
1044         PRIV_UNLOCK(priv);
1045 }
1046
1047 static void
1048 mlx5e_update_stats(void *arg)
1049 {
1050         struct mlx5e_priv *priv = arg;
1051
1052         queue_work(priv->wq, &priv->update_stats_work);
1053
1054         callout_reset(&priv->watchdog, hz, &mlx5e_update_stats, priv);
1055 }
1056
1057 static void
1058 mlx5e_async_event_sub(struct mlx5e_priv *priv,
1059     enum mlx5_dev_event event)
1060 {
1061         switch (event) {
1062         case MLX5_DEV_EVENT_PORT_UP:
1063         case MLX5_DEV_EVENT_PORT_DOWN:
1064                 queue_work(priv->wq, &priv->update_carrier_work);
1065                 break;
1066
1067         default:
1068                 break;
1069         }
1070 }
1071
1072 static void
1073 mlx5e_async_event(struct mlx5_core_dev *mdev, void *vpriv,
1074     enum mlx5_dev_event event, unsigned long param)
1075 {
1076         struct mlx5e_priv *priv = vpriv;
1077
1078         mtx_lock(&priv->async_events_mtx);
1079         if (test_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state))
1080                 mlx5e_async_event_sub(priv, event);
1081         mtx_unlock(&priv->async_events_mtx);
1082 }
1083
1084 static void
1085 mlx5e_enable_async_events(struct mlx5e_priv *priv)
1086 {
1087         set_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state);
1088 }
1089
1090 static void
1091 mlx5e_disable_async_events(struct mlx5e_priv *priv)
1092 {
1093         mtx_lock(&priv->async_events_mtx);
1094         clear_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state);
1095         mtx_unlock(&priv->async_events_mtx);
1096 }
1097
1098 static void mlx5e_calibration_callout(void *arg);
1099 static int mlx5e_calibration_duration = 20;
1100 static int mlx5e_fast_calibration = 1;
1101 static int mlx5e_normal_calibration = 30;
1102
1103 static SYSCTL_NODE(_hw_mlx5, OID_AUTO, calibr, CTLFLAG_RW, 0,
1104     "MLX5 timestamp calibration parameteres");
1105
1106 SYSCTL_INT(_hw_mlx5_calibr, OID_AUTO, duration, CTLFLAG_RWTUN,
1107     &mlx5e_calibration_duration, 0,
1108     "Duration of initial calibration");
1109 SYSCTL_INT(_hw_mlx5_calibr, OID_AUTO, fast, CTLFLAG_RWTUN,
1110     &mlx5e_fast_calibration, 0,
1111     "Recalibration interval during initial calibration");
1112 SYSCTL_INT(_hw_mlx5_calibr, OID_AUTO, normal, CTLFLAG_RWTUN,
1113     &mlx5e_normal_calibration, 0,
1114     "Recalibration interval during normal operations");
1115
1116 /*
1117  * Ignites the calibration process.
1118  */
1119 static void
1120 mlx5e_reset_calibration_callout(struct mlx5e_priv *priv)
1121 {
1122
1123         if (priv->clbr_done == 0)
1124                 mlx5e_calibration_callout(priv);
1125         else
1126                 callout_reset_curcpu(&priv->tstmp_clbr, (priv->clbr_done <
1127                     mlx5e_calibration_duration ? mlx5e_fast_calibration :
1128                     mlx5e_normal_calibration) * hz, mlx5e_calibration_callout,
1129                     priv);
1130 }
1131
1132 static uint64_t
1133 mlx5e_timespec2usec(const struct timespec *ts)
1134 {
1135
1136         return ((uint64_t)ts->tv_sec * 1000000000 + ts->tv_nsec);
1137 }
1138
1139 static uint64_t
1140 mlx5e_hw_clock(struct mlx5e_priv *priv)
1141 {
1142         struct mlx5_init_seg *iseg;
1143         uint32_t hw_h, hw_h1, hw_l;
1144
1145         iseg = priv->mdev->iseg;
1146         do {
1147                 hw_h = ioread32be(&iseg->internal_timer_h);
1148                 hw_l = ioread32be(&iseg->internal_timer_l);
1149                 hw_h1 = ioread32be(&iseg->internal_timer_h);
1150         } while (hw_h1 != hw_h);
1151         return (((uint64_t)hw_h << 32) | hw_l);
1152 }
1153
1154 /*
1155  * The calibration callout, it runs either in the context of the
1156  * thread which enables calibration, or in callout.  It takes the
1157  * snapshot of system and adapter clocks, then advances the pointers to
1158  * the calibration point to allow rx path to read the consistent data
1159  * lockless.
1160  */
1161 static void
1162 mlx5e_calibration_callout(void *arg)
1163 {
1164         struct mlx5e_priv *priv;
1165         struct mlx5e_clbr_point *next, *curr;
1166         struct timespec ts;
1167         int clbr_curr_next;
1168
1169         priv = arg;
1170         curr = &priv->clbr_points[priv->clbr_curr];
1171         clbr_curr_next = priv->clbr_curr + 1;
1172         if (clbr_curr_next >= nitems(priv->clbr_points))
1173                 clbr_curr_next = 0;
1174         next = &priv->clbr_points[clbr_curr_next];
1175
1176         next->base_prev = curr->base_curr;
1177         next->clbr_hw_prev = curr->clbr_hw_curr;
1178
1179         next->clbr_hw_curr = mlx5e_hw_clock(priv);
1180         if (((next->clbr_hw_curr - curr->clbr_hw_curr) >> MLX5E_TSTMP_PREC) ==
1181             0) {
1182                 if (priv->clbr_done != 0) {
1183                         mlx5_en_err(priv->ifp,
1184                             "HW failed tstmp frozen %#jx %#jx, disabling\n",
1185                              next->clbr_hw_curr, curr->clbr_hw_prev);
1186                         priv->clbr_done = 0;
1187                 }
1188                 atomic_store_rel_int(&curr->clbr_gen, 0);
1189                 return;
1190         }
1191
1192         nanouptime(&ts);
1193         next->base_curr = mlx5e_timespec2usec(&ts);
1194
1195         curr->clbr_gen = 0;
1196         atomic_thread_fence_rel();
1197         priv->clbr_curr = clbr_curr_next;
1198         atomic_store_rel_int(&next->clbr_gen, ++(priv->clbr_gen));
1199
1200         if (priv->clbr_done < mlx5e_calibration_duration)
1201                 priv->clbr_done++;
1202         mlx5e_reset_calibration_callout(priv);
1203 }
1204
1205 static const char *mlx5e_rq_stats_desc[] = {
1206         MLX5E_RQ_STATS(MLX5E_STATS_DESC)
1207 };
1208
1209 static int
1210 mlx5e_create_rq(struct mlx5e_channel *c,
1211     struct mlx5e_rq_param *param,
1212     struct mlx5e_rq *rq)
1213 {
1214         struct mlx5e_priv *priv = c->priv;
1215         struct mlx5_core_dev *mdev = priv->mdev;
1216         char buffer[16];
1217         void *rqc = param->rqc;
1218         void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
1219         int wq_sz;
1220         int err;
1221         int i;
1222         u32 nsegs, wqe_sz;
1223
1224         err = mlx5e_get_wqe_sz(priv, &wqe_sz, &nsegs);
1225         if (err != 0)
1226                 goto done;
1227
1228         /* Create DMA descriptor TAG */
1229         if ((err = -bus_dma_tag_create(
1230             bus_get_dma_tag(mdev->pdev->dev.bsddev),
1231             1,                          /* any alignment */
1232             0,                          /* no boundary */
1233             BUS_SPACE_MAXADDR,          /* lowaddr */
1234             BUS_SPACE_MAXADDR,          /* highaddr */
1235             NULL, NULL,                 /* filter, filterarg */
1236             nsegs * MLX5E_MAX_RX_BYTES, /* maxsize */
1237             nsegs,                      /* nsegments */
1238             nsegs * MLX5E_MAX_RX_BYTES, /* maxsegsize */
1239             0,                          /* flags */
1240             NULL, NULL,                 /* lockfunc, lockfuncarg */
1241             &rq->dma_tag)))
1242                 goto done;
1243
1244         err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
1245             &rq->wq_ctrl);
1246         if (err)
1247                 goto err_free_dma_tag;
1248
1249         rq->wq.db = &rq->wq.db[MLX5_RCV_DBR];
1250
1251         err = mlx5e_get_wqe_sz(priv, &rq->wqe_sz, &rq->nsegs);
1252         if (err != 0)
1253                 goto err_rq_wq_destroy;
1254
1255         wq_sz = mlx5_wq_ll_get_size(&rq->wq);
1256
1257         err = -tcp_lro_init_args(&rq->lro, c->tag.m_snd_tag.ifp, TCP_LRO_ENTRIES, wq_sz);
1258         if (err)
1259                 goto err_rq_wq_destroy;
1260
1261         rq->mbuf = malloc(wq_sz * sizeof(rq->mbuf[0]), M_MLX5EN, M_WAITOK | M_ZERO);
1262         for (i = 0; i != wq_sz; i++) {
1263                 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
1264                 int j;
1265
1266                 err = -bus_dmamap_create(rq->dma_tag, 0, &rq->mbuf[i].dma_map);
1267                 if (err != 0) {
1268                         while (i--)
1269                                 bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map);
1270                         goto err_rq_mbuf_free;
1271                 }
1272
1273                 /* set value for constant fields */
1274                 for (j = 0; j < rq->nsegs; j++)
1275                         wqe->data[j].lkey = cpu_to_be32(priv->mr.key);
1276         }
1277
1278         INIT_WORK(&rq->dim.work, mlx5e_dim_work);
1279         if (priv->params.rx_cq_moderation_mode < 2) {
1280                 rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
1281         } else {
1282                 void *cqc = container_of(param,
1283                     struct mlx5e_channel_param, rq)->rx_cq.cqc;
1284
1285                 switch (MLX5_GET(cqc, cqc, cq_period_mode)) {
1286                 case MLX5_CQ_PERIOD_MODE_START_FROM_EQE:
1287                         rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
1288                         break;
1289                 case MLX5_CQ_PERIOD_MODE_START_FROM_CQE:
1290                         rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE;
1291                         break;
1292                 default:
1293                         rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
1294                         break;
1295                 }
1296         }
1297
1298         rq->ifp = c->tag.m_snd_tag.ifp;
1299         rq->channel = c;
1300         rq->ix = c->ix;
1301
1302         snprintf(buffer, sizeof(buffer), "rxstat%d", c->ix);
1303         mlx5e_create_stats(&rq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
1304             buffer, mlx5e_rq_stats_desc, MLX5E_RQ_STATS_NUM,
1305             rq->stats.arg);
1306         return (0);
1307
1308 err_rq_mbuf_free:
1309         free(rq->mbuf, M_MLX5EN);
1310         tcp_lro_free(&rq->lro);
1311 err_rq_wq_destroy:
1312         mlx5_wq_destroy(&rq->wq_ctrl);
1313 err_free_dma_tag:
1314         bus_dma_tag_destroy(rq->dma_tag);
1315 done:
1316         return (err);
1317 }
1318
1319 static void
1320 mlx5e_destroy_rq(struct mlx5e_rq *rq)
1321 {
1322         int wq_sz;
1323         int i;
1324
1325         /* destroy all sysctl nodes */
1326         sysctl_ctx_free(&rq->stats.ctx);
1327
1328         /* free leftover LRO packets, if any */
1329         tcp_lro_free(&rq->lro);
1330
1331         wq_sz = mlx5_wq_ll_get_size(&rq->wq);
1332         for (i = 0; i != wq_sz; i++) {
1333                 if (rq->mbuf[i].mbuf != NULL) {
1334                         bus_dmamap_unload(rq->dma_tag, rq->mbuf[i].dma_map);
1335                         m_freem(rq->mbuf[i].mbuf);
1336                 }
1337                 bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map);
1338         }
1339         free(rq->mbuf, M_MLX5EN);
1340         mlx5_wq_destroy(&rq->wq_ctrl);
1341         bus_dma_tag_destroy(rq->dma_tag);
1342 }
1343
1344 static int
1345 mlx5e_enable_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
1346 {
1347         struct mlx5e_channel *c = rq->channel;
1348         struct mlx5e_priv *priv = c->priv;
1349         struct mlx5_core_dev *mdev = priv->mdev;
1350
1351         void *in;
1352         void *rqc;
1353         void *wq;
1354         int inlen;
1355         int err;
1356
1357         inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
1358             sizeof(u64) * rq->wq_ctrl.buf.npages;
1359         in = mlx5_vzalloc(inlen);
1360         if (in == NULL)
1361                 return (-ENOMEM);
1362
1363         rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
1364         wq = MLX5_ADDR_OF(rqc, rqc, wq);
1365
1366         memcpy(rqc, param->rqc, sizeof(param->rqc));
1367
1368         MLX5_SET(rqc, rqc, cqn, c->rq.cq.mcq.cqn);
1369         MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
1370         MLX5_SET(rqc, rqc, flush_in_error_en, 1);
1371         if (priv->counter_set_id >= 0)
1372                 MLX5_SET(rqc, rqc, counter_set_id, priv->counter_set_id);
1373         MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift -
1374             PAGE_SHIFT);
1375         MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma);
1376
1377         mlx5_fill_page_array(&rq->wq_ctrl.buf,
1378             (__be64 *) MLX5_ADDR_OF(wq, wq, pas));
1379
1380         err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
1381
1382         kvfree(in);
1383
1384         return (err);
1385 }
1386
1387 static int
1388 mlx5e_modify_rq(struct mlx5e_rq *rq, int curr_state, int next_state)
1389 {
1390         struct mlx5e_channel *c = rq->channel;
1391         struct mlx5e_priv *priv = c->priv;
1392         struct mlx5_core_dev *mdev = priv->mdev;
1393
1394         void *in;
1395         void *rqc;
1396         int inlen;
1397         int err;
1398
1399         inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1400         in = mlx5_vzalloc(inlen);
1401         if (in == NULL)
1402                 return (-ENOMEM);
1403
1404         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
1405
1406         MLX5_SET(modify_rq_in, in, rqn, rq->rqn);
1407         MLX5_SET(modify_rq_in, in, rq_state, curr_state);
1408         MLX5_SET(rqc, rqc, state, next_state);
1409
1410         err = mlx5_core_modify_rq(mdev, in, inlen);
1411
1412         kvfree(in);
1413
1414         return (err);
1415 }
1416
1417 static void
1418 mlx5e_disable_rq(struct mlx5e_rq *rq)
1419 {
1420         struct mlx5e_channel *c = rq->channel;
1421         struct mlx5e_priv *priv = c->priv;
1422         struct mlx5_core_dev *mdev = priv->mdev;
1423
1424         mlx5_core_destroy_rq(mdev, rq->rqn);
1425 }
1426
1427 static int
1428 mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq)
1429 {
1430         struct mlx5e_channel *c = rq->channel;
1431         struct mlx5e_priv *priv = c->priv;
1432         struct mlx5_wq_ll *wq = &rq->wq;
1433         int i;
1434
1435         for (i = 0; i < 1000; i++) {
1436                 if (wq->cur_sz >= priv->params.min_rx_wqes)
1437                         return (0);
1438
1439                 msleep(4);
1440         }
1441         return (-ETIMEDOUT);
1442 }
1443
1444 static int
1445 mlx5e_open_rq(struct mlx5e_channel *c,
1446     struct mlx5e_rq_param *param,
1447     struct mlx5e_rq *rq)
1448 {
1449         int err;
1450
1451         err = mlx5e_create_rq(c, param, rq);
1452         if (err)
1453                 return (err);
1454
1455         err = mlx5e_enable_rq(rq, param);
1456         if (err)
1457                 goto err_destroy_rq;
1458
1459         err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
1460         if (err)
1461                 goto err_disable_rq;
1462
1463         c->rq.enabled = 1;
1464
1465         return (0);
1466
1467 err_disable_rq:
1468         mlx5e_disable_rq(rq);
1469 err_destroy_rq:
1470         mlx5e_destroy_rq(rq);
1471
1472         return (err);
1473 }
1474
1475 static void
1476 mlx5e_close_rq(struct mlx5e_rq *rq)
1477 {
1478         mtx_lock(&rq->mtx);
1479         rq->enabled = 0;
1480         callout_stop(&rq->watchdog);
1481         mtx_unlock(&rq->mtx);
1482
1483         mlx5e_modify_rq(rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
1484 }
1485
1486 static void
1487 mlx5e_close_rq_wait(struct mlx5e_rq *rq)
1488 {
1489
1490         mlx5e_disable_rq(rq);
1491         mlx5e_close_cq(&rq->cq);
1492         cancel_work_sync(&rq->dim.work);
1493         mlx5e_destroy_rq(rq);
1494 }
1495
1496 void
1497 mlx5e_free_sq_db(struct mlx5e_sq *sq)
1498 {
1499         int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1500         int x;
1501
1502         for (x = 0; x != wq_sz; x++) {
1503                 if (sq->mbuf[x].mbuf != NULL) {
1504                         bus_dmamap_unload(sq->dma_tag, sq->mbuf[x].dma_map);
1505                         m_freem(sq->mbuf[x].mbuf);
1506                 }
1507                 bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map);
1508         }
1509         free(sq->mbuf, M_MLX5EN);
1510 }
1511
1512 int
1513 mlx5e_alloc_sq_db(struct mlx5e_sq *sq)
1514 {
1515         int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1516         int err;
1517         int x;
1518
1519         sq->mbuf = malloc(wq_sz * sizeof(sq->mbuf[0]), M_MLX5EN, M_WAITOK | M_ZERO);
1520
1521         /* Create DMA descriptor MAPs */
1522         for (x = 0; x != wq_sz; x++) {
1523                 err = -bus_dmamap_create(sq->dma_tag, 0, &sq->mbuf[x].dma_map);
1524                 if (err != 0) {
1525                         while (x--)
1526                                 bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map);
1527                         free(sq->mbuf, M_MLX5EN);
1528                         return (err);
1529                 }
1530         }
1531         return (0);
1532 }
1533
1534 static const char *mlx5e_sq_stats_desc[] = {
1535         MLX5E_SQ_STATS(MLX5E_STATS_DESC)
1536 };
1537
1538 void
1539 mlx5e_update_sq_inline(struct mlx5e_sq *sq)
1540 {
1541         sq->max_inline = sq->priv->params.tx_max_inline;
1542         sq->min_inline_mode = sq->priv->params.tx_min_inline_mode;
1543
1544         /*
1545          * Check if trust state is DSCP or if inline mode is NONE which
1546          * indicates CX-5 or newer hardware.
1547          */
1548         if (sq->priv->params_ethtool.trust_state != MLX5_QPTS_TRUST_PCP ||
1549             sq->min_inline_mode == MLX5_INLINE_MODE_NONE) {
1550                 if (MLX5_CAP_ETH(sq->priv->mdev, wqe_vlan_insert))
1551                         sq->min_insert_caps = MLX5E_INSERT_VLAN | MLX5E_INSERT_NON_VLAN;
1552                 else
1553                         sq->min_insert_caps = MLX5E_INSERT_NON_VLAN;
1554         } else {
1555                 sq->min_insert_caps = 0;
1556         }
1557 }
1558
1559 static void
1560 mlx5e_refresh_sq_inline_sub(struct mlx5e_priv *priv, struct mlx5e_channel *c)
1561 {
1562         int i;
1563
1564         for (i = 0; i != priv->num_tc; i++) {
1565                 mtx_lock(&c->sq[i].lock);
1566                 mlx5e_update_sq_inline(&c->sq[i]);
1567                 mtx_unlock(&c->sq[i].lock);
1568         }
1569 }
1570
1571 void
1572 mlx5e_refresh_sq_inline(struct mlx5e_priv *priv)
1573 {
1574         int i;
1575
1576         /* check if channels are closed */
1577         if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
1578                 return;
1579
1580         for (i = 0; i < priv->params.num_channels; i++)
1581                 mlx5e_refresh_sq_inline_sub(priv, &priv->channel[i]);
1582 }
1583
1584 static int
1585 mlx5e_create_sq(struct mlx5e_channel *c,
1586     int tc,
1587     struct mlx5e_sq_param *param,
1588     struct mlx5e_sq *sq)
1589 {
1590         struct mlx5e_priv *priv = c->priv;
1591         struct mlx5_core_dev *mdev = priv->mdev;
1592         char buffer[16];
1593         void *sqc = param->sqc;
1594         void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq);
1595         int err;
1596
1597         /* Create DMA descriptor TAG */
1598         if ((err = -bus_dma_tag_create(
1599             bus_get_dma_tag(mdev->pdev->dev.bsddev),
1600             1,                          /* any alignment */
1601             0,                          /* no boundary */
1602             BUS_SPACE_MAXADDR,          /* lowaddr */
1603             BUS_SPACE_MAXADDR,          /* highaddr */
1604             NULL, NULL,                 /* filter, filterarg */
1605             MLX5E_MAX_TX_PAYLOAD_SIZE,  /* maxsize */
1606             MLX5E_MAX_TX_MBUF_FRAGS,    /* nsegments */
1607             MLX5E_MAX_TX_MBUF_SIZE,     /* maxsegsize */
1608             0,                          /* flags */
1609             NULL, NULL,                 /* lockfunc, lockfuncarg */
1610             &sq->dma_tag)))
1611                 goto done;
1612
1613         err = mlx5_alloc_map_uar(mdev, &sq->uar);
1614         if (err)
1615                 goto err_free_dma_tag;
1616
1617         err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq,
1618             &sq->wq_ctrl);
1619         if (err)
1620                 goto err_unmap_free_uar;
1621
1622         sq->wq.db = &sq->wq.db[MLX5_SND_DBR];
1623         sq->bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2;
1624
1625         err = mlx5e_alloc_sq_db(sq);
1626         if (err)
1627                 goto err_sq_wq_destroy;
1628
1629         sq->mkey_be = cpu_to_be32(priv->mr.key);
1630         sq->ifp = priv->ifp;
1631         sq->priv = priv;
1632         sq->tc = tc;
1633
1634         mlx5e_update_sq_inline(sq);
1635
1636         snprintf(buffer, sizeof(buffer), "txstat%dtc%d", c->ix, tc);
1637         mlx5e_create_stats(&sq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
1638             buffer, mlx5e_sq_stats_desc, MLX5E_SQ_STATS_NUM,
1639             sq->stats.arg);
1640
1641         return (0);
1642
1643 err_sq_wq_destroy:
1644         mlx5_wq_destroy(&sq->wq_ctrl);
1645
1646 err_unmap_free_uar:
1647         mlx5_unmap_free_uar(mdev, &sq->uar);
1648
1649 err_free_dma_tag:
1650         bus_dma_tag_destroy(sq->dma_tag);
1651 done:
1652         return (err);
1653 }
1654
1655 static void
1656 mlx5e_destroy_sq(struct mlx5e_sq *sq)
1657 {
1658         /* destroy all sysctl nodes */
1659         sysctl_ctx_free(&sq->stats.ctx);
1660
1661         mlx5e_free_sq_db(sq);
1662         mlx5_wq_destroy(&sq->wq_ctrl);
1663         mlx5_unmap_free_uar(sq->priv->mdev, &sq->uar);
1664         bus_dma_tag_destroy(sq->dma_tag);
1665 }
1666
1667 int
1668 mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param,
1669     int tis_num)
1670 {
1671         void *in;
1672         void *sqc;
1673         void *wq;
1674         int inlen;
1675         int err;
1676
1677         inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
1678             sizeof(u64) * sq->wq_ctrl.buf.npages;
1679         in = mlx5_vzalloc(inlen);
1680         if (in == NULL)
1681                 return (-ENOMEM);
1682
1683         sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1684         wq = MLX5_ADDR_OF(sqc, sqc, wq);
1685
1686         memcpy(sqc, param->sqc, sizeof(param->sqc));
1687
1688         MLX5_SET(sqc, sqc, tis_num_0, tis_num);
1689         MLX5_SET(sqc, sqc, cqn, sq->cq.mcq.cqn);
1690         MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
1691         MLX5_SET(sqc, sqc, tis_lst_sz, 1);
1692         MLX5_SET(sqc, sqc, flush_in_error_en, 1);
1693
1694         MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
1695         MLX5_SET(wq, wq, uar_page, sq->uar.index);
1696         MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift -
1697             PAGE_SHIFT);
1698         MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma);
1699
1700         mlx5_fill_page_array(&sq->wq_ctrl.buf,
1701             (__be64 *) MLX5_ADDR_OF(wq, wq, pas));
1702
1703         err = mlx5_core_create_sq(sq->priv->mdev, in, inlen, &sq->sqn);
1704
1705         kvfree(in);
1706
1707         return (err);
1708 }
1709
1710 int
1711 mlx5e_modify_sq(struct mlx5e_sq *sq, int curr_state, int next_state)
1712 {
1713         void *in;
1714         void *sqc;
1715         int inlen;
1716         int err;
1717
1718         inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1719         in = mlx5_vzalloc(inlen);
1720         if (in == NULL)
1721                 return (-ENOMEM);
1722
1723         sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
1724
1725         MLX5_SET(modify_sq_in, in, sqn, sq->sqn);
1726         MLX5_SET(modify_sq_in, in, sq_state, curr_state);
1727         MLX5_SET(sqc, sqc, state, next_state);
1728
1729         err = mlx5_core_modify_sq(sq->priv->mdev, in, inlen);
1730
1731         kvfree(in);
1732
1733         return (err);
1734 }
1735
1736 void
1737 mlx5e_disable_sq(struct mlx5e_sq *sq)
1738 {
1739
1740         mlx5_core_destroy_sq(sq->priv->mdev, sq->sqn);
1741 }
1742
1743 static int
1744 mlx5e_open_sq(struct mlx5e_channel *c,
1745     int tc,
1746     struct mlx5e_sq_param *param,
1747     struct mlx5e_sq *sq)
1748 {
1749         int err;
1750
1751         sq->cev_factor = c->priv->params_ethtool.tx_completion_fact;
1752
1753         /* ensure the TX completion event factor is not zero */
1754         if (sq->cev_factor == 0)
1755                 sq->cev_factor = 1;
1756
1757         err = mlx5e_create_sq(c, tc, param, sq);
1758         if (err)
1759                 return (err);
1760
1761         err = mlx5e_enable_sq(sq, param, c->priv->tisn[tc]);
1762         if (err)
1763                 goto err_destroy_sq;
1764
1765         err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY);
1766         if (err)
1767                 goto err_disable_sq;
1768
1769         WRITE_ONCE(sq->running, 1);
1770
1771         return (0);
1772
1773 err_disable_sq:
1774         mlx5e_disable_sq(sq);
1775 err_destroy_sq:
1776         mlx5e_destroy_sq(sq);
1777
1778         return (err);
1779 }
1780
1781 static void
1782 mlx5e_sq_send_nops_locked(struct mlx5e_sq *sq, int can_sleep)
1783 {
1784         /* fill up remainder with NOPs */
1785         while (sq->cev_counter != 0) {
1786                 while (!mlx5e_sq_has_room_for(sq, 1)) {
1787                         if (can_sleep != 0) {
1788                                 mtx_unlock(&sq->lock);
1789                                 msleep(4);
1790                                 mtx_lock(&sq->lock);
1791                         } else {
1792                                 goto done;
1793                         }
1794                 }
1795                 /* send a single NOP */
1796                 mlx5e_send_nop(sq, 1);
1797                 atomic_thread_fence_rel();
1798         }
1799 done:
1800         /* Check if we need to write the doorbell */
1801         if (likely(sq->doorbell.d64 != 0)) {
1802                 mlx5e_tx_notify_hw(sq, sq->doorbell.d32, 0);
1803                 sq->doorbell.d64 = 0;
1804         }
1805 }
1806
1807 void
1808 mlx5e_sq_cev_timeout(void *arg)
1809 {
1810         struct mlx5e_sq *sq = arg;
1811
1812         mtx_assert(&sq->lock, MA_OWNED);
1813
1814         /* check next state */
1815         switch (sq->cev_next_state) {
1816         case MLX5E_CEV_STATE_SEND_NOPS:
1817                 /* fill TX ring with NOPs, if any */
1818                 mlx5e_sq_send_nops_locked(sq, 0);
1819
1820                 /* check if completed */
1821                 if (sq->cev_counter == 0) {
1822                         sq->cev_next_state = MLX5E_CEV_STATE_INITIAL;
1823                         return;
1824                 }
1825                 break;
1826         default:
1827                 /* send NOPs on next timeout */
1828                 sq->cev_next_state = MLX5E_CEV_STATE_SEND_NOPS;
1829                 break;
1830         }
1831
1832         /* restart timer */
1833         callout_reset_curcpu(&sq->cev_callout, hz, mlx5e_sq_cev_timeout, sq);
1834 }
1835
1836 void
1837 mlx5e_drain_sq(struct mlx5e_sq *sq)
1838 {
1839         int error;
1840         struct mlx5_core_dev *mdev= sq->priv->mdev;
1841
1842         /*
1843          * Check if already stopped.
1844          *
1845          * NOTE: Serialization of this function is managed by the
1846          * caller ensuring the priv's state lock is locked or in case
1847          * of rate limit support, a single thread manages drain and
1848          * resume of SQs. The "running" variable can therefore safely
1849          * be read without any locks.
1850          */
1851         if (READ_ONCE(sq->running) == 0)
1852                 return;
1853
1854         /* don't put more packets into the SQ */
1855         WRITE_ONCE(sq->running, 0);
1856
1857         /* serialize access to DMA rings */
1858         mtx_lock(&sq->lock);
1859
1860         /* teardown event factor timer, if any */
1861         sq->cev_next_state = MLX5E_CEV_STATE_HOLD_NOPS;
1862         callout_stop(&sq->cev_callout);
1863
1864         /* send dummy NOPs in order to flush the transmit ring */
1865         mlx5e_sq_send_nops_locked(sq, 1);
1866         mtx_unlock(&sq->lock);
1867
1868         /* wait till SQ is empty or link is down */
1869         mtx_lock(&sq->lock);
1870         while (sq->cc != sq->pc &&
1871             (sq->priv->media_status_last & IFM_ACTIVE) != 0 &&
1872             mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) {
1873                 mtx_unlock(&sq->lock);
1874                 msleep(1);
1875                 sq->cq.mcq.comp(&sq->cq.mcq);
1876                 mtx_lock(&sq->lock);
1877         }
1878         mtx_unlock(&sq->lock);
1879
1880         /* error out remaining requests */
1881         error = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY, MLX5_SQC_STATE_ERR);
1882         if (error != 0) {
1883                 mlx5_en_err(sq->ifp,
1884                     "mlx5e_modify_sq() from RDY to ERR failed: %d\n", error);
1885         }
1886
1887         /* wait till SQ is empty */
1888         mtx_lock(&sq->lock);
1889         while (sq->cc != sq->pc &&
1890                mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) {
1891                 mtx_unlock(&sq->lock);
1892                 msleep(1);
1893                 sq->cq.mcq.comp(&sq->cq.mcq);
1894                 mtx_lock(&sq->lock);
1895         }
1896         mtx_unlock(&sq->lock);
1897 }
1898
1899 static void
1900 mlx5e_close_sq_wait(struct mlx5e_sq *sq)
1901 {
1902
1903         mlx5e_drain_sq(sq);
1904         mlx5e_disable_sq(sq);
1905         mlx5e_destroy_sq(sq);
1906 }
1907
1908 static int
1909 mlx5e_create_cq(struct mlx5e_priv *priv,
1910     struct mlx5e_cq_param *param,
1911     struct mlx5e_cq *cq,
1912     mlx5e_cq_comp_t *comp,
1913     int eq_ix)
1914 {
1915         struct mlx5_core_dev *mdev = priv->mdev;
1916         struct mlx5_core_cq *mcq = &cq->mcq;
1917         int eqn_not_used;
1918         int irqn;
1919         int err;
1920         u32 i;
1921
1922         param->wq.buf_numa_node = 0;
1923         param->wq.db_numa_node = 0;
1924
1925         err = mlx5_vector2eqn(mdev, eq_ix, &eqn_not_used, &irqn);
1926         if (err)
1927                 return (err);
1928
1929         err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
1930             &cq->wq_ctrl);
1931         if (err)
1932                 return (err);
1933
1934         mcq->cqe_sz = 64;
1935         mcq->set_ci_db = cq->wq_ctrl.db.db;
1936         mcq->arm_db = cq->wq_ctrl.db.db + 1;
1937         *mcq->set_ci_db = 0;
1938         *mcq->arm_db = 0;
1939         mcq->vector = eq_ix;
1940         mcq->comp = comp;
1941         mcq->event = mlx5e_cq_error_event;
1942         mcq->irqn = irqn;
1943         mcq->uar = &priv->cq_uar;
1944
1945         for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
1946                 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
1947
1948                 cqe->op_own = 0xf1;
1949         }
1950
1951         cq->priv = priv;
1952
1953         return (0);
1954 }
1955
1956 static void
1957 mlx5e_destroy_cq(struct mlx5e_cq *cq)
1958 {
1959         mlx5_wq_destroy(&cq->wq_ctrl);
1960 }
1961
1962 static int
1963 mlx5e_enable_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param, int eq_ix)
1964 {
1965         struct mlx5_core_cq *mcq = &cq->mcq;
1966         void *in;
1967         void *cqc;
1968         int inlen;
1969         int irqn_not_used;
1970         int eqn;
1971         int err;
1972
1973         inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
1974             sizeof(u64) * cq->wq_ctrl.buf.npages;
1975         in = mlx5_vzalloc(inlen);
1976         if (in == NULL)
1977                 return (-ENOMEM);
1978
1979         cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
1980
1981         memcpy(cqc, param->cqc, sizeof(param->cqc));
1982
1983         mlx5_fill_page_array(&cq->wq_ctrl.buf,
1984             (__be64 *) MLX5_ADDR_OF(create_cq_in, in, pas));
1985
1986         mlx5_vector2eqn(cq->priv->mdev, eq_ix, &eqn, &irqn_not_used);
1987
1988         MLX5_SET(cqc, cqc, c_eqn, eqn);
1989         MLX5_SET(cqc, cqc, uar_page, mcq->uar->index);
1990         MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
1991             PAGE_SHIFT);
1992         MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma);
1993
1994         err = mlx5_core_create_cq(cq->priv->mdev, mcq, in, inlen);
1995
1996         kvfree(in);
1997
1998         if (err)
1999                 return (err);
2000
2001         mlx5e_cq_arm(cq, MLX5_GET_DOORBELL_LOCK(&cq->priv->doorbell_lock));
2002
2003         return (0);
2004 }
2005
2006 static void
2007 mlx5e_disable_cq(struct mlx5e_cq *cq)
2008 {
2009
2010         mlx5_core_destroy_cq(cq->priv->mdev, &cq->mcq);
2011 }
2012
2013 int
2014 mlx5e_open_cq(struct mlx5e_priv *priv,
2015     struct mlx5e_cq_param *param,
2016     struct mlx5e_cq *cq,
2017     mlx5e_cq_comp_t *comp,
2018     int eq_ix)
2019 {
2020         int err;
2021
2022         err = mlx5e_create_cq(priv, param, cq, comp, eq_ix);
2023         if (err)
2024                 return (err);
2025
2026         err = mlx5e_enable_cq(cq, param, eq_ix);
2027         if (err)
2028                 goto err_destroy_cq;
2029
2030         return (0);
2031
2032 err_destroy_cq:
2033         mlx5e_destroy_cq(cq);
2034
2035         return (err);
2036 }
2037
2038 void
2039 mlx5e_close_cq(struct mlx5e_cq *cq)
2040 {
2041         mlx5e_disable_cq(cq);
2042         mlx5e_destroy_cq(cq);
2043 }
2044
2045 static int
2046 mlx5e_open_tx_cqs(struct mlx5e_channel *c,
2047     struct mlx5e_channel_param *cparam)
2048 {
2049         int err;
2050         int tc;
2051
2052         for (tc = 0; tc < c->priv->num_tc; tc++) {
2053                 /* open completion queue */
2054                 err = mlx5e_open_cq(c->priv, &cparam->tx_cq, &c->sq[tc].cq,
2055                     &mlx5e_tx_cq_comp, c->ix);
2056                 if (err)
2057                         goto err_close_tx_cqs;
2058         }
2059         return (0);
2060
2061 err_close_tx_cqs:
2062         for (tc--; tc >= 0; tc--)
2063                 mlx5e_close_cq(&c->sq[tc].cq);
2064
2065         return (err);
2066 }
2067
2068 static void
2069 mlx5e_close_tx_cqs(struct mlx5e_channel *c)
2070 {
2071         int tc;
2072
2073         for (tc = 0; tc < c->priv->num_tc; tc++)
2074                 mlx5e_close_cq(&c->sq[tc].cq);
2075 }
2076
2077 static int
2078 mlx5e_open_sqs(struct mlx5e_channel *c,
2079     struct mlx5e_channel_param *cparam)
2080 {
2081         int err;
2082         int tc;
2083
2084         for (tc = 0; tc < c->priv->num_tc; tc++) {
2085                 err = mlx5e_open_sq(c, tc, &cparam->sq, &c->sq[tc]);
2086                 if (err)
2087                         goto err_close_sqs;
2088         }
2089
2090         return (0);
2091
2092 err_close_sqs:
2093         for (tc--; tc >= 0; tc--)
2094                 mlx5e_close_sq_wait(&c->sq[tc]);
2095
2096         return (err);
2097 }
2098
2099 static void
2100 mlx5e_close_sqs_wait(struct mlx5e_channel *c)
2101 {
2102         int tc;
2103
2104         for (tc = 0; tc < c->priv->num_tc; tc++)
2105                 mlx5e_close_sq_wait(&c->sq[tc]);
2106 }
2107
2108 static void
2109 mlx5e_chan_static_init(struct mlx5e_priv *priv, struct mlx5e_channel *c, int ix)
2110 {
2111         int tc;
2112
2113         /* setup priv and channel number */
2114         c->priv = priv;
2115         c->ix = ix;
2116
2117         /* setup send tag */
2118         c->tag.m_snd_tag.ifp = priv->ifp;
2119         c->tag.type = IF_SND_TAG_TYPE_UNLIMITED;
2120
2121         mtx_init(&c->rq.mtx, "mlx5rx", MTX_NETWORK_LOCK, MTX_DEF);
2122
2123         callout_init_mtx(&c->rq.watchdog, &c->rq.mtx, 0);
2124
2125         for (tc = 0; tc != MLX5E_MAX_TX_NUM_TC; tc++) {
2126                 struct mlx5e_sq *sq = c->sq + tc;
2127
2128                 mtx_init(&sq->lock, "mlx5tx",
2129                     MTX_NETWORK_LOCK " TX", MTX_DEF);
2130                 mtx_init(&sq->comp_lock, "mlx5comp",
2131                     MTX_NETWORK_LOCK " TX", MTX_DEF);
2132
2133                 callout_init_mtx(&sq->cev_callout, &sq->lock, 0);
2134         }
2135 }
2136
2137 static void
2138 mlx5e_chan_static_destroy(struct mlx5e_channel *c)
2139 {
2140         int tc;
2141
2142         callout_drain(&c->rq.watchdog);
2143
2144         mtx_destroy(&c->rq.mtx);
2145
2146         for (tc = 0; tc != MLX5E_MAX_TX_NUM_TC; tc++) {
2147                 callout_drain(&c->sq[tc].cev_callout);
2148                 mtx_destroy(&c->sq[tc].lock);
2149                 mtx_destroy(&c->sq[tc].comp_lock);
2150         }
2151 }
2152
2153 static int
2154 mlx5e_open_channel(struct mlx5e_priv *priv,
2155     struct mlx5e_channel_param *cparam,
2156     struct mlx5e_channel *c)
2157 {
2158         int i, err;
2159
2160         /* zero non-persistant data */
2161         MLX5E_ZERO(&c->rq, mlx5e_rq_zero_start);
2162         for (i = 0; i != priv->num_tc; i++)
2163                 MLX5E_ZERO(&c->sq[i], mlx5e_sq_zero_start);
2164
2165         /* open transmit completion queue */
2166         err = mlx5e_open_tx_cqs(c, cparam);
2167         if (err)
2168                 goto err_free;
2169
2170         /* open receive completion queue */
2171         err = mlx5e_open_cq(c->priv, &cparam->rx_cq, &c->rq.cq,
2172             &mlx5e_rx_cq_comp, c->ix);
2173         if (err)
2174                 goto err_close_tx_cqs;
2175
2176         err = mlx5e_open_sqs(c, cparam);
2177         if (err)
2178                 goto err_close_rx_cq;
2179
2180         err = mlx5e_open_rq(c, &cparam->rq, &c->rq);
2181         if (err)
2182                 goto err_close_sqs;
2183
2184         /* poll receive queue initially */
2185         c->rq.cq.mcq.comp(&c->rq.cq.mcq);
2186
2187         return (0);
2188
2189 err_close_sqs:
2190         mlx5e_close_sqs_wait(c);
2191
2192 err_close_rx_cq:
2193         mlx5e_close_cq(&c->rq.cq);
2194
2195 err_close_tx_cqs:
2196         mlx5e_close_tx_cqs(c);
2197
2198 err_free:
2199         return (err);
2200 }
2201
2202 static void
2203 mlx5e_close_channel(struct mlx5e_channel *c)
2204 {
2205         mlx5e_close_rq(&c->rq);
2206 }
2207
2208 static void
2209 mlx5e_close_channel_wait(struct mlx5e_channel *c)
2210 {
2211         mlx5e_close_rq_wait(&c->rq);
2212         mlx5e_close_sqs_wait(c);
2213         mlx5e_close_tx_cqs(c);
2214 }
2215
2216 static int
2217 mlx5e_get_wqe_sz(struct mlx5e_priv *priv, u32 *wqe_sz, u32 *nsegs)
2218 {
2219         u32 r, n;
2220
2221         r = priv->params.hw_lro_en ? priv->params.lro_wqe_sz :
2222             MLX5E_SW2MB_MTU(priv->ifp->if_mtu);
2223         if (r > MJUM16BYTES)
2224                 return (-ENOMEM);
2225
2226         if (r > MJUM9BYTES)
2227                 r = MJUM16BYTES;
2228         else if (r > MJUMPAGESIZE)
2229                 r = MJUM9BYTES;
2230         else if (r > MCLBYTES)
2231                 r = MJUMPAGESIZE;
2232         else
2233                 r = MCLBYTES;
2234
2235         /*
2236          * n + 1 must be a power of two, because stride size must be.
2237          * Stride size is 16 * (n + 1), as the first segment is
2238          * control.
2239          */
2240         for (n = howmany(r, MLX5E_MAX_RX_BYTES); !powerof2(n + 1); n++)
2241                 ;
2242
2243         if (n > MLX5E_MAX_BUSDMA_RX_SEGS)
2244                 return (-ENOMEM);
2245
2246         *wqe_sz = r;
2247         *nsegs = n;
2248         return (0);
2249 }
2250
2251 static void
2252 mlx5e_build_rq_param(struct mlx5e_priv *priv,
2253     struct mlx5e_rq_param *param)
2254 {
2255         void *rqc = param->rqc;
2256         void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
2257         u32 wqe_sz, nsegs;
2258
2259         mlx5e_get_wqe_sz(priv, &wqe_sz, &nsegs);
2260         MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
2261         MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
2262         MLX5_SET(wq, wq, log_wq_stride, ilog2(sizeof(struct mlx5e_rx_wqe) +
2263             nsegs * sizeof(struct mlx5_wqe_data_seg)));
2264         MLX5_SET(wq, wq, log_wq_sz, priv->params.log_rq_size);
2265         MLX5_SET(wq, wq, pd, priv->pdn);
2266
2267         param->wq.buf_numa_node = 0;
2268         param->wq.db_numa_node = 0;
2269         param->wq.linear = 1;
2270 }
2271
2272 static void
2273 mlx5e_build_sq_param(struct mlx5e_priv *priv,
2274     struct mlx5e_sq_param *param)
2275 {
2276         void *sqc = param->sqc;
2277         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
2278
2279         MLX5_SET(wq, wq, log_wq_sz, priv->params.log_sq_size);
2280         MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
2281         MLX5_SET(wq, wq, pd, priv->pdn);
2282
2283         param->wq.buf_numa_node = 0;
2284         param->wq.db_numa_node = 0;
2285         param->wq.linear = 1;
2286 }
2287
2288 static void
2289 mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
2290     struct mlx5e_cq_param *param)
2291 {
2292         void *cqc = param->cqc;
2293
2294         MLX5_SET(cqc, cqc, uar_page, priv->cq_uar.index);
2295 }
2296
2297 static void
2298 mlx5e_get_default_profile(struct mlx5e_priv *priv, int mode, struct net_dim_cq_moder *ptr)
2299 {
2300
2301         *ptr = net_dim_get_profile(mode, MLX5E_DIM_DEFAULT_PROFILE);
2302
2303         /* apply LRO restrictions */
2304         if (priv->params.hw_lro_en &&
2305             ptr->pkts > MLX5E_DIM_MAX_RX_CQ_MODERATION_PKTS_WITH_LRO) {
2306                 ptr->pkts = MLX5E_DIM_MAX_RX_CQ_MODERATION_PKTS_WITH_LRO;
2307         }
2308 }
2309
2310 static void
2311 mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
2312     struct mlx5e_cq_param *param)
2313 {
2314         struct net_dim_cq_moder curr;
2315         void *cqc = param->cqc;
2316
2317         /*
2318          * We use MLX5_CQE_FORMAT_HASH because the RX hash mini CQE
2319          * format is more beneficial for FreeBSD use case.
2320          *
2321          * Adding support for MLX5_CQE_FORMAT_CSUM will require changes
2322          * in mlx5e_decompress_cqe.
2323          */
2324         if (priv->params.cqe_zipping_en) {
2325                 MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_HASH);
2326                 MLX5_SET(cqc, cqc, cqe_compression_en, 1);
2327         }
2328
2329         MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_rq_size);
2330
2331         switch (priv->params.rx_cq_moderation_mode) {
2332         case 0:
2333                 MLX5_SET(cqc, cqc, cq_period, priv->params.rx_cq_moderation_usec);
2334                 MLX5_SET(cqc, cqc, cq_max_count, priv->params.rx_cq_moderation_pkts);
2335                 MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2336                 break;
2337         case 1:
2338                 MLX5_SET(cqc, cqc, cq_period, priv->params.rx_cq_moderation_usec);
2339                 MLX5_SET(cqc, cqc, cq_max_count, priv->params.rx_cq_moderation_pkts);
2340                 if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
2341                         MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
2342                 else
2343                         MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2344                 break;
2345         case 2:
2346                 mlx5e_get_default_profile(priv, NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE, &curr);
2347                 MLX5_SET(cqc, cqc, cq_period, curr.usec);
2348                 MLX5_SET(cqc, cqc, cq_max_count, curr.pkts);
2349                 MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2350                 break;
2351         case 3:
2352                 mlx5e_get_default_profile(priv, NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE, &curr);
2353                 MLX5_SET(cqc, cqc, cq_period, curr.usec);
2354                 MLX5_SET(cqc, cqc, cq_max_count, curr.pkts);
2355                 if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
2356                         MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
2357                 else
2358                         MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2359                 break;
2360         default:
2361                 break;
2362         }
2363
2364         mlx5e_dim_build_cq_param(priv, param);
2365
2366         mlx5e_build_common_cq_param(priv, param);
2367 }
2368
2369 static void
2370 mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
2371     struct mlx5e_cq_param *param)
2372 {
2373         void *cqc = param->cqc;
2374
2375         MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_sq_size);
2376         MLX5_SET(cqc, cqc, cq_period, priv->params.tx_cq_moderation_usec);
2377         MLX5_SET(cqc, cqc, cq_max_count, priv->params.tx_cq_moderation_pkts);
2378
2379         switch (priv->params.tx_cq_moderation_mode) {
2380         case 0:
2381                 MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2382                 break;
2383         default:
2384                 if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
2385                         MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
2386                 else
2387                         MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2388                 break;
2389         }
2390
2391         mlx5e_build_common_cq_param(priv, param);
2392 }
2393
2394 static void
2395 mlx5e_build_channel_param(struct mlx5e_priv *priv,
2396     struct mlx5e_channel_param *cparam)
2397 {
2398         memset(cparam, 0, sizeof(*cparam));
2399
2400         mlx5e_build_rq_param(priv, &cparam->rq);
2401         mlx5e_build_sq_param(priv, &cparam->sq);
2402         mlx5e_build_rx_cq_param(priv, &cparam->rx_cq);
2403         mlx5e_build_tx_cq_param(priv, &cparam->tx_cq);
2404 }
2405
2406 static int
2407 mlx5e_open_channels(struct mlx5e_priv *priv)
2408 {
2409         struct mlx5e_channel_param *cparam;
2410         int err;
2411         int i;
2412         int j;
2413
2414         cparam = malloc(sizeof(*cparam), M_MLX5EN, M_WAITOK);
2415
2416         mlx5e_build_channel_param(priv, cparam);
2417         for (i = 0; i < priv->params.num_channels; i++) {
2418                 err = mlx5e_open_channel(priv, cparam, &priv->channel[i]);
2419                 if (err)
2420                         goto err_close_channels;
2421         }
2422
2423         for (j = 0; j < priv->params.num_channels; j++) {
2424                 err = mlx5e_wait_for_min_rx_wqes(&priv->channel[j].rq);
2425                 if (err)
2426                         goto err_close_channels;
2427         }
2428         free(cparam, M_MLX5EN);
2429         return (0);
2430
2431 err_close_channels:
2432         while (i--) {
2433                 mlx5e_close_channel(&priv->channel[i]);
2434                 mlx5e_close_channel_wait(&priv->channel[i]);
2435         }
2436         free(cparam, M_MLX5EN);
2437         return (err);
2438 }
2439
2440 static void
2441 mlx5e_close_channels(struct mlx5e_priv *priv)
2442 {
2443         int i;
2444
2445         for (i = 0; i < priv->params.num_channels; i++)
2446                 mlx5e_close_channel(&priv->channel[i]);
2447         for (i = 0; i < priv->params.num_channels; i++)
2448                 mlx5e_close_channel_wait(&priv->channel[i]);
2449 }
2450
2451 static int
2452 mlx5e_refresh_sq_params(struct mlx5e_priv *priv, struct mlx5e_sq *sq)
2453 {
2454
2455         if (MLX5_CAP_GEN(priv->mdev, cq_period_mode_modify)) {
2456                 uint8_t cq_mode;
2457
2458                 switch (priv->params.tx_cq_moderation_mode) {
2459                 case 0:
2460                 case 2:
2461                         cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
2462                         break;
2463                 default:
2464                         cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_CQE;
2465                         break;
2466                 }
2467
2468                 return (mlx5_core_modify_cq_moderation_mode(priv->mdev, &sq->cq.mcq,
2469                     priv->params.tx_cq_moderation_usec,
2470                     priv->params.tx_cq_moderation_pkts,
2471                     cq_mode));
2472         }
2473
2474         return (mlx5_core_modify_cq_moderation(priv->mdev, &sq->cq.mcq,
2475             priv->params.tx_cq_moderation_usec,
2476             priv->params.tx_cq_moderation_pkts));
2477 }
2478
2479 static int
2480 mlx5e_refresh_rq_params(struct mlx5e_priv *priv, struct mlx5e_rq *rq)
2481 {
2482
2483         if (MLX5_CAP_GEN(priv->mdev, cq_period_mode_modify)) {
2484                 uint8_t cq_mode;
2485                 uint8_t dim_mode;
2486                 int retval;
2487
2488                 switch (priv->params.rx_cq_moderation_mode) {
2489                 case 0:
2490                 case 2:
2491                         cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
2492                         dim_mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
2493                         break;
2494                 default:
2495                         cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_CQE;
2496                         dim_mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE;
2497                         break;
2498                 }
2499
2500                 /* tear down dynamic interrupt moderation */
2501                 mtx_lock(&rq->mtx);
2502                 rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
2503                 mtx_unlock(&rq->mtx);
2504
2505                 /* wait for dynamic interrupt moderation work task, if any */
2506                 cancel_work_sync(&rq->dim.work);
2507
2508                 if (priv->params.rx_cq_moderation_mode >= 2) {
2509                         struct net_dim_cq_moder curr;
2510
2511                         mlx5e_get_default_profile(priv, dim_mode, &curr);
2512
2513                         retval = mlx5_core_modify_cq_moderation_mode(priv->mdev, &rq->cq.mcq,
2514                             curr.usec, curr.pkts, cq_mode);
2515
2516                         /* set dynamic interrupt moderation mode and zero defaults */
2517                         mtx_lock(&rq->mtx);
2518                         rq->dim.mode = dim_mode;
2519                         rq->dim.state = 0;
2520                         rq->dim.profile_ix = MLX5E_DIM_DEFAULT_PROFILE;
2521                         mtx_unlock(&rq->mtx);
2522                 } else {
2523                         retval = mlx5_core_modify_cq_moderation_mode(priv->mdev, &rq->cq.mcq,
2524                             priv->params.rx_cq_moderation_usec,
2525                             priv->params.rx_cq_moderation_pkts,
2526                             cq_mode);
2527                 }
2528                 return (retval);
2529         }
2530
2531         return (mlx5_core_modify_cq_moderation(priv->mdev, &rq->cq.mcq,
2532             priv->params.rx_cq_moderation_usec,
2533             priv->params.rx_cq_moderation_pkts));
2534 }
2535
2536 static int
2537 mlx5e_refresh_channel_params_sub(struct mlx5e_priv *priv, struct mlx5e_channel *c)
2538 {
2539         int err;
2540         int i;
2541
2542         err = mlx5e_refresh_rq_params(priv, &c->rq);
2543         if (err)
2544                 goto done;
2545
2546         for (i = 0; i != priv->num_tc; i++) {
2547                 err = mlx5e_refresh_sq_params(priv, &c->sq[i]);
2548                 if (err)
2549                         goto done;
2550         }
2551 done:
2552         return (err);
2553 }
2554
2555 int
2556 mlx5e_refresh_channel_params(struct mlx5e_priv *priv)
2557 {
2558         int i;
2559
2560         /* check if channels are closed */
2561         if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
2562                 return (EINVAL);
2563
2564         for (i = 0; i < priv->params.num_channels; i++) {
2565                 int err;
2566
2567                 err = mlx5e_refresh_channel_params_sub(priv, &priv->channel[i]);
2568                 if (err)
2569                         return (err);
2570         }
2571         return (0);
2572 }
2573
2574 static int
2575 mlx5e_open_tis(struct mlx5e_priv *priv, int tc)
2576 {
2577         struct mlx5_core_dev *mdev = priv->mdev;
2578         u32 in[MLX5_ST_SZ_DW(create_tis_in)];
2579         void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
2580
2581         memset(in, 0, sizeof(in));
2582
2583         MLX5_SET(tisc, tisc, prio, tc);
2584         MLX5_SET(tisc, tisc, transport_domain, priv->tdn);
2585
2586         return (mlx5_core_create_tis(mdev, in, sizeof(in), &priv->tisn[tc]));
2587 }
2588
2589 static void
2590 mlx5e_close_tis(struct mlx5e_priv *priv, int tc)
2591 {
2592         mlx5_core_destroy_tis(priv->mdev, priv->tisn[tc]);
2593 }
2594
2595 static int
2596 mlx5e_open_tises(struct mlx5e_priv *priv)
2597 {
2598         int num_tc = priv->num_tc;
2599         int err;
2600         int tc;
2601
2602         for (tc = 0; tc < num_tc; tc++) {
2603                 err = mlx5e_open_tis(priv, tc);
2604                 if (err)
2605                         goto err_close_tises;
2606         }
2607
2608         return (0);
2609
2610 err_close_tises:
2611         for (tc--; tc >= 0; tc--)
2612                 mlx5e_close_tis(priv, tc);
2613
2614         return (err);
2615 }
2616
2617 static void
2618 mlx5e_close_tises(struct mlx5e_priv *priv)
2619 {
2620         int num_tc = priv->num_tc;
2621         int tc;
2622
2623         for (tc = 0; tc < num_tc; tc++)
2624                 mlx5e_close_tis(priv, tc);
2625 }
2626
2627 static int
2628 mlx5e_open_rqt(struct mlx5e_priv *priv)
2629 {
2630         struct mlx5_core_dev *mdev = priv->mdev;
2631         u32 *in;
2632         u32 out[MLX5_ST_SZ_DW(create_rqt_out)] = {0};
2633         void *rqtc;
2634         int inlen;
2635         int err;
2636         int sz;
2637         int i;
2638
2639         sz = 1 << priv->params.rx_hash_log_tbl_sz;
2640
2641         inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
2642         in = mlx5_vzalloc(inlen);
2643         if (in == NULL)
2644                 return (-ENOMEM);
2645         rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
2646
2647         MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
2648         MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
2649
2650         for (i = 0; i < sz; i++) {
2651                 int ix = i;
2652 #ifdef RSS
2653                 ix = rss_get_indirection_to_bucket(ix);
2654 #endif
2655                 /* ensure we don't overflow */
2656                 ix %= priv->params.num_channels;
2657
2658                 /* apply receive side scaling stride, if any */
2659                 ix -= ix % (int)priv->params.channels_rsss;
2660
2661                 MLX5_SET(rqtc, rqtc, rq_num[i], priv->channel[ix].rq.rqn);
2662         }
2663
2664         MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT);
2665
2666         err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out));
2667         if (!err)
2668                 priv->rqtn = MLX5_GET(create_rqt_out, out, rqtn);
2669
2670         kvfree(in);
2671
2672         return (err);
2673 }
2674
2675 static void
2676 mlx5e_close_rqt(struct mlx5e_priv *priv)
2677 {
2678         u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)] = {0};
2679         u32 out[MLX5_ST_SZ_DW(destroy_rqt_out)] = {0};
2680
2681         MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT);
2682         MLX5_SET(destroy_rqt_in, in, rqtn, priv->rqtn);
2683
2684         mlx5_cmd_exec(priv->mdev, in, sizeof(in), out, sizeof(out));
2685 }
2686
2687 static void
2688 mlx5e_build_tir_ctx(struct mlx5e_priv *priv, u32 * tirc, int tt)
2689 {
2690         void *hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
2691         __be32 *hkey;
2692
2693         MLX5_SET(tirc, tirc, transport_domain, priv->tdn);
2694
2695 #define ROUGH_MAX_L2_L3_HDR_SZ 256
2696
2697 #define MLX5_HASH_IP     (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2698                           MLX5_HASH_FIELD_SEL_DST_IP)
2699
2700 #define MLX5_HASH_ALL    (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2701                           MLX5_HASH_FIELD_SEL_DST_IP   |\
2702                           MLX5_HASH_FIELD_SEL_L4_SPORT |\
2703                           MLX5_HASH_FIELD_SEL_L4_DPORT)
2704
2705 #define MLX5_HASH_IP_IPSEC_SPI  (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2706                                  MLX5_HASH_FIELD_SEL_DST_IP   |\
2707                                  MLX5_HASH_FIELD_SEL_IPSEC_SPI)
2708
2709         if (priv->params.hw_lro_en) {
2710                 MLX5_SET(tirc, tirc, lro_enable_mask,
2711                     MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
2712                     MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO);
2713                 MLX5_SET(tirc, tirc, lro_max_msg_sz,
2714                     (priv->params.lro_wqe_sz -
2715                     ROUGH_MAX_L2_L3_HDR_SZ) >> 8);
2716                 /* TODO: add the option to choose timer value dynamically */
2717                 MLX5_SET(tirc, tirc, lro_timeout_period_usecs,
2718                     MLX5_CAP_ETH(priv->mdev,
2719                     lro_timer_supported_periods[2]));
2720         }
2721
2722         /* setup parameters for hashing TIR type, if any */
2723         switch (tt) {
2724         case MLX5E_TT_ANY:
2725                 MLX5_SET(tirc, tirc, disp_type,
2726                     MLX5_TIRC_DISP_TYPE_DIRECT);
2727                 MLX5_SET(tirc, tirc, inline_rqn,
2728                     priv->channel[0].rq.rqn);
2729                 break;
2730         default:
2731                 MLX5_SET(tirc, tirc, disp_type,
2732                     MLX5_TIRC_DISP_TYPE_INDIRECT);
2733                 MLX5_SET(tirc, tirc, indirect_table,
2734                     priv->rqtn);
2735                 MLX5_SET(tirc, tirc, rx_hash_fn,
2736                     MLX5_TIRC_RX_HASH_FN_HASH_TOEPLITZ);
2737                 hkey = (__be32 *) MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
2738 #ifdef RSS
2739                 /*
2740                  * The FreeBSD RSS implementation does currently not
2741                  * support symmetric Toeplitz hashes:
2742                  */
2743                 MLX5_SET(tirc, tirc, rx_hash_symmetric, 0);
2744                 rss_getkey((uint8_t *)hkey);
2745 #else
2746                 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
2747                 hkey[0] = cpu_to_be32(0xD181C62C);
2748                 hkey[1] = cpu_to_be32(0xF7F4DB5B);
2749                 hkey[2] = cpu_to_be32(0x1983A2FC);
2750                 hkey[3] = cpu_to_be32(0x943E1ADB);
2751                 hkey[4] = cpu_to_be32(0xD9389E6B);
2752                 hkey[5] = cpu_to_be32(0xD1039C2C);
2753                 hkey[6] = cpu_to_be32(0xA74499AD);
2754                 hkey[7] = cpu_to_be32(0x593D56D9);
2755                 hkey[8] = cpu_to_be32(0xF3253C06);
2756                 hkey[9] = cpu_to_be32(0x2ADC1FFC);
2757 #endif
2758                 break;
2759         }
2760
2761         switch (tt) {
2762         case MLX5E_TT_IPV4_TCP:
2763                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2764                     MLX5_L3_PROT_TYPE_IPV4);
2765                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2766                     MLX5_L4_PROT_TYPE_TCP);
2767 #ifdef RSS
2768                 if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4)) {
2769                         MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2770                             MLX5_HASH_IP);
2771                 } else
2772 #endif
2773                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2774                     MLX5_HASH_ALL);
2775                 break;
2776
2777         case MLX5E_TT_IPV6_TCP:
2778                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2779                     MLX5_L3_PROT_TYPE_IPV6);
2780                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2781                     MLX5_L4_PROT_TYPE_TCP);
2782 #ifdef RSS
2783                 if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV6)) {
2784                         MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2785                             MLX5_HASH_IP);
2786                 } else
2787 #endif
2788                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2789                     MLX5_HASH_ALL);
2790                 break;
2791
2792         case MLX5E_TT_IPV4_UDP:
2793                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2794                     MLX5_L3_PROT_TYPE_IPV4);
2795                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2796                     MLX5_L4_PROT_TYPE_UDP);
2797 #ifdef RSS
2798                 if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4)) {
2799                         MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2800                             MLX5_HASH_IP);
2801                 } else
2802 #endif
2803                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2804                     MLX5_HASH_ALL);
2805                 break;
2806
2807         case MLX5E_TT_IPV6_UDP:
2808                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2809                     MLX5_L3_PROT_TYPE_IPV6);
2810                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2811                     MLX5_L4_PROT_TYPE_UDP);
2812 #ifdef RSS
2813                 if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV6)) {
2814                         MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2815                             MLX5_HASH_IP);
2816                 } else
2817 #endif
2818                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2819                     MLX5_HASH_ALL);
2820                 break;
2821
2822         case MLX5E_TT_IPV4_IPSEC_AH:
2823                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2824                     MLX5_L3_PROT_TYPE_IPV4);
2825                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2826                     MLX5_HASH_IP_IPSEC_SPI);
2827                 break;
2828
2829         case MLX5E_TT_IPV6_IPSEC_AH:
2830                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2831                     MLX5_L3_PROT_TYPE_IPV6);
2832                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2833                     MLX5_HASH_IP_IPSEC_SPI);
2834                 break;
2835
2836         case MLX5E_TT_IPV4_IPSEC_ESP:
2837                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2838                     MLX5_L3_PROT_TYPE_IPV4);
2839                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2840                     MLX5_HASH_IP_IPSEC_SPI);
2841                 break;
2842
2843         case MLX5E_TT_IPV6_IPSEC_ESP:
2844                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2845                     MLX5_L3_PROT_TYPE_IPV6);
2846                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2847                     MLX5_HASH_IP_IPSEC_SPI);
2848                 break;
2849
2850         case MLX5E_TT_IPV4:
2851                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2852                     MLX5_L3_PROT_TYPE_IPV4);
2853                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2854                     MLX5_HASH_IP);
2855                 break;
2856
2857         case MLX5E_TT_IPV6:
2858                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2859                     MLX5_L3_PROT_TYPE_IPV6);
2860                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2861                     MLX5_HASH_IP);
2862                 break;
2863
2864         default:
2865                 break;
2866         }
2867 }
2868
2869 static int
2870 mlx5e_open_tir(struct mlx5e_priv *priv, int tt)
2871 {
2872         struct mlx5_core_dev *mdev = priv->mdev;
2873         u32 *in;
2874         void *tirc;
2875         int inlen;
2876         int err;
2877
2878         inlen = MLX5_ST_SZ_BYTES(create_tir_in);
2879         in = mlx5_vzalloc(inlen);
2880         if (in == NULL)
2881                 return (-ENOMEM);
2882         tirc = MLX5_ADDR_OF(create_tir_in, in, tir_context);
2883
2884         mlx5e_build_tir_ctx(priv, tirc, tt);
2885
2886         err = mlx5_core_create_tir(mdev, in, inlen, &priv->tirn[tt]);
2887
2888         kvfree(in);
2889
2890         return (err);
2891 }
2892
2893 static void
2894 mlx5e_close_tir(struct mlx5e_priv *priv, int tt)
2895 {
2896         mlx5_core_destroy_tir(priv->mdev, priv->tirn[tt]);
2897 }
2898
2899 static int
2900 mlx5e_open_tirs(struct mlx5e_priv *priv)
2901 {
2902         int err;
2903         int i;
2904
2905         for (i = 0; i < MLX5E_NUM_TT; i++) {
2906                 err = mlx5e_open_tir(priv, i);
2907                 if (err)
2908                         goto err_close_tirs;
2909         }
2910
2911         return (0);
2912
2913 err_close_tirs:
2914         for (i--; i >= 0; i--)
2915                 mlx5e_close_tir(priv, i);
2916
2917         return (err);
2918 }
2919
2920 static void
2921 mlx5e_close_tirs(struct mlx5e_priv *priv)
2922 {
2923         int i;
2924
2925         for (i = 0; i < MLX5E_NUM_TT; i++)
2926                 mlx5e_close_tir(priv, i);
2927 }
2928
2929 /*
2930  * SW MTU does not include headers,
2931  * HW MTU includes all headers and checksums.
2932  */
2933 static int
2934 mlx5e_set_dev_port_mtu(struct ifnet *ifp, int sw_mtu)
2935 {
2936         struct mlx5e_priv *priv = ifp->if_softc;
2937         struct mlx5_core_dev *mdev = priv->mdev;
2938         int hw_mtu;
2939         int err;
2940
2941         hw_mtu = MLX5E_SW2HW_MTU(sw_mtu);
2942
2943         err = mlx5_set_port_mtu(mdev, hw_mtu);
2944         if (err) {
2945                 mlx5_en_err(ifp, "mlx5_set_port_mtu failed setting %d, err=%d\n",
2946                     sw_mtu, err);
2947                 return (err);
2948         }
2949
2950         /* Update vport context MTU */
2951         err = mlx5_set_vport_mtu(mdev, hw_mtu);
2952         if (err) {
2953                 mlx5_en_err(ifp,
2954                     "Failed updating vport context with MTU size, err=%d\n",
2955                     err);
2956         }
2957
2958         ifp->if_mtu = sw_mtu;
2959
2960         err = mlx5_query_vport_mtu(mdev, &hw_mtu);
2961         if (err || !hw_mtu) {
2962                 /* fallback to port oper mtu */
2963                 err = mlx5_query_port_oper_mtu(mdev, &hw_mtu);
2964         }
2965         if (err) {
2966                 mlx5_en_err(ifp,
2967                     "Query port MTU, after setting new MTU value, failed\n");
2968                 return (err);
2969         } else if (MLX5E_HW2SW_MTU(hw_mtu) < sw_mtu) {
2970                 err = -E2BIG,
2971                 mlx5_en_err(ifp,
2972                     "Port MTU %d is smaller than ifp mtu %d\n",
2973                     hw_mtu, sw_mtu);
2974         } else if (MLX5E_HW2SW_MTU(hw_mtu) > sw_mtu) {
2975                 err = -EINVAL;
2976                 mlx5_en_err(ifp,
2977                     "Port MTU %d is bigger than ifp mtu %d\n",
2978                     hw_mtu, sw_mtu);
2979         }
2980         priv->params_ethtool.hw_mtu = hw_mtu;
2981
2982         return (err);
2983 }
2984
2985 int
2986 mlx5e_open_locked(struct ifnet *ifp)
2987 {
2988         struct mlx5e_priv *priv = ifp->if_softc;
2989         int err;
2990         u16 set_id;
2991
2992         /* check if already opened */
2993         if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
2994                 return (0);
2995
2996 #ifdef RSS
2997         if (rss_getnumbuckets() > priv->params.num_channels) {
2998                 mlx5_en_info(ifp,
2999                     "NOTE: There are more RSS buckets(%u) than channels(%u) available\n",
3000                     rss_getnumbuckets(), priv->params.num_channels);
3001         }
3002 #endif
3003         err = mlx5e_open_tises(priv);
3004         if (err) {
3005                 mlx5_en_err(ifp, "mlx5e_open_tises failed, %d\n", err);
3006                 return (err);
3007         }
3008         err = mlx5_vport_alloc_q_counter(priv->mdev,
3009             MLX5_INTERFACE_PROTOCOL_ETH, &set_id);
3010         if (err) {
3011                 mlx5_en_err(priv->ifp,
3012                     "mlx5_vport_alloc_q_counter failed: %d\n", err);
3013                 goto err_close_tises;
3014         }
3015         /* store counter set ID */
3016         priv->counter_set_id = set_id;
3017
3018         err = mlx5e_open_channels(priv);
3019         if (err) {
3020                 mlx5_en_err(ifp,
3021                     "mlx5e_open_channels failed, %d\n", err);
3022                 goto err_dalloc_q_counter;
3023         }
3024         err = mlx5e_open_rqt(priv);
3025         if (err) {
3026                 mlx5_en_err(ifp, "mlx5e_open_rqt failed, %d\n", err);
3027                 goto err_close_channels;
3028         }
3029         err = mlx5e_open_tirs(priv);
3030         if (err) {
3031                 mlx5_en_err(ifp, "mlx5e_open_tir failed, %d\n", err);
3032                 goto err_close_rqls;
3033         }
3034         err = mlx5e_open_flow_table(priv);
3035         if (err) {
3036                 mlx5_en_err(ifp,
3037                     "mlx5e_open_flow_table failed, %d\n", err);
3038                 goto err_close_tirs;
3039         }
3040         err = mlx5e_add_all_vlan_rules(priv);
3041         if (err) {
3042                 mlx5_en_err(ifp,
3043                     "mlx5e_add_all_vlan_rules failed, %d\n", err);
3044                 goto err_close_flow_table;
3045         }
3046         set_bit(MLX5E_STATE_OPENED, &priv->state);
3047
3048         mlx5e_update_carrier(priv);
3049         mlx5e_set_rx_mode_core(priv);
3050
3051         return (0);
3052
3053 err_close_flow_table:
3054         mlx5e_close_flow_table(priv);
3055
3056 err_close_tirs:
3057         mlx5e_close_tirs(priv);
3058
3059 err_close_rqls:
3060         mlx5e_close_rqt(priv);
3061
3062 err_close_channels:
3063         mlx5e_close_channels(priv);
3064
3065 err_dalloc_q_counter:
3066         mlx5_vport_dealloc_q_counter(priv->mdev,
3067             MLX5_INTERFACE_PROTOCOL_ETH, priv->counter_set_id);
3068
3069 err_close_tises:
3070         mlx5e_close_tises(priv);
3071
3072         return (err);
3073 }
3074
3075 static void
3076 mlx5e_open(void *arg)
3077 {
3078         struct mlx5e_priv *priv = arg;
3079
3080         PRIV_LOCK(priv);
3081         if (mlx5_set_port_status(priv->mdev, MLX5_PORT_UP))
3082                 mlx5_en_err(priv->ifp,
3083                     "Setting port status to up failed\n");
3084
3085         mlx5e_open_locked(priv->ifp);
3086         priv->ifp->if_drv_flags |= IFF_DRV_RUNNING;
3087         PRIV_UNLOCK(priv);
3088 }
3089
3090 int
3091 mlx5e_close_locked(struct ifnet *ifp)
3092 {
3093         struct mlx5e_priv *priv = ifp->if_softc;
3094
3095         /* check if already closed */
3096         if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
3097                 return (0);
3098
3099         clear_bit(MLX5E_STATE_OPENED, &priv->state);
3100
3101         mlx5e_set_rx_mode_core(priv);
3102         mlx5e_del_all_vlan_rules(priv);
3103         if_link_state_change(priv->ifp, LINK_STATE_DOWN);
3104         mlx5e_close_flow_table(priv);
3105         mlx5e_close_tirs(priv);
3106         mlx5e_close_rqt(priv);
3107         mlx5e_close_channels(priv);
3108         mlx5_vport_dealloc_q_counter(priv->mdev,
3109             MLX5_INTERFACE_PROTOCOL_ETH, priv->counter_set_id);
3110         mlx5e_close_tises(priv);
3111
3112         return (0);
3113 }
3114
3115 #if (__FreeBSD_version >= 1100000)
3116 static uint64_t
3117 mlx5e_get_counter(struct ifnet *ifp, ift_counter cnt)
3118 {
3119         struct mlx5e_priv *priv = ifp->if_softc;
3120         u64 retval;
3121
3122         /* PRIV_LOCK(priv); XXX not allowed */
3123         switch (cnt) {
3124         case IFCOUNTER_IPACKETS:
3125                 retval = priv->stats.vport.rx_packets;
3126                 break;
3127         case IFCOUNTER_IERRORS:
3128                 retval = priv->stats.pport.in_range_len_errors +
3129                     priv->stats.pport.out_of_range_len +
3130                     priv->stats.pport.too_long_errors +
3131                     priv->stats.pport.check_seq_err +
3132                     priv->stats.pport.alignment_err;
3133                 break;
3134         case IFCOUNTER_IQDROPS:
3135                 retval = priv->stats.vport.rx_out_of_buffer;
3136                 break;
3137         case IFCOUNTER_OPACKETS:
3138                 retval = priv->stats.vport.tx_packets;
3139                 break;
3140         case IFCOUNTER_OERRORS:
3141                 retval = priv->stats.port_stats_debug.out_discards;
3142                 break;
3143         case IFCOUNTER_IBYTES:
3144                 retval = priv->stats.vport.rx_bytes;
3145                 break;
3146         case IFCOUNTER_OBYTES:
3147                 retval = priv->stats.vport.tx_bytes;
3148                 break;
3149         case IFCOUNTER_IMCASTS:
3150                 retval = priv->stats.vport.rx_multicast_packets;
3151                 break;
3152         case IFCOUNTER_OMCASTS:
3153                 retval = priv->stats.vport.tx_multicast_packets;
3154                 break;
3155         case IFCOUNTER_OQDROPS:
3156                 retval = priv->stats.vport.tx_queue_dropped;
3157                 break;
3158         case IFCOUNTER_COLLISIONS:
3159                 retval = priv->stats.pport.collisions;
3160                 break;
3161         default:
3162                 retval = if_get_counter_default(ifp, cnt);
3163                 break;
3164         }
3165         /* PRIV_UNLOCK(priv); XXX not allowed */
3166         return (retval);
3167 }
3168 #endif
3169
3170 static void
3171 mlx5e_set_rx_mode(struct ifnet *ifp)
3172 {
3173         struct mlx5e_priv *priv = ifp->if_softc;
3174
3175         queue_work(priv->wq, &priv->set_rx_mode_work);
3176 }
3177
3178 static int
3179 mlx5e_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
3180 {
3181         struct mlx5e_priv *priv;
3182         struct ifreq *ifr;
3183         struct ifi2creq i2c;
3184         int error = 0;
3185         int mask = 0;
3186         int size_read = 0;
3187         int module_status;
3188         int module_num;
3189         int max_mtu;
3190         uint8_t read_addr;
3191
3192         priv = ifp->if_softc;
3193
3194         /* check if detaching */
3195         if (priv == NULL || priv->gone != 0)
3196                 return (ENXIO);
3197
3198         switch (command) {
3199         case SIOCSIFMTU:
3200                 ifr = (struct ifreq *)data;
3201
3202                 PRIV_LOCK(priv);
3203                 mlx5_query_port_max_mtu(priv->mdev, &max_mtu);
3204
3205                 if (ifr->ifr_mtu >= MLX5E_MTU_MIN &&
3206                     ifr->ifr_mtu <= MIN(MLX5E_MTU_MAX, max_mtu)) {
3207                         int was_opened;
3208
3209                         was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
3210                         if (was_opened)
3211                                 mlx5e_close_locked(ifp);
3212
3213                         /* set new MTU */
3214                         mlx5e_set_dev_port_mtu(ifp, ifr->ifr_mtu);
3215
3216                         if (was_opened)
3217                                 mlx5e_open_locked(ifp);
3218                 } else {
3219                         error = EINVAL;
3220                         mlx5_en_err(ifp,
3221                             "Invalid MTU value. Min val: %d, Max val: %d\n",
3222                             MLX5E_MTU_MIN, MIN(MLX5E_MTU_MAX, max_mtu));
3223                 }
3224                 PRIV_UNLOCK(priv);
3225                 break;
3226         case SIOCSIFFLAGS:
3227                 if ((ifp->if_flags & IFF_UP) &&
3228                     (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
3229                         mlx5e_set_rx_mode(ifp);
3230                         break;
3231                 }
3232                 PRIV_LOCK(priv);
3233                 if (ifp->if_flags & IFF_UP) {
3234                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
3235                                 if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
3236                                         mlx5e_open_locked(ifp);
3237                                 ifp->if_drv_flags |= IFF_DRV_RUNNING;
3238                                 mlx5_set_port_status(priv->mdev, MLX5_PORT_UP);
3239                         }
3240                 } else {
3241                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
3242                                 mlx5_set_port_status(priv->mdev,
3243                                     MLX5_PORT_DOWN);
3244                                 if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
3245                                         mlx5e_close_locked(ifp);
3246                                 mlx5e_update_carrier(priv);
3247                                 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3248                         }
3249                 }
3250                 PRIV_UNLOCK(priv);
3251                 break;
3252         case SIOCADDMULTI:
3253         case SIOCDELMULTI:
3254                 mlx5e_set_rx_mode(ifp);
3255                 break;
3256         case SIOCSIFMEDIA:
3257         case SIOCGIFMEDIA:
3258         case SIOCGIFXMEDIA:
3259                 ifr = (struct ifreq *)data;
3260                 error = ifmedia_ioctl(ifp, ifr, &priv->media, command);
3261                 break;
3262         case SIOCSIFCAP:
3263                 ifr = (struct ifreq *)data;
3264                 PRIV_LOCK(priv);
3265                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
3266
3267                 if (mask & IFCAP_TXCSUM) {
3268                         ifp->if_capenable ^= IFCAP_TXCSUM;
3269                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
3270
3271                         if (IFCAP_TSO4 & ifp->if_capenable &&
3272                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
3273                                 ifp->if_capenable &= ~IFCAP_TSO4;
3274                                 ifp->if_hwassist &= ~CSUM_IP_TSO;
3275                                 mlx5_en_err(ifp,
3276                                     "tso4 disabled due to -txcsum.\n");
3277                         }
3278                 }
3279                 if (mask & IFCAP_TXCSUM_IPV6) {
3280                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
3281                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
3282
3283                         if (IFCAP_TSO6 & ifp->if_capenable &&
3284                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
3285                                 ifp->if_capenable &= ~IFCAP_TSO6;
3286                                 ifp->if_hwassist &= ~CSUM_IP6_TSO;
3287                                 mlx5_en_err(ifp,
3288                                     "tso6 disabled due to -txcsum6.\n");
3289                         }
3290                 }
3291                 if (mask & IFCAP_RXCSUM)
3292                         ifp->if_capenable ^= IFCAP_RXCSUM;
3293                 if (mask & IFCAP_RXCSUM_IPV6)
3294                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
3295                 if (mask & IFCAP_TSO4) {
3296                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
3297                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
3298                                 mlx5_en_err(ifp, "enable txcsum first.\n");
3299                                 error = EAGAIN;
3300                                 goto out;
3301                         }
3302                         ifp->if_capenable ^= IFCAP_TSO4;
3303                         ifp->if_hwassist ^= CSUM_IP_TSO;
3304                 }
3305                 if (mask & IFCAP_TSO6) {
3306                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
3307                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
3308                                 mlx5_en_err(ifp, "enable txcsum6 first.\n");
3309                                 error = EAGAIN;
3310                                 goto out;
3311                         }
3312                         ifp->if_capenable ^= IFCAP_TSO6;
3313                         ifp->if_hwassist ^= CSUM_IP6_TSO;
3314                 }
3315                 if (mask & IFCAP_VLAN_HWFILTER) {
3316                         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
3317                                 mlx5e_disable_vlan_filter(priv);
3318                         else
3319                                 mlx5e_enable_vlan_filter(priv);
3320
3321                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
3322                 }
3323                 if (mask & IFCAP_VLAN_HWTAGGING)
3324                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
3325                 if (mask & IFCAP_WOL_MAGIC)
3326                         ifp->if_capenable ^= IFCAP_WOL_MAGIC;
3327
3328                 VLAN_CAPABILITIES(ifp);
3329                 /* turn off LRO means also turn of HW LRO - if it's on */
3330                 if (mask & IFCAP_LRO) {
3331                         int was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
3332                         bool need_restart = false;
3333
3334                         ifp->if_capenable ^= IFCAP_LRO;
3335
3336                         /* figure out if updating HW LRO is needed */
3337                         if (!(ifp->if_capenable & IFCAP_LRO)) {
3338                                 if (priv->params.hw_lro_en) {
3339                                         priv->params.hw_lro_en = false;
3340                                         need_restart = true;
3341                                 }
3342                         } else {
3343                                 if (priv->params.hw_lro_en == false &&
3344                                     priv->params_ethtool.hw_lro != 0) {
3345                                         priv->params.hw_lro_en = true;
3346                                         need_restart = true;
3347                                 }
3348                         }
3349                         if (was_opened && need_restart) {
3350                                 mlx5e_close_locked(ifp);
3351                                 mlx5e_open_locked(ifp);
3352                         }
3353                 }
3354                 if (mask & IFCAP_HWRXTSTMP) {
3355                         ifp->if_capenable ^= IFCAP_HWRXTSTMP;
3356                         if (ifp->if_capenable & IFCAP_HWRXTSTMP) {
3357                                 if (priv->clbr_done == 0)
3358                                         mlx5e_reset_calibration_callout(priv);
3359                         } else {
3360                                 callout_drain(&priv->tstmp_clbr);
3361                                 priv->clbr_done = 0;
3362                         }
3363                 }
3364 out:
3365                 PRIV_UNLOCK(priv);
3366                 break;
3367
3368         case SIOCGI2C:
3369                 ifr = (struct ifreq *)data;
3370
3371                 /*
3372                  * Copy from the user-space address ifr_data to the
3373                  * kernel-space address i2c
3374                  */
3375                 error = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c));
3376                 if (error)
3377                         break;
3378
3379                 if (i2c.len > sizeof(i2c.data)) {
3380                         error = EINVAL;
3381                         break;
3382                 }
3383
3384                 PRIV_LOCK(priv);
3385                 /* Get module_num which is required for the query_eeprom */
3386                 error = mlx5_query_module_num(priv->mdev, &module_num);
3387                 if (error) {
3388                         mlx5_en_err(ifp,
3389                             "Query module num failed, eeprom reading is not supported\n");
3390                         error = EINVAL;
3391                         goto err_i2c;
3392                 }
3393                 /* Check if module is present before doing an access */
3394                 module_status = mlx5_query_module_status(priv->mdev, module_num);
3395                 if (module_status != MLX5_MODULE_STATUS_PLUGGED_ENABLED) {
3396                         error = EINVAL;
3397                         goto err_i2c;
3398                 }
3399                 /*
3400                  * Currently 0XA0 and 0xA2 are the only addresses permitted.
3401                  * The internal conversion is as follows:
3402                  */
3403                 if (i2c.dev_addr == 0xA0)
3404                         read_addr = MLX5_I2C_ADDR_LOW;
3405                 else if (i2c.dev_addr == 0xA2)
3406                         read_addr = MLX5_I2C_ADDR_HIGH;
3407                 else {
3408                         mlx5_en_err(ifp,
3409                             "Query eeprom failed, Invalid Address: %X\n",
3410                             i2c.dev_addr);
3411                         error = EINVAL;
3412                         goto err_i2c;
3413                 }
3414                 error = mlx5_query_eeprom(priv->mdev,
3415                     read_addr, MLX5_EEPROM_LOW_PAGE,
3416                     (uint32_t)i2c.offset, (uint32_t)i2c.len, module_num,
3417                     (uint32_t *)i2c.data, &size_read);
3418                 if (error) {
3419                         mlx5_en_err(ifp,
3420                             "Query eeprom failed, eeprom reading is not supported\n");
3421                         error = EINVAL;
3422                         goto err_i2c;
3423                 }
3424
3425                 if (i2c.len > MLX5_EEPROM_MAX_BYTES) {
3426                         error = mlx5_query_eeprom(priv->mdev,
3427                             read_addr, MLX5_EEPROM_LOW_PAGE,
3428                             (uint32_t)(i2c.offset + size_read),
3429                             (uint32_t)(i2c.len - size_read), module_num,
3430                             (uint32_t *)(i2c.data + size_read), &size_read);
3431                 }
3432                 if (error) {
3433                         mlx5_en_err(ifp,
3434                             "Query eeprom failed, eeprom reading is not supported\n");
3435                         error = EINVAL;
3436                         goto err_i2c;
3437                 }
3438
3439                 error = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c));
3440 err_i2c:
3441                 PRIV_UNLOCK(priv);
3442                 break;
3443
3444         default:
3445                 error = ether_ioctl(ifp, command, data);
3446                 break;
3447         }
3448         return (error);
3449 }
3450
3451 static int
3452 mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev)
3453 {
3454         /*
3455          * TODO: uncoment once FW really sets all these bits if
3456          * (!mdev->caps.eth.rss_ind_tbl_cap || !mdev->caps.eth.csum_cap ||
3457          * !mdev->caps.eth.max_lso_cap || !mdev->caps.eth.vlan_cap ||
3458          * !(mdev->caps.gen.flags & MLX5_DEV_CAP_FLAG_SCQE_BRK_MOD)) return
3459          * -ENOTSUPP;
3460          */
3461
3462         /* TODO: add more must-to-have features */
3463
3464         if (MLX5_CAP_GEN(mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
3465                 return (-ENODEV);
3466
3467         return (0);
3468 }
3469
3470 static u16
3471 mlx5e_get_max_inline_cap(struct mlx5_core_dev *mdev)
3472 {
3473         uint32_t bf_buf_size = (1U << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2U;
3474
3475         bf_buf_size -= sizeof(struct mlx5e_tx_wqe) - 2;
3476
3477         /* verify against driver hardware limit */
3478         if (bf_buf_size > MLX5E_MAX_TX_INLINE)
3479                 bf_buf_size = MLX5E_MAX_TX_INLINE;
3480
3481         return (bf_buf_size);
3482 }
3483
3484 static int
3485 mlx5e_build_ifp_priv(struct mlx5_core_dev *mdev,
3486     struct mlx5e_priv *priv,
3487     int num_comp_vectors)
3488 {
3489         int err;
3490
3491         /*
3492          * TODO: Consider link speed for setting "log_sq_size",
3493          * "log_rq_size" and "cq_moderation_xxx":
3494          */
3495         priv->params.log_sq_size =
3496             MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
3497         priv->params.log_rq_size =
3498             MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
3499         priv->params.rx_cq_moderation_usec =
3500             MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
3501             MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE :
3502             MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
3503         priv->params.rx_cq_moderation_mode =
3504             MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? 1 : 0;
3505         priv->params.rx_cq_moderation_pkts =
3506             MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
3507         priv->params.tx_cq_moderation_usec =
3508             MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
3509         priv->params.tx_cq_moderation_pkts =
3510             MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
3511         priv->params.min_rx_wqes =
3512             MLX5E_PARAMS_DEFAULT_MIN_RX_WQES;
3513         priv->params.rx_hash_log_tbl_sz =
3514             (order_base_2(num_comp_vectors) >
3515             MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ) ?
3516             order_base_2(num_comp_vectors) :
3517             MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ;
3518         priv->params.num_tc = 1;
3519         priv->params.default_vlan_prio = 0;
3520         priv->counter_set_id = -1;
3521         priv->params.tx_max_inline = mlx5e_get_max_inline_cap(mdev);
3522
3523         err = mlx5_query_min_inline(mdev, &priv->params.tx_min_inline_mode);
3524         if (err)
3525                 return (err);
3526
3527         /*
3528          * hw lro is currently defaulted to off. when it won't anymore we
3529          * will consider the HW capability: "!!MLX5_CAP_ETH(mdev, lro_cap)"
3530          */
3531         priv->params.hw_lro_en = false;
3532         priv->params.lro_wqe_sz = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ;
3533
3534         /*
3535          * CQE zipping is currently defaulted to off. when it won't
3536          * anymore we will consider the HW capability:
3537          * "!!MLX5_CAP_GEN(mdev, cqe_compression)"
3538          */
3539         priv->params.cqe_zipping_en = false;
3540
3541         priv->mdev = mdev;
3542         priv->params.num_channels = num_comp_vectors;
3543         priv->params.channels_rsss = 1;
3544         priv->order_base_2_num_channels = order_base_2(num_comp_vectors);
3545         priv->queue_mapping_channel_mask =
3546             roundup_pow_of_two(num_comp_vectors) - 1;
3547         priv->num_tc = priv->params.num_tc;
3548         priv->default_vlan_prio = priv->params.default_vlan_prio;
3549
3550         INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
3551         INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
3552         INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
3553
3554         return (0);
3555 }
3556
3557 static int
3558 mlx5e_create_mkey(struct mlx5e_priv *priv, u32 pdn,
3559                   struct mlx5_core_mr *mkey)
3560 {
3561         struct ifnet *ifp = priv->ifp;
3562         struct mlx5_core_dev *mdev = priv->mdev;
3563         int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
3564         void *mkc;
3565         u32 *in;
3566         int err;
3567
3568         in = mlx5_vzalloc(inlen);
3569         if (in == NULL) {
3570                 mlx5_en_err(ifp, "failed to allocate inbox\n");
3571                 return (-ENOMEM);
3572         }
3573
3574         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
3575         MLX5_SET(mkc, mkc, access_mode, MLX5_ACCESS_MODE_PA);
3576         MLX5_SET(mkc, mkc, lw, 1);
3577         MLX5_SET(mkc, mkc, lr, 1);
3578
3579         MLX5_SET(mkc, mkc, pd, pdn);
3580         MLX5_SET(mkc, mkc, length64, 1);
3581         MLX5_SET(mkc, mkc, qpn, 0xffffff);
3582
3583         err = mlx5_core_create_mkey(mdev, mkey, in, inlen);
3584         if (err)
3585                 mlx5_en_err(ifp, "mlx5_core_create_mkey failed, %d\n",
3586                     err);
3587
3588         kvfree(in);
3589         return (err);
3590 }
3591
3592 static const char *mlx5e_vport_stats_desc[] = {
3593         MLX5E_VPORT_STATS(MLX5E_STATS_DESC)
3594 };
3595
3596 static const char *mlx5e_pport_stats_desc[] = {
3597         MLX5E_PPORT_STATS(MLX5E_STATS_DESC)
3598 };
3599
3600 static void
3601 mlx5e_priv_static_init(struct mlx5e_priv *priv, const uint32_t channels)
3602 {
3603         uint32_t x;
3604
3605         mtx_init(&priv->async_events_mtx, "mlx5async", MTX_NETWORK_LOCK, MTX_DEF);
3606         sx_init(&priv->state_lock, "mlx5state");
3607         callout_init_mtx(&priv->watchdog, &priv->async_events_mtx, 0);
3608         MLX5_INIT_DOORBELL_LOCK(&priv->doorbell_lock);
3609         for (x = 0; x != channels; x++)
3610                 mlx5e_chan_static_init(priv, &priv->channel[x], x);
3611 }
3612
3613 static void
3614 mlx5e_priv_static_destroy(struct mlx5e_priv *priv, const uint32_t channels)
3615 {
3616         uint32_t x;
3617
3618         for (x = 0; x != channels; x++)
3619                 mlx5e_chan_static_destroy(&priv->channel[x]);
3620         callout_drain(&priv->watchdog);
3621         mtx_destroy(&priv->async_events_mtx);
3622         sx_destroy(&priv->state_lock);
3623 }
3624
3625 static int
3626 sysctl_firmware(SYSCTL_HANDLER_ARGS)
3627 {
3628         /*
3629          * %d.%d%.d the string format.
3630          * fw_rev_{maj,min,sub} return u16, 2^16 = 65536.
3631          * We need at most 5 chars to store that.
3632          * It also has: two "." and NULL at the end, which means we need 18
3633          * (5*3 + 3) chars at most.
3634          */
3635         char fw[18];
3636         struct mlx5e_priv *priv = arg1;
3637         int error;
3638
3639         snprintf(fw, sizeof(fw), "%d.%d.%d", fw_rev_maj(priv->mdev), fw_rev_min(priv->mdev),
3640             fw_rev_sub(priv->mdev));
3641         error = sysctl_handle_string(oidp, fw, sizeof(fw), req);
3642         return (error);
3643 }
3644
3645 static void
3646 mlx5e_disable_tx_dma(struct mlx5e_channel *ch)
3647 {
3648         int i;
3649
3650         for (i = 0; i < ch->priv->num_tc; i++)
3651                 mlx5e_drain_sq(&ch->sq[i]);
3652 }
3653
3654 static void
3655 mlx5e_reset_sq_doorbell_record(struct mlx5e_sq *sq)
3656 {
3657
3658         sq->doorbell.d32[0] = cpu_to_be32(MLX5_OPCODE_NOP);
3659         sq->doorbell.d32[1] = cpu_to_be32(sq->sqn << 8);
3660         mlx5e_tx_notify_hw(sq, sq->doorbell.d32, 0);
3661         sq->doorbell.d64 = 0;
3662 }
3663
3664 void
3665 mlx5e_resume_sq(struct mlx5e_sq *sq)
3666 {
3667         int err;
3668
3669         /* check if already enabled */
3670         if (READ_ONCE(sq->running) != 0)
3671                 return;
3672
3673         err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_ERR,
3674             MLX5_SQC_STATE_RST);
3675         if (err != 0) {
3676                 mlx5_en_err(sq->ifp,
3677                     "mlx5e_modify_sq() from ERR to RST failed: %d\n", err);
3678         }
3679
3680         sq->cc = 0;
3681         sq->pc = 0;
3682
3683         /* reset doorbell prior to moving from RST to RDY */
3684         mlx5e_reset_sq_doorbell_record(sq);
3685
3686         err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST,
3687             MLX5_SQC_STATE_RDY);
3688         if (err != 0) {
3689                 mlx5_en_err(sq->ifp,
3690                     "mlx5e_modify_sq() from RST to RDY failed: %d\n", err);
3691         }
3692
3693         sq->cev_next_state = MLX5E_CEV_STATE_INITIAL;
3694         WRITE_ONCE(sq->running, 1);
3695 }
3696
3697 static void
3698 mlx5e_enable_tx_dma(struct mlx5e_channel *ch)
3699 {
3700         int i;
3701
3702         for (i = 0; i < ch->priv->num_tc; i++)
3703                 mlx5e_resume_sq(&ch->sq[i]);
3704 }
3705
3706 static void
3707 mlx5e_disable_rx_dma(struct mlx5e_channel *ch)
3708 {
3709         struct mlx5e_rq *rq = &ch->rq;
3710         int err;
3711
3712         mtx_lock(&rq->mtx);
3713         rq->enabled = 0;
3714         callout_stop(&rq->watchdog);
3715         mtx_unlock(&rq->mtx);
3716
3717         err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
3718         if (err != 0) {
3719                 mlx5_en_err(rq->ifp,
3720                     "mlx5e_modify_rq() from RDY to RST failed: %d\n", err);
3721         }
3722
3723         while (!mlx5_wq_ll_is_empty(&rq->wq)) {
3724                 msleep(1);
3725                 rq->cq.mcq.comp(&rq->cq.mcq);
3726         }
3727
3728         /*
3729          * Transitioning into RST state will allow the FW to track less ERR state queues,
3730          * thus reducing the recv queue flushing time
3731          */
3732         err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_ERR, MLX5_RQC_STATE_RST);
3733         if (err != 0) {
3734                 mlx5_en_err(rq->ifp,
3735                     "mlx5e_modify_rq() from ERR to RST failed: %d\n", err);
3736         }
3737 }
3738
3739 static void
3740 mlx5e_enable_rx_dma(struct mlx5e_channel *ch)
3741 {
3742         struct mlx5e_rq *rq = &ch->rq;
3743         int err;
3744
3745         rq->wq.wqe_ctr = 0;
3746         mlx5_wq_ll_update_db_record(&rq->wq);
3747         err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
3748         if (err != 0) {
3749                 mlx5_en_err(rq->ifp,
3750                     "mlx5e_modify_rq() from RST to RDY failed: %d\n", err);
3751         }
3752
3753         rq->enabled = 1;
3754
3755         rq->cq.mcq.comp(&rq->cq.mcq);
3756 }
3757
3758 void
3759 mlx5e_modify_tx_dma(struct mlx5e_priv *priv, uint8_t value)
3760 {
3761         int i;
3762
3763         if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
3764                 return;
3765
3766         for (i = 0; i < priv->params.num_channels; i++) {
3767                 if (value)
3768                         mlx5e_disable_tx_dma(&priv->channel[i]);
3769                 else
3770                         mlx5e_enable_tx_dma(&priv->channel[i]);
3771         }
3772 }
3773
3774 void
3775 mlx5e_modify_rx_dma(struct mlx5e_priv *priv, uint8_t value)
3776 {
3777         int i;
3778
3779         if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
3780                 return;
3781
3782         for (i = 0; i < priv->params.num_channels; i++) {
3783                 if (value)
3784                         mlx5e_disable_rx_dma(&priv->channel[i]);
3785                 else
3786                         mlx5e_enable_rx_dma(&priv->channel[i]);
3787         }
3788 }
3789
3790 static void
3791 mlx5e_add_hw_stats(struct mlx5e_priv *priv)
3792 {
3793         SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw),
3794             OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD, priv, 0,
3795             sysctl_firmware, "A", "HCA firmware version");
3796
3797         SYSCTL_ADD_STRING(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw),
3798             OID_AUTO, "board_id", CTLFLAG_RD, priv->mdev->board_id, 0,
3799             "Board ID");
3800 }
3801
3802 static int
3803 mlx5e_sysctl_tx_priority_flow_control(SYSCTL_HANDLER_ARGS)
3804 {
3805         struct mlx5e_priv *priv = arg1;
3806         uint8_t temp[MLX5E_MAX_PRIORITY];
3807         uint32_t tx_pfc;
3808         int err;
3809         int i;
3810
3811         PRIV_LOCK(priv);
3812
3813         tx_pfc = priv->params.tx_priority_flow_control;
3814
3815         for (i = 0; i != MLX5E_MAX_PRIORITY; i++)
3816                 temp[i] = (tx_pfc >> i) & 1;
3817
3818         err = SYSCTL_OUT(req, temp, MLX5E_MAX_PRIORITY);
3819         if (err || !req->newptr)
3820                 goto done;
3821         err = SYSCTL_IN(req, temp, MLX5E_MAX_PRIORITY);
3822         if (err)
3823                 goto done;
3824
3825         priv->params.tx_priority_flow_control = 0;
3826
3827         /* range check input value */
3828         for (i = 0; i != MLX5E_MAX_PRIORITY; i++) {
3829                 if (temp[i] > 1) {
3830                         err = ERANGE;
3831                         goto done;
3832                 }
3833                 priv->params.tx_priority_flow_control |= (temp[i] << i);
3834         }
3835
3836         /* check if update is required */
3837         if (tx_pfc != priv->params.tx_priority_flow_control)
3838                 err = -mlx5e_set_port_pfc(priv);
3839 done:
3840         if (err != 0)
3841                 priv->params.tx_priority_flow_control= tx_pfc;
3842         PRIV_UNLOCK(priv);
3843
3844         return (err);
3845 }
3846
3847 static int
3848 mlx5e_sysctl_rx_priority_flow_control(SYSCTL_HANDLER_ARGS)
3849 {
3850         struct mlx5e_priv *priv = arg1;
3851         uint8_t temp[MLX5E_MAX_PRIORITY];
3852         uint32_t rx_pfc;
3853         int err;
3854         int i;
3855
3856         PRIV_LOCK(priv);
3857
3858         rx_pfc = priv->params.rx_priority_flow_control;
3859
3860         for (i = 0; i != MLX5E_MAX_PRIORITY; i++)
3861                 temp[i] = (rx_pfc >> i) & 1;
3862
3863         err = SYSCTL_OUT(req, temp, MLX5E_MAX_PRIORITY);
3864         if (err || !req->newptr)
3865                 goto done;
3866         err = SYSCTL_IN(req, temp, MLX5E_MAX_PRIORITY);
3867         if (err)
3868                 goto done;
3869
3870         priv->params.rx_priority_flow_control = 0;
3871
3872         /* range check input value */
3873         for (i = 0; i != MLX5E_MAX_PRIORITY; i++) {
3874                 if (temp[i] > 1) {
3875                         err = ERANGE;
3876                         goto done;
3877                 }
3878                 priv->params.rx_priority_flow_control |= (temp[i] << i);
3879         }
3880
3881         /* check if update is required */
3882         if (rx_pfc != priv->params.rx_priority_flow_control) {
3883                 err = -mlx5e_set_port_pfc(priv);
3884                 if (err == 0 && priv->sw_is_port_buf_owner)
3885                         err = mlx5e_update_buf_lossy(priv);
3886         }
3887 done:
3888         if (err != 0)
3889                 priv->params.rx_priority_flow_control= rx_pfc;
3890         PRIV_UNLOCK(priv);
3891
3892         return (err);
3893 }
3894
3895 static void
3896 mlx5e_setup_pauseframes(struct mlx5e_priv *priv)
3897 {
3898 #if (__FreeBSD_version < 1100000)
3899         char path[96];
3900 #endif
3901         int error;
3902
3903         /* enable pauseframes by default */
3904         priv->params.tx_pauseframe_control = 1;
3905         priv->params.rx_pauseframe_control = 1;
3906
3907         /* disable ports flow control, PFC, by default */
3908         priv->params.tx_priority_flow_control = 0;
3909         priv->params.rx_priority_flow_control = 0;
3910
3911 #if (__FreeBSD_version < 1100000)
3912         /* compute path for sysctl */
3913         snprintf(path, sizeof(path), "dev.mce.%d.tx_pauseframe_control",
3914             device_get_unit(priv->mdev->pdev->dev.bsddev));
3915
3916         /* try to fetch tunable, if any */
3917         TUNABLE_INT_FETCH(path, &priv->params.tx_pauseframe_control);
3918
3919         /* compute path for sysctl */
3920         snprintf(path, sizeof(path), "dev.mce.%d.rx_pauseframe_control",
3921             device_get_unit(priv->mdev->pdev->dev.bsddev));
3922
3923         /* try to fetch tunable, if any */
3924         TUNABLE_INT_FETCH(path, &priv->params.rx_pauseframe_control);
3925 #endif
3926
3927         /* register pauseframe SYSCTLs */
3928         SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
3929             OID_AUTO, "tx_pauseframe_control", CTLFLAG_RDTUN,
3930             &priv->params.tx_pauseframe_control, 0,
3931             "Set to enable TX pause frames. Clear to disable.");
3932
3933         SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
3934             OID_AUTO, "rx_pauseframe_control", CTLFLAG_RDTUN,
3935             &priv->params.rx_pauseframe_control, 0,
3936             "Set to enable RX pause frames. Clear to disable.");
3937
3938         /* register priority flow control, PFC, SYSCTLs */
3939         SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
3940             OID_AUTO, "tx_priority_flow_control", CTLTYPE_U8 | CTLFLAG_RWTUN |
3941             CTLFLAG_MPSAFE, priv, 0, &mlx5e_sysctl_tx_priority_flow_control, "CU",
3942             "Set to enable TX ports flow control frames for priorities 0..7. Clear to disable.");
3943
3944         SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
3945             OID_AUTO, "rx_priority_flow_control", CTLTYPE_U8 | CTLFLAG_RWTUN |
3946             CTLFLAG_MPSAFE, priv, 0, &mlx5e_sysctl_rx_priority_flow_control, "CU",
3947             "Set to enable RX ports flow control frames for priorities 0..7. Clear to disable.");
3948
3949         PRIV_LOCK(priv);
3950
3951         /* range check */
3952         priv->params.tx_pauseframe_control =
3953             priv->params.tx_pauseframe_control ? 1 : 0;
3954         priv->params.rx_pauseframe_control =
3955             priv->params.rx_pauseframe_control ? 1 : 0;
3956
3957         /* update firmware */
3958         error = mlx5e_set_port_pause_and_pfc(priv);
3959         if (error == -EINVAL) {
3960                 mlx5_en_err(priv->ifp,
3961                     "Global pauseframes must be disabled before enabling PFC.\n");
3962                 priv->params.rx_priority_flow_control = 0;
3963                 priv->params.tx_priority_flow_control = 0;
3964
3965                 /* update firmware */
3966                 (void) mlx5e_set_port_pause_and_pfc(priv);
3967         }
3968         PRIV_UNLOCK(priv);
3969 }
3970
3971 static int
3972 mlx5e_ul_snd_tag_alloc(struct ifnet *ifp,
3973     union if_snd_tag_alloc_params *params,
3974     struct m_snd_tag **ppmt)
3975 {
3976         struct mlx5e_priv *priv;
3977         struct mlx5e_channel *pch;
3978
3979         priv = ifp->if_softc;
3980
3981         if (unlikely(priv->gone || params->hdr.flowtype == M_HASHTYPE_NONE)) {
3982                 return (EOPNOTSUPP);
3983         } else {
3984                 /* keep this code synced with mlx5e_select_queue() */
3985                 u32 ch = priv->params.num_channels;
3986 #ifdef RSS
3987                 u32 temp;
3988
3989                 if (rss_hash2bucket(params->hdr.flowid,
3990                     params->hdr.flowtype, &temp) == 0)
3991                         ch = temp % ch;
3992                 else
3993 #endif
3994                         ch = (params->hdr.flowid % 128) % ch;
3995
3996                 /*
3997                  * NOTE: The channels array is only freed at detach
3998                  * and it safe to return a pointer to the send tag
3999                  * inside the channels structure as long as we
4000                  * reference the priv.
4001                  */
4002                 pch = priv->channel + ch;
4003
4004                 /* check if send queue is not running */
4005                 if (unlikely(pch->sq[0].running == 0))
4006                         return (ENXIO);
4007                 mlx5e_ref_channel(priv);
4008                 *ppmt = &pch->tag.m_snd_tag;
4009                 return (0);
4010         }
4011 }
4012
4013 static int
4014 mlx5e_ul_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *params)
4015 {
4016         struct mlx5e_channel *pch =
4017             container_of(pmt, struct mlx5e_channel, tag.m_snd_tag);
4018
4019         params->unlimited.max_rate = -1ULL;
4020         params->unlimited.queue_level = mlx5e_sq_queue_level(&pch->sq[0]);
4021         return (0);
4022 }
4023
4024 static void
4025 mlx5e_ul_snd_tag_free(struct m_snd_tag *pmt)
4026 {
4027         struct mlx5e_channel *pch =
4028             container_of(pmt, struct mlx5e_channel, tag.m_snd_tag);
4029
4030         mlx5e_unref_channel(pch->priv);
4031 }
4032
4033 static int
4034 mlx5e_snd_tag_alloc(struct ifnet *ifp,
4035     union if_snd_tag_alloc_params *params,
4036     struct m_snd_tag **ppmt)
4037 {
4038
4039         switch (params->hdr.type) {
4040 #ifdef RATELIMIT
4041         case IF_SND_TAG_TYPE_RATE_LIMIT:
4042                 return (mlx5e_rl_snd_tag_alloc(ifp, params, ppmt));
4043 #endif
4044         case IF_SND_TAG_TYPE_UNLIMITED:
4045                 return (mlx5e_ul_snd_tag_alloc(ifp, params, ppmt));
4046         default:
4047                 return (EOPNOTSUPP);
4048         }
4049 }
4050
4051 static int
4052 mlx5e_snd_tag_modify(struct m_snd_tag *pmt, union if_snd_tag_modify_params *params)
4053 {
4054         struct mlx5e_snd_tag *tag =
4055             container_of(pmt, struct mlx5e_snd_tag, m_snd_tag);
4056
4057         switch (tag->type) {
4058 #ifdef RATELIMIT
4059         case IF_SND_TAG_TYPE_RATE_LIMIT:
4060                 return (mlx5e_rl_snd_tag_modify(pmt, params));
4061 #endif
4062         case IF_SND_TAG_TYPE_UNLIMITED:
4063         default:
4064                 return (EOPNOTSUPP);
4065         }
4066 }
4067
4068 static int
4069 mlx5e_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *params)
4070 {
4071         struct mlx5e_snd_tag *tag =
4072             container_of(pmt, struct mlx5e_snd_tag, m_snd_tag);
4073
4074         switch (tag->type) {
4075 #ifdef RATELIMIT
4076         case IF_SND_TAG_TYPE_RATE_LIMIT:
4077                 return (mlx5e_rl_snd_tag_query(pmt, params));
4078 #endif
4079         case IF_SND_TAG_TYPE_UNLIMITED:
4080                 return (mlx5e_ul_snd_tag_query(pmt, params));
4081         default:
4082                 return (EOPNOTSUPP);
4083         }
4084 }
4085
4086 static void
4087 mlx5e_snd_tag_free(struct m_snd_tag *pmt)
4088 {
4089         struct mlx5e_snd_tag *tag =
4090             container_of(pmt, struct mlx5e_snd_tag, m_snd_tag);
4091
4092         switch (tag->type) {
4093 #ifdef RATELIMIT
4094         case IF_SND_TAG_TYPE_RATE_LIMIT:
4095                 mlx5e_rl_snd_tag_free(pmt);
4096                 break;
4097 #endif
4098         case IF_SND_TAG_TYPE_UNLIMITED:
4099                 mlx5e_ul_snd_tag_free(pmt);
4100                 break;
4101         default:
4102                 break;
4103         }
4104 }
4105
4106 static void *
4107 mlx5e_create_ifp(struct mlx5_core_dev *mdev)
4108 {
4109         struct ifnet *ifp;
4110         struct mlx5e_priv *priv;
4111         u8 dev_addr[ETHER_ADDR_LEN] __aligned(4);
4112         u8 connector_type;
4113         struct sysctl_oid_list *child;
4114         int ncv = mdev->priv.eq_table.num_comp_vectors;
4115         char unit[16];
4116         int err;
4117         int i,j;
4118         u32 eth_proto_cap;
4119         u32 out[MLX5_ST_SZ_DW(ptys_reg)];
4120         bool ext = 0;
4121         u32 speeds_num;
4122         struct media media_entry = {};
4123
4124         if (mlx5e_check_required_hca_cap(mdev)) {
4125                 mlx5_core_dbg(mdev, "mlx5e_check_required_hca_cap() failed\n");
4126                 return (NULL);
4127         }
4128         /*
4129          * Try to allocate the priv and make room for worst-case
4130          * number of channel structures:
4131          */
4132         priv = malloc(sizeof(*priv) +
4133             (sizeof(priv->channel[0]) * mdev->priv.eq_table.num_comp_vectors),
4134             M_MLX5EN, M_WAITOK | M_ZERO);
4135
4136         ifp = priv->ifp = if_alloc(IFT_ETHER);
4137         if (ifp == NULL) {
4138                 mlx5_core_err(mdev, "if_alloc() failed\n");
4139                 goto err_free_priv;
4140         }
4141         /* setup all static fields */
4142         mlx5e_priv_static_init(priv, mdev->priv.eq_table.num_comp_vectors);
4143
4144         ifp->if_softc = priv;
4145         if_initname(ifp, "mce", device_get_unit(mdev->pdev->dev.bsddev));
4146         ifp->if_mtu = ETHERMTU;
4147         ifp->if_init = mlx5e_open;
4148         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
4149         ifp->if_ioctl = mlx5e_ioctl;
4150         ifp->if_transmit = mlx5e_xmit;
4151         ifp->if_qflush = if_qflush;
4152 #if (__FreeBSD_version >= 1100000)
4153         ifp->if_get_counter = mlx5e_get_counter;
4154 #endif
4155         ifp->if_snd.ifq_maxlen = ifqmaxlen;
4156         /*
4157          * Set driver features
4158          */
4159         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6;
4160         ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING;
4161         ifp->if_capabilities |= IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWFILTER;
4162         ifp->if_capabilities |= IFCAP_LINKSTATE | IFCAP_JUMBO_MTU;
4163         ifp->if_capabilities |= IFCAP_LRO;
4164         ifp->if_capabilities |= IFCAP_TSO | IFCAP_VLAN_HWTSO;
4165         ifp->if_capabilities |= IFCAP_HWSTATS | IFCAP_HWRXTSTMP;
4166         ifp->if_capabilities |= IFCAP_TXRTLMT;
4167         ifp->if_snd_tag_alloc = mlx5e_snd_tag_alloc;
4168         ifp->if_snd_tag_free = mlx5e_snd_tag_free;
4169         ifp->if_snd_tag_modify = mlx5e_snd_tag_modify;
4170         ifp->if_snd_tag_query = mlx5e_snd_tag_query;
4171
4172         /* set TSO limits so that we don't have to drop TX packets */
4173         ifp->if_hw_tsomax = MLX5E_MAX_TX_PAYLOAD_SIZE - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
4174         ifp->if_hw_tsomaxsegcount = MLX5E_MAX_TX_MBUF_FRAGS - 1 /* hdr */;
4175         ifp->if_hw_tsomaxsegsize = MLX5E_MAX_TX_MBUF_SIZE;
4176
4177         ifp->if_capenable = ifp->if_capabilities;
4178         ifp->if_hwassist = 0;
4179         if (ifp->if_capenable & IFCAP_TSO)
4180                 ifp->if_hwassist |= CSUM_TSO;
4181         if (ifp->if_capenable & IFCAP_TXCSUM)
4182                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP);
4183         if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
4184                 ifp->if_hwassist |= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
4185
4186         /* ifnet sysctl tree */
4187         sysctl_ctx_init(&priv->sysctl_ctx);
4188         priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_STATIC_CHILDREN(_dev),
4189             OID_AUTO, ifp->if_dname, CTLFLAG_RD, 0, "MLX5 ethernet - interface name");
4190         if (priv->sysctl_ifnet == NULL) {
4191                 mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
4192                 goto err_free_sysctl;
4193         }
4194         snprintf(unit, sizeof(unit), "%d", ifp->if_dunit);
4195         priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4196             OID_AUTO, unit, CTLFLAG_RD, 0, "MLX5 ethernet - interface unit");
4197         if (priv->sysctl_ifnet == NULL) {
4198                 mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
4199                 goto err_free_sysctl;
4200         }
4201
4202         /* HW sysctl tree */
4203         child = SYSCTL_CHILDREN(device_get_sysctl_tree(mdev->pdev->dev.bsddev));
4204         priv->sysctl_hw = SYSCTL_ADD_NODE(&priv->sysctl_ctx, child,
4205             OID_AUTO, "hw", CTLFLAG_RD, 0, "MLX5 ethernet dev hw");
4206         if (priv->sysctl_hw == NULL) {
4207                 mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
4208                 goto err_free_sysctl;
4209         }
4210
4211         err = mlx5e_build_ifp_priv(mdev, priv, ncv);
4212         if (err) {
4213                 mlx5_core_err(mdev, "mlx5e_build_ifp_priv() failed (%d)\n", err);
4214                 goto err_free_sysctl;
4215         }
4216
4217         /* reuse mlx5core's watchdog workqueue */
4218         priv->wq = mdev->priv.health.wq_watchdog;
4219
4220         err = mlx5_alloc_map_uar(mdev, &priv->cq_uar);
4221         if (err) {
4222                 mlx5_en_err(ifp, "mlx5_alloc_map_uar failed, %d\n", err);
4223                 goto err_free_wq;
4224         }
4225         err = mlx5_core_alloc_pd(mdev, &priv->pdn);
4226         if (err) {
4227                 mlx5_en_err(ifp, "mlx5_core_alloc_pd failed, %d\n", err);
4228                 goto err_unmap_free_uar;
4229         }
4230         err = mlx5_alloc_transport_domain(mdev, &priv->tdn);
4231         if (err) {
4232                 mlx5_en_err(ifp,
4233                     "mlx5_alloc_transport_domain failed, %d\n", err);
4234                 goto err_dealloc_pd;
4235         }
4236         err = mlx5e_create_mkey(priv, priv->pdn, &priv->mr);
4237         if (err) {
4238                 mlx5_en_err(ifp, "mlx5e_create_mkey failed, %d\n", err);
4239                 goto err_dealloc_transport_domain;
4240         }
4241         mlx5_query_nic_vport_mac_address(priv->mdev, 0, dev_addr);
4242
4243         /* check if we should generate a random MAC address */
4244         if (MLX5_CAP_GEN(priv->mdev, vport_group_manager) == 0 &&
4245             is_zero_ether_addr(dev_addr)) {
4246                 random_ether_addr(dev_addr);
4247                 mlx5_en_err(ifp, "Assigned random MAC address\n");
4248         }
4249 #ifdef RATELIMIT
4250         err = mlx5e_rl_init(priv);
4251         if (err) {
4252                 mlx5_en_err(ifp, "mlx5e_rl_init failed, %d\n", err);
4253                 goto err_create_mkey;
4254         }
4255 #endif
4256
4257         /* set default MTU */
4258         mlx5e_set_dev_port_mtu(ifp, ifp->if_mtu);
4259
4260         /* Set default media status */
4261         priv->media_status_last = IFM_AVALID;
4262         priv->media_active_last = IFM_ETHER | IFM_AUTO |
4263             IFM_ETH_RXPAUSE | IFM_FDX;
4264
4265         /* setup default pauseframes configuration */
4266         mlx5e_setup_pauseframes(priv);
4267
4268         /* Setup supported medias */
4269         //TODO: If we failed to query ptys is it ok to proceed??
4270         if (!mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN, 1)) {
4271                 ext = MLX5_CAP_PCAM_FEATURE(mdev,
4272                     ptys_extended_ethernet);
4273                 eth_proto_cap = MLX5_GET_ETH_PROTO(ptys_reg, out, ext,
4274                     eth_proto_capability);
4275                 if (MLX5_CAP_PCAM_FEATURE(mdev, ptys_connector_type))
4276                         connector_type = MLX5_GET(ptys_reg, out,
4277                             connector_type);
4278         } else {
4279                 eth_proto_cap = 0;
4280                 mlx5_en_err(ifp, "Query port media capability failed, %d\n", err);
4281         }
4282
4283         ifmedia_init(&priv->media, IFM_IMASK | IFM_ETH_FMASK,
4284             mlx5e_media_change, mlx5e_media_status);
4285
4286         speeds_num = ext ? MLX5E_EXT_LINK_SPEEDS_NUMBER : MLX5E_LINK_SPEEDS_NUMBER;
4287         for (i = 0; i != speeds_num; i++) {
4288                 for (j = 0; j < MLX5E_LINK_MODES_NUMBER ; ++j) {
4289                         media_entry = ext ? mlx5e_ext_mode_table[i][j] :
4290                             mlx5e_mode_table[i][j];
4291                         if (media_entry.baudrate == 0)
4292                                 continue;
4293                         if (MLX5E_PROT_MASK(i) & eth_proto_cap) {
4294                                 ifmedia_add(&priv->media,
4295                                     media_entry.subtype |
4296                                     IFM_ETHER, 0, NULL);
4297                                 ifmedia_add(&priv->media,
4298                                     media_entry.subtype |
4299                                     IFM_ETHER | IFM_FDX |
4300                                     IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE, 0, NULL);
4301                         }
4302                 }
4303         }
4304
4305         ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO, 0, NULL);
4306         ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO | IFM_FDX |
4307             IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE, 0, NULL);
4308
4309         /* Set autoselect by default */
4310         ifmedia_set(&priv->media, IFM_ETHER | IFM_AUTO | IFM_FDX |
4311             IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE);
4312         ether_ifattach(ifp, dev_addr);
4313
4314         /* Register for VLAN events */
4315         priv->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
4316             mlx5e_vlan_rx_add_vid, priv, EVENTHANDLER_PRI_FIRST);
4317         priv->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
4318             mlx5e_vlan_rx_kill_vid, priv, EVENTHANDLER_PRI_FIRST);
4319
4320         /* Link is down by default */
4321         if_link_state_change(ifp, LINK_STATE_DOWN);
4322
4323         mlx5e_enable_async_events(priv);
4324
4325         mlx5e_add_hw_stats(priv);
4326
4327         mlx5e_create_stats(&priv->stats.vport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4328             "vstats", mlx5e_vport_stats_desc, MLX5E_VPORT_STATS_NUM,
4329             priv->stats.vport.arg);
4330
4331         mlx5e_create_stats(&priv->stats.pport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4332             "pstats", mlx5e_pport_stats_desc, MLX5E_PPORT_STATS_NUM,
4333             priv->stats.pport.arg);
4334
4335         mlx5e_create_ethtool(priv);
4336
4337         mtx_lock(&priv->async_events_mtx);
4338         mlx5e_update_stats(priv);
4339         mtx_unlock(&priv->async_events_mtx);
4340
4341         SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4342             OID_AUTO, "rx_clbr_done", CTLFLAG_RD,
4343             &priv->clbr_done, 0,
4344             "RX timestamps calibration state");
4345         callout_init(&priv->tstmp_clbr, CALLOUT_DIRECT);
4346         mlx5e_reset_calibration_callout(priv);
4347
4348         return (priv);
4349
4350 #ifdef RATELIMIT
4351 err_create_mkey:
4352         mlx5_core_destroy_mkey(priv->mdev, &priv->mr);
4353 #endif
4354 err_dealloc_transport_domain:
4355         mlx5_dealloc_transport_domain(mdev, priv->tdn);
4356
4357 err_dealloc_pd:
4358         mlx5_core_dealloc_pd(mdev, priv->pdn);
4359
4360 err_unmap_free_uar:
4361         mlx5_unmap_free_uar(mdev, &priv->cq_uar);
4362
4363 err_free_wq:
4364         flush_workqueue(priv->wq);
4365
4366 err_free_sysctl:
4367         sysctl_ctx_free(&priv->sysctl_ctx);
4368         if (priv->sysctl_debug)
4369                 sysctl_ctx_free(&priv->stats.port_stats_debug.ctx);
4370         mlx5e_priv_static_destroy(priv, mdev->priv.eq_table.num_comp_vectors);
4371         if_free(ifp);
4372
4373 err_free_priv:
4374         free(priv, M_MLX5EN);
4375         return (NULL);
4376 }
4377
4378 static void
4379 mlx5e_destroy_ifp(struct mlx5_core_dev *mdev, void *vpriv)
4380 {
4381         struct mlx5e_priv *priv = vpriv;
4382         struct ifnet *ifp = priv->ifp;
4383
4384         /* don't allow more IOCTLs */
4385         priv->gone = 1;
4386
4387         /* XXX wait a bit to allow IOCTL handlers to complete */
4388         pause("W", hz);
4389
4390 #ifdef RATELIMIT
4391         /*
4392          * The kernel can have reference(s) via the m_snd_tag's into
4393          * the ratelimit channels, and these must go away before
4394          * detaching:
4395          */
4396         while (READ_ONCE(priv->rl.stats.tx_active_connections) != 0) {
4397                 mlx5_en_err(priv->ifp,
4398                     "Waiting for all ratelimit connections to terminate\n");
4399                 pause("W", hz);
4400         }
4401 #endif
4402         /* stop watchdog timer */
4403         callout_drain(&priv->watchdog);
4404
4405         callout_drain(&priv->tstmp_clbr);
4406
4407         if (priv->vlan_attach != NULL)
4408                 EVENTHANDLER_DEREGISTER(vlan_config, priv->vlan_attach);
4409         if (priv->vlan_detach != NULL)
4410                 EVENTHANDLER_DEREGISTER(vlan_unconfig, priv->vlan_detach);
4411
4412         /* make sure device gets closed */
4413         PRIV_LOCK(priv);
4414         mlx5e_close_locked(ifp);
4415         PRIV_UNLOCK(priv);
4416
4417         /* wait for all unlimited send tags to go away */
4418         while (priv->channel_refs != 0) {
4419                 mlx5_en_err(priv->ifp,
4420                     "Waiting for all unlimited connections to terminate\n");
4421                 pause("W", hz);
4422         }
4423
4424         /* unregister device */
4425         ifmedia_removeall(&priv->media);
4426         ether_ifdetach(ifp);
4427
4428 #ifdef RATELIMIT
4429         mlx5e_rl_cleanup(priv);
4430 #endif
4431         /* destroy all remaining sysctl nodes */
4432         sysctl_ctx_free(&priv->stats.vport.ctx);
4433         sysctl_ctx_free(&priv->stats.pport.ctx);
4434         if (priv->sysctl_debug)
4435                 sysctl_ctx_free(&priv->stats.port_stats_debug.ctx);
4436         sysctl_ctx_free(&priv->sysctl_ctx);
4437
4438         mlx5_core_destroy_mkey(priv->mdev, &priv->mr);
4439         mlx5_dealloc_transport_domain(priv->mdev, priv->tdn);
4440         mlx5_core_dealloc_pd(priv->mdev, priv->pdn);
4441         mlx5_unmap_free_uar(priv->mdev, &priv->cq_uar);
4442         mlx5e_disable_async_events(priv);
4443         flush_workqueue(priv->wq);
4444         mlx5e_priv_static_destroy(priv, mdev->priv.eq_table.num_comp_vectors);
4445         if_free(ifp);
4446         free(priv, M_MLX5EN);
4447 }
4448
4449 static void *
4450 mlx5e_get_ifp(void *vpriv)
4451 {
4452         struct mlx5e_priv *priv = vpriv;
4453
4454         return (priv->ifp);
4455 }
4456
4457 static struct mlx5_interface mlx5e_interface = {
4458         .add = mlx5e_create_ifp,
4459         .remove = mlx5e_destroy_ifp,
4460         .event = mlx5e_async_event,
4461         .protocol = MLX5_INTERFACE_PROTOCOL_ETH,
4462         .get_dev = mlx5e_get_ifp,
4463 };
4464
4465 void
4466 mlx5e_init(void)
4467 {
4468         mlx5_register_interface(&mlx5e_interface);
4469 }
4470
4471 void
4472 mlx5e_cleanup(void)
4473 {
4474         mlx5_unregister_interface(&mlx5e_interface);
4475 }
4476
4477 static void
4478 mlx5e_show_version(void __unused *arg)
4479 {
4480
4481         printf("%s", mlx5e_version);
4482 }
4483 SYSINIT(mlx5e_show_version, SI_SUB_DRIVERS, SI_ORDER_ANY, mlx5e_show_version, NULL);
4484
4485 module_init_order(mlx5e_init, SI_ORDER_THIRD);
4486 module_exit_order(mlx5e_cleanup, SI_ORDER_THIRD);
4487
4488 #if (__FreeBSD_version >= 1100000)
4489 MODULE_DEPEND(mlx5en, linuxkpi, 1, 1, 1);
4490 #endif
4491 MODULE_DEPEND(mlx5en, mlx5, 1, 1, 1);
4492 MODULE_VERSION(mlx5en, 1);