sys/dev/mlx5/mlx5_en/mlx5_en_main.c

   1 /*-
   2  * Copyright (c) 2015-2018 Mellanox Technologies. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
  14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  16  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
  17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  23  * SUCH DAMAGE.
  24  *
  25  * $FreeBSD$
  26  */
  27
  28 #include "en.h"
  29
  30 #include <sys/sockio.h>
  31 #include <machine/atomic.h>
  32
  33 #ifndef ETH_DRIVER_VERSION
  34 #define ETH_DRIVER_VERSION      "3.5.0"
  35 #endif
  36 #define DRIVER_RELDATE  "November 2018"
  37
  38 static const char mlx5e_version[] = "mlx5en: Mellanox Ethernet driver "
  39         ETH_DRIVER_VERSION " (" DRIVER_RELDATE ")\n";
  40
  41 static int mlx5e_get_wqe_sz(struct mlx5e_priv *priv, u32 *wqe_sz, u32 *nsegs);
  42
  43 struct mlx5e_channel_param {
  44         struct mlx5e_rq_param rq;
  45         struct mlx5e_sq_param sq;
  46         struct mlx5e_cq_param rx_cq;
  47         struct mlx5e_cq_param tx_cq;
  48 };
  49
  50 struct media {
  51         u32     subtype;
  52         u64     baudrate;
  53 };
  54
  55 static const struct media mlx5e_mode_table[MLX5E_LINK_SPEEDS_NUMBER][MLX5E_LINK_MODES_NUMBER] = {
  56
  57         [MLX5E_1000BASE_CX_SGMII][MLX5E_SGMII] = {
  58                 .subtype = IFM_1000_CX_SGMII,
  59                 .baudrate = IF_Mbps(1000ULL),
  60         },
  61         [MLX5E_1000BASE_KX][MLX5E_KX] = {
  62                 .subtype = IFM_1000_KX,
  63                 .baudrate = IF_Mbps(1000ULL),
  64         },
  65         [MLX5E_10GBASE_CX4][MLX5E_CX4] = {
  66                 .subtype = IFM_10G_CX4,
  67                 .baudrate = IF_Gbps(10ULL),
  68         },
  69         [MLX5E_10GBASE_KX4][MLX5E_KX4] = {
  70                 .subtype = IFM_10G_KX4,
  71                 .baudrate = IF_Gbps(10ULL),
  72         },
  73         [MLX5E_10GBASE_KR][MLX5E_KR] = {
  74                 .subtype = IFM_10G_KR,
  75                 .baudrate = IF_Gbps(10ULL),
  76         },
  77         [MLX5E_20GBASE_KR2][MLX5E_KR2] = {
  78                 .subtype = IFM_20G_KR2,
  79                 .baudrate = IF_Gbps(20ULL),
  80         },
  81         [MLX5E_40GBASE_CR4][MLX5E_CR4] = {
  82                 .subtype = IFM_40G_CR4,
  83                 .baudrate = IF_Gbps(40ULL),
  84         },
  85         [MLX5E_40GBASE_KR4][MLX5E_KR4] = {
  86                 .subtype = IFM_40G_KR4,
  87                 .baudrate = IF_Gbps(40ULL),
  88         },
  89         [MLX5E_56GBASE_R4][MLX5E_R] = {
  90                 .subtype = IFM_56G_R4,
  91                 .baudrate = IF_Gbps(56ULL),
  92         },
  93         [MLX5E_10GBASE_CR][MLX5E_CR1] = {
  94                 .subtype = IFM_10G_CR1,
  95                 .baudrate = IF_Gbps(10ULL),
  96         },
  97         [MLX5E_10GBASE_SR][MLX5E_SR] = {
  98                 .subtype = IFM_10G_SR,
  99                 .baudrate = IF_Gbps(10ULL),
 100         },
 101         [MLX5E_10GBASE_ER_LR][MLX5E_ER] = {
 102                 .subtype = IFM_10G_ER,
 103                 .baudrate = IF_Gbps(10ULL),
 104         },
 105         [MLX5E_10GBASE_ER_LR][MLX5E_LR] = {
 106                 .subtype = IFM_10G_LR,
 107                 .baudrate = IF_Gbps(10ULL),
 108         },
 109         [MLX5E_40GBASE_SR4][MLX5E_SR4] = {
 110                 .subtype = IFM_40G_SR4,
 111                 .baudrate = IF_Gbps(40ULL),
 112         },
 113         [MLX5E_40GBASE_LR4_ER4][MLX5E_LR4] = {
 114                 .subtype = IFM_40G_LR4,
 115                 .baudrate = IF_Gbps(40ULL),
 116         },
 117         [MLX5E_40GBASE_LR4_ER4][MLX5E_ER4] = {
 118                 .subtype = IFM_40G_ER4,
 119                 .baudrate = IF_Gbps(40ULL),
 120         },
 121         [MLX5E_100GBASE_CR4][MLX5E_CR4] = {
 122                 .subtype = IFM_100G_CR4,
 123                 .baudrate = IF_Gbps(100ULL),
 124         },
 125         [MLX5E_100GBASE_SR4][MLX5E_SR4] = {
 126                 .subtype = IFM_100G_SR4,
 127                 .baudrate = IF_Gbps(100ULL),
 128         },
 129         [MLX5E_100GBASE_KR4][MLX5E_KR4] = {
 130                 .subtype = IFM_100G_KR4,
 131                 .baudrate = IF_Gbps(100ULL),
 132         },
 133         [MLX5E_100GBASE_LR4][MLX5E_LR4] = {
 134                 .subtype = IFM_100G_LR4,
 135                 .baudrate = IF_Gbps(100ULL),
 136         },
 137         [MLX5E_100BASE_TX][MLX5E_TX] = {
 138                 .subtype = IFM_100_TX,
 139                 .baudrate = IF_Mbps(100ULL),
 140         },
 141         [MLX5E_1000BASE_T][MLX5E_T] = {
 142                 .subtype = IFM_1000_T,
 143                 .baudrate = IF_Mbps(1000ULL),
 144         },
 145         [MLX5E_10GBASE_T][MLX5E_T] = {
 146                 .subtype = IFM_10G_T,
 147                 .baudrate = IF_Gbps(10ULL),
 148         },
 149         [MLX5E_25GBASE_CR][MLX5E_CR] = {
 150                 .subtype = IFM_25G_CR,
 151                 .baudrate = IF_Gbps(25ULL),
 152         },
 153         [MLX5E_25GBASE_KR][MLX5E_KR] = {
 154                 .subtype = IFM_25G_KR,
 155                 .baudrate = IF_Gbps(25ULL),
 156         },
 157         [MLX5E_25GBASE_SR][MLX5E_SR] = {
 158                 .subtype = IFM_25G_SR,
 159                 .baudrate = IF_Gbps(25ULL),
 160         },
 161         [MLX5E_50GBASE_CR2][MLX5E_CR2] = {
 162                 .subtype = IFM_50G_CR2,
 163                 .baudrate = IF_Gbps(50ULL),
 164         },
 165         [MLX5E_50GBASE_KR2][MLX5E_KR2] = {
 166                 .subtype = IFM_50G_KR2,
 167                 .baudrate = IF_Gbps(50ULL),
 168         },
 169 };
 170
 171 static const struct media mlx5e_ext_mode_table[MLX5E_EXT_LINK_SPEEDS_NUMBER][MLX5E_LINK_MODES_NUMBER] = {
 172         [MLX5E_SGMII_100M][MLX5E_SGMII] = {
 173                 .subtype = IFM_100_SGMII,
 174                 .baudrate = IF_Mbps(100),
 175         },
 176         [MLX5E_1000BASE_X_SGMII][MLX5E_KX] = {
 177                 .subtype = IFM_1000_KX,
 178                 .baudrate = IF_Mbps(1000),
 179         },
 180         [MLX5E_1000BASE_X_SGMII][MLX5E_CX_SGMII] = {
 181                 .subtype = IFM_1000_CX_SGMII,
 182                 .baudrate = IF_Mbps(1000),
 183         },
 184         [MLX5E_1000BASE_X_SGMII][MLX5E_CX] = {
 185                 .subtype = IFM_1000_CX,
 186                 .baudrate = IF_Mbps(1000),
 187         },
 188         [MLX5E_1000BASE_X_SGMII][MLX5E_LX] = {
 189                 .subtype = IFM_1000_LX,
 190                 .baudrate = IF_Mbps(1000),
 191         },
 192         [MLX5E_1000BASE_X_SGMII][MLX5E_SX] = {
 193                 .subtype = IFM_1000_SX,
 194                 .baudrate = IF_Mbps(1000),
 195         },
 196         [MLX5E_1000BASE_X_SGMII][MLX5E_T] = {
 197                 .subtype = IFM_1000_T,
 198                 .baudrate = IF_Mbps(1000),
 199         },
 200         [MLX5E_5GBASE_R][MLX5E_T] = {
 201                 .subtype = IFM_5000_T,
 202                 .baudrate = IF_Mbps(5000),
 203         },
 204         [MLX5E_5GBASE_R][MLX5E_KR] = {
 205                 .subtype = IFM_5000_KR,
 206                 .baudrate = IF_Mbps(5000),
 207         },
 208         [MLX5E_5GBASE_R][MLX5E_KR1] = {
 209                 .subtype = IFM_5000_KR1,
 210                 .baudrate = IF_Mbps(5000),
 211         },
 212         [MLX5E_5GBASE_R][MLX5E_KR_S] = {
 213                 .subtype = IFM_5000_KR_S,
 214                 .baudrate = IF_Mbps(5000),
 215         },
 216         [MLX5E_10GBASE_XFI_XAUI_1][MLX5E_ER] = {
 217                 .subtype = IFM_10G_ER,
 218                 .baudrate = IF_Gbps(10ULL),
 219         },
 220         [MLX5E_10GBASE_XFI_XAUI_1][MLX5E_KR] = {
 221                 .subtype = IFM_10G_KR,
 222                 .baudrate = IF_Gbps(10ULL),
 223         },
 224         [MLX5E_10GBASE_XFI_XAUI_1][MLX5E_LR] = {
 225                 .subtype = IFM_10G_LR,
 226                 .baudrate = IF_Gbps(10ULL),
 227         },
 228         [MLX5E_10GBASE_XFI_XAUI_1][MLX5E_SR] = {
 229                 .subtype = IFM_10G_SR,
 230                 .baudrate = IF_Gbps(10ULL),
 231         },
 232         [MLX5E_10GBASE_XFI_XAUI_1][MLX5E_T] = {
 233                 .subtype = IFM_10G_T,
 234                 .baudrate = IF_Gbps(10ULL),
 235         },
 236         [MLX5E_10GBASE_XFI_XAUI_1][MLX5E_AOC] = {
 237                 .subtype = IFM_10G_AOC,
 238                 .baudrate = IF_Gbps(10ULL),
 239         },
 240         [MLX5E_10GBASE_XFI_XAUI_1][MLX5E_CR1] = {
 241                 .subtype = IFM_10G_CR1,
 242                 .baudrate = IF_Gbps(10ULL),
 243         },
 244         [MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_CR4] = {
 245                 .subtype = IFM_40G_CR4,
 246                 .baudrate = IF_Gbps(40ULL),
 247         },
 248         [MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_KR4] = {
 249                 .subtype = IFM_40G_KR4,
 250                 .baudrate = IF_Gbps(40ULL),
 251         },
 252         [MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_LR4] = {
 253                 .subtype = IFM_40G_LR4,
 254                 .baudrate = IF_Gbps(40ULL),
 255         },
 256         [MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_SR4] = {
 257                 .subtype = IFM_40G_SR4,
 258                 .baudrate = IF_Gbps(40ULL),
 259         },
 260         [MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_ER4] = {
 261                 .subtype = IFM_40G_ER4,
 262                 .baudrate = IF_Gbps(40ULL),
 263         },
 264
 265         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CR] = {
 266                 .subtype = IFM_25G_CR,
 267                 .baudrate = IF_Gbps(25ULL),
 268         },
 269         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_KR] = {
 270                 .subtype = IFM_25G_KR,
 271                 .baudrate = IF_Gbps(25ULL),
 272         },
 273         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_SR] = {
 274                 .subtype = IFM_25G_SR,
 275                 .baudrate = IF_Gbps(25ULL),
 276         },
 277         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_ACC] = {
 278                 .subtype = IFM_25G_ACC,
 279                 .baudrate = IF_Gbps(25ULL),
 280         },
 281         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_AOC] = {
 282                 .subtype = IFM_25G_AOC,
 283                 .baudrate = IF_Gbps(25ULL),
 284         },
 285         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CR1] = {
 286                 .subtype = IFM_25G_CR1,
 287                 .baudrate = IF_Gbps(25ULL),
 288         },
 289         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CR_S] = {
 290                 .subtype = IFM_25G_CR_S,
 291                 .baudrate = IF_Gbps(25ULL),
 292         },
 293         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_KR1] = {
 294                 .subtype = IFM_5000_KR1,
 295                 .baudrate = IF_Gbps(25ULL),
 296         },
 297         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_KR_S] = {
 298                 .subtype = IFM_25G_KR_S,
 299                 .baudrate = IF_Gbps(25ULL),
 300         },
 301         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_LR] = {
 302                 .subtype = IFM_25G_LR,
 303                 .baudrate = IF_Gbps(25ULL),
 304         },
 305         [MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_T] = {
 306                 .subtype = IFM_25G_T,
 307                 .baudrate = IF_Gbps(25ULL),
 308         },
 309         [MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_CR2] = {
 310                 .subtype = IFM_50G_CR2,
 311                 .baudrate = IF_Gbps(50ULL),
 312         },
 313         [MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_KR2] = {
 314                 .subtype = IFM_50G_KR2,
 315                 .baudrate = IF_Gbps(50ULL),
 316         },
 317         [MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_SR2] = {
 318                 .subtype = IFM_50G_SR2,
 319                 .baudrate = IF_Gbps(50ULL),
 320         },
 321         [MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_LR2] = {
 322                 .subtype = IFM_50G_LR2,
 323                 .baudrate = IF_Gbps(50ULL),
 324         },
 325         [MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_LR] = {
 326                 .subtype = IFM_50G_LR,
 327                 .baudrate = IF_Gbps(50ULL),
 328         },
 329         [MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_SR] = {
 330                 .subtype = IFM_50G_SR,
 331                 .baudrate = IF_Gbps(50ULL),
 332         },
 333         [MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_CP] = {
 334                 .subtype = IFM_50G_CP,
 335                 .baudrate = IF_Gbps(50ULL),
 336         },
 337         [MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_FR] = {
 338                 .subtype = IFM_50G_FR,
 339                 .baudrate = IF_Gbps(50ULL),
 340         },
 341         [MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_KR_PAM4] = {
 342                 .subtype = IFM_50G_KR_PAM4,
 343                 .baudrate = IF_Gbps(50ULL),
 344         },
 345         [MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_CR4] = {
 346                 .subtype = IFM_100G_CR4,
 347                 .baudrate = IF_Gbps(100ULL),
 348         },
 349         [MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_KR4] = {
 350                 .subtype = IFM_100G_KR4,
 351                 .baudrate = IF_Gbps(100ULL),
 352         },
 353         [MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_LR4] = {
 354                 .subtype = IFM_100G_LR4,
 355                 .baudrate = IF_Gbps(100ULL),
 356         },
 357         [MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_SR4] = {
 358                 .subtype = IFM_100G_SR4,
 359                 .baudrate = IF_Gbps(100ULL),
 360         },
 361         [MLX5E_100GAUI_2_100GBASE_CR2_KR2][MLX5E_SR2] = {
 362                 .subtype = IFM_100G_SR2,
 363                 .baudrate = IF_Gbps(100ULL),
 364         },
 365         [MLX5E_100GAUI_2_100GBASE_CR2_KR2][MLX5E_CP2] = {
 366                 .subtype = IFM_100G_CP2,
 367                 .baudrate = IF_Gbps(100ULL),
 368         },
 369         [MLX5E_100GAUI_2_100GBASE_CR2_KR2][MLX5E_KR2_PAM4] = {
 370                 .subtype = IFM_100G_KR2_PAM4,
 371                 .baudrate = IF_Gbps(100ULL),
 372         },
 373         [MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_DR4] = {
 374                 .subtype = IFM_200G_DR4,
 375                 .baudrate = IF_Gbps(200ULL),
 376         },
 377         [MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_LR4] = {
 378                 .subtype = IFM_200G_LR4,
 379                 .baudrate = IF_Gbps(200ULL),
 380         },
 381         [MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_SR4] = {
 382                 .subtype = IFM_200G_SR4,
 383                 .baudrate = IF_Gbps(200ULL),
 384         },
 385         [MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_FR4] = {
 386                 .subtype = IFM_200G_FR4,
 387                 .baudrate = IF_Gbps(200ULL),
 388         },
 389         [MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_CR4_PAM4] = {
 390                 .subtype = IFM_200G_CR4_PAM4,
 391                 .baudrate = IF_Gbps(200ULL),
 392         },
 393         [MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_KR4_PAM4] = {
 394                 .subtype = IFM_200G_KR4_PAM4,
 395                 .baudrate = IF_Gbps(200ULL),
 396         },
 397 };
 398
 399 MALLOC_DEFINE(M_MLX5EN, "MLX5EN", "MLX5 Ethernet");
 400
 401 static void
 402 mlx5e_update_carrier(struct mlx5e_priv *priv)
 403 {
 404         struct mlx5_core_dev *mdev = priv->mdev;
 405         u32 out[MLX5_ST_SZ_DW(ptys_reg)];
 406         u32 eth_proto_oper;
 407         int error;
 408         u8 port_state;
 409         u8 is_er_type;
 410         u8 i, j;
 411         bool ext;
 412         struct media media_entry = {};
 413
 414         port_state = mlx5_query_vport_state(mdev,
 415             MLX5_QUERY_VPORT_STATE_IN_OP_MOD_VNIC_VPORT, 0);
 416
 417         if (port_state == VPORT_STATE_UP) {
 418                 priv->media_status_last |= IFM_ACTIVE;
 419         } else {
 420                 priv->media_status_last &= ~IFM_ACTIVE;
 421                 priv->media_active_last = IFM_ETHER;
 422                 if_link_state_change(priv->ifp, LINK_STATE_DOWN);
 423                 return;
 424         }
 425
 426         error = mlx5_query_port_ptys(mdev, out, sizeof(out),
 427             MLX5_PTYS_EN, 1);
 428         if (error) {
 429                 priv->media_active_last = IFM_ETHER;
 430                 priv->ifp->if_baudrate = 1;
 431                 if_printf(priv->ifp, "%s: query port ptys failed: "
 432                     "0x%x\n", __func__, error);
 433                 return;
 434         }
 435
 436         ext = MLX5_CAP_PCAM_FEATURE(mdev, ptys_extended_ethernet);
 437         eth_proto_oper = MLX5_GET_ETH_PROTO(ptys_reg, out, ext,
 438             eth_proto_oper);
 439
 440         i = ilog2(eth_proto_oper);
 441
 442         for (j = 0; j != MLX5E_LINK_MODES_NUMBER; j++) {
 443                 media_entry = ext ? mlx5e_ext_mode_table[i][j] :
 444                     mlx5e_mode_table[i][j];
 445                 if (media_entry.baudrate != 0)
 446                         break;
 447         }
 448
 449         if (media_entry.subtype == 0) {
 450                 if_printf(priv->ifp, "%s: Could not find operational "
 451                     "media subtype\n", __func__);
 452                 return;
 453         }
 454
 455         switch (media_entry.subtype) {
 456         case IFM_10G_ER:
 457                 error = mlx5_query_pddr_range_info(mdev, 1, &is_er_type);
 458                 if (error != 0) {
 459                         if_printf(priv->ifp, "%s: query port pddr failed: %d\n",
 460                                   __func__, error);
 461                 }
 462                 if (error != 0 || is_er_type == 0)
 463                         media_entry.subtype = IFM_10G_LR;
 464                 break;
 465         case IFM_40G_LR4:
 466                 error = mlx5_query_pddr_range_info(mdev, 1, &is_er_type);
 467                 if (error != 0) {
 468                         if_printf(priv->ifp, "%s: query port pddr failed: %d\n",
 469                                   __func__, error);
 470                 }
 471                 if (error == 0 && is_er_type != 0)
 472                         media_entry.subtype = IFM_40G_ER4;
 473                 break;
 474         }
 475         priv->media_active_last = media_entry.subtype | IFM_ETHER | IFM_FDX;
 476         priv->ifp->if_baudrate = media_entry.baudrate;
 477
 478         if_link_state_change(priv->ifp, LINK_STATE_UP);
 479 }
 480
 481 static void
 482 mlx5e_media_status(struct ifnet *dev, struct ifmediareq *ifmr)
 483 {
 484         struct mlx5e_priv *priv = dev->if_softc;
 485
 486         ifmr->ifm_status = priv->media_status_last;
 487         ifmr->ifm_active = priv->media_active_last |
 488             (priv->params.rx_pauseframe_control ? IFM_ETH_RXPAUSE : 0) |
 489             (priv->params.tx_pauseframe_control ? IFM_ETH_TXPAUSE : 0);
 490
 491 }
 492
 493 static u32
 494 mlx5e_find_link_mode(u32 subtype, bool ext)
 495 {
 496         u32 i;
 497         u32 j;
 498         u32 link_mode = 0;
 499         u32 speeds_num = 0;
 500         struct media media_entry = {};
 501
 502         switch (subtype) {
 503         case IFM_10G_LR:
 504                 subtype = IFM_10G_ER;
 505                 break;
 506         case IFM_40G_ER4:
 507                 subtype = IFM_40G_LR4;
 508                 break;
 509         }
 510
 511         speeds_num = ext ? MLX5E_EXT_LINK_SPEEDS_NUMBER :
 512             MLX5E_LINK_SPEEDS_NUMBER;
 513
 514         for (i = 0; i != speeds_num; i++) {
 515                 for (j = 0; j < MLX5E_LINK_MODES_NUMBER ; ++j) {
 516                         media_entry = ext ? mlx5e_ext_mode_table[i][j] :
 517                             mlx5e_mode_table[i][j];
 518                         if (media_entry.baudrate == 0)
 519                                 continue;
 520                         if (media_entry.subtype == subtype) {
 521                                 link_mode |= MLX5E_PROT_MASK(i);
 522                         }
 523                 }
 524         }
 525
 526         return (link_mode);
 527 }
 528
 529 static int
 530 mlx5e_set_port_pause_and_pfc(struct mlx5e_priv *priv)
 531 {
 532         return (mlx5_set_port_pause_and_pfc(priv->mdev, 1,
 533             priv->params.rx_pauseframe_control,
 534             priv->params.tx_pauseframe_control,
 535             priv->params.rx_priority_flow_control,
 536             priv->params.tx_priority_flow_control));
 537 }
 538
 539 static int
 540 mlx5e_set_port_pfc(struct mlx5e_priv *priv)
 541 {
 542         int error;
 543
 544         if (priv->gone != 0) {
 545                 error = -ENXIO;
 546         } else if (priv->params.rx_pauseframe_control ||
 547             priv->params.tx_pauseframe_control) {
 548                 if_printf(priv->ifp,
 549                     "Global pauseframes must be disabled before "
 550                     "enabling PFC.\n");
 551                 error = -EINVAL;
 552         } else {
 553                 error = mlx5e_set_port_pause_and_pfc(priv);
 554         }
 555         return (error);
 556 }
 557
 558 static int
 559 mlx5e_media_change(struct ifnet *dev)
 560 {
 561         struct mlx5e_priv *priv = dev->if_softc;
 562         struct mlx5_core_dev *mdev = priv->mdev;
 563         u32 eth_proto_cap;
 564         u32 link_mode;
 565         u32 out[MLX5_ST_SZ_DW(ptys_reg)];
 566         int was_opened;
 567         int locked;
 568         int error;
 569         bool ext;
 570
 571         locked = PRIV_LOCKED(priv);
 572         if (!locked)
 573                 PRIV_LOCK(priv);
 574
 575         if (IFM_TYPE(priv->media.ifm_media) != IFM_ETHER) {
 576                 error = EINVAL;
 577                 goto done;
 578         }
 579
 580         error = mlx5_query_port_ptys(mdev, out, sizeof(out),
 581             MLX5_PTYS_EN, 1);
 582         if (error != 0) {
 583                 if_printf(dev, "Query port media capability failed\n");
 584                 goto done;
 585         }
 586
 587         ext = MLX5_CAP_PCAM_FEATURE(mdev, ptys_extended_ethernet);
 588         link_mode = mlx5e_find_link_mode(IFM_SUBTYPE(priv->media.ifm_media), ext);
 589
 590         /* query supported capabilities */
 591         eth_proto_cap = MLX5_GET_ETH_PROTO(ptys_reg, out, ext,
 592             eth_proto_capability);
 593
 594         /* check for autoselect */
 595         if (IFM_SUBTYPE(priv->media.ifm_media) == IFM_AUTO) {
 596                 link_mode = eth_proto_cap;
 597                 if (link_mode == 0) {
 598                         if_printf(dev, "Port media capability is zero\n");
 599                         error = EINVAL;
 600                         goto done;
 601                 }
 602         } else {
 603                 link_mode = link_mode & eth_proto_cap;
 604                 if (link_mode == 0) {
 605                         if_printf(dev, "Not supported link mode requested\n");
 606                         error = EINVAL;
 607                         goto done;
 608                 }
 609         }
 610         if (priv->media.ifm_media & (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE)) {
 611                 /* check if PFC is enabled */
 612                 if (priv->params.rx_priority_flow_control ||
 613                     priv->params.tx_priority_flow_control) {
 614                         if_printf(dev, "PFC must be disabled before enabling global pauseframes.\n");
 615                         error = EINVAL;
 616                         goto done;
 617                 }
 618         }
 619         /* update pauseframe control bits */
 620         priv->params.rx_pauseframe_control =
 621             (priv->media.ifm_media & IFM_ETH_RXPAUSE) ? 1 : 0;
 622         priv->params.tx_pauseframe_control =
 623             (priv->media.ifm_media & IFM_ETH_TXPAUSE) ? 1 : 0;
 624
 625         /* check if device is opened */
 626         was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
 627
 628         /* reconfigure the hardware */
 629         mlx5_set_port_status(mdev, MLX5_PORT_DOWN);
 630         mlx5_set_port_proto(mdev, link_mode, MLX5_PTYS_EN, ext);
 631         error = -mlx5e_set_port_pause_and_pfc(priv);
 632         if (was_opened)
 633                 mlx5_set_port_status(mdev, MLX5_PORT_UP);
 634
 635 done:
 636         if (!locked)
 637                 PRIV_UNLOCK(priv);
 638         return (error);
 639 }
 640
 641 static void
 642 mlx5e_update_carrier_work(struct work_struct *work)
 643 {
 644         struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
 645             update_carrier_work);
 646
 647         PRIV_LOCK(priv);
 648         if (test_bit(MLX5E_STATE_OPENED, &priv->state))
 649                 mlx5e_update_carrier(priv);
 650         PRIV_UNLOCK(priv);
 651 }
 652
 653 #define MLX5E_PCIE_PERF_GET_64(a,b,c,d,e,f)    \
 654         s_debug->c = MLX5_GET64(mpcnt_reg, out, counter_set.f.c);
 655
 656 #define MLX5E_PCIE_PERF_GET_32(a,b,c,d,e,f)    \
 657         s_debug->c = MLX5_GET(mpcnt_reg, out, counter_set.f.c);
 658
 659 static void
 660 mlx5e_update_pcie_counters(struct mlx5e_priv *priv)
 661 {
 662         struct mlx5_core_dev *mdev = priv->mdev;
 663         struct mlx5e_port_stats_debug *s_debug = &priv->stats.port_stats_debug;
 664         const unsigned sz = MLX5_ST_SZ_BYTES(mpcnt_reg);
 665         void *out;
 666         void *in;
 667         int err;
 668
 669         /* allocate firmware request structures */
 670         in = mlx5_vzalloc(sz);
 671         out = mlx5_vzalloc(sz);
 672         if (in == NULL || out == NULL)
 673                 goto free_out;
 674
 675         MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_PERFORMANCE_COUNTERS_GROUP);
 676         err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
 677         if (err != 0)
 678                 goto free_out;
 679
 680         MLX5E_PCIE_PERFORMANCE_COUNTERS_64(MLX5E_PCIE_PERF_GET_64)
 681         MLX5E_PCIE_PERFORMANCE_COUNTERS_32(MLX5E_PCIE_PERF_GET_32)
 682
 683         MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_TIMERS_AND_STATES_COUNTERS_GROUP);
 684         err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
 685         if (err != 0)
 686                 goto free_out;
 687
 688         MLX5E_PCIE_TIMERS_AND_STATES_COUNTERS_32(MLX5E_PCIE_PERF_GET_32)
 689
 690         MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_LANE_COUNTERS_GROUP);
 691         err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
 692         if (err != 0)
 693                 goto free_out;
 694
 695         MLX5E_PCIE_LANE_COUNTERS_32(MLX5E_PCIE_PERF_GET_32)
 696
 697 free_out:
 698         /* free firmware request structures */
 699         kvfree(in);
 700         kvfree(out);
 701 }
 702
 703 /*
 704  * This function reads the physical port counters from the firmware
 705  * using a pre-defined layout defined by various MLX5E_PPORT_XXX()
 706  * macros. The output is converted from big-endian 64-bit values into
 707  * host endian ones and stored in the "priv->stats.pport" structure.
 708  */
 709 static void
 710 mlx5e_update_pport_counters(struct mlx5e_priv *priv)
 711 {
 712         struct mlx5_core_dev *mdev = priv->mdev;
 713         struct mlx5e_pport_stats *s = &priv->stats.pport;
 714         struct mlx5e_port_stats_debug *s_debug = &priv->stats.port_stats_debug;
 715         u32 *in;
 716         u32 *out;
 717         const u64 *ptr;
 718         unsigned sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
 719         unsigned x;
 720         unsigned y;
 721         unsigned z;
 722
 723         /* allocate firmware request structures */
 724         in = mlx5_vzalloc(sz);
 725         out = mlx5_vzalloc(sz);
 726         if (in == NULL || out == NULL)
 727                 goto free_out;
 728
 729         /*
 730          * Get pointer to the 64-bit counter set which is located at a
 731          * fixed offset in the output firmware request structure:
 732          */
 733         ptr = (const uint64_t *)MLX5_ADDR_OF(ppcnt_reg, out, counter_set);
 734
 735         MLX5_SET(ppcnt_reg, in, local_port, 1);
 736
 737         /* read IEEE802_3 counter group using predefined counter layout */
 738         MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
 739         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 740         for (x = 0, y = MLX5E_PPORT_PER_PRIO_STATS_NUM;
 741              x != MLX5E_PPORT_IEEE802_3_STATS_NUM; x++, y++)
 742                 s->arg[y] = be64toh(ptr[x]);
 743
 744         /* read RFC2819 counter group using predefined counter layout */
 745         MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2819_COUNTERS_GROUP);
 746         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 747         for (x = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM; x++, y++)
 748                 s->arg[y] = be64toh(ptr[x]);
 749
 750         for (y = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM +
 751             MLX5E_PPORT_RFC2819_STATS_DEBUG_NUM; x++, y++)
 752                 s_debug->arg[y] = be64toh(ptr[x]);
 753
 754         /* read RFC2863 counter group using predefined counter layout */
 755         MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2863_COUNTERS_GROUP);
 756         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 757         for (x = 0; x != MLX5E_PPORT_RFC2863_STATS_DEBUG_NUM; x++, y++)
 758                 s_debug->arg[y] = be64toh(ptr[x]);
 759
 760         /* read physical layer stats counter group using predefined counter layout */
 761         MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP);
 762         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 763         for (x = 0; x != MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG_NUM; x++, y++)
 764                 s_debug->arg[y] = be64toh(ptr[x]);
 765
 766         /* read Extended Ethernet counter group using predefined counter layout */
 767         MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);
 768         mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 769         for (x = 0; x != MLX5E_PPORT_ETHERNET_EXTENDED_STATS_DEBUG_NUM; x++, y++)
 770                 s_debug->arg[y] = be64toh(ptr[x]);
 771
 772         /* read PCIE counters */
 773         mlx5e_update_pcie_counters(priv);
 774
 775         /* read per-priority counters */
 776         MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_PRIORITY_COUNTERS_GROUP);
 777
 778         /* iterate all the priorities */
 779         for (y = z = 0; z != MLX5E_PPORT_PER_PRIO_STATS_NUM_PRIO; z++) {
 780                 MLX5_SET(ppcnt_reg, in, prio_tc, z);
 781                 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 782
 783                 /* read per priority stats counter group using predefined counter layout */
 784                 for (x = 0; x != (MLX5E_PPORT_PER_PRIO_STATS_NUM /
 785                     MLX5E_PPORT_PER_PRIO_STATS_NUM_PRIO); x++, y++)
 786                         s->arg[y] = be64toh(ptr[x]);
 787         }
 788
 789 free_out:
 790         /* free firmware request structures */
 791         kvfree(in);
 792         kvfree(out);
 793 }
 794
 795 static void
 796 mlx5e_grp_vnic_env_update_stats(struct mlx5e_priv *priv)
 797 {
 798         u32 out[MLX5_ST_SZ_DW(query_vnic_env_out)] = {};
 799         u32 in[MLX5_ST_SZ_DW(query_vnic_env_in)] = {};
 800
 801         if (!MLX5_CAP_GEN(priv->mdev, nic_receive_steering_discard))
 802                 return;
 803
 804         MLX5_SET(query_vnic_env_in, in, opcode,
 805             MLX5_CMD_OP_QUERY_VNIC_ENV);
 806         MLX5_SET(query_vnic_env_in, in, op_mod, 0);
 807         MLX5_SET(query_vnic_env_in, in, other_vport, 0);
 808
 809         if (mlx5_cmd_exec(priv->mdev, in, sizeof(in), out, sizeof(out)) != 0)
 810                 return;
 811
 812         priv->stats.vport.rx_steer_missed_packets =
 813             MLX5_GET64(query_vnic_env_out, out,
 814             vport_env.nic_receive_steering_discard);
 815 }
 816
 817 /*
 818  * This function is called regularly to collect all statistics
 819  * counters from the firmware. The values can be viewed through the
 820  * sysctl interface. Execution is serialized using the priv's global
 821  * configuration lock.
 822  */
 823 static void
 824 mlx5e_update_stats_locked(struct mlx5e_priv *priv)
 825 {
 826         struct mlx5_core_dev *mdev = priv->mdev;
 827         struct mlx5e_vport_stats *s = &priv->stats.vport;
 828         struct mlx5e_sq_stats *sq_stats;
 829         struct buf_ring *sq_br;
 830 #if (__FreeBSD_version < 1100000)
 831         struct ifnet *ifp = priv->ifp;
 832 #endif
 833
 834         u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)];
 835         u32 *out;
 836         int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
 837         u64 tso_packets = 0;
 838         u64 tso_bytes = 0;
 839         u64 tx_queue_dropped = 0;
 840         u64 tx_defragged = 0;
 841         u64 tx_offload_none = 0;
 842         u64 lro_packets = 0;
 843         u64 lro_bytes = 0;
 844         u64 sw_lro_queued = 0;
 845         u64 sw_lro_flushed = 0;
 846         u64 rx_csum_none = 0;
 847         u64 rx_wqe_err = 0;
 848         u64 rx_packets = 0;
 849         u64 rx_bytes = 0;
 850         u32 rx_out_of_buffer = 0;
 851         int i;
 852         int j;
 853
 854         out = mlx5_vzalloc(outlen);
 855         if (out == NULL)
 856                 goto free_out;
 857
 858         /* Collect firts the SW counters and then HW for consistency */
 859         for (i = 0; i < priv->params.num_channels; i++) {
 860                 struct mlx5e_channel *pch = priv->channel + i;
 861                 struct mlx5e_rq *rq = &pch->rq;
 862                 struct mlx5e_rq_stats *rq_stats = &pch->rq.stats;
 863
 864                 /* collect stats from LRO */
 865                 rq_stats->sw_lro_queued = rq->lro.lro_queued;
 866                 rq_stats->sw_lro_flushed = rq->lro.lro_flushed;
 867                 sw_lro_queued += rq_stats->sw_lro_queued;
 868                 sw_lro_flushed += rq_stats->sw_lro_flushed;
 869                 lro_packets += rq_stats->lro_packets;
 870                 lro_bytes += rq_stats->lro_bytes;
 871                 rx_csum_none += rq_stats->csum_none;
 872                 rx_wqe_err += rq_stats->wqe_err;
 873                 rx_packets += rq_stats->packets;
 874                 rx_bytes += rq_stats->bytes;
 875
 876                 for (j = 0; j < priv->num_tc; j++) {
 877                         sq_stats = &pch->sq[j].stats;
 878                         sq_br = pch->sq[j].br;
 879
 880                         tso_packets += sq_stats->tso_packets;
 881                         tso_bytes += sq_stats->tso_bytes;
 882                         tx_queue_dropped += sq_stats->dropped;
 883                         if (sq_br != NULL)
 884                                 tx_queue_dropped += sq_br->br_drops;
 885                         tx_defragged += sq_stats->defragged;
 886                         tx_offload_none += sq_stats->csum_offload_none;
 887                 }
 888         }
 889
 890         /* update counters */
 891         s->tso_packets = tso_packets;
 892         s->tso_bytes = tso_bytes;
 893         s->tx_queue_dropped = tx_queue_dropped;
 894         s->tx_defragged = tx_defragged;
 895         s->lro_packets = lro_packets;
 896         s->lro_bytes = lro_bytes;
 897         s->sw_lro_queued = sw_lro_queued;
 898         s->sw_lro_flushed = sw_lro_flushed;
 899         s->rx_csum_none = rx_csum_none;
 900         s->rx_wqe_err = rx_wqe_err;
 901         s->rx_packets = rx_packets;
 902         s->rx_bytes = rx_bytes;
 903
 904         mlx5e_grp_vnic_env_update_stats(priv);
 905
 906         /* HW counters */
 907         memset(in, 0, sizeof(in));
 908
 909         MLX5_SET(query_vport_counter_in, in, opcode,
 910             MLX5_CMD_OP_QUERY_VPORT_COUNTER);
 911         MLX5_SET(query_vport_counter_in, in, op_mod, 0);
 912         MLX5_SET(query_vport_counter_in, in, other_vport, 0);
 913
 914         memset(out, 0, outlen);
 915
 916         /* get number of out-of-buffer drops first */
 917         if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0 &&
 918             mlx5_vport_query_out_of_rx_buffer(mdev, priv->counter_set_id,
 919             &rx_out_of_buffer) == 0) {
 920                 s->rx_out_of_buffer = rx_out_of_buffer;
 921         }
 922
 923         /* get port statistics */
 924         if (mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen) == 0) {
 925 #define MLX5_GET_CTR(out, x) \
 926         MLX5_GET64(query_vport_counter_out, out, x)
 927
 928                 s->rx_error_packets =
 929                     MLX5_GET_CTR(out, received_errors.packets);
 930                 s->rx_error_bytes =
 931                     MLX5_GET_CTR(out, received_errors.octets);
 932                 s->tx_error_packets =
 933                     MLX5_GET_CTR(out, transmit_errors.packets);
 934                 s->tx_error_bytes =
 935                     MLX5_GET_CTR(out, transmit_errors.octets);
 936
 937                 s->rx_unicast_packets =
 938                     MLX5_GET_CTR(out, received_eth_unicast.packets);
 939                 s->rx_unicast_bytes =
 940                     MLX5_GET_CTR(out, received_eth_unicast.octets);
 941                 s->tx_unicast_packets =
 942                     MLX5_GET_CTR(out, transmitted_eth_unicast.packets);
 943                 s->tx_unicast_bytes =
 944                     MLX5_GET_CTR(out, transmitted_eth_unicast.octets);
 945
 946                 s->rx_multicast_packets =
 947                     MLX5_GET_CTR(out, received_eth_multicast.packets);
 948                 s->rx_multicast_bytes =
 949                     MLX5_GET_CTR(out, received_eth_multicast.octets);
 950                 s->tx_multicast_packets =
 951                     MLX5_GET_CTR(out, transmitted_eth_multicast.packets);
 952                 s->tx_multicast_bytes =
 953                     MLX5_GET_CTR(out, transmitted_eth_multicast.octets);
 954
 955                 s->rx_broadcast_packets =
 956                     MLX5_GET_CTR(out, received_eth_broadcast.packets);
 957                 s->rx_broadcast_bytes =
 958                     MLX5_GET_CTR(out, received_eth_broadcast.octets);
 959                 s->tx_broadcast_packets =
 960                     MLX5_GET_CTR(out, transmitted_eth_broadcast.packets);
 961                 s->tx_broadcast_bytes =
 962                     MLX5_GET_CTR(out, transmitted_eth_broadcast.octets);
 963
 964                 s->tx_packets = s->tx_unicast_packets +
 965                     s->tx_multicast_packets + s->tx_broadcast_packets;
 966                 s->tx_bytes = s->tx_unicast_bytes + s->tx_multicast_bytes +
 967                     s->tx_broadcast_bytes;
 968
 969                 /* Update calculated offload counters */
 970                 s->tx_csum_offload = s->tx_packets - tx_offload_none;
 971                 s->rx_csum_good = s->rx_packets - s->rx_csum_none;
 972         }
 973
 974         /* Get physical port counters */
 975         mlx5e_update_pport_counters(priv);
 976
 977         s->tx_jumbo_packets =
 978             priv->stats.port_stats_debug.tx_stat_p1519to2047octets +
 979             priv->stats.port_stats_debug.tx_stat_p2048to4095octets +
 980             priv->stats.port_stats_debug.tx_stat_p4096to8191octets +
 981             priv->stats.port_stats_debug.tx_stat_p8192to10239octets;
 982
 983 #if (__FreeBSD_version < 1100000)
 984         /* no get_counters interface in fbsd 10 */
 985         ifp->if_ipackets = s->rx_packets;
 986         ifp->if_ierrors = priv->stats.pport.in_range_len_errors +
 987             priv->stats.pport.out_of_range_len +
 988             priv->stats.pport.too_long_errors +
 989             priv->stats.pport.check_seq_err +
 990             priv->stats.pport.alignment_err;
 991         ifp->if_iqdrops = s->rx_out_of_buffer;
 992         ifp->if_opackets = s->tx_packets;
 993         ifp->if_oerrors = priv->stats.port_stats_debug.out_discards;
 994         ifp->if_snd.ifq_drops = s->tx_queue_dropped;
 995         ifp->if_ibytes = s->rx_bytes;
 996         ifp->if_obytes = s->tx_bytes;
 997         ifp->if_collisions =
 998             priv->stats.pport.collisions;
 999 #endif
1000
1001 free_out:
1002         kvfree(out);
1003
1004         /* Update diagnostics, if any */
1005         if (priv->params_ethtool.diag_pci_enable ||
1006             priv->params_ethtool.diag_general_enable) {
1007                 int error = mlx5_core_get_diagnostics_full(mdev,
1008                     priv->params_ethtool.diag_pci_enable ? &priv->params_pci : NULL,
1009                     priv->params_ethtool.diag_general_enable ? &priv->params_general : NULL);
1010                 if (error != 0)
1011                         if_printf(priv->ifp, "Failed reading diagnostics: %d\n", error);
1012         }
1013 }
1014
1015 static void
1016 mlx5e_update_stats_work(struct work_struct *work)
1017 {
1018         struct mlx5e_priv *priv;
1019
1020         priv  = container_of(work, struct mlx5e_priv, update_stats_work);
1021         PRIV_LOCK(priv);
1022         if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
1023                 mlx5e_update_stats_locked(priv);
1024         PRIV_UNLOCK(priv);
1025 }
1026
1027 static void
1028 mlx5e_update_stats(void *arg)
1029 {
1030         struct mlx5e_priv *priv = arg;
1031
1032         queue_work(priv->wq, &priv->update_stats_work);
1033
1034         callout_reset(&priv->watchdog, hz, &mlx5e_update_stats, priv);
1035 }
1036
1037 static void
1038 mlx5e_async_event_sub(struct mlx5e_priv *priv,
1039     enum mlx5_dev_event event)
1040 {
1041         switch (event) {
1042         case MLX5_DEV_EVENT_PORT_UP:
1043         case MLX5_DEV_EVENT_PORT_DOWN:
1044                 queue_work(priv->wq, &priv->update_carrier_work);
1045                 break;
1046
1047         default:
1048                 break;
1049         }
1050 }
1051
1052 static void
1053 mlx5e_async_event(struct mlx5_core_dev *mdev, void *vpriv,
1054     enum mlx5_dev_event event, unsigned long param)
1055 {
1056         struct mlx5e_priv *priv = vpriv;
1057
1058         mtx_lock(&priv->async_events_mtx);
1059         if (test_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state))
1060                 mlx5e_async_event_sub(priv, event);
1061         mtx_unlock(&priv->async_events_mtx);
1062 }
1063
1064 static void
1065 mlx5e_enable_async_events(struct mlx5e_priv *priv)
1066 {
1067         set_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state);
1068 }
1069
1070 static void
1071 mlx5e_disable_async_events(struct mlx5e_priv *priv)
1072 {
1073         mtx_lock(&priv->async_events_mtx);
1074         clear_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state);
1075         mtx_unlock(&priv->async_events_mtx);
1076 }
1077
1078 static const char *mlx5e_rq_stats_desc[] = {
1079         MLX5E_RQ_STATS(MLX5E_STATS_DESC)
1080 };
1081
1082 static int
1083 mlx5e_create_rq(struct mlx5e_channel *c,
1084     struct mlx5e_rq_param *param,
1085     struct mlx5e_rq *rq)
1086 {
1087         struct mlx5e_priv *priv = c->priv;
1088         struct mlx5_core_dev *mdev = priv->mdev;
1089         char buffer[16];
1090         void *rqc = param->rqc;
1091         void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
1092         int wq_sz;
1093         int err;
1094         int i;
1095         u32 nsegs, wqe_sz;
1096
1097         err = mlx5e_get_wqe_sz(priv, &wqe_sz, &nsegs);
1098         if (err != 0)
1099                 goto done;
1100
1101         /* Create DMA descriptor TAG */
1102         if ((err = -bus_dma_tag_create(
1103             bus_get_dma_tag(mdev->pdev->dev.bsddev),
1104             1,                          /* any alignment */
1105             0,                          /* no boundary */
1106             BUS_SPACE_MAXADDR,          /* lowaddr */
1107             BUS_SPACE_MAXADDR,          /* highaddr */
1108             NULL, NULL,                 /* filter, filterarg */
1109             nsegs * MLX5E_MAX_RX_BYTES, /* maxsize */
1110             nsegs,                      /* nsegments */
1111             nsegs * MLX5E_MAX_RX_BYTES, /* maxsegsize */
1112             0,                          /* flags */
1113             NULL, NULL,                 /* lockfunc, lockfuncarg */
1114             &rq->dma_tag)))
1115                 goto done;
1116
1117         err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
1118             &rq->wq_ctrl);
1119         if (err)
1120                 goto err_free_dma_tag;
1121
1122         rq->wq.db = &rq->wq.db[MLX5_RCV_DBR];
1123
1124         err = mlx5e_get_wqe_sz(priv, &rq->wqe_sz, &rq->nsegs);
1125         if (err != 0)
1126                 goto err_rq_wq_destroy;
1127
1128         wq_sz = mlx5_wq_ll_get_size(&rq->wq);
1129
1130         err = -tcp_lro_init_args(&rq->lro, c->ifp, TCP_LRO_ENTRIES, wq_sz);
1131         if (err)
1132                 goto err_rq_wq_destroy;
1133
1134         rq->mbuf = malloc(wq_sz * sizeof(rq->mbuf[0]), M_MLX5EN, M_WAITOK | M_ZERO);
1135         for (i = 0; i != wq_sz; i++) {
1136                 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
1137 #if (MLX5E_MAX_RX_SEGS == 1)
1138                 uint32_t byte_count = rq->wqe_sz - MLX5E_NET_IP_ALIGN;
1139 #else
1140                 int j;
1141 #endif
1142
1143                 err = -bus_dmamap_create(rq->dma_tag, 0, &rq->mbuf[i].dma_map);
1144                 if (err != 0) {
1145                         while (i--)
1146                                 bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map);
1147                         goto err_rq_mbuf_free;
1148                 }
1149
1150                 /* set value for constant fields */
1151 #if (MLX5E_MAX_RX_SEGS == 1)
1152                 wqe->data[0].lkey = c->mkey_be;
1153                 wqe->data[0].byte_count = cpu_to_be32(byte_count | MLX5_HW_START_PADDING);
1154 #else
1155                 for (j = 0; j < rq->nsegs; j++)
1156                         wqe->data[j].lkey = c->mkey_be;
1157 #endif
1158         }
1159
1160         INIT_WORK(&rq->dim.work, mlx5e_dim_work);
1161         if (priv->params.rx_cq_moderation_mode < 2) {
1162                 rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
1163         } else {
1164                 void *cqc = container_of(param,
1165                     struct mlx5e_channel_param, rq)->rx_cq.cqc;
1166
1167                 switch (MLX5_GET(cqc, cqc, cq_period_mode)) {
1168                 case MLX5_CQ_PERIOD_MODE_START_FROM_EQE:
1169                         rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
1170                         break;
1171                 case MLX5_CQ_PERIOD_MODE_START_FROM_CQE:
1172                         rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE;
1173                         break;
1174                 default:
1175                         rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
1176                         break;
1177                 }
1178         }
1179
1180         rq->ifp = c->ifp;
1181         rq->channel = c;
1182         rq->ix = c->ix;
1183
1184         snprintf(buffer, sizeof(buffer), "rxstat%d", c->ix);
1185         mlx5e_create_stats(&rq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
1186             buffer, mlx5e_rq_stats_desc, MLX5E_RQ_STATS_NUM,
1187             rq->stats.arg);
1188         return (0);
1189
1190 err_rq_mbuf_free:
1191         free(rq->mbuf, M_MLX5EN);
1192         tcp_lro_free(&rq->lro);
1193 err_rq_wq_destroy:
1194         mlx5_wq_destroy(&rq->wq_ctrl);
1195 err_free_dma_tag:
1196         bus_dma_tag_destroy(rq->dma_tag);
1197 done:
1198         return (err);
1199 }
1200
1201 static void
1202 mlx5e_destroy_rq(struct mlx5e_rq *rq)
1203 {
1204         int wq_sz;
1205         int i;
1206
1207         /* destroy all sysctl nodes */
1208         sysctl_ctx_free(&rq->stats.ctx);
1209
1210         /* free leftover LRO packets, if any */
1211         tcp_lro_free(&rq->lro);
1212
1213         wq_sz = mlx5_wq_ll_get_size(&rq->wq);
1214         for (i = 0; i != wq_sz; i++) {
1215                 if (rq->mbuf[i].mbuf != NULL) {
1216                         bus_dmamap_unload(rq->dma_tag, rq->mbuf[i].dma_map);
1217                         m_freem(rq->mbuf[i].mbuf);
1218                 }
1219                 bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map);
1220         }
1221         free(rq->mbuf, M_MLX5EN);
1222         mlx5_wq_destroy(&rq->wq_ctrl);
1223 }
1224
1225 static int
1226 mlx5e_enable_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
1227 {
1228         struct mlx5e_channel *c = rq->channel;
1229         struct mlx5e_priv *priv = c->priv;
1230         struct mlx5_core_dev *mdev = priv->mdev;
1231
1232         void *in;
1233         void *rqc;
1234         void *wq;
1235         int inlen;
1236         int err;
1237
1238         inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
1239             sizeof(u64) * rq->wq_ctrl.buf.npages;
1240         in = mlx5_vzalloc(inlen);
1241         if (in == NULL)
1242                 return (-ENOMEM);
1243
1244         rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
1245         wq = MLX5_ADDR_OF(rqc, rqc, wq);
1246
1247         memcpy(rqc, param->rqc, sizeof(param->rqc));
1248
1249         MLX5_SET(rqc, rqc, cqn, c->rq.cq.mcq.cqn);
1250         MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
1251         MLX5_SET(rqc, rqc, flush_in_error_en, 1);
1252         if (priv->counter_set_id >= 0)
1253                 MLX5_SET(rqc, rqc, counter_set_id, priv->counter_set_id);
1254         MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift -
1255             PAGE_SHIFT);
1256         MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma);
1257
1258         mlx5_fill_page_array(&rq->wq_ctrl.buf,
1259             (__be64 *) MLX5_ADDR_OF(wq, wq, pas));
1260
1261         err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
1262
1263         kvfree(in);
1264
1265         return (err);
1266 }
1267
1268 static int
1269 mlx5e_modify_rq(struct mlx5e_rq *rq, int curr_state, int next_state)
1270 {
1271         struct mlx5e_channel *c = rq->channel;
1272         struct mlx5e_priv *priv = c->priv;
1273         struct mlx5_core_dev *mdev = priv->mdev;
1274
1275         void *in;
1276         void *rqc;
1277         int inlen;
1278         int err;
1279
1280         inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1281         in = mlx5_vzalloc(inlen);
1282         if (in == NULL)
1283                 return (-ENOMEM);
1284
1285         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
1286
1287         MLX5_SET(modify_rq_in, in, rqn, rq->rqn);
1288         MLX5_SET(modify_rq_in, in, rq_state, curr_state);
1289         MLX5_SET(rqc, rqc, state, next_state);
1290
1291         err = mlx5_core_modify_rq(mdev, in, inlen);
1292
1293         kvfree(in);
1294
1295         return (err);
1296 }
1297
1298 static void
1299 mlx5e_disable_rq(struct mlx5e_rq *rq)
1300 {
1301         struct mlx5e_channel *c = rq->channel;
1302         struct mlx5e_priv *priv = c->priv;
1303         struct mlx5_core_dev *mdev = priv->mdev;
1304
1305         mlx5_core_destroy_rq(mdev, rq->rqn);
1306 }
1307
1308 static int
1309 mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq)
1310 {
1311         struct mlx5e_channel *c = rq->channel;
1312         struct mlx5e_priv *priv = c->priv;
1313         struct mlx5_wq_ll *wq = &rq->wq;
1314         int i;
1315
1316         for (i = 0; i < 1000; i++) {
1317                 if (wq->cur_sz >= priv->params.min_rx_wqes)
1318                         return (0);
1319
1320                 msleep(4);
1321         }
1322         return (-ETIMEDOUT);
1323 }
1324
1325 static int
1326 mlx5e_open_rq(struct mlx5e_channel *c,
1327     struct mlx5e_rq_param *param,
1328     struct mlx5e_rq *rq)
1329 {
1330         int err;
1331
1332         err = mlx5e_create_rq(c, param, rq);
1333         if (err)
1334                 return (err);
1335
1336         err = mlx5e_enable_rq(rq, param);
1337         if (err)
1338                 goto err_destroy_rq;
1339
1340         err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
1341         if (err)
1342                 goto err_disable_rq;
1343
1344         c->rq.enabled = 1;
1345
1346         return (0);
1347
1348 err_disable_rq:
1349         mlx5e_disable_rq(rq);
1350 err_destroy_rq:
1351         mlx5e_destroy_rq(rq);
1352
1353         return (err);
1354 }
1355
1356 static void
1357 mlx5e_close_rq(struct mlx5e_rq *rq)
1358 {
1359         mtx_lock(&rq->mtx);
1360         rq->enabled = 0;
1361         callout_stop(&rq->watchdog);
1362         mtx_unlock(&rq->mtx);
1363
1364         callout_drain(&rq->watchdog);
1365
1366         mlx5e_modify_rq(rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
1367 }
1368
1369 static void
1370 mlx5e_close_rq_wait(struct mlx5e_rq *rq)
1371 {
1372
1373         mlx5e_disable_rq(rq);
1374         mlx5e_close_cq(&rq->cq);
1375         cancel_work_sync(&rq->dim.work);
1376         mlx5e_destroy_rq(rq);
1377 }
1378
1379 void
1380 mlx5e_free_sq_db(struct mlx5e_sq *sq)
1381 {
1382         int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1383         int x;
1384
1385         for (x = 0; x != wq_sz; x++) {
1386                 if (sq->mbuf[x].mbuf != NULL) {
1387                         bus_dmamap_unload(sq->dma_tag, sq->mbuf[x].dma_map);
1388                         m_freem(sq->mbuf[x].mbuf);
1389                 }
1390                 bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map);
1391         }
1392         free(sq->mbuf, M_MLX5EN);
1393 }
1394
1395 int
1396 mlx5e_alloc_sq_db(struct mlx5e_sq *sq)
1397 {
1398         int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1399         int err;
1400         int x;
1401
1402         sq->mbuf = malloc(wq_sz * sizeof(sq->mbuf[0]), M_MLX5EN, M_WAITOK | M_ZERO);
1403
1404         /* Create DMA descriptor MAPs */
1405         for (x = 0; x != wq_sz; x++) {
1406                 err = -bus_dmamap_create(sq->dma_tag, 0, &sq->mbuf[x].dma_map);
1407                 if (err != 0) {
1408                         while (x--)
1409                                 bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map);
1410                         free(sq->mbuf, M_MLX5EN);
1411                         return (err);
1412                 }
1413         }
1414         return (0);
1415 }
1416
1417 static const char *mlx5e_sq_stats_desc[] = {
1418         MLX5E_SQ_STATS(MLX5E_STATS_DESC)
1419 };
1420
1421 void
1422 mlx5e_update_sq_inline(struct mlx5e_sq *sq)
1423 {
1424         sq->max_inline = sq->priv->params.tx_max_inline;
1425         sq->min_inline_mode = sq->priv->params.tx_min_inline_mode;
1426
1427         /*
1428          * Check if trust state is DSCP or if inline mode is NONE which
1429          * indicates CX-5 or newer hardware.
1430          */
1431         if (sq->priv->params_ethtool.trust_state != MLX5_QPTS_TRUST_PCP ||
1432             sq->min_inline_mode == MLX5_INLINE_MODE_NONE) {
1433                 if (MLX5_CAP_ETH(sq->priv->mdev, wqe_vlan_insert))
1434                         sq->min_insert_caps = MLX5E_INSERT_VLAN | MLX5E_INSERT_NON_VLAN;
1435                 else
1436                         sq->min_insert_caps = MLX5E_INSERT_NON_VLAN;
1437         } else {
1438                 sq->min_insert_caps = 0;
1439         }
1440 }
1441
1442 static void
1443 mlx5e_refresh_sq_inline_sub(struct mlx5e_priv *priv, struct mlx5e_channel *c)
1444 {
1445         int i;
1446
1447         for (i = 0; i != c->num_tc; i++) {
1448                 mtx_lock(&c->sq[i].lock);
1449                 mlx5e_update_sq_inline(&c->sq[i]);
1450                 mtx_unlock(&c->sq[i].lock);
1451         }
1452 }
1453
1454 void
1455 mlx5e_refresh_sq_inline(struct mlx5e_priv *priv)
1456 {
1457         int i;
1458
1459         /* check if channels are closed */
1460         if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
1461                 return;
1462
1463         for (i = 0; i < priv->params.num_channels; i++)
1464                 mlx5e_refresh_sq_inline_sub(priv, &priv->channel[i]);
1465 }
1466
1467 static int
1468 mlx5e_create_sq(struct mlx5e_channel *c,
1469     int tc,
1470     struct mlx5e_sq_param *param,
1471     struct mlx5e_sq *sq)
1472 {
1473         struct mlx5e_priv *priv = c->priv;
1474         struct mlx5_core_dev *mdev = priv->mdev;
1475         char buffer[16];
1476         void *sqc = param->sqc;
1477         void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq);
1478         int err;
1479
1480         /* Create DMA descriptor TAG */
1481         if ((err = -bus_dma_tag_create(
1482             bus_get_dma_tag(mdev->pdev->dev.bsddev),
1483             1,                          /* any alignment */
1484             0,                          /* no boundary */
1485             BUS_SPACE_MAXADDR,          /* lowaddr */
1486             BUS_SPACE_MAXADDR,          /* highaddr */
1487             NULL, NULL,                 /* filter, filterarg */
1488             MLX5E_MAX_TX_PAYLOAD_SIZE,  /* maxsize */
1489             MLX5E_MAX_TX_MBUF_FRAGS,    /* nsegments */
1490             MLX5E_MAX_TX_MBUF_SIZE,     /* maxsegsize */
1491             0,                          /* flags */
1492             NULL, NULL,                 /* lockfunc, lockfuncarg */
1493             &sq->dma_tag)))
1494                 goto done;
1495
1496         err = mlx5_alloc_map_uar(mdev, &sq->uar);
1497         if (err)
1498                 goto err_free_dma_tag;
1499
1500         err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq,
1501             &sq->wq_ctrl);
1502         if (err)
1503                 goto err_unmap_free_uar;
1504
1505         sq->wq.db = &sq->wq.db[MLX5_SND_DBR];
1506         sq->bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2;
1507
1508         err = mlx5e_alloc_sq_db(sq);
1509         if (err)
1510                 goto err_sq_wq_destroy;
1511
1512         sq->mkey_be = c->mkey_be;
1513         sq->ifp = priv->ifp;
1514         sq->priv = priv;
1515         sq->tc = tc;
1516
1517         mlx5e_update_sq_inline(sq);
1518
1519         snprintf(buffer, sizeof(buffer), "txstat%dtc%d", c->ix, tc);
1520         mlx5e_create_stats(&sq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
1521             buffer, mlx5e_sq_stats_desc, MLX5E_SQ_STATS_NUM,
1522             sq->stats.arg);
1523
1524         return (0);
1525
1526 err_sq_wq_destroy:
1527         mlx5_wq_destroy(&sq->wq_ctrl);
1528
1529 err_unmap_free_uar:
1530         mlx5_unmap_free_uar(mdev, &sq->uar);
1531
1532 err_free_dma_tag:
1533         bus_dma_tag_destroy(sq->dma_tag);
1534 done:
1535         return (err);
1536 }
1537
1538 static void
1539 mlx5e_destroy_sq(struct mlx5e_sq *sq)
1540 {
1541         /* destroy all sysctl nodes */
1542         sysctl_ctx_free(&sq->stats.ctx);
1543
1544         mlx5e_free_sq_db(sq);
1545         mlx5_wq_destroy(&sq->wq_ctrl);
1546         mlx5_unmap_free_uar(sq->priv->mdev, &sq->uar);
1547 }
1548
1549 int
1550 mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param,
1551     int tis_num)
1552 {
1553         void *in;
1554         void *sqc;
1555         void *wq;
1556         int inlen;
1557         int err;
1558
1559         inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
1560             sizeof(u64) * sq->wq_ctrl.buf.npages;
1561         in = mlx5_vzalloc(inlen);
1562         if (in == NULL)
1563                 return (-ENOMEM);
1564
1565         sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1566         wq = MLX5_ADDR_OF(sqc, sqc, wq);
1567
1568         memcpy(sqc, param->sqc, sizeof(param->sqc));
1569
1570         MLX5_SET(sqc, sqc, tis_num_0, tis_num);
1571         MLX5_SET(sqc, sqc, cqn, sq->cq.mcq.cqn);
1572         MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
1573         MLX5_SET(sqc, sqc, tis_lst_sz, 1);
1574         MLX5_SET(sqc, sqc, flush_in_error_en, 1);
1575
1576         MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
1577         MLX5_SET(wq, wq, uar_page, sq->uar.index);
1578         MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift -
1579             PAGE_SHIFT);
1580         MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma);
1581
1582         mlx5_fill_page_array(&sq->wq_ctrl.buf,
1583             (__be64 *) MLX5_ADDR_OF(wq, wq, pas));
1584
1585         err = mlx5_core_create_sq(sq->priv->mdev, in, inlen, &sq->sqn);
1586
1587         kvfree(in);
1588
1589         return (err);
1590 }
1591
1592 int
1593 mlx5e_modify_sq(struct mlx5e_sq *sq, int curr_state, int next_state)
1594 {
1595         void *in;
1596         void *sqc;
1597         int inlen;
1598         int err;
1599
1600         inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1601         in = mlx5_vzalloc(inlen);
1602         if (in == NULL)
1603                 return (-ENOMEM);
1604
1605         sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
1606
1607         MLX5_SET(modify_sq_in, in, sqn, sq->sqn);
1608         MLX5_SET(modify_sq_in, in, sq_state, curr_state);
1609         MLX5_SET(sqc, sqc, state, next_state);
1610
1611         err = mlx5_core_modify_sq(sq->priv->mdev, in, inlen);
1612
1613         kvfree(in);
1614
1615         return (err);
1616 }
1617
1618 void
1619 mlx5e_disable_sq(struct mlx5e_sq *sq)
1620 {
1621
1622         mlx5_core_destroy_sq(sq->priv->mdev, sq->sqn);
1623 }
1624
1625 static int
1626 mlx5e_open_sq(struct mlx5e_channel *c,
1627     int tc,
1628     struct mlx5e_sq_param *param,
1629     struct mlx5e_sq *sq)
1630 {
1631         int err;
1632
1633         err = mlx5e_create_sq(c, tc, param, sq);
1634         if (err)
1635                 return (err);
1636
1637         err = mlx5e_enable_sq(sq, param, c->priv->tisn[tc]);
1638         if (err)
1639                 goto err_destroy_sq;
1640
1641         err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY);
1642         if (err)
1643                 goto err_disable_sq;
1644
1645         WRITE_ONCE(sq->running, 1);
1646
1647         return (0);
1648
1649 err_disable_sq:
1650         mlx5e_disable_sq(sq);
1651 err_destroy_sq:
1652         mlx5e_destroy_sq(sq);
1653
1654         return (err);
1655 }
1656
1657 static void
1658 mlx5e_sq_send_nops_locked(struct mlx5e_sq *sq, int can_sleep)
1659 {
1660         /* fill up remainder with NOPs */
1661         while (sq->cev_counter != 0) {
1662                 while (!mlx5e_sq_has_room_for(sq, 1)) {
1663                         if (can_sleep != 0) {
1664                                 mtx_unlock(&sq->lock);
1665                                 msleep(4);
1666                                 mtx_lock(&sq->lock);
1667                         } else {
1668                                 goto done;
1669                         }
1670                 }
1671                 /* send a single NOP */
1672                 mlx5e_send_nop(sq, 1);
1673                 atomic_thread_fence_rel();
1674         }
1675 done:
1676         /* Check if we need to write the doorbell */
1677         if (likely(sq->doorbell.d64 != 0)) {
1678                 mlx5e_tx_notify_hw(sq, sq->doorbell.d32, 0);
1679                 sq->doorbell.d64 = 0;
1680         }
1681 }
1682
1683 void
1684 mlx5e_sq_cev_timeout(void *arg)
1685 {
1686         struct mlx5e_sq *sq = arg;
1687
1688         mtx_assert(&sq->lock, MA_OWNED);
1689
1690         /* check next state */
1691         switch (sq->cev_next_state) {
1692         case MLX5E_CEV_STATE_SEND_NOPS:
1693                 /* fill TX ring with NOPs, if any */
1694                 mlx5e_sq_send_nops_locked(sq, 0);
1695
1696                 /* check if completed */
1697                 if (sq->cev_counter == 0) {
1698                         sq->cev_next_state = MLX5E_CEV_STATE_INITIAL;
1699                         return;
1700                 }
1701                 break;
1702         default:
1703                 /* send NOPs on next timeout */
1704                 sq->cev_next_state = MLX5E_CEV_STATE_SEND_NOPS;
1705                 break;
1706         }
1707
1708         /* restart timer */
1709         callout_reset_curcpu(&sq->cev_callout, hz, mlx5e_sq_cev_timeout, sq);
1710 }
1711
1712 void
1713 mlx5e_drain_sq(struct mlx5e_sq *sq)
1714 {
1715         int error;
1716         struct mlx5_core_dev *mdev= sq->priv->mdev;
1717
1718         /*
1719          * Check if already stopped.
1720          *
1721          * NOTE: Serialization of this function is managed by the
1722          * caller ensuring the priv's state lock is locked or in case
1723          * of rate limit support, a single thread manages drain and
1724          * resume of SQs. The "running" variable can therefore safely
1725          * be read without any locks.
1726          */
1727         if (READ_ONCE(sq->running) == 0)
1728                 return;
1729
1730         /* don't put more packets into the SQ */
1731         WRITE_ONCE(sq->running, 0);
1732
1733         /* serialize access to DMA rings */
1734         mtx_lock(&sq->lock);
1735
1736         /* teardown event factor timer, if any */
1737         sq->cev_next_state = MLX5E_CEV_STATE_HOLD_NOPS;
1738         callout_stop(&sq->cev_callout);
1739
1740         /* send dummy NOPs in order to flush the transmit ring */
1741         mlx5e_sq_send_nops_locked(sq, 1);
1742         mtx_unlock(&sq->lock);
1743
1744         /* make sure it is safe to free the callout */
1745         callout_drain(&sq->cev_callout);
1746
1747         /* wait till SQ is empty or link is down */
1748         mtx_lock(&sq->lock);
1749         while (sq->cc != sq->pc &&
1750             (sq->priv->media_status_last & IFM_ACTIVE) != 0 &&
1751             mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) {
1752                 mtx_unlock(&sq->lock);
1753                 msleep(1);
1754                 sq->cq.mcq.comp(&sq->cq.mcq);
1755                 mtx_lock(&sq->lock);
1756         }
1757         mtx_unlock(&sq->lock);
1758
1759         /* error out remaining requests */
1760         error = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY, MLX5_SQC_STATE_ERR);
1761         if (error != 0) {
1762                 if_printf(sq->ifp,
1763                     "mlx5e_modify_sq() from RDY to ERR failed: %d\n", error);
1764         }
1765
1766         /* wait till SQ is empty */
1767         mtx_lock(&sq->lock);
1768         while (sq->cc != sq->pc &&
1769                mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) {
1770                 mtx_unlock(&sq->lock);
1771                 msleep(1);
1772                 sq->cq.mcq.comp(&sq->cq.mcq);
1773                 mtx_lock(&sq->lock);
1774         }
1775         mtx_unlock(&sq->lock);
1776 }
1777
1778 static void
1779 mlx5e_close_sq_wait(struct mlx5e_sq *sq)
1780 {
1781
1782         mlx5e_drain_sq(sq);
1783         mlx5e_disable_sq(sq);
1784         mlx5e_destroy_sq(sq);
1785 }
1786
1787 static int
1788 mlx5e_create_cq(struct mlx5e_priv *priv,
1789     struct mlx5e_cq_param *param,
1790     struct mlx5e_cq *cq,
1791     mlx5e_cq_comp_t *comp,
1792     int eq_ix)
1793 {
1794         struct mlx5_core_dev *mdev = priv->mdev;
1795         struct mlx5_core_cq *mcq = &cq->mcq;
1796         int eqn_not_used;
1797         int irqn;
1798         int err;
1799         u32 i;
1800
1801         param->wq.buf_numa_node = 0;
1802         param->wq.db_numa_node = 0;
1803
1804         err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
1805             &cq->wq_ctrl);
1806         if (err)
1807                 return (err);
1808
1809         mlx5_vector2eqn(mdev, eq_ix, &eqn_not_used, &irqn);
1810
1811         mcq->cqe_sz = 64;
1812         mcq->set_ci_db = cq->wq_ctrl.db.db;
1813         mcq->arm_db = cq->wq_ctrl.db.db + 1;
1814         *mcq->set_ci_db = 0;
1815         *mcq->arm_db = 0;
1816         mcq->vector = eq_ix;
1817         mcq->comp = comp;
1818         mcq->event = mlx5e_cq_error_event;
1819         mcq->irqn = irqn;
1820         mcq->uar = &priv->cq_uar;
1821
1822         for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
1823                 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
1824
1825                 cqe->op_own = 0xf1;
1826         }
1827
1828         cq->priv = priv;
1829
1830         return (0);
1831 }
1832
1833 static void
1834 mlx5e_destroy_cq(struct mlx5e_cq *cq)
1835 {
1836         mlx5_wq_destroy(&cq->wq_ctrl);
1837 }
1838
1839 static int
1840 mlx5e_enable_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param, int eq_ix)
1841 {
1842         struct mlx5_core_cq *mcq = &cq->mcq;
1843         void *in;
1844         void *cqc;
1845         int inlen;
1846         int irqn_not_used;
1847         int eqn;
1848         int err;
1849
1850         inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
1851             sizeof(u64) * cq->wq_ctrl.buf.npages;
1852         in = mlx5_vzalloc(inlen);
1853         if (in == NULL)
1854                 return (-ENOMEM);
1855
1856         cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
1857
1858         memcpy(cqc, param->cqc, sizeof(param->cqc));
1859
1860         mlx5_fill_page_array(&cq->wq_ctrl.buf,
1861             (__be64 *) MLX5_ADDR_OF(create_cq_in, in, pas));
1862
1863         mlx5_vector2eqn(cq->priv->mdev, eq_ix, &eqn, &irqn_not_used);
1864
1865         MLX5_SET(cqc, cqc, c_eqn, eqn);
1866         MLX5_SET(cqc, cqc, uar_page, mcq->uar->index);
1867         MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
1868             PAGE_SHIFT);
1869         MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma);
1870
1871         err = mlx5_core_create_cq(cq->priv->mdev, mcq, in, inlen);
1872
1873         kvfree(in);
1874
1875         if (err)
1876                 return (err);
1877
1878         mlx5e_cq_arm(cq, MLX5_GET_DOORBELL_LOCK(&cq->priv->doorbell_lock));
1879
1880         return (0);
1881 }
1882
1883 static void
1884 mlx5e_disable_cq(struct mlx5e_cq *cq)
1885 {
1886
1887         mlx5_core_destroy_cq(cq->priv->mdev, &cq->mcq);
1888 }
1889
1890 int
1891 mlx5e_open_cq(struct mlx5e_priv *priv,
1892     struct mlx5e_cq_param *param,
1893     struct mlx5e_cq *cq,
1894     mlx5e_cq_comp_t *comp,
1895     int eq_ix)
1896 {
1897         int err;
1898
1899         err = mlx5e_create_cq(priv, param, cq, comp, eq_ix);
1900         if (err)
1901                 return (err);
1902
1903         err = mlx5e_enable_cq(cq, param, eq_ix);
1904         if (err)
1905                 goto err_destroy_cq;
1906
1907         return (0);
1908
1909 err_destroy_cq:
1910         mlx5e_destroy_cq(cq);
1911
1912         return (err);
1913 }
1914
1915 void
1916 mlx5e_close_cq(struct mlx5e_cq *cq)
1917 {
1918         mlx5e_disable_cq(cq);
1919         mlx5e_destroy_cq(cq);
1920 }
1921
1922 static int
1923 mlx5e_open_tx_cqs(struct mlx5e_channel *c,
1924     struct mlx5e_channel_param *cparam)
1925 {
1926         int err;
1927         int tc;
1928
1929         for (tc = 0; tc < c->num_tc; tc++) {
1930                 /* open completion queue */
1931                 err = mlx5e_open_cq(c->priv, &cparam->tx_cq, &c->sq[tc].cq,
1932                     &mlx5e_tx_cq_comp, c->ix);
1933                 if (err)
1934                         goto err_close_tx_cqs;
1935         }
1936         return (0);
1937
1938 err_close_tx_cqs:
1939         for (tc--; tc >= 0; tc--)
1940                 mlx5e_close_cq(&c->sq[tc].cq);
1941
1942         return (err);
1943 }
1944
1945 static void
1946 mlx5e_close_tx_cqs(struct mlx5e_channel *c)
1947 {
1948         int tc;
1949
1950         for (tc = 0; tc < c->num_tc; tc++)
1951                 mlx5e_close_cq(&c->sq[tc].cq);
1952 }
1953
1954 static int
1955 mlx5e_open_sqs(struct mlx5e_channel *c,
1956     struct mlx5e_channel_param *cparam)
1957 {
1958         int err;
1959         int tc;
1960
1961         for (tc = 0; tc < c->num_tc; tc++) {
1962                 err = mlx5e_open_sq(c, tc, &cparam->sq, &c->sq[tc]);
1963                 if (err)
1964                         goto err_close_sqs;
1965         }
1966
1967         return (0);
1968
1969 err_close_sqs:
1970         for (tc--; tc >= 0; tc--)
1971                 mlx5e_close_sq_wait(&c->sq[tc]);
1972
1973         return (err);
1974 }
1975
1976 static void
1977 mlx5e_close_sqs_wait(struct mlx5e_channel *c)
1978 {
1979         int tc;
1980
1981         for (tc = 0; tc < c->num_tc; tc++)
1982                 mlx5e_close_sq_wait(&c->sq[tc]);
1983 }
1984
1985 static void
1986 mlx5e_chan_mtx_init(struct mlx5e_channel *c)
1987 {
1988         int tc;
1989
1990         mtx_init(&c->rq.mtx, "mlx5rx", MTX_NETWORK_LOCK, MTX_DEF);
1991
1992         callout_init_mtx(&c->rq.watchdog, &c->rq.mtx, 0);
1993
1994         for (tc = 0; tc < c->num_tc; tc++) {
1995                 struct mlx5e_sq *sq = c->sq + tc;
1996
1997                 mtx_init(&sq->lock, "mlx5tx",
1998                     MTX_NETWORK_LOCK " TX", MTX_DEF);
1999                 mtx_init(&sq->comp_lock, "mlx5comp",
2000                     MTX_NETWORK_LOCK " TX", MTX_DEF);
2001
2002                 callout_init_mtx(&sq->cev_callout, &sq->lock, 0);
2003
2004                 sq->cev_factor = c->priv->params_ethtool.tx_completion_fact;
2005
2006                 /* ensure the TX completion event factor is not zero */
2007                 if (sq->cev_factor == 0)
2008                         sq->cev_factor = 1;
2009         }
2010 }
2011
2012 static void
2013 mlx5e_chan_mtx_destroy(struct mlx5e_channel *c)
2014 {
2015         int tc;
2016
2017         mtx_destroy(&c->rq.mtx);
2018
2019         for (tc = 0; tc < c->num_tc; tc++) {
2020                 mtx_destroy(&c->sq[tc].lock);
2021                 mtx_destroy(&c->sq[tc].comp_lock);
2022         }
2023 }
2024
2025 static int
2026 mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
2027     struct mlx5e_channel_param *cparam,
2028     struct mlx5e_channel *c)
2029 {
2030         int err;
2031
2032         memset(c, 0, sizeof(*c));
2033
2034         c->priv = priv;
2035         c->ix = ix;
2036         c->ifp = priv->ifp;
2037         c->mkey_be = cpu_to_be32(priv->mr.key);
2038         c->num_tc = priv->num_tc;
2039
2040         /* init mutexes */
2041         mlx5e_chan_mtx_init(c);
2042
2043         /* open transmit completion queue */
2044         err = mlx5e_open_tx_cqs(c, cparam);
2045         if (err)
2046                 goto err_free;
2047
2048         /* open receive completion queue */
2049         err = mlx5e_open_cq(c->priv, &cparam->rx_cq, &c->rq.cq,
2050             &mlx5e_rx_cq_comp, c->ix);
2051         if (err)
2052                 goto err_close_tx_cqs;
2053
2054         err = mlx5e_open_sqs(c, cparam);
2055         if (err)
2056                 goto err_close_rx_cq;
2057
2058         err = mlx5e_open_rq(c, &cparam->rq, &c->rq);
2059         if (err)
2060                 goto err_close_sqs;
2061
2062         /* poll receive queue initially */
2063         c->rq.cq.mcq.comp(&c->rq.cq.mcq);
2064
2065         return (0);
2066
2067 err_close_sqs:
2068         mlx5e_close_sqs_wait(c);
2069
2070 err_close_rx_cq:
2071         mlx5e_close_cq(&c->rq.cq);
2072
2073 err_close_tx_cqs:
2074         mlx5e_close_tx_cqs(c);
2075
2076 err_free:
2077         /* destroy mutexes */
2078         mlx5e_chan_mtx_destroy(c);
2079         return (err);
2080 }
2081
2082 static void
2083 mlx5e_close_channel(struct mlx5e_channel *c)
2084 {
2085         mlx5e_close_rq(&c->rq);
2086 }
2087
2088 static void
2089 mlx5e_close_channel_wait(struct mlx5e_channel *c)
2090 {
2091         mlx5e_close_rq_wait(&c->rq);
2092         mlx5e_close_sqs_wait(c);
2093         mlx5e_close_tx_cqs(c);
2094         /* destroy mutexes */
2095         mlx5e_chan_mtx_destroy(c);
2096 }
2097
2098 static int
2099 mlx5e_get_wqe_sz(struct mlx5e_priv *priv, u32 *wqe_sz, u32 *nsegs)
2100 {
2101         u32 r, n;
2102
2103         r = priv->params.hw_lro_en ? priv->params.lro_wqe_sz :
2104             MLX5E_SW2MB_MTU(priv->ifp->if_mtu);
2105         if (r > MJUM16BYTES)
2106                 return (-ENOMEM);
2107
2108         if (r > MJUM9BYTES)
2109                 r = MJUM16BYTES;
2110         else if (r > MJUMPAGESIZE)
2111                 r = MJUM9BYTES;
2112         else if (r > MCLBYTES)
2113                 r = MJUMPAGESIZE;
2114         else
2115                 r = MCLBYTES;
2116
2117         /*
2118          * n + 1 must be a power of two, because stride size must be.
2119          * Stride size is 16 * (n + 1), as the first segment is
2120          * control.
2121          */
2122         for (n = howmany(r, MLX5E_MAX_RX_BYTES); !powerof2(n + 1); n++)
2123                 ;
2124
2125         *wqe_sz = r;
2126         *nsegs = n;
2127         return (0);
2128 }
2129
2130 static void
2131 mlx5e_build_rq_param(struct mlx5e_priv *priv,
2132     struct mlx5e_rq_param *param)
2133 {
2134         void *rqc = param->rqc;
2135         void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
2136         u32 wqe_sz, nsegs;
2137
2138         mlx5e_get_wqe_sz(priv, &wqe_sz, &nsegs);
2139         MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
2140         MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
2141         MLX5_SET(wq, wq, log_wq_stride, ilog2(sizeof(struct mlx5e_rx_wqe) +
2142             nsegs * sizeof(struct mlx5_wqe_data_seg)));
2143         MLX5_SET(wq, wq, log_wq_sz, priv->params.log_rq_size);
2144         MLX5_SET(wq, wq, pd, priv->pdn);
2145
2146         param->wq.buf_numa_node = 0;
2147         param->wq.db_numa_node = 0;
2148         param->wq.linear = 1;
2149 }
2150
2151 static void
2152 mlx5e_build_sq_param(struct mlx5e_priv *priv,
2153     struct mlx5e_sq_param *param)
2154 {
2155         void *sqc = param->sqc;
2156         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
2157
2158         MLX5_SET(wq, wq, log_wq_sz, priv->params.log_sq_size);
2159         MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
2160         MLX5_SET(wq, wq, pd, priv->pdn);
2161
2162         param->wq.buf_numa_node = 0;
2163         param->wq.db_numa_node = 0;
2164         param->wq.linear = 1;
2165 }
2166
2167 static void
2168 mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
2169     struct mlx5e_cq_param *param)
2170 {
2171         void *cqc = param->cqc;
2172
2173         MLX5_SET(cqc, cqc, uar_page, priv->cq_uar.index);
2174 }
2175
2176 static void
2177 mlx5e_get_default_profile(struct mlx5e_priv *priv, int mode, struct net_dim_cq_moder *ptr)
2178 {
2179
2180         *ptr = net_dim_get_profile(mode, MLX5E_DIM_DEFAULT_PROFILE);
2181
2182         /* apply LRO restrictions */
2183         if (priv->params.hw_lro_en &&
2184             ptr->pkts > MLX5E_DIM_MAX_RX_CQ_MODERATION_PKTS_WITH_LRO) {
2185                 ptr->pkts = MLX5E_DIM_MAX_RX_CQ_MODERATION_PKTS_WITH_LRO;
2186         }
2187 }
2188
2189 static void
2190 mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
2191     struct mlx5e_cq_param *param)
2192 {
2193         struct net_dim_cq_moder curr;
2194         void *cqc = param->cqc;
2195
2196         /*
2197          * We use MLX5_CQE_FORMAT_HASH because the RX hash mini CQE
2198          * format is more beneficial for FreeBSD use case.
2199          *
2200          * Adding support for MLX5_CQE_FORMAT_CSUM will require changes
2201          * in mlx5e_decompress_cqe.
2202          */
2203         if (priv->params.cqe_zipping_en) {
2204                 MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_HASH);
2205                 MLX5_SET(cqc, cqc, cqe_compression_en, 1);
2206         }
2207
2208         MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_rq_size);
2209
2210         switch (priv->params.rx_cq_moderation_mode) {
2211         case 0:
2212                 MLX5_SET(cqc, cqc, cq_period, priv->params.rx_cq_moderation_usec);
2213                 MLX5_SET(cqc, cqc, cq_max_count, priv->params.rx_cq_moderation_pkts);
2214                 MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2215                 break;
2216         case 1:
2217                 MLX5_SET(cqc, cqc, cq_period, priv->params.rx_cq_moderation_usec);
2218                 MLX5_SET(cqc, cqc, cq_max_count, priv->params.rx_cq_moderation_pkts);
2219                 if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
2220                         MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
2221                 else
2222                         MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2223                 break;
2224         case 2:
2225                 mlx5e_get_default_profile(priv, NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE, &curr);
2226                 MLX5_SET(cqc, cqc, cq_period, curr.usec);
2227                 MLX5_SET(cqc, cqc, cq_max_count, curr.pkts);
2228                 MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2229                 break;
2230         case 3:
2231                 mlx5e_get_default_profile(priv, NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE, &curr);
2232                 MLX5_SET(cqc, cqc, cq_period, curr.usec);
2233                 MLX5_SET(cqc, cqc, cq_max_count, curr.pkts);
2234                 if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
2235                         MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
2236                 else
2237                         MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2238                 break;
2239         default:
2240                 break;
2241         }
2242
2243         mlx5e_dim_build_cq_param(priv, param);
2244
2245         mlx5e_build_common_cq_param(priv, param);
2246 }
2247
2248 static void
2249 mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
2250     struct mlx5e_cq_param *param)
2251 {
2252         void *cqc = param->cqc;
2253
2254         MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_sq_size);
2255         MLX5_SET(cqc, cqc, cq_period, priv->params.tx_cq_moderation_usec);
2256         MLX5_SET(cqc, cqc, cq_max_count, priv->params.tx_cq_moderation_pkts);
2257
2258         switch (priv->params.tx_cq_moderation_mode) {
2259         case 0:
2260                 MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2261                 break;
2262         default:
2263                 if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
2264                         MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
2265                 else
2266                         MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2267                 break;
2268         }
2269
2270         mlx5e_build_common_cq_param(priv, param);
2271 }
2272
2273 static void
2274 mlx5e_build_channel_param(struct mlx5e_priv *priv,
2275     struct mlx5e_channel_param *cparam)
2276 {
2277         memset(cparam, 0, sizeof(*cparam));
2278
2279         mlx5e_build_rq_param(priv, &cparam->rq);
2280         mlx5e_build_sq_param(priv, &cparam->sq);
2281         mlx5e_build_rx_cq_param(priv, &cparam->rx_cq);
2282         mlx5e_build_tx_cq_param(priv, &cparam->tx_cq);
2283 }
2284
2285 static int
2286 mlx5e_open_channels(struct mlx5e_priv *priv)
2287 {
2288         struct mlx5e_channel_param cparam;
2289         int err;
2290         int i;
2291         int j;
2292
2293         mlx5e_build_channel_param(priv, &cparam);
2294         for (i = 0; i < priv->params.num_channels; i++) {
2295                 err = mlx5e_open_channel(priv, i, &cparam, &priv->channel[i]);
2296                 if (err)
2297                         goto err_close_channels;
2298         }
2299
2300         for (j = 0; j < priv->params.num_channels; j++) {
2301                 err = mlx5e_wait_for_min_rx_wqes(&priv->channel[j].rq);
2302                 if (err)
2303                         goto err_close_channels;
2304         }
2305
2306         return (0);
2307
2308 err_close_channels:
2309         while (i--) {
2310                 mlx5e_close_channel(&priv->channel[i]);
2311                 mlx5e_close_channel_wait(&priv->channel[i]);
2312         }
2313         return (err);
2314 }
2315
2316 static void
2317 mlx5e_close_channels(struct mlx5e_priv *priv)
2318 {
2319         int i;
2320
2321         for (i = 0; i < priv->params.num_channels; i++)
2322                 mlx5e_close_channel(&priv->channel[i]);
2323         for (i = 0; i < priv->params.num_channels; i++)
2324                 mlx5e_close_channel_wait(&priv->channel[i]);
2325 }
2326
2327 static int
2328 mlx5e_refresh_sq_params(struct mlx5e_priv *priv, struct mlx5e_sq *sq)
2329 {
2330
2331         if (MLX5_CAP_GEN(priv->mdev, cq_period_mode_modify)) {
2332                 uint8_t cq_mode;
2333
2334                 switch (priv->params.tx_cq_moderation_mode) {
2335                 case 0:
2336                 case 2:
2337                         cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
2338                         break;
2339                 default:
2340                         cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_CQE;
2341                         break;
2342                 }
2343
2344                 return (mlx5_core_modify_cq_moderation_mode(priv->mdev, &sq->cq.mcq,
2345                     priv->params.tx_cq_moderation_usec,
2346                     priv->params.tx_cq_moderation_pkts,
2347                     cq_mode));
2348         }
2349
2350         return (mlx5_core_modify_cq_moderation(priv->mdev, &sq->cq.mcq,
2351             priv->params.tx_cq_moderation_usec,
2352             priv->params.tx_cq_moderation_pkts));
2353 }
2354
2355 static int
2356 mlx5e_refresh_rq_params(struct mlx5e_priv *priv, struct mlx5e_rq *rq)
2357 {
2358
2359         if (MLX5_CAP_GEN(priv->mdev, cq_period_mode_modify)) {
2360                 uint8_t cq_mode;
2361                 uint8_t dim_mode;
2362                 int retval;
2363
2364                 switch (priv->params.rx_cq_moderation_mode) {
2365                 case 0:
2366                 case 2:
2367                         cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
2368                         dim_mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
2369                         break;
2370                 default:
2371                         cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_CQE;
2372                         dim_mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE;
2373                         break;
2374                 }
2375
2376                 /* tear down dynamic interrupt moderation */
2377                 mtx_lock(&rq->mtx);
2378                 rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
2379                 mtx_unlock(&rq->mtx);
2380
2381                 /* wait for dynamic interrupt moderation work task, if any */
2382                 cancel_work_sync(&rq->dim.work);
2383
2384                 if (priv->params.rx_cq_moderation_mode >= 2) {
2385                         struct net_dim_cq_moder curr;
2386
2387                         mlx5e_get_default_profile(priv, dim_mode, &curr);
2388
2389                         retval = mlx5_core_modify_cq_moderation_mode(priv->mdev, &rq->cq.mcq,
2390                             curr.usec, curr.pkts, cq_mode);
2391
2392                         /* set dynamic interrupt moderation mode and zero defaults */
2393                         mtx_lock(&rq->mtx);
2394                         rq->dim.mode = dim_mode;
2395                         rq->dim.state = 0;
2396                         rq->dim.profile_ix = MLX5E_DIM_DEFAULT_PROFILE;
2397                         mtx_unlock(&rq->mtx);
2398                 } else {
2399                         retval = mlx5_core_modify_cq_moderation_mode(priv->mdev, &rq->cq.mcq,
2400                             priv->params.rx_cq_moderation_usec,
2401                             priv->params.rx_cq_moderation_pkts,
2402                             cq_mode);
2403                 }
2404                 return (retval);
2405         }
2406
2407         return (mlx5_core_modify_cq_moderation(priv->mdev, &rq->cq.mcq,
2408             priv->params.rx_cq_moderation_usec,
2409             priv->params.rx_cq_moderation_pkts));
2410 }
2411
2412 static int
2413 mlx5e_refresh_channel_params_sub(struct mlx5e_priv *priv, struct mlx5e_channel *c)
2414 {
2415         int err;
2416         int i;
2417
2418         err = mlx5e_refresh_rq_params(priv, &c->rq);
2419         if (err)
2420                 goto done;
2421
2422         for (i = 0; i != c->num_tc; i++) {
2423                 err = mlx5e_refresh_sq_params(priv, &c->sq[i]);
2424                 if (err)
2425                         goto done;
2426         }
2427 done:
2428         return (err);
2429 }
2430
2431 int
2432 mlx5e_refresh_channel_params(struct mlx5e_priv *priv)
2433 {
2434         int i;
2435
2436         /* check if channels are closed */
2437         if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
2438                 return (EINVAL);
2439
2440         for (i = 0; i < priv->params.num_channels; i++) {
2441                 int err;
2442
2443                 err = mlx5e_refresh_channel_params_sub(priv, &priv->channel[i]);
2444                 if (err)
2445                         return (err);
2446         }
2447         return (0);
2448 }
2449
2450 static int
2451 mlx5e_open_tis(struct mlx5e_priv *priv, int tc)
2452 {
2453         struct mlx5_core_dev *mdev = priv->mdev;
2454         u32 in[MLX5_ST_SZ_DW(create_tis_in)];
2455         void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
2456
2457         memset(in, 0, sizeof(in));
2458
2459         MLX5_SET(tisc, tisc, prio, tc);
2460         MLX5_SET(tisc, tisc, transport_domain, priv->tdn);
2461
2462         return (mlx5_core_create_tis(mdev, in, sizeof(in), &priv->tisn[tc]));
2463 }
2464
2465 static void
2466 mlx5e_close_tis(struct mlx5e_priv *priv, int tc)
2467 {
2468         mlx5_core_destroy_tis(priv->mdev, priv->tisn[tc]);
2469 }
2470
2471 static int
2472 mlx5e_open_tises(struct mlx5e_priv *priv)
2473 {
2474         int num_tc = priv->num_tc;
2475         int err;
2476         int tc;
2477
2478         for (tc = 0; tc < num_tc; tc++) {
2479                 err = mlx5e_open_tis(priv, tc);
2480                 if (err)
2481                         goto err_close_tises;
2482         }
2483
2484         return (0);
2485
2486 err_close_tises:
2487         for (tc--; tc >= 0; tc--)
2488                 mlx5e_close_tis(priv, tc);
2489
2490         return (err);
2491 }
2492
2493 static void
2494 mlx5e_close_tises(struct mlx5e_priv *priv)
2495 {
2496         int num_tc = priv->num_tc;
2497         int tc;
2498
2499         for (tc = 0; tc < num_tc; tc++)
2500                 mlx5e_close_tis(priv, tc);
2501 }
2502
2503 static int
2504 mlx5e_open_rqt(struct mlx5e_priv *priv)
2505 {
2506         struct mlx5_core_dev *mdev = priv->mdev;
2507         u32 *in;
2508         u32 out[MLX5_ST_SZ_DW(create_rqt_out)] = {0};
2509         void *rqtc;
2510         int inlen;
2511         int err;
2512         int sz;
2513         int i;
2514
2515         sz = 1 << priv->params.rx_hash_log_tbl_sz;
2516
2517         inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
2518         in = mlx5_vzalloc(inlen);
2519         if (in == NULL)
2520                 return (-ENOMEM);
2521         rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
2522
2523         MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
2524         MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
2525
2526         for (i = 0; i < sz; i++) {
2527                 int ix = i;
2528 #ifdef RSS
2529                 ix = rss_get_indirection_to_bucket(ix);
2530 #endif
2531                 /* ensure we don't overflow */
2532                 ix %= priv->params.num_channels;
2533
2534                 /* apply receive side scaling stride, if any */
2535                 ix -= ix % (int)priv->params.channels_rsss;
2536
2537                 MLX5_SET(rqtc, rqtc, rq_num[i], priv->channel[ix].rq.rqn);
2538         }
2539
2540         MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT);
2541
2542         err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out));
2543         if (!err)
2544                 priv->rqtn = MLX5_GET(create_rqt_out, out, rqtn);
2545
2546         kvfree(in);
2547
2548         return (err);
2549 }
2550
2551 static void
2552 mlx5e_close_rqt(struct mlx5e_priv *priv)
2553 {
2554         u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)] = {0};
2555         u32 out[MLX5_ST_SZ_DW(destroy_rqt_out)] = {0};
2556
2557         MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT);
2558         MLX5_SET(destroy_rqt_in, in, rqtn, priv->rqtn);
2559
2560         mlx5_cmd_exec(priv->mdev, in, sizeof(in), out, sizeof(out));
2561 }
2562
2563 static void
2564 mlx5e_build_tir_ctx(struct mlx5e_priv *priv, u32 * tirc, int tt)
2565 {
2566         void *hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
2567         __be32 *hkey;
2568
2569         MLX5_SET(tirc, tirc, transport_domain, priv->tdn);
2570
2571 #define ROUGH_MAX_L2_L3_HDR_SZ 256
2572
2573 #define MLX5_HASH_IP     (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2574                           MLX5_HASH_FIELD_SEL_DST_IP)
2575
2576 #define MLX5_HASH_ALL    (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2577                           MLX5_HASH_FIELD_SEL_DST_IP   |\
2578                           MLX5_HASH_FIELD_SEL_L4_SPORT |\
2579                           MLX5_HASH_FIELD_SEL_L4_DPORT)
2580
2581 #define MLX5_HASH_IP_IPSEC_SPI  (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2582                                  MLX5_HASH_FIELD_SEL_DST_IP   |\
2583                                  MLX5_HASH_FIELD_SEL_IPSEC_SPI)
2584
2585         if (priv->params.hw_lro_en) {
2586                 MLX5_SET(tirc, tirc, lro_enable_mask,
2587                     MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
2588                     MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO);
2589                 MLX5_SET(tirc, tirc, lro_max_msg_sz,
2590                     (priv->params.lro_wqe_sz -
2591                     ROUGH_MAX_L2_L3_HDR_SZ) >> 8);
2592                 /* TODO: add the option to choose timer value dynamically */
2593                 MLX5_SET(tirc, tirc, lro_timeout_period_usecs,
2594                     MLX5_CAP_ETH(priv->mdev,
2595                     lro_timer_supported_periods[2]));
2596         }
2597
2598         /* setup parameters for hashing TIR type, if any */
2599         switch (tt) {
2600         case MLX5E_TT_ANY:
2601                 MLX5_SET(tirc, tirc, disp_type,
2602                     MLX5_TIRC_DISP_TYPE_DIRECT);
2603                 MLX5_SET(tirc, tirc, inline_rqn,
2604                     priv->channel[0].rq.rqn);
2605                 break;
2606         default:
2607                 MLX5_SET(tirc, tirc, disp_type,
2608                     MLX5_TIRC_DISP_TYPE_INDIRECT);
2609                 MLX5_SET(tirc, tirc, indirect_table,
2610                     priv->rqtn);
2611                 MLX5_SET(tirc, tirc, rx_hash_fn,
2612                     MLX5_TIRC_RX_HASH_FN_HASH_TOEPLITZ);
2613                 hkey = (__be32 *) MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
2614 #ifdef RSS
2615                 /*
2616                  * The FreeBSD RSS implementation does currently not
2617                  * support symmetric Toeplitz hashes:
2618                  */
2619                 MLX5_SET(tirc, tirc, rx_hash_symmetric, 0);
2620                 rss_getkey((uint8_t *)hkey);
2621 #else
2622                 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
2623                 hkey[0] = cpu_to_be32(0xD181C62C);
2624                 hkey[1] = cpu_to_be32(0xF7F4DB5B);
2625                 hkey[2] = cpu_to_be32(0x1983A2FC);
2626                 hkey[3] = cpu_to_be32(0x943E1ADB);
2627                 hkey[4] = cpu_to_be32(0xD9389E6B);
2628                 hkey[5] = cpu_to_be32(0xD1039C2C);
2629                 hkey[6] = cpu_to_be32(0xA74499AD);
2630                 hkey[7] = cpu_to_be32(0x593D56D9);
2631                 hkey[8] = cpu_to_be32(0xF3253C06);
2632                 hkey[9] = cpu_to_be32(0x2ADC1FFC);
2633 #endif
2634                 break;
2635         }
2636
2637         switch (tt) {
2638         case MLX5E_TT_IPV4_TCP:
2639                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2640                     MLX5_L3_PROT_TYPE_IPV4);
2641                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2642                     MLX5_L4_PROT_TYPE_TCP);
2643 #ifdef RSS
2644                 if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4)) {
2645                         MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2646                             MLX5_HASH_IP);
2647                 } else
2648 #endif
2649                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2650                     MLX5_HASH_ALL);
2651                 break;
2652
2653         case MLX5E_TT_IPV6_TCP:
2654                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2655                     MLX5_L3_PROT_TYPE_IPV6);
2656                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2657                     MLX5_L4_PROT_TYPE_TCP);
2658 #ifdef RSS
2659                 if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV6)) {
2660                         MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2661                             MLX5_HASH_IP);
2662                 } else
2663 #endif
2664                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2665                     MLX5_HASH_ALL);
2666                 break;
2667
2668         case MLX5E_TT_IPV4_UDP:
2669                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2670                     MLX5_L3_PROT_TYPE_IPV4);
2671                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2672                     MLX5_L4_PROT_TYPE_UDP);
2673 #ifdef RSS
2674                 if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4)) {
2675                         MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2676                             MLX5_HASH_IP);
2677                 } else
2678 #endif
2679                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2680                     MLX5_HASH_ALL);
2681                 break;
2682
2683         case MLX5E_TT_IPV6_UDP:
2684                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2685                     MLX5_L3_PROT_TYPE_IPV6);
2686                 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2687                     MLX5_L4_PROT_TYPE_UDP);
2688 #ifdef RSS
2689                 if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV6)) {
2690                         MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2691                             MLX5_HASH_IP);
2692                 } else
2693 #endif
2694                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2695                     MLX5_HASH_ALL);
2696                 break;
2697
2698         case MLX5E_TT_IPV4_IPSEC_AH:
2699                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2700                     MLX5_L3_PROT_TYPE_IPV4);
2701                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2702                     MLX5_HASH_IP_IPSEC_SPI);
2703                 break;
2704
2705         case MLX5E_TT_IPV6_IPSEC_AH:
2706                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2707                     MLX5_L3_PROT_TYPE_IPV6);
2708                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2709                     MLX5_HASH_IP_IPSEC_SPI);
2710                 break;
2711
2712         case MLX5E_TT_IPV4_IPSEC_ESP:
2713                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2714                     MLX5_L3_PROT_TYPE_IPV4);
2715                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2716                     MLX5_HASH_IP_IPSEC_SPI);
2717                 break;
2718
2719         case MLX5E_TT_IPV6_IPSEC_ESP:
2720                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2721                     MLX5_L3_PROT_TYPE_IPV6);
2722                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2723                     MLX5_HASH_IP_IPSEC_SPI);
2724                 break;
2725
2726         case MLX5E_TT_IPV4:
2727                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2728                     MLX5_L3_PROT_TYPE_IPV4);
2729                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2730                     MLX5_HASH_IP);
2731                 break;
2732
2733         case MLX5E_TT_IPV6:
2734                 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2735                     MLX5_L3_PROT_TYPE_IPV6);
2736                 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2737                     MLX5_HASH_IP);
2738                 break;
2739
2740         default:
2741                 break;
2742         }
2743 }
2744
2745 static int
2746 mlx5e_open_tir(struct mlx5e_priv *priv, int tt)
2747 {
2748         struct mlx5_core_dev *mdev = priv->mdev;
2749         u32 *in;
2750         void *tirc;
2751         int inlen;
2752         int err;
2753
2754         inlen = MLX5_ST_SZ_BYTES(create_tir_in);
2755         in = mlx5_vzalloc(inlen);
2756         if (in == NULL)
2757                 return (-ENOMEM);
2758         tirc = MLX5_ADDR_OF(create_tir_in, in, tir_context);
2759
2760         mlx5e_build_tir_ctx(priv, tirc, tt);
2761
2762         err = mlx5_core_create_tir(mdev, in, inlen, &priv->tirn[tt]);
2763
2764         kvfree(in);
2765
2766         return (err);
2767 }
2768
2769 static void
2770 mlx5e_close_tir(struct mlx5e_priv *priv, int tt)
2771 {
2772         mlx5_core_destroy_tir(priv->mdev, priv->tirn[tt]);
2773 }
2774
2775 static int
2776 mlx5e_open_tirs(struct mlx5e_priv *priv)
2777 {
2778         int err;
2779         int i;
2780
2781         for (i = 0; i < MLX5E_NUM_TT; i++) {
2782                 err = mlx5e_open_tir(priv, i);
2783                 if (err)
2784                         goto err_close_tirs;
2785         }
2786
2787         return (0);
2788
2789 err_close_tirs:
2790         for (i--; i >= 0; i--)
2791                 mlx5e_close_tir(priv, i);
2792
2793         return (err);
2794 }
2795
2796 static void
2797 mlx5e_close_tirs(struct mlx5e_priv *priv)
2798 {
2799         int i;
2800
2801         for (i = 0; i < MLX5E_NUM_TT; i++)
2802                 mlx5e_close_tir(priv, i);
2803 }
2804
2805 /*
2806  * SW MTU does not include headers,
2807  * HW MTU includes all headers and checksums.
2808  */
2809 static int
2810 mlx5e_set_dev_port_mtu(struct ifnet *ifp, int sw_mtu)
2811 {
2812         struct mlx5e_priv *priv = ifp->if_softc;
2813         struct mlx5_core_dev *mdev = priv->mdev;
2814         int hw_mtu;
2815         int err;
2816
2817         hw_mtu = MLX5E_SW2HW_MTU(sw_mtu);
2818
2819         err = mlx5_set_port_mtu(mdev, hw_mtu);
2820         if (err) {
2821                 if_printf(ifp, "%s: mlx5_set_port_mtu failed setting %d, err=%d\n",
2822                     __func__, sw_mtu, err);
2823                 return (err);
2824         }
2825
2826         /* Update vport context MTU */
2827         err = mlx5_set_vport_mtu(mdev, hw_mtu);
2828         if (err) {
2829                 if_printf(ifp, "%s: Failed updating vport context with MTU size, err=%d\n",
2830                     __func__, err);
2831         }
2832
2833         ifp->if_mtu = sw_mtu;
2834
2835         err = mlx5_query_vport_mtu(mdev, &hw_mtu);
2836         if (err || !hw_mtu) {
2837                 /* fallback to port oper mtu */
2838                 err = mlx5_query_port_oper_mtu(mdev, &hw_mtu);
2839         }
2840         if (err) {
2841                 if_printf(ifp, "Query port MTU, after setting new "
2842                     "MTU value, failed\n");
2843                 return (err);
2844         } else if (MLX5E_HW2SW_MTU(hw_mtu) < sw_mtu) {
2845                 err = -E2BIG,
2846                 if_printf(ifp, "Port MTU %d is smaller than "
2847                     "ifp mtu %d\n", hw_mtu, sw_mtu);
2848         } else if (MLX5E_HW2SW_MTU(hw_mtu) > sw_mtu) {
2849                 err = -EINVAL;
2850                 if_printf(ifp, "Port MTU %d is bigger than "
2851                     "ifp mtu %d\n", hw_mtu, sw_mtu);
2852         }
2853         priv->params_ethtool.hw_mtu = hw_mtu;
2854
2855         return (err);
2856 }
2857
2858 int
2859 mlx5e_open_locked(struct ifnet *ifp)
2860 {
2861         struct mlx5e_priv *priv = ifp->if_softc;
2862         int err;
2863         u16 set_id;
2864
2865         /* check if already opened */
2866         if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
2867                 return (0);
2868
2869 #ifdef RSS
2870         if (rss_getnumbuckets() > priv->params.num_channels) {
2871                 if_printf(ifp, "NOTE: There are more RSS buckets(%u) than "
2872                     "channels(%u) available\n", rss_getnumbuckets(),
2873                     priv->params.num_channels);
2874         }
2875 #endif
2876         err = mlx5e_open_tises(priv);
2877         if (err) {
2878                 if_printf(ifp, "%s: mlx5e_open_tises failed, %d\n",
2879                     __func__, err);
2880                 return (err);
2881         }
2882         err = mlx5_vport_alloc_q_counter(priv->mdev,
2883             MLX5_INTERFACE_PROTOCOL_ETH, &set_id);
2884         if (err) {
2885                 if_printf(priv->ifp,
2886                     "%s: mlx5_vport_alloc_q_counter failed: %d\n",
2887                     __func__, err);
2888                 goto err_close_tises;
2889         }
2890         /* store counter set ID */
2891         priv->counter_set_id = set_id;
2892
2893         err = mlx5e_open_channels(priv);
2894         if (err) {
2895                 if_printf(ifp, "%s: mlx5e_open_channels failed, %d\n",
2896                     __func__, err);
2897                 goto err_dalloc_q_counter;
2898         }
2899         err = mlx5e_open_rqt(priv);
2900         if (err) {
2901                 if_printf(ifp, "%s: mlx5e_open_rqt failed, %d\n",
2902                     __func__, err);
2903                 goto err_close_channels;
2904         }
2905         err = mlx5e_open_tirs(priv);
2906         if (err) {
2907                 if_printf(ifp, "%s: mlx5e_open_tir failed, %d\n",
2908                     __func__, err);
2909                 goto err_close_rqls;
2910         }
2911         err = mlx5e_open_flow_table(priv);
2912         if (err) {
2913                 if_printf(ifp, "%s: mlx5e_open_flow_table failed, %d\n",
2914                     __func__, err);
2915                 goto err_close_tirs;
2916         }
2917         err = mlx5e_add_all_vlan_rules(priv);
2918         if (err) {
2919                 if_printf(ifp, "%s: mlx5e_add_all_vlan_rules failed, %d\n",
2920                     __func__, err);
2921                 goto err_close_flow_table;
2922         }
2923         set_bit(MLX5E_STATE_OPENED, &priv->state);
2924
2925         mlx5e_update_carrier(priv);
2926         mlx5e_set_rx_mode_core(priv);
2927
2928         return (0);
2929
2930 err_close_flow_table:
2931         mlx5e_close_flow_table(priv);
2932
2933 err_close_tirs:
2934         mlx5e_close_tirs(priv);
2935
2936 err_close_rqls:
2937         mlx5e_close_rqt(priv);
2938
2939 err_close_channels:
2940         mlx5e_close_channels(priv);
2941
2942 err_dalloc_q_counter:
2943         mlx5_vport_dealloc_q_counter(priv->mdev,
2944             MLX5_INTERFACE_PROTOCOL_ETH, priv->counter_set_id);
2945
2946 err_close_tises:
2947         mlx5e_close_tises(priv);
2948
2949         return (err);
2950 }
2951
2952 static void
2953 mlx5e_open(void *arg)
2954 {
2955         struct mlx5e_priv *priv = arg;
2956
2957         PRIV_LOCK(priv);
2958         if (mlx5_set_port_status(priv->mdev, MLX5_PORT_UP))
2959                 if_printf(priv->ifp,
2960                     "%s: Setting port status to up failed\n",
2961                     __func__);
2962
2963         mlx5e_open_locked(priv->ifp);
2964         priv->ifp->if_drv_flags |= IFF_DRV_RUNNING;
2965         PRIV_UNLOCK(priv);
2966 }
2967
2968 int
2969 mlx5e_close_locked(struct ifnet *ifp)
2970 {
2971         struct mlx5e_priv *priv = ifp->if_softc;
2972
2973         /* check if already closed */
2974         if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
2975                 return (0);
2976
2977         clear_bit(MLX5E_STATE_OPENED, &priv->state);
2978
2979         mlx5e_set_rx_mode_core(priv);
2980         mlx5e_del_all_vlan_rules(priv);
2981         if_link_state_change(priv->ifp, LINK_STATE_DOWN);
2982         mlx5e_close_flow_table(priv);
2983         mlx5e_close_tirs(priv);
2984         mlx5e_close_rqt(priv);
2985         mlx5e_close_channels(priv);
2986         mlx5_vport_dealloc_q_counter(priv->mdev,
2987             MLX5_INTERFACE_PROTOCOL_ETH, priv->counter_set_id);
2988         mlx5e_close_tises(priv);
2989
2990         return (0);
2991 }
2992
2993 #if (__FreeBSD_version >= 1100000)
2994 static uint64_t
2995 mlx5e_get_counter(struct ifnet *ifp, ift_counter cnt)
2996 {
2997         struct mlx5e_priv *priv = ifp->if_softc;
2998         u64 retval;
2999
3000         /* PRIV_LOCK(priv); XXX not allowed */
3001         switch (cnt) {
3002         case IFCOUNTER_IPACKETS:
3003                 retval = priv->stats.vport.rx_packets;
3004                 break;
3005         case IFCOUNTER_IERRORS:
3006                 retval = priv->stats.pport.in_range_len_errors +
3007                     priv->stats.pport.out_of_range_len +
3008                     priv->stats.pport.too_long_errors +
3009                     priv->stats.pport.check_seq_err +
3010                     priv->stats.pport.alignment_err;
3011                 break;
3012         case IFCOUNTER_IQDROPS:
3013                 retval = priv->stats.vport.rx_out_of_buffer;
3014                 break;
3015         case IFCOUNTER_OPACKETS:
3016                 retval = priv->stats.vport.tx_packets;
3017                 break;
3018         case IFCOUNTER_OERRORS:
3019                 retval = priv->stats.port_stats_debug.out_discards;
3020                 break;
3021         case IFCOUNTER_IBYTES:
3022                 retval = priv->stats.vport.rx_bytes;
3023                 break;
3024         case IFCOUNTER_OBYTES:
3025                 retval = priv->stats.vport.tx_bytes;
3026                 break;
3027         case IFCOUNTER_IMCASTS:
3028                 retval = priv->stats.vport.rx_multicast_packets;
3029                 break;
3030         case IFCOUNTER_OMCASTS:
3031                 retval = priv->stats.vport.tx_multicast_packets;
3032                 break;
3033         case IFCOUNTER_OQDROPS:
3034                 retval = priv->stats.vport.tx_queue_dropped;
3035                 break;
3036         case IFCOUNTER_COLLISIONS:
3037                 retval = priv->stats.pport.collisions;
3038                 break;
3039         default:
3040                 retval = if_get_counter_default(ifp, cnt);
3041                 break;
3042         }
3043         /* PRIV_UNLOCK(priv); XXX not allowed */
3044         return (retval);
3045 }
3046 #endif
3047
3048 static void
3049 mlx5e_set_rx_mode(struct ifnet *ifp)
3050 {
3051         struct mlx5e_priv *priv = ifp->if_softc;
3052
3053         queue_work(priv->wq, &priv->set_rx_mode_work);
3054 }
3055
3056 static int
3057 mlx5e_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
3058 {
3059         struct mlx5e_priv *priv;
3060         struct ifreq *ifr;
3061         struct ifi2creq i2c;
3062         int error = 0;
3063         int mask = 0;
3064         int size_read = 0;
3065         int module_status;
3066         int module_num;
3067         int max_mtu;
3068         uint8_t read_addr;
3069
3070         priv = ifp->if_softc;
3071
3072         /* check if detaching */
3073         if (priv == NULL || priv->gone != 0)
3074                 return (ENXIO);
3075
3076         switch (command) {
3077         case SIOCSIFMTU:
3078                 ifr = (struct ifreq *)data;
3079
3080                 PRIV_LOCK(priv);
3081                 mlx5_query_port_max_mtu(priv->mdev, &max_mtu);
3082
3083                 if (ifr->ifr_mtu >= MLX5E_MTU_MIN &&
3084                     ifr->ifr_mtu <= MIN(MLX5E_MTU_MAX, max_mtu)) {
3085                         int was_opened;
3086
3087                         was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
3088                         if (was_opened)
3089                                 mlx5e_close_locked(ifp);
3090
3091                         /* set new MTU */
3092                         mlx5e_set_dev_port_mtu(ifp, ifr->ifr_mtu);
3093
3094                         if (was_opened)
3095                                 mlx5e_open_locked(ifp);
3096                 } else {
3097                         error = EINVAL;
3098                         if_printf(ifp, "Invalid MTU value. Min val: %d, Max val: %d\n",
3099                             MLX5E_MTU_MIN, MIN(MLX5E_MTU_MAX, max_mtu));
3100                 }
3101                 PRIV_UNLOCK(priv);
3102                 break;
3103         case SIOCSIFFLAGS:
3104                 if ((ifp->if_flags & IFF_UP) &&
3105                     (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
3106                         mlx5e_set_rx_mode(ifp);
3107                         break;
3108                 }
3109                 PRIV_LOCK(priv);
3110                 if (ifp->if_flags & IFF_UP) {
3111                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
3112                                 if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
3113                                         mlx5e_open_locked(ifp);
3114                                 ifp->if_drv_flags |= IFF_DRV_RUNNING;
3115                                 mlx5_set_port_status(priv->mdev, MLX5_PORT_UP);
3116                         }
3117                 } else {
3118                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
3119                                 mlx5_set_port_status(priv->mdev,
3120                                     MLX5_PORT_DOWN);
3121                                 if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
3122                                         mlx5e_close_locked(ifp);
3123                                 mlx5e_update_carrier(priv);
3124                                 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3125                         }
3126                 }
3127                 PRIV_UNLOCK(priv);
3128                 break;
3129         case SIOCADDMULTI:
3130         case SIOCDELMULTI:
3131                 mlx5e_set_rx_mode(ifp);
3132                 break;
3133         case SIOCSIFMEDIA:
3134         case SIOCGIFMEDIA:
3135         case SIOCGIFXMEDIA:
3136                 ifr = (struct ifreq *)data;
3137                 error = ifmedia_ioctl(ifp, ifr, &priv->media, command);
3138                 break;
3139         case SIOCSIFCAP:
3140                 ifr = (struct ifreq *)data;
3141                 PRIV_LOCK(priv);
3142                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
3143
3144                 if (mask & IFCAP_TXCSUM) {
3145                         ifp->if_capenable ^= IFCAP_TXCSUM;
3146                         ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
3147
3148                         if (IFCAP_TSO4 & ifp->if_capenable &&
3149                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
3150                                 ifp->if_capenable &= ~IFCAP_TSO4;
3151                                 ifp->if_hwassist &= ~CSUM_IP_TSO;
3152                                 if_printf(ifp,
3153                                     "tso4 disabled due to -txcsum.\n");
3154                         }
3155                 }
3156                 if (mask & IFCAP_TXCSUM_IPV6) {
3157                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
3158                         ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
3159
3160                         if (IFCAP_TSO6 & ifp->if_capenable &&
3161                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
3162                                 ifp->if_capenable &= ~IFCAP_TSO6;
3163                                 ifp->if_hwassist &= ~CSUM_IP6_TSO;
3164                                 if_printf(ifp,
3165                                     "tso6 disabled due to -txcsum6.\n");
3166                         }
3167                 }
3168                 if (mask & IFCAP_RXCSUM)
3169                         ifp->if_capenable ^= IFCAP_RXCSUM;
3170                 if (mask & IFCAP_RXCSUM_IPV6)
3171                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
3172                 if (mask & IFCAP_TSO4) {
3173                         if (!(IFCAP_TSO4 & ifp->if_capenable) &&
3174                             !(IFCAP_TXCSUM & ifp->if_capenable)) {
3175                                 if_printf(ifp, "enable txcsum first.\n");
3176                                 error = EAGAIN;
3177                                 goto out;
3178                         }
3179                         ifp->if_capenable ^= IFCAP_TSO4;
3180                         ifp->if_hwassist ^= CSUM_IP_TSO;
3181                 }
3182                 if (mask & IFCAP_TSO6) {
3183                         if (!(IFCAP_TSO6 & ifp->if_capenable) &&
3184                             !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
3185                                 if_printf(ifp, "enable txcsum6 first.\n");
3186                                 error = EAGAIN;
3187                                 goto out;
3188                         }
3189                         ifp->if_capenable ^= IFCAP_TSO6;
3190                         ifp->if_hwassist ^= CSUM_IP6_TSO;
3191                 }
3192                 if (mask & IFCAP_VLAN_HWFILTER) {
3193                         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
3194                                 mlx5e_disable_vlan_filter(priv);
3195                         else
3196                                 mlx5e_enable_vlan_filter(priv);
3197
3198                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
3199                 }
3200                 if (mask & IFCAP_VLAN_HWTAGGING)
3201                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
3202                 if (mask & IFCAP_WOL_MAGIC)
3203                         ifp->if_capenable ^= IFCAP_WOL_MAGIC;
3204
3205                 VLAN_CAPABILITIES(ifp);
3206                 /* turn off LRO means also turn of HW LRO - if it's on */
3207                 if (mask & IFCAP_LRO) {
3208                         int was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
3209                         bool need_restart = false;
3210
3211                         ifp->if_capenable ^= IFCAP_LRO;
3212
3213                         /* figure out if updating HW LRO is needed */
3214                         if (!(ifp->if_capenable & IFCAP_LRO)) {
3215                                 if (priv->params.hw_lro_en) {
3216                                         priv->params.hw_lro_en = false;
3217                                         need_restart = true;
3218                                 }
3219                         } else {
3220                                 if (priv->params.hw_lro_en == false &&
3221                                     priv->params_ethtool.hw_lro != 0) {
3222                                         priv->params.hw_lro_en = true;
3223                                         need_restart = true;
3224                                 }
3225                         }
3226                         if (was_opened && need_restart) {
3227                                 mlx5e_close_locked(ifp);
3228                                 mlx5e_open_locked(ifp);
3229                         }
3230                 }
3231 out:
3232                 PRIV_UNLOCK(priv);
3233                 break;
3234
3235         case SIOCGI2C:
3236                 ifr = (struct ifreq *)data;
3237
3238                 /*
3239                  * Copy from the user-space address ifr_data to the
3240                  * kernel-space address i2c
3241                  */
3242                 error = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c));
3243                 if (error)
3244                         break;
3245
3246                 if (i2c.len > sizeof(i2c.data)) {
3247                         error = EINVAL;
3248                         break;
3249                 }
3250
3251                 PRIV_LOCK(priv);
3252                 /* Get module_num which is required for the query_eeprom */
3253                 error = mlx5_query_module_num(priv->mdev, &module_num);
3254                 if (error) {
3255                         if_printf(ifp, "Query module num failed, eeprom "
3256                             "reading is not supported\n");
3257                         error = EINVAL;
3258                         goto err_i2c;
3259                 }
3260                 /* Check if module is present before doing an access */
3261                 module_status = mlx5_query_module_status(priv->mdev, module_num);
3262                 if (module_status != MLX5_MODULE_STATUS_PLUGGED_ENABLED &&
3263                     module_status != MLX5_MODULE_STATUS_PLUGGED_DISABLED) {
3264                         error = EINVAL;
3265                         goto err_i2c;
3266                 }
3267                 /*
3268                  * Currently 0XA0 and 0xA2 are the only addresses permitted.
3269                  * The internal conversion is as follows:
3270                  */
3271                 if (i2c.dev_addr == 0xA0)
3272                         read_addr = MLX5E_I2C_ADDR_LOW;
3273                 else if (i2c.dev_addr == 0xA2)
3274                         read_addr = MLX5E_I2C_ADDR_HIGH;
3275                 else {
3276                         if_printf(ifp, "Query eeprom failed, "
3277                             "Invalid Address: %X\n", i2c.dev_addr);
3278                         error = EINVAL;
3279                         goto err_i2c;
3280                 }
3281                 error = mlx5_query_eeprom(priv->mdev,
3282                     read_addr, MLX5E_EEPROM_LOW_PAGE,
3283                     (uint32_t)i2c.offset, (uint32_t)i2c.len, module_num,
3284                     (uint32_t *)i2c.data, &size_read);
3285                 if (error) {
3286                         if_printf(ifp, "Query eeprom failed, eeprom "
3287                             "reading is not supported\n");
3288                         error = EINVAL;
3289                         goto err_i2c;
3290                 }
3291
3292                 if (i2c.len > MLX5_EEPROM_MAX_BYTES) {
3293                         error = mlx5_query_eeprom(priv->mdev,
3294                             read_addr, MLX5E_EEPROM_LOW_PAGE,
3295                             (uint32_t)(i2c.offset + size_read),
3296                             (uint32_t)(i2c.len - size_read), module_num,
3297                             (uint32_t *)(i2c.data + size_read), &size_read);
3298                 }
3299                 if (error) {
3300                         if_printf(ifp, "Query eeprom failed, eeprom "
3301                             "reading is not supported\n");
3302                         error = EINVAL;
3303                         goto err_i2c;
3304                 }
3305
3306                 error = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c));
3307 err_i2c:
3308                 PRIV_UNLOCK(priv);
3309                 break;
3310
3311         default:
3312                 error = ether_ioctl(ifp, command, data);
3313                 break;
3314         }
3315         return (error);
3316 }
3317
3318 static int
3319 mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev)
3320 {
3321         /*
3322          * TODO: uncoment once FW really sets all these bits if
3323          * (!mdev->caps.eth.rss_ind_tbl_cap || !mdev->caps.eth.csum_cap ||
3324          * !mdev->caps.eth.max_lso_cap || !mdev->caps.eth.vlan_cap ||
3325          * !(mdev->caps.gen.flags & MLX5_DEV_CAP_FLAG_SCQE_BRK_MOD)) return
3326          * -ENOTSUPP;
3327          */
3328
3329         /* TODO: add more must-to-have features */
3330
3331         if (MLX5_CAP_GEN(mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
3332                 return (-ENODEV);
3333
3334         return (0);
3335 }
3336
3337 static u16
3338 mlx5e_get_max_inline_cap(struct mlx5_core_dev *mdev)
3339 {
3340         uint32_t bf_buf_size = (1U << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2U;
3341
3342         bf_buf_size -= sizeof(struct mlx5e_tx_wqe) - 2;
3343
3344         /* verify against driver hardware limit */
3345         if (bf_buf_size > MLX5E_MAX_TX_INLINE)
3346                 bf_buf_size = MLX5E_MAX_TX_INLINE;
3347
3348         return (bf_buf_size);
3349 }
3350
3351 static int
3352 mlx5e_build_ifp_priv(struct mlx5_core_dev *mdev,
3353     struct mlx5e_priv *priv,
3354     int num_comp_vectors)
3355 {
3356         int err;
3357
3358         /*
3359          * TODO: Consider link speed for setting "log_sq_size",
3360          * "log_rq_size" and "cq_moderation_xxx":
3361          */
3362         priv->params.log_sq_size =
3363             MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
3364         priv->params.log_rq_size =
3365             MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
3366         priv->params.rx_cq_moderation_usec =
3367             MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
3368             MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE :
3369             MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
3370         priv->params.rx_cq_moderation_mode =
3371             MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? 1 : 0;
3372         priv->params.rx_cq_moderation_pkts =
3373             MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
3374         priv->params.tx_cq_moderation_usec =
3375             MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
3376         priv->params.tx_cq_moderation_pkts =
3377             MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
3378         priv->params.min_rx_wqes =
3379             MLX5E_PARAMS_DEFAULT_MIN_RX_WQES;
3380         priv->params.rx_hash_log_tbl_sz =
3381             (order_base_2(num_comp_vectors) >
3382             MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ) ?
3383             order_base_2(num_comp_vectors) :
3384             MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ;
3385         priv->params.num_tc = 1;
3386         priv->params.default_vlan_prio = 0;
3387         priv->counter_set_id = -1;
3388         priv->params.tx_max_inline = mlx5e_get_max_inline_cap(mdev);
3389
3390         err = mlx5_query_min_inline(mdev, &priv->params.tx_min_inline_mode);
3391         if (err)
3392                 return (err);
3393
3394         /*
3395          * hw lro is currently defaulted to off. when it won't anymore we
3396          * will consider the HW capability: "!!MLX5_CAP_ETH(mdev, lro_cap)"
3397          */
3398         priv->params.hw_lro_en = false;
3399         priv->params.lro_wqe_sz = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ;
3400
3401         /*
3402          * CQE zipping is currently defaulted to off. when it won't
3403          * anymore we will consider the HW capability:
3404          * "!!MLX5_CAP_GEN(mdev, cqe_compression)"
3405          */
3406         priv->params.cqe_zipping_en = false;
3407
3408         priv->mdev = mdev;
3409         priv->params.num_channels = num_comp_vectors;
3410         priv->params.channels_rsss = 1;
3411         priv->order_base_2_num_channels = order_base_2(num_comp_vectors);
3412         priv->queue_mapping_channel_mask =
3413             roundup_pow_of_two(num_comp_vectors) - 1;
3414         priv->num_tc = priv->params.num_tc;
3415         priv->default_vlan_prio = priv->params.default_vlan_prio;
3416
3417         INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
3418         INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
3419         INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
3420
3421         return (0);
3422 }
3423
3424 static int
3425 mlx5e_create_mkey(struct mlx5e_priv *priv, u32 pdn,
3426                   struct mlx5_core_mr *mkey)
3427 {
3428         struct ifnet *ifp = priv->ifp;
3429         struct mlx5_core_dev *mdev = priv->mdev;
3430         int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
3431         void *mkc;
3432         u32 *in;
3433         int err;
3434
3435         in = mlx5_vzalloc(inlen);
3436         if (in == NULL) {
3437                 if_printf(ifp, "%s: failed to allocate inbox\n", __func__);
3438                 return (-ENOMEM);
3439         }
3440
3441         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
3442         MLX5_SET(mkc, mkc, access_mode, MLX5_ACCESS_MODE_PA);
3443         MLX5_SET(mkc, mkc, lw, 1);
3444         MLX5_SET(mkc, mkc, lr, 1);
3445
3446         MLX5_SET(mkc, mkc, pd, pdn);
3447         MLX5_SET(mkc, mkc, length64, 1);
3448         MLX5_SET(mkc, mkc, qpn, 0xffffff);
3449
3450         err = mlx5_core_create_mkey(mdev, mkey, in, inlen);
3451         if (err)
3452                 if_printf(ifp, "%s: mlx5_core_create_mkey failed, %d\n",
3453                     __func__, err);
3454
3455         kvfree(in);
3456         return (err);
3457 }
3458
3459 static const char *mlx5e_vport_stats_desc[] = {
3460         MLX5E_VPORT_STATS(MLX5E_STATS_DESC)
3461 };
3462
3463 static const char *mlx5e_pport_stats_desc[] = {
3464         MLX5E_PPORT_STATS(MLX5E_STATS_DESC)
3465 };
3466
3467 static void
3468 mlx5e_priv_mtx_init(struct mlx5e_priv *priv)
3469 {
3470         mtx_init(&priv->async_events_mtx, "mlx5async", MTX_NETWORK_LOCK, MTX_DEF);
3471         sx_init(&priv->state_lock, "mlx5state");
3472         callout_init_mtx(&priv->watchdog, &priv->async_events_mtx, 0);
3473         MLX5_INIT_DOORBELL_LOCK(&priv->doorbell_lock);
3474 }
3475
3476 static void
3477 mlx5e_priv_mtx_destroy(struct mlx5e_priv *priv)
3478 {
3479         mtx_destroy(&priv->async_events_mtx);
3480         sx_destroy(&priv->state_lock);
3481 }
3482
3483 static int
3484 sysctl_firmware(SYSCTL_HANDLER_ARGS)
3485 {
3486         /*
3487          * %d.%d%.d the string format.
3488          * fw_rev_{maj,min,sub} return u16, 2^16 = 65536.
3489          * We need at most 5 chars to store that.
3490          * It also has: two "." and NULL at the end, which means we need 18
3491          * (5*3 + 3) chars at most.
3492          */
3493         char fw[18];
3494         struct mlx5e_priv *priv = arg1;
3495         int error;
3496
3497         snprintf(fw, sizeof(fw), "%d.%d.%d", fw_rev_maj(priv->mdev), fw_rev_min(priv->mdev),
3498             fw_rev_sub(priv->mdev));
3499         error = sysctl_handle_string(oidp, fw, sizeof(fw), req);
3500         return (error);
3501 }
3502
3503 static void
3504 mlx5e_disable_tx_dma(struct mlx5e_channel *ch)
3505 {
3506         int i;
3507
3508         for (i = 0; i < ch->num_tc; i++)
3509                 mlx5e_drain_sq(&ch->sq[i]);
3510 }
3511
3512 static void
3513 mlx5e_reset_sq_doorbell_record(struct mlx5e_sq *sq)
3514 {
3515
3516         sq->doorbell.d32[0] = cpu_to_be32(MLX5_OPCODE_NOP);
3517         sq->doorbell.d32[1] = cpu_to_be32(sq->sqn << 8);
3518         mlx5e_tx_notify_hw(sq, sq->doorbell.d32, 0);
3519         sq->doorbell.d64 = 0;
3520 }
3521
3522 void
3523 mlx5e_resume_sq(struct mlx5e_sq *sq)
3524 {
3525         int err;
3526
3527         /* check if already enabled */
3528         if (READ_ONCE(sq->running) != 0)
3529                 return;
3530
3531         err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_ERR,
3532             MLX5_SQC_STATE_RST);
3533         if (err != 0) {
3534                 if_printf(sq->ifp,
3535                     "mlx5e_modify_sq() from ERR to RST failed: %d\n", err);
3536         }
3537
3538         sq->cc = 0;
3539         sq->pc = 0;
3540
3541         /* reset doorbell prior to moving from RST to RDY */
3542         mlx5e_reset_sq_doorbell_record(sq);
3543
3544         err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST,
3545             MLX5_SQC_STATE_RDY);
3546         if (err != 0) {
3547                 if_printf(sq->ifp,
3548                     "mlx5e_modify_sq() from RST to RDY failed: %d\n", err);
3549         }
3550
3551         sq->cev_next_state = MLX5E_CEV_STATE_INITIAL;
3552         WRITE_ONCE(sq->running, 1);
3553 }
3554
3555 static void
3556 mlx5e_enable_tx_dma(struct mlx5e_channel *ch)
3557 {
3558         int i;
3559
3560         for (i = 0; i < ch->num_tc; i++)
3561                 mlx5e_resume_sq(&ch->sq[i]);
3562 }
3563
3564 static void
3565 mlx5e_disable_rx_dma(struct mlx5e_channel *ch)
3566 {
3567         struct mlx5e_rq *rq = &ch->rq;
3568         int err;
3569
3570         mtx_lock(&rq->mtx);
3571         rq->enabled = 0;
3572         callout_stop(&rq->watchdog);
3573         mtx_unlock(&rq->mtx);
3574
3575         callout_drain(&rq->watchdog);
3576
3577         err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
3578         if (err != 0) {
3579                 if_printf(rq->ifp,
3580                     "mlx5e_modify_rq() from RDY to RST failed: %d\n", err);
3581         }
3582
3583         while (!mlx5_wq_ll_is_empty(&rq->wq)) {
3584                 msleep(1);
3585                 rq->cq.mcq.comp(&rq->cq.mcq);
3586         }
3587
3588         /*
3589          * Transitioning into RST state will allow the FW to track less ERR state queues,
3590          * thus reducing the recv queue flushing time
3591          */
3592         err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_ERR, MLX5_RQC_STATE_RST);
3593         if (err != 0) {
3594                 if_printf(rq->ifp,
3595                     "mlx5e_modify_rq() from ERR to RST failed: %d\n", err);
3596         }
3597 }
3598
3599 static void
3600 mlx5e_enable_rx_dma(struct mlx5e_channel *ch)
3601 {
3602         struct mlx5e_rq *rq = &ch->rq;
3603         int err;
3604
3605         rq->wq.wqe_ctr = 0;
3606         mlx5_wq_ll_update_db_record(&rq->wq);
3607         err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
3608         if (err != 0) {
3609                 if_printf(rq->ifp,
3610                     "mlx5e_modify_rq() from RST to RDY failed: %d\n", err);
3611         }
3612
3613         rq->enabled = 1;
3614
3615         rq->cq.mcq.comp(&rq->cq.mcq);
3616 }
3617
3618 void
3619 mlx5e_modify_tx_dma(struct mlx5e_priv *priv, uint8_t value)
3620 {
3621         int i;
3622
3623         if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
3624                 return;
3625
3626         for (i = 0; i < priv->params.num_channels; i++) {
3627                 if (value)
3628                         mlx5e_disable_tx_dma(&priv->channel[i]);
3629                 else
3630                         mlx5e_enable_tx_dma(&priv->channel[i]);
3631         }
3632 }
3633
3634 void
3635 mlx5e_modify_rx_dma(struct mlx5e_priv *priv, uint8_t value)
3636 {
3637         int i;
3638
3639         if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
3640                 return;
3641
3642         for (i = 0; i < priv->params.num_channels; i++) {
3643                 if (value)
3644                         mlx5e_disable_rx_dma(&priv->channel[i]);
3645                 else
3646                         mlx5e_enable_rx_dma(&priv->channel[i]);
3647         }
3648 }
3649
3650 static void
3651 mlx5e_add_hw_stats(struct mlx5e_priv *priv)
3652 {
3653         SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw),
3654             OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD, priv, 0,
3655             sysctl_firmware, "A", "HCA firmware version");
3656
3657         SYSCTL_ADD_STRING(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw),
3658             OID_AUTO, "board_id", CTLFLAG_RD, priv->mdev->board_id, 0,
3659             "Board ID");
3660 }
3661
3662 static int
3663 mlx5e_sysctl_tx_priority_flow_control(SYSCTL_HANDLER_ARGS)
3664 {
3665         struct mlx5e_priv *priv = arg1;
3666         uint8_t temp[MLX5E_MAX_PRIORITY];
3667         uint32_t tx_pfc;
3668         int err;
3669         int i;
3670
3671         PRIV_LOCK(priv);
3672
3673         tx_pfc = priv->params.tx_priority_flow_control;
3674
3675         for (i = 0; i != MLX5E_MAX_PRIORITY; i++)
3676                 temp[i] = (tx_pfc >> i) & 1;
3677
3678         err = SYSCTL_OUT(req, temp, MLX5E_MAX_PRIORITY);
3679         if (err || !req->newptr)
3680                 goto done;
3681         err = SYSCTL_IN(req, temp, MLX5E_MAX_PRIORITY);
3682         if (err)
3683                 goto done;
3684
3685         priv->params.tx_priority_flow_control = 0;
3686
3687         /* range check input value */
3688         for (i = 0; i != MLX5E_MAX_PRIORITY; i++) {
3689                 if (temp[i] > 1) {
3690                         err = ERANGE;
3691                         goto done;
3692                 }
3693                 priv->params.tx_priority_flow_control |= (temp[i] << i);
3694         }
3695
3696         /* check if update is required */
3697         if (tx_pfc != priv->params.tx_priority_flow_control)
3698                 err = -mlx5e_set_port_pfc(priv);
3699 done:
3700         if (err != 0)
3701                 priv->params.tx_priority_flow_control= tx_pfc;
3702         PRIV_UNLOCK(priv);
3703
3704         return (err);
3705 }
3706
3707 static int
3708 mlx5e_sysctl_rx_priority_flow_control(SYSCTL_HANDLER_ARGS)
3709 {
3710         struct mlx5e_priv *priv = arg1;
3711         uint8_t temp[MLX5E_MAX_PRIORITY];
3712         uint32_t rx_pfc;
3713         int err;
3714         int i;
3715
3716         PRIV_LOCK(priv);
3717
3718         rx_pfc = priv->params.rx_priority_flow_control;
3719
3720         for (i = 0; i != MLX5E_MAX_PRIORITY; i++)
3721                 temp[i] = (rx_pfc >> i) & 1;
3722
3723         err = SYSCTL_OUT(req, temp, MLX5E_MAX_PRIORITY);
3724         if (err || !req->newptr)
3725                 goto done;
3726         err = SYSCTL_IN(req, temp, MLX5E_MAX_PRIORITY);
3727         if (err)
3728                 goto done;
3729
3730         priv->params.rx_priority_flow_control = 0;
3731
3732         /* range check input value */
3733         for (i = 0; i != MLX5E_MAX_PRIORITY; i++) {
3734                 if (temp[i] > 1) {
3735                         err = ERANGE;
3736                         goto done;
3737                 }
3738                 priv->params.rx_priority_flow_control |= (temp[i] << i);
3739         }
3740
3741         /* check if update is required */
3742         if (rx_pfc != priv->params.rx_priority_flow_control)
3743                 err = -mlx5e_set_port_pfc(priv);
3744 done:
3745         if (err != 0)
3746                 priv->params.rx_priority_flow_control= rx_pfc;
3747         PRIV_UNLOCK(priv);
3748
3749         return (err);
3750 }
3751
3752 static void
3753 mlx5e_setup_pauseframes(struct mlx5e_priv *priv)
3754 {
3755 #if (__FreeBSD_version < 1100000)
3756         char path[96];
3757 #endif
3758         int error;
3759
3760         /* enable pauseframes by default */
3761         priv->params.tx_pauseframe_control = 1;
3762         priv->params.rx_pauseframe_control = 1;
3763
3764         /* disable ports flow control, PFC, by default */
3765         priv->params.tx_priority_flow_control = 0;
3766         priv->params.rx_priority_flow_control = 0;
3767
3768 #if (__FreeBSD_version < 1100000)
3769         /* compute path for sysctl */
3770         snprintf(path, sizeof(path), "dev.mce.%d.tx_pauseframe_control",
3771             device_get_unit(priv->mdev->pdev->dev.bsddev));
3772
3773         /* try to fetch tunable, if any */
3774         TUNABLE_INT_FETCH(path, &priv->params.tx_pauseframe_control);
3775
3776         /* compute path for sysctl */
3777         snprintf(path, sizeof(path), "dev.mce.%d.rx_pauseframe_control",
3778             device_get_unit(priv->mdev->pdev->dev.bsddev));
3779
3780         /* try to fetch tunable, if any */
3781         TUNABLE_INT_FETCH(path, &priv->params.rx_pauseframe_control);
3782 #endif
3783
3784         /* register pauseframe SYSCTLs */
3785         SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
3786             OID_AUTO, "tx_pauseframe_control", CTLFLAG_RDTUN,
3787             &priv->params.tx_pauseframe_control, 0,
3788             "Set to enable TX pause frames. Clear to disable.");
3789
3790         SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
3791             OID_AUTO, "rx_pauseframe_control", CTLFLAG_RDTUN,
3792             &priv->params.rx_pauseframe_control, 0,
3793             "Set to enable RX pause frames. Clear to disable.");
3794
3795         /* register priority flow control, PFC, SYSCTLs */
3796         SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
3797             OID_AUTO, "tx_priority_flow_control", CTLTYPE_U8 | CTLFLAG_RWTUN |
3798             CTLFLAG_MPSAFE, priv, 0, &mlx5e_sysctl_tx_priority_flow_control, "CU",
3799             "Set to enable TX ports flow control frames for priorities 0..7. Clear to disable.");
3800
3801         SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
3802             OID_AUTO, "rx_priority_flow_control", CTLTYPE_U8 | CTLFLAG_RWTUN |
3803             CTLFLAG_MPSAFE, priv, 0, &mlx5e_sysctl_rx_priority_flow_control, "CU",
3804             "Set to enable RX ports flow control frames for priorities 0..7. Clear to disable.");
3805
3806         PRIV_LOCK(priv);
3807
3808         /* range check */
3809         priv->params.tx_pauseframe_control =
3810             priv->params.tx_pauseframe_control ? 1 : 0;
3811         priv->params.rx_pauseframe_control =
3812             priv->params.rx_pauseframe_control ? 1 : 0;
3813
3814         /* update firmware */
3815         error = mlx5e_set_port_pause_and_pfc(priv);
3816         if (error == -EINVAL) {
3817                 if_printf(priv->ifp,
3818                     "Global pauseframes must be disabled before enabling PFC.\n");
3819                 priv->params.rx_priority_flow_control = 0;
3820                 priv->params.tx_priority_flow_control = 0;
3821
3822                 /* update firmware */
3823                 (void) mlx5e_set_port_pause_and_pfc(priv);
3824         }
3825         PRIV_UNLOCK(priv);
3826 }
3827
3828 static void *
3829 mlx5e_create_ifp(struct mlx5_core_dev *mdev)
3830 {
3831         struct ifnet *ifp;
3832         struct mlx5e_priv *priv;
3833         u8 dev_addr[ETHER_ADDR_LEN] __aligned(4);
3834         u8 connector_type;
3835         struct sysctl_oid_list *child;
3836         int ncv = mdev->priv.eq_table.num_comp_vectors;
3837         char unit[16];
3838         int err;
3839         int i,j;
3840         u32 eth_proto_cap;
3841         u32 out[MLX5_ST_SZ_DW(ptys_reg)];
3842         bool ext = 0;
3843         u32 speeds_num;
3844         struct media media_entry = {};
3845
3846         if (mlx5e_check_required_hca_cap(mdev)) {
3847                 mlx5_core_dbg(mdev, "mlx5e_check_required_hca_cap() failed\n");
3848                 return (NULL);
3849         }
3850         /*
3851          * Try to allocate the priv and make room for worst-case
3852          * number of channel structures:
3853          */
3854         priv = malloc(sizeof(*priv) +
3855             (sizeof(priv->channel[0]) * mdev->priv.eq_table.num_comp_vectors),
3856             M_MLX5EN, M_WAITOK | M_ZERO);
3857         mlx5e_priv_mtx_init(priv);
3858
3859         ifp = priv->ifp = if_alloc(IFT_ETHER);
3860         if (ifp == NULL) {
3861                 mlx5_core_err(mdev, "if_alloc() failed\n");
3862                 goto err_free_priv;
3863         }
3864         ifp->if_softc = priv;
3865         if_initname(ifp, "mce", device_get_unit(mdev->pdev->dev.bsddev));
3866         ifp->if_mtu = ETHERMTU;
3867         ifp->if_init = mlx5e_open;
3868         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
3869         ifp->if_ioctl = mlx5e_ioctl;
3870         ifp->if_transmit = mlx5e_xmit;
3871         ifp->if_qflush = if_qflush;
3872 #if (__FreeBSD_version >= 1100000)
3873         ifp->if_get_counter = mlx5e_get_counter;
3874 #endif
3875         ifp->if_snd.ifq_maxlen = ifqmaxlen;
3876         /*
3877          * Set driver features
3878          */
3879         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6;
3880         ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING;
3881         ifp->if_capabilities |= IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWFILTER;
3882         ifp->if_capabilities |= IFCAP_LINKSTATE | IFCAP_JUMBO_MTU;
3883         ifp->if_capabilities |= IFCAP_LRO;
3884         ifp->if_capabilities |= IFCAP_TSO | IFCAP_VLAN_HWTSO;
3885         ifp->if_capabilities |= IFCAP_HWSTATS;
3886
3887         /* set TSO limits so that we don't have to drop TX packets */
3888         ifp->if_hw_tsomax = MLX5E_MAX_TX_PAYLOAD_SIZE - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
3889         ifp->if_hw_tsomaxsegcount = MLX5E_MAX_TX_MBUF_FRAGS - 1 /* hdr */;
3890         ifp->if_hw_tsomaxsegsize = MLX5E_MAX_TX_MBUF_SIZE;
3891
3892         ifp->if_capenable = ifp->if_capabilities;
3893         ifp->if_hwassist = 0;
3894         if (ifp->if_capenable & IFCAP_TSO)
3895                 ifp->if_hwassist |= CSUM_TSO;
3896         if (ifp->if_capenable & IFCAP_TXCSUM)
3897                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP);
3898         if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
3899                 ifp->if_hwassist |= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
3900
3901         /* ifnet sysctl tree */
3902         sysctl_ctx_init(&priv->sysctl_ctx);
3903         priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_STATIC_CHILDREN(_dev),
3904             OID_AUTO, ifp->if_dname, CTLFLAG_RD, 0, "MLX5 ethernet - interface name");
3905         if (priv->sysctl_ifnet == NULL) {
3906                 mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
3907                 goto err_free_sysctl;
3908         }
3909         snprintf(unit, sizeof(unit), "%d", ifp->if_dunit);
3910         priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
3911             OID_AUTO, unit, CTLFLAG_RD, 0, "MLX5 ethernet - interface unit");
3912         if (priv->sysctl_ifnet == NULL) {
3913                 mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
3914                 goto err_free_sysctl;
3915         }
3916
3917         /* HW sysctl tree */
3918         child = SYSCTL_CHILDREN(device_get_sysctl_tree(mdev->pdev->dev.bsddev));
3919         priv->sysctl_hw = SYSCTL_ADD_NODE(&priv->sysctl_ctx, child,
3920             OID_AUTO, "hw", CTLFLAG_RD, 0, "MLX5 ethernet dev hw");
3921         if (priv->sysctl_hw == NULL) {
3922                 mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
3923                 goto err_free_sysctl;
3924         }
3925
3926         err = mlx5e_build_ifp_priv(mdev, priv, ncv);
3927         if (err) {
3928                 mlx5_core_err(mdev, "mlx5e_build_ifp_priv() failed (%d)\n", err);
3929                 goto err_free_sysctl;
3930         }
3931
3932         /* reuse mlx5core's watchdog workqueue */
3933         priv->wq = mdev->priv.health.wq_watchdog;
3934
3935         err = mlx5_alloc_map_uar(mdev, &priv->cq_uar);
3936         if (err) {
3937                 if_printf(ifp, "%s: mlx5_alloc_map_uar failed, %d\n",
3938                     __func__, err);
3939                 goto err_free_wq;
3940         }
3941         err = mlx5_core_alloc_pd(mdev, &priv->pdn);
3942         if (err) {
3943                 if_printf(ifp, "%s: mlx5_core_alloc_pd failed, %d\n",
3944                     __func__, err);
3945                 goto err_unmap_free_uar;
3946         }
3947         err = mlx5_alloc_transport_domain(mdev, &priv->tdn);
3948         if (err) {
3949                 if_printf(ifp, "%s: mlx5_alloc_transport_domain failed, %d\n",
3950                     __func__, err);
3951                 goto err_dealloc_pd;
3952         }
3953         err = mlx5e_create_mkey(priv, priv->pdn, &priv->mr);
3954         if (err) {
3955                 if_printf(ifp, "%s: mlx5e_create_mkey failed, %d\n",
3956                     __func__, err);
3957                 goto err_dealloc_transport_domain;
3958         }
3959         mlx5_query_nic_vport_mac_address(priv->mdev, 0, dev_addr);
3960
3961         /* check if we should generate a random MAC address */
3962         if (MLX5_CAP_GEN(priv->mdev, vport_group_manager) == 0 &&
3963             is_zero_ether_addr(dev_addr)) {
3964                 random_ether_addr(dev_addr);
3965                 if_printf(ifp, "Assigned random MAC address\n");
3966         }
3967
3968         /* set default MTU */
3969         mlx5e_set_dev_port_mtu(ifp, ifp->if_mtu);
3970
3971         /* Set default media status */
3972         priv->media_status_last = IFM_AVALID;
3973         priv->media_active_last = IFM_ETHER | IFM_AUTO |
3974             IFM_ETH_RXPAUSE | IFM_FDX;
3975
3976         /* setup default pauseframes configuration */
3977         mlx5e_setup_pauseframes(priv);
3978
3979         /* Setup supported medias */
3980         //TODO: If we failed to query ptys is it ok to proceed??
3981         if (!mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN, 1)) {
3982                 ext = MLX5_CAP_PCAM_FEATURE(mdev,
3983                     ptys_extended_ethernet);
3984                 eth_proto_cap = MLX5_GET_ETH_PROTO(ptys_reg, out, ext,
3985                     eth_proto_capability);
3986                 if (MLX5_CAP_PCAM_FEATURE(mdev, ptys_connector_type))
3987                         connector_type = MLX5_GET(ptys_reg, out,
3988                             connector_type);
3989         } else {
3990                 eth_proto_cap = 0;
3991                 if_printf(ifp, "%s: Query port media capability failed,"
3992                     " %d\n", __func__, err);
3993         }
3994
3995         ifmedia_init(&priv->media, IFM_IMASK | IFM_ETH_FMASK,
3996             mlx5e_media_change, mlx5e_media_status);
3997
3998         speeds_num = ext ? MLX5E_EXT_LINK_SPEEDS_NUMBER : MLX5E_LINK_SPEEDS_NUMBER;
3999         for (i = 0; i != speeds_num; i++) {
4000                 for (j = 0; j < MLX5E_LINK_MODES_NUMBER ; ++j) {
4001                         media_entry = ext ? mlx5e_ext_mode_table[i][j] :
4002                             mlx5e_mode_table[i][j];
4003                         if (media_entry.baudrate == 0)
4004                                 continue;
4005                         if (MLX5E_PROT_MASK(i) & eth_proto_cap) {
4006                                 ifmedia_add(&priv->media,
4007                                     media_entry.subtype |
4008                                     IFM_ETHER, 0, NULL);
4009                                 ifmedia_add(&priv->media,
4010                                     media_entry.subtype |
4011                                     IFM_ETHER | IFM_FDX |
4012                                     IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE, 0, NULL);
4013                         }
4014                 }
4015         }
4016
4017         ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO, 0, NULL);
4018         ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO | IFM_FDX |
4019             IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE, 0, NULL);
4020
4021         /* Set autoselect by default */
4022         ifmedia_set(&priv->media, IFM_ETHER | IFM_AUTO | IFM_FDX |
4023             IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE);
4024         ether_ifattach(ifp, dev_addr);
4025
4026         /* Register for VLAN events */
4027         priv->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
4028             mlx5e_vlan_rx_add_vid, priv, EVENTHANDLER_PRI_FIRST);
4029         priv->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
4030             mlx5e_vlan_rx_kill_vid, priv, EVENTHANDLER_PRI_FIRST);
4031
4032         /* Link is down by default */
4033         if_link_state_change(ifp, LINK_STATE_DOWN);
4034
4035         mlx5e_enable_async_events(priv);
4036
4037         mlx5e_add_hw_stats(priv);
4038
4039         mlx5e_create_stats(&priv->stats.vport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4040             "vstats", mlx5e_vport_stats_desc, MLX5E_VPORT_STATS_NUM,
4041             priv->stats.vport.arg);
4042
4043         mlx5e_create_stats(&priv->stats.pport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4044             "pstats", mlx5e_pport_stats_desc, MLX5E_PPORT_STATS_NUM,
4045             priv->stats.pport.arg);
4046
4047         mlx5e_create_ethtool(priv);
4048
4049         mtx_lock(&priv->async_events_mtx);
4050         mlx5e_update_stats(priv);
4051         mtx_unlock(&priv->async_events_mtx);
4052
4053         return (priv);
4054
4055 err_dealloc_transport_domain:
4056         mlx5_dealloc_transport_domain(mdev, priv->tdn);
4057
4058 err_dealloc_pd:
4059         mlx5_core_dealloc_pd(mdev, priv->pdn);
4060
4061 err_unmap_free_uar:
4062         mlx5_unmap_free_uar(mdev, &priv->cq_uar);
4063
4064 err_free_wq:
4065         flush_workqueue(priv->wq);
4066
4067 err_free_sysctl:
4068         sysctl_ctx_free(&priv->sysctl_ctx);
4069         if (priv->sysctl_debug)
4070                 sysctl_ctx_free(&priv->stats.port_stats_debug.ctx);
4071         if_free(ifp);
4072
4073 err_free_priv:
4074         mlx5e_priv_mtx_destroy(priv);
4075         free(priv, M_MLX5EN);
4076         return (NULL);
4077 }
4078
4079 static void
4080 mlx5e_destroy_ifp(struct mlx5_core_dev *mdev, void *vpriv)
4081 {
4082         struct mlx5e_priv *priv = vpriv;
4083         struct ifnet *ifp = priv->ifp;
4084
4085         /* don't allow more IOCTLs */
4086         priv->gone = 1;
4087
4088         /* XXX wait a bit to allow IOCTL handlers to complete */
4089         pause("W", hz);
4090
4091         /* stop watchdog timer */
4092         callout_drain(&priv->watchdog);
4093
4094         if (priv->vlan_attach != NULL)
4095                 EVENTHANDLER_DEREGISTER(vlan_config, priv->vlan_attach);
4096         if (priv->vlan_detach != NULL)
4097                 EVENTHANDLER_DEREGISTER(vlan_unconfig, priv->vlan_detach);
4098
4099         /* make sure device gets closed */
4100         PRIV_LOCK(priv);
4101         mlx5e_close_locked(ifp);
4102         PRIV_UNLOCK(priv);
4103
4104         /* unregister device */
4105         ifmedia_removeall(&priv->media);
4106         ether_ifdetach(ifp);
4107         if_free(ifp);
4108
4109         /* destroy all remaining sysctl nodes */
4110         sysctl_ctx_free(&priv->stats.vport.ctx);
4111         sysctl_ctx_free(&priv->stats.pport.ctx);
4112         if (priv->sysctl_debug)
4113                 sysctl_ctx_free(&priv->stats.port_stats_debug.ctx);
4114         sysctl_ctx_free(&priv->sysctl_ctx);
4115
4116         mlx5_core_destroy_mkey(priv->mdev, &priv->mr);
4117         mlx5_dealloc_transport_domain(priv->mdev, priv->tdn);
4118         mlx5_core_dealloc_pd(priv->mdev, priv->pdn);
4119         mlx5_unmap_free_uar(priv->mdev, &priv->cq_uar);
4120         mlx5e_disable_async_events(priv);
4121         flush_workqueue(priv->wq);
4122         mlx5e_priv_mtx_destroy(priv);
4123         free(priv, M_MLX5EN);
4124 }
4125
4126 static void *
4127 mlx5e_get_ifp(void *vpriv)
4128 {
4129         struct mlx5e_priv *priv = vpriv;
4130
4131         return (priv->ifp);
4132 }
4133
4134 static struct mlx5_interface mlx5e_interface = {
4135         .add = mlx5e_create_ifp,
4136         .remove = mlx5e_destroy_ifp,
4137         .event = mlx5e_async_event,
4138         .protocol = MLX5_INTERFACE_PROTOCOL_ETH,
4139         .get_dev = mlx5e_get_ifp,
4140 };
4141
4142 void
4143 mlx5e_init(void)
4144 {
4145         mlx5_register_interface(&mlx5e_interface);
4146 }
4147
4148 void
4149 mlx5e_cleanup(void)
4150 {
4151         mlx5_unregister_interface(&mlx5e_interface);
4152 }
4153
4154 static void
4155 mlx5e_show_version(void __unused *arg)
4156 {
4157
4158         printf("%s", mlx5e_version);
4159 }
4160 SYSINIT(mlx5e_show_version, SI_SUB_DRIVERS, SI_ORDER_ANY, mlx5e_show_version, NULL);
4161
4162 module_init_order(mlx5e_init, SI_ORDER_THIRD);
4163 module_exit_order(mlx5e_cleanup, SI_ORDER_THIRD);
4164
4165 #if (__FreeBSD_version >= 1100000)
4166 MODULE_DEPEND(mlx5en, linuxkpi, 1, 1, 1);
4167 #endif
4168 MODULE_DEPEND(mlx5en, mlx5, 1, 1, 1);
4169 MODULE_VERSION(mlx5en, 1);