sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c

   1 /*-
   2  * Copyright (c) 2015-2021 Mellanox Technologies. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
  14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  16  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
  17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  23  * SUCH DAMAGE.
  24  */
  25
  26 #include "opt_rss.h"
  27 #include "opt_ratelimit.h"
  28
  29 #include <dev/mlx5/mlx5_en/en.h>
  30
  31 #include <linux/list.h>
  32 #include <dev/mlx5/fs.h>
  33 #include <dev/mlx5/mpfs.h>
  34 #include <dev/mlx5/mlx5_core/fs_tcp.h>
  35
  36 /*
  37  * The flow tables with rules define the packet processing on receive.
  38  * Currently the following structure is set up to handle different
  39  * offloads like TLS RX offload, VLAN decapsulation, packet
  40  * classification, RSS hashing, VxLAN checksum offloading:
  41  *
  42  *   +=========+       +=========+      +=================+
  43  *   |TCP/IPv4 |       |TCP/IPv4 |      |TCP/IPv4 Match   |
  44  *   |Flowtable|------>|         |----->|Outer Proto Match|=====> TLS TIR n
  45  *   |         |       |Catch-all|\     |                 |
  46  *   +=========+       +=========+|     +=================+
  47  *                                |
  48  *       +------------------------+
  49  *       V
  50  *   +=========+       +=========+      +=================+
  51  *   |TCP/IPv6 |       |TCP/IPv6 |      |TCP/IPv6 Match   |
  52  *   |Flowtable|------>|         |----->|Outer Proto Match|=====> TLS TIR n
  53  *   |         |       |Catch-all|\     |                 |
  54  *   +=========+       +=========+|     +=================+
  55  *                                |
  56  *       +------------------------+
  57  *       V
  58  *   +=========+       +=========+      +=================+
  59  *   |VLAN ft: |       |VxLAN    |      |VxLAN Main       |
  60  *   |CTAG/STAG|------>|      VNI|----->|Inner Proto Match|=====> Inner TIR n
  61  *   |VID/noVID|/      |Catch-all|\     |                 |
  62  *   +=========+       +=========+|     +=================+
  63  *                                |
  64  *                                |
  65  *                                |
  66  *                                v
  67  *                      +=================+
  68  *                      |Main             |
  69  *                      |Outer Proto Match|=====> TIR n
  70  *                      |                 |
  71  *                      +=================+
  72  *
  73  * The path through flow rules directs each packet into an appropriate TIR,
  74  * according to the:
  75  * - VLAN encapsulation
  76  * - Outer protocol
  77  * - Presence of inner protocol
  78  */
  79
  80 #define MLX5_SET_CFG(p, f, v) MLX5_SET(create_flow_group_in, p, f, v)
  81
  82 enum {
  83         MLX5E_FULLMATCH = 0,
  84         MLX5E_ALLMULTI = 1,
  85         MLX5E_PROMISC = 2,
  86 };
  87
  88 enum {
  89         MLX5E_UC = 0,
  90         MLX5E_MC_IPV4 = 1,
  91         MLX5E_MC_IPV6 = 2,
  92         MLX5E_MC_OTHER = 3,
  93 };
  94
  95 enum {
  96         MLX5E_ACTION_NONE = 0,
  97         MLX5E_ACTION_ADD = 1,
  98         MLX5E_ACTION_DEL = 2,
  99 };
 100
 101 struct mlx5e_eth_addr_hash_node {
 102         LIST_ENTRY(mlx5e_eth_addr_hash_node) hlist;
 103         u8      action;
 104         u32     mpfs_index;
 105         struct mlx5e_eth_addr_info ai;
 106 };
 107
 108 static void mlx5e_del_all_vlan_rules(struct mlx5e_priv *);
 109
 110 static inline int
 111 mlx5e_hash_eth_addr(const u8 * addr)
 112 {
 113         return (addr[5]);
 114 }
 115
 116 static bool
 117 mlx5e_add_eth_addr_to_hash(struct mlx5e_eth_addr_hash_head *hash,
 118     struct mlx5e_eth_addr_hash_node *hn_new)
 119 {
 120         struct mlx5e_eth_addr_hash_node *hn;
 121         u32 ix = mlx5e_hash_eth_addr(hn_new->ai.addr);
 122
 123         LIST_FOREACH(hn, &hash[ix], hlist) {
 124                 if (bcmp(hn->ai.addr, hn_new->ai.addr, ETHER_ADDR_LEN) == 0) {
 125                         if (hn->action == MLX5E_ACTION_DEL)
 126                                 hn->action = MLX5E_ACTION_NONE;
 127                         free(hn_new, M_MLX5EN);
 128                         return (false);
 129                 }
 130         }
 131         LIST_INSERT_HEAD(&hash[ix], hn_new, hlist);
 132         return (true);
 133 }
 134
 135 static void
 136 mlx5e_del_eth_addr_from_hash(struct mlx5e_eth_addr_hash_node *hn)
 137 {
 138         LIST_REMOVE(hn, hlist);
 139         free(hn, M_MLX5EN);
 140 }
 141
 142 static void
 143 mlx5e_del_eth_addr_from_flow_table(struct mlx5e_priv *priv,
 144     struct mlx5e_eth_addr_info *ai)
 145 {
 146         mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_IPSEC_ESP]);
 147         mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_IPSEC_ESP]);
 148         mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_IPSEC_AH]);
 149         mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_IPSEC_AH]);
 150         mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_TCP]);
 151         mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_TCP]);
 152         mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_UDP]);
 153         mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_UDP]);
 154         mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6]);
 155         mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4]);
 156         mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_ANY]);
 157 }
 158
 159 static int
 160 mlx5e_get_eth_addr_type(const u8 * addr)
 161 {
 162         if (ETHER_IS_MULTICAST(addr) == 0)
 163                 return (MLX5E_UC);
 164
 165         if ((addr[0] == 0x01) &&
 166             (addr[1] == 0x00) &&
 167             (addr[2] == 0x5e) &&
 168             !(addr[3] & 0x80))
 169                 return (MLX5E_MC_IPV4);
 170
 171         if ((addr[0] == 0x33) &&
 172             (addr[1] == 0x33))
 173                 return (MLX5E_MC_IPV6);
 174
 175         return (MLX5E_MC_OTHER);
 176 }
 177
 178 static  u32
 179 mlx5e_get_tt_vec(struct mlx5e_eth_addr_info *ai, int type)
 180 {
 181         int eth_addr_type;
 182         u32 ret;
 183
 184         switch (type) {
 185         case MLX5E_FULLMATCH:
 186                 eth_addr_type = mlx5e_get_eth_addr_type(ai->addr);
 187                 switch (eth_addr_type) {
 188                 case MLX5E_UC:
 189                         ret =
 190                             (1 << MLX5E_TT_IPV4_TCP) |
 191                             (1 << MLX5E_TT_IPV6_TCP) |
 192                             (1 << MLX5E_TT_IPV4_UDP) |
 193                             (1 << MLX5E_TT_IPV6_UDP) |
 194                             (1 << MLX5E_TT_IPV4) |
 195                             (1 << MLX5E_TT_IPV6) |
 196                             (1 << MLX5E_TT_ANY) |
 197                             0;
 198                         break;
 199
 200                 case MLX5E_MC_IPV4:
 201                         ret =
 202                             (1 << MLX5E_TT_IPV4_UDP) |
 203                             (1 << MLX5E_TT_IPV4) |
 204                             0;
 205                         break;
 206
 207                 case MLX5E_MC_IPV6:
 208                         ret =
 209                             (1 << MLX5E_TT_IPV6_UDP) |
 210                             (1 << MLX5E_TT_IPV6) |
 211                             0;
 212                         break;
 213
 214                 default:
 215                         ret =
 216                             (1 << MLX5E_TT_ANY) |
 217                             0;
 218                         break;
 219                 }
 220                 break;
 221
 222         case MLX5E_ALLMULTI:
 223                 ret =
 224                     (1 << MLX5E_TT_IPV4_UDP) |
 225                     (1 << MLX5E_TT_IPV6_UDP) |
 226                     (1 << MLX5E_TT_IPV4) |
 227                     (1 << MLX5E_TT_IPV6) |
 228                     (1 << MLX5E_TT_ANY) |
 229                     0;
 230                 break;
 231
 232         default:                        /* MLX5E_PROMISC */
 233                 ret =
 234                     (1 << MLX5E_TT_IPV4_TCP) |
 235                     (1 << MLX5E_TT_IPV6_TCP) |
 236                     (1 << MLX5E_TT_IPV4_UDP) |
 237                     (1 << MLX5E_TT_IPV6_UDP) |
 238                     (1 << MLX5E_TT_IPV4) |
 239                     (1 << MLX5E_TT_IPV6) |
 240                     (1 << MLX5E_TT_ANY) |
 241                     0;
 242                 break;
 243         }
 244
 245         return (ret);
 246 }
 247
 248 static int
 249 mlx5e_add_eth_addr_rule_sub(struct mlx5e_priv *priv,
 250     struct mlx5e_eth_addr_info *ai, int type,
 251     u32 *mc, u32 *mv)
 252 {
 253         struct mlx5_flow_destination dest = {};
 254         u8 mc_enable = 0;
 255         struct mlx5_flow_rule **rule_p;
 256         struct mlx5_flow_table *ft = priv->fts.main.t;
 257         u8 *mc_dmac = MLX5_ADDR_OF(fte_match_param, mc,
 258                                    outer_headers.dmac_47_16);
 259         u8 *mv_dmac = MLX5_ADDR_OF(fte_match_param, mv,
 260                                    outer_headers.dmac_47_16);
 261         u32 *tirn = priv->tirn;
 262         u32 tt_vec;
 263         int err = 0;
 264
 265         dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
 266
 267         switch (type) {
 268         case MLX5E_FULLMATCH:
 269                 mc_enable = MLX5_MATCH_OUTER_HEADERS;
 270                 memset(mc_dmac, 0xff, ETH_ALEN);
 271                 ether_addr_copy(mv_dmac, ai->addr);
 272                 break;
 273
 274         case MLX5E_ALLMULTI:
 275                 mc_enable = MLX5_MATCH_OUTER_HEADERS;
 276                 mc_dmac[0] = 0x01;
 277                 mv_dmac[0] = 0x01;
 278                 break;
 279
 280         case MLX5E_PROMISC:
 281                 break;
 282         default:
 283                 break;
 284         }
 285
 286         tt_vec = mlx5e_get_tt_vec(ai, type);
 287
 288         if (tt_vec & BIT(MLX5E_TT_ANY)) {
 289                 rule_p = &ai->ft_rule[MLX5E_TT_ANY];
 290                 dest.tir_num = tirn[MLX5E_TT_ANY];
 291                 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 292                                              MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
 293                                              MLX5_FS_ETH_FLOW_TAG, &dest);
 294                 if (IS_ERR_OR_NULL(*rule_p))
 295                         goto err_del_ai;
 296         }
 297
 298         mc_enable = MLX5_MATCH_OUTER_HEADERS;
 299         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
 300
 301         if (tt_vec & BIT(MLX5E_TT_IPV4)) {
 302                 rule_p = &ai->ft_rule[MLX5E_TT_IPV4];
 303                 dest.tir_num = tirn[MLX5E_TT_IPV4];
 304                 MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 305                          ETHERTYPE_IP);
 306                 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 307                                              MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
 308                                              MLX5_FS_ETH_FLOW_TAG, &dest);
 309                 if (IS_ERR_OR_NULL(*rule_p))
 310                         goto err_del_ai;
 311         }
 312
 313         if (tt_vec & BIT(MLX5E_TT_IPV6)) {
 314                 rule_p = &ai->ft_rule[MLX5E_TT_IPV6];
 315                 dest.tir_num = tirn[MLX5E_TT_IPV6];
 316                 MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 317                          ETHERTYPE_IPV6);
 318                 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 319                                              MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
 320                                              MLX5_FS_ETH_FLOW_TAG, &dest);
 321                 if (IS_ERR_OR_NULL(*rule_p))
 322                         goto err_del_ai;
 323         }
 324
 325         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol);
 326         MLX5_SET(fte_match_param, mv, outer_headers.ip_protocol, IPPROTO_UDP);
 327
 328         if (tt_vec & BIT(MLX5E_TT_IPV4_UDP)) {
 329                 rule_p = &ai->ft_rule[MLX5E_TT_IPV4_UDP];
 330                 dest.tir_num = tirn[MLX5E_TT_IPV4_UDP];
 331                 MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 332                          ETHERTYPE_IP);
 333                 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 334                                              MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
 335                                              MLX5_FS_ETH_FLOW_TAG, &dest);
 336                 if (IS_ERR_OR_NULL(*rule_p))
 337                         goto err_del_ai;
 338         }
 339
 340         if (tt_vec & BIT(MLX5E_TT_IPV6_UDP)) {
 341                 rule_p = &ai->ft_rule[MLX5E_TT_IPV6_UDP];
 342                 dest.tir_num = tirn[MLX5E_TT_IPV6_UDP];
 343                 MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 344                          ETHERTYPE_IPV6);
 345                 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 346                                              MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
 347                                              MLX5_FS_ETH_FLOW_TAG, &dest);
 348                 if (IS_ERR_OR_NULL(*rule_p))
 349                         goto err_del_ai;
 350         }
 351
 352         MLX5_SET(fte_match_param, mv, outer_headers.ip_protocol, IPPROTO_TCP);
 353
 354         if (tt_vec & BIT(MLX5E_TT_IPV4_TCP)) {
 355                 rule_p = &ai->ft_rule[MLX5E_TT_IPV4_TCP];
 356                 dest.tir_num = tirn[MLX5E_TT_IPV4_TCP];
 357                 MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 358                          ETHERTYPE_IP);
 359                 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 360                                              MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
 361                                              MLX5_FS_ETH_FLOW_TAG, &dest);
 362                 if (IS_ERR_OR_NULL(*rule_p))
 363                         goto err_del_ai;
 364         }
 365
 366         if (tt_vec & BIT(MLX5E_TT_IPV6_TCP)) {
 367                 rule_p = &ai->ft_rule[MLX5E_TT_IPV6_TCP];
 368                 dest.tir_num = tirn[MLX5E_TT_IPV6_TCP];
 369                 MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 370                          ETHERTYPE_IPV6);
 371                 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 372                                              MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
 373                                              MLX5_FS_ETH_FLOW_TAG, &dest);
 374                 if (IS_ERR_OR_NULL(*rule_p))
 375                         goto err_del_ai;
 376         }
 377
 378         MLX5_SET(fte_match_param, mv, outer_headers.ip_protocol, IPPROTO_AH);
 379
 380         if (tt_vec & BIT(MLX5E_TT_IPV4_IPSEC_AH)) {
 381                 rule_p = &ai->ft_rule[MLX5E_TT_IPV4_IPSEC_AH];
 382                 dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_AH];
 383                 MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 384                          ETHERTYPE_IP);
 385                 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 386                                              MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
 387                                              MLX5_FS_ETH_FLOW_TAG, &dest);
 388                 if (IS_ERR_OR_NULL(*rule_p))
 389                         goto err_del_ai;
 390         }
 391
 392         if (tt_vec & BIT(MLX5E_TT_IPV6_IPSEC_AH)) {
 393                 rule_p = &ai->ft_rule[MLX5E_TT_IPV6_IPSEC_AH];
 394                 dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_AH];
 395                 MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 396                          ETHERTYPE_IPV6);
 397                 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 398                                              MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
 399                                              MLX5_FS_ETH_FLOW_TAG, &dest);
 400                 if (IS_ERR_OR_NULL(*rule_p))
 401                         goto err_del_ai;
 402         }
 403
 404         MLX5_SET(fte_match_param, mv, outer_headers.ip_protocol, IPPROTO_ESP);
 405
 406         if (tt_vec & BIT(MLX5E_TT_IPV4_IPSEC_ESP)) {
 407                 rule_p = &ai->ft_rule[MLX5E_TT_IPV4_IPSEC_ESP];
 408                 dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_ESP];
 409                 MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 410                          ETHERTYPE_IP);
 411                 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 412                                              MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
 413                                              MLX5_FS_ETH_FLOW_TAG, &dest);
 414                 if (IS_ERR_OR_NULL(*rule_p))
 415                         goto err_del_ai;
 416         }
 417
 418         if (tt_vec & BIT(MLX5E_TT_IPV6_IPSEC_ESP)) {
 419                 rule_p = &ai->ft_rule[MLX5E_TT_IPV6_IPSEC_ESP];
 420                 dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_ESP];
 421                 MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 422                          ETHERTYPE_IPV6);
 423                 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 424                                              MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
 425                                              MLX5_FS_ETH_FLOW_TAG, &dest);
 426                 if (IS_ERR_OR_NULL(*rule_p))
 427                         goto err_del_ai;
 428         }
 429
 430         return 0;
 431
 432 err_del_ai:
 433         err = PTR_ERR(*rule_p);
 434         *rule_p = NULL;
 435         mlx5e_del_eth_addr_from_flow_table(priv, ai);
 436
 437         return err;
 438 }
 439
 440 static int
 441 mlx5e_add_eth_addr_rule(struct mlx5e_priv *priv,
 442     struct mlx5e_eth_addr_info *ai, int type)
 443 {
 444         u32 *match_criteria;
 445         u32 *match_value;
 446         int err = 0;
 447
 448         match_value     = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
 449         match_criteria  = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
 450         if (!match_value || !match_criteria) {
 451                 mlx5_en_err(priv->ifp, "alloc failed\n");
 452                 err = -ENOMEM;
 453                 goto add_eth_addr_rule_out;
 454         }
 455         err = mlx5e_add_eth_addr_rule_sub(priv, ai, type, match_criteria,
 456             match_value);
 457
 458 add_eth_addr_rule_out:
 459         kvfree(match_criteria);
 460         kvfree(match_value);
 461
 462         return (err);
 463 }
 464
 465 static void
 466 mlx5e_del_main_vxlan_rules(struct mlx5e_priv *priv)
 467 {
 468         mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_ESP]);
 469         mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_ESP]);
 470         mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_AH]);
 471         mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_AH]);
 472         mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_TCP]);
 473         mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_TCP]);
 474         mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_UDP]);
 475         mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_UDP]);
 476         mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6]);
 477         mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4]);
 478         mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_ANY]);
 479 }
 480
 481 static int
 482 mlx5e_add_main_vxlan_rules_sub(struct mlx5e_priv *priv, u32 *mc, u32 *mv)
 483 {
 484         struct mlx5_flow_destination dest = {};
 485         u8 mc_enable = 0;
 486         struct mlx5_flow_rule **rule_p;
 487         struct mlx5_flow_table *ft = priv->fts.main_vxlan.t;
 488         u32 *tirn = priv->tirn_inner_vxlan;
 489         int err = 0;
 490
 491         dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
 492
 493         mc_enable = MLX5_MATCH_INNER_HEADERS;
 494         MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ethertype);
 495
 496         rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4];
 497         dest.tir_num = tirn[MLX5E_TT_IPV4];
 498         MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
 499         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 500             MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, MLX5_FS_ETH_FLOW_TAG, &dest);
 501         if (IS_ERR_OR_NULL(*rule_p))
 502                 goto err_del_ai;
 503
 504         rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6];
 505         dest.tir_num = tirn[MLX5E_TT_IPV6];
 506         MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6);
 507         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 508              MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, MLX5_FS_ETH_FLOW_TAG, &dest);
 509         if (IS_ERR_OR_NULL(*rule_p))
 510                 goto err_del_ai;
 511
 512         MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ip_protocol);
 513         MLX5_SET(fte_match_param, mv, inner_headers.ip_protocol, IPPROTO_UDP);
 514
 515         rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_UDP];
 516         dest.tir_num = tirn[MLX5E_TT_IPV4_UDP];
 517         MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
 518         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 519             MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, MLX5_FS_ETH_FLOW_TAG, &dest);
 520         if (IS_ERR_OR_NULL(*rule_p))
 521                 goto err_del_ai;
 522
 523         rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_UDP];
 524         dest.tir_num = tirn[MLX5E_TT_IPV6_UDP];
 525         MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6);
 526         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 527             MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, MLX5_FS_ETH_FLOW_TAG, &dest);
 528         if (IS_ERR_OR_NULL(*rule_p))
 529                 goto err_del_ai;
 530
 531         MLX5_SET(fte_match_param, mv, inner_headers.ip_protocol, IPPROTO_TCP);
 532
 533         rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_TCP];
 534         dest.tir_num = tirn[MLX5E_TT_IPV4_TCP];
 535         MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
 536         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 537             MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, MLX5_FS_ETH_FLOW_TAG, &dest);
 538         if (IS_ERR_OR_NULL(*rule_p))
 539                 goto err_del_ai;
 540
 541         rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_TCP];
 542         dest.tir_num = tirn[MLX5E_TT_IPV6_TCP];
 543         MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6);
 544         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 545             MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, MLX5_FS_ETH_FLOW_TAG, &dest);
 546         if (IS_ERR_OR_NULL(*rule_p))
 547                 goto err_del_ai;
 548
 549         MLX5_SET(fte_match_param, mv, inner_headers.ip_protocol, IPPROTO_AH);
 550
 551         rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_AH];
 552         dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_AH];
 553         MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
 554         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 555             MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, MLX5_FS_ETH_FLOW_TAG, &dest);
 556         if (IS_ERR_OR_NULL(*rule_p))
 557                 goto err_del_ai;
 558
 559         rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_AH];
 560         dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_AH];
 561         MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6);
 562         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 563             MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, MLX5_FS_ETH_FLOW_TAG, &dest);
 564         if (IS_ERR_OR_NULL(*rule_p))
 565                 goto err_del_ai;
 566
 567         MLX5_SET(fte_match_param, mv, inner_headers.ip_protocol, IPPROTO_ESP);
 568
 569         rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_ESP];
 570         dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_ESP];
 571         MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
 572         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 573             MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, MLX5_FS_ETH_FLOW_TAG, &dest);
 574         if (IS_ERR_OR_NULL(*rule_p))
 575                         goto err_del_ai;
 576
 577         rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_ESP];
 578         dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_ESP];
 579         MLX5_SET(fte_match_param, mv, inner_headers.ethertype,
 580                  ETHERTYPE_IPV6);
 581         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 582             MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, MLX5_FS_ETH_FLOW_TAG, &dest);
 583         if (IS_ERR_OR_NULL(*rule_p))
 584                 goto err_del_ai;
 585
 586         mc_enable = 0;
 587         memset(mv, 0, MLX5_ST_SZ_BYTES(fte_match_param));
 588         memset(mc, 0, MLX5_ST_SZ_BYTES(fte_match_param));
 589         rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_ANY];
 590         dest.tir_num = tirn[MLX5E_TT_ANY];
 591         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 592             MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, MLX5_FS_ETH_FLOW_TAG, &dest);
 593         if (IS_ERR_OR_NULL(*rule_p))
 594                 goto err_del_ai;
 595
 596         return (0);
 597
 598 err_del_ai:
 599         err = PTR_ERR(*rule_p);
 600         *rule_p = NULL;
 601         mlx5e_del_main_vxlan_rules(priv);
 602
 603         return (err);
 604 }
 605
 606 static int
 607 mlx5e_add_main_vxlan_rules(struct mlx5e_priv *priv)
 608 {
 609         u32 *match_criteria;
 610         u32 *match_value;
 611         int err = 0;
 612
 613         match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
 614         match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
 615         if (match_value == NULL || match_criteria == NULL) {
 616                 mlx5_en_err(priv->ifp, "alloc failed\n");
 617                 err = -ENOMEM;
 618                 goto add_main_vxlan_rules_out;
 619         }
 620         err = mlx5e_add_main_vxlan_rules_sub(priv, match_criteria, match_value);
 621
 622 add_main_vxlan_rules_out:
 623         kvfree(match_criteria);
 624         kvfree(match_value);
 625
 626         return (err);
 627 }
 628
 629 static int mlx5e_vport_context_update_vlans(struct mlx5e_priv *priv)
 630 {
 631         if_t ifp = priv->ifp;
 632         int max_list_size;
 633         int list_size;
 634         u16 *vlans;
 635         int vlan;
 636         int err;
 637         int i;
 638
 639         list_size = 0;
 640         for_each_set_bit(vlan, priv->vlan.active_vlans, VLAN_N_VID)
 641                 list_size++;
 642
 643         max_list_size = 1 << MLX5_CAP_GEN(priv->mdev, log_max_vlan_list);
 644
 645         if (list_size > max_list_size) {
 646                 mlx5_en_err(ifp,
 647                             "ifnet vlans list size (%d) > (%d) max vport list size, some vlans will be dropped\n",
 648                             list_size, max_list_size);
 649                 list_size = max_list_size;
 650         }
 651
 652         vlans = kcalloc(list_size, sizeof(*vlans), GFP_KERNEL);
 653         if (!vlans)
 654                 return -ENOMEM;
 655
 656         i = 0;
 657         for_each_set_bit(vlan, priv->vlan.active_vlans, VLAN_N_VID) {
 658                 if (i >= list_size)
 659                         break;
 660                 vlans[i++] = vlan;
 661         }
 662
 663         err = mlx5_modify_nic_vport_vlans(priv->mdev, vlans, list_size);
 664         if (err)
 665                 mlx5_en_err(ifp, "Failed to modify vport vlans list err(%d)\n",
 666                            err);
 667
 668         kfree(vlans);
 669         return err;
 670 }
 671
 672 enum mlx5e_vlan_rule_type {
 673         MLX5E_VLAN_RULE_TYPE_UNTAGGED,
 674         MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID,
 675         MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID,
 676         MLX5E_VLAN_RULE_TYPE_MATCH_VID,
 677 };
 678
 679 static int
 680 mlx5e_add_vlan_rule_sub(struct mlx5e_priv *priv,
 681     enum mlx5e_vlan_rule_type rule_type, u16 vid,
 682     u32 *mc, u32 *mv)
 683 {
 684         struct mlx5_flow_table *ft = priv->fts.vlan.t;
 685         struct mlx5_flow_destination dest = {};
 686         u8 mc_enable = 0;
 687         struct mlx5_flow_rule **rule_p;
 688         int err = 0;
 689
 690         dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 691         dest.ft = priv->fts.vxlan.t;
 692
 693         mc_enable = MLX5_MATCH_OUTER_HEADERS;
 694
 695         switch (rule_type) {
 696         case MLX5E_VLAN_RULE_TYPE_UNTAGGED:
 697                 rule_p = &priv->vlan.untagged_ft_rule;
 698                 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.cvlan_tag);
 699                 break;
 700         case MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID:
 701                 rule_p = &priv->vlan.any_cvlan_ft_rule;
 702                 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.cvlan_tag);
 703                 MLX5_SET(fte_match_param, mv, outer_headers.cvlan_tag, 1);
 704                 break;
 705         case MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID:
 706                 rule_p = &priv->vlan.any_svlan_ft_rule;
 707                 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.svlan_tag);
 708                 MLX5_SET(fte_match_param, mv, outer_headers.svlan_tag, 1);
 709                 break;
 710         default: /* MLX5E_VLAN_RULE_TYPE_MATCH_VID */
 711                 rule_p = &priv->vlan.active_vlans_ft_rule[vid];
 712                 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.cvlan_tag);
 713                 MLX5_SET(fte_match_param, mv, outer_headers.cvlan_tag, 1);
 714                 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.first_vid);
 715                 MLX5_SET(fte_match_param, mv, outer_headers.first_vid, vid);
 716                 mlx5e_vport_context_update_vlans(priv);
 717                 break;
 718         }
 719
 720         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 721                                      MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
 722                                      MLX5_FS_ETH_FLOW_TAG,
 723                                      &dest);
 724
 725         if (IS_ERR(*rule_p)) {
 726                 err = PTR_ERR(*rule_p);
 727                 *rule_p = NULL;
 728                 mlx5_en_err(priv->ifp, "add rule failed\n");
 729         }
 730
 731         return (err);
 732 }
 733
 734 static int
 735 mlx5e_add_vlan_rule(struct mlx5e_priv *priv,
 736     enum mlx5e_vlan_rule_type rule_type, u16 vid)
 737 {
 738         u32 *match_criteria;
 739         u32 *match_value;
 740         int err = 0;
 741
 742         match_value     = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
 743         match_criteria  = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
 744         if (!match_value || !match_criteria) {
 745                 mlx5_en_err(priv->ifp, "alloc failed\n");
 746                 err = -ENOMEM;
 747                 goto add_vlan_rule_out;
 748         }
 749
 750         err = mlx5e_add_vlan_rule_sub(priv, rule_type, vid, match_criteria,
 751                                     match_value);
 752
 753 add_vlan_rule_out:
 754         kvfree(match_criteria);
 755         kvfree(match_value);
 756
 757         return (err);
 758 }
 759
 760 static void
 761 mlx5e_del_vlan_rule(struct mlx5e_priv *priv,
 762     enum mlx5e_vlan_rule_type rule_type, u16 vid)
 763 {
 764         switch (rule_type) {
 765         case MLX5E_VLAN_RULE_TYPE_UNTAGGED:
 766                 mlx5_del_flow_rule(&priv->vlan.untagged_ft_rule);
 767                 break;
 768         case MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID:
 769                 mlx5_del_flow_rule(&priv->vlan.any_cvlan_ft_rule);
 770                 break;
 771         case MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID:
 772                 mlx5_del_flow_rule(&priv->vlan.any_svlan_ft_rule);
 773                 break;
 774         case MLX5E_VLAN_RULE_TYPE_MATCH_VID:
 775                 mlx5_del_flow_rule(&priv->vlan.active_vlans_ft_rule[vid]);
 776                 mlx5e_vport_context_update_vlans(priv);
 777                 break;
 778         default:
 779                 break;
 780         }
 781 }
 782
 783 static void
 784 mlx5e_del_any_vid_rules(struct mlx5e_priv *priv)
 785 {
 786         mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0);
 787         mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID, 0);
 788 }
 789
 790 static int
 791 mlx5e_add_any_vid_rules(struct mlx5e_priv *priv)
 792 {
 793         int err;
 794
 795         err = mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0);
 796         if (err)
 797                 return (err);
 798
 799         err = mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID, 0);
 800         if (err)
 801                 mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0);
 802
 803         return (err);
 804 }
 805
 806 void
 807 mlx5e_enable_vlan_filter(struct mlx5e_priv *priv)
 808 {
 809         if (priv->vlan.filter_disabled) {
 810                 priv->vlan.filter_disabled = false;
 811                 if (if_getflags(priv->ifp) & IFF_PROMISC)
 812                         return;
 813                 if (test_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state))
 814                         mlx5e_del_any_vid_rules(priv);
 815         }
 816 }
 817
 818 void
 819 mlx5e_disable_vlan_filter(struct mlx5e_priv *priv)
 820 {
 821         if (!priv->vlan.filter_disabled) {
 822                 priv->vlan.filter_disabled = true;
 823                 if (if_getflags(priv->ifp) & IFF_PROMISC)
 824                         return;
 825                 if (test_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state))
 826                         mlx5e_add_any_vid_rules(priv);
 827         }
 828 }
 829
 830 void
 831 mlx5e_vlan_rx_add_vid(void *arg, if_t ifp, u16 vid)
 832 {
 833         struct mlx5e_priv *priv = arg;
 834
 835         if (ifp != priv->ifp)
 836                 return;
 837
 838         PRIV_LOCK(priv);
 839         if (!test_and_set_bit(vid, priv->vlan.active_vlans) &&
 840             test_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state))
 841                 mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_VID, vid);
 842         PRIV_UNLOCK(priv);
 843 }
 844
 845 void
 846 mlx5e_vlan_rx_kill_vid(void *arg, if_t ifp, u16 vid)
 847 {
 848         struct mlx5e_priv *priv = arg;
 849
 850         if (ifp != priv->ifp)
 851                 return;
 852
 853         PRIV_LOCK(priv);
 854         clear_bit(vid, priv->vlan.active_vlans);
 855         if (test_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state))
 856                 mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_VID, vid);
 857         PRIV_UNLOCK(priv);
 858 }
 859
 860 static int
 861 mlx5e_add_all_vlan_rules(struct mlx5e_priv *priv)
 862 {
 863         int err;
 864         int i;
 865
 866         set_bit(0, priv->vlan.active_vlans);
 867         for_each_set_bit(i, priv->vlan.active_vlans, VLAN_N_VID) {
 868                 err = mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_VID,
 869                                           i);
 870                 if (err)
 871                         goto error;
 872         }
 873
 874         err = mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_UNTAGGED, 0);
 875         if (err)
 876                 goto error;
 877
 878         if (priv->vlan.filter_disabled) {
 879                 err = mlx5e_add_any_vid_rules(priv);
 880                 if (err)
 881                         goto error;
 882         }
 883         return (0);
 884 error:
 885         mlx5e_del_all_vlan_rules(priv);
 886         return (err);
 887 }
 888
 889 static void
 890 mlx5e_del_all_vlan_rules(struct mlx5e_priv *priv)
 891 {
 892         int i;
 893
 894         if (priv->vlan.filter_disabled)
 895                 mlx5e_del_any_vid_rules(priv);
 896
 897         mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_UNTAGGED, 0);
 898
 899         for_each_set_bit(i, priv->vlan.active_vlans, VLAN_N_VID)
 900                 mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_VID, i);
 901         clear_bit(0, priv->vlan.active_vlans);
 902 }
 903
 904 #define mlx5e_for_each_hash_node(hn, tmp, hash, i) \
 905         for (i = 0; i < MLX5E_ETH_ADDR_HASH_SIZE; i++) \
 906                 LIST_FOREACH_SAFE(hn, &(hash)[i], hlist, tmp)
 907
 908 static void
 909 mlx5e_execute_action(struct mlx5e_priv *priv,
 910     struct mlx5e_eth_addr_hash_node *hn)
 911 {
 912         switch (hn->action) {
 913         case MLX5E_ACTION_ADD:
 914                 mlx5e_add_eth_addr_rule(priv, &hn->ai, MLX5E_FULLMATCH);
 915                 hn->action = MLX5E_ACTION_NONE;
 916                 break;
 917
 918         case MLX5E_ACTION_DEL:
 919                 mlx5e_del_eth_addr_from_flow_table(priv, &hn->ai);
 920                 if (hn->mpfs_index != -1U)
 921                         mlx5_mpfs_del_mac(priv->mdev, hn->mpfs_index);
 922                 mlx5e_del_eth_addr_from_hash(hn);
 923                 break;
 924
 925         default:
 926                 break;
 927         }
 928 }
 929
 930 static struct mlx5e_eth_addr_hash_node *
 931 mlx5e_move_hn(struct mlx5e_eth_addr_hash_head *fh, struct mlx5e_eth_addr_hash_head *uh)
 932 {
 933         struct mlx5e_eth_addr_hash_node *hn;
 934
 935         hn = LIST_FIRST(fh);
 936         if (hn != NULL) {
 937                 LIST_REMOVE(hn, hlist);
 938                 LIST_INSERT_HEAD(uh, hn, hlist);
 939         }
 940         return (hn);
 941 }
 942
 943 static struct mlx5e_eth_addr_hash_node *
 944 mlx5e_remove_hn(struct mlx5e_eth_addr_hash_head *fh)
 945 {
 946         struct mlx5e_eth_addr_hash_node *hn;
 947
 948         hn = LIST_FIRST(fh);
 949         if (hn != NULL)
 950                 LIST_REMOVE(hn, hlist);
 951         return (hn);
 952 }
 953
 954 struct mlx5e_copy_addr_ctx {
 955         struct mlx5e_eth_addr_hash_head *free;
 956         struct mlx5e_eth_addr_hash_head *fill;
 957         bool success;
 958 };
 959
 960 static u_int
 961 mlx5e_copy_addr(void *arg, struct sockaddr_dl *sdl, u_int cnt)
 962 {
 963         struct mlx5e_copy_addr_ctx *ctx = arg;
 964         struct mlx5e_eth_addr_hash_node *hn;
 965
 966         hn = mlx5e_move_hn(ctx->free, ctx->fill);
 967         if (hn == NULL) {
 968                 ctx->success = false;
 969                 return (0);
 970         }
 971         ether_addr_copy(hn->ai.addr, LLADDR(sdl));
 972
 973         return (1);
 974 }
 975
 976 static void
 977 mlx5e_sync_ifp_addr(struct mlx5e_priv *priv)
 978 {
 979         struct mlx5e_copy_addr_ctx ctx;
 980         struct mlx5e_eth_addr_hash_head head_free;
 981         struct mlx5e_eth_addr_hash_head head_uc;
 982         struct mlx5e_eth_addr_hash_head head_mc;
 983         struct mlx5e_eth_addr_hash_node *hn;
 984         if_t ifp = priv->ifp;
 985         size_t x;
 986         size_t num;
 987
 988         PRIV_ASSERT_LOCKED(priv);
 989
 990 retry:
 991         LIST_INIT(&head_free);
 992         LIST_INIT(&head_uc);
 993         LIST_INIT(&head_mc);
 994         num = 1 + if_lladdr_count(ifp) + if_llmaddr_count(ifp);
 995
 996         /* allocate place holders */
 997         for (x = 0; x != num; x++) {
 998                 hn = malloc(sizeof(*hn), M_MLX5EN, M_WAITOK | M_ZERO);
 999                 hn->action = MLX5E_ACTION_ADD;
1000                 hn->mpfs_index = -1U;
1001                 LIST_INSERT_HEAD(&head_free, hn, hlist);
1002         }
1003
1004         hn = mlx5e_move_hn(&head_free, &head_uc);
1005         MPASS(hn != NULL);
1006
1007         ether_addr_copy(hn->ai.addr, if_getlladdr(ifp));
1008
1009         ctx.free = &head_free;
1010         ctx.fill = &head_uc;
1011         ctx.success = true;
1012         if_foreach_lladdr(ifp, mlx5e_copy_addr, &ctx);
1013         if (ctx.success == false)
1014                 goto cleanup;
1015
1016         ctx.fill = &head_mc;
1017         if_foreach_llmaddr(ifp, mlx5e_copy_addr, &ctx);
1018         if (ctx.success == false)
1019                 goto cleanup;
1020
1021         /* insert L2 unicast addresses into hash list */
1022
1023         while ((hn = mlx5e_remove_hn(&head_uc)) != NULL) {
1024                 if (mlx5e_add_eth_addr_to_hash(priv->eth_addr.if_uc, hn) == 0)
1025                         continue;
1026                 if (hn->mpfs_index == -1U)
1027                         mlx5_mpfs_add_mac(priv->mdev, &hn->mpfs_index,
1028                             hn->ai.addr, 0, 0);
1029         }
1030
1031         /* insert L2 multicast addresses into hash list */
1032
1033         while ((hn = mlx5e_remove_hn(&head_mc)) != NULL) {
1034                 if (mlx5e_add_eth_addr_to_hash(priv->eth_addr.if_mc, hn) == 0)
1035                         continue;
1036         }
1037
1038 cleanup:
1039         while ((hn = mlx5e_remove_hn(&head_uc)) != NULL)
1040                 free(hn, M_MLX5EN);
1041         while ((hn = mlx5e_remove_hn(&head_mc)) != NULL)
1042                 free(hn, M_MLX5EN);
1043         while ((hn = mlx5e_remove_hn(&head_free)) != NULL)
1044                 free(hn, M_MLX5EN);
1045
1046         if (ctx.success == false)
1047                 goto retry;
1048 }
1049
1050 static void mlx5e_fill_addr_array(struct mlx5e_priv *priv, int list_type,
1051                                   u8 addr_array[][ETH_ALEN], int size)
1052 {
1053         bool is_uc = (list_type == MLX5_NIC_VPORT_LIST_TYPE_UC);
1054         if_t ifp = priv->ifp;
1055         struct mlx5e_eth_addr_hash_node *hn;
1056         struct mlx5e_eth_addr_hash_head *addr_list;
1057         struct mlx5e_eth_addr_hash_node *tmp;
1058         int i = 0;
1059         int hi;
1060
1061         addr_list = is_uc ? priv->eth_addr.if_uc : priv->eth_addr.if_mc;
1062
1063         if (is_uc) /* Make sure our own address is pushed first */
1064                 ether_addr_copy(addr_array[i++], if_getlladdr(ifp));
1065         else if (priv->eth_addr.broadcast_enabled)
1066                 ether_addr_copy(addr_array[i++], if_getbroadcastaddr(ifp));
1067
1068         mlx5e_for_each_hash_node(hn, tmp, addr_list, hi) {
1069                 if (ether_addr_equal(if_getlladdr(ifp), hn->ai.addr))
1070                         continue;
1071                 if (i >= size)
1072                         break;
1073                 ether_addr_copy(addr_array[i++], hn->ai.addr);
1074         }
1075 }
1076
1077 static void mlx5e_vport_context_update_addr_list(struct mlx5e_priv *priv,
1078                                                  int list_type)
1079 {
1080         bool is_uc = (list_type == MLX5_NIC_VPORT_LIST_TYPE_UC);
1081         struct mlx5e_eth_addr_hash_node *hn;
1082         u8 (*addr_array)[ETH_ALEN] = NULL;
1083         struct mlx5e_eth_addr_hash_head *addr_list;
1084         struct mlx5e_eth_addr_hash_node *tmp;
1085         int max_size;
1086         int size;
1087         int err;
1088         int hi;
1089
1090         size = is_uc ? 0 : (priv->eth_addr.broadcast_enabled ? 1 : 0);
1091         max_size = is_uc ?
1092                 1 << MLX5_CAP_GEN(priv->mdev, log_max_current_uc_list) :
1093                 1 << MLX5_CAP_GEN(priv->mdev, log_max_current_mc_list);
1094
1095         addr_list = is_uc ? priv->eth_addr.if_uc : priv->eth_addr.if_mc;
1096         mlx5e_for_each_hash_node(hn, tmp, addr_list, hi)
1097                 size++;
1098
1099         if (size > max_size) {
1100                 mlx5_en_err(priv->ifp,
1101                             "ifp %s list size (%d) > (%d) max vport list size, some addresses will be dropped\n",
1102                             is_uc ? "UC" : "MC", size, max_size);
1103                 size = max_size;
1104         }
1105
1106         if (size) {
1107                 addr_array = kcalloc(size, ETH_ALEN, GFP_KERNEL);
1108                 if (!addr_array) {
1109                         err = -ENOMEM;
1110                         goto out;
1111                 }
1112                 mlx5e_fill_addr_array(priv, list_type, addr_array, size);
1113         }
1114
1115         err = mlx5_modify_nic_vport_mac_list(priv->mdev, list_type, addr_array, size);
1116 out:
1117         if (err)
1118                 mlx5_en_err(priv->ifp,
1119                            "Failed to modify vport %s list err(%d)\n",
1120                            is_uc ? "UC" : "MC", err);
1121         kfree(addr_array);
1122 }
1123
1124 static void mlx5e_vport_context_update(struct mlx5e_priv *priv)
1125 {
1126         struct mlx5e_eth_addr_db *ea = &priv->eth_addr;
1127
1128         mlx5e_vport_context_update_addr_list(priv, MLX5_NIC_VPORT_LIST_TYPE_UC);
1129         mlx5e_vport_context_update_addr_list(priv, MLX5_NIC_VPORT_LIST_TYPE_MC);
1130         mlx5_modify_nic_vport_promisc(priv->mdev, 0,
1131                                       ea->allmulti_enabled,
1132                                       ea->promisc_enabled);
1133 }
1134
1135 static void
1136 mlx5e_apply_ifp_addr(struct mlx5e_priv *priv)
1137 {
1138         struct mlx5e_eth_addr_hash_node *hn;
1139         struct mlx5e_eth_addr_hash_node *tmp;
1140         int i;
1141
1142         mlx5e_for_each_hash_node(hn, tmp, priv->eth_addr.if_uc, i)
1143             mlx5e_execute_action(priv, hn);
1144
1145         mlx5e_for_each_hash_node(hn, tmp, priv->eth_addr.if_mc, i)
1146             mlx5e_execute_action(priv, hn);
1147 }
1148
1149 static void
1150 mlx5e_handle_ifp_addr(struct mlx5e_priv *priv, bool rx_mode_enable)
1151 {
1152         struct mlx5e_eth_addr_hash_node *hn;
1153         struct mlx5e_eth_addr_hash_node *tmp;
1154         int i;
1155
1156         mlx5e_for_each_hash_node(hn, tmp, priv->eth_addr.if_uc, i)
1157             hn->action = MLX5E_ACTION_DEL;
1158         mlx5e_for_each_hash_node(hn, tmp, priv->eth_addr.if_mc, i)
1159             hn->action = MLX5E_ACTION_DEL;
1160
1161         if (rx_mode_enable)
1162                 mlx5e_sync_ifp_addr(priv);
1163
1164         mlx5e_apply_ifp_addr(priv);
1165 }
1166
1167 static void
1168 mlx5e_set_rx_mode_core(struct mlx5e_priv *priv, bool rx_mode_enable)
1169 {
1170         struct mlx5e_eth_addr_db *ea = &priv->eth_addr;
1171         if_t ndev = priv->ifp;
1172         int ndev_flags = if_getflags(ndev);
1173
1174         bool promisc_enabled = rx_mode_enable && (ndev_flags & IFF_PROMISC);
1175         bool allmulti_enabled = rx_mode_enable && (ndev_flags & IFF_ALLMULTI);
1176         bool broadcast_enabled = rx_mode_enable;
1177
1178         bool enable_promisc = !ea->promisc_enabled && promisc_enabled;
1179         bool disable_promisc = ea->promisc_enabled && !promisc_enabled;
1180         bool enable_allmulti = !ea->allmulti_enabled && allmulti_enabled;
1181         bool disable_allmulti = ea->allmulti_enabled && !allmulti_enabled;
1182         bool enable_broadcast = !ea->broadcast_enabled && broadcast_enabled;
1183         bool disable_broadcast = ea->broadcast_enabled && !broadcast_enabled;
1184
1185         /* update broadcast address */
1186         ether_addr_copy(priv->eth_addr.broadcast.addr,
1187             if_getbroadcastaddr(priv->ifp));
1188
1189         if (enable_promisc) {
1190                 mlx5e_add_eth_addr_rule(priv, &ea->promisc, MLX5E_PROMISC);
1191                 if (!priv->vlan.filter_disabled)
1192                         mlx5e_add_any_vid_rules(priv);
1193         }
1194         if (enable_allmulti)
1195                 mlx5e_add_eth_addr_rule(priv, &ea->allmulti, MLX5E_ALLMULTI);
1196         if (enable_broadcast)
1197                 mlx5e_add_eth_addr_rule(priv, &ea->broadcast, MLX5E_FULLMATCH);
1198
1199         mlx5e_handle_ifp_addr(priv, rx_mode_enable);
1200
1201         if (disable_broadcast)
1202                 mlx5e_del_eth_addr_from_flow_table(priv, &ea->broadcast);
1203         if (disable_allmulti)
1204                 mlx5e_del_eth_addr_from_flow_table(priv, &ea->allmulti);
1205         if (disable_promisc) {
1206                 if (!priv->vlan.filter_disabled)
1207                         mlx5e_del_any_vid_rules(priv);
1208                 mlx5e_del_eth_addr_from_flow_table(priv, &ea->promisc);
1209         }
1210
1211         ea->promisc_enabled = promisc_enabled;
1212         ea->allmulti_enabled = allmulti_enabled;
1213         ea->broadcast_enabled = broadcast_enabled;
1214
1215         mlx5e_vport_context_update(priv);
1216 }
1217
1218 void
1219 mlx5e_set_rx_mode_work(struct work_struct *work)
1220 {
1221         struct mlx5e_priv *priv =
1222             container_of(work, struct mlx5e_priv, set_rx_mode_work);
1223
1224         PRIV_LOCK(priv);
1225         if (test_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state))
1226                 mlx5e_set_rx_mode_core(priv, true);
1227         PRIV_UNLOCK(priv);
1228 }
1229
1230 static void
1231 mlx5e_destroy_groups(struct mlx5e_flow_table *ft)
1232 {
1233         int i;
1234
1235         for (i = ft->num_groups - 1; i >= 0; i--) {
1236                 if (!IS_ERR_OR_NULL(ft->g[i]))
1237                         mlx5_destroy_flow_group(ft->g[i]);
1238                 ft->g[i] = NULL;
1239         }
1240         ft->num_groups = 0;
1241 }
1242
1243 static void
1244 mlx5e_destroy_flow_table(struct mlx5e_flow_table *ft)
1245 {
1246         mlx5e_destroy_groups(ft);
1247         kfree(ft->g);
1248         mlx5_destroy_flow_table(ft->t);
1249         ft->t = NULL;
1250 }
1251
1252 #define MLX5E_NUM_MAIN_GROUPS   10
1253 #define MLX5E_MAIN_GROUP0_SIZE  BIT(4)
1254 #define MLX5E_MAIN_GROUP1_SIZE  BIT(3)
1255 #define MLX5E_MAIN_GROUP2_SIZE  BIT(1)
1256 #define MLX5E_MAIN_GROUP3_SIZE  BIT(0)
1257 #define MLX5E_MAIN_GROUP4_SIZE  BIT(14)
1258 #define MLX5E_MAIN_GROUP5_SIZE  BIT(13)
1259 #define MLX5E_MAIN_GROUP6_SIZE  BIT(11)
1260 #define MLX5E_MAIN_GROUP7_SIZE  BIT(2)
1261 #define MLX5E_MAIN_GROUP8_SIZE  BIT(1)
1262 #define MLX5E_MAIN_GROUP9_SIZE  BIT(0)
1263 #define MLX5E_MAIN_TABLE_SIZE   (MLX5E_MAIN_GROUP0_SIZE +\
1264                                  MLX5E_MAIN_GROUP1_SIZE +\
1265                                  MLX5E_MAIN_GROUP2_SIZE +\
1266                                  MLX5E_MAIN_GROUP3_SIZE +\
1267                                  MLX5E_MAIN_GROUP4_SIZE +\
1268                                  MLX5E_MAIN_GROUP5_SIZE +\
1269                                  MLX5E_MAIN_GROUP6_SIZE +\
1270                                  MLX5E_MAIN_GROUP7_SIZE +\
1271                                  MLX5E_MAIN_GROUP8_SIZE +\
1272                                  MLX5E_MAIN_GROUP9_SIZE +\
1273                                  0)
1274
1275 static int
1276 mlx5e_create_main_groups_sub(struct mlx5e_flow_table *ft, u32 *in,
1277                                       int inlen)
1278 {
1279         u8 *mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
1280         u8 *dmac = MLX5_ADDR_OF(create_flow_group_in, in,
1281                                 match_criteria.outer_headers.dmac_47_16);
1282         int err;
1283         int ix = 0;
1284
1285         /* Tunnel rules need to be first in this list of groups */
1286
1287         /* Start tunnel rules */
1288         memset(in, 0, inlen);
1289         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1290         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
1291         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol);
1292         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.udp_dport);
1293         MLX5_SET_CFG(in, start_flow_index, ix);
1294         ix += MLX5E_MAIN_GROUP0_SIZE;
1295         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1296         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1297         if (IS_ERR(ft->g[ft->num_groups]))
1298                 goto err_destory_groups;
1299         ft->num_groups++;
1300         /* End Tunnel Rules */
1301
1302         memset(in, 0, inlen);
1303         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1304         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
1305         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol);
1306         MLX5_SET_CFG(in, start_flow_index, ix);
1307         ix += MLX5E_MAIN_GROUP1_SIZE;
1308         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1309         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1310         if (IS_ERR(ft->g[ft->num_groups]))
1311                 goto err_destory_groups;
1312         ft->num_groups++;
1313
1314         memset(in, 0, inlen);
1315         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1316         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
1317         MLX5_SET_CFG(in, start_flow_index, ix);
1318         ix += MLX5E_MAIN_GROUP2_SIZE;
1319         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1320         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1321         if (IS_ERR(ft->g[ft->num_groups]))
1322                 goto err_destory_groups;
1323         ft->num_groups++;
1324
1325         memset(in, 0, inlen);
1326         MLX5_SET_CFG(in, start_flow_index, ix);
1327         ix += MLX5E_MAIN_GROUP3_SIZE;
1328         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1329         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1330         if (IS_ERR(ft->g[ft->num_groups]))
1331                 goto err_destory_groups;
1332         ft->num_groups++;
1333
1334         memset(in, 0, inlen);
1335         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1336         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
1337         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol);
1338         memset(dmac, 0xff, ETH_ALEN);
1339         MLX5_SET_CFG(in, start_flow_index, ix);
1340         ix += MLX5E_MAIN_GROUP4_SIZE;
1341         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1342         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1343         if (IS_ERR(ft->g[ft->num_groups]))
1344                 goto err_destory_groups;
1345         ft->num_groups++;
1346
1347         memset(in, 0, inlen);
1348         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1349         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
1350         memset(dmac, 0xff, ETH_ALEN);
1351         MLX5_SET_CFG(in, start_flow_index, ix);
1352         ix += MLX5E_MAIN_GROUP5_SIZE;
1353         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1354         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1355         if (IS_ERR(ft->g[ft->num_groups]))
1356                 goto err_destory_groups;
1357         ft->num_groups++;
1358
1359         memset(in, 0, inlen);
1360         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1361         memset(dmac, 0xff, ETH_ALEN);
1362         MLX5_SET_CFG(in, start_flow_index, ix);
1363         ix += MLX5E_MAIN_GROUP6_SIZE;
1364         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1365         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1366         if (IS_ERR(ft->g[ft->num_groups]))
1367                 goto err_destory_groups;
1368         ft->num_groups++;
1369
1370         memset(in, 0, inlen);
1371         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1372         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
1373         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol);
1374         dmac[0] = 0x01;
1375         MLX5_SET_CFG(in, start_flow_index, ix);
1376         ix += MLX5E_MAIN_GROUP7_SIZE;
1377         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1378         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1379         if (IS_ERR(ft->g[ft->num_groups]))
1380                 goto err_destory_groups;
1381         ft->num_groups++;
1382
1383         memset(in, 0, inlen);
1384         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1385         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
1386         dmac[0] = 0x01;
1387         MLX5_SET_CFG(in, start_flow_index, ix);
1388         ix += MLX5E_MAIN_GROUP8_SIZE;
1389         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1390         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1391         if (IS_ERR(ft->g[ft->num_groups]))
1392                 goto err_destory_groups;
1393         ft->num_groups++;
1394
1395         memset(in, 0, inlen);
1396         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1397         dmac[0] = 0x01;
1398         MLX5_SET_CFG(in, start_flow_index, ix);
1399         ix += MLX5E_MAIN_GROUP9_SIZE;
1400         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1401         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1402         if (IS_ERR(ft->g[ft->num_groups]))
1403                 goto err_destory_groups;
1404         ft->num_groups++;
1405
1406         return (0);
1407
1408 err_destory_groups:
1409         err = PTR_ERR(ft->g[ft->num_groups]);
1410         ft->g[ft->num_groups] = NULL;
1411         mlx5e_destroy_groups(ft);
1412
1413         return (err);
1414 }
1415
1416 static int
1417 mlx5e_create_main_groups(struct mlx5e_flow_table *ft)
1418 {
1419         u32 *in;
1420         int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1421         int err;
1422
1423         in = mlx5_vzalloc(inlen);
1424         if (!in)
1425                 return (-ENOMEM);
1426
1427         err = mlx5e_create_main_groups_sub(ft, in, inlen);
1428
1429         kvfree(in);
1430         return (err);
1431 }
1432
1433 #define MLX5E_MAIN_VXLAN_GROUP0_SIZE    BIT(3)
1434 #define MLX5E_MAIN_VXLAN_GROUP1_SIZE    BIT(3)
1435 #define MLX5E_MAIN_VXLAN_GROUP2_SIZE    BIT(0)
1436 static int
1437 mlx5e_create_main_vxlan_groups_sub(struct mlx5e_flow_table *ft, u32 *in,
1438     int inlen)
1439 {
1440         u8 *mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
1441         int err;
1442         int ix = 0;
1443
1444         memset(in, 0, inlen);
1445         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_INNER_HEADERS);
1446         MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ethertype);
1447         MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ip_protocol);
1448         MLX5_SET_CFG(in, start_flow_index, ix);
1449         ix += MLX5E_MAIN_VXLAN_GROUP0_SIZE;
1450         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1451         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1452         if (IS_ERR(ft->g[ft->num_groups]))
1453                 goto err_destory_groups;
1454         ft->num_groups++;
1455
1456         memset(in, 0, inlen);
1457         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_INNER_HEADERS);
1458         MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ethertype);
1459         MLX5_SET_CFG(in, start_flow_index, ix);
1460         ix += MLX5E_MAIN_VXLAN_GROUP1_SIZE;
1461         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1462         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1463         if (IS_ERR(ft->g[ft->num_groups]))
1464                 goto err_destory_groups;
1465         ft->num_groups++;
1466
1467         memset(in, 0, inlen);
1468         MLX5_SET_CFG(in, start_flow_index, ix);
1469         ix += MLX5E_MAIN_VXLAN_GROUP2_SIZE;
1470         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1471         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1472         if (IS_ERR(ft->g[ft->num_groups]))
1473                 goto err_destory_groups;
1474         ft->num_groups++;
1475
1476         return (0);
1477
1478 err_destory_groups:
1479         err = PTR_ERR(ft->g[ft->num_groups]);
1480         ft->g[ft->num_groups] = NULL;
1481         mlx5e_destroy_groups(ft);
1482
1483         return (err);
1484 }
1485
1486 static int
1487 mlx5e_create_main_vxlan_groups(struct mlx5e_flow_table *ft)
1488 {
1489         u32 *in;
1490         int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1491         int err;
1492
1493         in = mlx5_vzalloc(inlen);
1494         if (!in)
1495                 return (-ENOMEM);
1496
1497         err = mlx5e_create_main_vxlan_groups_sub(ft, in, inlen);
1498
1499         kvfree(in);
1500         return (err);
1501 }
1502
1503
1504 static int
1505 mlx5e_create_main_flow_table(struct mlx5e_priv *priv, bool inner_vxlan)
1506 {
1507         struct mlx5e_flow_table *ft = inner_vxlan ? &priv->fts.main_vxlan :
1508             &priv->fts.main;
1509         int err;
1510
1511         ft->num_groups = 0;
1512         ft->t = mlx5_create_flow_table(priv->fts.ns, 0,
1513             inner_vxlan ? "vxlan_main" : "main", MLX5E_MAIN_TABLE_SIZE);
1514
1515         if (IS_ERR(ft->t)) {
1516                 err = PTR_ERR(ft->t);
1517                 ft->t = NULL;
1518                 return (err);
1519         }
1520         ft->g = kcalloc(MLX5E_NUM_MAIN_GROUPS, sizeof(*ft->g), GFP_KERNEL);
1521         if (!ft->g) {
1522                 err = -ENOMEM;
1523                 goto err_destroy_main_flow_table;
1524         }
1525
1526         err = inner_vxlan ? mlx5e_create_main_vxlan_groups(ft) :
1527             mlx5e_create_main_groups(ft);
1528         if (err)
1529                 goto err_free_g;
1530         return (0);
1531
1532 err_free_g:
1533         kfree(ft->g);
1534
1535 err_destroy_main_flow_table:
1536         mlx5_destroy_flow_table(ft->t);
1537         ft->t = NULL;
1538
1539         return (err);
1540 }
1541
1542 static void mlx5e_destroy_main_flow_table(struct mlx5e_priv *priv)
1543 {
1544         mlx5e_destroy_flow_table(&priv->fts.main);
1545 }
1546
1547 static void mlx5e_destroy_main_vxlan_flow_table(struct mlx5e_priv *priv)
1548 {
1549         mlx5e_destroy_flow_table(&priv->fts.main_vxlan);
1550 }
1551
1552 #define MLX5E_NUM_VLAN_GROUPS   3
1553 #define MLX5E_VLAN_GROUP0_SIZE  BIT(12)
1554 #define MLX5E_VLAN_GROUP1_SIZE  BIT(1)
1555 #define MLX5E_VLAN_GROUP2_SIZE  BIT(0)
1556 #define MLX5E_VLAN_TABLE_SIZE   (MLX5E_VLAN_GROUP0_SIZE +\
1557                                  MLX5E_VLAN_GROUP1_SIZE +\
1558                                  MLX5E_VLAN_GROUP2_SIZE +\
1559                                  0)
1560
1561 static int
1562 mlx5e_create_vlan_groups_sub(struct mlx5e_flow_table *ft, u32 *in,
1563                                       int inlen)
1564 {
1565         int err;
1566         int ix = 0;
1567         u8 *mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
1568
1569         memset(in, 0, inlen);
1570         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1571         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.cvlan_tag);
1572         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.first_vid);
1573         MLX5_SET_CFG(in, start_flow_index, ix);
1574         ix += MLX5E_VLAN_GROUP0_SIZE;
1575         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1576         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1577         if (IS_ERR(ft->g[ft->num_groups]))
1578                 goto err_destory_groups;
1579         ft->num_groups++;
1580
1581         memset(in, 0, inlen);
1582         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1583         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.cvlan_tag);
1584         MLX5_SET_CFG(in, start_flow_index, ix);
1585         ix += MLX5E_VLAN_GROUP1_SIZE;
1586         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1587         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1588         if (IS_ERR(ft->g[ft->num_groups]))
1589                 goto err_destory_groups;
1590         ft->num_groups++;
1591
1592         memset(in, 0, inlen);
1593         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1594         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.svlan_tag);
1595         MLX5_SET_CFG(in, start_flow_index, ix);
1596         ix += MLX5E_VLAN_GROUP2_SIZE;
1597         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1598         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1599         if (IS_ERR(ft->g[ft->num_groups]))
1600                 goto err_destory_groups;
1601         ft->num_groups++;
1602
1603         return (0);
1604
1605 err_destory_groups:
1606         err = PTR_ERR(ft->g[ft->num_groups]);
1607         ft->g[ft->num_groups] = NULL;
1608         mlx5e_destroy_groups(ft);
1609
1610         return (err);
1611 }
1612
1613 static int
1614 mlx5e_create_vlan_groups(struct mlx5e_flow_table *ft)
1615 {
1616         u32 *in;
1617         int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1618         int err;
1619
1620         in = mlx5_vzalloc(inlen);
1621         if (!in)
1622                 return (-ENOMEM);
1623
1624         err = mlx5e_create_vlan_groups_sub(ft, in, inlen);
1625
1626         kvfree(in);
1627         return (err);
1628 }
1629
1630 static int
1631 mlx5e_create_vlan_flow_table(struct mlx5e_priv *priv)
1632 {
1633         struct mlx5e_flow_table *ft = &priv->fts.vlan;
1634         int err;
1635
1636         ft->num_groups = 0;
1637         ft->t = mlx5_create_flow_table(priv->fts.ns, 0, "vlan",
1638                                        MLX5E_VLAN_TABLE_SIZE);
1639
1640         if (IS_ERR(ft->t)) {
1641                 err = PTR_ERR(ft->t);
1642                 ft->t = NULL;
1643                 return (err);
1644         }
1645         ft->g = kcalloc(MLX5E_NUM_VLAN_GROUPS, sizeof(*ft->g), GFP_KERNEL);
1646         if (!ft->g) {
1647                 err = -ENOMEM;
1648                 goto err_destroy_vlan_flow_table;
1649         }
1650
1651         err = mlx5e_create_vlan_groups(ft);
1652         if (err)
1653                 goto err_free_g;
1654
1655         return (0);
1656
1657 err_free_g:
1658         kfree(ft->g);
1659
1660 err_destroy_vlan_flow_table:
1661         mlx5_destroy_flow_table(ft->t);
1662         ft->t = NULL;
1663
1664         return (err);
1665 }
1666
1667 static void
1668 mlx5e_destroy_vlan_flow_table(struct mlx5e_priv *priv)
1669 {
1670         mlx5e_destroy_flow_table(&priv->fts.vlan);
1671 }
1672
1673 static int
1674 mlx5e_add_vxlan_rule_sub(struct mlx5e_priv *priv, u32 *mc, u32 *mv,
1675     struct mlx5e_vxlan_db_el *el)
1676 {
1677         struct mlx5_flow_table *ft = priv->fts.vxlan.t;
1678         struct mlx5_flow_destination dest = {};
1679         u8 mc_enable;
1680         struct mlx5_flow_rule **rule_p;
1681         int err = 0;
1682
1683         dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1684         dest.ft = priv->fts.main_vxlan.t;
1685
1686         mc_enable = MLX5_MATCH_OUTER_HEADERS;
1687         rule_p = &el->vxlan_ft_rule;
1688         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
1689         MLX5_SET(fte_match_param, mv, outer_headers.ethertype, el->proto);
1690         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol);
1691         MLX5_SET(fte_match_param, mv, outer_headers.ip_protocol, IPPROTO_UDP);
1692         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.udp_dport);
1693         MLX5_SET(fte_match_param, mv, outer_headers.udp_dport, el->port);
1694
1695         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
1696             MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, MLX5_FS_ETH_FLOW_TAG, &dest);
1697
1698         if (IS_ERR(*rule_p)) {
1699                 err = PTR_ERR(*rule_p);
1700                 *rule_p = NULL;
1701                 mlx5_en_err(priv->ifp, "add rule failed\n");
1702         }
1703
1704         return (err);
1705 }
1706
1707 static struct mlx5e_vxlan_db_el *
1708 mlx5e_vxlan_find_db_el(struct mlx5e_priv *priv, u_int proto, u_int port)
1709 {
1710         struct mlx5e_vxlan_db_el *el;
1711
1712         TAILQ_FOREACH(el, &priv->vxlan.head, link) {
1713                 if (el->proto == proto && el->port == port)
1714                         return (el);
1715         }
1716         return (NULL);
1717 }
1718
1719 static struct mlx5e_vxlan_db_el *
1720 mlx5e_vxlan_alloc_db_el(struct mlx5e_priv *priv, u_int proto, u_int port)
1721 {
1722         struct mlx5e_vxlan_db_el *el;
1723
1724         el = mlx5_vzalloc(sizeof(*el));
1725         el->refcount = 1;
1726         el->proto = proto;
1727         el->port = port;
1728         el->vxlan_ft_rule = NULL;
1729         return (el);
1730 }
1731
1732 static int
1733 mlx5e_vxlan_family_to_proto(sa_family_t family, u_int *proto)
1734 {
1735         switch (family) {
1736         case AF_INET:
1737                 *proto = ETHERTYPE_IP;
1738                 return (0);
1739         case AF_INET6:
1740                 *proto = ETHERTYPE_IPV6;
1741                 return (0);
1742         default:
1743                 return (-EINVAL);
1744         }
1745 }
1746
1747 static int
1748 mlx5e_add_vxlan_rule_from_db(struct mlx5e_priv *priv,
1749     struct mlx5e_vxlan_db_el *el)
1750 {
1751         u32 *match_criteria;
1752         u32 *match_value;
1753         int err;
1754
1755         match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
1756         match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
1757         if (match_value == NULL || match_criteria == NULL) {
1758                 mlx5_en_err(priv->ifp, "alloc failed\n");
1759                 err = -ENOMEM;
1760                 goto add_vxlan_rule_out;
1761         }
1762
1763         err = mlx5e_add_vxlan_rule_sub(priv, match_criteria, match_value, el);
1764
1765 add_vxlan_rule_out:
1766         kvfree(match_criteria);
1767         kvfree(match_value);
1768
1769         return (err);
1770 }
1771
1772 static int
1773 mlx5e_add_vxlan_rule(struct mlx5e_priv *priv, sa_family_t family, u_int port)
1774 {
1775         struct mlx5e_vxlan_db_el *el;
1776         u_int proto;
1777         int err;
1778
1779         err = mlx5e_vxlan_family_to_proto(family, &proto);
1780         if (err != 0)
1781                 return (err);
1782
1783         el = mlx5e_vxlan_find_db_el(priv, proto, port);
1784         if (el != NULL) {
1785                 el->refcount++;
1786                 if (el->installed)
1787                         return (0);
1788         }
1789         el = mlx5e_vxlan_alloc_db_el(priv, proto, port);
1790
1791         if ((if_getcapenable(priv->ifp) & IFCAP_VXLAN_HWCSUM) != 0) {
1792                 err = mlx5e_add_vxlan_rule_from_db(priv, el);
1793                 if (err == 0)
1794                         el->installed = true;
1795         }
1796         if (err == 0)
1797                 TAILQ_INSERT_TAIL(&priv->vxlan.head, el, link);
1798         else
1799                 kvfree(el);
1800
1801         return (err);
1802 }
1803
1804 static int
1805 mlx5e_add_vxlan_catchall_rule_sub(struct mlx5e_priv *priv, u32 *mc, u32 *mv)
1806 {
1807         struct mlx5_flow_table *ft = priv->fts.vxlan.t;
1808         struct mlx5_flow_destination dest = {};
1809         u8 mc_enable = 0;
1810         struct mlx5_flow_rule **rule_p;
1811         int err = 0;
1812
1813         dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1814         dest.ft = priv->fts.main.t;
1815
1816         rule_p = &priv->fts.vxlan_catchall_ft_rule;
1817         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
1818             MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, MLX5_FS_ETH_FLOW_TAG, &dest);
1819
1820         if (IS_ERR(*rule_p)) {
1821                 err = PTR_ERR(*rule_p);
1822                 *rule_p = NULL;
1823                 mlx5_en_err(priv->ifp, "add rule failed\n");
1824         }
1825
1826         return (err);
1827 }
1828
1829
1830 static int
1831 mlx5e_add_vxlan_catchall_rule(struct mlx5e_priv *priv)
1832 {
1833         u32 *match_criteria;
1834         u32 *match_value;
1835         int err;
1836
1837         match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
1838         match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
1839         if (match_value == NULL || match_criteria == NULL) {
1840                 mlx5_en_err(priv->ifp, "alloc failed\n");
1841                 err = -ENOMEM;
1842                 goto add_vxlan_rule_out;
1843         }
1844
1845         err = mlx5e_add_vxlan_catchall_rule_sub(priv, match_criteria,
1846             match_value);
1847
1848 add_vxlan_rule_out:
1849         kvfree(match_criteria);
1850         kvfree(match_value);
1851
1852         return (err);
1853 }
1854
1855 int
1856 mlx5e_add_all_vxlan_rules(struct mlx5e_priv *priv)
1857 {
1858         struct mlx5e_vxlan_db_el *el;
1859         int err;
1860
1861         err = 0;
1862         TAILQ_FOREACH(el, &priv->vxlan.head, link) {
1863                 if (el->installed)
1864                         continue;
1865                 err = mlx5e_add_vxlan_rule_from_db(priv, el);
1866                 if (err != 0)
1867                         break;
1868                 el->installed = true;
1869         }
1870
1871         return (err);
1872 }
1873
1874 static int
1875 mlx5e_del_vxlan_rule(struct mlx5e_priv *priv, sa_family_t family, u_int port)
1876 {
1877         struct mlx5e_vxlan_db_el *el;
1878         u_int proto;
1879         int err;
1880
1881         err = mlx5e_vxlan_family_to_proto(family, &proto);
1882         if (err != 0)
1883                 return (err);
1884
1885         el = mlx5e_vxlan_find_db_el(priv, proto, port);
1886         if (el == NULL)
1887                 return (0);
1888         if (el->refcount > 1) {
1889                 el->refcount--;
1890                 return (0);
1891         }
1892
1893         if (el->installed)
1894                 mlx5_del_flow_rule(&el->vxlan_ft_rule);
1895         TAILQ_REMOVE(&priv->vxlan.head, el, link);
1896         kvfree(el);
1897         return (0);
1898 }
1899
1900 void
1901 mlx5e_del_all_vxlan_rules(struct mlx5e_priv *priv)
1902 {
1903         struct mlx5e_vxlan_db_el *el;
1904
1905         TAILQ_FOREACH(el, &priv->vxlan.head, link) {
1906                 if (!el->installed)
1907                         continue;
1908                 mlx5_del_flow_rule(&el->vxlan_ft_rule);
1909                 el->installed = false;
1910         }
1911 }
1912
1913 static void
1914 mlx5e_del_vxlan_catchall_rule(struct mlx5e_priv *priv)
1915 {
1916         mlx5_del_flow_rule(&priv->fts.vxlan_catchall_ft_rule);
1917 }
1918
1919 void
1920 mlx5e_vxlan_start(void *arg, if_t ifp __unused, sa_family_t family,
1921     u_int port)
1922 {
1923         struct mlx5e_priv *priv = arg;
1924         int err;
1925
1926         PRIV_LOCK(priv);
1927         err = mlx5_vxlan_udp_port_add(priv->mdev, port);
1928         if (err == 0 && test_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state))
1929                 mlx5e_add_vxlan_rule(priv, family, port);
1930         PRIV_UNLOCK(priv);
1931 }
1932
1933 void
1934 mlx5e_vxlan_stop(void *arg, if_t ifp __unused, sa_family_t family,
1935     u_int port)
1936 {
1937         struct mlx5e_priv *priv = arg;
1938
1939         PRIV_LOCK(priv);
1940         if (test_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state))
1941                 mlx5e_del_vxlan_rule(priv, family, port);
1942         (void)mlx5_vxlan_udp_port_delete(priv->mdev, port);
1943         PRIV_UNLOCK(priv);
1944 }
1945
1946 #define MLX5E_VXLAN_GROUP0_SIZE BIT(3)  /* XXXKIB */
1947 #define MLX5E_VXLAN_GROUP1_SIZE BIT(0)
1948 #define MLX5E_NUM_VXLAN_GROUPS  BIT(1)
1949 #define MLX5E_VXLAN_TABLE_SIZE  \
1950     (MLX5E_VXLAN_GROUP0_SIZE + MLX5E_VXLAN_GROUP1_SIZE)
1951
1952 static int
1953 mlx5e_create_vxlan_groups_sub(struct mlx5e_flow_table *ft, u32 *in,
1954                                       int inlen)
1955 {
1956         int err;
1957         int ix = 0;
1958         u8 *mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
1959
1960         memset(in, 0, inlen);
1961         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1962         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
1963         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol);
1964         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.udp_dport);
1965         MLX5_SET_CFG(in, start_flow_index, ix);
1966         ix += MLX5E_VXLAN_GROUP0_SIZE;
1967         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1968         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1969         if (IS_ERR(ft->g[ft->num_groups]))
1970                 goto err_destory_groups;
1971         ft->num_groups++;
1972
1973         memset(in, 0, inlen);
1974         MLX5_SET_CFG(in, start_flow_index, ix);
1975         ix += MLX5E_VXLAN_GROUP1_SIZE;
1976         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1977         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1978         if (IS_ERR(ft->g[ft->num_groups]))
1979                 goto err_destory_groups;
1980         ft->num_groups++;
1981
1982         return (0);
1983
1984 err_destory_groups:
1985         err = PTR_ERR(ft->g[ft->num_groups]);
1986         ft->g[ft->num_groups] = NULL;
1987         mlx5e_destroy_groups(ft);
1988
1989         return (err);
1990 }
1991
1992 static int
1993 mlx5e_create_vxlan_groups(struct mlx5e_flow_table *ft)
1994 {
1995         u32 *in;
1996         int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1997         int err;
1998
1999         in = mlx5_vzalloc(inlen);
2000         if (!in)
2001                 return (-ENOMEM);
2002
2003         err = mlx5e_create_vxlan_groups_sub(ft, in, inlen);
2004
2005         kvfree(in);
2006         return (err);
2007 }
2008
2009 static int
2010 mlx5e_create_vxlan_flow_table(struct mlx5e_priv *priv)
2011 {
2012         struct mlx5e_flow_table *ft = &priv->fts.vxlan;
2013         int err;
2014
2015         ft->num_groups = 0;
2016         ft->t = mlx5_create_flow_table(priv->fts.ns, 0, "vxlan",
2017                                        MLX5E_VXLAN_TABLE_SIZE);
2018
2019         if (IS_ERR(ft->t)) {
2020                 err = PTR_ERR(ft->t);
2021                 ft->t = NULL;
2022                 return (err);
2023         }
2024         ft->g = kcalloc(MLX5E_NUM_VXLAN_GROUPS, sizeof(*ft->g), GFP_KERNEL);
2025         if (!ft->g) {
2026                 err = -ENOMEM;
2027                 goto err_destroy_vxlan_flow_table;
2028         }
2029
2030         err = mlx5e_create_vxlan_groups(ft);
2031         if (err)
2032                 goto err_free_g;
2033
2034         TAILQ_INIT(&priv->vxlan.head);
2035         return (0);
2036
2037 err_free_g:
2038         kfree(ft->g);
2039
2040 err_destroy_vxlan_flow_table:
2041         mlx5_destroy_flow_table(ft->t);
2042         ft->t = NULL;
2043
2044         return (err);
2045 }
2046
2047 #define MLX5E_NUM_INNER_RSS_GROUPS      3
2048 #define MLX5E_INNER_RSS_GROUP0_SIZE     BIT(3)
2049 #define MLX5E_INNER_RSS_GROUP1_SIZE     BIT(1)
2050 #define MLX5E_INNER_RSS_GROUP2_SIZE     BIT(0)
2051 #define MLX5E_INNER_RSS_TABLE_SIZE      (MLX5E_INNER_RSS_GROUP0_SIZE +\
2052                                          MLX5E_INNER_RSS_GROUP1_SIZE +\
2053                                          MLX5E_INNER_RSS_GROUP2_SIZE +\
2054                                          0)
2055
2056 static int
2057 mlx5e_create_inner_rss_groups_sub(struct mlx5e_flow_table *ft, u32 *in,
2058                                            int inlen)
2059 {
2060         u8 *mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
2061         int err;
2062         int ix = 0;
2063
2064         memset(in, 0, inlen);
2065         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_INNER_HEADERS);
2066         MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ethertype);
2067         MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ip_protocol);
2068         MLX5_SET_CFG(in, start_flow_index, ix);
2069         ix += MLX5E_INNER_RSS_GROUP0_SIZE;
2070         MLX5_SET_CFG(in, end_flow_index, ix - 1);
2071         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
2072         if (IS_ERR(ft->g[ft->num_groups]))
2073                 goto err_destory_groups;
2074         ft->num_groups++;
2075
2076         memset(in, 0, inlen);
2077         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_INNER_HEADERS);
2078         MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ethertype);
2079         MLX5_SET_CFG(in, start_flow_index, ix);
2080         ix += MLX5E_INNER_RSS_GROUP1_SIZE;
2081         MLX5_SET_CFG(in, end_flow_index, ix - 1);
2082         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
2083         if (IS_ERR(ft->g[ft->num_groups]))
2084                 goto err_destory_groups;
2085         ft->num_groups++;
2086
2087         memset(in, 0, inlen);
2088         MLX5_SET_CFG(in, start_flow_index, ix);
2089         ix += MLX5E_INNER_RSS_GROUP2_SIZE;
2090         MLX5_SET_CFG(in, end_flow_index, ix - 1);
2091         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
2092         if (IS_ERR(ft->g[ft->num_groups]))
2093                 goto err_destory_groups;
2094         ft->num_groups++;
2095
2096         return (0);
2097
2098 err_destory_groups:
2099         err = PTR_ERR(ft->g[ft->num_groups]);
2100         ft->g[ft->num_groups] = NULL;
2101         mlx5e_destroy_groups(ft);
2102
2103         return (err);
2104 }
2105
2106 static int
2107 mlx5e_create_inner_rss_groups(struct mlx5e_flow_table *ft)
2108 {
2109         u32 *in;
2110         int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2111         int err;
2112
2113         in = mlx5_vzalloc(inlen);
2114         if (!in)
2115                 return (-ENOMEM);
2116
2117         err = mlx5e_create_inner_rss_groups_sub(ft, in, inlen);
2118
2119         kvfree(in);
2120         return (err);
2121 }
2122
2123 static int
2124 mlx5e_create_inner_rss_flow_table(struct mlx5e_priv *priv)
2125 {
2126         struct mlx5e_flow_table *ft = &priv->fts.inner_rss;
2127         int err;
2128
2129         ft->num_groups = 0;
2130         ft->t = mlx5_create_flow_table(priv->fts.ns, 0, "inner_rss",
2131                                        MLX5E_INNER_RSS_TABLE_SIZE);
2132
2133         if (IS_ERR(ft->t)) {
2134                 err = PTR_ERR(ft->t);
2135                 ft->t = NULL;
2136                 return (err);
2137         }
2138         ft->g = kcalloc(MLX5E_NUM_INNER_RSS_GROUPS, sizeof(*ft->g),
2139                         GFP_KERNEL);
2140         if (!ft->g) {
2141                 err = -ENOMEM;
2142                 goto err_destroy_inner_rss_flow_table;
2143         }
2144
2145         err = mlx5e_create_inner_rss_groups(ft);
2146         if (err)
2147                 goto err_free_g;
2148
2149         return (0);
2150
2151 err_free_g:
2152         kfree(ft->g);
2153
2154 err_destroy_inner_rss_flow_table:
2155         mlx5_destroy_flow_table(ft->t);
2156         ft->t = NULL;
2157
2158         return (err);
2159 }
2160
2161 static void mlx5e_destroy_inner_rss_flow_table(struct mlx5e_priv *priv)
2162 {
2163         mlx5e_destroy_flow_table(&priv->fts.inner_rss);
2164 }
2165
2166 static void
2167 mlx5e_destroy_vxlan_flow_table(struct mlx5e_priv *priv)
2168 {
2169         mlx5e_destroy_flow_table(&priv->fts.vxlan);
2170 }
2171
2172 int
2173 mlx5e_open_flow_tables(struct mlx5e_priv *priv)
2174 {
2175         int err;
2176
2177         /* setup namespace pointer */
2178         priv->fts.ns = mlx5_get_flow_namespace(
2179             priv->mdev, MLX5_FLOW_NAMESPACE_KERNEL);
2180
2181         err = mlx5e_create_vlan_flow_table(priv);
2182         if (err)
2183                 return (err);
2184
2185         err = mlx5e_create_vxlan_flow_table(priv);
2186         if (err)
2187                 goto err_destroy_vlan_flow_table;
2188
2189         err = mlx5e_create_main_flow_table(priv, true);
2190         if (err)
2191                 goto err_destroy_vxlan_flow_table;
2192
2193         err = mlx5e_create_inner_rss_flow_table(priv);
2194         if (err)
2195                 goto err_destroy_main_flow_table_true;
2196
2197         err = mlx5e_create_main_flow_table(priv, false);
2198         if (err)
2199                 goto err_destroy_inner_rss_flow_table;
2200
2201         err = mlx5e_add_vxlan_catchall_rule(priv);
2202         if (err)
2203                 goto err_destroy_main_flow_table_false;
2204
2205         err = mlx5e_accel_fs_tcp_create(priv);
2206         if (err)
2207                 goto err_del_vxlan_catchall_rule;
2208
2209         return (0);
2210
2211 err_del_vxlan_catchall_rule:
2212         mlx5e_del_vxlan_catchall_rule(priv);
2213 err_destroy_main_flow_table_false:
2214         mlx5e_destroy_main_flow_table(priv);
2215 err_destroy_inner_rss_flow_table:
2216         mlx5e_destroy_inner_rss_flow_table(priv);
2217 err_destroy_main_flow_table_true:
2218         mlx5e_destroy_main_vxlan_flow_table(priv);
2219 err_destroy_vxlan_flow_table:
2220         mlx5e_destroy_vxlan_flow_table(priv);
2221 err_destroy_vlan_flow_table:
2222         mlx5e_destroy_vlan_flow_table(priv);
2223
2224         return (err);
2225 }
2226
2227 void
2228 mlx5e_close_flow_tables(struct mlx5e_priv *priv)
2229 {
2230         mlx5e_accel_fs_tcp_destroy(priv);
2231         mlx5e_del_vxlan_catchall_rule(priv);
2232         mlx5e_destroy_main_flow_table(priv);
2233         mlx5e_destroy_inner_rss_flow_table(priv);
2234         mlx5e_destroy_main_vxlan_flow_table(priv);
2235         mlx5e_destroy_vxlan_flow_table(priv);
2236         mlx5e_destroy_vlan_flow_table(priv);
2237 }
2238
2239 int
2240 mlx5e_open_flow_rules(struct mlx5e_priv *priv)
2241 {
2242         int err;
2243
2244         err = mlx5e_add_all_vlan_rules(priv);
2245         if (err)
2246                 return (err);
2247
2248         err = mlx5e_add_main_vxlan_rules(priv);
2249         if (err)
2250                 goto err_del_all_vlan_rules;
2251
2252         err = mlx5e_add_all_vxlan_rules(priv);
2253         if (err)
2254                 goto err_del_main_vxlan_rules;
2255
2256         mlx5e_set_rx_mode_core(priv, true);
2257
2258         set_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state);
2259
2260         return (0);
2261
2262 err_del_main_vxlan_rules:
2263         mlx5e_del_main_vxlan_rules(priv);
2264
2265 err_del_all_vlan_rules:
2266         mlx5e_del_all_vlan_rules(priv);
2267
2268         return (err);
2269 }
2270
2271 void
2272 mlx5e_close_flow_rules(struct mlx5e_priv *priv)
2273 {
2274         clear_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state);
2275
2276         mlx5e_set_rx_mode_core(priv, false);
2277         mlx5e_del_all_vxlan_rules(priv);
2278         mlx5e_del_main_vxlan_rules(priv);
2279         mlx5e_del_all_vlan_rules(priv);
2280 }