sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c

   1 /*-
   2  * Copyright (c) 2015-2021 Mellanox Technologies. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
  14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  16  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
  17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  23  * SUCH DAMAGE.
  24  */
  25
  26 #include "opt_rss.h"
  27 #include "opt_ratelimit.h"
  28
  29 #include <dev/mlx5/mlx5_en/en.h>
  30
  31 #include <linux/list.h>
  32 #include <dev/mlx5/fs.h>
  33 #include <dev/mlx5/mpfs.h>
  34 #include <dev/mlx5/mlx5_core/fs_tcp.h>
  35
  36 /*
  37  * The flow tables with rules define the packet processing on receive.
  38  * Currently the following structure is set up to handle different
  39  * offloads like TLS RX offload, VLAN decapsulation, packet
  40  * classification, RSS hashing, VxLAN checksum offloading:
  41  *
  42  *   +=========+       +=========+      +=================+
  43  *   |TCP/IPv4 |       |TCP/IPv4 |      |TCP/IPv4 Match   |
  44  *   |Flowtable|------>|         |----->|Outer Proto Match|=====> TLS TIR n
  45  *   |         |       |Catch-all|\     |                 |
  46  *   +=========+       +=========+|     +=================+
  47  *                                |
  48  *       +------------------------+
  49  *       V
  50  *   +=========+       +=========+      +=================+
  51  *   |TCP/IPv6 |       |TCP/IPv6 |      |TCP/IPv6 Match   |
  52  *   |Flowtable|------>|         |----->|Outer Proto Match|=====> TLS TIR n
  53  *   |         |       |Catch-all|\     |                 |
  54  *   +=========+       +=========+|     +=================+
  55  *                                |
  56  *       +------------------------+
  57  *       V
  58  *   +=========+       +=========+      +=================+
  59  *   |VLAN ft: |       |VxLAN    |      |VxLAN Main       |
  60  *   |CTAG/STAG|------>|      VNI|----->|Inner Proto Match|=====> Inner TIR n
  61  *   |VID/noVID|/      |Catch-all|\     |                 |
  62  *   +=========+       +=========+|     +=================+
  63  *                                |
  64  *                                |
  65  *                                |
  66  *                                v
  67  *                      +=================+
  68  *                      |Main             |
  69  *                      |Outer Proto Match|=====> TIR n
  70  *                      |                 |
  71  *                      +=================+
  72  *
  73  * The path through flow rules directs each packet into an appropriate TIR,
  74  * according to the:
  75  * - VLAN encapsulation
  76  * - Outer protocol
  77  * - Presence of inner protocol
  78  */
  79
  80 #define MLX5_SET_CFG(p, f, v) MLX5_SET(create_flow_group_in, p, f, v)
  81
  82 enum {
  83         MLX5E_FULLMATCH = 0,
  84         MLX5E_ALLMULTI = 1,
  85         MLX5E_PROMISC = 2,
  86 };
  87
  88 enum {
  89         MLX5E_UC = 0,
  90         MLX5E_MC_IPV4 = 1,
  91         MLX5E_MC_IPV6 = 2,
  92         MLX5E_MC_OTHER = 3,
  93 };
  94
  95 enum {
  96         MLX5E_ACTION_NONE = 0,
  97         MLX5E_ACTION_ADD = 1,
  98         MLX5E_ACTION_DEL = 2,
  99 };
 100
 101 struct mlx5e_eth_addr_hash_node {
 102         LIST_ENTRY(mlx5e_eth_addr_hash_node) hlist;
 103         u8      action;
 104         u32     mpfs_index;
 105         struct mlx5e_eth_addr_info ai;
 106 };
 107
 108 static void mlx5e_del_all_vlan_rules(struct mlx5e_priv *);
 109
 110 static inline int
 111 mlx5e_hash_eth_addr(const u8 * addr)
 112 {
 113         return (addr[5]);
 114 }
 115
 116 static bool
 117 mlx5e_add_eth_addr_to_hash(struct mlx5e_eth_addr_hash_head *hash,
 118     struct mlx5e_eth_addr_hash_node *hn_new)
 119 {
 120         struct mlx5e_eth_addr_hash_node *hn;
 121         u32 ix = mlx5e_hash_eth_addr(hn_new->ai.addr);
 122
 123         LIST_FOREACH(hn, &hash[ix], hlist) {
 124                 if (bcmp(hn->ai.addr, hn_new->ai.addr, ETHER_ADDR_LEN) == 0) {
 125                         if (hn->action == MLX5E_ACTION_DEL)
 126                                 hn->action = MLX5E_ACTION_NONE;
 127                         free(hn_new, M_MLX5EN);
 128                         return (false);
 129                 }
 130         }
 131         LIST_INSERT_HEAD(&hash[ix], hn_new, hlist);
 132         return (true);
 133 }
 134
 135 static void
 136 mlx5e_del_eth_addr_from_hash(struct mlx5e_eth_addr_hash_node *hn)
 137 {
 138         LIST_REMOVE(hn, hlist);
 139         free(hn, M_MLX5EN);
 140 }
 141
 142 static void
 143 mlx5e_del_eth_addr_from_flow_table(struct mlx5e_priv *priv,
 144     struct mlx5e_eth_addr_info *ai)
 145 {
 146         mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_IPSEC_ESP]);
 147         mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_IPSEC_ESP]);
 148         mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_IPSEC_AH]);
 149         mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_IPSEC_AH]);
 150         mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_TCP]);
 151         mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_TCP]);
 152         mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_UDP]);
 153         mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_UDP]);
 154         mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6]);
 155         mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4]);
 156         mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_ANY]);
 157 }
 158
 159 static int
 160 mlx5e_get_eth_addr_type(const u8 * addr)
 161 {
 162         if (ETHER_IS_MULTICAST(addr) == 0)
 163                 return (MLX5E_UC);
 164
 165         if ((addr[0] == 0x01) &&
 166             (addr[1] == 0x00) &&
 167             (addr[2] == 0x5e) &&
 168             !(addr[3] & 0x80))
 169                 return (MLX5E_MC_IPV4);
 170
 171         if ((addr[0] == 0x33) &&
 172             (addr[1] == 0x33))
 173                 return (MLX5E_MC_IPV6);
 174
 175         return (MLX5E_MC_OTHER);
 176 }
 177
 178 static  u32
 179 mlx5e_get_tt_vec(struct mlx5e_eth_addr_info *ai, int type)
 180 {
 181         int eth_addr_type;
 182         u32 ret;
 183
 184         switch (type) {
 185         case MLX5E_FULLMATCH:
 186                 eth_addr_type = mlx5e_get_eth_addr_type(ai->addr);
 187                 switch (eth_addr_type) {
 188                 case MLX5E_UC:
 189                         ret =
 190                             (1 << MLX5E_TT_IPV4_TCP) |
 191                             (1 << MLX5E_TT_IPV6_TCP) |
 192                             (1 << MLX5E_TT_IPV4_UDP) |
 193                             (1 << MLX5E_TT_IPV6_UDP) |
 194                             (1 << MLX5E_TT_IPV4) |
 195                             (1 << MLX5E_TT_IPV6) |
 196                             (1 << MLX5E_TT_ANY) |
 197                             0;
 198                         break;
 199
 200                 case MLX5E_MC_IPV4:
 201                         ret =
 202                             (1 << MLX5E_TT_IPV4_UDP) |
 203                             (1 << MLX5E_TT_IPV4) |
 204                             0;
 205                         break;
 206
 207                 case MLX5E_MC_IPV6:
 208                         ret =
 209                             (1 << MLX5E_TT_IPV6_UDP) |
 210                             (1 << MLX5E_TT_IPV6) |
 211                             0;
 212                         break;
 213
 214                 default:
 215                         ret =
 216                             (1 << MLX5E_TT_ANY) |
 217                             0;
 218                         break;
 219                 }
 220                 break;
 221
 222         case MLX5E_ALLMULTI:
 223                 ret =
 224                     (1 << MLX5E_TT_IPV4_UDP) |
 225                     (1 << MLX5E_TT_IPV6_UDP) |
 226                     (1 << MLX5E_TT_IPV4) |
 227                     (1 << MLX5E_TT_IPV6) |
 228                     (1 << MLX5E_TT_ANY) |
 229                     0;
 230                 break;
 231
 232         default:                        /* MLX5E_PROMISC */
 233                 ret =
 234                     (1 << MLX5E_TT_IPV4_TCP) |
 235                     (1 << MLX5E_TT_IPV6_TCP) |
 236                     (1 << MLX5E_TT_IPV4_UDP) |
 237                     (1 << MLX5E_TT_IPV6_UDP) |
 238                     (1 << MLX5E_TT_IPV4) |
 239                     (1 << MLX5E_TT_IPV6) |
 240                     (1 << MLX5E_TT_ANY) |
 241                     0;
 242                 break;
 243         }
 244
 245         return (ret);
 246 }
 247
 248 static int
 249 mlx5e_add_eth_addr_rule_sub(struct mlx5e_priv *priv,
 250     struct mlx5e_eth_addr_info *ai, int type,
 251     u32 *mc, u32 *mv)
 252 {
 253         struct mlx5_flow_destination dest = {};
 254         u8 mc_enable = 0;
 255         struct mlx5_flow_rule **rule_p;
 256         struct mlx5_flow_table *ft = priv->fts.main.t;
 257         u8 *mc_dmac = MLX5_ADDR_OF(fte_match_param, mc,
 258                                    outer_headers.dmac_47_16);
 259         u8 *mv_dmac = MLX5_ADDR_OF(fte_match_param, mv,
 260                                    outer_headers.dmac_47_16);
 261         u32 *tirn = priv->tirn;
 262         u32 tt_vec;
 263         int err = 0;
 264         struct mlx5_flow_act flow_act = {
 265                 .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
 266                 .flow_tag = MLX5_FS_ETH_FLOW_TAG,
 267         };
 268
 269         dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
 270
 271         switch (type) {
 272         case MLX5E_FULLMATCH:
 273                 mc_enable = MLX5_MATCH_OUTER_HEADERS;
 274                 memset(mc_dmac, 0xff, ETH_ALEN);
 275                 ether_addr_copy(mv_dmac, ai->addr);
 276                 break;
 277
 278         case MLX5E_ALLMULTI:
 279                 mc_enable = MLX5_MATCH_OUTER_HEADERS;
 280                 mc_dmac[0] = 0x01;
 281                 mv_dmac[0] = 0x01;
 282                 break;
 283
 284         case MLX5E_PROMISC:
 285                 break;
 286         default:
 287                 break;
 288         }
 289
 290         tt_vec = mlx5e_get_tt_vec(ai, type);
 291
 292         if (tt_vec & BIT(MLX5E_TT_ANY)) {
 293                 rule_p = &ai->ft_rule[MLX5E_TT_ANY];
 294                 dest.tir_num = tirn[MLX5E_TT_ANY];
 295                 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 296                                              MLX5_FLOW_RULE_FWD_ACTION_DEST,
 297                                              &flow_act, &dest);
 298                 if (IS_ERR_OR_NULL(*rule_p))
 299                         goto err_del_ai;
 300         }
 301
 302         mc_enable = MLX5_MATCH_OUTER_HEADERS;
 303         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
 304
 305         if (tt_vec & BIT(MLX5E_TT_IPV4)) {
 306                 rule_p = &ai->ft_rule[MLX5E_TT_IPV4];
 307                 dest.tir_num = tirn[MLX5E_TT_IPV4];
 308                 MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 309                          ETHERTYPE_IP);
 310                 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 311                                              MLX5_FLOW_RULE_FWD_ACTION_DEST,
 312                                              &flow_act, &dest);
 313                 if (IS_ERR_OR_NULL(*rule_p))
 314                         goto err_del_ai;
 315         }
 316
 317         if (tt_vec & BIT(MLX5E_TT_IPV6)) {
 318                 rule_p = &ai->ft_rule[MLX5E_TT_IPV6];
 319                 dest.tir_num = tirn[MLX5E_TT_IPV6];
 320                 MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 321                          ETHERTYPE_IPV6);
 322                 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 323                                              MLX5_FLOW_RULE_FWD_ACTION_DEST,
 324                                              &flow_act, &dest);
 325                 if (IS_ERR_OR_NULL(*rule_p))
 326                         goto err_del_ai;
 327         }
 328
 329         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol);
 330         MLX5_SET(fte_match_param, mv, outer_headers.ip_protocol, IPPROTO_UDP);
 331
 332         if (tt_vec & BIT(MLX5E_TT_IPV4_UDP)) {
 333                 rule_p = &ai->ft_rule[MLX5E_TT_IPV4_UDP];
 334                 dest.tir_num = tirn[MLX5E_TT_IPV4_UDP];
 335                 MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 336                          ETHERTYPE_IP);
 337                 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 338                                              MLX5_FLOW_RULE_FWD_ACTION_DEST,
 339                                              &flow_act, &dest);
 340                 if (IS_ERR_OR_NULL(*rule_p))
 341                         goto err_del_ai;
 342         }
 343
 344         if (tt_vec & BIT(MLX5E_TT_IPV6_UDP)) {
 345                 rule_p = &ai->ft_rule[MLX5E_TT_IPV6_UDP];
 346                 dest.tir_num = tirn[MLX5E_TT_IPV6_UDP];
 347                 MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 348                          ETHERTYPE_IPV6);
 349                 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 350                                              MLX5_FLOW_RULE_FWD_ACTION_DEST,
 351                                              &flow_act, &dest);
 352                 if (IS_ERR_OR_NULL(*rule_p))
 353                         goto err_del_ai;
 354         }
 355
 356         MLX5_SET(fte_match_param, mv, outer_headers.ip_protocol, IPPROTO_TCP);
 357
 358         if (tt_vec & BIT(MLX5E_TT_IPV4_TCP)) {
 359                 rule_p = &ai->ft_rule[MLX5E_TT_IPV4_TCP];
 360                 dest.tir_num = tirn[MLX5E_TT_IPV4_TCP];
 361                 MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 362                          ETHERTYPE_IP);
 363                 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 364                                              MLX5_FLOW_RULE_FWD_ACTION_DEST,
 365                                              &flow_act, &dest);
 366                 if (IS_ERR_OR_NULL(*rule_p))
 367                         goto err_del_ai;
 368         }
 369
 370         if (tt_vec & BIT(MLX5E_TT_IPV6_TCP)) {
 371                 rule_p = &ai->ft_rule[MLX5E_TT_IPV6_TCP];
 372                 dest.tir_num = tirn[MLX5E_TT_IPV6_TCP];
 373                 MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 374                          ETHERTYPE_IPV6);
 375                 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 376                                              MLX5_FLOW_RULE_FWD_ACTION_DEST,
 377                                              &flow_act, &dest);
 378                 if (IS_ERR_OR_NULL(*rule_p))
 379                         goto err_del_ai;
 380         }
 381
 382         MLX5_SET(fte_match_param, mv, outer_headers.ip_protocol, IPPROTO_AH);
 383
 384         if (tt_vec & BIT(MLX5E_TT_IPV4_IPSEC_AH)) {
 385                 rule_p = &ai->ft_rule[MLX5E_TT_IPV4_IPSEC_AH];
 386                 dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_AH];
 387                 MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 388                          ETHERTYPE_IP);
 389                 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 390                                              MLX5_FLOW_RULE_FWD_ACTION_DEST,
 391                                              &flow_act, &dest);
 392                 if (IS_ERR_OR_NULL(*rule_p))
 393                         goto err_del_ai;
 394         }
 395
 396         if (tt_vec & BIT(MLX5E_TT_IPV6_IPSEC_AH)) {
 397                 rule_p = &ai->ft_rule[MLX5E_TT_IPV6_IPSEC_AH];
 398                 dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_AH];
 399                 MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 400                          ETHERTYPE_IPV6);
 401                 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 402                                              MLX5_FLOW_RULE_FWD_ACTION_DEST,
 403                                              &flow_act, &dest);
 404                 if (IS_ERR_OR_NULL(*rule_p))
 405                         goto err_del_ai;
 406         }
 407
 408         MLX5_SET(fte_match_param, mv, outer_headers.ip_protocol, IPPROTO_ESP);
 409
 410         if (tt_vec & BIT(MLX5E_TT_IPV4_IPSEC_ESP)) {
 411                 rule_p = &ai->ft_rule[MLX5E_TT_IPV4_IPSEC_ESP];
 412                 dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_ESP];
 413                 MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 414                          ETHERTYPE_IP);
 415                 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 416                                              MLX5_FLOW_RULE_FWD_ACTION_DEST,
 417                                              &flow_act, &dest);
 418                 if (IS_ERR_OR_NULL(*rule_p))
 419                         goto err_del_ai;
 420         }
 421
 422         if (tt_vec & BIT(MLX5E_TT_IPV6_IPSEC_ESP)) {
 423                 rule_p = &ai->ft_rule[MLX5E_TT_IPV6_IPSEC_ESP];
 424                 dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_ESP];
 425                 MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
 426                          ETHERTYPE_IPV6);
 427                 *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 428                                              MLX5_FLOW_RULE_FWD_ACTION_DEST,
 429                                              &flow_act, &dest);
 430                 if (IS_ERR_OR_NULL(*rule_p))
 431                         goto err_del_ai;
 432         }
 433
 434         return 0;
 435
 436 err_del_ai:
 437         err = PTR_ERR(*rule_p);
 438         *rule_p = NULL;
 439         mlx5e_del_eth_addr_from_flow_table(priv, ai);
 440
 441         return err;
 442 }
 443
 444 static int
 445 mlx5e_add_eth_addr_rule(struct mlx5e_priv *priv,
 446     struct mlx5e_eth_addr_info *ai, int type)
 447 {
 448         u32 *match_criteria;
 449         u32 *match_value;
 450         int err = 0;
 451
 452         match_value     = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
 453         match_criteria  = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
 454         if (!match_value || !match_criteria) {
 455                 mlx5_en_err(priv->ifp, "alloc failed\n");
 456                 err = -ENOMEM;
 457                 goto add_eth_addr_rule_out;
 458         }
 459         err = mlx5e_add_eth_addr_rule_sub(priv, ai, type, match_criteria,
 460             match_value);
 461
 462 add_eth_addr_rule_out:
 463         kvfree(match_criteria);
 464         kvfree(match_value);
 465
 466         return (err);
 467 }
 468
 469 static void
 470 mlx5e_del_main_vxlan_rules(struct mlx5e_priv *priv)
 471 {
 472         mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_ESP]);
 473         mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_ESP]);
 474         mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_AH]);
 475         mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_AH]);
 476         mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_TCP]);
 477         mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_TCP]);
 478         mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_UDP]);
 479         mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_UDP]);
 480         mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6]);
 481         mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4]);
 482         mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_ANY]);
 483 }
 484
 485 static int
 486 mlx5e_add_main_vxlan_rules_sub(struct mlx5e_priv *priv, u32 *mc, u32 *mv)
 487 {
 488         struct mlx5_flow_destination dest = {};
 489         u8 mc_enable = 0;
 490         struct mlx5_flow_rule **rule_p;
 491         struct mlx5_flow_table *ft = priv->fts.main_vxlan.t;
 492         u32 *tirn = priv->tirn_inner_vxlan;
 493         struct mlx5_flow_act flow_act = {
 494                 .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
 495                 .flow_tag = MLX5_FS_ETH_FLOW_TAG,
 496         };
 497         int err = 0;
 498
 499         dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
 500
 501         mc_enable = MLX5_MATCH_INNER_HEADERS;
 502         MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ethertype);
 503
 504         rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4];
 505         dest.tir_num = tirn[MLX5E_TT_IPV4];
 506         MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
 507         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 508             MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
 509         if (IS_ERR_OR_NULL(*rule_p))
 510                 goto err_del_ai;
 511
 512         rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6];
 513         dest.tir_num = tirn[MLX5E_TT_IPV6];
 514         MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6);
 515         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 516              MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
 517         if (IS_ERR_OR_NULL(*rule_p))
 518                 goto err_del_ai;
 519
 520         MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ip_protocol);
 521         MLX5_SET(fte_match_param, mv, inner_headers.ip_protocol, IPPROTO_UDP);
 522
 523         rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_UDP];
 524         dest.tir_num = tirn[MLX5E_TT_IPV4_UDP];
 525         MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
 526         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 527             MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
 528         if (IS_ERR_OR_NULL(*rule_p))
 529                 goto err_del_ai;
 530
 531         rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_UDP];
 532         dest.tir_num = tirn[MLX5E_TT_IPV6_UDP];
 533         MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6);
 534         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 535             MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
 536         if (IS_ERR_OR_NULL(*rule_p))
 537                 goto err_del_ai;
 538
 539         MLX5_SET(fte_match_param, mv, inner_headers.ip_protocol, IPPROTO_TCP);
 540
 541         rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_TCP];
 542         dest.tir_num = tirn[MLX5E_TT_IPV4_TCP];
 543         MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
 544         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 545             MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
 546         if (IS_ERR_OR_NULL(*rule_p))
 547                 goto err_del_ai;
 548
 549         rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_TCP];
 550         dest.tir_num = tirn[MLX5E_TT_IPV6_TCP];
 551         MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6);
 552         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 553             MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
 554         if (IS_ERR_OR_NULL(*rule_p))
 555                 goto err_del_ai;
 556
 557         MLX5_SET(fte_match_param, mv, inner_headers.ip_protocol, IPPROTO_AH);
 558
 559         rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_AH];
 560         dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_AH];
 561         MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
 562         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 563             MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
 564         if (IS_ERR_OR_NULL(*rule_p))
 565                 goto err_del_ai;
 566
 567         rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_AH];
 568         dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_AH];
 569         MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6);
 570         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 571             MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
 572         if (IS_ERR_OR_NULL(*rule_p))
 573                 goto err_del_ai;
 574
 575         MLX5_SET(fte_match_param, mv, inner_headers.ip_protocol, IPPROTO_ESP);
 576
 577         rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_ESP];
 578         dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_ESP];
 579         MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
 580         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 581             MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
 582         if (IS_ERR_OR_NULL(*rule_p))
 583                         goto err_del_ai;
 584
 585         rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_ESP];
 586         dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_ESP];
 587         MLX5_SET(fte_match_param, mv, inner_headers.ethertype,
 588                  ETHERTYPE_IPV6);
 589         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 590             MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
 591         if (IS_ERR_OR_NULL(*rule_p))
 592                 goto err_del_ai;
 593
 594         mc_enable = 0;
 595         memset(mv, 0, MLX5_ST_SZ_BYTES(fte_match_param));
 596         memset(mc, 0, MLX5_ST_SZ_BYTES(fte_match_param));
 597         rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_ANY];
 598         dest.tir_num = tirn[MLX5E_TT_ANY];
 599         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 600             MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
 601         if (IS_ERR_OR_NULL(*rule_p))
 602                 goto err_del_ai;
 603
 604         return (0);
 605
 606 err_del_ai:
 607         err = PTR_ERR(*rule_p);
 608         *rule_p = NULL;
 609         mlx5e_del_main_vxlan_rules(priv);
 610
 611         return (err);
 612 }
 613
 614 static int
 615 mlx5e_add_main_vxlan_rules(struct mlx5e_priv *priv)
 616 {
 617         u32 *match_criteria;
 618         u32 *match_value;
 619         int err = 0;
 620
 621         match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
 622         match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
 623         if (match_value == NULL || match_criteria == NULL) {
 624                 mlx5_en_err(priv->ifp, "alloc failed\n");
 625                 err = -ENOMEM;
 626                 goto add_main_vxlan_rules_out;
 627         }
 628         err = mlx5e_add_main_vxlan_rules_sub(priv, match_criteria, match_value);
 629
 630 add_main_vxlan_rules_out:
 631         kvfree(match_criteria);
 632         kvfree(match_value);
 633
 634         return (err);
 635 }
 636
 637 static int mlx5e_vport_context_update_vlans(struct mlx5e_priv *priv)
 638 {
 639         if_t ifp = priv->ifp;
 640         int max_list_size;
 641         int list_size;
 642         u16 *vlans;
 643         int vlan;
 644         int err;
 645         int i;
 646
 647         list_size = 0;
 648         for_each_set_bit(vlan, priv->vlan.active_vlans, VLAN_N_VID)
 649                 list_size++;
 650
 651         max_list_size = 1 << MLX5_CAP_GEN(priv->mdev, log_max_vlan_list);
 652
 653         if (list_size > max_list_size) {
 654                 mlx5_en_err(ifp,
 655                             "ifnet vlans list size (%d) > (%d) max vport list size, some vlans will be dropped\n",
 656                             list_size, max_list_size);
 657                 list_size = max_list_size;
 658         }
 659
 660         vlans = kcalloc(list_size, sizeof(*vlans), GFP_KERNEL);
 661         if (!vlans)
 662                 return -ENOMEM;
 663
 664         i = 0;
 665         for_each_set_bit(vlan, priv->vlan.active_vlans, VLAN_N_VID) {
 666                 if (i >= list_size)
 667                         break;
 668                 vlans[i++] = vlan;
 669         }
 670
 671         err = mlx5_modify_nic_vport_vlans(priv->mdev, vlans, list_size);
 672         if (err)
 673                 mlx5_en_err(ifp, "Failed to modify vport vlans list err(%d)\n",
 674                            err);
 675
 676         kfree(vlans);
 677         return err;
 678 }
 679
 680 enum mlx5e_vlan_rule_type {
 681         MLX5E_VLAN_RULE_TYPE_UNTAGGED,
 682         MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID,
 683         MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID,
 684         MLX5E_VLAN_RULE_TYPE_MATCH_VID,
 685 };
 686
 687 static int
 688 mlx5e_add_vlan_rule_sub(struct mlx5e_priv *priv,
 689     enum mlx5e_vlan_rule_type rule_type, u16 vid,
 690     u32 *mc, u32 *mv)
 691 {
 692         struct mlx5_flow_table *ft = priv->fts.vlan.t;
 693         struct mlx5_flow_destination dest = {};
 694         u8 mc_enable = 0;
 695         struct mlx5_flow_rule **rule_p;
 696         int err = 0;
 697         struct mlx5_flow_act flow_act = {
 698                 .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
 699                 .flow_tag = MLX5_FS_ETH_FLOW_TAG,
 700         };
 701
 702         dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 703         dest.ft = priv->fts.vxlan.t;
 704
 705         mc_enable = MLX5_MATCH_OUTER_HEADERS;
 706
 707         switch (rule_type) {
 708         case MLX5E_VLAN_RULE_TYPE_UNTAGGED:
 709                 rule_p = &priv->vlan.untagged_ft_rule;
 710                 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.cvlan_tag);
 711                 break;
 712         case MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID:
 713                 rule_p = &priv->vlan.any_cvlan_ft_rule;
 714                 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.cvlan_tag);
 715                 MLX5_SET(fte_match_param, mv, outer_headers.cvlan_tag, 1);
 716                 break;
 717         case MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID:
 718                 rule_p = &priv->vlan.any_svlan_ft_rule;
 719                 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.svlan_tag);
 720                 MLX5_SET(fte_match_param, mv, outer_headers.svlan_tag, 1);
 721                 break;
 722         default: /* MLX5E_VLAN_RULE_TYPE_MATCH_VID */
 723                 rule_p = &priv->vlan.active_vlans_ft_rule[vid];
 724                 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.cvlan_tag);
 725                 MLX5_SET(fte_match_param, mv, outer_headers.cvlan_tag, 1);
 726                 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.first_vid);
 727                 MLX5_SET(fte_match_param, mv, outer_headers.first_vid, vid);
 728                 mlx5e_vport_context_update_vlans(priv);
 729                 break;
 730         }
 731
 732         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
 733                                      MLX5_FLOW_RULE_FWD_ACTION_DEST,
 734                                      &flow_act,
 735                                      &dest);
 736
 737         if (IS_ERR(*rule_p)) {
 738                 err = PTR_ERR(*rule_p);
 739                 *rule_p = NULL;
 740                 mlx5_en_err(priv->ifp, "add rule failed\n");
 741         }
 742
 743         return (err);
 744 }
 745
 746 static int
 747 mlx5e_add_vlan_rule(struct mlx5e_priv *priv,
 748     enum mlx5e_vlan_rule_type rule_type, u16 vid)
 749 {
 750         u32 *match_criteria;
 751         u32 *match_value;
 752         int err = 0;
 753
 754         match_value     = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
 755         match_criteria  = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
 756         if (!match_value || !match_criteria) {
 757                 mlx5_en_err(priv->ifp, "alloc failed\n");
 758                 err = -ENOMEM;
 759                 goto add_vlan_rule_out;
 760         }
 761
 762         err = mlx5e_add_vlan_rule_sub(priv, rule_type, vid, match_criteria,
 763                                     match_value);
 764
 765 add_vlan_rule_out:
 766         kvfree(match_criteria);
 767         kvfree(match_value);
 768
 769         return (err);
 770 }
 771
 772 static void
 773 mlx5e_del_vlan_rule(struct mlx5e_priv *priv,
 774     enum mlx5e_vlan_rule_type rule_type, u16 vid)
 775 {
 776         switch (rule_type) {
 777         case MLX5E_VLAN_RULE_TYPE_UNTAGGED:
 778                 mlx5_del_flow_rule(&priv->vlan.untagged_ft_rule);
 779                 break;
 780         case MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID:
 781                 mlx5_del_flow_rule(&priv->vlan.any_cvlan_ft_rule);
 782                 break;
 783         case MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID:
 784                 mlx5_del_flow_rule(&priv->vlan.any_svlan_ft_rule);
 785                 break;
 786         case MLX5E_VLAN_RULE_TYPE_MATCH_VID:
 787                 mlx5_del_flow_rule(&priv->vlan.active_vlans_ft_rule[vid]);
 788                 mlx5e_vport_context_update_vlans(priv);
 789                 break;
 790         default:
 791                 break;
 792         }
 793 }
 794
 795 static void
 796 mlx5e_del_any_vid_rules(struct mlx5e_priv *priv)
 797 {
 798         mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0);
 799         mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID, 0);
 800 }
 801
 802 static int
 803 mlx5e_add_any_vid_rules(struct mlx5e_priv *priv)
 804 {
 805         int err;
 806
 807         err = mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0);
 808         if (err)
 809                 return (err);
 810
 811         err = mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID, 0);
 812         if (err)
 813                 mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0);
 814
 815         return (err);
 816 }
 817
 818 void
 819 mlx5e_enable_vlan_filter(struct mlx5e_priv *priv)
 820 {
 821         if (priv->vlan.filter_disabled) {
 822                 priv->vlan.filter_disabled = false;
 823                 if (if_getflags(priv->ifp) & IFF_PROMISC)
 824                         return;
 825                 if (test_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state))
 826                         mlx5e_del_any_vid_rules(priv);
 827         }
 828 }
 829
 830 void
 831 mlx5e_disable_vlan_filter(struct mlx5e_priv *priv)
 832 {
 833         if (!priv->vlan.filter_disabled) {
 834                 priv->vlan.filter_disabled = true;
 835                 if (if_getflags(priv->ifp) & IFF_PROMISC)
 836                         return;
 837                 if (test_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state))
 838                         mlx5e_add_any_vid_rules(priv);
 839         }
 840 }
 841
 842 void
 843 mlx5e_vlan_rx_add_vid(void *arg, if_t ifp, u16 vid)
 844 {
 845         struct mlx5e_priv *priv = arg;
 846
 847         if (ifp != priv->ifp)
 848                 return;
 849
 850         PRIV_LOCK(priv);
 851         if (!test_and_set_bit(vid, priv->vlan.active_vlans) &&
 852             test_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state))
 853                 mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_VID, vid);
 854         PRIV_UNLOCK(priv);
 855 }
 856
 857 void
 858 mlx5e_vlan_rx_kill_vid(void *arg, if_t ifp, u16 vid)
 859 {
 860         struct mlx5e_priv *priv = arg;
 861
 862         if (ifp != priv->ifp)
 863                 return;
 864
 865         PRIV_LOCK(priv);
 866         clear_bit(vid, priv->vlan.active_vlans);
 867         if (test_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state))
 868                 mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_VID, vid);
 869         PRIV_UNLOCK(priv);
 870 }
 871
 872 static int
 873 mlx5e_add_all_vlan_rules(struct mlx5e_priv *priv)
 874 {
 875         int err;
 876         int i;
 877
 878         set_bit(0, priv->vlan.active_vlans);
 879         for_each_set_bit(i, priv->vlan.active_vlans, VLAN_N_VID) {
 880                 err = mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_VID,
 881                                           i);
 882                 if (err)
 883                         goto error;
 884         }
 885
 886         err = mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_UNTAGGED, 0);
 887         if (err)
 888                 goto error;
 889
 890         if (priv->vlan.filter_disabled) {
 891                 err = mlx5e_add_any_vid_rules(priv);
 892                 if (err)
 893                         goto error;
 894         }
 895         return (0);
 896 error:
 897         mlx5e_del_all_vlan_rules(priv);
 898         return (err);
 899 }
 900
 901 static void
 902 mlx5e_del_all_vlan_rules(struct mlx5e_priv *priv)
 903 {
 904         int i;
 905
 906         if (priv->vlan.filter_disabled)
 907                 mlx5e_del_any_vid_rules(priv);
 908
 909         mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_UNTAGGED, 0);
 910
 911         for_each_set_bit(i, priv->vlan.active_vlans, VLAN_N_VID)
 912                 mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_VID, i);
 913         clear_bit(0, priv->vlan.active_vlans);
 914 }
 915
 916 #define mlx5e_for_each_hash_node(hn, tmp, hash, i) \
 917         for (i = 0; i < MLX5E_ETH_ADDR_HASH_SIZE; i++) \
 918                 LIST_FOREACH_SAFE(hn, &(hash)[i], hlist, tmp)
 919
 920 static void
 921 mlx5e_execute_action(struct mlx5e_priv *priv,
 922     struct mlx5e_eth_addr_hash_node *hn)
 923 {
 924         switch (hn->action) {
 925         case MLX5E_ACTION_ADD:
 926                 mlx5e_add_eth_addr_rule(priv, &hn->ai, MLX5E_FULLMATCH);
 927                 hn->action = MLX5E_ACTION_NONE;
 928                 break;
 929
 930         case MLX5E_ACTION_DEL:
 931                 mlx5e_del_eth_addr_from_flow_table(priv, &hn->ai);
 932                 if (hn->mpfs_index != -1U)
 933                         mlx5_mpfs_del_mac(priv->mdev, hn->mpfs_index);
 934                 mlx5e_del_eth_addr_from_hash(hn);
 935                 break;
 936
 937         default:
 938                 break;
 939         }
 940 }
 941
 942 static struct mlx5e_eth_addr_hash_node *
 943 mlx5e_move_hn(struct mlx5e_eth_addr_hash_head *fh, struct mlx5e_eth_addr_hash_head *uh)
 944 {
 945         struct mlx5e_eth_addr_hash_node *hn;
 946
 947         hn = LIST_FIRST(fh);
 948         if (hn != NULL) {
 949                 LIST_REMOVE(hn, hlist);
 950                 LIST_INSERT_HEAD(uh, hn, hlist);
 951         }
 952         return (hn);
 953 }
 954
 955 static struct mlx5e_eth_addr_hash_node *
 956 mlx5e_remove_hn(struct mlx5e_eth_addr_hash_head *fh)
 957 {
 958         struct mlx5e_eth_addr_hash_node *hn;
 959
 960         hn = LIST_FIRST(fh);
 961         if (hn != NULL)
 962                 LIST_REMOVE(hn, hlist);
 963         return (hn);
 964 }
 965
 966 struct mlx5e_copy_addr_ctx {
 967         struct mlx5e_eth_addr_hash_head *free;
 968         struct mlx5e_eth_addr_hash_head *fill;
 969         bool success;
 970 };
 971
 972 static u_int
 973 mlx5e_copy_addr(void *arg, struct sockaddr_dl *sdl, u_int cnt)
 974 {
 975         struct mlx5e_copy_addr_ctx *ctx = arg;
 976         struct mlx5e_eth_addr_hash_node *hn;
 977
 978         hn = mlx5e_move_hn(ctx->free, ctx->fill);
 979         if (hn == NULL) {
 980                 ctx->success = false;
 981                 return (0);
 982         }
 983         ether_addr_copy(hn->ai.addr, LLADDR(sdl));
 984
 985         return (1);
 986 }
 987
 988 static void
 989 mlx5e_sync_ifp_addr(struct mlx5e_priv *priv)
 990 {
 991         struct mlx5e_copy_addr_ctx ctx;
 992         struct mlx5e_eth_addr_hash_head head_free;
 993         struct mlx5e_eth_addr_hash_head head_uc;
 994         struct mlx5e_eth_addr_hash_head head_mc;
 995         struct mlx5e_eth_addr_hash_node *hn;
 996         if_t ifp = priv->ifp;
 997         size_t x;
 998         size_t num;
 999
1000         PRIV_ASSERT_LOCKED(priv);
1001
1002 retry:
1003         LIST_INIT(&head_free);
1004         LIST_INIT(&head_uc);
1005         LIST_INIT(&head_mc);
1006         num = 1 + if_lladdr_count(ifp) + if_llmaddr_count(ifp);
1007
1008         /* allocate place holders */
1009         for (x = 0; x != num; x++) {
1010                 hn = malloc(sizeof(*hn), M_MLX5EN, M_WAITOK | M_ZERO);
1011                 hn->action = MLX5E_ACTION_ADD;
1012                 hn->mpfs_index = -1U;
1013                 LIST_INSERT_HEAD(&head_free, hn, hlist);
1014         }
1015
1016         hn = mlx5e_move_hn(&head_free, &head_uc);
1017         MPASS(hn != NULL);
1018
1019         ether_addr_copy(hn->ai.addr, if_getlladdr(ifp));
1020
1021         ctx.free = &head_free;
1022         ctx.fill = &head_uc;
1023         ctx.success = true;
1024         if_foreach_lladdr(ifp, mlx5e_copy_addr, &ctx);
1025         if (ctx.success == false)
1026                 goto cleanup;
1027
1028         ctx.fill = &head_mc;
1029         if_foreach_llmaddr(ifp, mlx5e_copy_addr, &ctx);
1030         if (ctx.success == false)
1031                 goto cleanup;
1032
1033         /* insert L2 unicast addresses into hash list */
1034
1035         while ((hn = mlx5e_remove_hn(&head_uc)) != NULL) {
1036                 if (mlx5e_add_eth_addr_to_hash(priv->eth_addr.if_uc, hn) == 0)
1037                         continue;
1038                 if (hn->mpfs_index == -1U)
1039                         mlx5_mpfs_add_mac(priv->mdev, &hn->mpfs_index,
1040                             hn->ai.addr, 0, 0);
1041         }
1042
1043         /* insert L2 multicast addresses into hash list */
1044
1045         while ((hn = mlx5e_remove_hn(&head_mc)) != NULL) {
1046                 if (mlx5e_add_eth_addr_to_hash(priv->eth_addr.if_mc, hn) == 0)
1047                         continue;
1048         }
1049
1050 cleanup:
1051         while ((hn = mlx5e_remove_hn(&head_uc)) != NULL)
1052                 free(hn, M_MLX5EN);
1053         while ((hn = mlx5e_remove_hn(&head_mc)) != NULL)
1054                 free(hn, M_MLX5EN);
1055         while ((hn = mlx5e_remove_hn(&head_free)) != NULL)
1056                 free(hn, M_MLX5EN);
1057
1058         if (ctx.success == false)
1059                 goto retry;
1060 }
1061
1062 static void mlx5e_fill_addr_array(struct mlx5e_priv *priv, int list_type,
1063                                   u8 addr_array[][ETH_ALEN], int size)
1064 {
1065         bool is_uc = (list_type == MLX5_NIC_VPORT_LIST_TYPE_UC);
1066         if_t ifp = priv->ifp;
1067         struct mlx5e_eth_addr_hash_node *hn;
1068         struct mlx5e_eth_addr_hash_head *addr_list;
1069         struct mlx5e_eth_addr_hash_node *tmp;
1070         int i = 0;
1071         int hi;
1072
1073         addr_list = is_uc ? priv->eth_addr.if_uc : priv->eth_addr.if_mc;
1074
1075         if (is_uc) /* Make sure our own address is pushed first */
1076                 ether_addr_copy(addr_array[i++], if_getlladdr(ifp));
1077         else if (priv->eth_addr.broadcast_enabled)
1078                 ether_addr_copy(addr_array[i++], if_getbroadcastaddr(ifp));
1079
1080         mlx5e_for_each_hash_node(hn, tmp, addr_list, hi) {
1081                 if (ether_addr_equal(if_getlladdr(ifp), hn->ai.addr))
1082                         continue;
1083                 if (i >= size)
1084                         break;
1085                 ether_addr_copy(addr_array[i++], hn->ai.addr);
1086         }
1087 }
1088
1089 static void mlx5e_vport_context_update_addr_list(struct mlx5e_priv *priv,
1090                                                  int list_type)
1091 {
1092         bool is_uc = (list_type == MLX5_NIC_VPORT_LIST_TYPE_UC);
1093         struct mlx5e_eth_addr_hash_node *hn;
1094         u8 (*addr_array)[ETH_ALEN] = NULL;
1095         struct mlx5e_eth_addr_hash_head *addr_list;
1096         struct mlx5e_eth_addr_hash_node *tmp;
1097         int max_size;
1098         int size;
1099         int err;
1100         int hi;
1101
1102         size = is_uc ? 0 : (priv->eth_addr.broadcast_enabled ? 1 : 0);
1103         max_size = is_uc ?
1104                 1 << MLX5_CAP_GEN(priv->mdev, log_max_current_uc_list) :
1105                 1 << MLX5_CAP_GEN(priv->mdev, log_max_current_mc_list);
1106
1107         addr_list = is_uc ? priv->eth_addr.if_uc : priv->eth_addr.if_mc;
1108         mlx5e_for_each_hash_node(hn, tmp, addr_list, hi)
1109                 size++;
1110
1111         if (size > max_size) {
1112                 mlx5_en_err(priv->ifp,
1113                             "ifp %s list size (%d) > (%d) max vport list size, some addresses will be dropped\n",
1114                             is_uc ? "UC" : "MC", size, max_size);
1115                 size = max_size;
1116         }
1117
1118         if (size) {
1119                 addr_array = kcalloc(size, ETH_ALEN, GFP_KERNEL);
1120                 if (!addr_array) {
1121                         err = -ENOMEM;
1122                         goto out;
1123                 }
1124                 mlx5e_fill_addr_array(priv, list_type, addr_array, size);
1125         }
1126
1127         err = mlx5_modify_nic_vport_mac_list(priv->mdev, list_type, addr_array, size);
1128 out:
1129         if (err)
1130                 mlx5_en_err(priv->ifp,
1131                            "Failed to modify vport %s list err(%d)\n",
1132                            is_uc ? "UC" : "MC", err);
1133         kfree(addr_array);
1134 }
1135
1136 static void mlx5e_vport_context_update(struct mlx5e_priv *priv)
1137 {
1138         struct mlx5e_eth_addr_db *ea = &priv->eth_addr;
1139
1140         mlx5e_vport_context_update_addr_list(priv, MLX5_NIC_VPORT_LIST_TYPE_UC);
1141         mlx5e_vport_context_update_addr_list(priv, MLX5_NIC_VPORT_LIST_TYPE_MC);
1142         mlx5_modify_nic_vport_promisc(priv->mdev, 0,
1143                                       ea->allmulti_enabled,
1144                                       ea->promisc_enabled);
1145 }
1146
1147 static void
1148 mlx5e_apply_ifp_addr(struct mlx5e_priv *priv)
1149 {
1150         struct mlx5e_eth_addr_hash_node *hn;
1151         struct mlx5e_eth_addr_hash_node *tmp;
1152         int i;
1153
1154         mlx5e_for_each_hash_node(hn, tmp, priv->eth_addr.if_uc, i)
1155             mlx5e_execute_action(priv, hn);
1156
1157         mlx5e_for_each_hash_node(hn, tmp, priv->eth_addr.if_mc, i)
1158             mlx5e_execute_action(priv, hn);
1159 }
1160
1161 static void
1162 mlx5e_handle_ifp_addr(struct mlx5e_priv *priv, bool rx_mode_enable)
1163 {
1164         struct mlx5e_eth_addr_hash_node *hn;
1165         struct mlx5e_eth_addr_hash_node *tmp;
1166         int i;
1167
1168         mlx5e_for_each_hash_node(hn, tmp, priv->eth_addr.if_uc, i)
1169             hn->action = MLX5E_ACTION_DEL;
1170         mlx5e_for_each_hash_node(hn, tmp, priv->eth_addr.if_mc, i)
1171             hn->action = MLX5E_ACTION_DEL;
1172
1173         if (rx_mode_enable)
1174                 mlx5e_sync_ifp_addr(priv);
1175
1176         mlx5e_apply_ifp_addr(priv);
1177 }
1178
1179 static void
1180 mlx5e_set_rx_mode_core(struct mlx5e_priv *priv, bool rx_mode_enable)
1181 {
1182         struct mlx5e_eth_addr_db *ea = &priv->eth_addr;
1183         if_t ndev = priv->ifp;
1184         int ndev_flags = if_getflags(ndev);
1185
1186         bool promisc_enabled = rx_mode_enable && (ndev_flags & IFF_PROMISC);
1187         bool allmulti_enabled = rx_mode_enable && (ndev_flags & IFF_ALLMULTI);
1188         bool broadcast_enabled = rx_mode_enable;
1189
1190         bool enable_promisc = !ea->promisc_enabled && promisc_enabled;
1191         bool disable_promisc = ea->promisc_enabled && !promisc_enabled;
1192         bool enable_allmulti = !ea->allmulti_enabled && allmulti_enabled;
1193         bool disable_allmulti = ea->allmulti_enabled && !allmulti_enabled;
1194         bool enable_broadcast = !ea->broadcast_enabled && broadcast_enabled;
1195         bool disable_broadcast = ea->broadcast_enabled && !broadcast_enabled;
1196
1197         /* update broadcast address */
1198         ether_addr_copy(priv->eth_addr.broadcast.addr,
1199             if_getbroadcastaddr(priv->ifp));
1200
1201         if (enable_promisc) {
1202                 mlx5e_add_eth_addr_rule(priv, &ea->promisc, MLX5E_PROMISC);
1203                 if (!priv->vlan.filter_disabled)
1204                         mlx5e_add_any_vid_rules(priv);
1205         }
1206         if (enable_allmulti)
1207                 mlx5e_add_eth_addr_rule(priv, &ea->allmulti, MLX5E_ALLMULTI);
1208         if (enable_broadcast)
1209                 mlx5e_add_eth_addr_rule(priv, &ea->broadcast, MLX5E_FULLMATCH);
1210
1211         mlx5e_handle_ifp_addr(priv, rx_mode_enable);
1212
1213         if (disable_broadcast)
1214                 mlx5e_del_eth_addr_from_flow_table(priv, &ea->broadcast);
1215         if (disable_allmulti)
1216                 mlx5e_del_eth_addr_from_flow_table(priv, &ea->allmulti);
1217         if (disable_promisc) {
1218                 if (!priv->vlan.filter_disabled)
1219                         mlx5e_del_any_vid_rules(priv);
1220                 mlx5e_del_eth_addr_from_flow_table(priv, &ea->promisc);
1221         }
1222
1223         ea->promisc_enabled = promisc_enabled;
1224         ea->allmulti_enabled = allmulti_enabled;
1225         ea->broadcast_enabled = broadcast_enabled;
1226
1227         mlx5e_vport_context_update(priv);
1228 }
1229
1230 void
1231 mlx5e_set_rx_mode_work(struct work_struct *work)
1232 {
1233         struct mlx5e_priv *priv =
1234             container_of(work, struct mlx5e_priv, set_rx_mode_work);
1235
1236         PRIV_LOCK(priv);
1237         if (test_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state))
1238                 mlx5e_set_rx_mode_core(priv, true);
1239         PRIV_UNLOCK(priv);
1240 }
1241
1242 static void
1243 mlx5e_destroy_groups(struct mlx5e_flow_table *ft)
1244 {
1245         int i;
1246
1247         for (i = ft->num_groups - 1; i >= 0; i--) {
1248                 if (!IS_ERR_OR_NULL(ft->g[i]))
1249                         mlx5_destroy_flow_group(ft->g[i]);
1250                 ft->g[i] = NULL;
1251         }
1252         ft->num_groups = 0;
1253 }
1254
1255 static void
1256 mlx5e_destroy_flow_table(struct mlx5e_flow_table *ft)
1257 {
1258         mlx5e_destroy_groups(ft);
1259         kfree(ft->g);
1260         mlx5_destroy_flow_table(ft->t);
1261         ft->t = NULL;
1262 }
1263
1264 #define MLX5E_NUM_MAIN_GROUPS   10
1265 #define MLX5E_MAIN_GROUP0_SIZE  BIT(4)
1266 #define MLX5E_MAIN_GROUP1_SIZE  BIT(3)
1267 #define MLX5E_MAIN_GROUP2_SIZE  BIT(1)
1268 #define MLX5E_MAIN_GROUP3_SIZE  BIT(0)
1269 #define MLX5E_MAIN_GROUP4_SIZE  BIT(14)
1270 #define MLX5E_MAIN_GROUP5_SIZE  BIT(13)
1271 #define MLX5E_MAIN_GROUP6_SIZE  BIT(11)
1272 #define MLX5E_MAIN_GROUP7_SIZE  BIT(2)
1273 #define MLX5E_MAIN_GROUP8_SIZE  BIT(1)
1274 #define MLX5E_MAIN_GROUP9_SIZE  BIT(0)
1275 #define MLX5E_MAIN_TABLE_SIZE   (MLX5E_MAIN_GROUP0_SIZE +\
1276                                  MLX5E_MAIN_GROUP1_SIZE +\
1277                                  MLX5E_MAIN_GROUP2_SIZE +\
1278                                  MLX5E_MAIN_GROUP3_SIZE +\
1279                                  MLX5E_MAIN_GROUP4_SIZE +\
1280                                  MLX5E_MAIN_GROUP5_SIZE +\
1281                                  MLX5E_MAIN_GROUP6_SIZE +\
1282                                  MLX5E_MAIN_GROUP7_SIZE +\
1283                                  MLX5E_MAIN_GROUP8_SIZE +\
1284                                  MLX5E_MAIN_GROUP9_SIZE +\
1285                                  0)
1286
1287 static int
1288 mlx5e_create_main_groups_sub(struct mlx5e_flow_table *ft, u32 *in,
1289                                       int inlen)
1290 {
1291         u8 *mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
1292         u8 *dmac = MLX5_ADDR_OF(create_flow_group_in, in,
1293                                 match_criteria.outer_headers.dmac_47_16);
1294         int err;
1295         int ix = 0;
1296
1297         /* Tunnel rules need to be first in this list of groups */
1298
1299         /* Start tunnel rules */
1300         memset(in, 0, inlen);
1301         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1302         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
1303         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol);
1304         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.udp_dport);
1305         MLX5_SET_CFG(in, start_flow_index, ix);
1306         ix += MLX5E_MAIN_GROUP0_SIZE;
1307         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1308         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1309         if (IS_ERR(ft->g[ft->num_groups]))
1310                 goto err_destory_groups;
1311         ft->num_groups++;
1312         /* End Tunnel Rules */
1313
1314         memset(in, 0, inlen);
1315         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1316         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
1317         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol);
1318         MLX5_SET_CFG(in, start_flow_index, ix);
1319         ix += MLX5E_MAIN_GROUP1_SIZE;
1320         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1321         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1322         if (IS_ERR(ft->g[ft->num_groups]))
1323                 goto err_destory_groups;
1324         ft->num_groups++;
1325
1326         memset(in, 0, inlen);
1327         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1328         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
1329         MLX5_SET_CFG(in, start_flow_index, ix);
1330         ix += MLX5E_MAIN_GROUP2_SIZE;
1331         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1332         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1333         if (IS_ERR(ft->g[ft->num_groups]))
1334                 goto err_destory_groups;
1335         ft->num_groups++;
1336
1337         memset(in, 0, inlen);
1338         MLX5_SET_CFG(in, start_flow_index, ix);
1339         ix += MLX5E_MAIN_GROUP3_SIZE;
1340         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1341         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1342         if (IS_ERR(ft->g[ft->num_groups]))
1343                 goto err_destory_groups;
1344         ft->num_groups++;
1345
1346         memset(in, 0, inlen);
1347         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1348         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
1349         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol);
1350         memset(dmac, 0xff, ETH_ALEN);
1351         MLX5_SET_CFG(in, start_flow_index, ix);
1352         ix += MLX5E_MAIN_GROUP4_SIZE;
1353         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1354         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1355         if (IS_ERR(ft->g[ft->num_groups]))
1356                 goto err_destory_groups;
1357         ft->num_groups++;
1358
1359         memset(in, 0, inlen);
1360         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1361         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
1362         memset(dmac, 0xff, ETH_ALEN);
1363         MLX5_SET_CFG(in, start_flow_index, ix);
1364         ix += MLX5E_MAIN_GROUP5_SIZE;
1365         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1366         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1367         if (IS_ERR(ft->g[ft->num_groups]))
1368                 goto err_destory_groups;
1369         ft->num_groups++;
1370
1371         memset(in, 0, inlen);
1372         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1373         memset(dmac, 0xff, ETH_ALEN);
1374         MLX5_SET_CFG(in, start_flow_index, ix);
1375         ix += MLX5E_MAIN_GROUP6_SIZE;
1376         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1377         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1378         if (IS_ERR(ft->g[ft->num_groups]))
1379                 goto err_destory_groups;
1380         ft->num_groups++;
1381
1382         memset(in, 0, inlen);
1383         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1384         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
1385         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol);
1386         dmac[0] = 0x01;
1387         MLX5_SET_CFG(in, start_flow_index, ix);
1388         ix += MLX5E_MAIN_GROUP7_SIZE;
1389         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1390         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1391         if (IS_ERR(ft->g[ft->num_groups]))
1392                 goto err_destory_groups;
1393         ft->num_groups++;
1394
1395         memset(in, 0, inlen);
1396         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1397         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
1398         dmac[0] = 0x01;
1399         MLX5_SET_CFG(in, start_flow_index, ix);
1400         ix += MLX5E_MAIN_GROUP8_SIZE;
1401         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1402         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1403         if (IS_ERR(ft->g[ft->num_groups]))
1404                 goto err_destory_groups;
1405         ft->num_groups++;
1406
1407         memset(in, 0, inlen);
1408         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1409         dmac[0] = 0x01;
1410         MLX5_SET_CFG(in, start_flow_index, ix);
1411         ix += MLX5E_MAIN_GROUP9_SIZE;
1412         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1413         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1414         if (IS_ERR(ft->g[ft->num_groups]))
1415                 goto err_destory_groups;
1416         ft->num_groups++;
1417
1418         return (0);
1419
1420 err_destory_groups:
1421         err = PTR_ERR(ft->g[ft->num_groups]);
1422         ft->g[ft->num_groups] = NULL;
1423         mlx5e_destroy_groups(ft);
1424
1425         return (err);
1426 }
1427
1428 static int
1429 mlx5e_create_main_groups(struct mlx5e_flow_table *ft)
1430 {
1431         u32 *in;
1432         int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1433         int err;
1434
1435         in = mlx5_vzalloc(inlen);
1436         if (!in)
1437                 return (-ENOMEM);
1438
1439         err = mlx5e_create_main_groups_sub(ft, in, inlen);
1440
1441         kvfree(in);
1442         return (err);
1443 }
1444
1445 #define MLX5E_MAIN_VXLAN_GROUP0_SIZE    BIT(3)
1446 #define MLX5E_MAIN_VXLAN_GROUP1_SIZE    BIT(3)
1447 #define MLX5E_MAIN_VXLAN_GROUP2_SIZE    BIT(0)
1448 static int
1449 mlx5e_create_main_vxlan_groups_sub(struct mlx5e_flow_table *ft, u32 *in,
1450     int inlen)
1451 {
1452         u8 *mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
1453         int err;
1454         int ix = 0;
1455
1456         memset(in, 0, inlen);
1457         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_INNER_HEADERS);
1458         MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ethertype);
1459         MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ip_protocol);
1460         MLX5_SET_CFG(in, start_flow_index, ix);
1461         ix += MLX5E_MAIN_VXLAN_GROUP0_SIZE;
1462         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1463         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1464         if (IS_ERR(ft->g[ft->num_groups]))
1465                 goto err_destory_groups;
1466         ft->num_groups++;
1467
1468         memset(in, 0, inlen);
1469         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_INNER_HEADERS);
1470         MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ethertype);
1471         MLX5_SET_CFG(in, start_flow_index, ix);
1472         ix += MLX5E_MAIN_VXLAN_GROUP1_SIZE;
1473         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1474         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1475         if (IS_ERR(ft->g[ft->num_groups]))
1476                 goto err_destory_groups;
1477         ft->num_groups++;
1478
1479         memset(in, 0, inlen);
1480         MLX5_SET_CFG(in, start_flow_index, ix);
1481         ix += MLX5E_MAIN_VXLAN_GROUP2_SIZE;
1482         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1483         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1484         if (IS_ERR(ft->g[ft->num_groups]))
1485                 goto err_destory_groups;
1486         ft->num_groups++;
1487
1488         return (0);
1489
1490 err_destory_groups:
1491         err = PTR_ERR(ft->g[ft->num_groups]);
1492         ft->g[ft->num_groups] = NULL;
1493         mlx5e_destroy_groups(ft);
1494
1495         return (err);
1496 }
1497
1498 static int
1499 mlx5e_create_main_vxlan_groups(struct mlx5e_flow_table *ft)
1500 {
1501         u32 *in;
1502         int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1503         int err;
1504
1505         in = mlx5_vzalloc(inlen);
1506         if (!in)
1507                 return (-ENOMEM);
1508
1509         err = mlx5e_create_main_vxlan_groups_sub(ft, in, inlen);
1510
1511         kvfree(in);
1512         return (err);
1513 }
1514
1515
1516 static int
1517 mlx5e_create_main_flow_table(struct mlx5e_priv *priv, bool inner_vxlan)
1518 {
1519         struct mlx5e_flow_table *ft = inner_vxlan ? &priv->fts.main_vxlan :
1520             &priv->fts.main;
1521         int err;
1522
1523         ft->num_groups = 0;
1524         ft->t = mlx5_create_flow_table(priv->fts.ns, 0,
1525             inner_vxlan ? "vxlan_main" : "main", MLX5E_MAIN_TABLE_SIZE);
1526
1527         if (IS_ERR(ft->t)) {
1528                 err = PTR_ERR(ft->t);
1529                 ft->t = NULL;
1530                 return (err);
1531         }
1532         ft->g = kcalloc(MLX5E_NUM_MAIN_GROUPS, sizeof(*ft->g), GFP_KERNEL);
1533         if (!ft->g) {
1534                 err = -ENOMEM;
1535                 goto err_destroy_main_flow_table;
1536         }
1537
1538         err = inner_vxlan ? mlx5e_create_main_vxlan_groups(ft) :
1539             mlx5e_create_main_groups(ft);
1540         if (err)
1541                 goto err_free_g;
1542         return (0);
1543
1544 err_free_g:
1545         kfree(ft->g);
1546
1547 err_destroy_main_flow_table:
1548         mlx5_destroy_flow_table(ft->t);
1549         ft->t = NULL;
1550
1551         return (err);
1552 }
1553
1554 static void mlx5e_destroy_main_flow_table(struct mlx5e_priv *priv)
1555 {
1556         mlx5e_destroy_flow_table(&priv->fts.main);
1557 }
1558
1559 static void mlx5e_destroy_main_vxlan_flow_table(struct mlx5e_priv *priv)
1560 {
1561         mlx5e_destroy_flow_table(&priv->fts.main_vxlan);
1562 }
1563
1564 #define MLX5E_NUM_VLAN_GROUPS   3
1565 #define MLX5E_VLAN_GROUP0_SIZE  BIT(12)
1566 #define MLX5E_VLAN_GROUP1_SIZE  BIT(1)
1567 #define MLX5E_VLAN_GROUP2_SIZE  BIT(0)
1568 #define MLX5E_VLAN_TABLE_SIZE   (MLX5E_VLAN_GROUP0_SIZE +\
1569                                  MLX5E_VLAN_GROUP1_SIZE +\
1570                                  MLX5E_VLAN_GROUP2_SIZE +\
1571                                  0)
1572
1573 static int
1574 mlx5e_create_vlan_groups_sub(struct mlx5e_flow_table *ft, u32 *in,
1575                                       int inlen)
1576 {
1577         int err;
1578         int ix = 0;
1579         u8 *mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
1580
1581         memset(in, 0, inlen);
1582         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1583         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.cvlan_tag);
1584         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.first_vid);
1585         MLX5_SET_CFG(in, start_flow_index, ix);
1586         ix += MLX5E_VLAN_GROUP0_SIZE;
1587         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1588         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1589         if (IS_ERR(ft->g[ft->num_groups]))
1590                 goto err_destory_groups;
1591         ft->num_groups++;
1592
1593         memset(in, 0, inlen);
1594         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1595         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.cvlan_tag);
1596         MLX5_SET_CFG(in, start_flow_index, ix);
1597         ix += MLX5E_VLAN_GROUP1_SIZE;
1598         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1599         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1600         if (IS_ERR(ft->g[ft->num_groups]))
1601                 goto err_destory_groups;
1602         ft->num_groups++;
1603
1604         memset(in, 0, inlen);
1605         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1606         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.svlan_tag);
1607         MLX5_SET_CFG(in, start_flow_index, ix);
1608         ix += MLX5E_VLAN_GROUP2_SIZE;
1609         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1610         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1611         if (IS_ERR(ft->g[ft->num_groups]))
1612                 goto err_destory_groups;
1613         ft->num_groups++;
1614
1615         return (0);
1616
1617 err_destory_groups:
1618         err = PTR_ERR(ft->g[ft->num_groups]);
1619         ft->g[ft->num_groups] = NULL;
1620         mlx5e_destroy_groups(ft);
1621
1622         return (err);
1623 }
1624
1625 static int
1626 mlx5e_create_vlan_groups(struct mlx5e_flow_table *ft)
1627 {
1628         u32 *in;
1629         int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1630         int err;
1631
1632         in = mlx5_vzalloc(inlen);
1633         if (!in)
1634                 return (-ENOMEM);
1635
1636         err = mlx5e_create_vlan_groups_sub(ft, in, inlen);
1637
1638         kvfree(in);
1639         return (err);
1640 }
1641
1642 static int
1643 mlx5e_create_vlan_flow_table(struct mlx5e_priv *priv)
1644 {
1645         struct mlx5e_flow_table *ft = &priv->fts.vlan;
1646         int err;
1647
1648         ft->num_groups = 0;
1649         ft->t = mlx5_create_flow_table(priv->fts.ns, 0, "vlan",
1650                                        MLX5E_VLAN_TABLE_SIZE);
1651
1652         if (IS_ERR(ft->t)) {
1653                 err = PTR_ERR(ft->t);
1654                 ft->t = NULL;
1655                 return (err);
1656         }
1657         ft->g = kcalloc(MLX5E_NUM_VLAN_GROUPS, sizeof(*ft->g), GFP_KERNEL);
1658         if (!ft->g) {
1659                 err = -ENOMEM;
1660                 goto err_destroy_vlan_flow_table;
1661         }
1662
1663         err = mlx5e_create_vlan_groups(ft);
1664         if (err)
1665                 goto err_free_g;
1666
1667         return (0);
1668
1669 err_free_g:
1670         kfree(ft->g);
1671
1672 err_destroy_vlan_flow_table:
1673         mlx5_destroy_flow_table(ft->t);
1674         ft->t = NULL;
1675
1676         return (err);
1677 }
1678
1679 static void
1680 mlx5e_destroy_vlan_flow_table(struct mlx5e_priv *priv)
1681 {
1682         mlx5e_destroy_flow_table(&priv->fts.vlan);
1683 }
1684
1685 static int
1686 mlx5e_add_vxlan_rule_sub(struct mlx5e_priv *priv, u32 *mc, u32 *mv,
1687     struct mlx5e_vxlan_db_el *el)
1688 {
1689         struct mlx5_flow_table *ft = priv->fts.vxlan.t;
1690         struct mlx5_flow_destination dest = {};
1691         u8 mc_enable;
1692         struct mlx5_flow_rule **rule_p;
1693         int err = 0;
1694         struct mlx5_flow_act flow_act = {
1695                 .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
1696                 .flow_tag = MLX5_FS_ETH_FLOW_TAG,
1697         };
1698
1699         dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1700         dest.ft = priv->fts.main_vxlan.t;
1701
1702         mc_enable = MLX5_MATCH_OUTER_HEADERS;
1703         rule_p = &el->vxlan_ft_rule;
1704         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
1705         MLX5_SET(fte_match_param, mv, outer_headers.ethertype, el->proto);
1706         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol);
1707         MLX5_SET(fte_match_param, mv, outer_headers.ip_protocol, IPPROTO_UDP);
1708         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.udp_dport);
1709         MLX5_SET(fte_match_param, mv, outer_headers.udp_dport, el->port);
1710
1711         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
1712             MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
1713
1714         if (IS_ERR(*rule_p)) {
1715                 err = PTR_ERR(*rule_p);
1716                 *rule_p = NULL;
1717                 mlx5_en_err(priv->ifp, "add rule failed\n");
1718         }
1719
1720         return (err);
1721 }
1722
1723 static struct mlx5e_vxlan_db_el *
1724 mlx5e_vxlan_find_db_el(struct mlx5e_priv *priv, u_int proto, u_int port)
1725 {
1726         struct mlx5e_vxlan_db_el *el;
1727
1728         TAILQ_FOREACH(el, &priv->vxlan.head, link) {
1729                 if (el->proto == proto && el->port == port)
1730                         return (el);
1731         }
1732         return (NULL);
1733 }
1734
1735 static struct mlx5e_vxlan_db_el *
1736 mlx5e_vxlan_alloc_db_el(struct mlx5e_priv *priv, u_int proto, u_int port)
1737 {
1738         struct mlx5e_vxlan_db_el *el;
1739
1740         el = mlx5_vzalloc(sizeof(*el));
1741         el->refcount = 1;
1742         el->proto = proto;
1743         el->port = port;
1744         el->vxlan_ft_rule = NULL;
1745         return (el);
1746 }
1747
1748 static int
1749 mlx5e_vxlan_family_to_proto(sa_family_t family, u_int *proto)
1750 {
1751         switch (family) {
1752         case AF_INET:
1753                 *proto = ETHERTYPE_IP;
1754                 return (0);
1755         case AF_INET6:
1756                 *proto = ETHERTYPE_IPV6;
1757                 return (0);
1758         default:
1759                 return (-EINVAL);
1760         }
1761 }
1762
1763 static int
1764 mlx5e_add_vxlan_rule_from_db(struct mlx5e_priv *priv,
1765     struct mlx5e_vxlan_db_el *el)
1766 {
1767         u32 *match_criteria;
1768         u32 *match_value;
1769         int err;
1770
1771         match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
1772         match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
1773         if (match_value == NULL || match_criteria == NULL) {
1774                 mlx5_en_err(priv->ifp, "alloc failed\n");
1775                 err = -ENOMEM;
1776                 goto add_vxlan_rule_out;
1777         }
1778
1779         err = mlx5e_add_vxlan_rule_sub(priv, match_criteria, match_value, el);
1780
1781 add_vxlan_rule_out:
1782         kvfree(match_criteria);
1783         kvfree(match_value);
1784
1785         return (err);
1786 }
1787
1788 static int
1789 mlx5e_add_vxlan_rule(struct mlx5e_priv *priv, sa_family_t family, u_int port)
1790 {
1791         struct mlx5e_vxlan_db_el *el;
1792         u_int proto;
1793         int err;
1794
1795         err = mlx5e_vxlan_family_to_proto(family, &proto);
1796         if (err != 0)
1797                 return (err);
1798
1799         el = mlx5e_vxlan_find_db_el(priv, proto, port);
1800         if (el != NULL) {
1801                 el->refcount++;
1802                 if (el->installed)
1803                         return (0);
1804         }
1805         el = mlx5e_vxlan_alloc_db_el(priv, proto, port);
1806
1807         if ((if_getcapenable(priv->ifp) & IFCAP_VXLAN_HWCSUM) != 0) {
1808                 err = mlx5e_add_vxlan_rule_from_db(priv, el);
1809                 if (err == 0)
1810                         el->installed = true;
1811         }
1812         if (err == 0)
1813                 TAILQ_INSERT_TAIL(&priv->vxlan.head, el, link);
1814         else
1815                 kvfree(el);
1816
1817         return (err);
1818 }
1819
1820 static int
1821 mlx5e_add_vxlan_catchall_rule_sub(struct mlx5e_priv *priv, u32 *mc, u32 *mv)
1822 {
1823         struct mlx5_flow_table *ft = priv->fts.vxlan.t;
1824         struct mlx5_flow_destination dest = {};
1825         u8 mc_enable = 0;
1826         struct mlx5_flow_rule **rule_p;
1827         int err = 0;
1828         struct mlx5_flow_act flow_act = {
1829                 .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
1830                 .flow_tag = MLX5_FS_ETH_FLOW_TAG,
1831         };
1832
1833         dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1834         dest.ft = priv->fts.main.t;
1835
1836         rule_p = &priv->fts.vxlan_catchall_ft_rule;
1837         *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
1838             MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
1839
1840         if (IS_ERR(*rule_p)) {
1841                 err = PTR_ERR(*rule_p);
1842                 *rule_p = NULL;
1843                 mlx5_en_err(priv->ifp, "add rule failed\n");
1844         }
1845
1846         return (err);
1847 }
1848
1849
1850 static int
1851 mlx5e_add_vxlan_catchall_rule(struct mlx5e_priv *priv)
1852 {
1853         u32 *match_criteria;
1854         u32 *match_value;
1855         int err;
1856
1857         match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
1858         match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
1859         if (match_value == NULL || match_criteria == NULL) {
1860                 mlx5_en_err(priv->ifp, "alloc failed\n");
1861                 err = -ENOMEM;
1862                 goto add_vxlan_rule_out;
1863         }
1864
1865         err = mlx5e_add_vxlan_catchall_rule_sub(priv, match_criteria,
1866             match_value);
1867
1868 add_vxlan_rule_out:
1869         kvfree(match_criteria);
1870         kvfree(match_value);
1871
1872         return (err);
1873 }
1874
1875 int
1876 mlx5e_add_all_vxlan_rules(struct mlx5e_priv *priv)
1877 {
1878         struct mlx5e_vxlan_db_el *el;
1879         int err;
1880
1881         err = 0;
1882         TAILQ_FOREACH(el, &priv->vxlan.head, link) {
1883                 if (el->installed)
1884                         continue;
1885                 err = mlx5e_add_vxlan_rule_from_db(priv, el);
1886                 if (err != 0)
1887                         break;
1888                 el->installed = true;
1889         }
1890
1891         return (err);
1892 }
1893
1894 static int
1895 mlx5e_del_vxlan_rule(struct mlx5e_priv *priv, sa_family_t family, u_int port)
1896 {
1897         struct mlx5e_vxlan_db_el *el;
1898         u_int proto;
1899         int err;
1900
1901         err = mlx5e_vxlan_family_to_proto(family, &proto);
1902         if (err != 0)
1903                 return (err);
1904
1905         el = mlx5e_vxlan_find_db_el(priv, proto, port);
1906         if (el == NULL)
1907                 return (0);
1908         if (el->refcount > 1) {
1909                 el->refcount--;
1910                 return (0);
1911         }
1912
1913         if (el->installed)
1914                 mlx5_del_flow_rule(&el->vxlan_ft_rule);
1915         TAILQ_REMOVE(&priv->vxlan.head, el, link);
1916         kvfree(el);
1917         return (0);
1918 }
1919
1920 void
1921 mlx5e_del_all_vxlan_rules(struct mlx5e_priv *priv)
1922 {
1923         struct mlx5e_vxlan_db_el *el;
1924
1925         TAILQ_FOREACH(el, &priv->vxlan.head, link) {
1926                 if (!el->installed)
1927                         continue;
1928                 mlx5_del_flow_rule(&el->vxlan_ft_rule);
1929                 el->installed = false;
1930         }
1931 }
1932
1933 static void
1934 mlx5e_del_vxlan_catchall_rule(struct mlx5e_priv *priv)
1935 {
1936         mlx5_del_flow_rule(&priv->fts.vxlan_catchall_ft_rule);
1937 }
1938
1939 void
1940 mlx5e_vxlan_start(void *arg, if_t ifp __unused, sa_family_t family,
1941     u_int port)
1942 {
1943         struct mlx5e_priv *priv = arg;
1944         int err;
1945
1946         PRIV_LOCK(priv);
1947         err = mlx5_vxlan_udp_port_add(priv->mdev, port);
1948         if (err == 0 && test_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state))
1949                 mlx5e_add_vxlan_rule(priv, family, port);
1950         PRIV_UNLOCK(priv);
1951 }
1952
1953 void
1954 mlx5e_vxlan_stop(void *arg, if_t ifp __unused, sa_family_t family,
1955     u_int port)
1956 {
1957         struct mlx5e_priv *priv = arg;
1958
1959         PRIV_LOCK(priv);
1960         if (test_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state))
1961                 mlx5e_del_vxlan_rule(priv, family, port);
1962         (void)mlx5_vxlan_udp_port_delete(priv->mdev, port);
1963         PRIV_UNLOCK(priv);
1964 }
1965
1966 #define MLX5E_VXLAN_GROUP0_SIZE BIT(3)  /* XXXKIB */
1967 #define MLX5E_VXLAN_GROUP1_SIZE BIT(0)
1968 #define MLX5E_NUM_VXLAN_GROUPS  BIT(1)
1969 #define MLX5E_VXLAN_TABLE_SIZE  \
1970     (MLX5E_VXLAN_GROUP0_SIZE + MLX5E_VXLAN_GROUP1_SIZE)
1971
1972 static int
1973 mlx5e_create_vxlan_groups_sub(struct mlx5e_flow_table *ft, u32 *in,
1974                                       int inlen)
1975 {
1976         int err;
1977         int ix = 0;
1978         u8 *mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
1979
1980         memset(in, 0, inlen);
1981         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1982         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
1983         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol);
1984         MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.udp_dport);
1985         MLX5_SET_CFG(in, start_flow_index, ix);
1986         ix += MLX5E_VXLAN_GROUP0_SIZE;
1987         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1988         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1989         if (IS_ERR(ft->g[ft->num_groups]))
1990                 goto err_destory_groups;
1991         ft->num_groups++;
1992
1993         memset(in, 0, inlen);
1994         MLX5_SET_CFG(in, start_flow_index, ix);
1995         ix += MLX5E_VXLAN_GROUP1_SIZE;
1996         MLX5_SET_CFG(in, end_flow_index, ix - 1);
1997         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
1998         if (IS_ERR(ft->g[ft->num_groups]))
1999                 goto err_destory_groups;
2000         ft->num_groups++;
2001
2002         return (0);
2003
2004 err_destory_groups:
2005         err = PTR_ERR(ft->g[ft->num_groups]);
2006         ft->g[ft->num_groups] = NULL;
2007         mlx5e_destroy_groups(ft);
2008
2009         return (err);
2010 }
2011
2012 static int
2013 mlx5e_create_vxlan_groups(struct mlx5e_flow_table *ft)
2014 {
2015         u32 *in;
2016         int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2017         int err;
2018
2019         in = mlx5_vzalloc(inlen);
2020         if (!in)
2021                 return (-ENOMEM);
2022
2023         err = mlx5e_create_vxlan_groups_sub(ft, in, inlen);
2024
2025         kvfree(in);
2026         return (err);
2027 }
2028
2029 static int
2030 mlx5e_create_vxlan_flow_table(struct mlx5e_priv *priv)
2031 {
2032         struct mlx5e_flow_table *ft = &priv->fts.vxlan;
2033         int err;
2034
2035         ft->num_groups = 0;
2036         ft->t = mlx5_create_flow_table(priv->fts.ns, 0, "vxlan",
2037                                        MLX5E_VXLAN_TABLE_SIZE);
2038
2039         if (IS_ERR(ft->t)) {
2040                 err = PTR_ERR(ft->t);
2041                 ft->t = NULL;
2042                 return (err);
2043         }
2044         ft->g = kcalloc(MLX5E_NUM_VXLAN_GROUPS, sizeof(*ft->g), GFP_KERNEL);
2045         if (!ft->g) {
2046                 err = -ENOMEM;
2047                 goto err_destroy_vxlan_flow_table;
2048         }
2049
2050         err = mlx5e_create_vxlan_groups(ft);
2051         if (err)
2052                 goto err_free_g;
2053
2054         TAILQ_INIT(&priv->vxlan.head);
2055         return (0);
2056
2057 err_free_g:
2058         kfree(ft->g);
2059
2060 err_destroy_vxlan_flow_table:
2061         mlx5_destroy_flow_table(ft->t);
2062         ft->t = NULL;
2063
2064         return (err);
2065 }
2066
2067 #define MLX5E_NUM_INNER_RSS_GROUPS      3
2068 #define MLX5E_INNER_RSS_GROUP0_SIZE     BIT(3)
2069 #define MLX5E_INNER_RSS_GROUP1_SIZE     BIT(1)
2070 #define MLX5E_INNER_RSS_GROUP2_SIZE     BIT(0)
2071 #define MLX5E_INNER_RSS_TABLE_SIZE      (MLX5E_INNER_RSS_GROUP0_SIZE +\
2072                                          MLX5E_INNER_RSS_GROUP1_SIZE +\
2073                                          MLX5E_INNER_RSS_GROUP2_SIZE +\
2074                                          0)
2075
2076 static int
2077 mlx5e_create_inner_rss_groups_sub(struct mlx5e_flow_table *ft, u32 *in,
2078                                            int inlen)
2079 {
2080         u8 *mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
2081         int err;
2082         int ix = 0;
2083
2084         memset(in, 0, inlen);
2085         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_INNER_HEADERS);
2086         MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ethertype);
2087         MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ip_protocol);
2088         MLX5_SET_CFG(in, start_flow_index, ix);
2089         ix += MLX5E_INNER_RSS_GROUP0_SIZE;
2090         MLX5_SET_CFG(in, end_flow_index, ix - 1);
2091         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
2092         if (IS_ERR(ft->g[ft->num_groups]))
2093                 goto err_destory_groups;
2094         ft->num_groups++;
2095
2096         memset(in, 0, inlen);
2097         MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_INNER_HEADERS);
2098         MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ethertype);
2099         MLX5_SET_CFG(in, start_flow_index, ix);
2100         ix += MLX5E_INNER_RSS_GROUP1_SIZE;
2101         MLX5_SET_CFG(in, end_flow_index, ix - 1);
2102         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
2103         if (IS_ERR(ft->g[ft->num_groups]))
2104                 goto err_destory_groups;
2105         ft->num_groups++;
2106
2107         memset(in, 0, inlen);
2108         MLX5_SET_CFG(in, start_flow_index, ix);
2109         ix += MLX5E_INNER_RSS_GROUP2_SIZE;
2110         MLX5_SET_CFG(in, end_flow_index, ix - 1);
2111         ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
2112         if (IS_ERR(ft->g[ft->num_groups]))
2113                 goto err_destory_groups;
2114         ft->num_groups++;
2115
2116         return (0);
2117
2118 err_destory_groups:
2119         err = PTR_ERR(ft->g[ft->num_groups]);
2120         ft->g[ft->num_groups] = NULL;
2121         mlx5e_destroy_groups(ft);
2122
2123         return (err);
2124 }
2125
2126 static int
2127 mlx5e_create_inner_rss_groups(struct mlx5e_flow_table *ft)
2128 {
2129         u32 *in;
2130         int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2131         int err;
2132
2133         in = mlx5_vzalloc(inlen);
2134         if (!in)
2135                 return (-ENOMEM);
2136
2137         err = mlx5e_create_inner_rss_groups_sub(ft, in, inlen);
2138
2139         kvfree(in);
2140         return (err);
2141 }
2142
2143 static int
2144 mlx5e_create_inner_rss_flow_table(struct mlx5e_priv *priv)
2145 {
2146         struct mlx5e_flow_table *ft = &priv->fts.inner_rss;
2147         int err;
2148
2149         ft->num_groups = 0;
2150         ft->t = mlx5_create_flow_table(priv->fts.ns, 0, "inner_rss",
2151                                        MLX5E_INNER_RSS_TABLE_SIZE);
2152
2153         if (IS_ERR(ft->t)) {
2154                 err = PTR_ERR(ft->t);
2155                 ft->t = NULL;
2156                 return (err);
2157         }
2158         ft->g = kcalloc(MLX5E_NUM_INNER_RSS_GROUPS, sizeof(*ft->g),
2159                         GFP_KERNEL);
2160         if (!ft->g) {
2161                 err = -ENOMEM;
2162                 goto err_destroy_inner_rss_flow_table;
2163         }
2164
2165         err = mlx5e_create_inner_rss_groups(ft);
2166         if (err)
2167                 goto err_free_g;
2168
2169         return (0);
2170
2171 err_free_g:
2172         kfree(ft->g);
2173
2174 err_destroy_inner_rss_flow_table:
2175         mlx5_destroy_flow_table(ft->t);
2176         ft->t = NULL;
2177
2178         return (err);
2179 }
2180
2181 static void mlx5e_destroy_inner_rss_flow_table(struct mlx5e_priv *priv)
2182 {
2183         mlx5e_destroy_flow_table(&priv->fts.inner_rss);
2184 }
2185
2186 static void
2187 mlx5e_destroy_vxlan_flow_table(struct mlx5e_priv *priv)
2188 {
2189         mlx5e_destroy_flow_table(&priv->fts.vxlan);
2190 }
2191
2192 int
2193 mlx5e_open_flow_tables(struct mlx5e_priv *priv)
2194 {
2195         int err;
2196
2197         /* setup namespace pointer */
2198         priv->fts.ns = mlx5_get_flow_namespace(
2199             priv->mdev, MLX5_FLOW_NAMESPACE_KERNEL);
2200
2201         err = mlx5e_create_vlan_flow_table(priv);
2202         if (err)
2203                 return (err);
2204
2205         err = mlx5e_create_vxlan_flow_table(priv);
2206         if (err)
2207                 goto err_destroy_vlan_flow_table;
2208
2209         err = mlx5e_create_main_flow_table(priv, true);
2210         if (err)
2211                 goto err_destroy_vxlan_flow_table;
2212
2213         err = mlx5e_create_inner_rss_flow_table(priv);
2214         if (err)
2215                 goto err_destroy_main_flow_table_true;
2216
2217         err = mlx5e_create_main_flow_table(priv, false);
2218         if (err)
2219                 goto err_destroy_inner_rss_flow_table;
2220
2221         err = mlx5e_add_vxlan_catchall_rule(priv);
2222         if (err)
2223                 goto err_destroy_main_flow_table_false;
2224
2225         err = mlx5e_accel_fs_tcp_create(priv);
2226         if (err)
2227                 goto err_del_vxlan_catchall_rule;
2228
2229         return (0);
2230
2231 err_del_vxlan_catchall_rule:
2232         mlx5e_del_vxlan_catchall_rule(priv);
2233 err_destroy_main_flow_table_false:
2234         mlx5e_destroy_main_flow_table(priv);
2235 err_destroy_inner_rss_flow_table:
2236         mlx5e_destroy_inner_rss_flow_table(priv);
2237 err_destroy_main_flow_table_true:
2238         mlx5e_destroy_main_vxlan_flow_table(priv);
2239 err_destroy_vxlan_flow_table:
2240         mlx5e_destroy_vxlan_flow_table(priv);
2241 err_destroy_vlan_flow_table:
2242         mlx5e_destroy_vlan_flow_table(priv);
2243
2244         return (err);
2245 }
2246
2247 void
2248 mlx5e_close_flow_tables(struct mlx5e_priv *priv)
2249 {
2250         mlx5e_accel_fs_tcp_destroy(priv);
2251         mlx5e_del_vxlan_catchall_rule(priv);
2252         mlx5e_destroy_main_flow_table(priv);
2253         mlx5e_destroy_inner_rss_flow_table(priv);
2254         mlx5e_destroy_main_vxlan_flow_table(priv);
2255         mlx5e_destroy_vxlan_flow_table(priv);
2256         mlx5e_destroy_vlan_flow_table(priv);
2257 }
2258
2259 int
2260 mlx5e_open_flow_rules(struct mlx5e_priv *priv)
2261 {
2262         int err;
2263
2264         err = mlx5e_add_all_vlan_rules(priv);
2265         if (err)
2266                 return (err);
2267
2268         err = mlx5e_add_main_vxlan_rules(priv);
2269         if (err)
2270                 goto err_del_all_vlan_rules;
2271
2272         err = mlx5e_add_all_vxlan_rules(priv);
2273         if (err)
2274                 goto err_del_main_vxlan_rules;
2275
2276         mlx5e_set_rx_mode_core(priv, true);
2277
2278         set_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state);
2279
2280         return (0);
2281
2282 err_del_main_vxlan_rules:
2283         mlx5e_del_main_vxlan_rules(priv);
2284
2285 err_del_all_vlan_rules:
2286         mlx5e_del_all_vlan_rules(priv);
2287
2288         return (err);
2289 }
2290
2291 void
2292 mlx5e_close_flow_rules(struct mlx5e_priv *priv)
2293 {
2294         clear_bit(MLX5E_STATE_FLOW_RULES_READY, &priv->state);
2295
2296         mlx5e_set_rx_mode_core(priv, false);
2297         mlx5e_del_all_vxlan_rules(priv);
2298         mlx5e_del_main_vxlan_rules(priv);
2299         mlx5e_del_all_vlan_rules(priv);
2300 }