2 * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 #include <dev/mlx5/driver.h>
29 #include <dev/mlx5/flow_table.h>
30 #include "mlx5_core.h"
33 struct mlx5_flow_table_group g;
38 struct mlx5_flow_table {
39 struct mlx5_core_dev *dev;
44 struct mutex mutex; /* sync bitmap alloc */
46 struct mlx5_ftg *group;
47 unsigned long *bitmap;
51 static int mlx5_set_flow_entry_cmd(struct mlx5_flow_table *ft, u32 group_ix,
52 u32 flow_index, void *flow_context)
54 u32 out[MLX5_ST_SZ_DW(set_fte_out)];
56 void *in_flow_context;
58 MLX5_GET(flow_context, flow_context, destination_list_size) *
59 MLX5_ST_SZ_BYTES(dest_format_struct);
60 int inlen = MLX5_ST_SZ_BYTES(set_fte_in) + fcdls;
63 in = mlx5_vzalloc(inlen);
65 mlx5_core_warn(ft->dev, "failed to allocate inbox\n");
69 MLX5_SET(set_fte_in, in, vport_number, ft->vport);
70 MLX5_SET(set_fte_in, in, other_vport, !!ft->vport);
71 MLX5_SET(set_fte_in, in, table_type, ft->type);
72 MLX5_SET(set_fte_in, in, table_id, ft->id);
73 MLX5_SET(set_fte_in, in, flow_index, flow_index);
74 MLX5_SET(set_fte_in, in, opcode, MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY);
76 in_flow_context = MLX5_ADDR_OF(set_fte_in, in, flow_context);
77 memcpy(in_flow_context, flow_context,
78 MLX5_ST_SZ_BYTES(flow_context) + fcdls);
80 MLX5_SET(flow_context, in_flow_context, group_id, ft->group[group_ix].id);
82 memset(out, 0, sizeof(out));
83 err = mlx5_cmd_exec_check_status(ft->dev, in, inlen, out,
90 static int mlx5_del_flow_entry_cmd(struct mlx5_flow_table *ft, u32 flow_index)
92 u32 in[MLX5_ST_SZ_DW(delete_fte_in)];
93 u32 out[MLX5_ST_SZ_DW(delete_fte_out)];
95 memset(in, 0, sizeof(in));
96 memset(out, 0, sizeof(out));
98 #define MLX5_SET_DFTEI(p, x, v) MLX5_SET(delete_fte_in, p, x, v)
99 MLX5_SET_DFTEI(in, vport_number, ft->vport);
100 MLX5_SET_DFTEI(in, other_vport, !!ft->vport);
101 MLX5_SET_DFTEI(in, table_type, ft->type);
102 MLX5_SET_DFTEI(in, table_id, ft->id);
103 MLX5_SET_DFTEI(in, flow_index, flow_index);
104 MLX5_SET_DFTEI(in, opcode, MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
106 return mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out,
110 static void mlx5_destroy_flow_group_cmd(struct mlx5_flow_table *ft, int i)
112 u32 in[MLX5_ST_SZ_DW(destroy_flow_group_in)];
113 u32 out[MLX5_ST_SZ_DW(destroy_flow_group_out)];
115 memset(in, 0, sizeof(in));
116 memset(out, 0, sizeof(out));
118 #define MLX5_SET_DFGI(p, x, v) MLX5_SET(destroy_flow_group_in, p, x, v)
119 MLX5_SET_DFGI(in, vport_number, ft->vport);
120 MLX5_SET_DFGI(in, other_vport, !!ft->vport);
121 MLX5_SET_DFGI(in, table_type, ft->type);
122 MLX5_SET_DFGI(in, table_id, ft->id);
123 MLX5_SET_DFGI(in, opcode, MLX5_CMD_OP_DESTROY_FLOW_GROUP);
124 MLX5_SET_DFGI(in, group_id, ft->group[i].id);
125 mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out, sizeof(out));
128 static int mlx5_create_flow_group_cmd(struct mlx5_flow_table *ft, int i)
130 u32 out[MLX5_ST_SZ_DW(create_flow_group_out)];
132 void *in_match_criteria;
133 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
134 struct mlx5_flow_table_group *g = &ft->group[i].g;
135 u32 start_ix = ft->group[i].start_ix;
136 u32 end_ix = start_ix + (1 << g->log_sz) - 1;
139 in = mlx5_vzalloc(inlen);
141 mlx5_core_warn(ft->dev, "failed to allocate inbox\n");
144 in_match_criteria = MLX5_ADDR_OF(create_flow_group_in, in,
147 memset(out, 0, sizeof(out));
149 #define MLX5_SET_CFGI(p, x, v) MLX5_SET(create_flow_group_in, p, x, v)
150 MLX5_SET_CFGI(in, vport_number, ft->vport);
151 MLX5_SET_CFGI(in, other_vport, !!ft->vport);
152 MLX5_SET_CFGI(in, table_type, ft->type);
153 MLX5_SET_CFGI(in, table_id, ft->id);
154 MLX5_SET_CFGI(in, opcode, MLX5_CMD_OP_CREATE_FLOW_GROUP);
155 MLX5_SET_CFGI(in, start_flow_index, start_ix);
156 MLX5_SET_CFGI(in, end_flow_index, end_ix);
157 MLX5_SET_CFGI(in, match_criteria_enable, g->match_criteria_enable);
159 memcpy(in_match_criteria, g->match_criteria,
160 MLX5_ST_SZ_BYTES(fte_match_param));
162 err = mlx5_cmd_exec_check_status(ft->dev, in, inlen, out,
165 ft->group[i].id = MLX5_GET(create_flow_group_out, out,
173 static void mlx5_destroy_flow_table_groups(struct mlx5_flow_table *ft)
177 for (i = 0; i < ft->num_groups; i++)
178 mlx5_destroy_flow_group_cmd(ft, i);
181 static int mlx5_create_flow_table_groups(struct mlx5_flow_table *ft)
186 for (i = 0; i < ft->num_groups; i++) {
187 err = mlx5_create_flow_group_cmd(ft, i);
189 goto err_destroy_flow_table_groups;
194 err_destroy_flow_table_groups:
195 for (i--; i >= 0; i--)
196 mlx5_destroy_flow_group_cmd(ft, i);
201 static int mlx5_create_flow_table_cmd(struct mlx5_flow_table *ft)
203 u32 in[MLX5_ST_SZ_DW(create_flow_table_in)];
204 u32 out[MLX5_ST_SZ_DW(create_flow_table_out)];
207 memset(in, 0, sizeof(in));
209 MLX5_SET(create_flow_table_in, in, vport_number, ft->vport);
210 MLX5_SET(create_flow_table_in, in, other_vport, !!ft->vport);
211 MLX5_SET(create_flow_table_in, in, table_type, ft->type);
212 MLX5_SET(create_flow_table_in, in, level, ft->level);
213 MLX5_SET(create_flow_table_in, in, log_size, order_base_2(ft->size));
215 MLX5_SET(create_flow_table_in, in, opcode,
216 MLX5_CMD_OP_CREATE_FLOW_TABLE);
218 memset(out, 0, sizeof(out));
219 err = mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out,
224 ft->id = MLX5_GET(create_flow_table_out, out, table_id);
229 static void mlx5_destroy_flow_table_cmd(struct mlx5_flow_table *ft)
231 u32 in[MLX5_ST_SZ_DW(destroy_flow_table_in)];
232 u32 out[MLX5_ST_SZ_DW(destroy_flow_table_out)];
234 memset(in, 0, sizeof(in));
235 memset(out, 0, sizeof(out));
237 #define MLX5_SET_DFTI(p, x, v) MLX5_SET(destroy_flow_table_in, p, x, v)
238 MLX5_SET_DFTI(in, vport_number, ft->vport);
239 MLX5_SET_DFTI(in, other_vport, !!ft->vport);
240 MLX5_SET_DFTI(in, table_type, ft->type);
241 MLX5_SET_DFTI(in, table_id, ft->id);
242 MLX5_SET_DFTI(in, opcode, MLX5_CMD_OP_DESTROY_FLOW_TABLE);
244 mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out, sizeof(out));
247 static int mlx5_find_group(struct mlx5_flow_table *ft, u8 match_criteria_enable,
248 u32 *match_criteria, int *group_ix)
250 void *mc_outer = MLX5_ADDR_OF(fte_match_param, match_criteria,
252 void *mc_misc = MLX5_ADDR_OF(fte_match_param, match_criteria,
254 void *mc_inner = MLX5_ADDR_OF(fte_match_param, match_criteria,
256 int mc_outer_sz = MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4);
257 int mc_misc_sz = MLX5_ST_SZ_BYTES(fte_match_set_misc);
258 int mc_inner_sz = MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4);
261 for (i = 0; i < ft->num_groups; i++) {
262 struct mlx5_flow_table_group *g = &ft->group[i].g;
263 void *gmc_outer = MLX5_ADDR_OF(fte_match_param,
266 void *gmc_misc = MLX5_ADDR_OF(fte_match_param,
269 void *gmc_inner = MLX5_ADDR_OF(fte_match_param,
273 if (g->match_criteria_enable != match_criteria_enable)
276 if (match_criteria_enable & MLX5_MATCH_OUTER_HEADERS)
277 if (memcmp(mc_outer, gmc_outer, mc_outer_sz))
280 if (match_criteria_enable & MLX5_MATCH_MISC_PARAMETERS)
281 if (memcmp(mc_misc, gmc_misc, mc_misc_sz))
284 if (match_criteria_enable & MLX5_MATCH_INNER_HEADERS)
285 if (memcmp(mc_inner, gmc_inner, mc_inner_sz))
295 static int alloc_flow_index(struct mlx5_flow_table *ft, int group_ix, u32 *ix)
297 struct mlx5_ftg *g = &ft->group[group_ix];
300 mutex_lock(&ft->mutex);
302 *ix = find_next_zero_bit(ft->bitmap, ft->size, g->start_ix);
303 if (*ix >= (g->start_ix + (1 << g->g.log_sz)))
306 __set_bit(*ix, ft->bitmap);
308 mutex_unlock(&ft->mutex);
313 static void mlx5_free_flow_index(struct mlx5_flow_table *ft, u32 ix)
315 __clear_bit(ix, ft->bitmap);
318 int mlx5_add_flow_table_entry(void *flow_table, u8 match_criteria_enable,
319 void *match_criteria, void *flow_context,
322 struct mlx5_flow_table *ft = flow_table;
326 err = mlx5_find_group(ft, match_criteria_enable, match_criteria,
329 mlx5_core_warn(ft->dev, "mlx5_find_group failed\n");
333 err = alloc_flow_index(ft, group_ix, flow_index);
335 mlx5_core_warn(ft->dev, "alloc_flow_index failed\n");
339 err = mlx5_set_flow_entry_cmd(ft, group_ix, *flow_index, flow_context);
341 mlx5_free_flow_index(ft, *flow_index);
345 EXPORT_SYMBOL(mlx5_add_flow_table_entry);
347 int mlx5_del_flow_table_entry(void *flow_table, u32 flow_index)
349 struct mlx5_flow_table *ft = flow_table;
352 ret = mlx5_del_flow_entry_cmd(ft, flow_index);
354 mlx5_free_flow_index(ft, flow_index);
357 EXPORT_SYMBOL(mlx5_del_flow_table_entry);
359 void *mlx5_create_flow_table(struct mlx5_core_dev *dev, u8 level, u8 table_type,
362 struct mlx5_flow_table_group *group)
364 struct mlx5_flow_table *ft;
372 for (i = 0; i < num_groups; i++)
373 ft_size += (1 << group[i].log_sz);
375 ft = kzalloc(sizeof(*ft), GFP_KERNEL);
376 gr = kcalloc(num_groups, sizeof(struct mlx5_ftg), GFP_KERNEL);
377 bm = kcalloc(BITS_TO_LONGS(ft_size), sizeof(uintptr_t), GFP_KERNEL);
381 ft->num_groups = num_groups;
384 ft->type = table_type;
387 mutex_init(&ft->mutex);
389 for (i = 0; i < ft->num_groups; i++) {
390 memcpy(&ft->group[i].g, &group[i], sizeof(*group));
391 ft->group[i].start_ix = start_ix;
392 start_ix += 1 << group[i].log_sz;
395 err = mlx5_create_flow_table_cmd(ft);
399 err = mlx5_create_flow_table_groups(ft);
401 goto err_destroy_flow_table_cmd;
405 err_destroy_flow_table_cmd:
406 mlx5_destroy_flow_table_cmd(ft);
409 mlx5_core_warn(dev, "failed to alloc flow table\n");
416 EXPORT_SYMBOL(mlx5_create_flow_table);
418 void mlx5_destroy_flow_table(void *flow_table)
420 struct mlx5_flow_table *ft = flow_table;
422 mlx5_destroy_flow_table_groups(ft);
423 mlx5_destroy_flow_table_cmd(ft);
428 EXPORT_SYMBOL(mlx5_destroy_flow_table);
430 u32 mlx5_get_flow_table_id(void *flow_table)
432 struct mlx5_flow_table *ft = flow_table;
436 EXPORT_SYMBOL(mlx5_get_flow_table_id);
438 int mlx5_set_flow_table_root(struct mlx5_core_dev *mdev, u16 op_mod,
439 u8 vport_num, u8 table_type, u32 table_id,
442 u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)];
443 u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)];
445 int is_group_manager;
447 is_group_manager = MLX5_CAP_GEN(mdev, vport_group_manager);
449 memset(in, 0, sizeof(in));
451 MLX5_SET(set_flow_table_root_in, in, op_mod, op_mod);
452 MLX5_SET(set_flow_table_root_in, in, table_type, table_type);
453 MLX5_SET(set_flow_table_root_in, in, underlay_qpn, underlay_qpn);
454 if (op_mod == MLX5_SET_FLOW_TABLE_ROOT_OPMOD_SET)
455 MLX5_SET(set_flow_table_root_in, in, table_id, table_id);
457 MLX5_SET(set_flow_table_root_in, in, opcode,
458 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT);
461 if (is_group_manager) {
462 MLX5_SET(set_flow_table_root_in, in, other_vport,
464 MLX5_SET(set_flow_table_root_in, in, vport_number,
471 memset(out, 0, sizeof(out));
472 err = mlx5_cmd_exec_check_status(mdev, in, sizeof(in), out,
479 EXPORT_SYMBOL(mlx5_set_flow_table_root);