2 * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 static int mlx5e_rl_open_workers(struct mlx5e_priv *);
33 static void mlx5e_rl_close_workers(struct mlx5e_priv *);
34 static int mlx5e_rl_sysctl_show_rate_table(SYSCTL_HANDLER_ARGS);
35 static void mlx5e_rl_sysctl_add_u64_oid(struct mlx5e_rl_priv_data *, unsigned x,
36 struct sysctl_oid *, const char *name, const char *desc);
37 static void mlx5e_rl_sysctl_add_stats_u64_oid(struct mlx5e_rl_priv_data *rl, unsigned x,
38 struct sysctl_oid *node, const char *name, const char *desc);
39 static int mlx5e_rl_tx_limit_add(struct mlx5e_rl_priv_data *, uint64_t value);
40 static int mlx5e_rl_tx_limit_clr(struct mlx5e_rl_priv_data *, uint64_t value);
43 mlx5e_rl_build_sq_param(struct mlx5e_rl_priv_data *rl,
44 struct mlx5e_sq_param *param)
46 void *sqc = param->sqc;
47 void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
48 uint8_t log_sq_size = order_base_2(rl->param.tx_queue_size);
50 MLX5_SET(wq, wq, log_wq_sz, log_sq_size);
51 MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
52 MLX5_SET(wq, wq, pd, rl->priv->pdn);
54 param->wq.buf_numa_node = 0;
55 param->wq.db_numa_node = 0;
60 mlx5e_rl_build_cq_param(struct mlx5e_rl_priv_data *rl,
61 struct mlx5e_cq_param *param)
63 void *cqc = param->cqc;
64 uint8_t log_sq_size = order_base_2(rl->param.tx_queue_size);
66 MLX5_SET(cqc, cqc, log_cq_size, log_sq_size);
67 MLX5_SET(cqc, cqc, cq_period, rl->param.tx_coalesce_usecs);
68 MLX5_SET(cqc, cqc, cq_max_count, rl->param.tx_coalesce_pkts);
70 switch (rl->param.tx_coalesce_mode) {
72 MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
75 if (MLX5_CAP_GEN(rl->priv->mdev, cq_period_start_from_cqe))
76 MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
78 MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
84 mlx5e_rl_build_channel_param(struct mlx5e_rl_priv_data *rl,
85 struct mlx5e_rl_channel_param *cparam)
87 memset(cparam, 0, sizeof(*cparam));
89 mlx5e_rl_build_sq_param(rl, &cparam->sq);
90 mlx5e_rl_build_cq_param(rl, &cparam->cq);
94 mlx5e_rl_create_sq(struct mlx5e_priv *priv, struct mlx5e_sq *sq,
95 struct mlx5e_sq_param *param, int ix)
97 struct mlx5_core_dev *mdev = priv->mdev;
98 void *sqc = param->sqc;
99 void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq);
102 /* Create DMA descriptor TAG */
103 if ((err = -bus_dma_tag_create(
104 bus_get_dma_tag(mdev->pdev->dev.bsddev),
105 1, /* any alignment */
107 BUS_SPACE_MAXADDR, /* lowaddr */
108 BUS_SPACE_MAXADDR, /* highaddr */
109 NULL, NULL, /* filter, filterarg */
110 MLX5E_MAX_TX_PAYLOAD_SIZE, /* maxsize */
111 MLX5E_MAX_TX_MBUF_FRAGS, /* nsegments */
112 MLX5E_MAX_TX_MBUF_SIZE, /* maxsegsize */
114 NULL, NULL, /* lockfunc, lockfuncarg */
119 sq->uar = priv->rl.sq_uar;
121 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, &sq->wq,
124 goto err_free_dma_tag;
126 sq->wq.db = &sq->wq.db[MLX5_SND_DBR];
128 * The sq->bf_buf_size variable is intentionally left zero so
129 * that the doorbell writes will occur at the same memory
133 err = mlx5e_alloc_sq_db(sq);
135 goto err_sq_wq_destroy;
137 sq->mkey_be = cpu_to_be32(priv->mr.key);
140 sq->max_inline = priv->params.tx_max_inline;
141 sq->min_inline_mode = priv->params.tx_min_inline_mode;
142 sq->vlan_inline_cap = MLX5_CAP_ETH(mdev, wqe_vlan_insert);
147 mlx5_wq_destroy(&sq->wq_ctrl);
149 bus_dma_tag_destroy(sq->dma_tag);
155 mlx5e_rl_destroy_sq(struct mlx5e_sq *sq)
158 mlx5e_free_sq_db(sq);
159 mlx5_wq_destroy(&sq->wq_ctrl);
163 mlx5e_rl_open_sq(struct mlx5e_priv *priv, struct mlx5e_sq *sq,
164 struct mlx5e_sq_param *param, int ix)
168 err = mlx5e_rl_create_sq(priv, sq, param, ix);
172 err = mlx5e_enable_sq(sq, param, priv->rl.tisn);
176 err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY);
183 mlx5e_disable_sq(sq);
185 mlx5e_rl_destroy_sq(sq);
191 mlx5e_rl_chan_mtx_init(struct mlx5e_priv *priv, struct mlx5e_sq *sq)
193 mtx_init(&sq->lock, "mlx5tx-rl", NULL, MTX_DEF);
194 mtx_init(&sq->comp_lock, "mlx5comp-rl", NULL, MTX_DEF);
196 callout_init_mtx(&sq->cev_callout, &sq->lock, 0);
198 sq->cev_factor = priv->rl.param.tx_completion_fact;
200 /* ensure the TX completion event factor is not zero */
201 if (sq->cev_factor == 0)
206 mlx5e_rl_open_channel(struct mlx5e_rl_worker *rlw, int eq_ix,
207 struct mlx5e_rl_channel_param *cparam,
208 struct mlx5e_sq *volatile *ppsq)
210 struct mlx5e_priv *priv = rlw->priv;
214 sq = malloc(sizeof(*sq), M_MLX5EN, M_WAITOK | M_ZERO);
217 mlx5e_rl_chan_mtx_init(priv, sq);
219 /* open TX completion queue */
220 err = mlx5e_open_cq(priv, &cparam->cq, &sq->cq,
221 &mlx5e_tx_cq_comp, eq_ix);
225 err = mlx5e_rl_open_sq(priv, sq, &cparam->sq, eq_ix);
227 goto err_close_tx_cq;
229 /* store TX channel pointer */
232 /* poll TX queue initially */
233 sq->cq.mcq.comp(&sq->cq.mcq);
238 mlx5e_close_cq(&sq->cq);
241 /* destroy mutexes */
242 mtx_destroy(&sq->lock);
243 mtx_destroy(&sq->comp_lock);
245 atomic_add_64(&priv->rl.stats.tx_allocate_resource_failure, 1ULL);
250 mlx5e_rl_close_channel(struct mlx5e_sq *volatile *ppsq)
252 struct mlx5e_sq *sq = *ppsq;
254 /* check if channel is already closed */
257 /* ensure channel pointer is no longer used */
260 /* teardown and destroy SQ */
262 mlx5e_disable_sq(sq);
263 mlx5e_rl_destroy_sq(sq);
266 mlx5e_close_cq(&sq->cq);
268 /* destroy mutexes */
269 mtx_destroy(&sq->lock);
270 mtx_destroy(&sq->comp_lock);
276 mlx5e_rl_sync_tx_completion_fact(struct mlx5e_rl_priv_data *rl)
279 * Limit the maximum distance between completion events to
280 * half of the currently set TX queue size.
282 * The maximum number of queue entries a single IP packet can
283 * consume is given by MLX5_SEND_WQE_MAX_WQEBBS.
285 * The worst case max value is then given as below:
287 uint64_t max = rl->param.tx_queue_size /
288 (2 * MLX5_SEND_WQE_MAX_WQEBBS);
291 * Update the maximum completion factor value in case the
292 * tx_queue_size field changed. Ensure we don't overflow
297 else if (max > 65535)
299 rl->param.tx_completion_fact_max = max;
302 * Verify that the current TX completion factor is within the
305 if (rl->param.tx_completion_fact < 1)
306 rl->param.tx_completion_fact = 1;
307 else if (rl->param.tx_completion_fact > max)
308 rl->param.tx_completion_fact = max;
312 mlx5e_rl_modify_sq(struct mlx5e_sq *sq, uint16_t rl_index)
314 struct mlx5e_priv *priv = sq->priv;
315 struct mlx5_core_dev *mdev = priv->mdev;
322 inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
323 in = mlx5_vzalloc(inlen);
327 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
329 MLX5_SET(modify_sq_in, in, sqn, sq->sqn);
330 MLX5_SET(modify_sq_in, in, sq_state, MLX5_SQC_STATE_RDY);
331 MLX5_SET64(modify_sq_in, in, modify_bitmask, 1);
332 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RDY);
333 MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, rl_index);
335 err = mlx5_core_modify_sq(mdev, in, inlen);
343 * This function will search the configured rate limit table for the
344 * best match to avoid that a single socket based application can
345 * allocate all the available hardware rates. If the user selected
346 * rate deviates too much from the closes rate available in the rate
347 * limit table, unlimited rate will be selected.
350 mlx5e_rl_find_best_rate_locked(struct mlx5e_rl_priv_data *rl, uint64_t user_rate)
352 uint64_t distance = -1ULL;
354 uint64_t retval = 0; /* unlimited */
357 /* search for closest rate */
358 for (x = 0; x != rl->param.tx_rates_def; x++) {
359 uint64_t rate = rl->rate_limit_table[x];
363 if (rate > user_rate)
364 diff = rate - user_rate;
366 diff = user_rate - rate;
368 /* check if distance is smaller than previous rate */
369 if (diff < distance) {
375 /* range check for multiplication below */
376 if (user_rate > rl->param.tx_limit_max)
377 user_rate = rl->param.tx_limit_max;
379 /* fallback to unlimited, if rate deviates too much */
380 if (distance > howmany(user_rate *
381 rl->param.tx_allowed_deviation, 1000ULL))
388 * This function sets the requested rate for a rate limit channel, in
389 * bits per second. The requested rate will be filtered through the
390 * find best rate function above.
393 mlx5e_rlw_channel_set_rate_locked(struct mlx5e_rl_worker *rlw,
394 struct mlx5e_rl_channel *channel, uint64_t rate)
396 struct mlx5e_rl_priv_data *rl = &rlw->priv->rl;
404 MLX5E_RL_WORKER_UNLOCK(rlw);
408 /* get current burst size in bytes */
409 temp = rl->param.tx_burst_size *
410 MLX5E_SW2HW_MTU(rlw->priv->ifp->if_mtu);
412 /* limit burst size to 64K currently */
418 rate = mlx5e_rl_find_best_rate_locked(rl, rate);
420 MLX5E_RL_RUNLOCK(rl);
423 /* rate doesn't exist, fallback to unlimited */
427 atomic_add_64(&rlw->priv->rl.stats.tx_modify_rate_failure, 1ULL);
429 /* get a reference on the new rate */
430 error = -mlx5_rl_add_rate(rlw->priv->mdev,
431 howmany(rate, 1000), burst, &index);
434 /* adding rate failed, fallback to unlimited */
437 atomic_add_64(&rlw->priv->rl.stats.tx_add_new_rate_failure, 1ULL);
440 MLX5E_RL_WORKER_LOCK(rlw);
443 burst = 0; /* default */
446 /* atomically swap rates */
447 temp = channel->last_rate;
448 channel->last_rate = rate;
451 /* atomically swap burst size */
452 temp = channel->last_burst;
453 channel->last_burst = burst;
456 MLX5E_RL_WORKER_UNLOCK(rlw);
457 /* put reference on the old rate, if any */
459 mlx5_rl_remove_rate(rlw->priv->mdev,
460 howmany(rate, 1000), burst);
466 error = mlx5e_rl_modify_sq(sq, index);
468 atomic_add_64(&rlw->priv->rl.stats.tx_modify_rate_failure, 1ULL);
471 MLX5E_RL_WORKER_LOCK(rlw);
477 mlx5e_rl_worker(void *arg)
480 struct mlx5e_rl_worker *rlw = arg;
481 struct mlx5e_rl_channel *channel;
482 struct mlx5e_priv *priv;
487 /* set thread priority */
491 sched_prio(td, PI_SWI(SWI_NET));
496 /* compute completion vector */
497 ix = (rlw - priv->rl.workers) %
498 priv->mdev->priv.eq_table.num_comp_vectors;
500 /* TODO bind to CPU */
502 /* open all the SQs */
503 MLX5E_RL_WORKER_LOCK(rlw);
504 for (x = 0; x < priv->rl.param.tx_channels_per_worker_def; x++) {
505 struct mlx5e_rl_channel *channel = rlw->channels + x;
507 #if !defined(HAVE_RL_PRE_ALLOCATE_CHANNELS)
508 if (channel->state == MLX5E_RL_ST_FREE)
511 MLX5E_RL_WORKER_UNLOCK(rlw);
513 MLX5E_RL_RLOCK(&priv->rl);
514 error = mlx5e_rl_open_channel(rlw, ix,
515 &priv->rl.chan_param, &channel->sq);
516 MLX5E_RL_RUNLOCK(&priv->rl);
518 MLX5E_RL_WORKER_LOCK(rlw);
521 "mlx5e_rl_open_channel failed: %d\n", error);
524 mlx5e_rlw_channel_set_rate_locked(rlw, channel, channel->init_rate);
527 if (STAILQ_FIRST(&rlw->process_head) == NULL) {
528 /* check if we are tearing down */
529 if (rlw->worker_done != 0)
531 cv_wait(&rlw->cv, &rlw->mtx);
533 /* check if we are tearing down */
534 if (rlw->worker_done != 0)
536 channel = STAILQ_FIRST(&rlw->process_head);
537 if (channel != NULL) {
538 STAILQ_REMOVE_HEAD(&rlw->process_head, entry);
540 switch (channel->state) {
541 case MLX5E_RL_ST_MODIFY:
542 channel->state = MLX5E_RL_ST_USED;
543 MLX5E_RL_WORKER_UNLOCK(rlw);
545 /* create channel by demand */
546 if (channel->sq == NULL) {
547 MLX5E_RL_RLOCK(&priv->rl);
548 error = mlx5e_rl_open_channel(rlw, ix,
549 &priv->rl.chan_param, &channel->sq);
550 MLX5E_RL_RUNLOCK(&priv->rl);
554 "mlx5e_rl_open_channel failed: %d\n", error);
556 atomic_add_64(&rlw->priv->rl.stats.tx_open_queues, 1ULL);
559 mlx5e_resume_sq(channel->sq);
562 MLX5E_RL_WORKER_LOCK(rlw);
563 /* convert from bytes/s to bits/s and set new rate */
564 error = mlx5e_rlw_channel_set_rate_locked(rlw, channel,
565 channel->new_rate * 8ULL);
568 "mlx5e_rlw_channel_set_rate_locked failed: %d\n",
573 case MLX5E_RL_ST_DESTROY:
574 error = mlx5e_rlw_channel_set_rate_locked(rlw, channel, 0);
577 "mlx5e_rlw_channel_set_rate_locked failed: %d\n",
580 if (channel->sq != NULL) {
582 * Make sure all packets are
583 * transmitted before SQ is
584 * returned to free list:
586 MLX5E_RL_WORKER_UNLOCK(rlw);
587 mlx5e_drain_sq(channel->sq);
588 MLX5E_RL_WORKER_LOCK(rlw);
590 /* put the channel back into the free list */
591 STAILQ_INSERT_HEAD(&rlw->index_list_head, channel, entry);
592 channel->state = MLX5E_RL_ST_FREE;
593 atomic_add_64(&priv->rl.stats.tx_active_connections, -1ULL);
602 /* close all the SQs */
603 for (x = 0; x < priv->rl.param.tx_channels_per_worker_def; x++) {
604 struct mlx5e_rl_channel *channel = rlw->channels + x;
606 /* update the initial rate */
607 channel->init_rate = channel->last_rate;
609 /* make sure we free up the rate resource */
610 mlx5e_rlw_channel_set_rate_locked(rlw, channel, 0);
612 if (channel->sq != NULL) {
613 MLX5E_RL_WORKER_UNLOCK(rlw);
614 mlx5e_rl_close_channel(&channel->sq);
615 atomic_add_64(&rlw->priv->rl.stats.tx_open_queues, -1ULL);
616 MLX5E_RL_WORKER_LOCK(rlw);
620 rlw->worker_done = 0;
621 cv_broadcast(&rlw->cv);
622 MLX5E_RL_WORKER_UNLOCK(rlw);
628 mlx5e_rl_open_tis(struct mlx5e_priv *priv)
630 struct mlx5_core_dev *mdev = priv->mdev;
631 u32 in[MLX5_ST_SZ_DW(create_tis_in)];
632 void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
634 memset(in, 0, sizeof(in));
636 MLX5_SET(tisc, tisc, prio, 0);
637 MLX5_SET(tisc, tisc, transport_domain, priv->tdn);
639 return (mlx5_core_create_tis(mdev, in, sizeof(in), &priv->rl.tisn));
643 mlx5e_rl_close_tis(struct mlx5e_priv *priv)
645 mlx5_core_destroy_tis(priv->mdev, priv->rl.tisn);
649 mlx5e_rl_set_default_params(struct mlx5e_rl_params *param,
650 struct mlx5_core_dev *mdev)
652 /* ratelimit workers */
653 param->tx_worker_threads_def = mdev->priv.eq_table.num_comp_vectors;
654 param->tx_worker_threads_max = MLX5E_RL_MAX_WORKERS;
657 if (param->tx_worker_threads_def == 0 ||
658 param->tx_worker_threads_def > param->tx_worker_threads_max)
659 param->tx_worker_threads_def = param->tx_worker_threads_max;
661 /* ratelimit channels */
662 param->tx_channels_per_worker_def = MLX5E_RL_MAX_SQS /
663 param->tx_worker_threads_def;
664 param->tx_channels_per_worker_max = MLX5E_RL_MAX_SQS;
667 if (param->tx_channels_per_worker_def > MLX5E_RL_DEF_SQ_PER_WORKER)
668 param->tx_channels_per_worker_def = MLX5E_RL_DEF_SQ_PER_WORKER;
670 /* set default burst size */
671 param->tx_burst_size = 4; /* MTUs */
674 * Set maximum burst size
676 * The burst size is multiplied by the MTU and clamped to the
677 * range 0 ... 65535 bytes inclusivly before fed into the
680 * NOTE: If the burst size or MTU is changed only ratelimit
681 * connections made after the change will use the new burst
684 param->tx_burst_size_max = 255;
686 /* get firmware rate limits in 1000bit/s and convert them to bit/s */
687 param->tx_limit_min = mdev->priv.rl_table.min_rate * 1000ULL;
688 param->tx_limit_max = mdev->priv.rl_table.max_rate * 1000ULL;
690 /* ratelimit table size */
691 param->tx_rates_max = mdev->priv.rl_table.max_size;
694 if (param->tx_rates_max > MLX5E_RL_MAX_TX_RATES)
695 param->tx_rates_max = MLX5E_RL_MAX_TX_RATES;
697 /* set default number of rates */
698 param->tx_rates_def = param->tx_rates_max;
700 /* set maximum allowed rate deviation */
701 if (param->tx_limit_max != 0) {
703 * Make sure the deviation multiplication doesn't
704 * overflow unsigned 64-bit:
706 param->tx_allowed_deviation_max = -1ULL /
709 /* set default rate deviation */
710 param->tx_allowed_deviation = 50; /* 5.0% */
712 /* channel parameters */
713 param->tx_queue_size = (1 << MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE);
714 param->tx_coalesce_usecs = MLX5E_RL_TX_COAL_USEC_DEFAULT;
715 param->tx_coalesce_pkts = MLX5E_RL_TX_COAL_PKTS_DEFAULT;
716 param->tx_coalesce_mode = MLX5E_RL_TX_COAL_MODE_DEFAULT;
717 param->tx_completion_fact = MLX5E_RL_TX_COMP_FACT_DEFAULT;
720 static const char *mlx5e_rl_params_desc[] = {
721 MLX5E_RL_PARAMS(MLX5E_STATS_DESC)
724 static const char *mlx5e_rl_table_params_desc[] = {
725 MLX5E_RL_TABLE_PARAMS(MLX5E_STATS_DESC)
728 static const char *mlx5e_rl_stats_desc[] = {
729 MLX5E_RL_STATS(MLX5E_STATS_DESC)
733 mlx5e_rl_init(struct mlx5e_priv *priv)
735 struct mlx5e_rl_priv_data *rl = &priv->rl;
736 struct sysctl_oid *node;
737 struct sysctl_oid *stats;
743 /* check if there is support for packet pacing */
744 if (!MLX5_CAP_GEN(priv->mdev, qos) || !MLX5_CAP_QOS(priv->mdev, packet_pacing))
749 sysctl_ctx_init(&rl->ctx);
751 sx_init(&rl->rl_sxlock, "ratelimit-sxlock");
753 /* allocate shared UAR for SQs */
754 error = mlx5_alloc_map_uar(priv->mdev, &rl->sq_uar);
758 /* open own TIS domain for ratelimit SQs */
759 error = mlx5e_rl_open_tis(priv);
763 /* setup default value for parameters */
764 mlx5e_rl_set_default_params(&rl->param, priv->mdev);
766 /* update the completion factor */
767 mlx5e_rl_sync_tx_completion_fact(rl);
769 /* create root node */
770 node = SYSCTL_ADD_NODE(&rl->ctx,
771 SYSCTL_CHILDREN(priv->sysctl_ifnet), OID_AUTO,
772 "rate_limit", CTLFLAG_RW, NULL, "Rate limiting support");
776 for (i = 0; i != MLX5E_RL_PARAMS_NUM; i++) {
777 mlx5e_rl_sysctl_add_u64_oid(rl,
778 MLX5E_RL_PARAMS_INDEX(arg[i]),
779 node, mlx5e_rl_params_desc[2 * i],
780 mlx5e_rl_params_desc[2 * i + 1]);
783 stats = SYSCTL_ADD_NODE(&rl->ctx, SYSCTL_CHILDREN(node),
784 OID_AUTO, "stats", CTLFLAG_RD, NULL,
785 "Rate limiting statistics");
788 for (i = 0; i != MLX5E_RL_STATS_NUM; i++) {
789 mlx5e_rl_sysctl_add_stats_u64_oid(rl, i,
790 stats, mlx5e_rl_stats_desc[2 * i],
791 mlx5e_rl_stats_desc[2 * i + 1]);
796 /* allocate workers array */
797 rl->workers = malloc(sizeof(rl->workers[0]) *
798 rl->param.tx_worker_threads_def, M_MLX5EN, M_WAITOK | M_ZERO);
800 /* allocate rate limit array */
801 rl->rate_limit_table = malloc(sizeof(rl->rate_limit_table[0]) *
802 rl->param.tx_rates_def, M_MLX5EN, M_WAITOK | M_ZERO);
805 /* create more SYSCTls */
806 SYSCTL_ADD_PROC(&rl->ctx, SYSCTL_CHILDREN(node), OID_AUTO,
807 "tx_rate_show", CTLTYPE_STRING | CTLFLAG_RD |
808 CTLFLAG_MPSAFE, rl, 0, &mlx5e_rl_sysctl_show_rate_table,
809 "A", "Show table of all configured TX rates");
811 /* try to fetch rate table from kernel environment */
812 for (i = 0; i != rl->param.tx_rates_def; i++) {
813 /* compute path for tunable */
814 snprintf(buf, sizeof(buf), "dev.mce.%d.rate_limit.tx_rate_add_%d",
815 device_get_unit(priv->mdev->pdev->dev.bsddev), (int)i);
816 if (TUNABLE_QUAD_FETCH(buf, &j))
817 mlx5e_rl_tx_limit_add(rl, j);
820 /* setup rate table sysctls */
821 for (i = 0; i != MLX5E_RL_TABLE_PARAMS_NUM; i++) {
822 mlx5e_rl_sysctl_add_u64_oid(rl,
823 MLX5E_RL_PARAMS_INDEX(table_arg[i]),
824 node, mlx5e_rl_table_params_desc[2 * i],
825 mlx5e_rl_table_params_desc[2 * i + 1]);
829 for (j = 0; j < rl->param.tx_worker_threads_def; j++) {
830 struct mlx5e_rl_worker *rlw = rl->workers + j;
834 cv_init(&rlw->cv, "mlx5-worker-cv");
835 mtx_init(&rlw->mtx, "mlx5-worker-mtx", NULL, MTX_DEF);
836 STAILQ_INIT(&rlw->index_list_head);
837 STAILQ_INIT(&rlw->process_head);
839 rlw->channels = malloc(sizeof(rlw->channels[0]) *
840 rl->param.tx_channels_per_worker_def, M_MLX5EN, M_WAITOK | M_ZERO);
842 MLX5E_RL_WORKER_LOCK(rlw);
843 for (i = 0; i < rl->param.tx_channels_per_worker_def; i++) {
844 struct mlx5e_rl_channel *channel = rlw->channels + i;
845 channel->worker = rlw;
846 channel->m_snd_tag.ifp = priv->ifp;
847 STAILQ_INSERT_TAIL(&rlw->index_list_head, channel, entry);
849 MLX5E_RL_WORKER_UNLOCK(rlw);
853 error = mlx5e_rl_open_workers(priv);
858 "mlx5e_rl_open_workers failed: %d\n", error);
864 mlx5_unmap_free_uar(priv->mdev, &rl->sq_uar);
866 sysctl_ctx_free(&rl->ctx);
867 sx_destroy(&rl->rl_sxlock);
872 mlx5e_rl_open_workers(struct mlx5e_priv *priv)
874 struct mlx5e_rl_priv_data *rl = &priv->rl;
875 struct thread *rl_thread = NULL;
876 struct proc *rl_proc = NULL;
880 if (priv->gone || rl->opened)
884 /* compute channel parameters once */
885 mlx5e_rl_build_channel_param(rl, &rl->chan_param);
886 MLX5E_RL_WUNLOCK(rl);
888 for (j = 0; j < rl->param.tx_worker_threads_def; j++) {
889 struct mlx5e_rl_worker *rlw = rl->workers + j;
891 /* start worker thread */
892 error = kproc_kthread_add(mlx5e_rl_worker, rlw, &rl_proc, &rl_thread,
893 RFHIGHPID, 0, "mlx5-ratelimit", "mlx5-rl-worker-thread-%d", (int)j);
895 if_printf(rl->priv->ifp,
896 "kproc_kthread_add failed: %d\n", error);
897 rlw->worker_done = 1;
907 mlx5e_rl_close_workers(struct mlx5e_priv *priv)
909 struct mlx5e_rl_priv_data *rl = &priv->rl;
915 /* tear down worker threads simultaneously */
916 for (y = 0; y < rl->param.tx_worker_threads_def; y++) {
917 struct mlx5e_rl_worker *rlw = rl->workers + y;
919 /* tear down worker before freeing SQs */
920 MLX5E_RL_WORKER_LOCK(rlw);
921 if (rlw->worker_done == 0) {
922 rlw->worker_done = 1;
923 cv_broadcast(&rlw->cv);
925 /* XXX thread not started */
926 rlw->worker_done = 0;
928 MLX5E_RL_WORKER_UNLOCK(rlw);
931 /* wait for worker threads to exit */
932 for (y = 0; y < rl->param.tx_worker_threads_def; y++) {
933 struct mlx5e_rl_worker *rlw = rl->workers + y;
935 /* tear down worker before freeing SQs */
936 MLX5E_RL_WORKER_LOCK(rlw);
937 while (rlw->worker_done != 0)
938 cv_wait(&rlw->cv, &rlw->mtx);
939 MLX5E_RL_WORKER_UNLOCK(rlw);
946 mlx5e_rl_reset_rates(struct mlx5e_rl_priv_data *rl)
951 for (x = 0; x != rl->param.tx_rates_def; x++)
952 rl->rate_limit_table[x] = 0;
953 MLX5E_RL_WUNLOCK(rl);
957 mlx5e_rl_cleanup(struct mlx5e_priv *priv)
959 struct mlx5e_rl_priv_data *rl = &priv->rl;
962 /* check if there is support for packet pacing */
963 if (!MLX5_CAP_GEN(priv->mdev, qos) || !MLX5_CAP_QOS(priv->mdev, packet_pacing))
966 /* TODO check if there is support for packet pacing */
968 sysctl_ctx_free(&rl->ctx);
971 mlx5e_rl_close_workers(priv);
974 mlx5e_rl_reset_rates(rl);
976 /* free shared UAR for SQs */
977 mlx5_unmap_free_uar(priv->mdev, &rl->sq_uar);
979 /* close TIS domain */
980 mlx5e_rl_close_tis(priv);
982 for (y = 0; y < rl->param.tx_worker_threads_def; y++) {
983 struct mlx5e_rl_worker *rlw = rl->workers + y;
985 cv_destroy(&rlw->cv);
986 mtx_destroy(&rlw->mtx);
987 free(rlw->channels, M_MLX5EN);
989 free(rl->rate_limit_table, M_MLX5EN);
990 free(rl->workers, M_MLX5EN);
991 sx_destroy(&rl->rl_sxlock);
995 mlx5e_rlw_queue_channel_locked(struct mlx5e_rl_worker *rlw,
996 struct mlx5e_rl_channel *channel)
998 STAILQ_INSERT_TAIL(&rlw->process_head, channel, entry);
999 cv_broadcast(&rlw->cv);
1003 mlx5e_rl_free(struct mlx5e_rl_worker *rlw, struct mlx5e_rl_channel *channel)
1005 if (channel == NULL)
1008 MLX5E_RL_WORKER_LOCK(rlw);
1009 switch (channel->state) {
1010 case MLX5E_RL_ST_MODIFY:
1011 channel->state = MLX5E_RL_ST_DESTROY;
1013 case MLX5E_RL_ST_USED:
1014 channel->state = MLX5E_RL_ST_DESTROY;
1015 mlx5e_rlw_queue_channel_locked(rlw, channel);
1020 MLX5E_RL_WORKER_UNLOCK(rlw);
1024 mlx5e_rl_modify(struct mlx5e_rl_worker *rlw, struct mlx5e_rl_channel *channel, uint64_t rate)
1027 MLX5E_RL_WORKER_LOCK(rlw);
1028 channel->new_rate = rate;
1029 switch (channel->state) {
1030 case MLX5E_RL_ST_USED:
1031 channel->state = MLX5E_RL_ST_MODIFY;
1032 mlx5e_rlw_queue_channel_locked(rlw, channel);
1037 MLX5E_RL_WORKER_UNLOCK(rlw);
1043 mlx5e_rl_query(struct mlx5e_rl_worker *rlw, struct mlx5e_rl_channel *channel, uint64_t *prate)
1047 MLX5E_RL_WORKER_LOCK(rlw);
1048 switch (channel->state) {
1049 case MLX5E_RL_ST_USED:
1050 *prate = channel->last_rate;
1053 case MLX5E_RL_ST_MODIFY:
1060 MLX5E_RL_WORKER_UNLOCK(rlw);
1066 mlx5e_find_available_tx_ring_index(struct mlx5e_rl_worker *rlw,
1067 struct mlx5e_rl_channel **pchannel)
1069 struct mlx5e_rl_channel *channel;
1070 int retval = ENOMEM;
1072 MLX5E_RL_WORKER_LOCK(rlw);
1073 /* Check for available channel in free list */
1074 if ((channel = STAILQ_FIRST(&rlw->index_list_head)) != NULL) {
1076 /* Remove head index from available list */
1077 STAILQ_REMOVE_HEAD(&rlw->index_list_head, entry);
1078 channel->state = MLX5E_RL_ST_USED;
1079 atomic_add_64(&rlw->priv->rl.stats.tx_active_connections, 1ULL);
1081 atomic_add_64(&rlw->priv->rl.stats.tx_available_resource_failure, 1ULL);
1083 MLX5E_RL_WORKER_UNLOCK(rlw);
1085 *pchannel = channel;
1086 #ifdef RATELIMIT_DEBUG
1087 if_printf(rlw->priv->ifp, "Channel pointer for rate limit connection is %p\n", channel);
1093 mlx5e_rl_snd_tag_alloc(struct ifnet *ifp,
1094 union if_snd_tag_alloc_params *params,
1095 struct m_snd_tag **ppmt)
1097 struct mlx5e_rl_channel *channel;
1098 struct mlx5e_rl_worker *rlw;
1099 struct mlx5e_priv *priv;
1102 priv = ifp->if_softc;
1104 /* check if there is support for packet pacing or if device is going away */
1105 if (!MLX5_CAP_GEN(priv->mdev, qos) ||
1106 !MLX5_CAP_QOS(priv->mdev, packet_pacing) || priv->gone ||
1107 params->rate_limit.hdr.type != IF_SND_TAG_TYPE_RATE_LIMIT)
1108 return (EOPNOTSUPP);
1110 /* compute worker thread this TCP connection belongs to */
1111 rlw = priv->rl.workers + ((params->rate_limit.hdr.flowid % 128) %
1112 priv->rl.param.tx_worker_threads_def);
1114 error = mlx5e_find_available_tx_ring_index(rlw, &channel);
1118 error = mlx5e_rl_modify(rlw, channel, params->rate_limit.max_rate);
1120 mlx5e_rl_free(rlw, channel);
1124 /* store pointer to mbuf tag */
1125 *ppmt = &channel->m_snd_tag;
1132 mlx5e_rl_snd_tag_modify(struct m_snd_tag *pmt, union if_snd_tag_modify_params *params)
1134 struct mlx5e_rl_channel *channel =
1135 container_of(pmt, struct mlx5e_rl_channel, m_snd_tag);
1137 return (mlx5e_rl_modify(channel->worker, channel, params->rate_limit.max_rate));
1141 mlx5e_rl_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *params)
1143 struct mlx5e_rl_channel *channel =
1144 container_of(pmt, struct mlx5e_rl_channel, m_snd_tag);
1146 return (mlx5e_rl_query(channel->worker, channel, ¶ms->rate_limit.max_rate));
1150 mlx5e_rl_snd_tag_free(struct m_snd_tag *pmt)
1152 struct mlx5e_rl_channel *channel =
1153 container_of(pmt, struct mlx5e_rl_channel, m_snd_tag);
1155 mlx5e_rl_free(channel->worker, channel);
1159 mlx5e_rl_sysctl_show_rate_table(SYSCTL_HANDLER_ARGS)
1161 struct mlx5e_rl_priv_data *rl = arg1;
1162 struct mlx5e_priv *priv = rl->priv;
1167 error = sysctl_wire_old_buffer(req, 0);
1173 sbuf_new_for_sysctl(&sbuf, NULL, 128 * rl->param.tx_rates_def, req);
1176 "\n\n" "\t" "ENTRY" "\t" "BURST" "\t" "RATE [bit/s]\n"
1177 "\t" "--------------------------------------------\n");
1180 for (x = 0; x != rl->param.tx_rates_def; x++) {
1181 if (rl->rate_limit_table[x] == 0)
1184 sbuf_printf(&sbuf, "\t" "%3u" "\t" "%3u" "\t" "%lld\n",
1185 x, (unsigned)rl->param.tx_burst_size,
1186 (long long)rl->rate_limit_table[x]);
1188 MLX5E_RL_RUNLOCK(rl);
1190 error = sbuf_finish(&sbuf);
1199 mlx5e_rl_refresh_channel_params(struct mlx5e_rl_priv_data *rl)
1205 /* compute channel parameters once */
1206 mlx5e_rl_build_channel_param(rl, &rl->chan_param);
1207 MLX5E_RL_WUNLOCK(rl);
1209 for (y = 0; y != rl->param.tx_worker_threads_def; y++) {
1210 struct mlx5e_rl_worker *rlw = rl->workers + y;
1212 for (x = 0; x != rl->param.tx_channels_per_worker_def; x++) {
1213 struct mlx5e_rl_channel *channel;
1214 struct mlx5e_sq *sq;
1216 channel = rlw->channels + x;
1222 if (MLX5_CAP_GEN(rl->priv->mdev, cq_period_mode_modify)) {
1223 mlx5_core_modify_cq_moderation_mode(rl->priv->mdev, &sq->cq.mcq,
1224 rl->param.tx_coalesce_usecs,
1225 rl->param.tx_coalesce_pkts,
1226 rl->param.tx_coalesce_mode);
1228 mlx5_core_modify_cq_moderation(rl->priv->mdev, &sq->cq.mcq,
1229 rl->param.tx_coalesce_usecs,
1230 rl->param.tx_coalesce_pkts);
1238 mlx5e_rl_tx_limit_add(struct mlx5e_rl_priv_data *rl, uint64_t value)
1244 mlx5_rl_is_in_range(rl->priv->mdev, howmany(value, 1000), 0) == 0)
1250 /* check if rate already exists */
1251 for (x = 0; x != rl->param.tx_rates_def; x++) {
1252 if (rl->rate_limit_table[x] != value)
1258 /* check if there is a free rate entry */
1259 if (x == rl->param.tx_rates_def) {
1260 for (x = 0; x != rl->param.tx_rates_def; x++) {
1261 if (rl->rate_limit_table[x] != 0)
1263 rl->rate_limit_table[x] = value;
1268 MLX5E_RL_WUNLOCK(rl);
1274 mlx5e_rl_tx_limit_clr(struct mlx5e_rl_priv_data *rl, uint64_t value)
1284 /* check if rate already exists */
1285 for (x = 0; x != rl->param.tx_rates_def; x++) {
1286 if (rl->rate_limit_table[x] != value)
1289 rl->rate_limit_table[x] = 0;
1293 /* check if there is a free rate entry */
1294 if (x == rl->param.tx_rates_def)
1298 MLX5E_RL_WUNLOCK(rl);
1304 mlx5e_rl_sysctl_handler(SYSCTL_HANDLER_ARGS)
1306 struct mlx5e_rl_priv_data *rl = arg1;
1307 struct mlx5e_priv *priv = rl->priv;
1308 unsigned mode_modify;
1309 unsigned was_opened;
1317 value = rl->param.arg[arg2];
1318 MLX5E_RL_RUNLOCK(rl);
1322 error = sysctl_handle_64(oidp, &value, 0, req);
1323 if (error || req->newptr == NULL ||
1324 value == rl->param.arg[arg2])
1331 /* check if device is gone */
1336 was_opened = rl->opened;
1337 mode_modify = MLX5_CAP_GEN(priv->mdev, cq_period_mode_modify);
1339 switch (MLX5E_RL_PARAMS_INDEX(arg[arg2])) {
1340 case MLX5E_RL_PARAMS_INDEX(tx_worker_threads_def):
1341 if (value > rl->param.tx_worker_threads_max)
1342 value = rl->param.tx_worker_threads_max;
1346 /* store new value */
1347 rl->param.arg[arg2] = value;
1350 case MLX5E_RL_PARAMS_INDEX(tx_channels_per_worker_def):
1351 if (value > rl->param.tx_channels_per_worker_max)
1352 value = rl->param.tx_channels_per_worker_max;
1356 /* store new value */
1357 rl->param.arg[arg2] = value;
1360 case MLX5E_RL_PARAMS_INDEX(tx_rates_def):
1361 if (value > rl->param.tx_rates_max)
1362 value = rl->param.tx_rates_max;
1366 /* store new value */
1367 rl->param.arg[arg2] = value;
1370 case MLX5E_RL_PARAMS_INDEX(tx_coalesce_usecs):
1374 else if (value > MLX5E_FLD_MAX(cqc, cq_period))
1375 value = MLX5E_FLD_MAX(cqc, cq_period);
1377 /* store new value */
1378 rl->param.arg[arg2] = value;
1380 /* check to avoid down and up the network interface */
1382 error = mlx5e_rl_refresh_channel_params(rl);
1385 case MLX5E_RL_PARAMS_INDEX(tx_coalesce_pkts):
1386 /* import TX coal pkts */
1389 else if (value > MLX5E_FLD_MAX(cqc, cq_max_count))
1390 value = MLX5E_FLD_MAX(cqc, cq_max_count);
1392 /* store new value */
1393 rl->param.arg[arg2] = value;
1395 /* check to avoid down and up the network interface */
1397 error = mlx5e_rl_refresh_channel_params(rl);
1400 case MLX5E_RL_PARAMS_INDEX(tx_coalesce_mode):
1401 /* network interface must be down */
1402 if (was_opened != 0 && mode_modify == 0)
1403 mlx5e_rl_close_workers(priv);
1405 /* import TX coalesce mode */
1409 /* store new value */
1410 rl->param.arg[arg2] = value;
1412 /* restart network interface, if any */
1413 if (was_opened != 0) {
1414 if (mode_modify == 0)
1415 mlx5e_rl_open_workers(priv);
1417 error = mlx5e_rl_refresh_channel_params(rl);
1421 case MLX5E_RL_PARAMS_INDEX(tx_queue_size):
1422 /* network interface must be down */
1424 mlx5e_rl_close_workers(priv);
1426 /* import TX queue size */
1427 if (value < (1 << MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE))
1428 value = (1 << MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE);
1429 else if (value > priv->params_ethtool.tx_queue_size_max)
1430 value = priv->params_ethtool.tx_queue_size_max;
1432 /* store actual TX queue size */
1433 value = 1ULL << order_base_2(value);
1435 /* store new value */
1436 rl->param.arg[arg2] = value;
1438 /* verify TX completion factor */
1439 mlx5e_rl_sync_tx_completion_fact(rl);
1441 /* restart network interface, if any */
1443 mlx5e_rl_open_workers(priv);
1446 case MLX5E_RL_PARAMS_INDEX(tx_completion_fact):
1447 /* network interface must be down */
1449 mlx5e_rl_close_workers(priv);
1451 /* store new value */
1452 rl->param.arg[arg2] = value;
1454 /* verify parameter */
1455 mlx5e_rl_sync_tx_completion_fact(rl);
1457 /* restart network interface, if any */
1459 mlx5e_rl_open_workers(priv);
1462 case MLX5E_RL_PARAMS_INDEX(tx_limit_add):
1463 error = mlx5e_rl_tx_limit_add(rl, value);
1466 case MLX5E_RL_PARAMS_INDEX(tx_limit_clr):
1467 error = mlx5e_rl_tx_limit_clr(rl, value);
1470 case MLX5E_RL_PARAMS_INDEX(tx_allowed_deviation):
1472 if (value > rl->param.tx_allowed_deviation_max)
1473 value = rl->param.tx_allowed_deviation_max;
1474 else if (value < rl->param.tx_allowed_deviation_min)
1475 value = rl->param.tx_allowed_deviation_min;
1478 rl->param.arg[arg2] = value;
1479 MLX5E_RL_WUNLOCK(rl);
1482 case MLX5E_RL_PARAMS_INDEX(tx_burst_size):
1484 if (value > rl->param.tx_burst_size_max)
1485 value = rl->param.tx_burst_size_max;
1486 else if (value < rl->param.tx_burst_size_min)
1487 value = rl->param.tx_burst_size_min;
1490 rl->param.arg[arg2] = value;
1491 MLX5E_RL_WUNLOCK(rl);
1503 mlx5e_rl_sysctl_add_u64_oid(struct mlx5e_rl_priv_data *rl, unsigned x,
1504 struct sysctl_oid *node, const char *name, const char *desc)
1507 * NOTE: In FreeBSD-11 and newer the CTLFLAG_RWTUN flag will
1508 * take care of loading default sysctl value from the kernel
1509 * environment, if any:
1511 if (strstr(name, "_max") != 0 || strstr(name, "_min") != 0) {
1512 /* read-only SYSCTLs */
1513 SYSCTL_ADD_PROC(&rl->ctx, SYSCTL_CHILDREN(node), OID_AUTO,
1514 name, CTLTYPE_U64 | CTLFLAG_RD |
1515 CTLFLAG_MPSAFE, rl, x, &mlx5e_rl_sysctl_handler, "QU", desc);
1517 if (strstr(name, "_def") != 0) {
1518 #ifdef RATELIMIT_DEBUG
1519 /* tunable read-only advanced SYSCTLs */
1520 SYSCTL_ADD_PROC(&rl->ctx, SYSCTL_CHILDREN(node), OID_AUTO,
1521 name, CTLTYPE_U64 | CTLFLAG_RDTUN |
1522 CTLFLAG_MPSAFE, rl, x, &mlx5e_rl_sysctl_handler, "QU", desc);
1525 /* read-write SYSCTLs */
1526 SYSCTL_ADD_PROC(&rl->ctx, SYSCTL_CHILDREN(node), OID_AUTO,
1527 name, CTLTYPE_U64 | CTLFLAG_RWTUN |
1528 CTLFLAG_MPSAFE, rl, x, &mlx5e_rl_sysctl_handler, "QU", desc);
1534 mlx5e_rl_sysctl_add_stats_u64_oid(struct mlx5e_rl_priv_data *rl, unsigned x,
1535 struct sysctl_oid *node, const char *name, const char *desc)
1537 /* read-only SYSCTLs */
1538 SYSCTL_ADD_U64(&rl->ctx, SYSCTL_CHILDREN(node), OID_AUTO, name,
1539 CTLFLAG_RD, &rl->stats.arg[x], 0, desc);