2 * Copyright (c) 2017 Chelsio Communications, Inc.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
32 #include "opt_inet6.h"
33 #include "opt_ratelimit.h"
35 #include <sys/types.h>
36 #include <sys/malloc.h>
37 #include <sys/queue.h>
39 #include <sys/taskqueue.h>
40 #include <sys/sysctl.h>
42 #include "common/common.h"
43 #include "common/t4_regs.h"
44 #include "common/t4_regs_values.h"
45 #include "common/t4_msg.h"
49 in_range(int val, int lo, int hi)
52 return (val < 0 || (val <= hi && val >= lo));
56 set_sched_class_config(struct adapter *sc, int minmax)
63 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4sscc");
66 rc = -t4_sched_config(sc, FW_SCHED_TYPE_PKTSCHED, minmax, 1);
67 end_synchronized_op(sc, 0);
73 set_sched_class_params(struct adapter *sc, struct t4_sched_class_params *p,
76 int rc, top_speed, fw_level, fw_mode, fw_rateunit, fw_ratemode;
78 struct tx_cl_rl_params *tc;
80 if (p->level == SCHED_CLASS_LEVEL_CL_RL)
81 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL;
82 else if (p->level == SCHED_CLASS_LEVEL_CL_WRR)
83 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR;
84 else if (p->level == SCHED_CLASS_LEVEL_CH_RL)
85 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL;
89 if (p->mode == SCHED_CLASS_MODE_CLASS)
90 fw_mode = FW_SCHED_PARAMS_MODE_CLASS;
91 else if (p->mode == SCHED_CLASS_MODE_FLOW)
92 fw_mode = FW_SCHED_PARAMS_MODE_FLOW;
96 if (p->rateunit == SCHED_CLASS_RATEUNIT_BITS)
97 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
98 else if (p->rateunit == SCHED_CLASS_RATEUNIT_PKTS)
99 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE;
103 if (p->ratemode == SCHED_CLASS_RATEMODE_REL)
104 fw_ratemode = FW_SCHED_PARAMS_RATE_REL;
105 else if (p->ratemode == SCHED_CLASS_RATEMODE_ABS)
106 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS;
110 /* Vet our parameters ... */
111 if (!in_range(p->channel, 0, sc->chip_params->nchan - 1))
114 pi = sc->port[sc->chan_map[p->channel]];
117 MPASS(pi->tx_chan == p->channel);
118 top_speed = port_top_speed(pi) * 1000000; /* Gbps -> Kbps */
120 if (!in_range(p->cl, 0, sc->chip_params->nsched_cls) ||
121 !in_range(p->minrate, 0, top_speed) ||
122 !in_range(p->maxrate, 0, top_speed) ||
123 !in_range(p->weight, 0, 100))
127 * Translate any unset parameters into the firmware's
128 * nomenclature and/or fail the call if the parameters
131 if (p->rateunit < 0 || p->ratemode < 0 || p->channel < 0 || p->cl < 0)
136 if (p->maxrate < 0) {
137 if (p->level == SCHED_CLASS_LEVEL_CL_RL ||
138 p->level == SCHED_CLASS_LEVEL_CH_RL)
144 if (p->level == SCHED_CLASS_LEVEL_CL_WRR)
149 if (p->pktsize < 0) {
150 if (p->level == SCHED_CLASS_LEVEL_CL_RL ||
151 p->level == SCHED_CLASS_LEVEL_CH_RL)
157 rc = begin_synchronized_op(sc, NULL,
158 sleep_ok ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4sscp");
161 if (p->level == SCHED_CLASS_LEVEL_CL_RL) {
162 tc = &pi->sched_params->cl_rl[p->cl];
163 if (tc->refcount > 0) {
167 tc->ratemode = fw_ratemode;
168 tc->rateunit = fw_rateunit;
170 tc->maxrate = p->maxrate;
171 tc->pktsize = p->pktsize;
174 rc = -t4_sched_params(sc, FW_SCHED_TYPE_PKTSCHED, fw_level, fw_mode,
175 fw_rateunit, fw_ratemode, p->channel, p->cl, p->minrate, p->maxrate,
176 p->weight, p->pktsize, sleep_ok);
177 if (p->level == SCHED_CLASS_LEVEL_CL_RL && rc != 0) {
179 * Unknown state at this point, see parameters in tc for what
182 tc->flags |= TX_CLRL_ERROR;
185 end_synchronized_op(sc, sleep_ok ? 0 : LOCK_HELD);
191 update_tx_sched(void *context, int pending)
193 int i, j, mode, rateunit, ratemode, maxrate, pktsize, rc;
194 struct port_info *pi;
195 struct tx_cl_rl_params *tc;
196 struct adapter *sc = context;
197 const int n = sc->chip_params->nsched_cls;
199 mtx_lock(&sc->tc_lock);
200 for_each_port(sc, i) {
202 tc = &pi->sched_params->cl_rl[0];
203 for (j = 0; j < n; j++, tc++) {
204 MPASS(mtx_owned(&sc->tc_lock));
205 if ((tc->flags & TX_CLRL_REFRESH) == 0)
209 rateunit = tc->rateunit;
210 ratemode = tc->ratemode;
211 maxrate = tc->maxrate;
212 pktsize = tc->pktsize;
213 mtx_unlock(&sc->tc_lock);
215 if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
217 mtx_lock(&sc->tc_lock);
220 rc = t4_sched_params(sc, FW_SCHED_TYPE_PKTSCHED,
221 FW_SCHED_PARAMS_LEVEL_CL_RL, mode, rateunit,
222 ratemode, pi->tx_chan, j, 0, maxrate, 0, pktsize,
224 end_synchronized_op(sc, 0);
226 mtx_lock(&sc->tc_lock);
228 tc->flags |= TX_CLRL_ERROR;
229 } else if (tc->mode == mode &&
230 tc->rateunit == rateunit &&
231 tc->maxrate == maxrate &&
232 tc->pktsize == tc->pktsize) {
233 tc->flags &= ~(TX_CLRL_REFRESH | TX_CLRL_ERROR);
237 mtx_unlock(&sc->tc_lock);
241 t4_set_sched_class(struct adapter *sc, struct t4_sched_params *p)
244 if (p->type != SCHED_CLASS_TYPE_PACKET)
247 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG)
248 return (set_sched_class_config(sc, p->u.config.minmax));
250 if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS)
251 return (set_sched_class_params(sc, &p->u.params, 1));
257 t4_set_sched_queue(struct adapter *sc, struct t4_sched_queue *p)
259 struct port_info *pi = NULL;
262 uint32_t fw_mnem, fw_queue, fw_class;
265 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq");
269 if (p->port >= sc->params.nports) {
274 /* XXX: Only supported for the main VI. */
275 pi = sc->port[p->port];
277 if (!(vi->flags & VI_INIT_DONE)) {
278 /* tx queues not set up yet */
283 if (!in_range(p->queue, 0, vi->ntxq - 1) ||
284 !in_range(p->cl, 0, sc->chip_params->nsched_cls - 1)) {
290 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX
291 * Scheduling Class in this case).
293 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
294 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH));
295 fw_class = p->cl < 0 ? 0xffffffff : p->cl;
298 * If op.queue is non-negative, then we're only changing the scheduling
299 * on a single specified TX queue.
302 txq = &sc->sge.txq[vi->first_txq + p->queue];
303 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
304 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
310 * Change the scheduling on all the TX queues for the
313 for_each_txq(vi, i, txq) {
314 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id));
315 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue,
323 end_synchronized_op(sc, 0);
328 t4_init_tx_sched(struct adapter *sc)
331 const int n = sc->chip_params->nsched_cls;
332 struct port_info *pi;
333 struct tx_cl_rl_params *tc;
334 static const uint32_t init_kbps[] = {
353 mtx_init(&sc->tc_lock, "tx_sched lock", NULL, MTX_DEF);
354 TASK_INIT(&sc->tc_task, 0, update_tx_sched, sc);
355 for_each_port(sc, i) {
357 pi->sched_params = malloc(sizeof(*pi->sched_params) +
358 n * sizeof(*tc), M_CXGBE, M_ZERO | M_WAITOK);
359 tc = &pi->sched_params->cl_rl[0];
360 for (j = 0; j < n; j++, tc++) {
362 tc->ratemode = FW_SCHED_PARAMS_RATE_ABS;
363 tc->rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
364 tc->mode = FW_SCHED_PARAMS_MODE_FLOW;
365 tc->maxrate = init_kbps[min(j, nitems(init_kbps) - 1)];
366 tc->pktsize = ETHERMTU; /* XXX */
368 if (t4_sched_params_cl_rl_kbps(sc, pi->tx_chan, j,
369 tc->mode, tc->maxrate, tc->pktsize, 1) == 0)
372 tc->flags = TX_CLRL_ERROR;
380 t4_free_tx_sched(struct adapter *sc)
384 taskqueue_drain(taskqueue_thread, &sc->tc_task);
386 for_each_port(sc, i) {
387 if (sc->port[i] != NULL)
388 free(sc->port[i]->sched_params, M_CXGBE);
391 if (mtx_initialized(&sc->tc_lock))
392 mtx_destroy(&sc->tc_lock);
398 t4_update_tx_sched(struct adapter *sc)
401 taskqueue_enqueue(taskqueue_thread, &sc->tc_task);
405 t4_reserve_cl_rl_kbps(struct adapter *sc, int port_id, u_int maxrate,
408 int rc = 0, fa = -1, i;
409 struct tx_cl_rl_params *tc;
411 MPASS(port_id >= 0 && port_id < sc->params.nports);
413 tc = &sc->port[port_id]->sched_params->cl_rl[0];
414 mtx_lock(&sc->tc_lock);
415 for (i = 0; i < sc->chip_params->nsched_cls; i++, tc++) {
416 if (fa < 0 && tc->refcount == 0)
419 if (tc->ratemode == FW_SCHED_PARAMS_RATE_ABS &&
420 tc->rateunit == FW_SCHED_PARAMS_UNIT_BITRATE &&
421 tc->mode == FW_SCHED_PARAMS_MODE_FLOW &&
422 tc->maxrate == maxrate) {
429 MPASS(i == sc->chip_params->nsched_cls);
431 tc = &sc->port[port_id]->sched_params->cl_rl[fa];
432 tc->flags = TX_CLRL_REFRESH;
434 tc->ratemode = FW_SCHED_PARAMS_RATE_ABS;
435 tc->rateunit = FW_SCHED_PARAMS_UNIT_BITRATE;
436 tc->mode = FW_SCHED_PARAMS_MODE_FLOW;
437 tc->maxrate = maxrate;
438 tc->pktsize = ETHERMTU; /* XXX */
440 t4_update_tx_sched(sc);
446 mtx_unlock(&sc->tc_lock);
451 t4_release_cl_rl_kbps(struct adapter *sc, int port_id, int tc_idx)
453 struct tx_cl_rl_params *tc;
455 MPASS(port_id >= 0 && port_id < sc->params.nports);
456 MPASS(tc_idx >= 0 && tc_idx < sc->chip_params->nsched_cls);
458 mtx_lock(&sc->tc_lock);
459 tc = &sc->port[port_id]->sched_params->cl_rl[tc_idx];
460 MPASS(tc->refcount > 0);
461 MPASS(tc->ratemode == FW_SCHED_PARAMS_RATE_ABS);
462 MPASS(tc->rateunit == FW_SCHED_PARAMS_UNIT_BITRATE);
463 MPASS(tc->mode == FW_SCHED_PARAMS_MODE_FLOW);
465 mtx_unlock(&sc->tc_lock);
470 t4_init_etid_table(struct adapter *sc)
475 if (!is_ethoffload(sc))
479 MPASS(t->netids > 0);
481 mtx_init(&t->etid_lock, "etid lock", NULL, MTX_DEF);
482 t->etid_tab = malloc(sizeof(*t->etid_tab) * t->netids, M_CXGBE,
484 t->efree = t->etid_tab;
486 for (i = 1; i < t->netids; i++)
487 t->etid_tab[i - 1].next = &t->etid_tab[i];
488 t->etid_tab[t->netids - 1].next = NULL;
492 t4_free_etid_table(struct adapter *sc)
496 if (!is_ethoffload(sc))
500 MPASS(t->netids > 0);
502 free(t->etid_tab, M_CXGBE);
505 if (mtx_initialized(&t->etid_lock))
506 mtx_destroy(&t->etid_lock);
510 static int alloc_etid(struct adapter *, struct cxgbe_snd_tag *);
511 static void free_etid(struct adapter *, int);
514 alloc_etid(struct adapter *sc, struct cxgbe_snd_tag *cst)
516 struct tid_info *t = &sc->tids;
519 mtx_lock(&t->etid_lock);
521 union etid_entry *p = t->efree;
523 etid = p - t->etid_tab + t->etid_base;
528 mtx_unlock(&t->etid_lock);
533 struct cxgbe_snd_tag *
534 lookup_etid(struct adapter *sc, int etid)
536 struct tid_info *t = &sc->tids;
538 return (t->etid_tab[etid - t->etid_base].cst);
543 free_etid(struct adapter *sc, int etid)
545 struct tid_info *t = &sc->tids;
546 union etid_entry *p = &t->etid_tab[etid - t->etid_base];
548 mtx_lock(&t->etid_lock);
552 mtx_unlock(&t->etid_lock);
556 cxgbe_snd_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
557 struct m_snd_tag **pt)
560 struct vi_info *vi = ifp->if_softc;
561 struct port_info *pi = vi->pi;
562 struct adapter *sc = pi->adapter;
563 struct cxgbe_snd_tag *cst;
565 if (params->hdr.type != IF_SND_TAG_TYPE_RATE_LIMIT)
568 rc = t4_reserve_cl_rl_kbps(sc, pi->port_id,
569 (params->rate_limit.max_rate * 8ULL / 1000), &schedcl);
572 MPASS(schedcl >= 0 && schedcl < sc->chip_params->nsched_cls);
574 cst = malloc(sizeof(*cst), M_CXGBE, M_ZERO | M_NOWAIT);
577 t4_release_cl_rl_kbps(sc, pi->port_id, schedcl);
581 cst->etid = alloc_etid(sc, cst);
587 mtx_init(&cst->lock, "cst_lock", NULL, MTX_DEF);
590 cst->port_id = pi->port_id;
591 cst->schedcl = schedcl;
592 cst->max_rate = params->rate_limit.max_rate;
593 cst->next_credits = -1;
594 cst->tx_credits = sc->params.ofldq_wr_cred;
595 cst->tx_total = cst->tx_credits;
598 * Queues will be selected later when the connection flowid is available.
606 * Change in parameters, no change in ifp.
609 cxgbe_snd_tag_modify(struct m_snd_tag *mst,
610 union if_snd_tag_modify_params *params)
613 struct cxgbe_snd_tag *cst = mst_to_cst(mst);
614 struct adapter *sc = cst->adapter;
616 /* XXX: is schedcl -1 ok here? */
617 MPASS(cst->schedcl >= 0 && cst->schedcl < sc->chip_params->nsched_cls);
619 rc = t4_reserve_cl_rl_kbps(sc, cst->port_id,
620 (params->rate_limit.max_rate * 8ULL / 1000), &schedcl);
623 MPASS(schedcl >= 0 && schedcl < sc->chip_params->nsched_cls);
624 t4_release_cl_rl_kbps(sc, cst->port_id, cst->schedcl);
625 cst->schedcl = schedcl;
626 cst->max_rate = params->rate_limit.max_rate;
632 cxgbe_snd_tag_query(struct m_snd_tag *mst,
633 union if_snd_tag_query_params *params)
635 struct cxgbe_snd_tag *cst = mst_to_cst(mst);
637 params->rate_limit.max_rate = cst->max_rate;
639 #define CST_TO_MST_QLEVEL_SCALE (IF_SND_QUEUE_LEVEL_MAX / cst->tx_total)
640 params->rate_limit.queue_level =
641 (cst->tx_total - cst->tx_credits) * CST_TO_MST_QLEVEL_SCALE;
647 cxgbe_snd_tag_free(struct m_snd_tag *mst)
649 struct cxgbe_snd_tag *cst = mst_to_cst(mst);
650 struct adapter *sc = cst->adapter;
653 free_etid(sc, cst->etid);
654 if (cst->schedcl != -1)
655 t4_release_cl_rl_kbps(sc, cst->port_id, cst->schedcl);
656 if (mtx_initialized(&cst->lock))
657 mtx_destroy(&cst->lock);