2 * Copyright (c) 2015 Mellanox Technologies. All rights reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 #include <sys/sockio.h>
31 #include <machine/atomic.h>
33 #define ETH_DRIVER_VERSION "3.1.0-dev"
34 char mlx5e_version[] = "Mellanox Ethernet driver"
35 " (" ETH_DRIVER_VERSION ")";
37 struct mlx5e_channel_param {
38 struct mlx5e_rq_param rq;
39 struct mlx5e_sq_param sq;
40 struct mlx5e_cq_param rx_cq;
41 struct mlx5e_cq_param tx_cq;
47 } mlx5e_mode_table[MLX5E_LINK_MODES_NUMBER] = {
49 [MLX5E_1000BASE_CX_SGMII] = {
50 .subtype = IFM_1000_CX_SGMII,
51 .baudrate = IF_Mbps(1000ULL),
53 [MLX5E_1000BASE_KX] = {
54 .subtype = IFM_1000_KX,
55 .baudrate = IF_Mbps(1000ULL),
57 [MLX5E_10GBASE_CX4] = {
58 .subtype = IFM_10G_CX4,
59 .baudrate = IF_Gbps(10ULL),
61 [MLX5E_10GBASE_KX4] = {
62 .subtype = IFM_10G_KX4,
63 .baudrate = IF_Gbps(10ULL),
65 [MLX5E_10GBASE_KR] = {
66 .subtype = IFM_10G_KR,
67 .baudrate = IF_Gbps(10ULL),
69 [MLX5E_20GBASE_KR2] = {
70 .subtype = IFM_20G_KR2,
71 .baudrate = IF_Gbps(20ULL),
73 [MLX5E_40GBASE_CR4] = {
74 .subtype = IFM_40G_CR4,
75 .baudrate = IF_Gbps(40ULL),
77 [MLX5E_40GBASE_KR4] = {
78 .subtype = IFM_40G_KR4,
79 .baudrate = IF_Gbps(40ULL),
81 [MLX5E_56GBASE_R4] = {
82 .subtype = IFM_56G_R4,
83 .baudrate = IF_Gbps(56ULL),
85 [MLX5E_10GBASE_CR] = {
86 .subtype = IFM_10G_CR1,
87 .baudrate = IF_Gbps(10ULL),
89 [MLX5E_10GBASE_SR] = {
90 .subtype = IFM_10G_SR,
91 .baudrate = IF_Gbps(10ULL),
93 [MLX5E_10GBASE_LR] = {
94 .subtype = IFM_10G_LR,
95 .baudrate = IF_Gbps(10ULL),
97 [MLX5E_40GBASE_SR4] = {
98 .subtype = IFM_40G_SR4,
99 .baudrate = IF_Gbps(40ULL),
101 [MLX5E_40GBASE_LR4] = {
102 .subtype = IFM_40G_LR4,
103 .baudrate = IF_Gbps(40ULL),
105 [MLX5E_100GBASE_CR4] = {
106 .subtype = IFM_100G_CR4,
107 .baudrate = IF_Gbps(100ULL),
109 [MLX5E_100GBASE_SR4] = {
110 .subtype = IFM_100G_SR4,
111 .baudrate = IF_Gbps(100ULL),
113 [MLX5E_100GBASE_KR4] = {
114 .subtype = IFM_100G_KR4,
115 .baudrate = IF_Gbps(100ULL),
117 [MLX5E_100GBASE_LR4] = {
118 .subtype = IFM_100G_LR4,
119 .baudrate = IF_Gbps(100ULL),
121 [MLX5E_100BASE_TX] = {
122 .subtype = IFM_100_TX,
123 .baudrate = IF_Mbps(100ULL),
125 [MLX5E_100BASE_T] = {
126 .subtype = IFM_100_T,
127 .baudrate = IF_Mbps(100ULL),
129 [MLX5E_10GBASE_T] = {
130 .subtype = IFM_10G_T,
131 .baudrate = IF_Gbps(10ULL),
133 [MLX5E_25GBASE_CR] = {
134 .subtype = IFM_25G_CR,
135 .baudrate = IF_Gbps(25ULL),
137 [MLX5E_25GBASE_KR] = {
138 .subtype = IFM_25G_KR,
139 .baudrate = IF_Gbps(25ULL),
141 [MLX5E_25GBASE_SR] = {
142 .subtype = IFM_25G_SR,
143 .baudrate = IF_Gbps(25ULL),
145 [MLX5E_50GBASE_CR2] = {
146 .subtype = IFM_50G_CR2,
147 .baudrate = IF_Gbps(50ULL),
149 [MLX5E_50GBASE_KR2] = {
150 .subtype = IFM_50G_KR2,
151 .baudrate = IF_Gbps(50ULL),
155 MALLOC_DEFINE(M_MLX5EN, "MLX5EN", "MLX5 Ethernet");
158 mlx5e_update_carrier(struct mlx5e_priv *priv)
160 struct mlx5_core_dev *mdev = priv->mdev;
161 u32 out[MLX5_ST_SZ_DW(ptys_reg)];
167 port_state = mlx5_query_vport_state(mdev,
168 MLX5_QUERY_VPORT_STATE_IN_OP_MOD_VNIC_VPORT, 0);
170 if (port_state == VPORT_STATE_UP) {
171 priv->media_status_last |= IFM_ACTIVE;
173 priv->media_status_last &= ~IFM_ACTIVE;
174 priv->media_active_last = IFM_ETHER;
175 if_link_state_change(priv->ifp, LINK_STATE_DOWN);
179 error = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN);
181 priv->media_active_last = IFM_ETHER;
182 priv->ifp->if_baudrate = 1;
183 if_printf(priv->ifp, "%s: query port ptys failed: 0x%x\n",
187 eth_proto_oper = MLX5_GET(ptys_reg, out, eth_proto_oper);
189 for (i = 0; i != MLX5E_LINK_MODES_NUMBER; i++) {
190 if (mlx5e_mode_table[i].baudrate == 0)
192 if (MLX5E_PROT_MASK(i) & eth_proto_oper) {
193 priv->ifp->if_baudrate =
194 mlx5e_mode_table[i].baudrate;
195 priv->media_active_last =
196 mlx5e_mode_table[i].subtype | IFM_ETHER | IFM_FDX;
199 if_link_state_change(priv->ifp, LINK_STATE_UP);
203 mlx5e_media_status(struct ifnet *dev, struct ifmediareq *ifmr)
205 struct mlx5e_priv *priv = dev->if_softc;
207 ifmr->ifm_status = priv->media_status_last;
208 ifmr->ifm_active = priv->media_active_last |
209 (priv->params.rx_pauseframe_control ? IFM_ETH_RXPAUSE : 0) |
210 (priv->params.tx_pauseframe_control ? IFM_ETH_TXPAUSE : 0);
215 mlx5e_find_link_mode(u32 subtype)
220 for (i = 0; i < MLX5E_LINK_MODES_NUMBER; ++i) {
221 if (mlx5e_mode_table[i].baudrate == 0)
223 if (mlx5e_mode_table[i].subtype == subtype)
224 link_mode |= MLX5E_PROT_MASK(i);
231 mlx5e_media_change(struct ifnet *dev)
233 struct mlx5e_priv *priv = dev->if_softc;
234 struct mlx5_core_dev *mdev = priv->mdev;
241 locked = PRIV_LOCKED(priv);
245 if (IFM_TYPE(priv->media.ifm_media) != IFM_ETHER) {
249 link_mode = mlx5e_find_link_mode(IFM_SUBTYPE(priv->media.ifm_media));
251 /* query supported capabilities */
252 error = mlx5_query_port_proto_cap(mdev, ð_proto_cap, MLX5_PTYS_EN);
254 if_printf(dev, "Query port media capability failed\n");
257 /* check for autoselect */
258 if (IFM_SUBTYPE(priv->media.ifm_media) == IFM_AUTO) {
259 link_mode = eth_proto_cap;
260 if (link_mode == 0) {
261 if_printf(dev, "Port media capability is zero\n");
266 link_mode = link_mode & eth_proto_cap;
267 if (link_mode == 0) {
268 if_printf(dev, "Not supported link mode requested\n");
273 /* update pauseframe control bits */
274 priv->params.rx_pauseframe_control =
275 (priv->media.ifm_media & IFM_ETH_RXPAUSE) ? 1 : 0;
276 priv->params.tx_pauseframe_control =
277 (priv->media.ifm_media & IFM_ETH_TXPAUSE) ? 1 : 0;
279 /* check if device is opened */
280 was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
282 /* reconfigure the hardware */
283 mlx5_set_port_status(mdev, MLX5_PORT_DOWN);
284 mlx5_set_port_proto(mdev, link_mode, MLX5_PTYS_EN);
285 mlx5_set_port_pause(mdev, 1,
286 priv->params.rx_pauseframe_control,
287 priv->params.tx_pauseframe_control);
289 mlx5_set_port_status(mdev, MLX5_PORT_UP);
298 mlx5e_update_carrier_work(struct work_struct *work)
300 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
301 update_carrier_work);
304 if (test_bit(MLX5E_STATE_OPENED, &priv->state))
305 mlx5e_update_carrier(priv);
310 * This function reads the physical port counters from the firmware
311 * using a pre-defined layout defined by various MLX5E_PPORT_XXX()
312 * macros. The output is converted from big-endian 64-bit values into
313 * host endian ones and stored in the "priv->stats.pport" structure.
316 mlx5e_update_pport_counters(struct mlx5e_priv *priv)
318 struct mlx5_core_dev *mdev = priv->mdev;
319 struct mlx5e_pport_stats *s = &priv->stats.pport;
320 struct mlx5e_port_stats_debug *s_debug = &priv->stats.port_stats_debug;
324 unsigned sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
328 /* allocate firmware request structures */
329 in = mlx5_vzalloc(sz);
330 out = mlx5_vzalloc(sz);
331 if (in == NULL || out == NULL)
335 * Get pointer to the 64-bit counter set which is located at a
336 * fixed offset in the output firmware request structure:
338 ptr = (const uint64_t *)MLX5_ADDR_OF(ppcnt_reg, out, counter_set);
340 MLX5_SET(ppcnt_reg, in, local_port, 1);
342 /* read IEEE802_3 counter group using predefined counter layout */
343 MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
344 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
345 for (x = y = 0; x != MLX5E_PPORT_IEEE802_3_STATS_NUM; x++, y++)
346 s->arg[y] = be64toh(ptr[x]);
348 /* read RFC2819 counter group using predefined counter layout */
349 MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2819_COUNTERS_GROUP);
350 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
351 for (x = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM; x++, y++)
352 s->arg[y] = be64toh(ptr[x]);
353 for (y = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM +
354 MLX5E_PPORT_RFC2819_STATS_DEBUG_NUM; x++, y++)
355 s_debug->arg[y] = be64toh(ptr[x]);
357 /* read RFC2863 counter group using predefined counter layout */
358 MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2863_COUNTERS_GROUP);
359 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
360 for (x = 0; x != MLX5E_PPORT_RFC2863_STATS_DEBUG_NUM; x++, y++)
361 s_debug->arg[y] = be64toh(ptr[x]);
363 /* read physical layer stats counter group using predefined counter layout */
364 MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP);
365 mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
366 for (x = 0; x != MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG_NUM; x++, y++)
367 s_debug->arg[y] = be64toh(ptr[x]);
369 /* free firmware request structures */
375 * This function is called regularly to collect all statistics
376 * counters from the firmware. The values can be viewed through the
377 * sysctl interface. Execution is serialized using the priv's global
378 * configuration lock.
381 mlx5e_update_stats_work(struct work_struct *work)
383 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
385 struct mlx5_core_dev *mdev = priv->mdev;
386 struct mlx5e_vport_stats *s = &priv->stats.vport;
387 struct mlx5e_rq_stats *rq_stats;
388 struct mlx5e_sq_stats *sq_stats;
389 struct buf_ring *sq_br;
390 #if (__FreeBSD_version < 1100000)
391 struct ifnet *ifp = priv->ifp;
394 u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)];
396 int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
399 u64 tx_queue_dropped = 0;
400 u64 tx_defragged = 0;
401 u64 tx_offload_none = 0;
404 u64 sw_lro_queued = 0;
405 u64 sw_lro_flushed = 0;
406 u64 rx_csum_none = 0;
408 u32 rx_out_of_buffer = 0;
413 out = mlx5_vzalloc(outlen);
416 if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
419 /* Collect firts the SW counters and then HW for consistency */
420 for (i = 0; i < priv->params.num_channels; i++) {
421 struct mlx5e_rq *rq = &priv->channel[i]->rq;
423 rq_stats = &priv->channel[i]->rq.stats;
425 /* collect stats from LRO */
426 rq_stats->sw_lro_queued = rq->lro.lro_queued;
427 rq_stats->sw_lro_flushed = rq->lro.lro_flushed;
428 sw_lro_queued += rq_stats->sw_lro_queued;
429 sw_lro_flushed += rq_stats->sw_lro_flushed;
430 lro_packets += rq_stats->lro_packets;
431 lro_bytes += rq_stats->lro_bytes;
432 rx_csum_none += rq_stats->csum_none;
433 rx_wqe_err += rq_stats->wqe_err;
435 for (j = 0; j < priv->num_tc; j++) {
436 sq_stats = &priv->channel[i]->sq[j].stats;
437 sq_br = priv->channel[i]->sq[j].br;
439 tso_packets += sq_stats->tso_packets;
440 tso_bytes += sq_stats->tso_bytes;
441 tx_queue_dropped += sq_stats->dropped;
442 tx_queue_dropped += sq_br->br_drops;
443 tx_defragged += sq_stats->defragged;
444 tx_offload_none += sq_stats->csum_offload_none;
448 /* update counters */
449 s->tso_packets = tso_packets;
450 s->tso_bytes = tso_bytes;
451 s->tx_queue_dropped = tx_queue_dropped;
452 s->tx_defragged = tx_defragged;
453 s->lro_packets = lro_packets;
454 s->lro_bytes = lro_bytes;
455 s->sw_lro_queued = sw_lro_queued;
456 s->sw_lro_flushed = sw_lro_flushed;
457 s->rx_csum_none = rx_csum_none;
458 s->rx_wqe_err = rx_wqe_err;
461 memset(in, 0, sizeof(in));
463 MLX5_SET(query_vport_counter_in, in, opcode,
464 MLX5_CMD_OP_QUERY_VPORT_COUNTER);
465 MLX5_SET(query_vport_counter_in, in, op_mod, 0);
466 MLX5_SET(query_vport_counter_in, in, other_vport, 0);
468 memset(out, 0, outlen);
470 /* get number of out-of-buffer drops first */
471 if (mlx5_vport_query_out_of_rx_buffer(mdev, priv->counter_set_id,
475 /* accumulate difference into a 64-bit counter */
476 s->rx_out_of_buffer += (u64)(u32)(rx_out_of_buffer - s->rx_out_of_buffer_prev);
477 s->rx_out_of_buffer_prev = rx_out_of_buffer;
479 /* get port statistics */
480 if (mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen))
483 #define MLX5_GET_CTR(out, x) \
484 MLX5_GET64(query_vport_counter_out, out, x)
486 s->rx_error_packets =
487 MLX5_GET_CTR(out, received_errors.packets);
489 MLX5_GET_CTR(out, received_errors.octets);
490 s->tx_error_packets =
491 MLX5_GET_CTR(out, transmit_errors.packets);
493 MLX5_GET_CTR(out, transmit_errors.octets);
495 s->rx_unicast_packets =
496 MLX5_GET_CTR(out, received_eth_unicast.packets);
497 s->rx_unicast_bytes =
498 MLX5_GET_CTR(out, received_eth_unicast.octets);
499 s->tx_unicast_packets =
500 MLX5_GET_CTR(out, transmitted_eth_unicast.packets);
501 s->tx_unicast_bytes =
502 MLX5_GET_CTR(out, transmitted_eth_unicast.octets);
504 s->rx_multicast_packets =
505 MLX5_GET_CTR(out, received_eth_multicast.packets);
506 s->rx_multicast_bytes =
507 MLX5_GET_CTR(out, received_eth_multicast.octets);
508 s->tx_multicast_packets =
509 MLX5_GET_CTR(out, transmitted_eth_multicast.packets);
510 s->tx_multicast_bytes =
511 MLX5_GET_CTR(out, transmitted_eth_multicast.octets);
513 s->rx_broadcast_packets =
514 MLX5_GET_CTR(out, received_eth_broadcast.packets);
515 s->rx_broadcast_bytes =
516 MLX5_GET_CTR(out, received_eth_broadcast.octets);
517 s->tx_broadcast_packets =
518 MLX5_GET_CTR(out, transmitted_eth_broadcast.packets);
519 s->tx_broadcast_bytes =
520 MLX5_GET_CTR(out, transmitted_eth_broadcast.octets);
523 s->rx_unicast_packets +
524 s->rx_multicast_packets +
525 s->rx_broadcast_packets -
528 s->rx_unicast_bytes +
529 s->rx_multicast_bytes +
530 s->rx_broadcast_bytes;
532 s->tx_unicast_packets +
533 s->tx_multicast_packets +
534 s->tx_broadcast_packets;
536 s->tx_unicast_bytes +
537 s->tx_multicast_bytes +
538 s->tx_broadcast_bytes;
540 /* Update calculated offload counters */
541 s->tx_csum_offload = s->tx_packets - tx_offload_none;
542 s->rx_csum_good = s->rx_packets - s->rx_csum_none;
544 /* Update per port counters */
545 mlx5e_update_pport_counters(priv);
547 #if (__FreeBSD_version < 1100000)
548 /* no get_counters interface in fbsd 10 */
549 ifp->if_ipackets = s->rx_packets;
550 ifp->if_ierrors = s->rx_error_packets;
551 ifp->if_iqdrops = s->rx_out_of_buffer;
552 ifp->if_opackets = s->tx_packets;
553 ifp->if_oerrors = s->tx_error_packets;
554 ifp->if_snd.ifq_drops = s->tx_queue_dropped;
555 ifp->if_ibytes = s->rx_bytes;
556 ifp->if_obytes = s->tx_bytes;
565 mlx5e_update_stats(void *arg)
567 struct mlx5e_priv *priv = arg;
569 schedule_work(&priv->update_stats_work);
571 callout_reset(&priv->watchdog, hz, &mlx5e_update_stats, priv);
575 mlx5e_async_event_sub(struct mlx5e_priv *priv,
576 enum mlx5_dev_event event)
579 case MLX5_DEV_EVENT_PORT_UP:
580 case MLX5_DEV_EVENT_PORT_DOWN:
581 schedule_work(&priv->update_carrier_work);
590 mlx5e_async_event(struct mlx5_core_dev *mdev, void *vpriv,
591 enum mlx5_dev_event event, unsigned long param)
593 struct mlx5e_priv *priv = vpriv;
595 mtx_lock(&priv->async_events_mtx);
596 if (test_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state))
597 mlx5e_async_event_sub(priv, event);
598 mtx_unlock(&priv->async_events_mtx);
602 mlx5e_enable_async_events(struct mlx5e_priv *priv)
604 set_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state);
608 mlx5e_disable_async_events(struct mlx5e_priv *priv)
610 mtx_lock(&priv->async_events_mtx);
611 clear_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state);
612 mtx_unlock(&priv->async_events_mtx);
615 static const char *mlx5e_rq_stats_desc[] = {
616 MLX5E_RQ_STATS(MLX5E_STATS_DESC)
620 mlx5e_create_rq(struct mlx5e_channel *c,
621 struct mlx5e_rq_param *param,
624 struct mlx5e_priv *priv = c->priv;
625 struct mlx5_core_dev *mdev = priv->mdev;
627 void *rqc = param->rqc;
628 void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
633 /* Create DMA descriptor TAG */
634 if ((err = -bus_dma_tag_create(
635 bus_get_dma_tag(mdev->pdev->dev.bsddev),
636 1, /* any alignment */
638 BUS_SPACE_MAXADDR, /* lowaddr */
639 BUS_SPACE_MAXADDR, /* highaddr */
640 NULL, NULL, /* filter, filterarg */
641 MJUM16BYTES, /* maxsize */
643 MJUM16BYTES, /* maxsegsize */
645 NULL, NULL, /* lockfunc, lockfuncarg */
649 err = mlx5_wq_ll_create(mdev, ¶m->wq, rqc_wq, &rq->wq,
652 goto err_free_dma_tag;
654 rq->wq.db = &rq->wq.db[MLX5_RCV_DBR];
656 if (priv->params.hw_lro_en) {
657 rq->wqe_sz = priv->params.lro_wqe_sz;
659 rq->wqe_sz = MLX5E_SW2MB_MTU(priv->ifp->if_mtu);
661 if (rq->wqe_sz > MJUM16BYTES) {
663 goto err_rq_wq_destroy;
664 } else if (rq->wqe_sz > MJUM9BYTES) {
665 rq->wqe_sz = MJUM16BYTES;
666 } else if (rq->wqe_sz > MJUMPAGESIZE) {
667 rq->wqe_sz = MJUM9BYTES;
668 } else if (rq->wqe_sz > MCLBYTES) {
669 rq->wqe_sz = MJUMPAGESIZE;
671 rq->wqe_sz = MCLBYTES;
674 wq_sz = mlx5_wq_ll_get_size(&rq->wq);
676 err = -tcp_lro_init_args(&rq->lro, c->ifp, TCP_LRO_ENTRIES, wq_sz);
678 goto err_rq_wq_destroy;
680 rq->mbuf = malloc(wq_sz * sizeof(rq->mbuf[0]), M_MLX5EN, M_WAITOK | M_ZERO);
681 for (i = 0; i != wq_sz; i++) {
682 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
683 uint32_t byte_count = rq->wqe_sz - MLX5E_NET_IP_ALIGN;
685 err = -bus_dmamap_create(rq->dma_tag, 0, &rq->mbuf[i].dma_map);
688 bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map);
689 goto err_rq_mbuf_free;
691 wqe->data.lkey = c->mkey_be;
692 wqe->data.byte_count = cpu_to_be32(byte_count | MLX5_HW_START_PADDING);
699 snprintf(buffer, sizeof(buffer), "rxstat%d", c->ix);
700 mlx5e_create_stats(&rq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
701 buffer, mlx5e_rq_stats_desc, MLX5E_RQ_STATS_NUM,
706 free(rq->mbuf, M_MLX5EN);
707 tcp_lro_free(&rq->lro);
709 mlx5_wq_destroy(&rq->wq_ctrl);
711 bus_dma_tag_destroy(rq->dma_tag);
717 mlx5e_destroy_rq(struct mlx5e_rq *rq)
722 /* destroy all sysctl nodes */
723 sysctl_ctx_free(&rq->stats.ctx);
725 /* free leftover LRO packets, if any */
726 tcp_lro_free(&rq->lro);
728 wq_sz = mlx5_wq_ll_get_size(&rq->wq);
729 for (i = 0; i != wq_sz; i++) {
730 if (rq->mbuf[i].mbuf != NULL) {
731 bus_dmamap_unload(rq->dma_tag,
732 rq->mbuf[i].dma_map);
733 m_freem(rq->mbuf[i].mbuf);
735 bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map);
737 free(rq->mbuf, M_MLX5EN);
738 mlx5_wq_destroy(&rq->wq_ctrl);
742 mlx5e_enable_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
744 struct mlx5e_channel *c = rq->channel;
745 struct mlx5e_priv *priv = c->priv;
746 struct mlx5_core_dev *mdev = priv->mdev;
754 inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
755 sizeof(u64) * rq->wq_ctrl.buf.npages;
756 in = mlx5_vzalloc(inlen);
760 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
761 wq = MLX5_ADDR_OF(rqc, rqc, wq);
763 memcpy(rqc, param->rqc, sizeof(param->rqc));
765 MLX5_SET(rqc, rqc, cqn, c->rq.cq.mcq.cqn);
766 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
767 MLX5_SET(rqc, rqc, flush_in_error_en, 1);
768 if (priv->counter_set_id >= 0)
769 MLX5_SET(rqc, rqc, counter_set_id, priv->counter_set_id);
770 MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift -
772 MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma);
774 mlx5_fill_page_array(&rq->wq_ctrl.buf,
775 (__be64 *) MLX5_ADDR_OF(wq, wq, pas));
777 err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
785 mlx5e_modify_rq(struct mlx5e_rq *rq, int curr_state, int next_state)
787 struct mlx5e_channel *c = rq->channel;
788 struct mlx5e_priv *priv = c->priv;
789 struct mlx5_core_dev *mdev = priv->mdev;
796 inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
797 in = mlx5_vzalloc(inlen);
801 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
803 MLX5_SET(modify_rq_in, in, rqn, rq->rqn);
804 MLX5_SET(modify_rq_in, in, rq_state, curr_state);
805 MLX5_SET(rqc, rqc, state, next_state);
807 err = mlx5_core_modify_rq(mdev, in, inlen);
815 mlx5e_disable_rq(struct mlx5e_rq *rq)
817 struct mlx5e_channel *c = rq->channel;
818 struct mlx5e_priv *priv = c->priv;
819 struct mlx5_core_dev *mdev = priv->mdev;
821 mlx5_core_destroy_rq(mdev, rq->rqn);
825 mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq)
827 struct mlx5e_channel *c = rq->channel;
828 struct mlx5e_priv *priv = c->priv;
829 struct mlx5_wq_ll *wq = &rq->wq;
832 for (i = 0; i < 1000; i++) {
833 if (wq->cur_sz >= priv->params.min_rx_wqes)
842 mlx5e_open_rq(struct mlx5e_channel *c,
843 struct mlx5e_rq_param *param,
848 err = mlx5e_create_rq(c, param, rq);
852 err = mlx5e_enable_rq(rq, param);
856 err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
865 mlx5e_disable_rq(rq);
867 mlx5e_destroy_rq(rq);
873 mlx5e_close_rq(struct mlx5e_rq *rq)
877 callout_stop(&rq->watchdog);
878 mtx_unlock(&rq->mtx);
880 callout_drain(&rq->watchdog);
882 mlx5e_modify_rq(rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
886 mlx5e_close_rq_wait(struct mlx5e_rq *rq)
888 /* wait till RQ is empty */
889 while (!mlx5_wq_ll_is_empty(&rq->wq)) {
891 rq->cq.mcq.comp(&rq->cq.mcq);
894 mlx5e_disable_rq(rq);
895 mlx5e_destroy_rq(rq);
899 mlx5e_free_sq_db(struct mlx5e_sq *sq)
901 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
904 for (x = 0; x != wq_sz; x++)
905 bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map);
906 free(sq->mbuf, M_MLX5EN);
910 mlx5e_alloc_sq_db(struct mlx5e_sq *sq)
912 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
916 sq->mbuf = malloc(wq_sz * sizeof(sq->mbuf[0]), M_MLX5EN, M_WAITOK | M_ZERO);
918 /* Create DMA descriptor MAPs */
919 for (x = 0; x != wq_sz; x++) {
920 err = -bus_dmamap_create(sq->dma_tag, 0, &sq->mbuf[x].dma_map);
923 bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map);
924 free(sq->mbuf, M_MLX5EN);
931 static const char *mlx5e_sq_stats_desc[] = {
932 MLX5E_SQ_STATS(MLX5E_STATS_DESC)
936 mlx5e_create_sq(struct mlx5e_channel *c,
938 struct mlx5e_sq_param *param,
941 struct mlx5e_priv *priv = c->priv;
942 struct mlx5_core_dev *mdev = priv->mdev;
945 void *sqc = param->sqc;
946 void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq);
953 /* Create DMA descriptor TAG */
954 if ((err = -bus_dma_tag_create(
955 bus_get_dma_tag(mdev->pdev->dev.bsddev),
956 1, /* any alignment */
958 BUS_SPACE_MAXADDR, /* lowaddr */
959 BUS_SPACE_MAXADDR, /* highaddr */
960 NULL, NULL, /* filter, filterarg */
961 MLX5E_MAX_TX_PAYLOAD_SIZE, /* maxsize */
962 MLX5E_MAX_TX_MBUF_FRAGS, /* nsegments */
963 MLX5E_MAX_TX_MBUF_SIZE, /* maxsegsize */
965 NULL, NULL, /* lockfunc, lockfuncarg */
969 err = mlx5_alloc_map_uar(mdev, &sq->uar);
971 goto err_free_dma_tag;
973 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, &sq->wq,
976 goto err_unmap_free_uar;
978 sq->wq.db = &sq->wq.db[MLX5_SND_DBR];
979 sq->bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2;
981 err = mlx5e_alloc_sq_db(sq);
983 goto err_sq_wq_destroy;
985 sq->mkey_be = c->mkey_be;
990 sq->br = buf_ring_alloc(MLX5E_SQ_TX_QUEUE_SIZE, M_MLX5EN,
991 M_WAITOK, &sq->lock);
992 if (sq->br == NULL) {
993 if_printf(c->ifp, "%s: Failed allocating sq drbr buffer\n",
999 sq->sq_tq = taskqueue_create_fast("mlx5e_que", M_WAITOK,
1000 taskqueue_thread_enqueue, &sq->sq_tq);
1001 if (sq->sq_tq == NULL) {
1002 if_printf(c->ifp, "%s: Failed allocating taskqueue\n",
1008 TASK_INIT(&sq->sq_task, 0, mlx5e_tx_que, sq);
1010 cpu_id = rss_getcpu(c->ix % rss_getnumbuckets());
1011 CPU_SETOF(cpu_id, &cpu_mask);
1012 taskqueue_start_threads_cpuset(&sq->sq_tq, 1, PI_NET, &cpu_mask,
1013 "%s TX SQ%d.%d CPU%d", c->ifp->if_xname, c->ix, tc, cpu_id);
1015 taskqueue_start_threads(&sq->sq_tq, 1, PI_NET,
1016 "%s TX SQ%d.%d", c->ifp->if_xname, c->ix, tc);
1018 snprintf(buffer, sizeof(buffer), "txstat%dtc%d", c->ix, tc);
1019 mlx5e_create_stats(&sq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
1020 buffer, mlx5e_sq_stats_desc, MLX5E_SQ_STATS_NUM,
1026 buf_ring_free(sq->br, M_MLX5EN);
1028 mlx5e_free_sq_db(sq);
1030 mlx5_wq_destroy(&sq->wq_ctrl);
1033 mlx5_unmap_free_uar(mdev, &sq->uar);
1036 bus_dma_tag_destroy(sq->dma_tag);
1042 mlx5e_destroy_sq(struct mlx5e_sq *sq)
1044 /* destroy all sysctl nodes */
1045 sysctl_ctx_free(&sq->stats.ctx);
1047 mlx5e_free_sq_db(sq);
1048 mlx5_wq_destroy(&sq->wq_ctrl);
1049 mlx5_unmap_free_uar(sq->priv->mdev, &sq->uar);
1050 taskqueue_drain(sq->sq_tq, &sq->sq_task);
1051 taskqueue_free(sq->sq_tq);
1052 buf_ring_free(sq->br, M_MLX5EN);
1056 mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param,
1065 inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
1066 sizeof(u64) * sq->wq_ctrl.buf.npages;
1067 in = mlx5_vzalloc(inlen);
1071 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1072 wq = MLX5_ADDR_OF(sqc, sqc, wq);
1074 memcpy(sqc, param->sqc, sizeof(param->sqc));
1076 MLX5_SET(sqc, sqc, tis_num_0, tis_num);
1077 MLX5_SET(sqc, sqc, cqn, sq->cq.mcq.cqn);
1078 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
1079 MLX5_SET(sqc, sqc, tis_lst_sz, 1);
1080 MLX5_SET(sqc, sqc, flush_in_error_en, 1);
1082 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
1083 MLX5_SET(wq, wq, uar_page, sq->uar.index);
1084 MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift -
1086 MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma);
1088 mlx5_fill_page_array(&sq->wq_ctrl.buf,
1089 (__be64 *) MLX5_ADDR_OF(wq, wq, pas));
1091 err = mlx5_core_create_sq(sq->priv->mdev, in, inlen, &sq->sqn);
1099 mlx5e_modify_sq(struct mlx5e_sq *sq, int curr_state, int next_state)
1106 inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1107 in = mlx5_vzalloc(inlen);
1111 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
1113 MLX5_SET(modify_sq_in, in, sqn, sq->sqn);
1114 MLX5_SET(modify_sq_in, in, sq_state, curr_state);
1115 MLX5_SET(sqc, sqc, state, next_state);
1117 err = mlx5_core_modify_sq(sq->priv->mdev, in, inlen);
1125 mlx5e_disable_sq(struct mlx5e_sq *sq)
1128 mlx5_core_destroy_sq(sq->priv->mdev, sq->sqn);
1132 mlx5e_open_sq(struct mlx5e_channel *c,
1134 struct mlx5e_sq_param *param,
1135 struct mlx5e_sq *sq)
1139 err = mlx5e_create_sq(c, tc, param, sq);
1143 err = mlx5e_enable_sq(sq, param, c->priv->tisn[tc]);
1145 goto err_destroy_sq;
1147 err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY);
1149 goto err_disable_sq;
1151 atomic_store_rel_int(&sq->queue_state, MLX5E_SQ_READY);
1156 mlx5e_disable_sq(sq);
1158 mlx5e_destroy_sq(sq);
1164 mlx5e_sq_send_nops_locked(struct mlx5e_sq *sq, int can_sleep)
1166 /* fill up remainder with NOPs */
1167 while (sq->cev_counter != 0) {
1168 while (!mlx5e_sq_has_room_for(sq, 1)) {
1169 if (can_sleep != 0) {
1170 mtx_unlock(&sq->lock);
1172 mtx_lock(&sq->lock);
1177 /* send a single NOP */
1178 mlx5e_send_nop(sq, 1);
1182 /* Check if we need to write the doorbell */
1183 if (likely(sq->doorbell.d64 != 0)) {
1184 mlx5e_tx_notify_hw(sq, sq->doorbell.d32, 0);
1185 sq->doorbell.d64 = 0;
1191 mlx5e_sq_cev_timeout(void *arg)
1193 struct mlx5e_sq *sq = arg;
1195 mtx_assert(&sq->lock, MA_OWNED);
1197 /* check next state */
1198 switch (sq->cev_next_state) {
1199 case MLX5E_CEV_STATE_SEND_NOPS:
1200 /* fill TX ring with NOPs, if any */
1201 mlx5e_sq_send_nops_locked(sq, 0);
1203 /* check if completed */
1204 if (sq->cev_counter == 0) {
1205 sq->cev_next_state = MLX5E_CEV_STATE_INITIAL;
1210 /* send NOPs on next timeout */
1211 sq->cev_next_state = MLX5E_CEV_STATE_SEND_NOPS;
1216 callout_reset_curcpu(&sq->cev_callout, hz, mlx5e_sq_cev_timeout, sq);
1220 mlx5e_drain_sq(struct mlx5e_sq *sq)
1223 mtx_lock(&sq->lock);
1224 /* teardown event factor timer, if any */
1225 sq->cev_next_state = MLX5E_CEV_STATE_HOLD_NOPS;
1226 callout_stop(&sq->cev_callout);
1228 /* send dummy NOPs in order to flush the transmit ring */
1229 mlx5e_sq_send_nops_locked(sq, 1);
1230 mtx_unlock(&sq->lock);
1232 /* make sure it is safe to free the callout */
1233 callout_drain(&sq->cev_callout);
1235 /* error out remaining requests */
1236 mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY, MLX5_SQC_STATE_ERR);
1238 /* wait till SQ is empty */
1239 mtx_lock(&sq->lock);
1240 while (sq->cc != sq->pc) {
1241 mtx_unlock(&sq->lock);
1243 sq->cq.mcq.comp(&sq->cq.mcq);
1244 mtx_lock(&sq->lock);
1246 mtx_unlock(&sq->lock);
1250 mlx5e_close_sq_wait(struct mlx5e_sq *sq)
1254 mlx5e_disable_sq(sq);
1255 mlx5e_destroy_sq(sq);
1259 mlx5e_create_cq(struct mlx5e_priv *priv,
1260 struct mlx5e_cq_param *param,
1261 struct mlx5e_cq *cq,
1262 mlx5e_cq_comp_t *comp,
1265 struct mlx5_core_dev *mdev = priv->mdev;
1266 struct mlx5_core_cq *mcq = &cq->mcq;
1272 param->wq.buf_numa_node = 0;
1273 param->wq.db_numa_node = 0;
1275 err = mlx5_cqwq_create(mdev, ¶m->wq, param->cqc, &cq->wq,
1280 mlx5_vector2eqn(mdev, eq_ix, &eqn_not_used, &irqn);
1283 mcq->set_ci_db = cq->wq_ctrl.db.db;
1284 mcq->arm_db = cq->wq_ctrl.db.db + 1;
1285 *mcq->set_ci_db = 0;
1287 mcq->vector = eq_ix;
1289 mcq->event = mlx5e_cq_error_event;
1291 mcq->uar = &priv->cq_uar;
1293 for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
1294 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
1305 mlx5e_destroy_cq(struct mlx5e_cq *cq)
1307 mlx5_wq_destroy(&cq->wq_ctrl);
1311 mlx5e_enable_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param, int eq_ix)
1313 struct mlx5_core_cq *mcq = &cq->mcq;
1321 inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
1322 sizeof(u64) * cq->wq_ctrl.buf.npages;
1323 in = mlx5_vzalloc(inlen);
1327 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
1329 memcpy(cqc, param->cqc, sizeof(param->cqc));
1331 mlx5_fill_page_array(&cq->wq_ctrl.buf,
1332 (__be64 *) MLX5_ADDR_OF(create_cq_in, in, pas));
1334 mlx5_vector2eqn(cq->priv->mdev, eq_ix, &eqn, &irqn_not_used);
1336 MLX5_SET(cqc, cqc, c_eqn, eqn);
1337 MLX5_SET(cqc, cqc, uar_page, mcq->uar->index);
1338 MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
1340 MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma);
1342 err = mlx5_core_create_cq(cq->priv->mdev, mcq, in, inlen);
1355 mlx5e_disable_cq(struct mlx5e_cq *cq)
1358 mlx5_core_destroy_cq(cq->priv->mdev, &cq->mcq);
1362 mlx5e_open_cq(struct mlx5e_priv *priv,
1363 struct mlx5e_cq_param *param,
1364 struct mlx5e_cq *cq,
1365 mlx5e_cq_comp_t *comp,
1370 err = mlx5e_create_cq(priv, param, cq, comp, eq_ix);
1374 err = mlx5e_enable_cq(cq, param, eq_ix);
1376 goto err_destroy_cq;
1381 mlx5e_destroy_cq(cq);
1387 mlx5e_close_cq(struct mlx5e_cq *cq)
1389 mlx5e_disable_cq(cq);
1390 mlx5e_destroy_cq(cq);
1394 mlx5e_open_tx_cqs(struct mlx5e_channel *c,
1395 struct mlx5e_channel_param *cparam)
1400 for (tc = 0; tc < c->num_tc; tc++) {
1401 /* open completion queue */
1402 err = mlx5e_open_cq(c->priv, &cparam->tx_cq, &c->sq[tc].cq,
1403 &mlx5e_tx_cq_comp, c->ix);
1405 goto err_close_tx_cqs;
1410 for (tc--; tc >= 0; tc--)
1411 mlx5e_close_cq(&c->sq[tc].cq);
1417 mlx5e_close_tx_cqs(struct mlx5e_channel *c)
1421 for (tc = 0; tc < c->num_tc; tc++)
1422 mlx5e_close_cq(&c->sq[tc].cq);
1426 mlx5e_open_sqs(struct mlx5e_channel *c,
1427 struct mlx5e_channel_param *cparam)
1432 for (tc = 0; tc < c->num_tc; tc++) {
1433 err = mlx5e_open_sq(c, tc, &cparam->sq, &c->sq[tc]);
1441 for (tc--; tc >= 0; tc--)
1442 mlx5e_close_sq_wait(&c->sq[tc]);
1448 mlx5e_close_sqs_wait(struct mlx5e_channel *c)
1452 for (tc = 0; tc < c->num_tc; tc++)
1453 mlx5e_close_sq_wait(&c->sq[tc]);
1457 mlx5e_chan_mtx_init(struct mlx5e_channel *c)
1461 mtx_init(&c->rq.mtx, "mlx5rx", MTX_NETWORK_LOCK, MTX_DEF);
1463 callout_init_mtx(&c->rq.watchdog, &c->rq.mtx, 0);
1465 for (tc = 0; tc < c->num_tc; tc++) {
1466 struct mlx5e_sq *sq = c->sq + tc;
1468 mtx_init(&sq->lock, "mlx5tx", MTX_NETWORK_LOCK, MTX_DEF);
1469 mtx_init(&sq->comp_lock, "mlx5comp", MTX_NETWORK_LOCK,
1472 callout_init_mtx(&sq->cev_callout, &sq->lock, 0);
1474 sq->cev_factor = c->priv->params_ethtool.tx_completion_fact;
1476 /* ensure the TX completion event factor is not zero */
1477 if (sq->cev_factor == 0)
1483 mlx5e_chan_mtx_destroy(struct mlx5e_channel *c)
1487 mtx_destroy(&c->rq.mtx);
1489 for (tc = 0; tc < c->num_tc; tc++) {
1490 mtx_destroy(&c->sq[tc].lock);
1491 mtx_destroy(&c->sq[tc].comp_lock);
1496 mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
1497 struct mlx5e_channel_param *cparam,
1498 struct mlx5e_channel *volatile *cp)
1500 struct mlx5e_channel *c;
1503 c = malloc(sizeof(*c), M_MLX5EN, M_WAITOK | M_ZERO);
1508 c->mkey_be = cpu_to_be32(priv->mr.key);
1509 c->num_tc = priv->num_tc;
1512 mlx5e_chan_mtx_init(c);
1514 /* open transmit completion queue */
1515 err = mlx5e_open_tx_cqs(c, cparam);
1519 /* open receive completion queue */
1520 err = mlx5e_open_cq(c->priv, &cparam->rx_cq, &c->rq.cq,
1521 &mlx5e_rx_cq_comp, c->ix);
1523 goto err_close_tx_cqs;
1525 err = mlx5e_open_sqs(c, cparam);
1527 goto err_close_rx_cq;
1529 err = mlx5e_open_rq(c, &cparam->rq, &c->rq);
1533 /* store channel pointer */
1536 /* poll receive queue initially */
1537 c->rq.cq.mcq.comp(&c->rq.cq.mcq);
1542 mlx5e_close_sqs_wait(c);
1545 mlx5e_close_cq(&c->rq.cq);
1548 mlx5e_close_tx_cqs(c);
1551 /* destroy mutexes */
1552 mlx5e_chan_mtx_destroy(c);
1558 mlx5e_close_channel(struct mlx5e_channel *volatile *pp)
1560 struct mlx5e_channel *c = *pp;
1562 /* check if channel is already closed */
1565 mlx5e_close_rq(&c->rq);
1569 mlx5e_close_channel_wait(struct mlx5e_channel *volatile *pp)
1571 struct mlx5e_channel *c = *pp;
1573 /* check if channel is already closed */
1576 /* ensure channel pointer is no longer used */
1579 mlx5e_close_rq_wait(&c->rq);
1580 mlx5e_close_sqs_wait(c);
1581 mlx5e_close_cq(&c->rq.cq);
1582 mlx5e_close_tx_cqs(c);
1583 /* destroy mutexes */
1584 mlx5e_chan_mtx_destroy(c);
1589 mlx5e_build_rq_param(struct mlx5e_priv *priv,
1590 struct mlx5e_rq_param *param)
1592 void *rqc = param->rqc;
1593 void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
1595 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
1596 MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
1597 MLX5_SET(wq, wq, log_wq_stride, ilog2(sizeof(struct mlx5e_rx_wqe)));
1598 MLX5_SET(wq, wq, log_wq_sz, priv->params.log_rq_size);
1599 MLX5_SET(wq, wq, pd, priv->pdn);
1601 param->wq.buf_numa_node = 0;
1602 param->wq.db_numa_node = 0;
1603 param->wq.linear = 1;
1607 mlx5e_build_sq_param(struct mlx5e_priv *priv,
1608 struct mlx5e_sq_param *param)
1610 void *sqc = param->sqc;
1611 void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
1613 MLX5_SET(wq, wq, log_wq_sz, priv->params.log_sq_size);
1614 MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
1615 MLX5_SET(wq, wq, pd, priv->pdn);
1617 param->wq.buf_numa_node = 0;
1618 param->wq.db_numa_node = 0;
1619 param->wq.linear = 1;
1623 mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
1624 struct mlx5e_cq_param *param)
1626 void *cqc = param->cqc;
1628 MLX5_SET(cqc, cqc, uar_page, priv->cq_uar.index);
1632 mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
1633 struct mlx5e_cq_param *param)
1635 void *cqc = param->cqc;
1639 * TODO The sysctl to control on/off is a bool value for now, which means
1640 * we only support CSUM, once HASH is implemnted we'll need to address that.
1642 if (priv->params.cqe_zipping_en) {
1643 MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_CSUM);
1644 MLX5_SET(cqc, cqc, cqe_compression_en, 1);
1647 MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_rq_size);
1648 MLX5_SET(cqc, cqc, cq_period, priv->params.rx_cq_moderation_usec);
1649 MLX5_SET(cqc, cqc, cq_max_count, priv->params.rx_cq_moderation_pkts);
1651 switch (priv->params.rx_cq_moderation_mode) {
1653 MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
1656 if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
1657 MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
1659 MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
1663 mlx5e_build_common_cq_param(priv, param);
1667 mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
1668 struct mlx5e_cq_param *param)
1670 void *cqc = param->cqc;
1672 MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_sq_size);
1673 MLX5_SET(cqc, cqc, cq_period, priv->params.tx_cq_moderation_usec);
1674 MLX5_SET(cqc, cqc, cq_max_count, priv->params.tx_cq_moderation_pkts);
1676 switch (priv->params.tx_cq_moderation_mode) {
1678 MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
1681 if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
1682 MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
1684 MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
1688 mlx5e_build_common_cq_param(priv, param);
1692 mlx5e_build_channel_param(struct mlx5e_priv *priv,
1693 struct mlx5e_channel_param *cparam)
1695 memset(cparam, 0, sizeof(*cparam));
1697 mlx5e_build_rq_param(priv, &cparam->rq);
1698 mlx5e_build_sq_param(priv, &cparam->sq);
1699 mlx5e_build_rx_cq_param(priv, &cparam->rx_cq);
1700 mlx5e_build_tx_cq_param(priv, &cparam->tx_cq);
1704 mlx5e_open_channels(struct mlx5e_priv *priv)
1706 struct mlx5e_channel_param cparam;
1712 priv->channel = malloc(priv->params.num_channels *
1713 sizeof(struct mlx5e_channel *), M_MLX5EN, M_WAITOK | M_ZERO);
1715 mlx5e_build_channel_param(priv, &cparam);
1716 for (i = 0; i < priv->params.num_channels; i++) {
1717 err = mlx5e_open_channel(priv, i, &cparam, &priv->channel[i]);
1719 goto err_close_channels;
1722 for (j = 0; j < priv->params.num_channels; j++) {
1723 err = mlx5e_wait_for_min_rx_wqes(&priv->channel[j]->rq);
1725 goto err_close_channels;
1731 for (i--; i >= 0; i--) {
1732 mlx5e_close_channel(&priv->channel[i]);
1733 mlx5e_close_channel_wait(&priv->channel[i]);
1736 /* remove "volatile" attribute from "channel" pointer */
1737 ptr = __DECONST(void *, priv->channel);
1738 priv->channel = NULL;
1740 free(ptr, M_MLX5EN);
1746 mlx5e_close_channels(struct mlx5e_priv *priv)
1751 if (priv->channel == NULL)
1754 for (i = 0; i < priv->params.num_channels; i++)
1755 mlx5e_close_channel(&priv->channel[i]);
1756 for (i = 0; i < priv->params.num_channels; i++)
1757 mlx5e_close_channel_wait(&priv->channel[i]);
1759 /* remove "volatile" attribute from "channel" pointer */
1760 ptr = __DECONST(void *, priv->channel);
1761 priv->channel = NULL;
1763 free(ptr, M_MLX5EN);
1767 mlx5e_refresh_sq_params(struct mlx5e_priv *priv, struct mlx5e_sq *sq)
1769 return (mlx5_core_modify_cq_moderation(priv->mdev, &sq->cq.mcq,
1770 priv->params.tx_cq_moderation_usec,
1771 priv->params.tx_cq_moderation_pkts));
1775 mlx5e_refresh_rq_params(struct mlx5e_priv *priv, struct mlx5e_rq *rq)
1777 return (mlx5_core_modify_cq_moderation(priv->mdev, &rq->cq.mcq,
1778 priv->params.rx_cq_moderation_usec,
1779 priv->params.rx_cq_moderation_pkts));
1783 mlx5e_refresh_channel_params_sub(struct mlx5e_priv *priv, struct mlx5e_channel *c)
1791 err = mlx5e_refresh_rq_params(priv, &c->rq);
1795 for (i = 0; i != c->num_tc; i++) {
1796 err = mlx5e_refresh_sq_params(priv, &c->sq[i]);
1805 mlx5e_refresh_channel_params(struct mlx5e_priv *priv)
1809 if (priv->channel == NULL)
1812 for (i = 0; i < priv->params.num_channels; i++) {
1815 err = mlx5e_refresh_channel_params_sub(priv, priv->channel[i]);
1823 mlx5e_open_tis(struct mlx5e_priv *priv, int tc)
1825 struct mlx5_core_dev *mdev = priv->mdev;
1826 u32 in[MLX5_ST_SZ_DW(create_tis_in)];
1827 void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
1829 memset(in, 0, sizeof(in));
1831 MLX5_SET(tisc, tisc, prio, tc);
1832 MLX5_SET(tisc, tisc, transport_domain, priv->tdn);
1834 return (mlx5_core_create_tis(mdev, in, sizeof(in), &priv->tisn[tc]));
1838 mlx5e_close_tis(struct mlx5e_priv *priv, int tc)
1840 mlx5_core_destroy_tis(priv->mdev, priv->tisn[tc]);
1844 mlx5e_open_tises(struct mlx5e_priv *priv)
1846 int num_tc = priv->num_tc;
1850 for (tc = 0; tc < num_tc; tc++) {
1851 err = mlx5e_open_tis(priv, tc);
1853 goto err_close_tises;
1859 for (tc--; tc >= 0; tc--)
1860 mlx5e_close_tis(priv, tc);
1866 mlx5e_close_tises(struct mlx5e_priv *priv)
1868 int num_tc = priv->num_tc;
1871 for (tc = 0; tc < num_tc; tc++)
1872 mlx5e_close_tis(priv, tc);
1876 mlx5e_open_rqt(struct mlx5e_priv *priv)
1878 struct mlx5_core_dev *mdev = priv->mdev;
1880 u32 out[MLX5_ST_SZ_DW(create_rqt_out)];
1887 sz = 1 << priv->params.rx_hash_log_tbl_sz;
1889 inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
1890 in = mlx5_vzalloc(inlen);
1893 rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
1895 MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
1896 MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
1898 for (i = 0; i < sz; i++) {
1901 ix = rss_get_indirection_to_bucket(i);
1905 /* ensure we don't overflow */
1906 ix %= priv->params.num_channels;
1907 MLX5_SET(rqtc, rqtc, rq_num[i], priv->channel[ix]->rq.rqn);
1910 MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT);
1912 memset(out, 0, sizeof(out));
1913 err = mlx5_cmd_exec_check_status(mdev, in, inlen, out, sizeof(out));
1915 priv->rqtn = MLX5_GET(create_rqt_out, out, rqtn);
1923 mlx5e_close_rqt(struct mlx5e_priv *priv)
1925 u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)];
1926 u32 out[MLX5_ST_SZ_DW(destroy_rqt_out)];
1928 memset(in, 0, sizeof(in));
1930 MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT);
1931 MLX5_SET(destroy_rqt_in, in, rqtn, priv->rqtn);
1933 mlx5_cmd_exec_check_status(priv->mdev, in, sizeof(in), out,
1938 mlx5e_build_tir_ctx(struct mlx5e_priv *priv, u32 * tirc, int tt)
1940 void *hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
1943 MLX5_SET(tirc, tirc, transport_domain, priv->tdn);
1945 #define ROUGH_MAX_L2_L3_HDR_SZ 256
1947 #define MLX5_HASH_IP (MLX5_HASH_FIELD_SEL_SRC_IP |\
1948 MLX5_HASH_FIELD_SEL_DST_IP)
1950 #define MLX5_HASH_ALL (MLX5_HASH_FIELD_SEL_SRC_IP |\
1951 MLX5_HASH_FIELD_SEL_DST_IP |\
1952 MLX5_HASH_FIELD_SEL_L4_SPORT |\
1953 MLX5_HASH_FIELD_SEL_L4_DPORT)
1955 #define MLX5_HASH_IP_IPSEC_SPI (MLX5_HASH_FIELD_SEL_SRC_IP |\
1956 MLX5_HASH_FIELD_SEL_DST_IP |\
1957 MLX5_HASH_FIELD_SEL_IPSEC_SPI)
1959 if (priv->params.hw_lro_en) {
1960 MLX5_SET(tirc, tirc, lro_enable_mask,
1961 MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
1962 MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO);
1963 MLX5_SET(tirc, tirc, lro_max_msg_sz,
1964 (priv->params.lro_wqe_sz -
1965 ROUGH_MAX_L2_L3_HDR_SZ) >> 8);
1966 /* TODO: add the option to choose timer value dynamically */
1967 MLX5_SET(tirc, tirc, lro_timeout_period_usecs,
1968 MLX5_CAP_ETH(priv->mdev,
1969 lro_timer_supported_periods[2]));
1972 /* setup parameters for hashing TIR type, if any */
1975 MLX5_SET(tirc, tirc, disp_type,
1976 MLX5_TIRC_DISP_TYPE_DIRECT);
1977 MLX5_SET(tirc, tirc, inline_rqn,
1978 priv->channel[0]->rq.rqn);
1981 MLX5_SET(tirc, tirc, disp_type,
1982 MLX5_TIRC_DISP_TYPE_INDIRECT);
1983 MLX5_SET(tirc, tirc, indirect_table,
1985 MLX5_SET(tirc, tirc, rx_hash_fn,
1986 MLX5_TIRC_RX_HASH_FN_HASH_TOEPLITZ);
1987 hkey = (__be32 *) MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
1990 * The FreeBSD RSS implementation does currently not
1991 * support symmetric Toeplitz hashes:
1993 MLX5_SET(tirc, tirc, rx_hash_symmetric, 0);
1994 rss_getkey((uint8_t *)hkey);
1996 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
1997 hkey[0] = cpu_to_be32(0xD181C62C);
1998 hkey[1] = cpu_to_be32(0xF7F4DB5B);
1999 hkey[2] = cpu_to_be32(0x1983A2FC);
2000 hkey[3] = cpu_to_be32(0x943E1ADB);
2001 hkey[4] = cpu_to_be32(0xD9389E6B);
2002 hkey[5] = cpu_to_be32(0xD1039C2C);
2003 hkey[6] = cpu_to_be32(0xA74499AD);
2004 hkey[7] = cpu_to_be32(0x593D56D9);
2005 hkey[8] = cpu_to_be32(0xF3253C06);
2006 hkey[9] = cpu_to_be32(0x2ADC1FFC);
2012 case MLX5E_TT_IPV4_TCP:
2013 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2014 MLX5_L3_PROT_TYPE_IPV4);
2015 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2016 MLX5_L4_PROT_TYPE_TCP);
2018 if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4)) {
2019 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2023 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2027 case MLX5E_TT_IPV6_TCP:
2028 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2029 MLX5_L3_PROT_TYPE_IPV6);
2030 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2031 MLX5_L4_PROT_TYPE_TCP);
2033 if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV6)) {
2034 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2038 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2042 case MLX5E_TT_IPV4_UDP:
2043 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2044 MLX5_L3_PROT_TYPE_IPV4);
2045 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2046 MLX5_L4_PROT_TYPE_UDP);
2048 if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4)) {
2049 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2053 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2057 case MLX5E_TT_IPV6_UDP:
2058 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2059 MLX5_L3_PROT_TYPE_IPV6);
2060 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2061 MLX5_L4_PROT_TYPE_UDP);
2063 if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV6)) {
2064 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2068 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2072 case MLX5E_TT_IPV4_IPSEC_AH:
2073 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2074 MLX5_L3_PROT_TYPE_IPV4);
2075 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2076 MLX5_HASH_IP_IPSEC_SPI);
2079 case MLX5E_TT_IPV6_IPSEC_AH:
2080 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2081 MLX5_L3_PROT_TYPE_IPV6);
2082 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2083 MLX5_HASH_IP_IPSEC_SPI);
2086 case MLX5E_TT_IPV4_IPSEC_ESP:
2087 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2088 MLX5_L3_PROT_TYPE_IPV4);
2089 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2090 MLX5_HASH_IP_IPSEC_SPI);
2093 case MLX5E_TT_IPV6_IPSEC_ESP:
2094 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2095 MLX5_L3_PROT_TYPE_IPV6);
2096 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2097 MLX5_HASH_IP_IPSEC_SPI);
2101 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2102 MLX5_L3_PROT_TYPE_IPV4);
2103 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2108 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2109 MLX5_L3_PROT_TYPE_IPV6);
2110 MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2120 mlx5e_open_tir(struct mlx5e_priv *priv, int tt)
2122 struct mlx5_core_dev *mdev = priv->mdev;
2128 inlen = MLX5_ST_SZ_BYTES(create_tir_in);
2129 in = mlx5_vzalloc(inlen);
2132 tirc = MLX5_ADDR_OF(create_tir_in, in, tir_context);
2134 mlx5e_build_tir_ctx(priv, tirc, tt);
2136 err = mlx5_core_create_tir(mdev, in, inlen, &priv->tirn[tt]);
2144 mlx5e_close_tir(struct mlx5e_priv *priv, int tt)
2146 mlx5_core_destroy_tir(priv->mdev, priv->tirn[tt]);
2150 mlx5e_open_tirs(struct mlx5e_priv *priv)
2155 for (i = 0; i < MLX5E_NUM_TT; i++) {
2156 err = mlx5e_open_tir(priv, i);
2158 goto err_close_tirs;
2164 for (i--; i >= 0; i--)
2165 mlx5e_close_tir(priv, i);
2171 mlx5e_close_tirs(struct mlx5e_priv *priv)
2175 for (i = 0; i < MLX5E_NUM_TT; i++)
2176 mlx5e_close_tir(priv, i);
2180 * SW MTU does not include headers,
2181 * HW MTU includes all headers and checksums.
2184 mlx5e_set_dev_port_mtu(struct ifnet *ifp, int sw_mtu)
2186 struct mlx5e_priv *priv = ifp->if_softc;
2187 struct mlx5_core_dev *mdev = priv->mdev;
2191 err = mlx5_set_port_mtu(mdev, MLX5E_SW2HW_MTU(sw_mtu));
2193 if_printf(ifp, "%s: mlx5_set_port_mtu failed setting %d, err=%d\n",
2194 __func__, sw_mtu, err);
2197 err = mlx5_query_port_oper_mtu(mdev, &hw_mtu);
2199 if_printf(ifp, "Query port MTU, after setting new "
2200 "MTU value, failed\n");
2201 } else if (MLX5E_HW2SW_MTU(hw_mtu) < sw_mtu) {
2203 if_printf(ifp, "Port MTU %d is smaller than "
2204 "ifp mtu %d\n", hw_mtu, sw_mtu);
2205 } else if (MLX5E_HW2SW_MTU(hw_mtu) > sw_mtu) {
2207 if_printf(ifp, "Port MTU %d is bigger than "
2208 "ifp mtu %d\n", hw_mtu, sw_mtu);
2210 ifp->if_mtu = sw_mtu;
2215 mlx5e_open_locked(struct ifnet *ifp)
2217 struct mlx5e_priv *priv = ifp->if_softc;
2221 /* check if already opened */
2222 if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
2226 if (rss_getnumbuckets() > priv->params.num_channels) {
2227 if_printf(ifp, "NOTE: There are more RSS buckets(%u) than "
2228 "channels(%u) available\n", rss_getnumbuckets(),
2229 priv->params.num_channels);
2232 err = mlx5e_open_tises(priv);
2234 if_printf(ifp, "%s: mlx5e_open_tises failed, %d\n",
2238 err = mlx5_vport_alloc_q_counter(priv->mdev,
2239 MLX5_INTERFACE_PROTOCOL_ETH, &set_id);
2241 if_printf(priv->ifp,
2242 "%s: mlx5_vport_alloc_q_counter failed: %d\n",
2244 goto err_close_tises;
2246 /* store counter set ID */
2247 priv->counter_set_id = set_id;
2249 err = mlx5e_open_channels(priv);
2251 if_printf(ifp, "%s: mlx5e_open_channels failed, %d\n",
2253 goto err_dalloc_q_counter;
2255 err = mlx5e_open_rqt(priv);
2257 if_printf(ifp, "%s: mlx5e_open_rqt failed, %d\n",
2259 goto err_close_channels;
2261 err = mlx5e_open_tirs(priv);
2263 if_printf(ifp, "%s: mlx5e_open_tir failed, %d\n",
2265 goto err_close_rqls;
2267 err = mlx5e_open_flow_table(priv);
2269 if_printf(ifp, "%s: mlx5e_open_flow_table failed, %d\n",
2271 goto err_close_tirs;
2273 err = mlx5e_add_all_vlan_rules(priv);
2275 if_printf(ifp, "%s: mlx5e_add_all_vlan_rules failed, %d\n",
2277 goto err_close_flow_table;
2279 set_bit(MLX5E_STATE_OPENED, &priv->state);
2281 mlx5e_update_carrier(priv);
2282 mlx5e_set_rx_mode_core(priv);
2286 err_close_flow_table:
2287 mlx5e_close_flow_table(priv);
2290 mlx5e_close_tirs(priv);
2293 mlx5e_close_rqt(priv);
2296 mlx5e_close_channels(priv);
2298 err_dalloc_q_counter:
2299 mlx5_vport_dealloc_q_counter(priv->mdev,
2300 MLX5_INTERFACE_PROTOCOL_ETH, priv->counter_set_id);
2303 mlx5e_close_tises(priv);
2309 mlx5e_open(void *arg)
2311 struct mlx5e_priv *priv = arg;
2314 if (mlx5_set_port_status(priv->mdev, MLX5_PORT_UP))
2315 if_printf(priv->ifp,
2316 "%s: Setting port status to up failed\n",
2319 mlx5e_open_locked(priv->ifp);
2320 priv->ifp->if_drv_flags |= IFF_DRV_RUNNING;
2325 mlx5e_close_locked(struct ifnet *ifp)
2327 struct mlx5e_priv *priv = ifp->if_softc;
2329 /* check if already closed */
2330 if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
2333 clear_bit(MLX5E_STATE_OPENED, &priv->state);
2335 mlx5e_set_rx_mode_core(priv);
2336 mlx5e_del_all_vlan_rules(priv);
2337 if_link_state_change(priv->ifp, LINK_STATE_DOWN);
2338 mlx5e_close_flow_table(priv);
2339 mlx5e_close_tirs(priv);
2340 mlx5e_close_rqt(priv);
2341 mlx5e_close_channels(priv);
2342 mlx5_vport_dealloc_q_counter(priv->mdev,
2343 MLX5_INTERFACE_PROTOCOL_ETH, priv->counter_set_id);
2344 mlx5e_close_tises(priv);
2349 #if (__FreeBSD_version >= 1100000)
2351 mlx5e_get_counter(struct ifnet *ifp, ift_counter cnt)
2353 struct mlx5e_priv *priv = ifp->if_softc;
2356 /* PRIV_LOCK(priv); XXX not allowed */
2358 case IFCOUNTER_IPACKETS:
2359 retval = priv->stats.vport.rx_packets;
2361 case IFCOUNTER_IERRORS:
2362 retval = priv->stats.vport.rx_error_packets;
2364 case IFCOUNTER_IQDROPS:
2365 retval = priv->stats.vport.rx_out_of_buffer;
2367 case IFCOUNTER_OPACKETS:
2368 retval = priv->stats.vport.tx_packets;
2370 case IFCOUNTER_OERRORS:
2371 retval = priv->stats.vport.tx_error_packets;
2373 case IFCOUNTER_IBYTES:
2374 retval = priv->stats.vport.rx_bytes;
2376 case IFCOUNTER_OBYTES:
2377 retval = priv->stats.vport.tx_bytes;
2379 case IFCOUNTER_IMCASTS:
2380 retval = priv->stats.vport.rx_multicast_packets;
2382 case IFCOUNTER_OMCASTS:
2383 retval = priv->stats.vport.tx_multicast_packets;
2385 case IFCOUNTER_OQDROPS:
2386 retval = priv->stats.vport.tx_queue_dropped;
2389 retval = if_get_counter_default(ifp, cnt);
2392 /* PRIV_UNLOCK(priv); XXX not allowed */
2398 mlx5e_set_rx_mode(struct ifnet *ifp)
2400 struct mlx5e_priv *priv = ifp->if_softc;
2402 schedule_work(&priv->set_rx_mode_work);
2406 mlx5e_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
2408 struct mlx5e_priv *priv;
2410 struct ifi2creq i2c;
2418 priv = ifp->if_softc;
2420 /* check if detaching */
2421 if (priv == NULL || priv->gone != 0)
2426 ifr = (struct ifreq *)data;
2429 mlx5_query_port_max_mtu(priv->mdev, &max_mtu);
2431 if (ifr->ifr_mtu >= MLX5E_MTU_MIN &&
2432 ifr->ifr_mtu <= MIN(MLX5E_MTU_MAX, max_mtu)) {
2435 was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
2437 mlx5e_close_locked(ifp);
2440 mlx5e_set_dev_port_mtu(ifp, ifr->ifr_mtu);
2443 mlx5e_open_locked(ifp);
2446 if_printf(ifp, "Invalid MTU value. Min val: %d, Max val: %d\n",
2447 MLX5E_MTU_MIN, MIN(MLX5E_MTU_MAX, max_mtu));
2452 if ((ifp->if_flags & IFF_UP) &&
2453 (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
2454 mlx5e_set_rx_mode(ifp);
2458 if (ifp->if_flags & IFF_UP) {
2459 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
2460 if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
2461 mlx5e_open_locked(ifp);
2462 ifp->if_drv_flags |= IFF_DRV_RUNNING;
2463 mlx5_set_port_status(priv->mdev, MLX5_PORT_UP);
2466 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2467 mlx5_set_port_status(priv->mdev,
2469 if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
2470 mlx5e_close_locked(ifp);
2471 mlx5e_update_carrier(priv);
2472 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2479 mlx5e_set_rx_mode(ifp);
2484 ifr = (struct ifreq *)data;
2485 error = ifmedia_ioctl(ifp, ifr, &priv->media, command);
2488 ifr = (struct ifreq *)data;
2490 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
2492 if (mask & IFCAP_TXCSUM) {
2493 ifp->if_capenable ^= IFCAP_TXCSUM;
2494 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
2496 if (IFCAP_TSO4 & ifp->if_capenable &&
2497 !(IFCAP_TXCSUM & ifp->if_capenable)) {
2498 ifp->if_capenable &= ~IFCAP_TSO4;
2499 ifp->if_hwassist &= ~CSUM_IP_TSO;
2501 "tso4 disabled due to -txcsum.\n");
2504 if (mask & IFCAP_TXCSUM_IPV6) {
2505 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
2506 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
2508 if (IFCAP_TSO6 & ifp->if_capenable &&
2509 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
2510 ifp->if_capenable &= ~IFCAP_TSO6;
2511 ifp->if_hwassist &= ~CSUM_IP6_TSO;
2513 "tso6 disabled due to -txcsum6.\n");
2516 if (mask & IFCAP_RXCSUM)
2517 ifp->if_capenable ^= IFCAP_RXCSUM;
2518 if (mask & IFCAP_RXCSUM_IPV6)
2519 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
2520 if (mask & IFCAP_TSO4) {
2521 if (!(IFCAP_TSO4 & ifp->if_capenable) &&
2522 !(IFCAP_TXCSUM & ifp->if_capenable)) {
2523 if_printf(ifp, "enable txcsum first.\n");
2527 ifp->if_capenable ^= IFCAP_TSO4;
2528 ifp->if_hwassist ^= CSUM_IP_TSO;
2530 if (mask & IFCAP_TSO6) {
2531 if (!(IFCAP_TSO6 & ifp->if_capenable) &&
2532 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
2533 if_printf(ifp, "enable txcsum6 first.\n");
2537 ifp->if_capenable ^= IFCAP_TSO6;
2538 ifp->if_hwassist ^= CSUM_IP6_TSO;
2540 if (mask & IFCAP_VLAN_HWFILTER) {
2541 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
2542 mlx5e_disable_vlan_filter(priv);
2544 mlx5e_enable_vlan_filter(priv);
2546 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
2548 if (mask & IFCAP_VLAN_HWTAGGING)
2549 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
2550 if (mask & IFCAP_WOL_MAGIC)
2551 ifp->if_capenable ^= IFCAP_WOL_MAGIC;
2553 VLAN_CAPABILITIES(ifp);
2554 /* turn off LRO means also turn of HW LRO - if it's on */
2555 if (mask & IFCAP_LRO) {
2556 int was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
2557 bool need_restart = false;
2559 ifp->if_capenable ^= IFCAP_LRO;
2560 if (!(ifp->if_capenable & IFCAP_LRO)) {
2561 if (priv->params.hw_lro_en) {
2562 priv->params.hw_lro_en = false;
2563 need_restart = true;
2564 /* Not sure this is the correct way */
2565 priv->params_ethtool.hw_lro = priv->params.hw_lro_en;
2568 if (was_opened && need_restart) {
2569 mlx5e_close_locked(ifp);
2570 mlx5e_open_locked(ifp);
2578 ifr = (struct ifreq *)data;
2581 * Copy from the user-space address ifr_data to the
2582 * kernel-space address i2c
2584 error = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
2588 if (i2c.len > sizeof(i2c.data)) {
2594 /* Get module_num which is required for the query_eeprom */
2595 error = mlx5_query_module_num(priv->mdev, &module_num);
2597 if_printf(ifp, "Query module num failed, eeprom "
2598 "reading is not supported\n");
2602 /* Check if module is present before doing an access */
2603 if (mlx5_query_module_status(priv->mdev, module_num) !=
2604 MLX5_MODULE_STATUS_PLUGGED) {
2609 * Currently 0XA0 and 0xA2 are the only addresses permitted.
2610 * The internal conversion is as follows:
2612 if (i2c.dev_addr == 0xA0)
2613 read_addr = MLX5E_I2C_ADDR_LOW;
2614 else if (i2c.dev_addr == 0xA2)
2615 read_addr = MLX5E_I2C_ADDR_HIGH;
2617 if_printf(ifp, "Query eeprom failed, "
2618 "Invalid Address: %X\n", i2c.dev_addr);
2622 error = mlx5_query_eeprom(priv->mdev,
2623 read_addr, MLX5E_EEPROM_LOW_PAGE,
2624 (uint32_t)i2c.offset, (uint32_t)i2c.len, module_num,
2625 (uint32_t *)i2c.data, &size_read);
2627 if_printf(ifp, "Query eeprom failed, eeprom "
2628 "reading is not supported\n");
2633 if (i2c.len > MLX5_EEPROM_MAX_BYTES) {
2634 error = mlx5_query_eeprom(priv->mdev,
2635 read_addr, MLX5E_EEPROM_LOW_PAGE,
2636 (uint32_t)(i2c.offset + size_read),
2637 (uint32_t)(i2c.len - size_read), module_num,
2638 (uint32_t *)(i2c.data + size_read), &size_read);
2641 if_printf(ifp, "Query eeprom failed, eeprom "
2642 "reading is not supported\n");
2647 error = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
2653 error = ether_ioctl(ifp, command, data);
2660 mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev)
2663 * TODO: uncoment once FW really sets all these bits if
2664 * (!mdev->caps.eth.rss_ind_tbl_cap || !mdev->caps.eth.csum_cap ||
2665 * !mdev->caps.eth.max_lso_cap || !mdev->caps.eth.vlan_cap ||
2666 * !(mdev->caps.gen.flags & MLX5_DEV_CAP_FLAG_SCQE_BRK_MOD)) return
2670 /* TODO: add more must-to-have features */
2672 if (MLX5_CAP_GEN(mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
2679 mlx5e_build_ifp_priv(struct mlx5_core_dev *mdev,
2680 struct mlx5e_priv *priv,
2681 int num_comp_vectors)
2684 * TODO: Consider link speed for setting "log_sq_size",
2685 * "log_rq_size" and "cq_moderation_xxx":
2687 priv->params.log_sq_size =
2688 MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
2689 priv->params.log_rq_size =
2690 MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
2691 priv->params.rx_cq_moderation_usec =
2692 MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
2693 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE :
2694 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
2695 priv->params.rx_cq_moderation_mode =
2696 MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? 1 : 0;
2697 priv->params.rx_cq_moderation_pkts =
2698 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
2699 priv->params.tx_cq_moderation_usec =
2700 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
2701 priv->params.tx_cq_moderation_pkts =
2702 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
2703 priv->params.min_rx_wqes =
2704 MLX5E_PARAMS_DEFAULT_MIN_RX_WQES;
2705 priv->params.rx_hash_log_tbl_sz =
2706 (order_base_2(num_comp_vectors) >
2707 MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ) ?
2708 order_base_2(num_comp_vectors) :
2709 MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ;
2710 priv->params.num_tc = 1;
2711 priv->params.default_vlan_prio = 0;
2712 priv->counter_set_id = -1;
2715 * hw lro is currently defaulted to off. when it won't anymore we
2716 * will consider the HW capability: "!!MLX5_CAP_ETH(mdev, lro_cap)"
2718 priv->params.hw_lro_en = false;
2719 priv->params.lro_wqe_sz = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ;
2721 priv->params.cqe_zipping_en = !!MLX5_CAP_GEN(mdev, cqe_compression);
2724 priv->params.num_channels = num_comp_vectors;
2725 priv->order_base_2_num_channels = order_base_2(num_comp_vectors);
2726 priv->queue_mapping_channel_mask =
2727 roundup_pow_of_two(num_comp_vectors) - 1;
2728 priv->num_tc = priv->params.num_tc;
2729 priv->default_vlan_prio = priv->params.default_vlan_prio;
2731 INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
2732 INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
2733 INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
2737 mlx5e_create_mkey(struct mlx5e_priv *priv, u32 pdn,
2738 struct mlx5_core_mr *mr)
2740 struct ifnet *ifp = priv->ifp;
2741 struct mlx5_core_dev *mdev = priv->mdev;
2742 struct mlx5_create_mkey_mbox_in *in;
2745 in = mlx5_vzalloc(sizeof(*in));
2747 if_printf(ifp, "%s: failed to allocate inbox\n", __func__);
2750 in->seg.flags = MLX5_PERM_LOCAL_WRITE |
2751 MLX5_PERM_LOCAL_READ |
2752 MLX5_ACCESS_MODE_PA;
2753 in->seg.flags_pd = cpu_to_be32(pdn | MLX5_MKEY_LEN64);
2754 in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
2756 err = mlx5_core_create_mkey(mdev, mr, in, sizeof(*in), NULL, NULL,
2759 if_printf(ifp, "%s: mlx5_core_create_mkey failed, %d\n",
2767 static const char *mlx5e_vport_stats_desc[] = {
2768 MLX5E_VPORT_STATS(MLX5E_STATS_DESC)
2771 static const char *mlx5e_pport_stats_desc[] = {
2772 MLX5E_PPORT_STATS(MLX5E_STATS_DESC)
2776 mlx5e_priv_mtx_init(struct mlx5e_priv *priv)
2778 mtx_init(&priv->async_events_mtx, "mlx5async", MTX_NETWORK_LOCK, MTX_DEF);
2779 sx_init(&priv->state_lock, "mlx5state");
2780 callout_init_mtx(&priv->watchdog, &priv->async_events_mtx, 0);
2781 MLX5_INIT_DOORBELL_LOCK(&priv->doorbell_lock);
2785 mlx5e_priv_mtx_destroy(struct mlx5e_priv *priv)
2787 mtx_destroy(&priv->async_events_mtx);
2788 sx_destroy(&priv->state_lock);
2792 sysctl_firmware(SYSCTL_HANDLER_ARGS)
2795 * %d.%d%.d the string format.
2796 * fw_rev_{maj,min,sub} return u16, 2^16 = 65536.
2797 * We need at most 5 chars to store that.
2798 * It also has: two "." and NULL at the end, which means we need 18
2799 * (5*3 + 3) chars at most.
2802 struct mlx5e_priv *priv = arg1;
2805 snprintf(fw, sizeof(fw), "%d.%d.%d", fw_rev_maj(priv->mdev), fw_rev_min(priv->mdev),
2806 fw_rev_sub(priv->mdev));
2807 error = sysctl_handle_string(oidp, fw, sizeof(fw), req);
2812 mlx5e_add_hw_stats(struct mlx5e_priv *priv)
2814 SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw),
2815 OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD, priv, 0,
2816 sysctl_firmware, "A", "HCA firmware version");
2818 SYSCTL_ADD_STRING(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw),
2819 OID_AUTO, "board_id", CTLFLAG_RD, priv->mdev->board_id, 0,
2824 mlx5e_setup_pauseframes(struct mlx5e_priv *priv)
2826 #if (__FreeBSD_version < 1100000)
2830 /* Only receiving pauseframes is enabled by default */
2831 priv->params.tx_pauseframe_control = 0;
2832 priv->params.rx_pauseframe_control = 1;
2834 #if (__FreeBSD_version < 1100000)
2835 /* compute path for sysctl */
2836 snprintf(path, sizeof(path), "dev.mce.%d.tx_pauseframe_control",
2837 device_get_unit(priv->mdev->pdev->dev.bsddev));
2839 /* try to fetch tunable, if any */
2840 TUNABLE_INT_FETCH(path, &priv->params.tx_pauseframe_control);
2842 /* compute path for sysctl */
2843 snprintf(path, sizeof(path), "dev.mce.%d.rx_pauseframe_control",
2844 device_get_unit(priv->mdev->pdev->dev.bsddev));
2846 /* try to fetch tunable, if any */
2847 TUNABLE_INT_FETCH(path, &priv->params.rx_pauseframe_control);
2850 /* register pausframe SYSCTLs */
2851 SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
2852 OID_AUTO, "tx_pauseframe_control", CTLFLAG_RDTUN,
2853 &priv->params.tx_pauseframe_control, 0,
2854 "Set to enable TX pause frames. Clear to disable.");
2856 SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
2857 OID_AUTO, "rx_pauseframe_control", CTLFLAG_RDTUN,
2858 &priv->params.rx_pauseframe_control, 0,
2859 "Set to enable RX pause frames. Clear to disable.");
2862 priv->params.tx_pauseframe_control =
2863 priv->params.tx_pauseframe_control ? 1 : 0;
2864 priv->params.rx_pauseframe_control =
2865 priv->params.rx_pauseframe_control ? 1 : 0;
2867 /* update firmware */
2868 mlx5_set_port_pause(priv->mdev, 1,
2869 priv->params.rx_pauseframe_control,
2870 priv->params.tx_pauseframe_control);
2874 mlx5e_create_ifp(struct mlx5_core_dev *mdev)
2876 static volatile int mlx5_en_unit;
2878 struct mlx5e_priv *priv;
2879 u8 dev_addr[ETHER_ADDR_LEN] __aligned(4);
2880 struct sysctl_oid_list *child;
2881 int ncv = mdev->priv.eq_table.num_comp_vectors;
2887 if (mlx5e_check_required_hca_cap(mdev)) {
2888 mlx5_core_dbg(mdev, "mlx5e_check_required_hca_cap() failed\n");
2891 priv = malloc(sizeof(*priv), M_MLX5EN, M_WAITOK | M_ZERO);
2892 mlx5e_priv_mtx_init(priv);
2894 ifp = priv->ifp = if_alloc(IFT_ETHER);
2896 mlx5_core_err(mdev, "if_alloc() failed\n");
2899 ifp->if_softc = priv;
2900 if_initname(ifp, "mce", atomic_fetchadd_int(&mlx5_en_unit, 1));
2901 ifp->if_mtu = ETHERMTU;
2902 ifp->if_init = mlx5e_open;
2903 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2904 ifp->if_ioctl = mlx5e_ioctl;
2905 ifp->if_transmit = mlx5e_xmit;
2906 ifp->if_qflush = if_qflush;
2907 #if (__FreeBSD_version >= 1100000)
2908 ifp->if_get_counter = mlx5e_get_counter;
2910 ifp->if_snd.ifq_maxlen = ifqmaxlen;
2912 * Set driver features
2914 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6;
2915 ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING;
2916 ifp->if_capabilities |= IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWFILTER;
2917 ifp->if_capabilities |= IFCAP_LINKSTATE | IFCAP_JUMBO_MTU;
2918 ifp->if_capabilities |= IFCAP_LRO;
2919 ifp->if_capabilities |= IFCAP_TSO | IFCAP_VLAN_HWTSO;
2920 ifp->if_capabilities |= IFCAP_HWSTATS;
2922 /* set TSO limits so that we don't have to drop TX packets */
2923 ifp->if_hw_tsomax = MLX5E_MAX_TX_PAYLOAD_SIZE - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
2924 ifp->if_hw_tsomaxsegcount = MLX5E_MAX_TX_MBUF_FRAGS - 1 /* hdr */;
2925 ifp->if_hw_tsomaxsegsize = MLX5E_MAX_TX_MBUF_SIZE;
2927 ifp->if_capenable = ifp->if_capabilities;
2928 ifp->if_hwassist = 0;
2929 if (ifp->if_capenable & IFCAP_TSO)
2930 ifp->if_hwassist |= CSUM_TSO;
2931 if (ifp->if_capenable & IFCAP_TXCSUM)
2932 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP);
2933 if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
2934 ifp->if_hwassist |= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
2936 /* ifnet sysctl tree */
2937 sysctl_ctx_init(&priv->sysctl_ctx);
2938 priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_STATIC_CHILDREN(_dev),
2939 OID_AUTO, ifp->if_dname, CTLFLAG_RD, 0, "MLX5 ethernet - interface name");
2940 if (priv->sysctl_ifnet == NULL) {
2941 mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
2942 goto err_free_sysctl;
2944 snprintf(unit, sizeof(unit), "%d", ifp->if_dunit);
2945 priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
2946 OID_AUTO, unit, CTLFLAG_RD, 0, "MLX5 ethernet - interface unit");
2947 if (priv->sysctl_ifnet == NULL) {
2948 mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
2949 goto err_free_sysctl;
2952 /* HW sysctl tree */
2953 child = SYSCTL_CHILDREN(device_get_sysctl_tree(mdev->pdev->dev.bsddev));
2954 priv->sysctl_hw = SYSCTL_ADD_NODE(&priv->sysctl_ctx, child,
2955 OID_AUTO, "hw", CTLFLAG_RD, 0, "MLX5 ethernet dev hw");
2956 if (priv->sysctl_hw == NULL) {
2957 mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
2958 goto err_free_sysctl;
2960 mlx5e_build_ifp_priv(mdev, priv, ncv);
2961 err = mlx5_alloc_map_uar(mdev, &priv->cq_uar);
2963 if_printf(ifp, "%s: mlx5_alloc_map_uar failed, %d\n",
2965 goto err_free_sysctl;
2967 err = mlx5_core_alloc_pd(mdev, &priv->pdn);
2969 if_printf(ifp, "%s: mlx5_core_alloc_pd failed, %d\n",
2971 goto err_unmap_free_uar;
2973 err = mlx5_alloc_transport_domain(mdev, &priv->tdn);
2975 if_printf(ifp, "%s: mlx5_alloc_transport_domain failed, %d\n",
2977 goto err_dealloc_pd;
2979 err = mlx5e_create_mkey(priv, priv->pdn, &priv->mr);
2981 if_printf(ifp, "%s: mlx5e_create_mkey failed, %d\n",
2983 goto err_dealloc_transport_domain;
2985 mlx5_query_nic_vport_mac_address(priv->mdev, 0, dev_addr);
2987 /* check if we should generate a random MAC address */
2988 if (MLX5_CAP_GEN(priv->mdev, vport_group_manager) == 0 &&
2989 is_zero_ether_addr(dev_addr)) {
2990 random_ether_addr(dev_addr);
2991 if_printf(ifp, "Assigned random MAC address\n");
2994 /* set default MTU */
2995 mlx5e_set_dev_port_mtu(ifp, ifp->if_mtu);
2998 device_set_desc(mdev->pdev->dev.bsddev, mlx5e_version);
3000 /* Set default media status */
3001 priv->media_status_last = IFM_AVALID;
3002 priv->media_active_last = IFM_ETHER | IFM_AUTO |
3003 IFM_ETH_RXPAUSE | IFM_FDX;
3005 /* setup default pauseframes configuration */
3006 mlx5e_setup_pauseframes(priv);
3008 err = mlx5_query_port_proto_cap(mdev, ð_proto_cap, MLX5_PTYS_EN);
3011 if_printf(ifp, "%s: Query port media capability failed, %d\n",
3015 /* Setup supported medias */
3016 ifmedia_init(&priv->media, IFM_IMASK | IFM_ETH_FMASK,
3017 mlx5e_media_change, mlx5e_media_status);
3019 for (i = 0; i < MLX5E_LINK_MODES_NUMBER; ++i) {
3020 if (mlx5e_mode_table[i].baudrate == 0)
3022 if (MLX5E_PROT_MASK(i) & eth_proto_cap) {
3023 ifmedia_add(&priv->media,
3024 mlx5e_mode_table[i].subtype |
3025 IFM_ETHER, 0, NULL);
3026 ifmedia_add(&priv->media,
3027 mlx5e_mode_table[i].subtype |
3028 IFM_ETHER | IFM_FDX |
3029 IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE, 0, NULL);
3033 ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO, 0, NULL);
3034 ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO | IFM_FDX |
3035 IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE, 0, NULL);
3037 /* Set autoselect by default */
3038 ifmedia_set(&priv->media, IFM_ETHER | IFM_AUTO | IFM_FDX |
3039 IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE);
3040 ether_ifattach(ifp, dev_addr);
3042 /* Register for VLAN events */
3043 priv->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
3044 mlx5e_vlan_rx_add_vid, priv, EVENTHANDLER_PRI_FIRST);
3045 priv->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
3046 mlx5e_vlan_rx_kill_vid, priv, EVENTHANDLER_PRI_FIRST);
3048 /* Link is down by default */
3049 if_link_state_change(ifp, LINK_STATE_DOWN);
3051 mlx5e_enable_async_events(priv);
3053 mlx5e_add_hw_stats(priv);
3055 mlx5e_create_stats(&priv->stats.vport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
3056 "vstats", mlx5e_vport_stats_desc, MLX5E_VPORT_STATS_NUM,
3057 priv->stats.vport.arg);
3059 mlx5e_create_stats(&priv->stats.pport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
3060 "pstats", mlx5e_pport_stats_desc, MLX5E_PPORT_STATS_NUM,
3061 priv->stats.pport.arg);
3063 mlx5e_create_ethtool(priv);
3065 mtx_lock(&priv->async_events_mtx);
3066 mlx5e_update_stats(priv);
3067 mtx_unlock(&priv->async_events_mtx);
3071 err_dealloc_transport_domain:
3072 mlx5_dealloc_transport_domain(mdev, priv->tdn);
3075 mlx5_core_dealloc_pd(mdev, priv->pdn);
3078 mlx5_unmap_free_uar(mdev, &priv->cq_uar);
3081 sysctl_ctx_free(&priv->sysctl_ctx);
3086 mlx5e_priv_mtx_destroy(priv);
3087 free(priv, M_MLX5EN);
3092 mlx5e_destroy_ifp(struct mlx5_core_dev *mdev, void *vpriv)
3094 struct mlx5e_priv *priv = vpriv;
3095 struct ifnet *ifp = priv->ifp;
3097 /* don't allow more IOCTLs */
3101 * Clear the device description to avoid use after free,
3102 * because the bsddev is not destroyed when this module is
3105 device_set_desc(mdev->pdev->dev.bsddev, NULL);
3107 /* XXX wait a bit to allow IOCTL handlers to complete */
3110 /* stop watchdog timer */
3111 callout_drain(&priv->watchdog);
3113 if (priv->vlan_attach != NULL)
3114 EVENTHANDLER_DEREGISTER(vlan_config, priv->vlan_attach);
3115 if (priv->vlan_detach != NULL)
3116 EVENTHANDLER_DEREGISTER(vlan_unconfig, priv->vlan_detach);
3118 /* make sure device gets closed */
3120 mlx5e_close_locked(ifp);
3123 /* unregister device */
3124 ifmedia_removeall(&priv->media);
3125 ether_ifdetach(ifp);
3128 /* destroy all remaining sysctl nodes */
3129 if (priv->sysctl_debug)
3130 sysctl_ctx_free(&priv->stats.port_stats_debug.ctx);
3131 sysctl_ctx_free(&priv->stats.vport.ctx);
3132 sysctl_ctx_free(&priv->stats.pport.ctx);
3133 sysctl_ctx_free(&priv->sysctl_ctx);
3135 mlx5_core_destroy_mkey(priv->mdev, &priv->mr);
3136 mlx5_dealloc_transport_domain(priv->mdev, priv->tdn);
3137 mlx5_core_dealloc_pd(priv->mdev, priv->pdn);
3138 mlx5_unmap_free_uar(priv->mdev, &priv->cq_uar);
3139 mlx5e_disable_async_events(priv);
3140 flush_scheduled_work();
3141 mlx5e_priv_mtx_destroy(priv);
3142 free(priv, M_MLX5EN);
3146 mlx5e_get_ifp(void *vpriv)
3148 struct mlx5e_priv *priv = vpriv;
3153 static struct mlx5_interface mlx5e_interface = {
3154 .add = mlx5e_create_ifp,
3155 .remove = mlx5e_destroy_ifp,
3156 .event = mlx5e_async_event,
3157 .protocol = MLX5_INTERFACE_PROTOCOL_ETH,
3158 .get_dev = mlx5e_get_ifp,
3164 mlx5_register_interface(&mlx5e_interface);
3170 mlx5_unregister_interface(&mlx5e_interface);
3173 module_init_order(mlx5e_init, SI_ORDER_THIRD);
3174 module_exit_order(mlx5e_cleanup, SI_ORDER_THIRD);
3176 #if (__FreeBSD_version >= 1100000)
3177 MODULE_DEPEND(mlx5en, linuxkpi, 1, 1, 1);
3179 MODULE_DEPEND(mlx5en, mlx5, 1, 1, 1);
3180 MODULE_VERSION(mlx5en, 1);