From 4f425342ac541fc11a8106250b72851033c836f6 Mon Sep 17 00:00:00 2001 From: hselasky Date: Tue, 14 Aug 2018 11:19:04 +0000 Subject: [PATCH] MFC r336450: Do not inline transmit headers and use HW VLAN tagging if supported by mlx5en(4). Query the minimal inline mode supported by the card. When creating a send queue, cache the queried mode and optimize the transmit if no inlining is required. In this case, we can avoid touching the headers cache line and avoid dirtying several more lines by copying headers into the send WQEs. Also, if no inline headers are used, hardware assists in the VLAN tag framing. Submitted by: kib@, slavash@ Sponsored by: Mellanox Technologies git-svn-id: svn://svn.freebsd.org/base/stable/10@337742 ccf9f872-aa2e-dd11-9fc8-001c23d0bc1f --- sys/dev/mlx5/device.h | 7 ++ sys/dev/mlx5/mlx5_core/mlx5_vport.c | 52 +++++++++++++++ sys/dev/mlx5/mlx5_en/en.h | 6 ++ sys/dev/mlx5/mlx5_en/mlx5_en_main.c | 29 ++++++++ sys/dev/mlx5/mlx5_en/mlx5_en_tx.c | 100 ++++++++++++++++++++-------- sys/dev/mlx5/qp.h | 12 +++- sys/dev/mlx5/vport.h | 12 ++++ 7 files changed, 189 insertions(+), 29 deletions(-) diff --git a/sys/dev/mlx5/device.h b/sys/dev/mlx5/device.h index 47b263ad7..19b4a4c60 100644 --- a/sys/dev/mlx5/device.h +++ b/sys/dev/mlx5/device.h @@ -1311,6 +1311,13 @@ enum { MLX5_CMD_HCA_CAP_MIN_WQE_INLINE_MODE_NOT_REQUIRED = 0x2 }; +enum mlx5_inline_modes { + MLX5_INLINE_MODE_NONE, + MLX5_INLINE_MODE_L2, + MLX5_INLINE_MODE_IP, + MLX5_INLINE_MODE_TCP_UDP, +}; + enum { MLX5_QUERY_VPORT_STATE_OUT_STATE_FOLLOW = 0x2, }; diff --git a/sys/dev/mlx5/mlx5_core/mlx5_vport.c b/sys/dev/mlx5/mlx5_core/mlx5_vport.c index 0e83332ec..43ad814ef 100644 --- a/sys/dev/mlx5/mlx5_core/mlx5_vport.c +++ b/sys/dev/mlx5/mlx5_core/mlx5_vport.c @@ -228,6 +228,58 @@ int mlx5_vport_query_out_of_rx_buffer(struct mlx5_core_dev *mdev, return err; } +int mlx5_query_nic_vport_min_inline(struct mlx5_core_dev *mdev, + u16 vport, u8 *min_inline) +{ + u32 out[MLX5_ST_SZ_DW(query_nic_vport_context_out)] = {0}; + int err; + + err = mlx5_query_nic_vport_context(mdev, vport, out, sizeof(out)); + if (!err) + *min_inline = MLX5_GET(query_nic_vport_context_out, out, + nic_vport_context.min_wqe_inline_mode); + return err; +} +EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_min_inline); + +void mlx5_query_min_inline(struct mlx5_core_dev *mdev, + u8 *min_inline_mode) +{ + switch (MLX5_CAP_ETH(mdev, wqe_inline_mode)) { + case MLX5_CAP_INLINE_MODE_L2: + *min_inline_mode = MLX5_INLINE_MODE_L2; + break; + case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: + mlx5_query_nic_vport_min_inline(mdev, 0, min_inline_mode); + break; + case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: + *min_inline_mode = MLX5_INLINE_MODE_NONE; + break; + } +} +EXPORT_SYMBOL_GPL(mlx5_query_min_inline); + +int mlx5_modify_nic_vport_min_inline(struct mlx5_core_dev *mdev, + u16 vport, u8 min_inline) +{ + u32 in[MLX5_ST_SZ_DW(modify_nic_vport_context_in)] = {0}; + int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in); + void *nic_vport_ctx; + + MLX5_SET(modify_nic_vport_context_in, in, + field_select.min_wqe_inline_mode, 1); + MLX5_SET(modify_nic_vport_context_in, in, vport_number, vport); + MLX5_SET(modify_nic_vport_context_in, in, other_vport, 1); + + nic_vport_ctx = MLX5_ADDR_OF(modify_nic_vport_context_in, + in, nic_vport_context); + MLX5_SET(nic_vport_context, nic_vport_ctx, + min_wqe_inline_mode, min_inline); + + return mlx5_modify_nic_vport_context(mdev, in, inlen); +} +EXPORT_SYMBOL_GPL(mlx5_modify_nic_vport_min_inline); + int mlx5_query_nic_vport_mac_address(struct mlx5_core_dev *mdev, u16 vport, u8 *addr) { diff --git a/sys/dev/mlx5/mlx5_en/en.h b/sys/dev/mlx5/mlx5_en/en.h index 955d8f977..39134a78b 100644 --- a/sys/dev/mlx5/mlx5_en/en.h +++ b/sys/dev/mlx5/mlx5_en/en.h @@ -404,6 +404,8 @@ struct mlx5e_params { u16 rx_hash_log_tbl_sz; u32 tx_pauseframe_control; u32 rx_pauseframe_control; + u16 tx_max_inline; + u8 tx_min_inline_mode; }; #define MLX5E_PARAMS(m) \ @@ -560,6 +562,9 @@ struct mlx5e_sq { u32 sqn; u32 bf_buf_size; u32 mkey_be; + u16 max_inline; + u8 min_inline_mode; + u8 vlan_inline_cap; /* control path */ struct mlx5_wq_ctrl wq_ctrl; @@ -851,5 +856,6 @@ int mlx5e_enable_sq(struct mlx5e_sq *, struct mlx5e_sq_param *, int tis_num); int mlx5e_modify_sq(struct mlx5e_sq *, int curr_state, int next_state); void mlx5e_disable_sq(struct mlx5e_sq *); void mlx5e_drain_sq(struct mlx5e_sq *); +u8 mlx5e_params_calculate_tx_min_inline(struct mlx5_core_dev *mdev); #endif /* _MLX5_EN_H_ */ diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c index 95fae4401..39d4453f4 100644 --- a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c @@ -1022,6 +1022,9 @@ mlx5e_create_sq(struct mlx5e_channel *c, sq->ifp = priv->ifp; sq->priv = priv; sq->tc = tc; + sq->max_inline = priv->params.tx_max_inline; + sq->min_inline_mode = priv->params.tx_min_inline_mode; + sq->vlan_inline_cap = MLX5_CAP_ETH(mdev, wqe_vlan_insert); /* check if we should allocate a second packet buffer */ if (priv->params_ethtool.tx_bufring_disable == 0) { @@ -2840,6 +2843,16 @@ mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev) return (0); } +static u16 +mlx5e_get_max_inline_cap(struct mlx5_core_dev *mdev) +{ + int bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2; + + return bf_buf_size - + sizeof(struct mlx5e_tx_wqe) + + 2 /*sizeof(mlx5e_tx_wqe.inline_hdr_start)*/; +} + static void mlx5e_build_ifp_priv(struct mlx5_core_dev *mdev, struct mlx5e_priv *priv, @@ -2875,6 +2888,8 @@ mlx5e_build_ifp_priv(struct mlx5_core_dev *mdev, priv->params.num_tc = 1; priv->params.default_vlan_prio = 0; priv->counter_set_id = -1; + priv->params.tx_max_inline = mlx5e_get_max_inline_cap(mdev); + mlx5_query_min_inline(mdev, &priv->params.tx_min_inline_mode); /* * hw lro is currently defaulted to off. when it won't anymore we @@ -2973,6 +2988,20 @@ sysctl_firmware(SYSCTL_HANDLER_ARGS) return (error); } +u8 +mlx5e_params_calculate_tx_min_inline(struct mlx5_core_dev *mdev) +{ + u8 min_inline_mode; + + min_inline_mode = MLX5_INLINE_MODE_L2; + mlx5_query_min_inline(mdev, &min_inline_mode); + if (min_inline_mode == MLX5_INLINE_MODE_NONE && + !MLX5_CAP_ETH(mdev, wqe_vlan_insert)) + min_inline_mode = MLX5_INLINE_MODE_L2; + + return (min_inline_mode); +} + static void mlx5e_add_hw_stats(struct mlx5e_priv *priv) { diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c index 6f20bd1fc..9c7e83f75 100644 --- a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c @@ -137,7 +137,43 @@ mlx5e_select_queue(struct ifnet *ifp, struct mbuf *mb) static inline u16 mlx5e_get_inline_hdr_size(struct mlx5e_sq *sq, struct mbuf *mb) { - return (MIN(MLX5E_MAX_TX_INLINE, mb->m_len)); + + switch(sq->min_inline_mode) { + case MLX5_INLINE_MODE_NONE: + /* + * When inline mode is NONE, we do not need to copy + * headers into WQEs, except when vlan tag framing is + * requested. Hardware might offload vlan tagging on + * transmit. This is a separate capability, which is + * known to be disabled on ConnectX-5 due to a hardware + * bug RM 931383. If vlan_inline_cap is not present and + * the packet has vlan tag, fall back to inlining. + */ + if ((mb->m_flags & M_VLANTAG) != 0 && + sq->vlan_inline_cap == 0) + break; + return (0); + case MLX5_INLINE_MODE_L2: + /* + * Due to hardware limitations, when trust mode is + * DSCP, the hardware may request MLX5_INLINE_MODE_L2 + * while it really needs all L2 headers and the 4 first + * bytes of the IP header (which include the + * TOS/traffic-class). + * + * To avoid doing a firmware command for querying the + * trust state and parsing the mbuf for doing + * unnecessary checks (VLAN/eth_type) in the fast path, + * we are going for the worth case (22 Bytes) if + * the mb->m_pkthdr.len allows it. + */ + if (mb->m_pkthdr.len > ETHER_HDR_LEN + + ETHER_VLAN_ENCAP_LEN + 4) + return (MIN(sq->max_inline, ETHER_HDR_LEN + + ETHER_VLAN_ENCAP_LEN + 4)); + break; + } + return (MIN(sq->max_inline, mb->m_pkthdr.len)); } static int @@ -275,37 +311,47 @@ mlx5e_sq_xmit(struct mlx5e_sq *sq, struct mbuf **mbp) sq->mbuf[pi].num_bytes = max_t (unsigned int, mb->m_pkthdr.len, ETHER_MIN_LEN - ETHER_CRC_LEN); } - if (mb->m_flags & M_VLANTAG) { - struct ether_vlan_header *eh = - (struct ether_vlan_header *)wqe->eth.inline_hdr_start; - - /* Range checks */ - if (ihs > (MLX5E_MAX_TX_INLINE - ETHER_VLAN_ENCAP_LEN)) - ihs = (MLX5E_MAX_TX_INLINE - ETHER_VLAN_ENCAP_LEN); - else if (ihs < ETHER_HDR_LEN) { - err = EINVAL; - goto tx_drop; + if (ihs == 0) { + if ((mb->m_flags & M_VLANTAG) != 0) { + wqe->eth.vlan_cmd = htons(0x8000); /* bit 0 CVLAN */ + wqe->eth.vlan_hdr = htons(mb->m_pkthdr.ether_vtag); + } else { + wqe->eth.inline_hdr_sz = 0; } - m_copydata(mb, 0, ETHER_HDR_LEN, (caddr_t)eh); - m_adj(mb, ETHER_HDR_LEN); - /* Insert 4 bytes VLAN tag into data stream */ - eh->evl_proto = eh->evl_encap_proto; - eh->evl_encap_proto = htons(ETHERTYPE_VLAN); - eh->evl_tag = htons(mb->m_pkthdr.ether_vtag); - /* Copy rest of header data, if any */ - m_copydata(mb, 0, ihs - ETHER_HDR_LEN, (caddr_t)(eh + 1)); - m_adj(mb, ihs - ETHER_HDR_LEN); - /* Extend header by 4 bytes */ - ihs += ETHER_VLAN_ENCAP_LEN; } else { - m_copydata(mb, 0, ihs, wqe->eth.inline_hdr_start); - m_adj(mb, ihs); + if ((mb->m_flags & M_VLANTAG) != 0) { + struct ether_vlan_header *eh = (struct ether_vlan_header + *)wqe->eth.inline_hdr_start; + + /* Range checks */ + if (ihs > (MLX5E_MAX_TX_INLINE - ETHER_VLAN_ENCAP_LEN)) + ihs = (MLX5E_MAX_TX_INLINE - + ETHER_VLAN_ENCAP_LEN); + else if (ihs < ETHER_HDR_LEN) { + err = EINVAL; + goto tx_drop; + } + m_copydata(mb, 0, ETHER_HDR_LEN, (caddr_t)eh); + m_adj(mb, ETHER_HDR_LEN); + /* Insert 4 bytes VLAN tag into data stream */ + eh->evl_proto = eh->evl_encap_proto; + eh->evl_encap_proto = htons(ETHERTYPE_VLAN); + eh->evl_tag = htons(mb->m_pkthdr.ether_vtag); + /* Copy rest of header data, if any */ + m_copydata(mb, 0, ihs - ETHER_HDR_LEN, (caddr_t)(eh + + 1)); + m_adj(mb, ihs - ETHER_HDR_LEN); + /* Extend header by 4 bytes */ + ihs += ETHER_VLAN_ENCAP_LEN; + } else { + m_copydata(mb, 0, ihs, wqe->eth.inline_hdr_start); + m_adj(mb, ihs); + } + wqe->eth.inline_hdr_sz = cpu_to_be16(ihs); } - wqe->eth.inline_hdr_sz = cpu_to_be16(ihs); - ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS; - if (likely(ihs > sizeof(wqe->eth.inline_hdr_start))) { + if (ihs > sizeof(wqe->eth.inline_hdr_start)) { ds_cnt += DIV_ROUND_UP(ihs - sizeof(wqe->eth.inline_hdr_start), MLX5_SEND_WQE_DS); } diff --git a/sys/dev/mlx5/qp.h b/sys/dev/mlx5/qp.h index 82e3edefc..ad88436a4 100644 --- a/sys/dev/mlx5/qp.h +++ b/sys/dev/mlx5/qp.h @@ -236,8 +236,16 @@ struct mlx5_wqe_eth_seg { u8 swp_flags; __be16 mss; __be32 rsvd2; - __be16 inline_hdr_sz; - u8 inline_hdr_start[2]; + union { + struct { + __be16 inline_hdr_sz; + u8 inline_hdr_start[2]; + }; + struct { + __be16 vlan_cmd; + __be16 vlan_hdr; + }; + }; }; struct mlx5_wqe_xrc_seg { diff --git a/sys/dev/mlx5/vport.h b/sys/dev/mlx5/vport.h index edb863580..749c67200 100644 --- a/sys/dev/mlx5/vport.h +++ b/sys/dev/mlx5/vport.h @@ -29,6 +29,13 @@ #define __MLX5_VPORT_H__ #include + +enum { + MLX5_CAP_INLINE_MODE_L2, + MLX5_CAP_INLINE_MODE_VPORT_CONTEXT, + MLX5_CAP_INLINE_MODE_NOT_REQUIRED, +}; + int mlx5_vport_alloc_q_counter(struct mlx5_core_dev *mdev, int client_id, u16 *counter_set_id); int mlx5_vport_dealloc_q_counter(struct mlx5_core_dev *mdev, int client_id, @@ -69,6 +76,11 @@ int mlx5_modify_nic_vport_mac_address(struct mlx5_core_dev *dev, u16 vport, u8 mac[ETH_ALEN]); int mlx5_set_nic_vport_current_mac(struct mlx5_core_dev *mdev, int vport, bool other_vport, u8 *addr); +int mlx5_query_nic_vport_min_inline(struct mlx5_core_dev *mdev, + u16 vport, u8 *min_inline); +void mlx5_query_min_inline(struct mlx5_core_dev *mdev, u8 *min_inline); +int mlx5_modify_nic_vport_min_inline(struct mlx5_core_dev *mdev, + u16 vport, u8 min_inline); int mlx5_modify_nic_vport_port_guid(struct mlx5_core_dev *mdev, u32 vport, u64 port_guid); int mlx5_modify_nic_vport_node_guid(struct mlx5_core_dev *mdev, -- 2.45.0