From 665c2d515beee8a92c4d59d423619074c7431b4a Mon Sep 17 00:00:00 2001 From: hselasky Date: Fri, 11 Dec 2015 14:21:58 +0000 Subject: [PATCH] MFC r290710, r291694, r291699 and r291793: - Fix print formatting compile warnings for Sparc64 and PowerPC platforms. - Updated the mlx4 and mlxen drivers to the latest version, v2.1.6: - Added support for dumping the SFP EEPROM content to dmesg. - Fixed handling of network interface capability IOCTLs. - Fixed race when loading and unloading the mlxen driver by applying appropriate locking. - Removed two unused C-files. - Convert the mlxen driver to use the BUSDMA(9) APIs instead of vtophys() when loading mbufs for transmission and reception. While at it all pointer arithmetic and cast qualifier issues were fixed, mostly related to transmission and reception. - Fix i386 build WITH_OFED=YES. Remove some redundant KASSERTs. Sponsored by: Mellanox Technologies Differential Revision: https://reviews.freebsd.org/D4283 Differential Revision: https://reviews.freebsd.org/D4284 git-svn-id: svn://svn.freebsd.org/base/stable/10@292107 ccf9f872-aa2e-dd11-9fc8-001c23d0bc1f --- sys/modules/mlxen/Makefile | 2 +- sys/ofed/drivers/net/mlx4/en_ethtool.c | 1616 ----------------------- sys/ofed/drivers/net/mlx4/en_main.c | 11 +- sys/ofed/drivers/net/mlx4/en_netdev.c | 276 +++- sys/ofed/drivers/net/mlx4/en_port.c | 1 + sys/ofed/drivers/net/mlx4/en_rx.c | 420 +++--- sys/ofed/drivers/net/mlx4/en_selftest.c | 178 --- sys/ofed/drivers/net/mlx4/en_tx.c | 852 ++++++------ sys/ofed/drivers/net/mlx4/main.c | 88 +- sys/ofed/drivers/net/mlx4/mlx4.h | 10 +- sys/ofed/drivers/net/mlx4/mlx4_en.h | 83 +- sys/ofed/drivers/net/mlx4/mlx4_stats.h | 1 + sys/ofed/drivers/net/mlx4/port.c | 7 +- sys/ofed/include/linux/mlx4/cq.h | 2 +- sys/ofed/include/linux/mlx4/device.h | 7 + sys/ofed/include/linux/mlx4/doorbell.h | 2 +- sys/ofed/include/linux/mlx4/qp.h | 8 + 17 files changed, 1019 insertions(+), 2545 deletions(-) delete mode 100644 sys/ofed/drivers/net/mlx4/en_ethtool.c delete mode 100644 sys/ofed/drivers/net/mlx4/en_selftest.c diff --git a/sys/modules/mlxen/Makefile b/sys/modules/mlxen/Makefile index 9c8c0b3cf..b75e0d03f 100644 --- a/sys/modules/mlxen/Makefile +++ b/sys/modules/mlxen/Makefile @@ -26,4 +26,4 @@ opt_inet6.h: .include -CFLAGS+= -Wno-cast-qual -Wno-pointer-arith ${GCC_MS_EXTENSIONS} +CFLAGS+= ${GCC_MS_EXTENSIONS} diff --git a/sys/ofed/drivers/net/mlx4/en_ethtool.c b/sys/ofed/drivers/net/mlx4/en_ethtool.c deleted file mode 100644 index 3ebeaf4be..000000000 --- a/sys/ofed/drivers/net/mlx4/en_ethtool.c +++ /dev/null @@ -1,1616 +0,0 @@ -/* - * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include -#include -#include -#include -#include -#include - -#include "mlx4_en.h" -#include "en_port.h" - -#define EN_ETHTOOL_QP_ATTACH (1ull << 63) - -union mlx4_ethtool_flow_union { - struct ethtool_tcpip4_spec tcp_ip4_spec; - struct ethtool_tcpip4_spec udp_ip4_spec; - struct ethtool_tcpip4_spec sctp_ip4_spec; - struct ethtool_ah_espip4_spec ah_ip4_spec; - struct ethtool_ah_espip4_spec esp_ip4_spec; - struct ethtool_usrip4_spec usr_ip4_spec; - struct ethhdr ether_spec; - __u8 hdata[52]; -}; - -struct mlx4_ethtool_flow_ext { - __u8 padding[2]; - unsigned char h_dest[ETH_ALEN]; - __be16 vlan_etype; - __be16 vlan_tci; - __be32 data[2]; -}; - -struct mlx4_ethtool_rx_flow_spec { - __u32 flow_type; - union mlx4_ethtool_flow_union h_u; - struct mlx4_ethtool_flow_ext h_ext; - union mlx4_ethtool_flow_union m_u; - struct mlx4_ethtool_flow_ext m_ext; - __u64 ring_cookie; - __u32 location; -}; - -struct mlx4_ethtool_rxnfc { - __u32 cmd; - __u32 flow_type; - __u64 data; - struct mlx4_ethtool_rx_flow_spec fs; - __u32 rule_cnt; - __u32 rule_locs[0]; -}; - -#ifndef FLOW_MAC_EXT -#define FLOW_MAC_EXT 0x40000000 -#endif - -static void -mlx4_en_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - - strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, DRV_VERSION " (" DRV_RELDATE ")", - sizeof(drvinfo->version)); - snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), - "%d.%d.%d", - (u16) (mdev->dev->caps.fw_ver >> 32), - (u16) ((mdev->dev->caps.fw_ver >> 16) & 0xffff), - (u16) (mdev->dev->caps.fw_ver & 0xffff)); - strlcpy(drvinfo->bus_info, pci_name(mdev->dev->pdev), - sizeof(drvinfo->bus_info)); - drvinfo->n_stats = 0; - drvinfo->regdump_len = 0; - drvinfo->eedump_len = 0; -} - -static const char main_strings[][ETH_GSTRING_LEN] = { - /* packet statistics */ - "rx_packets", - "rx_bytes", - "rx_multicast_packets", - "rx_broadcast_packets", - "rx_errors", - "rx_dropped", - "rx_length_errors", - "rx_over_errors", - "rx_crc_errors", - "rx_jabbers", - "rx_in_range_length_error", - "rx_out_range_length_error", - "rx_lt_64_bytes_packets", - "rx_127_bytes_packets", - "rx_255_bytes_packets", - "rx_511_bytes_packets", - "rx_1023_bytes_packets", - "rx_1518_bytes_packets", - "rx_1522_bytes_packets", - "rx_1548_bytes_packets", - "rx_gt_1548_bytes_packets", - "tx_packets", - "tx_bytes", - "tx_multicast_packets", - "tx_broadcast_packets", - "tx_errors", - "tx_dropped", - "tx_lt_64_bytes_packets", - "tx_127_bytes_packets", - "tx_255_bytes_packets", - "tx_511_bytes_packets", - "tx_1023_bytes_packets", - "tx_1518_bytes_packets", - "tx_1522_bytes_packets", - "tx_1548_bytes_packets", - "tx_gt_1548_bytes_packets", - "rx_prio_0_packets", "rx_prio_0_bytes", - "rx_prio_1_packets", "rx_prio_1_bytes", - "rx_prio_2_packets", "rx_prio_2_bytes", - "rx_prio_3_packets", "rx_prio_3_bytes", - "rx_prio_4_packets", "rx_prio_4_bytes", - "rx_prio_5_packets", "rx_prio_5_bytes", - "rx_prio_6_packets", "rx_prio_6_bytes", - "rx_prio_7_packets", "rx_prio_7_bytes", - "rx_novlan_packets", "rx_novlan_bytes", - "tx_prio_0_packets", "tx_prio_0_bytes", - "tx_prio_1_packets", "tx_prio_1_bytes", - "tx_prio_2_packets", "tx_prio_2_bytes", - "tx_prio_3_packets", "tx_prio_3_bytes", - "tx_prio_4_packets", "tx_prio_4_bytes", - "tx_prio_5_packets", "tx_prio_5_bytes", - "tx_prio_6_packets", "tx_prio_6_bytes", - "tx_prio_7_packets", "tx_prio_7_bytes", - "tx_novlan_packets", "tx_novlan_bytes", - - /* flow control statistics */ - "rx_pause_prio_0", "rx_pause_duration_prio_0", - "rx_pause_transition_prio_0", "tx_pause_prio_0", - "tx_pause_duration_prio_0", "tx_pause_transition_prio_0", - "rx_pause_prio_1", "rx_pause_duration_prio_1", - "rx_pause_transition_prio_1", "tx_pause_prio_1", - "tx_pause_duration_prio_1", "tx_pause_transition_prio_1", - "rx_pause_prio_2", "rx_pause_duration_prio_2", - "rx_pause_transition_prio_2", "tx_pause_prio_2", - "tx_pause_duration_prio_2", "tx_pause_transition_prio_2", - "rx_pause_prio_3", "rx_pause_duration_prio_3", - "rx_pause_transition_prio_3", "tx_pause_prio_3", - "tx_pause_duration_prio_3", "tx_pause_transition_prio_3", - "rx_pause_prio_4", "rx_pause_duration_prio_4", - "rx_pause_transition_prio_4", "tx_pause_prio_4", - "tx_pause_duration_prio_4", "tx_pause_transition_prio_4", - "rx_pause_prio_5", "rx_pause_duration_prio_5", - "rx_pause_transition_prio_5", "tx_pause_prio_5", - "tx_pause_duration_prio_5", "tx_pause_transition_prio_5", - "rx_pause_prio_6", "rx_pause_duration_prio_6", - "rx_pause_transition_prio_6", "tx_pause_prio_6", - "tx_pause_duration_prio_6", "tx_pause_transition_prio_6", - "rx_pause_prio_7", "rx_pause_duration_prio_7", - "rx_pause_transition_prio_7", "tx_pause_prio_7", - "tx_pause_duration_prio_7", "tx_pause_transition_prio_7", - - /* VF statistics */ - "rx_packets", - "rx_bytes", - "rx_multicast_packets", - "rx_broadcast_packets", - "rx_errors", - "rx_dropped", - "tx_packets", - "tx_bytes", - "tx_multicast_packets", - "tx_broadcast_packets", - "tx_errors", - - /* VPort statistics */ - "vport_rx_unicast_packets", - "vport_rx_unicast_bytes", - "vport_rx_multicast_packets", - "vport_rx_multicast_bytes", - "vport_rx_broadcast_packets", - "vport_rx_broadcast_bytes", - "vport_rx_dropped", - "vport_rx_errors", - "vport_tx_unicast_packets", - "vport_tx_unicast_bytes", - "vport_tx_multicast_packets", - "vport_tx_multicast_bytes", - "vport_tx_broadcast_packets", - "vport_tx_broadcast_bytes", - "vport_tx_errors", - - /* port statistics */ - "tx_tso_packets", - "tx_queue_stopped", "tx_wake_queue", "tx_timeout", "rx_alloc_failed", - "rx_csum_good", "rx_csum_none", "tx_chksum_offload", -}; - -static const char mlx4_en_test_names[][ETH_GSTRING_LEN]= { - "Interrupt Test", - "Link Test", - "Speed Test", - "Register Test", - "Loopback Test", -}; - -static u32 mlx4_en_get_msglevel(struct net_device *dev) -{ - return ((struct mlx4_en_priv *) netdev_priv(dev))->msg_enable; -} - -static void mlx4_en_set_msglevel(struct net_device *dev, u32 val) -{ - ((struct mlx4_en_priv *) netdev_priv(dev))->msg_enable = val; -} - -static void mlx4_en_get_wol(struct net_device *netdev, - struct ethtool_wolinfo *wol) -{ - struct mlx4_en_priv *priv = netdev_priv(netdev); - int err = 0; - u64 config = 0; - u64 mask; - - if ((priv->port < 1) || (priv->port > 2)) { - en_err(priv, "Failed to get WoL information\n"); - return; - } - - mask = (priv->port == 1) ? MLX4_DEV_CAP_FLAG_WOL_PORT1 : - MLX4_DEV_CAP_FLAG_WOL_PORT2; - - if (!(priv->mdev->dev->caps.flags & mask)) { - wol->supported = 0; - wol->wolopts = 0; - return; - } - - err = mlx4_wol_read(priv->mdev->dev, &config, priv->port); - if (err) { - en_err(priv, "Failed to get WoL information\n"); - return; - } - - if (config & MLX4_EN_WOL_MAGIC) - wol->supported = WAKE_MAGIC; - else - wol->supported = 0; - - if (config & MLX4_EN_WOL_ENABLED) - wol->wolopts = WAKE_MAGIC; - else - wol->wolopts = 0; -} - -static int mlx4_en_set_wol(struct net_device *netdev, - struct ethtool_wolinfo *wol) -{ - struct mlx4_en_priv *priv = netdev_priv(netdev); - u64 config = 0; - int err = 0; - u64 mask; - - if ((priv->port < 1) || (priv->port > 2)) - return -EOPNOTSUPP; - - mask = (priv->port == 1) ? MLX4_DEV_CAP_FLAG_WOL_PORT1 : - MLX4_DEV_CAP_FLAG_WOL_PORT2; - - if (!(priv->mdev->dev->caps.flags & mask)) - return -EOPNOTSUPP; - - if (wol->supported & ~WAKE_MAGIC) - return -EINVAL; - - err = mlx4_wol_read(priv->mdev->dev, &config, priv->port); - if (err) { - en_err(priv, "Failed to get WoL info, unable to modify\n"); - return err; - } - - if (wol->wolopts & WAKE_MAGIC) { - config |= MLX4_EN_WOL_DO_MODIFY | MLX4_EN_WOL_ENABLED | - MLX4_EN_WOL_MAGIC; - } else { - config &= ~(MLX4_EN_WOL_ENABLED | MLX4_EN_WOL_MAGIC); - config |= MLX4_EN_WOL_DO_MODIFY; - } - - err = mlx4_wol_write(priv->mdev->dev, config, priv->port); - if (err) - en_err(priv, "Failed to set WoL information\n"); - - return err; -} - -struct bitmap_sim_iterator { - bool advance_array; - unsigned long *stats_bitmap; - unsigned int count; - unsigned int j; -}; - -static inline void bitmap_sim_iterator_init(struct bitmap_sim_iterator *h, - unsigned long *stats_bitmap, - int count) -{ - h->j = 0; - h->advance_array = !bitmap_empty(stats_bitmap, count); - h->count = h->advance_array ? bitmap_weight(stats_bitmap, count) - : count; - h->stats_bitmap = stats_bitmap; -} - -static inline int bitmap_sim_iterator_test(struct bitmap_sim_iterator *h) -{ - return !h->advance_array ? 1 : test_bit(h->j, h->stats_bitmap); -} - -static inline int bitmap_sim_iterator_inc(struct bitmap_sim_iterator *h) -{ - return h->j++; -} - -static inline unsigned int bitmap_sim_iterator_count( - struct bitmap_sim_iterator *h) -{ - return h->count; -} - -int mlx4_en_get_sset_count(struct net_device *dev, int sset) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct bitmap_sim_iterator it; - - int num_of_stats = NUM_ALL_STATS - - ((priv->mdev->dev->caps.flags2 & - MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) ? 0 : NUM_FLOW_STATS); - - bitmap_sim_iterator_init(&it, priv->stats_bitmap, num_of_stats); - - switch (sset) { - case ETH_SS_STATS: - return bitmap_sim_iterator_count(&it) + - (priv->tx_ring_num * 2) + -#ifdef LL_EXTENDED_STATS - (priv->rx_ring_num * 5); -#else - (priv->rx_ring_num * 2); -#endif - case ETH_SS_TEST: - return MLX4_EN_NUM_SELF_TEST - !(priv->mdev->dev->caps.flags - & MLX4_DEV_CAP_FLAG_UC_LOOPBACK) * 2; - default: - return -EOPNOTSUPP; - } -} - -void mlx4_en_get_ethtool_stats(struct net_device *dev, - struct ethtool_stats *stats, u64 *data) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - int index = 0; - int i; - struct bitmap_sim_iterator it; - - int num_of_stats = NUM_ALL_STATS - - ((priv->mdev->dev->caps.flags2 & - MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) ? 0 : NUM_FLOW_STATS); - - bitmap_sim_iterator_init(&it, priv->stats_bitmap, num_of_stats); - - if (!data || !priv->port_up) - return; - - spin_lock_bh(&priv->stats_lock); - - for (i = 0; i < NUM_PKT_STATS; i++, - bitmap_sim_iterator_inc(&it)) - if (bitmap_sim_iterator_test(&it)) - data[index++] = - ((unsigned long *)&priv->pkstats)[i]; - for (i = 0; i < NUM_FLOW_STATS; i++, - bitmap_sim_iterator_inc(&it)) - if (priv->mdev->dev->caps.flags2 & - MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) - if (bitmap_sim_iterator_test(&it)) - data[index++] = - ((u64 *)&priv->flowstats)[i]; - for (i = 0; i < NUM_VF_STATS; i++, - bitmap_sim_iterator_inc(&it)) - if (bitmap_sim_iterator_test(&it)) - data[index++] = - ((unsigned long *)&priv->vf_stats)[i]; - for (i = 0; i < NUM_VPORT_STATS; i++, - bitmap_sim_iterator_inc(&it)) - if (bitmap_sim_iterator_test(&it)) - data[index++] = - ((unsigned long *)&priv->vport_stats)[i]; - for (i = 0; i < NUM_PORT_STATS; i++, - bitmap_sim_iterator_inc(&it)) - if (bitmap_sim_iterator_test(&it)) - data[index++] = - ((unsigned long *)&priv->port_stats)[i]; - - for (i = 0; i < priv->tx_ring_num; i++) { - data[index++] = priv->tx_ring[i]->packets; - data[index++] = priv->tx_ring[i]->bytes; - } - for (i = 0; i < priv->rx_ring_num; i++) { - data[index++] = priv->rx_ring[i]->packets; - data[index++] = priv->rx_ring[i]->bytes; -#ifdef LL_EXTENDED_STATS - data[index++] = priv->rx_ring[i]->yields; - data[index++] = priv->rx_ring[i]->misses; - data[index++] = priv->rx_ring[i]->cleaned; -#endif - } - spin_unlock_bh(&priv->stats_lock); - -} - -void mlx4_en_restore_ethtool_stats(struct mlx4_en_priv *priv, u64 *data) -{ - int index = 0; - int i; - struct bitmap_sim_iterator it; - - int num_of_stats = NUM_ALL_STATS - - ((priv->mdev->dev->caps.flags2 & - MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) ? 0 : NUM_FLOW_STATS); - - bitmap_sim_iterator_init(&it, priv->stats_bitmap, num_of_stats); - - if (!data || !priv->port_up) - return; - - spin_lock_bh(&priv->stats_lock); - - for (i = 0; i < NUM_PKT_STATS; i++, - bitmap_sim_iterator_inc(&it)) - if (bitmap_sim_iterator_test(&it)) - ((unsigned long *)&priv->pkstats)[i] = - data[index++]; - for (i = 0; i < NUM_FLOW_STATS; i++, - bitmap_sim_iterator_inc(&it)) - if (priv->mdev->dev->caps.flags2 & - MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) - if (bitmap_sim_iterator_test(&it)) - ((u64 *)&priv->flowstats)[i] = - data[index++]; - for (i = 0; i < NUM_VF_STATS; i++, - bitmap_sim_iterator_inc(&it)) - if (bitmap_sim_iterator_test(&it)) - ((unsigned long *)&priv->vf_stats)[i] = - data[index++]; - for (i = 0; i < NUM_VPORT_STATS; i++, - bitmap_sim_iterator_inc(&it)) - if (bitmap_sim_iterator_test(&it)) - ((unsigned long *)&priv->vport_stats)[i] = - data[index++]; - for (i = 0; i < NUM_PORT_STATS; i++, - bitmap_sim_iterator_inc(&it)) - if (bitmap_sim_iterator_test(&it)) - ((unsigned long *)&priv->port_stats)[i] = - data[index++]; - - for (i = 0; i < priv->tx_ring_num; i++) { - priv->tx_ring[i]->packets = data[index++]; - priv->tx_ring[i]->bytes = data[index++]; - } - for (i = 0; i < priv->rx_ring_num; i++) { - priv->rx_ring[i]->packets = data[index++]; - priv->rx_ring[i]->bytes = data[index++]; - } - spin_unlock_bh(&priv->stats_lock); -} - -static void mlx4_en_self_test(struct net_device *dev, - struct ethtool_test *etest, u64 *buf) -{ - mlx4_en_ex_selftest(dev, &etest->flags, buf); -} - -static void mlx4_en_get_strings(struct net_device *dev, - uint32_t stringset, uint8_t *data) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - int index = 0; - int i, k; - struct bitmap_sim_iterator it; - - int num_of_stats = NUM_ALL_STATS - - ((priv->mdev->dev->caps.flags2 & - MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) ? 0 : NUM_FLOW_STATS); - - bitmap_sim_iterator_init(&it, priv->stats_bitmap, num_of_stats); - - switch (stringset) { - case ETH_SS_TEST: - for (i = 0; i < MLX4_EN_NUM_SELF_TEST - 2; i++) - strcpy(data + i * ETH_GSTRING_LEN, mlx4_en_test_names[i]); - if (priv->mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_UC_LOOPBACK) - for (; i < MLX4_EN_NUM_SELF_TEST; i++) - strcpy(data + i * ETH_GSTRING_LEN, mlx4_en_test_names[i]); - break; - - case ETH_SS_STATS: - /* Add main counters */ - for (i = 0; i < NUM_PKT_STATS; i++, - bitmap_sim_iterator_inc(&it)) - if (bitmap_sim_iterator_test(&it)) - strcpy(data + (index++) * ETH_GSTRING_LEN, - main_strings[i]); - - for (k = 0; k < NUM_FLOW_STATS; k++, - bitmap_sim_iterator_inc(&it)) - if (priv->mdev->dev->caps.flags2 & - MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) - if (bitmap_sim_iterator_test(&it)) - strcpy(data + (index++) * - ETH_GSTRING_LEN, - main_strings[i + k]); - - for (; (i + k) < num_of_stats; i++, - bitmap_sim_iterator_inc(&it)) - if (bitmap_sim_iterator_test(&it)) - strcpy(data + (index++) * ETH_GSTRING_LEN, - main_strings[i + k]); - - for (i = 0; i < priv->tx_ring_num; i++) { - sprintf(data + (index++) * ETH_GSTRING_LEN, - "tx%d_packets", i); - sprintf(data + (index++) * ETH_GSTRING_LEN, - "tx%d_bytes", i); - } - for (i = 0; i < priv->rx_ring_num; i++) { - sprintf(data + (index++) * ETH_GSTRING_LEN, - "rx%d_packets", i); - sprintf(data + (index++) * ETH_GSTRING_LEN, - "rx%d_bytes", i); -#ifdef LL_EXTENDED_STATS - sprintf(data + (index++) * ETH_GSTRING_LEN, - "rx%d_napi_yield", i); - sprintf(data + (index++) * ETH_GSTRING_LEN, - "rx%d_misses", i); - sprintf(data + (index++) * ETH_GSTRING_LEN, - "rx%d_cleaned", i); -#endif - } - break; - } -} - -static u32 mlx4_en_autoneg_get(struct net_device *dev) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - u32 autoneg = AUTONEG_DISABLE; - - if ((mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ETH_BACKPL_AN_REP) && - priv->port_state.autoneg) { - autoneg = AUTONEG_ENABLE; - } - - return autoneg; -} - -static int mlx4_en_get_settings(struct net_device *dev, struct ethtool_cmd *cmd) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - int trans_type; - - /* SUPPORTED_1000baseT_Half isn't supported */ - cmd->supported = SUPPORTED_1000baseT_Full - |SUPPORTED_10000baseT_Full; - - cmd->advertising = ADVERTISED_1000baseT_Full - |ADVERTISED_10000baseT_Full; - - cmd->supported |= SUPPORTED_1000baseKX_Full - |SUPPORTED_10000baseKX4_Full - |SUPPORTED_10000baseKR_Full - |SUPPORTED_10000baseR_FEC - |SUPPORTED_40000baseKR4_Full - |SUPPORTED_40000baseCR4_Full - |SUPPORTED_40000baseSR4_Full - |SUPPORTED_40000baseLR4_Full; - - /* ADVERTISED_1000baseT_Half isn't advertised */ - cmd->advertising |= ADVERTISED_1000baseKX_Full - |ADVERTISED_10000baseKX4_Full - |ADVERTISED_10000baseKR_Full - |ADVERTISED_10000baseR_FEC - |ADVERTISED_40000baseKR4_Full - |ADVERTISED_40000baseCR4_Full - |ADVERTISED_40000baseSR4_Full - |ADVERTISED_40000baseLR4_Full; - - if (mlx4_en_QUERY_PORT(priv->mdev, priv->port)) - return -ENOMEM; - - cmd->autoneg = mlx4_en_autoneg_get(dev); - if (cmd->autoneg == AUTONEG_ENABLE) { - cmd->supported |= SUPPORTED_Autoneg; - cmd->advertising |= ADVERTISED_Autoneg; - } - - trans_type = priv->port_state.transciver; - if (netif_carrier_ok(dev)) { - ethtool_cmd_speed_set(cmd, priv->port_state.link_speed); - cmd->duplex = DUPLEX_FULL; - } else { - ethtool_cmd_speed_set(cmd, -1); - cmd->duplex = -1; - } - - if (trans_type > 0 && trans_type <= 0xC) { - cmd->port = PORT_FIBRE; - cmd->transceiver = XCVR_EXTERNAL; - cmd->supported |= SUPPORTED_FIBRE; - cmd->advertising |= ADVERTISED_FIBRE; - } else if (trans_type == 0x80 || trans_type == 0) { - cmd->port = PORT_TP; - cmd->transceiver = XCVR_INTERNAL; - cmd->supported |= SUPPORTED_TP; - cmd->advertising |= ADVERTISED_TP; - } else { - cmd->port = -1; - cmd->transceiver = -1; - } - return 0; -} - -static const char *mlx4_en_duplex_to_string(int duplex) -{ - switch (duplex) { - case DUPLEX_FULL: - return "FULL"; - case DUPLEX_HALF: - return "HALF"; - default: - break; - } - return "UNKNOWN"; -} - -static int mlx4_en_set_settings(struct net_device *dev, struct ethtool_cmd *cmd) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_port_state *port_state = &priv->port_state; - - if ((cmd->autoneg != port_state->autoneg) || - (ethtool_cmd_speed(cmd) != port_state->link_speed) || - (cmd->duplex != DUPLEX_FULL)) { - en_info(priv, "Changing port state properties (auto-negotiation" - " , speed/duplex) is not supported. Current:" - " auto-negotiation=%d speed/duplex=%d/%s\n", - port_state->autoneg, port_state->link_speed, - mlx4_en_duplex_to_string(DUPLEX_FULL)); - return -EOPNOTSUPP; - } - - /* User provided same port state properties that are currently set. - * Nothing to change - */ - return 0; -} - -static int mlx4_en_get_coalesce(struct net_device *dev, - struct ethtool_coalesce *coal) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - - coal->tx_coalesce_usecs = priv->tx_usecs; - coal->tx_max_coalesced_frames = priv->tx_frames; - coal->rx_coalesce_usecs = priv->rx_usecs; - coal->rx_max_coalesced_frames = priv->rx_frames; - - coal->pkt_rate_low = priv->pkt_rate_low; - coal->rx_coalesce_usecs_low = priv->rx_usecs_low; - coal->pkt_rate_high = priv->pkt_rate_high; - coal->rx_coalesce_usecs_high = priv->rx_usecs_high; - coal->rate_sample_interval = priv->sample_interval; - coal->use_adaptive_rx_coalesce = priv->adaptive_rx_coal; - return 0; -} - -static int mlx4_en_set_coalesce(struct net_device *dev, - struct ethtool_coalesce *coal) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - int err, i; - - priv->rx_frames = (coal->rx_max_coalesced_frames == - MLX4_EN_AUTO_CONF) ? - MLX4_EN_RX_COAL_TARGET / - priv->dev->mtu + 1 : - coal->rx_max_coalesced_frames; - priv->rx_usecs = (coal->rx_coalesce_usecs == - MLX4_EN_AUTO_CONF) ? - MLX4_EN_RX_COAL_TIME : - coal->rx_coalesce_usecs; - - /* Setting TX coalescing parameters */ - if (coal->tx_coalesce_usecs != priv->tx_usecs || - coal->tx_max_coalesced_frames != priv->tx_frames) { - priv->tx_usecs = coal->tx_coalesce_usecs; - priv->tx_frames = coal->tx_max_coalesced_frames; - if (priv->port_up) { - for (i = 0; i < priv->tx_ring_num; i++) { - priv->tx_cq[i]->moder_cnt = priv->tx_frames; - priv->tx_cq[i]->moder_time = priv->tx_usecs; - if (mlx4_en_set_cq_moder(priv, priv->tx_cq[i])) - en_warn(priv, "Failed changing moderation for TX cq %d\n", i); - } - } - } - - /* Set adaptive coalescing params */ - priv->pkt_rate_low = coal->pkt_rate_low; - priv->rx_usecs_low = coal->rx_coalesce_usecs_low; - priv->pkt_rate_high = coal->pkt_rate_high; - priv->rx_usecs_high = coal->rx_coalesce_usecs_high; - priv->sample_interval = coal->rate_sample_interval; - priv->adaptive_rx_coal = coal->use_adaptive_rx_coalesce; - if (priv->adaptive_rx_coal) - return 0; - - if (priv->port_up) { - for (i = 0; i < priv->rx_ring_num; i++) { - priv->rx_cq[i]->moder_cnt = priv->rx_frames; - priv->rx_cq[i]->moder_time = priv->rx_usecs; - priv->last_moder_time[i] = MLX4_EN_AUTO_CONF; - err = mlx4_en_set_cq_moder(priv, priv->rx_cq[i]); - if (err) - return err; - } - } - - return 0; -} - -static int mlx4_en_set_pauseparam(struct net_device *dev, - struct ethtool_pauseparam *pause) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - int err; - - if (pause->autoneg) - return -EOPNOTSUPP; - - priv->prof->tx_pause = pause->tx_pause != 0; - priv->prof->rx_pause = pause->rx_pause != 0; - err = mlx4_SET_PORT_general(mdev->dev, priv->port, - priv->rx_skb_size + ETH_FCS_LEN, - priv->prof->tx_pause, - priv->prof->tx_ppp, - priv->prof->rx_pause, - priv->prof->rx_ppp); - if (err) - en_err(priv, "Failed setting pause params\n"); - - return err; -} - -static void mlx4_en_get_pauseparam(struct net_device *dev, - struct ethtool_pauseparam *pause) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - - pause->tx_pause = priv->prof->tx_pause; - pause->rx_pause = priv->prof->rx_pause; - pause->autoneg = mlx4_en_autoneg_get(dev); -} - -/* rtnl lock must be taken before calling */ -int mlx4_en_pre_config(struct mlx4_en_priv *priv) -{ -#ifdef CONFIG_RFS_ACCEL - struct cpu_rmap *rmap; - - if (!priv->dev->rx_cpu_rmap) - return 0; - - /* Disable RFS events - * Must have all RFS jobs flushed before freeing resources - */ - rmap = priv->dev->rx_cpu_rmap; - priv->dev->rx_cpu_rmap = NULL; - - rtnl_unlock(); - free_irq_cpu_rmap(rmap); - rtnl_lock(); - - if (priv->dev->rx_cpu_rmap) - return -EBUSY; /* another configuration completed while lock - * was free - */ - - /* Make sure all currently running filter_work are being processed - * Other work will return immediatly because of disable_rfs - */ - flush_workqueue(priv->mdev->workqueue); - -#endif - - return 0; -} - -static int mlx4_en_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *param) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - u32 rx_size, tx_size; - int port_up = 0; - int err = 0; - int i, n_stats; - u64 *data = NULL; - - if (!priv->port_up) - return -ENOMEM; - - if (param->rx_jumbo_pending || param->rx_mini_pending) - return -EINVAL; - - rx_size = roundup_pow_of_two(param->rx_pending); - rx_size = max_t(u32, rx_size, MLX4_EN_MIN_RX_SIZE); - rx_size = min_t(u32, rx_size, MLX4_EN_MAX_RX_SIZE); - tx_size = roundup_pow_of_two(param->tx_pending); - tx_size = max_t(u32, tx_size, MLX4_EN_MIN_TX_SIZE); - tx_size = min_t(u32, tx_size, MLX4_EN_MAX_TX_SIZE); - - if (rx_size == (priv->port_up ? priv->rx_ring[0]->actual_size : - priv->rx_ring[0]->size) && - tx_size == priv->tx_ring[0]->size) - return 0; - err = mlx4_en_pre_config(priv); - if (err) - return err; - - mutex_lock(&mdev->state_lock); - if (priv->port_up) { - port_up = 1; - mlx4_en_stop_port(dev); - } - - /* Cache port statistics */ - n_stats = mlx4_en_get_sset_count(dev, ETH_SS_STATS); - if (n_stats > 0) { - data = kmalloc(n_stats * sizeof(u64), GFP_KERNEL); - if (data) - mlx4_en_get_ethtool_stats(dev, NULL, data); - } - - mlx4_en_free_resources(priv); - - priv->prof->tx_ring_size = tx_size; - priv->prof->rx_ring_size = rx_size; - - err = mlx4_en_alloc_resources(priv); - if (err) { - en_err(priv, "Failed reallocating port resources\n"); - goto out; - } - - /* Restore port statistics */ - if (n_stats > 0 && data) - mlx4_en_restore_ethtool_stats(priv, data); - - if (port_up) { - err = mlx4_en_start_port(dev); - if (err) { - en_err(priv, "Failed starting port\n"); - goto out; - } - - for (i = 0; i < priv->rx_ring_num; i++) { - priv->rx_cq[i]->moder_cnt = priv->rx_frames; - priv->rx_cq[i]->moder_time = priv->rx_usecs; - priv->last_moder_time[i] = MLX4_EN_AUTO_CONF; - err = mlx4_en_set_cq_moder(priv, priv->rx_cq[i]); - if (err) - goto out; - } - } - -out: - kfree(data); - mutex_unlock(&mdev->state_lock); - return err; -} - -static void mlx4_en_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *param) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - - if (!priv->port_up) - return; - - memset(param, 0, sizeof(*param)); - param->rx_max_pending = MLX4_EN_MAX_RX_SIZE; - param->tx_max_pending = MLX4_EN_MAX_TX_SIZE; - param->rx_pending = priv->port_up ? - priv->rx_ring[0]->actual_size : priv->rx_ring[0]->size; - param->tx_pending = priv->tx_ring[0]->size; -} - -static u32 mlx4_en_get_rxfh_indir_size(struct net_device *dev) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - - return priv->rx_ring_num; -} - -static int mlx4_en_get_rxfh_indir(struct net_device *dev, u32 *ring_index) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_rss_map *rss_map = &priv->rss_map; - int rss_rings; - size_t n = priv->rx_ring_num; - int err = 0; - - rss_rings = priv->prof->rss_rings ?: priv->rx_ring_num; - rss_rings = 1 << ilog2(rss_rings); - - while (n--) { - ring_index[n] = rss_map->qps[n % rss_rings].qpn - - rss_map->base_qpn; - } - - return err; -} - -static int mlx4_en_set_rxfh_indir(struct net_device *dev, - const u32 *ring_index) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - int port_up = 0; - int err = 0; - int i; - int rss_rings = 0; - - /* Calculate RSS table size and make sure flows are spread evenly - * between rings - */ - for (i = 0; i < priv->rx_ring_num; i++) { - if (i > 0 && !ring_index[i] && !rss_rings) - rss_rings = i; - - if (ring_index[i] != (i % (rss_rings ?: priv->rx_ring_num))) - return -EINVAL; - } - - if (!rss_rings) - rss_rings = priv->rx_ring_num; - - /* RSS table size must be an order of 2 */ - if (!is_power_of_2(rss_rings)) - return -EINVAL; - - mutex_lock(&mdev->state_lock); - if (priv->port_up) { - port_up = 1; - mlx4_en_stop_port(dev); - } - - priv->prof->rss_rings = rss_rings; - - if (port_up) { - err = mlx4_en_start_port(dev); - if (err) - en_err(priv, "Failed starting port\n"); - } - - mutex_unlock(&mdev->state_lock); - return err; -} - -#define all_zeros_or_all_ones(field) \ - ((field) == 0 || (field) == (__force typeof(field))-1) - -static int mlx4_en_validate_flow(struct net_device *dev, - struct mlx4_ethtool_rxnfc *cmd) -{ - struct ethtool_usrip4_spec *l3_mask; - struct ethtool_tcpip4_spec *l4_mask; - struct ethhdr *eth_mask; - - if (cmd->fs.location >= MAX_NUM_OF_FS_RULES) - return -EINVAL; - - if (cmd->fs.flow_type & FLOW_MAC_EXT) { - /* dest mac mask must be ff:ff:ff:ff:ff:ff */ - if (!is_broadcast_ether_addr(cmd->fs.m_ext.h_dest)) - return -EINVAL; - } - - switch (cmd->fs.flow_type & ~(FLOW_EXT | FLOW_MAC_EXT)) { - case TCP_V4_FLOW: - case UDP_V4_FLOW: - if (cmd->fs.m_u.tcp_ip4_spec.tos) - return -EINVAL; - l4_mask = &cmd->fs.m_u.tcp_ip4_spec; - /* don't allow mask which isn't all 0 or 1 */ - if (!all_zeros_or_all_ones(l4_mask->ip4src) || - !all_zeros_or_all_ones(l4_mask->ip4dst) || - !all_zeros_or_all_ones(l4_mask->psrc) || - !all_zeros_or_all_ones(l4_mask->pdst)) - return -EINVAL; - break; - case IP_USER_FLOW: - l3_mask = &cmd->fs.m_u.usr_ip4_spec; - if (l3_mask->l4_4_bytes || l3_mask->tos || l3_mask->proto || - cmd->fs.h_u.usr_ip4_spec.ip_ver != ETH_RX_NFC_IP4 || - (!l3_mask->ip4src && !l3_mask->ip4dst) || - !all_zeros_or_all_ones(l3_mask->ip4src) || - !all_zeros_or_all_ones(l3_mask->ip4dst)) - return -EINVAL; - break; - case ETHER_FLOW: - eth_mask = &cmd->fs.m_u.ether_spec; - /* source mac mask must not be set */ - if (!is_zero_ether_addr(eth_mask->h_source)) - return -EINVAL; - - /* dest mac mask must be ff:ff:ff:ff:ff:ff */ - if (!is_broadcast_ether_addr(eth_mask->h_dest)) - return -EINVAL; - - if (!all_zeros_or_all_ones(eth_mask->h_proto)) - return -EINVAL; - break; - default: - return -EINVAL; - } - - if ((cmd->fs.flow_type & FLOW_EXT)) { - if (cmd->fs.m_ext.vlan_etype || - !(cmd->fs.m_ext.vlan_tci == 0 || - cmd->fs.m_ext.vlan_tci == cpu_to_be16(0xfff))) - return -EINVAL; - if (cmd->fs.m_ext.vlan_tci) { - if (be16_to_cpu(cmd->fs.h_ext.vlan_tci) < - VLAN_MIN_VALUE || - be16_to_cpu(cmd->fs.h_ext.vlan_tci) > - VLAN_MAX_VALUE) - return -EINVAL; - } - } - - return 0; -} - -static int mlx4_en_ethtool_add_mac_rule(struct mlx4_ethtool_rxnfc *cmd, - struct list_head *rule_list_h, - struct mlx4_spec_list *spec_l2, - unsigned char *mac) -{ - int err = 0; - __be64 mac_msk = cpu_to_be64(MLX4_MAC_MASK << 16); - - spec_l2->id = MLX4_NET_TRANS_RULE_ID_ETH; - memcpy(spec_l2->eth.dst_mac_msk, &mac_msk, ETH_ALEN); - memcpy(spec_l2->eth.dst_mac, mac, ETH_ALEN); - - if ((cmd->fs.flow_type & FLOW_EXT) && cmd->fs.m_ext.vlan_tci) { - spec_l2->eth.vlan_id = cmd->fs.h_ext.vlan_tci; - spec_l2->eth.vlan_id_msk = cpu_to_be16(0xfff); - } - - list_add_tail(&spec_l2->list, rule_list_h); - - return err; -} - -static int mlx4_en_ethtool_add_mac_rule_by_ipv4(struct mlx4_en_priv *priv, - struct mlx4_ethtool_rxnfc *cmd, - struct list_head *rule_list_h, - struct mlx4_spec_list *spec_l2, - __be32 ipv4_dst) -{ - unsigned char mac[ETH_ALEN]; - - if (!ipv4_is_multicast(ipv4_dst)) { - if (cmd->fs.flow_type & FLOW_MAC_EXT) - memcpy(&mac, cmd->fs.h_ext.h_dest, ETH_ALEN); - else - memcpy(&mac, priv->dev->dev_addr, ETH_ALEN); - } else { - ip_eth_mc_map(ipv4_dst, mac); - } - - return mlx4_en_ethtool_add_mac_rule(cmd, rule_list_h, spec_l2, &mac[0]); -} - -static int add_ip_rule(struct mlx4_en_priv *priv, - struct mlx4_ethtool_rxnfc *cmd, - struct list_head *list_h) -{ - struct mlx4_spec_list *spec_l2 = NULL; - struct mlx4_spec_list *spec_l3 = NULL; - struct ethtool_usrip4_spec *l3_mask = &cmd->fs.m_u.usr_ip4_spec; - - spec_l3 = kzalloc(sizeof(*spec_l3), GFP_KERNEL); - spec_l2 = kzalloc(sizeof(*spec_l2), GFP_KERNEL); - if (!spec_l2 || !spec_l3) { - en_err(priv, "Fail to alloc ethtool rule.\n"); - kfree(spec_l2); - kfree(spec_l3); - return -ENOMEM; - } - - mlx4_en_ethtool_add_mac_rule_by_ipv4(priv, cmd, list_h, spec_l2, - cmd->fs.h_u. - usr_ip4_spec.ip4dst); - spec_l3->id = MLX4_NET_TRANS_RULE_ID_IPV4; - spec_l3->ipv4.src_ip = cmd->fs.h_u.usr_ip4_spec.ip4src; - if (l3_mask->ip4src) - spec_l3->ipv4.src_ip_msk = MLX4_BE_WORD_MASK; - spec_l3->ipv4.dst_ip = cmd->fs.h_u.usr_ip4_spec.ip4dst; - if (l3_mask->ip4dst) - spec_l3->ipv4.dst_ip_msk = MLX4_BE_WORD_MASK; - list_add_tail(&spec_l3->list, list_h); - - return 0; -} - -static int add_tcp_udp_rule(struct mlx4_en_priv *priv, - struct mlx4_ethtool_rxnfc *cmd, - struct list_head *list_h, int proto) -{ - struct mlx4_spec_list *spec_l2 = NULL; - struct mlx4_spec_list *spec_l3 = NULL; - struct mlx4_spec_list *spec_l4 = NULL; - struct ethtool_tcpip4_spec *l4_mask = &cmd->fs.m_u.tcp_ip4_spec; - - spec_l2 = kzalloc(sizeof(*spec_l2), GFP_KERNEL); - spec_l3 = kzalloc(sizeof(*spec_l3), GFP_KERNEL); - spec_l4 = kzalloc(sizeof(*spec_l4), GFP_KERNEL); - if (!spec_l2 || !spec_l3 || !spec_l4) { - en_err(priv, "Fail to alloc ethtool rule.\n"); - kfree(spec_l2); - kfree(spec_l3); - kfree(spec_l4); - return -ENOMEM; - } - - spec_l3->id = MLX4_NET_TRANS_RULE_ID_IPV4; - - if (proto == TCP_V4_FLOW) { - mlx4_en_ethtool_add_mac_rule_by_ipv4(priv, cmd, list_h, - spec_l2, - cmd->fs.h_u. - tcp_ip4_spec.ip4dst); - spec_l4->id = MLX4_NET_TRANS_RULE_ID_TCP; - spec_l3->ipv4.src_ip = cmd->fs.h_u.tcp_ip4_spec.ip4src; - spec_l3->ipv4.dst_ip = cmd->fs.h_u.tcp_ip4_spec.ip4dst; - spec_l4->tcp_udp.src_port = cmd->fs.h_u.tcp_ip4_spec.psrc; - spec_l4->tcp_udp.dst_port = cmd->fs.h_u.tcp_ip4_spec.pdst; - } else { - mlx4_en_ethtool_add_mac_rule_by_ipv4(priv, cmd, list_h, - spec_l2, - cmd->fs.h_u. - udp_ip4_spec.ip4dst); - spec_l4->id = MLX4_NET_TRANS_RULE_ID_UDP; - spec_l3->ipv4.src_ip = cmd->fs.h_u.udp_ip4_spec.ip4src; - spec_l3->ipv4.dst_ip = cmd->fs.h_u.udp_ip4_spec.ip4dst; - spec_l4->tcp_udp.src_port = cmd->fs.h_u.udp_ip4_spec.psrc; - spec_l4->tcp_udp.dst_port = cmd->fs.h_u.udp_ip4_spec.pdst; - } - - if (l4_mask->ip4src) - spec_l3->ipv4.src_ip_msk = MLX4_BE_WORD_MASK; - if (l4_mask->ip4dst) - spec_l3->ipv4.dst_ip_msk = MLX4_BE_WORD_MASK; - - if (l4_mask->psrc) - spec_l4->tcp_udp.src_port_msk = MLX4_BE_SHORT_MASK; - if (l4_mask->pdst) - spec_l4->tcp_udp.dst_port_msk = MLX4_BE_SHORT_MASK; - - list_add_tail(&spec_l3->list, list_h); - list_add_tail(&spec_l4->list, list_h); - - return 0; -} - -static int mlx4_en_ethtool_to_net_trans_rule(struct net_device *dev, - struct mlx4_ethtool_rxnfc *cmd, - struct list_head *rule_list_h) -{ - int err; - struct ethhdr *eth_spec; - struct mlx4_spec_list *spec_l2; - struct mlx4_en_priv *priv = netdev_priv(dev); - - err = mlx4_en_validate_flow(dev, cmd); - if (err) - return err; - - switch (cmd->fs.flow_type & ~(FLOW_EXT | FLOW_MAC_EXT)) { - case ETHER_FLOW: - spec_l2 = kzalloc(sizeof(*spec_l2), GFP_KERNEL); - if (!spec_l2) - return -ENOMEM; - - eth_spec = &cmd->fs.h_u.ether_spec; - mlx4_en_ethtool_add_mac_rule(cmd, rule_list_h, spec_l2, ð_spec->h_dest[0]); - spec_l2->eth.ether_type = eth_spec->h_proto; - if (eth_spec->h_proto) - spec_l2->eth.ether_type_enable = 1; - break; - case IP_USER_FLOW: - err = add_ip_rule(priv, cmd, rule_list_h); - break; - case TCP_V4_FLOW: - err = add_tcp_udp_rule(priv, cmd, rule_list_h, TCP_V4_FLOW); - break; - case UDP_V4_FLOW: - err = add_tcp_udp_rule(priv, cmd, rule_list_h, UDP_V4_FLOW); - break; - } - - return err; -} - -static int mlx4_en_flow_replace(struct net_device *dev, - struct mlx4_ethtool_rxnfc *cmd) -{ - int err; - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - struct ethtool_flow_id *loc_rule; - struct mlx4_spec_list *spec, *tmp_spec; - u32 qpn; - u64 reg_id; - - struct mlx4_net_trans_rule rule = { - .queue_mode = MLX4_NET_TRANS_Q_FIFO, - .exclusive = 0, - .allow_loopback = 1, - .promisc_mode = MLX4_FS_REGULAR, - }; - - rule.port = priv->port; - rule.priority = MLX4_DOMAIN_ETHTOOL | cmd->fs.location; - INIT_LIST_HEAD(&rule.list); - - /* Allow direct QP attaches if the EN_ETHTOOL_QP_ATTACH flag is set */ - if (cmd->fs.ring_cookie == RX_CLS_FLOW_DISC) - qpn = priv->drop_qp.qpn; - else if (cmd->fs.ring_cookie & EN_ETHTOOL_QP_ATTACH) { - qpn = cmd->fs.ring_cookie & (EN_ETHTOOL_QP_ATTACH - 1); - } else { - if (cmd->fs.ring_cookie >= priv->rx_ring_num) { - en_warn(priv, "rxnfc: RX ring (%llu) doesn't exist.\n", - cmd->fs.ring_cookie); - return -EINVAL; - } - qpn = priv->rss_map.qps[cmd->fs.ring_cookie].qpn; - if (!qpn) { - en_warn(priv, "rxnfc: RX ring (%llu) is inactive.\n", - cmd->fs.ring_cookie); - return -EINVAL; - } - } - rule.qpn = qpn; - err = mlx4_en_ethtool_to_net_trans_rule(dev, cmd, &rule.list); - if (err) - goto out_free_list; - - mutex_lock(&mdev->state_lock); - loc_rule = &priv->ethtool_rules[cmd->fs.location]; - if (loc_rule->id) { - err = mlx4_flow_detach(priv->mdev->dev, loc_rule->id); - if (err) { - en_err(priv, "Fail to detach network rule at location %d. registration id = %llx\n", - cmd->fs.location, loc_rule->id); - goto unlock; - } - loc_rule->id = 0; - memset(&loc_rule->flow_spec, 0, - sizeof(struct ethtool_rx_flow_spec)); - list_del(&loc_rule->list); - } - err = mlx4_flow_attach(priv->mdev->dev, &rule, ®_id); - if (err) { - en_err(priv, "Fail to attach network rule at location %d.\n", - cmd->fs.location); - goto unlock; - } - loc_rule->id = reg_id; - memcpy(&loc_rule->flow_spec, &cmd->fs, - sizeof(struct ethtool_rx_flow_spec)); - list_add_tail(&loc_rule->list, &priv->ethtool_list); - -unlock: - mutex_unlock(&mdev->state_lock); -out_free_list: - list_for_each_entry_safe(spec, tmp_spec, &rule.list, list) { - list_del(&spec->list); - kfree(spec); - } - return err; -} - -static int mlx4_en_flow_detach(struct net_device *dev, - struct mlx4_ethtool_rxnfc *cmd) -{ - int err = 0; - struct ethtool_flow_id *rule; - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - - if (cmd->fs.location >= MAX_NUM_OF_FS_RULES) - return -EINVAL; - - mutex_lock(&mdev->state_lock); - rule = &priv->ethtool_rules[cmd->fs.location]; - if (!rule->id) { - err = -ENOENT; - goto out; - } - - err = mlx4_flow_detach(priv->mdev->dev, rule->id); - if (err) { - en_err(priv, "Fail to detach network rule at location %d. registration id = 0x%llx\n", - cmd->fs.location, rule->id); - goto out; - } - rule->id = 0; - memset(&rule->flow_spec, 0, sizeof(struct ethtool_rx_flow_spec)); - - list_del(&rule->list); -out: - mutex_unlock(&mdev->state_lock); - return err; - -} - -static int mlx4_en_get_flow(struct net_device *dev, struct mlx4_ethtool_rxnfc *cmd, - int loc) -{ - int err = 0; - struct ethtool_flow_id *rule; - struct mlx4_en_priv *priv = netdev_priv(dev); - - if (loc < 0 || loc >= MAX_NUM_OF_FS_RULES) - return -EINVAL; - - rule = &priv->ethtool_rules[loc]; - if (rule->id) - memcpy(&cmd->fs, &rule->flow_spec, - sizeof(struct ethtool_rx_flow_spec)); - else - err = -ENOENT; - - return err; -} - -static int mlx4_en_get_num_flows(struct mlx4_en_priv *priv) -{ - - int i, res = 0; - for (i = 0; i < MAX_NUM_OF_FS_RULES; i++) { - if (priv->ethtool_rules[i].id) - res++; - } - return res; - -} - -static int mlx4_en_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *c, - u32 *rule_locs) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - int err = 0; - int i = 0, priority = 0; - struct mlx4_ethtool_rxnfc *cmd = (struct mlx4_ethtool_rxnfc *)c; - - if ((cmd->cmd == ETHTOOL_GRXCLSRLCNT || - cmd->cmd == ETHTOOL_GRXCLSRULE || - cmd->cmd == ETHTOOL_GRXCLSRLALL) && - (mdev->dev->caps.steering_mode != - MLX4_STEERING_MODE_DEVICE_MANAGED || !priv->port_up)) - return -EINVAL; - - switch (cmd->cmd) { - case ETHTOOL_GRXRINGS: - cmd->data = priv->rx_ring_num; - break; - case ETHTOOL_GRXCLSRLCNT: - cmd->rule_cnt = mlx4_en_get_num_flows(priv); - break; - case ETHTOOL_GRXCLSRULE: - err = mlx4_en_get_flow(dev, cmd, cmd->fs.location); - break; - case ETHTOOL_GRXCLSRLALL: - while ((!err || err == -ENOENT) && priority < cmd->rule_cnt) { - err = mlx4_en_get_flow(dev, cmd, i); - if (!err) - rule_locs[priority++] = i; - i++; - } - err = 0; - break; - default: - err = -EOPNOTSUPP; - break; - } - - return err; -} - -static int mlx4_en_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *c) -{ - int err = 0; - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_ethtool_rxnfc *cmd = (struct mlx4_ethtool_rxnfc *)c; - - if (mdev->dev->caps.steering_mode != - MLX4_STEERING_MODE_DEVICE_MANAGED || !priv->port_up) - return -EINVAL; - - switch (cmd->cmd) { - case ETHTOOL_SRXCLSRLINS: - err = mlx4_en_flow_replace(dev, cmd); - break; - case ETHTOOL_SRXCLSRLDEL: - err = mlx4_en_flow_detach(dev, cmd); - break; - default: - en_warn(priv, "Unsupported ethtool command. (%d)\n", cmd->cmd); - return -EINVAL; - } - - return err; -} - -static void mlx4_en_get_channels(struct net_device *dev, - struct ethtool_channels *channel) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - - memset(channel, 0, sizeof(*channel)); - - channel->max_rx = MAX_RX_RINGS; - channel->max_tx = MLX4_EN_MAX_TX_RING_P_UP; - - channel->rx_count = priv->rx_ring_num; - channel->tx_count = priv->tx_ring_num / MLX4_EN_NUM_UP; -} - -static int mlx4_en_set_channels(struct net_device *dev, - struct ethtool_channels *channel) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - int port_up = 0; - int i; - int err = 0; - - if (channel->other_count || channel->combined_count || - channel->tx_count > MLX4_EN_MAX_TX_RING_P_UP || - channel->rx_count > MAX_RX_RINGS || - !channel->tx_count || !channel->rx_count) - return -EINVAL; - - err = mlx4_en_pre_config(priv); - if (err) - return err; - - mutex_lock(&mdev->state_lock); - if (priv->port_up) { - port_up = 1; - mlx4_en_stop_port(dev); - } - - mlx4_en_free_resources(priv); - - priv->num_tx_rings_p_up = channel->tx_count; - priv->tx_ring_num = channel->tx_count * MLX4_EN_NUM_UP; - priv->rx_ring_num = channel->rx_count; - - err = mlx4_en_alloc_resources(priv); - if (err) { - en_err(priv, "Failed reallocating port resources\n"); - goto out; - } - - netif_set_real_num_tx_queues(dev, priv->tx_ring_num); - netif_set_real_num_rx_queues(dev, priv->rx_ring_num); - - mlx4_en_setup_tc(dev, MLX4_EN_NUM_UP); - - en_warn(priv, "Using %d TX rings\n", priv->tx_ring_num); - en_warn(priv, "Using %d RX rings\n", priv->rx_ring_num); - - if (port_up) { - err = mlx4_en_start_port(dev); - if (err) - en_err(priv, "Failed starting port\n"); - - for (i = 0; i < priv->rx_ring_num; i++) { - priv->rx_cq[i]->moder_cnt = priv->rx_frames; - priv->rx_cq[i]->moder_time = priv->rx_usecs; - priv->last_moder_time[i] = MLX4_EN_AUTO_CONF; - err = mlx4_en_set_cq_moder(priv, priv->rx_cq[i]); - if (err) - goto out; - } - } - -out: - mutex_unlock(&mdev->state_lock); - return err; -} - -static int mlx4_en_get_ts_info(struct net_device *dev, - struct ethtool_ts_info *info) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - int ret; - - ret = ethtool_op_get_ts_info(dev, info); - if (ret) - return ret; - - if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) { - info->so_timestamping |= - SOF_TIMESTAMPING_TX_HARDWARE | - SOF_TIMESTAMPING_RX_HARDWARE | - SOF_TIMESTAMPING_RAW_HARDWARE; - - info->tx_types = - (1 << HWTSTAMP_TX_OFF) | - (1 << HWTSTAMP_TX_ON); - - info->rx_filters = - (1 << HWTSTAMP_FILTER_NONE) | - (1 << HWTSTAMP_FILTER_ALL); - } - - return ret; -} - -const struct ethtool_ops mlx4_en_ethtool_ops = { - .get_drvinfo = mlx4_en_get_drvinfo, - .get_settings = mlx4_en_get_settings, - .set_settings = mlx4_en_set_settings, - .get_link = ethtool_op_get_link, - .get_strings = mlx4_en_get_strings, - .get_sset_count = mlx4_en_get_sset_count, - .get_ethtool_stats = mlx4_en_get_ethtool_stats, - .self_test = mlx4_en_self_test, - .get_wol = mlx4_en_get_wol, - .set_wol = mlx4_en_set_wol, - .get_msglevel = mlx4_en_get_msglevel, - .set_msglevel = mlx4_en_set_msglevel, - .get_coalesce = mlx4_en_get_coalesce, - .set_coalesce = mlx4_en_set_coalesce, - .get_pauseparam = mlx4_en_get_pauseparam, - .set_pauseparam = mlx4_en_set_pauseparam, - .get_ringparam = mlx4_en_get_ringparam, - .set_ringparam = mlx4_en_set_ringparam, - .get_rxnfc = mlx4_en_get_rxnfc, - .set_rxnfc = mlx4_en_set_rxnfc, - .get_rxfh_indir_size = mlx4_en_get_rxfh_indir_size, - .get_rxfh_indir = mlx4_en_get_rxfh_indir, - .set_rxfh_indir = mlx4_en_set_rxfh_indir, - .get_channels = mlx4_en_get_channels, - .set_channels = mlx4_en_set_channels, - .get_ts_info = mlx4_en_get_ts_info, -}; - - - - - diff --git a/sys/ofed/drivers/net/mlx4/en_main.c b/sys/ofed/drivers/net/mlx4/en_main.c index 757afe071..09c182474 100644 --- a/sys/ofed/drivers/net/mlx4/en_main.c +++ b/sys/ofed/drivers/net/mlx4/en_main.c @@ -42,14 +42,7 @@ #include "mlx4_en.h" -MODULE_AUTHOR("Liran Liss, Yevgeny Petrilin"); -MODULE_DESCRIPTION("Mellanox ConnectX HCA Ethernet driver"); -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_VERSION(DRV_VERSION " ("DRV_RELDATE")"); - -static const char mlx4_en_version[] = - DRV_NAME ": Mellanox ConnectX HCA Ethernet driver v" - DRV_VERSION " (" DRV_RELDATE ")\n"; +/* Mellanox ConnectX HCA Ethernet driver */ #define MLX4_EN_PARM_INT(X, def_val, desc) \ static unsigned int X = def_val;\ @@ -174,8 +167,6 @@ static void *mlx4_en_add(struct mlx4_dev *dev) int i; int err; - printk_once(KERN_INFO "%s", mlx4_en_version); - mdev = kzalloc(sizeof *mdev, GFP_KERNEL); if (!mdev) { dev_err(&dev->pdev->dev, "Device struct alloc failed, " diff --git a/sys/ofed/drivers/net/mlx4/en_netdev.c b/sys/ofed/drivers/net/mlx4/en_netdev.c index d8b2f4fbd..0c7b30020 100644 --- a/sys/ofed/drivers/net/mlx4/en_netdev.c +++ b/sys/ofed/drivers/net/mlx4/en_netdev.c @@ -659,8 +659,10 @@ static void mlx4_en_cache_mclist(struct net_device *dev) continue; /* Make sure the list didn't grow. */ tmp = kzalloc(sizeof(struct mlx4_en_mc_list), GFP_ATOMIC); - if (tmp == NULL) + if (tmp == NULL) { + en_err(priv, "Failed to allocate multicast list\n"); break; + } memcpy(tmp->addr, LLADDR((struct sockaddr_dl *)ifma->ifma_addr), ETH_ALEN); list_add_tail(&tmp->list, &priv->mc_list); @@ -971,12 +973,12 @@ static void mlx4_en_do_set_rx_mode(struct work_struct *work) if (!mlx4_en_QUERY_PORT(mdev, priv->port)) { if (priv->port_state.link_state) { priv->last_link_state = MLX4_DEV_EVENT_PORT_UP; - /* Important note: the following call for if_link_state_change - * is needed for interface up scenario (start port, link state - * change) */ /* update netif baudrate */ priv->dev->if_baudrate = IF_Mbps(priv->port_state.link_speed); + /* Important note: the following call for if_link_state_change + * is needed for interface up scenario (start port, link state + * change) */ if_link_state_change(priv->dev, LINK_STATE_UP); en_dbg(HW, priv, "Link Up\n"); } @@ -1196,8 +1198,8 @@ static void mlx4_en_linkstate(struct work_struct *work) /* update netif baudrate */ priv->dev->if_baudrate = 0; - /* make sure the port is up before notifying the OS. - * This is tricky since we get here on INIT_PORT and + /* make sure the port is up before notifying the OS. + * This is tricky since we get here on INIT_PORT and * in such case we can't tell the OS the port is up. * To solve this there is a call to if_link_state_change * in set_rx_mode. @@ -1246,7 +1248,6 @@ int mlx4_en_start_port(struct net_device *dev) PAGE_SIZE); priv->rx_alloc_order = get_order(priv->rx_alloc_size); priv->rx_buf_size = roundup_pow_of_two(priv->rx_mb_size); - priv->log_rx_info = ROUNDUP_LOG2(sizeof(struct mlx4_en_rx_buf)); en_dbg(DRV, priv, "Rx buf size:%d\n", priv->rx_mb_size); /* Configure rx cq's and rings */ @@ -1575,6 +1576,7 @@ static void mlx4_en_clear_stats(struct net_device *dev) priv->tx_ring[i]->bytes = 0; priv->tx_ring[i]->packets = 0; priv->tx_ring[i]->tx_csum = 0; + priv->tx_ring[i]->oversized_packets = 0; } for (i = 0; i < priv->rx_ring_num; i++) { priv->rx_ring[i]->bytes = 0; @@ -1644,8 +1646,6 @@ void mlx4_en_free_resources(struct mlx4_en_priv *priv) if (priv->sysctl) sysctl_ctx_free(&priv->stat_ctx); - - } int mlx4_en_alloc_resources(struct mlx4_en_priv *priv) @@ -1730,8 +1730,11 @@ void mlx4_en_destroy_netdev(struct net_device *dev) EVENTHANDLER_DEREGISTER(vlan_unconfig, priv->vlan_detach); /* Unregister device - this will close the port if it was up */ - if (priv->registered) + if (priv->registered) { + mutex_lock(&mdev->state_lock); ether_ifdetach(dev); + mutex_unlock(&mdev->state_lock); + } if (priv->allocated) mlx4_free_hwq_res(mdev->dev, &priv->res, MLX4_EN_PAGE_SIZE); @@ -1809,13 +1812,6 @@ static int mlx4_en_calc_media(struct mlx4_en_priv *priv) active = IFM_ETHER; if (priv->last_link_state == MLX4_DEV_EVENT_PORT_DOWN) return (active); - /* - * [ShaharK] mlx4_en_QUERY_PORT sleeps and cannot be called under a - * non-sleepable lock. - * I moved it to the periodic mlx4_en_do_get_stats. - if (mlx4_en_QUERY_PORT(priv->mdev, priv->port)) - return (active); - */ active |= IFM_FDX; trans_type = priv->port_state.transciver; /* XXX I don't know all of the transceiver values. */ @@ -1948,12 +1944,55 @@ static int mlx4_en_ioctl(struct ifnet *dev, u_long command, caddr_t data) case SIOCSIFCAP: mutex_lock(&mdev->state_lock); mask = ifr->ifr_reqcap ^ dev->if_capenable; - if (mask & IFCAP_HWCSUM) - dev->if_capenable ^= IFCAP_HWCSUM; - if (mask & IFCAP_TSO4) + if (mask & IFCAP_TXCSUM) { + dev->if_capenable ^= IFCAP_TXCSUM; + dev->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); + + if (IFCAP_TSO4 & dev->if_capenable && + !(IFCAP_TXCSUM & dev->if_capenable)) { + dev->if_capenable &= ~IFCAP_TSO4; + dev->if_hwassist &= ~CSUM_IP_TSO; + if_printf(dev, + "tso4 disabled due to -txcsum.\n"); + } + } + if (mask & IFCAP_TXCSUM_IPV6) { + dev->if_capenable ^= IFCAP_TXCSUM_IPV6; + dev->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); + + if (IFCAP_TSO6 & dev->if_capenable && + !(IFCAP_TXCSUM_IPV6 & dev->if_capenable)) { + dev->if_capenable &= ~IFCAP_TSO6; + dev->if_hwassist &= ~CSUM_IP6_TSO; + if_printf(dev, + "tso6 disabled due to -txcsum6.\n"); + } + } + if (mask & IFCAP_RXCSUM) + dev->if_capenable ^= IFCAP_RXCSUM; + if (mask & IFCAP_RXCSUM_IPV6) + dev->if_capenable ^= IFCAP_RXCSUM_IPV6; + + if (mask & IFCAP_TSO4) { + if (!(IFCAP_TSO4 & dev->if_capenable) && + !(IFCAP_TXCSUM & dev->if_capenable)) { + if_printf(dev, "enable txcsum first.\n"); + error = EAGAIN; + goto out; + } dev->if_capenable ^= IFCAP_TSO4; - if (mask & IFCAP_TSO6) + dev->if_hwassist ^= CSUM_IP_TSO; + } + if (mask & IFCAP_TSO6) { + if (!(IFCAP_TSO6 & dev->if_capenable) && + !(IFCAP_TXCSUM_IPV6 & dev->if_capenable)) { + if_printf(dev, "enable txcsum6 first.\n"); + error = EAGAIN; + goto out; + } dev->if_capenable ^= IFCAP_TSO6; + dev->if_hwassist ^= CSUM_IP6_TSO; + } if (mask & IFCAP_LRO) dev->if_capenable ^= IFCAP_LRO; if (mask & IFCAP_VLAN_HWTAGGING) @@ -1964,9 +2003,11 @@ static int mlx4_en_ioctl(struct ifnet *dev, u_long command, caddr_t data) dev->if_capenable ^= IFCAP_WOL_MAGIC; if (dev->if_drv_flags & IFF_DRV_RUNNING) mlx4_en_start_port(dev); +out: mutex_unlock(&mdev->state_lock); VLAN_CAPABILITIES(dev); break; +#if __FreeBSD_version >= 1100036 case SIOCGI2C: { struct ifi2creq i2c; @@ -1990,6 +2031,7 @@ static int mlx4_en_ioctl(struct ifnet *dev, u_long command, caddr_t data) error = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); break; } +#endif default: error = ether_ioctl(dev, command, data); break; @@ -2049,8 +2091,6 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, priv->port = port; priv->port_up = false; priv->flags = prof->flags; - priv->ctrl_flags = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE | - MLX4_WQE_CTRL_SOLICITED); priv->num_tx_rings_p_up = mdev->profile.num_tx_rings_p_up; priv->tx_ring_num = prof->tx_ring_num; @@ -2066,7 +2106,7 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, err = -ENOMEM; goto out; } - + priv->rx_ring_num = prof->rx_ring_num; priv->cqe_factor = (mdev->dev->caps.cqe_size == 64) ? 1 : 0; priv->mac_index = -1; @@ -2089,7 +2129,6 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i) INIT_HLIST_HEAD(&priv->mac_hash[i]); - /* Query for default mac and max mtu */ priv->max_mtu = mdev->dev->caps.eth_mtu_cap[priv->port]; priv->mac = mdev->dev->caps.def_mac[priv->port]; @@ -2105,8 +2144,6 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, goto out; } - - priv->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + DS_SIZE); @@ -2128,7 +2165,7 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, /* * Set driver features */ - dev->if_capabilities |= IFCAP_RXCSUM | IFCAP_TXCSUM; + dev->if_capabilities |= IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6; dev->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING; dev->if_capabilities |= IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWFILTER; dev->if_capabilities |= IFCAP_LINKSTATE | IFCAP_JUMBO_MTU; @@ -2137,10 +2174,12 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, if (mdev->LSO_support) dev->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6 | IFCAP_VLAN_HWTSO; +#if __FreeBSD_version >= 1100000 /* set TSO limits so that we don't have to drop TX packets */ - dev->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); - dev->if_hw_tsomaxsegcount = 16; - dev->if_hw_tsomaxsegsize = 65536; /* XXX can do up to 4GByte */ + dev->if_hw_tsomax = MLX4_EN_TX_MAX_PAYLOAD_SIZE - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN) /* hdr */; + dev->if_hw_tsomaxsegcount = MLX4_EN_TX_MAX_MBUF_FRAGS - 1 /* hdr */; + dev->if_hw_tsomaxsegsize = MLX4_EN_TX_MAX_MBUF_SIZE; +#endif dev->if_capenable = dev->if_capabilities; @@ -2149,6 +2188,8 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, dev->if_hwassist |= CSUM_TSO; if (dev->if_capenable & IFCAP_TXCSUM) dev->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP); + if (dev->if_capenable & IFCAP_TXCSUM_IPV6) + dev->if_hwassist |= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); /* Register for VLAN events */ @@ -2211,8 +2252,6 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) queue_delayed_work(mdev->workqueue, &priv->service_task, SERVICE_TASK_DELAY); - - return 0; out: @@ -2294,6 +2333,162 @@ static int mlx4_en_set_tx_ring_size(SYSCTL_HANDLER_ARGS) return (error); } +static int mlx4_en_get_module_info(struct net_device *dev, + struct ethtool_modinfo *modinfo) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + int ret; + u8 data[4]; + + /* Read first 2 bytes to get Module & REV ID */ + ret = mlx4_get_module_info(mdev->dev, priv->port, + 0/*offset*/, 2/*size*/, data); + + if (ret < 2) { + en_err(priv, "Failed to read eeprom module first two bytes, error: 0x%x\n", -ret); + return -EIO; + } + + switch (data[0] /* identifier */) { + case MLX4_MODULE_ID_QSFP: + modinfo->type = ETH_MODULE_SFF_8436; + modinfo->eeprom_len = ETH_MODULE_SFF_8436_LEN; + break; + case MLX4_MODULE_ID_QSFP_PLUS: + if (data[1] >= 0x3) { /* revision id */ + modinfo->type = ETH_MODULE_SFF_8636; + modinfo->eeprom_len = ETH_MODULE_SFF_8636_LEN; + } else { + modinfo->type = ETH_MODULE_SFF_8436; + modinfo->eeprom_len = ETH_MODULE_SFF_8436_LEN; + } + break; + case MLX4_MODULE_ID_QSFP28: + modinfo->type = ETH_MODULE_SFF_8636; + modinfo->eeprom_len = ETH_MODULE_SFF_8636_LEN; + break; + case MLX4_MODULE_ID_SFP: + modinfo->type = ETH_MODULE_SFF_8472; + modinfo->eeprom_len = ETH_MODULE_SFF_8472_LEN; + break; + default: + en_err(priv, "mlx4_en_get_module_info : Not recognized cable type\n"); + return -EINVAL; + } + + return 0; +} + +static int mlx4_en_get_module_eeprom(struct net_device *dev, + struct ethtool_eeprom *ee, + u8 *data) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + int offset = ee->offset; + int i = 0, ret; + + if (ee->len == 0) + return -EINVAL; + + memset(data, 0, ee->len); + + while (i < ee->len) { + en_dbg(DRV, priv, + "mlx4_get_module_info i(%d) offset(%d) len(%d)\n", + i, offset, ee->len - i); + + ret = mlx4_get_module_info(mdev->dev, priv->port, + offset, ee->len - i, data + i); + + if (!ret) /* Done reading */ + return 0; + + if (ret < 0) { + en_err(priv, + "mlx4_get_module_info i(%d) offset(%d) bytes_to_read(%d) - FAILED (0x%x)\n", + i, offset, ee->len - i, ret); + return -1; + } + + i += ret; + offset += ret; + } + return 0; +} + +static void mlx4_en_print_eeprom(u8 *data, __u32 len) +{ + int i; + int j = 0; + int row = 0; + const int NUM_OF_BYTES = 16; + + printf("\nOffset\t\tValues\n"); + printf("------\t\t------\n"); + while(row < len){ + printf("0x%04x\t\t",row); + for(i=0; i < NUM_OF_BYTES; i++){ + printf("%02x ", data[j]); + row++; + j++; + } + printf("\n"); + } +} + +/* Read cable EEPROM module information by first inspecting the first + * two bytes to get the length and then read the rest of the information. + * The information is printed to dmesg. */ +static int mlx4_en_read_eeprom(SYSCTL_HANDLER_ARGS) +{ + + u8* data; + int error; + int result = 0; + struct mlx4_en_priv *priv; + struct net_device *dev; + struct ethtool_modinfo modinfo; + struct ethtool_eeprom ee; + + error = sysctl_handle_int(oidp, &result, 0, req); + if (error || !req->newptr) + return (error); + + if (result == 1) { + priv = arg1; + dev = priv->dev; + data = kmalloc(PAGE_SIZE, GFP_KERNEL); + + error = mlx4_en_get_module_info(dev, &modinfo); + if (error) { + en_err(priv, + "mlx4_en_get_module_info returned with error - FAILED (0x%x)\n", + -error); + goto out; + } + + ee.len = modinfo.eeprom_len; + ee.offset = 0; + + error = mlx4_en_get_module_eeprom(dev, &ee, data); + if (error) { + en_err(priv, + "mlx4_en_get_module_eeprom returned with error - FAILED (0x%x)\n", + -error); + /* Continue printing partial information in case of an error */ + } + + /* EEPROM information will be printed in dmesg */ + mlx4_en_print_eeprom(data, ee.len); +out: + kfree(data); + } + /* Return zero to prevent sysctl failure. */ + return (0); +} + static int mlx4_en_set_tx_ppp(SYSCTL_HANDLER_ARGS) { struct mlx4_en_priv *priv; @@ -2419,7 +2614,7 @@ static void mlx4_en_sysctl_conf(struct mlx4_en_priv *priv) /* Add coalescer configuration. */ coal = SYSCTL_ADD_NODE(ctx, node_list, OID_AUTO, "coalesce", CTLFLAG_RD, NULL, "Interrupt coalesce configuration"); - coal_list = SYSCTL_CHILDREN(node); + coal_list = SYSCTL_CHILDREN(coal); SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "pkt_rate_low", CTLFLAG_RW, &priv->pkt_rate_low, 0, "Packets per-second for minimum delay"); @@ -2438,11 +2633,14 @@ static void mlx4_en_sysctl_conf(struct mlx4_en_priv *priv) SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "adaptive_rx_coal", CTLFLAG_RW, &priv->adaptive_rx_coal, 0, "Enable adaptive rx coalescing"); + /* EEPROM support */ + SYSCTL_ADD_PROC(ctx, node_list, OID_AUTO, "eeprom_info", + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0, + mlx4_en_read_eeprom, "I", "EEPROM information"); } static void mlx4_en_sysctl_stat(struct mlx4_en_priv *priv) { - struct net_device *dev; struct sysctl_ctx_list *ctx; struct sysctl_oid *node; struct sysctl_oid_list *node_list; @@ -2453,8 +2651,6 @@ static void mlx4_en_sysctl_stat(struct mlx4_en_priv *priv) char namebuf[128]; int i; - dev = priv->dev; - ctx = &priv->stat_ctx; sysctl_ctx_init(ctx); node = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(priv->sysctl), OID_AUTO, @@ -2482,6 +2678,8 @@ static void mlx4_en_sysctl_stat(struct mlx4_en_priv *priv) &priv->port_stats.wake_queue, "Queue resumed after full"); SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_timeout", CTLFLAG_RD, &priv->port_stats.tx_timeout, "Transmit timeouts"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_oversized_packets", CTLFLAG_RD, + &priv->port_stats.oversized_packets, "TX oversized packets, m_defrag failed"); SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_alloc_failed", CTLFLAG_RD, &priv->port_stats.rx_alloc_failed, "RX failed to allocate mbuf"); SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_chksum_good", CTLFLAG_RD, @@ -2565,7 +2763,7 @@ struct mlx4_en_pkt_stats { SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_packets", CTLFLAG_RD, &priv->pkstats.tx_packets, "TX packets"); SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_bytes", CTLFLAG_RD, - &priv->pkstats.tx_packets, "TX Bytes"); + &priv->pkstats.tx_bytes, "TX Bytes"); SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_multicast_packets", CTLFLAG_RD, &priv->pkstats.tx_multicast_packets, "TX Multicast Packets"); SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_broadcast_packets", CTLFLAG_RD, @@ -2606,8 +2804,8 @@ struct mlx4_en_pkt_stats { CTLFLAG_RD, &tx_ring->packets, "TX packets"); SYSCTL_ADD_ULONG(ctx, ring_list, OID_AUTO, "bytes", CTLFLAG_RD, &tx_ring->bytes, "TX bytes"); - } + for (i = 0; i < priv->rx_ring_num; i++) { rx_ring = priv->rx_ring[i]; snprintf(namebuf, sizeof(namebuf), "rx_ring%d", i); diff --git a/sys/ofed/drivers/net/mlx4/en_port.c b/sys/ofed/drivers/net/mlx4/en_port.c index 645c9c4c2..6c71962f2 100644 --- a/sys/ofed/drivers/net/mlx4/en_port.c +++ b/sys/ofed/drivers/net/mlx4/en_port.c @@ -194,6 +194,7 @@ int mlx4_en_DUMP_ETH_STATS(struct mlx4_en_dev *mdev, u8 port, u8 reset) priv->port_stats.tx_chksum_offload += priv->tx_ring[i]->tx_csum; priv->port_stats.queue_stopped += priv->tx_ring[i]->queue_stopped; priv->port_stats.wake_queue += priv->tx_ring[i]->wake_queue; + priv->port_stats.oversized_packets += priv->tx_ring[i]->oversized_packets; } /* RX Statistics */ priv->pkstats.rx_packets = be64_to_cpu(mlx4_en_stats->RTOT_prio_0) + diff --git a/sys/ofed/drivers/net/mlx4/en_rx.c b/sys/ofed/drivers/net/mlx4/en_rx.c index e011b8799..b57bd26c9 100644 --- a/sys/ofed/drivers/net/mlx4/en_rx.c +++ b/sys/ofed/drivers/net/mlx4/en_rx.c @@ -49,106 +49,135 @@ static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, int index) { - struct mlx4_en_rx_desc *rx_desc = ring->buf + ring->stride * index; + struct mlx4_en_rx_desc *rx_desc = (struct mlx4_en_rx_desc *) + (ring->buf + (ring->stride * index)); int possible_frags; int i; - /* Set size and memtype fields */ - for (i = 0; i < priv->num_frags; i++) { - rx_desc->data[i].byte_count = - cpu_to_be32(priv->frag_info[i].frag_size); - rx_desc->data[i].lkey = cpu_to_be32(priv->mdev->mr.key); - } - - /* If the number of used fragments does not fill up the ring stride, - * * remaining (unused) fragments must be padded with null address/size - * * and a special memory key */ + rx_desc->data[0].byte_count = cpu_to_be32(priv->rx_mb_size); + rx_desc->data[0].lkey = cpu_to_be32(priv->mdev->mr.key); + + /* + * If the number of used fragments does not fill up the ring + * stride, remaining (unused) fragments must be padded with + * null address/size and a special memory key: + */ possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE; - for (i = priv->num_frags; i < possible_frags; i++) { + for (i = 1; i < possible_frags; i++) { rx_desc->data[i].byte_count = 0; rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD); rx_desc->data[i].addr = 0; } - } -static int mlx4_en_alloc_buf(struct mlx4_en_priv *priv, - struct mlx4_en_rx_desc *rx_desc, - struct mbuf **mb_list, - int i) +static int +mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring, + __be64 *pdma, struct mlx4_en_rx_mbuf *mb_list) { - struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_en_frag_info *frag_info = &priv->frag_info[i]; + bus_dma_segment_t segs[1]; + bus_dmamap_t map; struct mbuf *mb; - dma_addr_t dma; + int nsegs; + int err; - if (i == 0) - mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, frag_info->frag_size); - else - mb = m_getjcl(M_NOWAIT, MT_DATA, 0, frag_info->frag_size); - if (mb == NULL) { - priv->port_stats.rx_alloc_failed++; - return -ENOMEM; + /* try to allocate a new spare mbuf */ + if (unlikely(ring->spare.mbuf == NULL)) { + mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size); + if (unlikely(mb == NULL)) + return (-ENOMEM); + /* setup correct length */ + mb->m_len = ring->rx_mb_size; + + /* load spare mbuf into BUSDMA */ + err = -bus_dmamap_load_mbuf_sg(ring->dma_tag, ring->spare.dma_map, + mb, segs, &nsegs, BUS_DMA_NOWAIT); + if (unlikely(err != 0)) { + m_freem(mb); + return (err); + } + + /* store spare info */ + ring->spare.mbuf = mb; + ring->spare.paddr_be = cpu_to_be64(segs[0].ds_addr); + + bus_dmamap_sync(ring->dma_tag, ring->spare.dma_map, + BUS_DMASYNC_PREREAD); + } + + /* synchronize and unload the current mbuf, if any */ + if (likely(mb_list->mbuf != NULL)) { + bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, + BUS_DMASYNC_POSTREAD); + bus_dmamap_unload(ring->dma_tag, mb_list->dma_map); } - dma = pci_map_single(mdev->pdev, mb->m_data, frag_info->frag_size, - PCI_DMA_FROMDEVICE); - rx_desc->data[i].addr = cpu_to_be64(dma); - mb_list[i] = mb; - return 0; -} + mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size); + if (unlikely(mb == NULL)) + goto use_spare; -static int mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring *ring, int index) -{ - struct mlx4_en_rx_desc *rx_desc = ring->buf + (index * ring->stride); - struct mbuf **mb_list = ring->rx_info + (index << priv->log_rx_info); - int i; + /* setup correct length */ + mb->m_len = ring->rx_mb_size; - for (i = 0; i < priv->num_frags; i++) - if (mlx4_en_alloc_buf(priv, rx_desc, mb_list, i)) - goto err; + err = -bus_dmamap_load_mbuf_sg(ring->dma_tag, mb_list->dma_map, + mb, segs, &nsegs, BUS_DMA_NOWAIT); + if (unlikely(err != 0)) { + m_freem(mb); + goto use_spare; + } + + *pdma = cpu_to_be64(segs[0].ds_addr); + mb_list->mbuf = mb; + + bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, BUS_DMASYNC_PREREAD); + return (0); - return 0; +use_spare: + /* swap DMA maps */ + map = mb_list->dma_map; + mb_list->dma_map = ring->spare.dma_map; + ring->spare.dma_map = map; -err: - while (i--) - m_free(mb_list[i]); - return -ENOMEM; + /* swap MBUFs */ + mb_list->mbuf = ring->spare.mbuf; + ring->spare.mbuf = NULL; + + /* store physical address */ + *pdma = ring->spare.paddr_be; + return (0); } -static inline void mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring) +static void +mlx4_en_free_buf(struct mlx4_en_rx_ring *ring, struct mlx4_en_rx_mbuf *mb_list) { - *ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff); + bus_dmamap_t map = mb_list->dma_map; + bus_dmamap_sync(ring->dma_tag, map, BUS_DMASYNC_POSTREAD); + bus_dmamap_unload(ring->dma_tag, map); + m_freem(mb_list->mbuf); + mb_list->mbuf = NULL; /* safety clearing */ } -static void mlx4_en_free_rx_desc(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring *ring, - int index) +static int +mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv, + struct mlx4_en_rx_ring *ring, int index) { - struct mlx4_en_frag_info *frag_info; - struct mlx4_en_dev *mdev = priv->mdev; - struct mbuf **mb_list; - struct mlx4_en_rx_desc *rx_desc = ring->buf + (index << ring->log_stride); - dma_addr_t dma; - int nr; - - mb_list = ring->rx_info + (index << priv->log_rx_info); - for (nr = 0; nr < priv->num_frags; nr++) { - en_dbg(DRV, priv, "Freeing fragment:%d\n", nr); - frag_info = &priv->frag_info[nr]; - dma = be64_to_cpu(rx_desc->data[nr].addr); - -#if BITS_PER_LONG == 64 - en_dbg(DRV, priv, "Unmaping buffer at dma:0x%lx\n", (u64) dma); -#elif BITS_PER_LONG == 32 - en_dbg(DRV, priv, "Unmaping buffer at dma:0x%llx\n", (u64) dma); -#endif - pci_unmap_single(mdev->pdev, dma, frag_info->frag_size, - PCI_DMA_FROMDEVICE); - m_free(mb_list[nr]); + struct mlx4_en_rx_desc *rx_desc = (struct mlx4_en_rx_desc *) + (ring->buf + (index * ring->stride)); + struct mlx4_en_rx_mbuf *mb_list = ring->mbuf + index; + + mb_list->mbuf = NULL; + + if (mlx4_en_alloc_buf(ring, &rx_desc->data[0].addr, mb_list)) { + priv->port_stats.rx_alloc_failed++; + return (-ENOMEM); } + return (0); +} + +static inline void +mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring) +{ + *ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff); } static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv *priv) @@ -191,7 +220,8 @@ static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv *priv) while (ring->actual_size > new_size) { ring->actual_size--; ring->prod--; - mlx4_en_free_rx_desc(priv, ring, ring->actual_size); + mlx4_en_free_buf(ring, + ring->mbuf + ring->actual_size); } } @@ -211,100 +241,106 @@ static void mlx4_en_free_rx_buf(struct mlx4_en_priv *priv, while (ring->cons != ring->prod) { index = ring->cons & ring->size_mask; en_dbg(DRV, priv, "Processing descriptor:%d\n", index); - mlx4_en_free_rx_desc(priv, ring, index); + mlx4_en_free_buf(ring, ring->mbuf + index); ++ring->cons; } } -#if MLX4_EN_MAX_RX_FRAGS == 3 -static int frag_sizes[] = { - FRAG_SZ0, - FRAG_SZ1, - FRAG_SZ2, -}; -#elif MLX4_EN_MAX_RX_FRAGS == 2 -static int frag_sizes[] = { - FRAG_SZ0, - FRAG_SZ1, -}; -#else -#error "Unknown MAX_RX_FRAGS" -#endif - void mlx4_en_calc_rx_buf(struct net_device *dev) { struct mlx4_en_priv *priv = netdev_priv(dev); int eff_mtu = dev->if_mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN; - int buf_size = 0; - int i, frag; - for (i = 0, frag = 0; buf_size < eff_mtu; frag++, i++) { - /* - * Allocate small to large but only as much as is needed for - * the tail. - */ - while (i > 0 && eff_mtu - buf_size <= frag_sizes[i - 1]) - i--; - priv->frag_info[frag].frag_size = frag_sizes[i]; - priv->frag_info[frag].frag_prefix_size = buf_size; - buf_size += priv->frag_info[frag].frag_size; - } + if (eff_mtu > MJUM16BYTES) { + en_err(priv, "MTU(%d) is too big\n", (int)dev->if_mtu); + eff_mtu = MJUM16BYTES; + } else if (eff_mtu > MJUM9BYTES) { + eff_mtu = MJUM16BYTES; + } else if (eff_mtu > MJUMPAGESIZE) { + eff_mtu = MJUM9BYTES; + } else if (eff_mtu > MCLBYTES) { + eff_mtu = MJUMPAGESIZE; + } else { + eff_mtu = MCLBYTES; + } - priv->num_frags = frag; priv->rx_mb_size = eff_mtu; - priv->log_rx_info = - ROUNDUP_LOG2(priv->num_frags * sizeof(struct mbuf *)); - en_dbg(DRV, priv, "Rx buffer scatter-list (effective-mtu:%d " - "num_frags:%d):\n", eff_mtu, priv->num_frags); - for (i = 0; i < priv->num_frags; i++) { - en_dbg(DRV, priv, " frag:%d - size:%d prefix:%d\n", i, - priv->frag_info[i].frag_size, - priv->frag_info[i].frag_prefix_size); - } + en_dbg(DRV, priv, "Effective RX MTU: %d bytes\n", eff_mtu); } - int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring **pring, u32 size, int node) { struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_en_rx_ring *ring; - int err = -ENOMEM; + int err; int tmp; + uint32_t x; ring = kzalloc(sizeof(struct mlx4_en_rx_ring), GFP_KERNEL); if (!ring) { en_err(priv, "Failed to allocate RX ring structure\n"); return -ENOMEM; } - + + /* Create DMA descriptor TAG */ + if ((err = -bus_dma_tag_create( + bus_get_dma_tag(mdev->pdev->dev.bsddev), + 1, /* any alignment */ + 0, /* no boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + MJUM16BYTES, /* maxsize */ + 1, /* nsegments */ + MJUM16BYTES, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockfuncarg */ + &ring->dma_tag))) { + en_err(priv, "Failed to create DMA tag\n"); + goto err_ring; + } + ring->prod = 0; ring->cons = 0; ring->size = size; ring->size_mask = size - 1; - ring->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + - DS_SIZE * MLX4_EN_MAX_RX_FRAGS); + ring->stride = roundup_pow_of_two( + sizeof(struct mlx4_en_rx_desc) + DS_SIZE); ring->log_stride = ffs(ring->stride) - 1; ring->buf_size = ring->size * ring->stride + TXBB_SIZE; - tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS * - sizeof(struct mbuf *)); + tmp = size * sizeof(struct mlx4_en_rx_mbuf); - ring->rx_info = kmalloc(tmp, GFP_KERNEL); - if (!ring->rx_info) { + ring->mbuf = kzalloc(tmp, GFP_KERNEL); + if (ring->mbuf == NULL) { err = -ENOMEM; - goto err_ring; + goto err_dma_tag; } - en_dbg(DRV, priv, "Allocated rx_info ring at addr:%p size:%d\n", - ring->rx_info, tmp); + err = -bus_dmamap_create(ring->dma_tag, 0, &ring->spare.dma_map); + if (err != 0) + goto err_info; + + for (x = 0; x != size; x++) { + err = -bus_dmamap_create(ring->dma_tag, 0, + &ring->mbuf[x].dma_map); + if (err != 0) { + while (x--) + bus_dmamap_destroy(ring->dma_tag, + ring->mbuf[x].dma_map); + goto err_info; + } + } + en_dbg(DRV, priv, "Allocated MBUF ring at addr:%p size:%d\n", + ring->mbuf, tmp); err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size, 2 * PAGE_SIZE); if (err) - goto err_info; + goto err_dma_map; err = mlx4_en_map_buffer(&ring->wqres.buf); if (err) { @@ -317,23 +353,29 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, err_hwq: mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); +err_dma_map: + for (x = 0; x != size; x++) { + bus_dmamap_destroy(ring->dma_tag, + ring->mbuf[x].dma_map); + } + bus_dmamap_destroy(ring->dma_tag, ring->spare.dma_map); err_info: - vfree(ring->rx_info); + vfree(ring->mbuf); +err_dma_tag: + bus_dma_tag_destroy(ring->dma_tag); err_ring: kfree(ring); - - return err; + return (err); } - int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv) { struct mlx4_en_rx_ring *ring; int i; int ring_ind; int err; - int stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + - DS_SIZE * priv->num_frags); + int stride = roundup_pow_of_two( + sizeof(struct mlx4_en_rx_desc) + DS_SIZE); for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { ring = priv->rx_ring[ring_ind]; @@ -409,10 +451,22 @@ void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, { struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_en_rx_ring *ring = *pring; + uint32_t x; mlx4_en_unmap_buffer(&ring->wqres.buf); mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE); - vfree(ring->rx_info); + for (x = 0; x != size; x++) + bus_dmamap_destroy(ring->dma_tag, ring->mbuf[x].dma_map); + /* free spare mbuf, if any */ + if (ring->spare.mbuf != NULL) { + bus_dmamap_sync(ring->dma_tag, ring->spare.dma_map, + BUS_DMASYNC_POSTREAD); + bus_dmamap_unload(ring->dma_tag, ring->spare.dma_map); + m_freem(ring->spare.mbuf); + } + bus_dmamap_destroy(ring->dma_tag, ring->spare.dma_map); + vfree(ring->mbuf); + bus_dma_tag_destroy(ring->dma_tag); kfree(ring); *pring = NULL; #ifdef CONFIG_RFS_ACCEL @@ -420,7 +474,6 @@ void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, #endif } - void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring) { @@ -469,69 +522,27 @@ static inline int invalid_cqe(struct mlx4_en_priv *priv, return 0; } - -/* Unmap a completed descriptor and free unused pages */ -static int mlx4_en_complete_rx_desc(struct mlx4_en_priv *priv, - struct mlx4_en_rx_desc *rx_desc, - struct mbuf **mb_list, - int length) +static struct mbuf * +mlx4_en_rx_mb(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, + struct mlx4_en_rx_desc *rx_desc, struct mlx4_en_rx_mbuf *mb_list, + int length) { - struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_en_frag_info *frag_info; - dma_addr_t dma; struct mbuf *mb; - int nr; - - mb = mb_list[0]; - mb->m_pkthdr.len = length; - /* Collect used fragments while replacing them in the HW descirptors */ - for (nr = 0; nr < priv->num_frags; nr++) { - frag_info = &priv->frag_info[nr]; - if (length <= frag_info->frag_prefix_size) - break; - if (nr) - mb->m_next = mb_list[nr]; - mb = mb_list[nr]; - mb->m_len = frag_info->frag_size; - dma = be64_to_cpu(rx_desc->data[nr].addr); - - /* Allocate a replacement page */ - if (mlx4_en_alloc_buf(priv, rx_desc, mb_list, nr)) - goto fail; - - /* Unmap buffer */ - pci_unmap_single(mdev->pdev, dma, frag_info->frag_size, - PCI_DMA_FROMDEVICE); - } - /* Adjust size of last fragment to match actual length */ - mb->m_len = length - priv->frag_info[nr - 1].frag_prefix_size; - mb->m_next = NULL; - return 0; -fail: - /* Drop all accumulated fragments (which have already been replaced in - * the descriptor) of this packet; remaining fragments are reused... */ - while (nr > 0) { - nr--; - m_free(mb_list[nr]); - } - return -ENOMEM; - -} + /* get mbuf */ + mb = mb_list->mbuf; -static struct mbuf *mlx4_en_rx_mb(struct mlx4_en_priv *priv, - struct mlx4_en_rx_desc *rx_desc, - struct mbuf **mb_list, - unsigned int length) -{ - struct mbuf *mb; + /* collect used fragment while atomically replacing it */ + if (mlx4_en_alloc_buf(ring, &rx_desc->data[0].addr, mb_list)) + return (NULL); - mb = mb_list[0]; - /* Move relevant fragments to mb */ - if (unlikely(mlx4_en_complete_rx_desc(priv, rx_desc, mb_list, length))) - return NULL; + /* range check hardware computed value */ + if (unlikely(length > mb->m_len)) + length = mb->m_len; - return mb; + /* update total packet length in packet header */ + mb->m_len = mb->m_pkthdr.len = length; + return (mb); } /* For cpu arch with cache line of 64B the performance is better when cqe size==64B @@ -545,7 +556,7 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_cqe *cqe; struct mlx4_en_rx_ring *ring = priv->rx_ring[cq->ring]; - struct mbuf **mb_list; + struct mlx4_en_rx_mbuf *mb_list; struct mlx4_en_rx_desc *rx_desc; struct mbuf *mb; struct mlx4_cq *mcq = &cq->mcq; @@ -573,8 +584,9 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud /* Process all completed CQEs */ while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, cons_index & size)) { - mb_list = ring->rx_info + (index << priv->log_rx_info); - rx_desc = ring->buf + (index << ring->log_stride); + mb_list = ring->mbuf + index; + rx_desc = (struct mlx4_en_rx_desc *) + (ring->buf + (index << ring->log_stride)); /* * make sure we read the CQE after we read the ownership bit @@ -589,8 +601,9 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud */ length = be32_to_cpu(cqe->byte_cnt); length -= ring->fcs_del; - mb = mlx4_en_rx_mb(priv, rx_desc, mb_list, length); - if (!mb) { + + mb = mlx4_en_rx_mb(priv, ring, rx_desc, mb_list, length); + if (unlikely(!mb)) { ring->errors++; goto next; } @@ -611,7 +624,8 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud mb->m_pkthdr.ether_vtag = be16_to_cpu(cqe->sl_vid); mb->m_flags |= M_VLANTAG; } - if (likely(dev->if_capabilities & IFCAP_RXCSUM) && + if (likely(dev->if_capenable & + (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) && (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) && (cqe->checksum == cpu_to_be16(0xffff))) { priv->port_stats.rx_chksum_good++; @@ -692,6 +706,7 @@ void mlx4_en_rx_irq(struct mlx4_cq *mcq) // Because there is no NAPI in freeBSD done = mlx4_en_poll_rx_cq(cq, MLX4_EN_RX_BUDGET); if (priv->port_up && (done == MLX4_EN_RX_BUDGET) ) { + cq->curr_poll_rx_cpu_id = curcpu; taskqueue_enqueue(cq->tq, &cq->cq_task); } else { @@ -702,8 +717,15 @@ void mlx4_en_rx_irq(struct mlx4_cq *mcq) void mlx4_en_rx_que(void *context, int pending) { struct mlx4_en_cq *cq; + struct thread *td; cq = context; + td = curthread; + + thread_lock(td); + sched_bind(td, cq->curr_poll_rx_cpu_id); + thread_unlock(td); + while (mlx4_en_poll_rx_cq(cq, MLX4_EN_RX_BUDGET) == MLX4_EN_RX_BUDGET); mlx4_en_arm_cq(cq->dev->if_softc, cq); @@ -841,8 +863,8 @@ int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv) else rss_rings = priv->prof->rss_rings; - ptr = ((void *) &context) + offsetof(struct mlx4_qp_context, pri_path) - + MLX4_RSS_OFFSET_IN_QPC_PRI_PATH; + ptr = ((u8 *)&context) + offsetof(struct mlx4_qp_context, pri_path) + + MLX4_RSS_OFFSET_IN_QPC_PRI_PATH; rss_context = ptr; rss_context->base_qpn = cpu_to_be32(ilog2(rss_rings) << 24 | (rss_map->base_qpn)); diff --git a/sys/ofed/drivers/net/mlx4/en_selftest.c b/sys/ofed/drivers/net/mlx4/en_selftest.c deleted file mode 100644 index fb13bd691..000000000 --- a/sys/ofed/drivers/net/mlx4/en_selftest.c +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include -#include -#include -#include - -#include "mlx4_en.h" - - -static int mlx4_en_test_registers(struct mlx4_en_priv *priv) -{ - return mlx4_cmd(priv->mdev->dev, 0, 0, 0, MLX4_CMD_HW_HEALTH_CHECK, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); -} - -static int mlx4_en_test_loopback_xmit(struct mlx4_en_priv *priv) -{ - struct sk_buff *skb; - struct ethhdr *ethh; - unsigned char *packet; - unsigned int packet_size = MLX4_LOOPBACK_TEST_PAYLOAD; - unsigned int i; - int err; - - - /* build the pkt before xmit */ - skb = netdev_alloc_skb(priv->dev, MLX4_LOOPBACK_TEST_PAYLOAD + ETH_HLEN + NET_IP_ALIGN); - if (!skb) { - en_err(priv, "-LOOPBACK_TEST_XMIT- failed to create skb for xmit\n"); - return -ENOMEM; - } - skb_reserve(skb, NET_IP_ALIGN); - - ethh = (struct ethhdr *)skb_put(skb, sizeof(struct ethhdr)); - packet = (unsigned char *)skb_put(skb, packet_size); - memcpy(ethh->h_dest, priv->dev->dev_addr, ETH_ALEN); - memset(ethh->h_source, 0, ETH_ALEN); - ethh->h_proto = htons(ETH_P_ARP); - skb_set_mac_header(skb, 0); - for (i = 0; i < packet_size; ++i) /* fill our packet */ - packet[i] = (unsigned char)(i & 0xff); - - /* xmit the pkt */ - err = mlx4_en_xmit(skb, priv->dev); - return err; -} - -static int mlx4_en_test_loopback(struct mlx4_en_priv *priv) -{ - u32 loopback_ok = 0; - int i; - - - priv->loopback_ok = 0; - priv->validate_loopback = 1; - - mlx4_en_update_loopback_state(priv->dev, priv->dev->features); - - /* xmit */ - if (mlx4_en_test_loopback_xmit(priv)) { - en_err(priv, "Transmitting loopback packet failed\n"); - goto mlx4_en_test_loopback_exit; - } - - /* polling for result */ - for (i = 0; i < MLX4_EN_LOOPBACK_RETRIES; ++i) { - msleep(MLX4_EN_LOOPBACK_TIMEOUT); - if (priv->loopback_ok) { - loopback_ok = 1; - break; - } - } - if (!loopback_ok) - en_err(priv, "Loopback packet didn't arrive\n"); - -mlx4_en_test_loopback_exit: - - priv->validate_loopback = 0; - mlx4_en_update_loopback_state(priv->dev, priv->dev->features); - return !loopback_ok; -} - - -static int mlx4_en_test_link(struct mlx4_en_priv *priv) -{ - if (mlx4_en_QUERY_PORT(priv->mdev, priv->port)) - return -ENOMEM; - if (priv->port_state.link_state == 1) - return 0; - else - return 1; -} - -static int mlx4_en_test_speed(struct mlx4_en_priv *priv) -{ - - if (mlx4_en_QUERY_PORT(priv->mdev, priv->port)) - return -ENOMEM; - - /* The device supports 1G, 10G and 40G speed */ - if (priv->port_state.link_speed != MLX4_EN_LINK_SPEED_1G && - priv->port_state.link_speed != MLX4_EN_LINK_SPEED_10G && - priv->port_state.link_speed != MLX4_EN_LINK_SPEED_40G) - return priv->port_state.link_speed; - return 0; -} - - -void mlx4_en_ex_selftest(struct net_device *dev, u32 *flags, u64 *buf) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - int i, carrier_ok; - - memset(buf, 0, sizeof(u64) * MLX4_EN_NUM_SELF_TEST); - - if (*flags & ETH_TEST_FL_OFFLINE) { - /* disable the interface */ - carrier_ok = netif_carrier_ok(dev); - - netif_carrier_off(dev); - /* Wait until all tx queues are empty. - * there should not be any additional incoming traffic - * since we turned the carrier off */ - msleep(200); - - if (priv->mdev->dev->caps.flags & - MLX4_DEV_CAP_FLAG_UC_LOOPBACK) { - buf[3] = mlx4_en_test_registers(priv); - if (priv->port_up) - buf[4] = mlx4_en_test_loopback(priv); - } - - if (carrier_ok) - netif_carrier_on(dev); - - } - buf[0] = mlx4_test_interrupts(mdev->dev); - buf[1] = mlx4_en_test_link(priv); - buf[2] = mlx4_en_test_speed(priv); - - for (i = 0; i < MLX4_EN_NUM_SELF_TEST; i++) { - if (buf[i]) - *flags |= ETH_TEST_FL_FAILED; - } -} diff --git a/sys/ofed/drivers/net/mlx4/en_tx.c b/sys/ofed/drivers/net/mlx4/en_tx.c index 115dd0beb..c147dd3b5 100644 --- a/sys/ofed/drivers/net/mlx4/en_tx.c +++ b/sys/ofed/drivers/net/mlx4/en_tx.c @@ -68,6 +68,7 @@ int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, { struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_en_tx_ring *ring; + uint32_t x; int tmp; int err; @@ -80,11 +81,26 @@ int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, } } + /* Create DMA descriptor TAG */ + if ((err = -bus_dma_tag_create( + bus_get_dma_tag(mdev->pdev->dev.bsddev), + 1, /* any alignment */ + 0, /* no boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + MLX4_EN_TX_MAX_PAYLOAD_SIZE, /* maxsize */ + MLX4_EN_TX_MAX_MBUF_FRAGS, /* nsegments */ + MLX4_EN_TX_MAX_MBUF_SIZE, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockfuncarg */ + &ring->dma_tag))) + goto done; + ring->size = size; ring->size_mask = size - 1; ring->stride = stride; - ring->full_size = ring->size - HEADROOM - MAX_DESC_TXBBS; - ring->inline_thold = min(inline_thold, MAX_INLINE); + ring->inline_thold = MAX(MIN_PKT_LEN, MIN(inline_thold, MAX_INLINE)); mtx_init(&ring->tx_lock.m, "mlx4 tx", NULL, MTX_DEF); mtx_init(&ring->comp_lock.m, "mlx4 comp", NULL, MTX_DEF); @@ -93,30 +109,36 @@ int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, M_WAITOK, &ring->tx_lock.m); if (ring->br == NULL) { en_err(priv, "Failed allocating tx_info ring\n"); - return -ENOMEM; + err = -ENOMEM; + goto err_free_dma_tag; } tmp = size * sizeof(struct mlx4_en_tx_info); - ring->tx_info = vmalloc_node(tmp, node); + ring->tx_info = kzalloc_node(tmp, GFP_KERNEL, node); if (!ring->tx_info) { - ring->tx_info = vmalloc(tmp); + ring->tx_info = kzalloc(tmp, GFP_KERNEL); if (!ring->tx_info) { err = -ENOMEM; goto err_ring; } } - en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n", - ring->tx_info, tmp); - - ring->bounce_buf = kmalloc_node(MAX_DESC_SIZE, GFP_KERNEL, node); - if (!ring->bounce_buf) { - ring->bounce_buf = kmalloc(MAX_DESC_SIZE, GFP_KERNEL); - if (!ring->bounce_buf) { - err = -ENOMEM; + /* Create DMA descriptor MAPs */ + for (x = 0; x != size; x++) { + err = -bus_dmamap_create(ring->dma_tag, 0, + &ring->tx_info[x].dma_map); + if (err != 0) { + while (x--) { + bus_dmamap_destroy(ring->dma_tag, + ring->tx_info[x].dma_map); + } goto err_info; } } + + en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n", + ring->tx_info, tmp); + ring->buf_size = ALIGN(size * ring->stride, MLX4_EN_PAGE_SIZE); /* Allocate HW buffers on provided NUMA node */ @@ -124,7 +146,7 @@ int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, 2 * PAGE_SIZE); if (err) { en_err(priv, "Failed allocating hwq resources\n"); - goto err_bounce; + goto err_dma_map; } err = mlx4_en_map_buffer(&ring->wqres.buf); @@ -174,12 +196,16 @@ int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, mlx4_en_unmap_buffer(&ring->wqres.buf); err_hwq_res: mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); -err_bounce: - kfree(ring->bounce_buf); +err_dma_map: + for (x = 0; x != size; x++) + bus_dmamap_destroy(ring->dma_tag, ring->tx_info[x].dma_map); err_info: vfree(ring->tx_info); err_ring: buf_ring_free(ring->br, M_DEVBUF); +err_free_dma_tag: + bus_dma_tag_destroy(ring->dma_tag); +done: kfree(ring); return err; } @@ -189,6 +215,7 @@ void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv, { struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_en_tx_ring *ring = *pring; + uint32_t x; en_dbg(DRV, priv, "Destroying tx ring, qpn: %d\n", ring->qpn); buf_ring_free(ring->br, M_DEVBUF); @@ -199,10 +226,12 @@ void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv, mlx4_qp_release_range(priv->mdev->dev, ring->qpn, 1); mlx4_en_unmap_buffer(&ring->wqres.buf); mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); - kfree(ring->bounce_buf); + for (x = 0; x != ring->size; x++) + bus_dmamap_destroy(ring->dma_tag, ring->tx_info[x].dma_map); vfree(ring->tx_info); mtx_destroy(&ring->tx_lock.m); mtx_destroy(&ring->comp_lock.m); + bus_dma_tag_destroy(ring->dma_tag); kfree(ring); *pring = NULL; } @@ -220,7 +249,6 @@ int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv, ring->last_nr_txbb = 1; ring->poll_cnt = 0; ring->blocked = 0; - memset(ring->tx_info, 0, ring->size * sizeof(struct mlx4_en_tx_info)); memset(ring->buf, 0, ring->buf_size); ring->qp_state = MLX4_QP_STATE_RST; @@ -245,96 +273,63 @@ void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv *priv, MLX4_QP_STATE_RST, NULL, 0, 0, &ring->qp); } -static void mlx4_en_stamp_wqe(struct mlx4_en_priv *priv, - struct mlx4_en_tx_ring *ring, - int index, u8 owner) +static volatile struct mlx4_wqe_data_seg * +mlx4_en_store_inline_lso_data(volatile struct mlx4_wqe_data_seg *dseg, + struct mbuf *mb, int len, __be32 owner_bit) { - struct mlx4_en_tx_info *tx_info = &ring->tx_info[index]; - struct mlx4_en_tx_desc *tx_desc = ring->buf + index * TXBB_SIZE; - void *end = ring->buf + ring->buf_size; - __be32 *ptr = (__be32 *)tx_desc; - __be32 stamp = cpu_to_be32(STAMP_VAL | (!!owner << STAMP_SHIFT)); - int i; - - /* Optimize the common case when there are no wraparounds */ - if (likely((void *)tx_desc + tx_info->nr_txbb * TXBB_SIZE <= end)) - /* Stamp the freed descriptor */ - for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE; i += STAMP_STRIDE) { - *ptr = stamp; - ptr += STAMP_DWORDS; - } - else - /* Stamp the freed descriptor */ - for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE; i += STAMP_STRIDE) { - *ptr = stamp; - ptr += STAMP_DWORDS; - if ((void *)ptr >= end) { - ptr = ring->buf; - stamp ^= cpu_to_be32(0x80000000); - } - } + uint8_t *inl = __DEVOLATILE(uint8_t *, dseg); + + /* copy data into place */ + m_copydata(mb, 0, len, inl + 4); + dseg += DIV_ROUND_UP(4 + len, DS_SIZE_ALIGNMENT); + return (dseg); } -static u32 mlx4_en_free_tx_desc(struct mlx4_en_priv *priv, - struct mlx4_en_tx_ring *ring, - int index, u8 owner, u64 timestamp) +static void +mlx4_en_store_inline_lso_header(volatile struct mlx4_wqe_data_seg *dseg, + int len, __be32 owner_bit) +{ +} + +static void +mlx4_en_stamp_wqe(struct mlx4_en_priv *priv, + struct mlx4_en_tx_ring *ring, u32 index, u8 owner) { - struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_en_tx_info *tx_info = &ring->tx_info[index]; - struct mlx4_en_tx_desc *tx_desc = ring->buf + index * TXBB_SIZE; - struct mlx4_wqe_data_seg *data = (void *) tx_desc + tx_info->data_offset; - struct mbuf *mb = tx_info->mb; - void *end = ring->buf + ring->buf_size; - int frags = tx_info->nr_segs;; - int i; - - /* Optimize the common case when there are no wraparounds */ - if (likely((void *) tx_desc + tx_info->nr_txbb * TXBB_SIZE <= end)) { - if (!tx_info->inl) { - if (tx_info->linear) { - dma_unmap_single(priv->ddev, - (dma_addr_t) be64_to_cpu(data->addr), - be32_to_cpu(data->byte_count), - PCI_DMA_TODEVICE); - ++data; - } + struct mlx4_en_tx_desc *tx_desc = (struct mlx4_en_tx_desc *) + (ring->buf + (index * TXBB_SIZE)); + volatile __be32 *ptr = (__be32 *)tx_desc; + const __be32 stamp = cpu_to_be32(STAMP_VAL | + ((u32)owner << STAMP_SHIFT)); + u32 i; + + /* Stamp the freed descriptor */ + for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE; i += STAMP_STRIDE) { + *ptr = stamp; + ptr += STAMP_DWORDS; + } +} - for (i = 0; i < frags; i++) { - pci_unmap_single(mdev->pdev, - (dma_addr_t) be64_to_cpu(data[i].addr), - data[i].byte_count, PCI_DMA_TODEVICE); - } - } - } else { - if (!tx_info->inl) { - if ((void *) data >= end) { - data = ring->buf + ((void *)data - end); - } +static u32 +mlx4_en_free_tx_desc(struct mlx4_en_priv *priv, + struct mlx4_en_tx_ring *ring, u32 index) +{ + struct mlx4_en_tx_info *tx_info; + struct mbuf *mb; - if (tx_info->linear) { - dma_unmap_single(priv->ddev, - (dma_addr_t) be64_to_cpu(data->addr), - be32_to_cpu(data->byte_count), - PCI_DMA_TODEVICE); - ++data; - } + tx_info = &ring->tx_info[index]; + mb = tx_info->mb; + + if (mb == NULL) + goto done; + + bus_dmamap_sync(ring->dma_tag, tx_info->dma_map, + BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(ring->dma_tag, tx_info->dma_map); - for (i = 0; i < frags; i++) { - /* Check for wraparound before unmapping */ - if ((void *) data >= end) - data = ring->buf; - pci_unmap_single(mdev->pdev, - (dma_addr_t) be64_to_cpu(data->addr), - data->byte_count, PCI_DMA_TODEVICE); - ++data; - } - } - } - /* Send a copy of the frame to the BPF listener */ - if (priv->dev && priv->dev->if_bpf) - ETHER_BPF_MTAP(priv->dev, mb); m_freem(mb); - return tx_info->nr_txbb; +done: + return (tx_info->nr_txbb); } int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring) @@ -354,8 +349,7 @@ int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring) while (ring->cons != ring->prod) { ring->last_nr_txbb = mlx4_en_free_tx_desc(priv, ring, - ring->cons & ring->size_mask, - !!(ring->cons & ring->size), 0); + ring->cons & ring->size_mask); ring->cons += ring->last_nr_txbb; cnt++; } @@ -366,6 +360,14 @@ int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring) return cnt; } +static bool +mlx4_en_tx_ring_is_full(struct mlx4_en_tx_ring *ring) +{ + int wqs; + wqs = ring->size - (ring->prod - ring->cons); + return (wqs < (HEADROOM + (2 * MLX4_EN_TX_WQE_MAX_WQEBBS))); +} + static int mlx4_en_process_tx_cq(struct net_device *dev, struct mlx4_en_cq *cq) { @@ -381,12 +383,7 @@ static int mlx4_en_process_tx_cq(struct net_device *dev, int size = cq->size; u32 size_mask = ring->size_mask; struct mlx4_cqe *buf = cq->buf; - u32 packets = 0; - u32 bytes = 0; int factor = priv->cqe_factor; - u64 timestamp = 0; - int done = 0; - if (!priv->port_up) return 0; @@ -421,16 +418,12 @@ static int mlx4_en_process_tx_cq(struct net_device *dev, ring_index = (ring_index + ring->last_nr_txbb) & size_mask; /* free next descriptor */ ring->last_nr_txbb = mlx4_en_free_tx_desc( - priv, ring, ring_index, - !!((ring->cons + txbbs_skipped) & - ring->size), timestamp); + priv, ring, ring_index); mlx4_en_stamp_wqe(priv, ring, stamp_index, !!((ring->cons + txbbs_stamp) & ring->size)); stamp_index = ring_index; txbbs_stamp = txbbs_skipped; - packets++; - bytes += ring->tx_info[ring_index].nr_bytes; } while (ring_index != new_index); ++cons_index; @@ -449,15 +442,14 @@ static int mlx4_en_process_tx_cq(struct net_device *dev, ring->cons += txbbs_skipped; /* Wakeup Tx queue if it was stopped and ring is not full */ - if (unlikely(ring->blocked) && - (ring->prod - ring->cons) <= ring->full_size) { + if (unlikely(ring->blocked) && !mlx4_en_tx_ring_is_full(ring)) { ring->blocked = 0; if (atomic_fetchadd_int(&priv->blocked, -1) == 1) atomic_clear_int(&dev->if_drv_flags ,IFF_DRV_OACTIVE); ring->wake_queue++; priv->port_stats.wake_queue++; } - return done; + return (0); } void mlx4_en_tx_irq(struct mlx4_cq *mcq) @@ -498,34 +490,6 @@ void mlx4_en_poll_tx_cq(unsigned long data) spin_unlock(&ring->comp_lock); } -static struct mlx4_en_tx_desc *mlx4_en_bounce_to_desc(struct mlx4_en_priv *priv, - struct mlx4_en_tx_ring *ring, - u32 index, - unsigned int desc_size) -{ - u32 copy = (ring->size - index) * TXBB_SIZE; - int i; - - for (i = desc_size - copy - 4; i >= 0; i -= 4) { - if ((i & (TXBB_SIZE - 1)) == 0) - wmb(); - - *((u32 *) (ring->buf + i)) = - *((u32 *) (ring->bounce_buf + copy + i)); - } - - for (i = copy - 4; i >= 4 ; i -= 4) { - if ((i & (TXBB_SIZE - 1)) == 0) - wmb(); - - *((u32 *) (ring->buf + index * TXBB_SIZE + i)) = - *((u32 *) (ring->bounce_buf + i)); - } - - /* Return real descriptor location */ - return ring->buf + index * TXBB_SIZE; -} - static inline void mlx4_en_xmit_poll(struct mlx4_en_priv *priv, int tx_ind) { struct mlx4_en_cq *cq = priv->tx_cq[tx_ind]; @@ -544,30 +508,22 @@ static inline void mlx4_en_xmit_poll(struct mlx4_en_priv *priv, int tx_ind) } } -static int is_inline(struct mbuf *mb, int thold) +static u16 +mlx4_en_get_inline_hdr_size(struct mlx4_en_tx_ring *ring, struct mbuf *mb) { - if (thold && mb->m_pkthdr.len <= thold && - (mb->m_pkthdr.csum_flags & CSUM_TSO) == 0) - return 1; + u16 retval; - return 0; -} - -static int inline_size(struct mbuf *mb) -{ - int len; + /* only copy from first fragment, if possible */ + retval = MIN(ring->inline_thold, mb->m_len); - len = mb->m_pkthdr.len; - if (len + CTRL_SIZE + sizeof(struct mlx4_wqe_inline_seg) - <= MLX4_INLINE_ALIGN) - return ALIGN(len + CTRL_SIZE + - sizeof(struct mlx4_wqe_inline_seg), 16); - else - return ALIGN(len + CTRL_SIZE + 2 * - sizeof(struct mlx4_wqe_inline_seg), 16); + /* check for too little data */ + if (unlikely(retval < MIN_PKT_LEN)) + retval = MIN(ring->inline_thold, mb->m_pkthdr.len); + return (retval); } -static int get_head_size(struct mbuf *mb) +static int +mlx4_en_get_header_size(struct mbuf *mb) { struct ether_vlan_header *eh; struct tcphdr *th; @@ -620,83 +576,48 @@ static int get_head_size(struct mbuf *mb) return (eth_hdr_len); } -static int get_real_size(struct mbuf *mb, struct net_device *dev, int *p_n_segs, - int *lso_header_size, int inl) +static volatile struct mlx4_wqe_data_seg * +mlx4_en_store_inline_data(volatile struct mlx4_wqe_data_seg *dseg, + struct mbuf *mb, int len, __be32 owner_bit) { - struct mbuf *m; - int nr_segs = 0; - - for (m = mb; m != NULL; m = m->m_next) - if (m->m_len) - nr_segs++; - - if (mb->m_pkthdr.csum_flags & CSUM_TSO) { - *lso_header_size = get_head_size(mb); - if (*lso_header_size) { - if (mb->m_len == *lso_header_size) - nr_segs--; - *p_n_segs = nr_segs; - return CTRL_SIZE + nr_segs * DS_SIZE + - ALIGN(*lso_header_size + 4, DS_SIZE); - } - } else - *lso_header_size = 0; - *p_n_segs = nr_segs; - if (inl) - return inline_size(mb); - return (CTRL_SIZE + nr_segs * DS_SIZE); -} - -static struct mbuf *mb_copy(struct mbuf *mb, int *offp, char *data, int len) -{ - int bytes; - int off; - - off = *offp; - while (len) { - bytes = min(mb->m_len - off, len); - if (bytes) - memcpy(data, mb->m_data + off, bytes); - len -= bytes; - data += bytes; - off += bytes; - if (off == mb->m_len) { - off = 0; - mb = mb->m_next; - } - } - *offp = off; - return (mb); + uint8_t *inl = __DEVOLATILE(uint8_t *, dseg); + const int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - 4; + + if (unlikely(len < MIN_PKT_LEN)) { + m_copydata(mb, 0, len, inl + 4); + memset(inl + 4 + len, 0, MIN_PKT_LEN - len); + dseg += DIV_ROUND_UP(4 + MIN_PKT_LEN, DS_SIZE_ALIGNMENT); + } else if (len <= spc) { + m_copydata(mb, 0, len, inl + 4); + dseg += DIV_ROUND_UP(4 + len, DS_SIZE_ALIGNMENT); + } else { + m_copydata(mb, 0, spc, inl + 4); + m_copydata(mb, spc, len - spc, inl + 8 + spc); + dseg += DIV_ROUND_UP(8 + len, DS_SIZE_ALIGNMENT); + } + return (dseg); } -static void build_inline_wqe(struct mlx4_en_tx_desc *tx_desc, struct mbuf *mb, - int real_size, u16 *vlan_tag, int tx_ind) +static void +mlx4_en_store_inline_header(volatile struct mlx4_wqe_data_seg *dseg, + int len, __be32 owner_bit) { - struct mlx4_wqe_inline_seg *inl = &tx_desc->inl; - int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - sizeof *inl; - int len; - int off; - - off = 0; - len = mb->m_pkthdr.len; - if (len <= spc) { - inl->byte_count = cpu_to_be32(1 << 31 | - (max_t(typeof(len), len, MIN_PKT_LEN))); - mb_copy(mb, &off, (void *)(inl + 1), len); - if (len < MIN_PKT_LEN) - memset(((void *)(inl + 1)) + len, 0, - MIN_PKT_LEN - len); + uint8_t *inl = __DEVOLATILE(uint8_t *, dseg); + const int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - 4; + + if (unlikely(len < MIN_PKT_LEN)) { + *(volatile uint32_t *)inl = + SET_BYTE_COUNT((1 << 31) | MIN_PKT_LEN); + } else if (len <= spc) { + *(volatile uint32_t *)inl = + SET_BYTE_COUNT((1 << 31) | len); } else { - inl->byte_count = cpu_to_be32(1 << 31 | spc); - mb = mb_copy(mb, &off, (void *)(inl + 1), spc); - inl = (void *) (inl + 1) + spc; - mb_copy(mb, &off, (void *)(inl + 1), len - spc); + *(volatile uint32_t *)(inl + 4 + spc) = + SET_BYTE_COUNT((1 << 31) | (len - spc)); wmb(); - inl->byte_count = cpu_to_be32(1 << 31 | (len - spc)); + *(volatile uint32_t *)inl = + SET_BYTE_COUNT((1 << 31) | spc); } - tx_desc->ctrl.vlan_tag = cpu_to_be16(*vlan_tag); - tx_desc->ctrl.ins_vlan = MLX4_WQE_CTRL_INS_VLAN * !!(*vlan_tag); - tx_desc->ctrl.fence_size = (real_size / 16) & 0x3f; } static unsigned long hashrandom; @@ -720,18 +641,14 @@ u16 mlx4_en_select_queue(struct net_device *dev, struct mbuf *mb) up = (vlan_tag >> 13) % MLX4_EN_NUM_UP; } #endif - /* check if flowid is set */ - if (M_HASHTYPE_GET(mb) != M_HASHTYPE_NONE) - queue_index = mb->m_pkthdr.flowid; - else - queue_index = mlx4_en_hashmbuf(MLX4_F_HASHL3 | MLX4_F_HASHL4, mb, hashrandom); + queue_index = mlx4_en_hashmbuf(MLX4_F_HASHL3 | MLX4_F_HASHL4, mb, hashrandom); return ((queue_index % rings_p_up) + (up * rings_p_up)); } -static void mlx4_bf_copy(void __iomem *dst, unsigned long *src, unsigned bytecnt) +static void mlx4_bf_copy(void __iomem *dst, volatile unsigned long *src, unsigned bytecnt) { - __iowrite64_copy(dst, src, bytecnt / 8); + __iowrite64_copy(dst, __DEVOLATILE(void *, src), bytecnt / 8); } static u64 mlx4_en_mac_to_u64(u8 *addr) @@ -746,168 +663,263 @@ static u64 mlx4_en_mac_to_u64(u8 *addr) return mac; } -static int mlx4_en_xmit(struct net_device *dev, int tx_ind, struct mbuf **mbp) +static int mlx4_en_xmit(struct mlx4_en_priv *priv, int tx_ind, struct mbuf **mbp) { - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_en_tx_ring *ring; - struct mlx4_en_cq *cq; - struct mlx4_en_tx_desc *tx_desc; - struct mlx4_wqe_data_seg *data; + enum { + DS_FACT = TXBB_SIZE / DS_SIZE_ALIGNMENT, + CTRL_FLAGS = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE | + MLX4_WQE_CTRL_SOLICITED), + }; + bus_dma_segment_t segs[MLX4_EN_TX_MAX_MBUF_FRAGS]; + volatile struct mlx4_wqe_data_seg *dseg; + volatile struct mlx4_wqe_data_seg *dseg_inline; + volatile struct mlx4_en_tx_desc *tx_desc; + struct mlx4_en_tx_ring *ring = priv->tx_ring[tx_ind]; + struct ifnet *ifp = priv->dev; struct mlx4_en_tx_info *tx_info; + struct mbuf *mb = *mbp; struct mbuf *m; - int nr_txbb; + __be32 owner_bit; int nr_segs; - int desc_size; - int real_size; - dma_addr_t dma; - u32 index, bf_index, ring_size; - __be32 op_own; - u16 vlan_tag = 0; - int i; - int lso_header_size; - bool bounce = false; - bool inl = false; - struct mbuf *mb; - mb = *mbp; - int defrag = 1; - - if (!priv->port_up) - goto tx_drop; - - ring = priv->tx_ring[tx_ind]; - ring_size = ring->size; - inl = is_inline(mb, ring->inline_thold); - -retry: - real_size = get_real_size(mb, dev, &nr_segs, &lso_header_size, inl); - if (unlikely(!real_size)) - goto tx_drop; + int pad; + int err; + u32 bf_size; + u32 bf_prod; + u32 opcode; + u16 index; + u16 ds_cnt; + u16 ihs; - /* Align descriptor to TXBB size */ - desc_size = ALIGN(real_size, TXBB_SIZE); - nr_txbb = desc_size / TXBB_SIZE; - if (unlikely(nr_txbb > MAX_DESC_TXBBS)) { - if (defrag) { - mb = m_defrag(*mbp, M_NOWAIT); - if (mb == NULL) { - mb = *mbp; - goto tx_drop; - } - *mbp = mb; - defrag = 0; - goto retry; - } - en_warn(priv, "Oversized header or SG list\n"); + if (unlikely(!priv->port_up)) { + err = EINVAL; goto tx_drop; } - /* Obtain VLAN information if present */ - if (mb->m_flags & M_VLANTAG) { - vlan_tag = mb->m_pkthdr.ether_vtag; - } - - /* Check available TXBBs and 2K spare for prefetch - * Even if netif_tx_stop_queue() will be called - * driver will send current packet to ensure - * that at least one completion will be issued after - * stopping the queue - */ - if (unlikely((int)(ring->prod - ring->cons) > ring->full_size)) { - /* every full Tx ring stops queue */ - if (ring->blocked == 0) - atomic_add_int(&priv->blocked, 1); - /* Set HW-queue-is-full flag */ - atomic_set_int(&dev->if_drv_flags, IFF_DRV_OACTIVE); + /* check if TX ring is full */ + if (unlikely(mlx4_en_tx_ring_is_full(ring))) { + /* every full native Tx ring stops queue */ + if (ring->blocked == 0) + atomic_add_int(&priv->blocked, 1); + /* Set HW-queue-is-full flag */ + atomic_set_int(&ifp->if_drv_flags, IFF_DRV_OACTIVE); + priv->port_stats.queue_stopped++; ring->blocked = 1; priv->port_stats.queue_stopped++; ring->queue_stopped++; /* Use interrupts to find out when queue opened */ - cq = priv->tx_cq[tx_ind]; - mlx4_en_arm_cq(priv, cq); - return EBUSY; + mlx4_en_arm_cq(priv, priv->tx_cq[tx_ind]); + return (ENOBUFS); } + /* sanity check we are not wrapping around */ + KASSERT(((~ring->prod) & ring->size_mask) >= + (MLX4_EN_TX_WQE_MAX_WQEBBS - 1), ("Wrapping around TX ring")); + /* Track current inflight packets for performance analysis */ AVG_PERF_COUNTER(priv->pstats.inflight_avg, (u32) (ring->prod - ring->cons - 1)); - /* Packet is good - grab an index and transmit it */ + /* Track current mbuf packet header length */ + AVG_PERF_COUNTER(priv->pstats.tx_pktsz_avg, mb->m_pkthdr.len); + + /* Grab an index and try to transmit packet */ + owner_bit = (ring->prod & ring->size) ? + cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0; index = ring->prod & ring->size_mask; - bf_index = ring->prod; + tx_desc = (volatile struct mlx4_en_tx_desc *) + (ring->buf + index * TXBB_SIZE); + tx_info = &ring->tx_info[index]; + dseg = &tx_desc->data; - /* See if we have enough space for whole descriptor TXBB for setting - * SW ownership on next descriptor; if not, use a bounce buffer. */ - if (likely(index + nr_txbb <= ring_size)) - tx_desc = ring->buf + index * TXBB_SIZE; - else { - tx_desc = (struct mlx4_en_tx_desc *) ring->bounce_buf; - bounce = true; + /* send a copy of the frame to the BPF listener, if any */ + if (ifp != NULL && ifp->if_bpf != NULL) + ETHER_BPF_MTAP(ifp, mb); + + /* get default flags */ + tx_desc->ctrl.srcrb_flags = CTRL_FLAGS; + + if (mb->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)) + tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM); + + if (mb->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | + CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) + tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_TCP_UDP_CSUM); + + /* do statistics */ + if (likely(tx_desc->ctrl.srcrb_flags != CTRL_FLAGS)) { + priv->port_stats.tx_chksum_offload++; + ring->tx_csum++; } - /* Save mb in tx_info ring */ - tx_info = &ring->tx_info[index]; - tx_info->mb = mb; - tx_info->nr_txbb = nr_txbb; - tx_info->nr_segs = nr_segs; - - if (lso_header_size) { - memcpy(tx_desc->lso.header, mb->m_data, lso_header_size); - data = ((void *)&tx_desc->lso + ALIGN(lso_header_size + 4, - DS_SIZE)); - /* lso header is part of m_data. - * need to omit when mapping DMA */ - mb->m_data += lso_header_size; - mb->m_len -= lso_header_size; + /* check for VLAN tag */ + if (mb->m_flags & M_VLANTAG) { + tx_desc->ctrl.vlan_tag = cpu_to_be16(mb->m_pkthdr.ether_vtag); + tx_desc->ctrl.ins_vlan = MLX4_WQE_CTRL_INS_VLAN; + } else { + tx_desc->ctrl.vlan_tag = 0; + tx_desc->ctrl.ins_vlan = 0; } - else - data = &tx_desc->data; - /* valid only for none inline segments */ - tx_info->data_offset = (void *)data - (void *)tx_desc; + /* clear immediate field */ + tx_desc->ctrl.imm = 0; - if (inl) { - tx_info->inl = 1; + /* Handle LSO (TSO) packets */ + if (mb->m_pkthdr.csum_flags & CSUM_TSO) { + u32 payload_len; + u32 mss = mb->m_pkthdr.tso_segsz; + u32 num_pkts; + + opcode = cpu_to_be32(MLX4_OPCODE_LSO | MLX4_WQE_CTRL_RR) | + owner_bit; + ihs = mlx4_en_get_header_size(mb); + if (unlikely(ihs > MAX_INLINE)) { + ring->oversized_packets++; + err = EINVAL; + goto tx_drop; + } + tx_desc->lso.mss_hdr_size = cpu_to_be32((mss << 16) | ihs); + payload_len = mb->m_pkthdr.len - ihs; + if (unlikely(payload_len == 0)) + num_pkts = 1; + else + num_pkts = DIV_ROUND_UP(payload_len, mss); + ring->bytes += payload_len + (num_pkts * ihs); + ring->packets += num_pkts; + priv->port_stats.tso_packets++; + /* store pointer to inline header */ + dseg_inline = dseg; + /* copy data inline */ + dseg = mlx4_en_store_inline_lso_data(dseg, + mb, ihs, owner_bit); } else { - for (i = 0, m = mb; i < nr_segs; i++, m = m->m_next) { - if (m->m_len == 0) { - i--; - continue; - } - dma = pci_map_single(mdev->dev->pdev, m->m_data, - m->m_len, PCI_DMA_TODEVICE); - data->addr = cpu_to_be64(dma); - data->lkey = cpu_to_be32(mdev->mr.key); - wmb(); - data->byte_count = cpu_to_be32(m->m_len); - data++; - } - if (lso_header_size) { - mb->m_data -= lso_header_size; - mb->m_len += lso_header_size; - } - tx_info->inl = 0; + opcode = cpu_to_be32(MLX4_OPCODE_SEND) | + owner_bit; + ihs = mlx4_en_get_inline_hdr_size(ring, mb); + ring->bytes += max_t (unsigned int, + mb->m_pkthdr.len, ETHER_MIN_LEN - ETHER_CRC_LEN); + ring->packets++; + /* store pointer to inline header */ + dseg_inline = dseg; + /* copy data inline */ + dseg = mlx4_en_store_inline_data(dseg, + mb, ihs, owner_bit); + } + m_adj(mb, ihs); + + /* trim off empty mbufs */ + while (mb->m_len == 0) { + mb = m_free(mb); + /* check if all data has been inlined */ + if (mb == NULL) { + nr_segs = 0; + goto skip_dma; + } } + err = bus_dmamap_load_mbuf_sg(ring->dma_tag, tx_info->dma_map, + mb, segs, &nr_segs, BUS_DMA_NOWAIT); + if (unlikely(err == EFBIG)) { + /* Too many mbuf fragments */ + m = m_defrag(mb, M_NOWAIT); + if (m == NULL) { + ring->oversized_packets++; + goto tx_drop; + } + mb = m; + /* Try again */ + err = bus_dmamap_load_mbuf_sg(ring->dma_tag, tx_info->dma_map, + mb, segs, &nr_segs, BUS_DMA_NOWAIT); + } + /* catch errors */ + if (unlikely(err != 0)) { + ring->oversized_packets++; + goto tx_drop; + } + /* make sure all mbuf data is written to RAM */ + bus_dmamap_sync(ring->dma_tag, tx_info->dma_map, + BUS_DMASYNC_PREWRITE); - /* Prepare ctrl segement apart opcode+ownership, which depends on - * whether LSO is used */ - tx_desc->ctrl.vlan_tag = cpu_to_be16(vlan_tag); - tx_desc->ctrl.ins_vlan = MLX4_WQE_CTRL_INS_VLAN * - !!vlan_tag; - tx_desc->ctrl.fence_size = (real_size / 16) & 0x3f; - tx_desc->ctrl.srcrb_flags = priv->ctrl_flags; - if (mb->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO | - CSUM_TCP | CSUM_UDP | CSUM_TCP_IPV6 | CSUM_UDP_IPV6)) { - if (mb->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)) - tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM); - if (mb->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | - CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) - tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_TCP_UDP_CSUM); - priv->port_stats.tx_chksum_offload++; - ring->tx_csum++; - } +skip_dma: + /* compute number of DS needed */ + ds_cnt = (dseg - ((volatile struct mlx4_wqe_data_seg *)tx_desc)) + nr_segs; + + /* + * Check if the next request can wrap around and fill the end + * of the current request with zero immediate data: + */ + pad = DIV_ROUND_UP(ds_cnt, DS_FACT); + pad = (~(ring->prod + pad)) & ring->size_mask; + + if (unlikely(pad < (MLX4_EN_TX_WQE_MAX_WQEBBS - 1))) { + /* + * Compute the least number of DS blocks we need to + * pad in order to achieve a TX ring wraparound: + */ + pad = (DS_FACT * (pad + 1)); + } else { + /* + * The hardware will automatically jump to the next + * TXBB. No need for padding. + */ + pad = 0; + } + + /* compute total number of DS blocks */ + ds_cnt += pad; + /* + * When modifying this code, please ensure that the following + * computation is always less than or equal to 0x3F: + * + * ((MLX4_EN_TX_WQE_MAX_WQEBBS - 1) * DS_FACT) + + * (MLX4_EN_TX_WQE_MAX_WQEBBS * DS_FACT) + * + * Else the "ds_cnt" variable can become too big. + */ + tx_desc->ctrl.fence_size = (ds_cnt & 0x3f); + + /* store pointer to mbuf */ + tx_info->mb = mb; + tx_info->nr_txbb = DIV_ROUND_UP(ds_cnt, DS_FACT); + bf_size = ds_cnt * DS_SIZE_ALIGNMENT; + bf_prod = ring->prod; + + /* compute end of "dseg" array */ + dseg += nr_segs + pad; + + /* pad using zero immediate dseg */ + while (pad--) { + dseg--; + dseg->addr = 0; + dseg->lkey = 0; + wmb(); + dseg->byte_count = SET_BYTE_COUNT((1 << 31)|0); + } + + /* fill segment list */ + while (nr_segs--) { + if (unlikely(segs[nr_segs].ds_len == 0)) { + dseg--; + dseg->addr = 0; + dseg->lkey = 0; + wmb(); + dseg->byte_count = SET_BYTE_COUNT((1 << 31)|0); + } else { + dseg--; + dseg->addr = cpu_to_be64((uint64_t)segs[nr_segs].ds_addr); + dseg->lkey = cpu_to_be32(priv->mdev->mr.key); + wmb(); + dseg->byte_count = SET_BYTE_COUNT((uint32_t)segs[nr_segs].ds_len); + } + } + + wmb(); + + /* write owner bits in reverse order */ + if ((opcode & cpu_to_be32(0x1F)) == cpu_to_be32(MLX4_OPCODE_LSO)) + mlx4_en_store_inline_lso_header(dseg_inline, ihs, owner_bit); + else + mlx4_en_store_inline_header(dseg_inline, ihs, owner_bit); if (unlikely(priv->validate_loopback)) { /* Copy dst mac address to wqe */ @@ -925,77 +937,46 @@ static int mlx4_en_xmit(struct net_device *dev, int tx_ind, struct mbuf **mbp) } } - /* Handle LSO (TSO) packets */ - if (lso_header_size) { - int segsz; - /* Mark opcode as LSO */ - op_own = cpu_to_be32(MLX4_OPCODE_LSO | (1 << 6)) | - ((ring->prod & ring_size) ? - cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0); - - /* Fill in the LSO prefix */ - tx_desc->lso.mss_hdr_size = cpu_to_be32( - mb->m_pkthdr.tso_segsz << 16 | lso_header_size); - - priv->port_stats.tso_packets++; - segsz = mb->m_pkthdr.tso_segsz; - i = ((mb->m_pkthdr.len - lso_header_size + segsz - 1) / segsz); - tx_info->nr_bytes= mb->m_pkthdr.len + (i - 1) * lso_header_size; - ring->packets += i; - } else { - /* Normal (Non LSO) packet */ - op_own = cpu_to_be32(MLX4_OPCODE_SEND) | - ((ring->prod & ring_size) ? - cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0); - tx_info->nr_bytes = max(mb->m_pkthdr.len, - (unsigned int)ETHER_MIN_LEN - ETHER_CRC_LEN); - ring->packets++; - - } - ring->bytes += tx_info->nr_bytes; - AVG_PERF_COUNTER(priv->pstats.tx_pktsz_avg, mb->m_pkthdr.len); + /* update producer counter */ + ring->prod += tx_info->nr_txbb; - if (tx_info->inl) { - build_inline_wqe(tx_desc, mb, real_size, &vlan_tag, tx_ind); - tx_info->inl = 1; - } + if (ring->bf_enabled && bf_size <= MAX_BF && + (tx_desc->ctrl.ins_vlan != MLX4_WQE_CTRL_INS_VLAN)) { - ring->prod += nr_txbb; + /* store doorbell number */ + *(volatile __be32 *) (&tx_desc->ctrl.vlan_tag) |= cpu_to_be32(ring->doorbell_qpn); + /* or in producer number for this WQE */ + opcode |= cpu_to_be32((bf_prod & 0xffff) << 8); - /* If we used a bounce buffer then copy descriptor back into place */ - if (unlikely(bounce)) - tx_desc = mlx4_en_bounce_to_desc(priv, ring, index, desc_size); - if (ring->bf_enabled && desc_size <= MAX_BF && !bounce && !vlan_tag) { - *(__be32 *) (&tx_desc->ctrl.vlan_tag) |= cpu_to_be32(ring->doorbell_qpn); - op_own |= htonl((bf_index & 0xffff) << 8); - /* Ensure new descirptor hits memory - * before setting ownership of this descriptor to HW */ + /* + * Ensure the new descriptor hits memory before + * setting ownership of this descriptor to HW: + */ wmb(); - tx_desc->ctrl.owner_opcode = op_own; - + tx_desc->ctrl.owner_opcode = opcode; wmb(); - - mlx4_bf_copy(ring->bf.reg + ring->bf.offset, (unsigned long *) &tx_desc->ctrl, - desc_size); - + mlx4_bf_copy(((u8 *)ring->bf.reg) + ring->bf.offset, + (volatile unsigned long *) &tx_desc->ctrl, bf_size); wmb(); - ring->bf.offset ^= ring->bf.buf_size; } else { - /* Ensure new descirptor hits memory - * before setting ownership of this descriptor to HW */ + /* + * Ensure the new descriptor hits memory before + * setting ownership of this descriptor to HW: + */ wmb(); - tx_desc->ctrl.owner_opcode = op_own; + tx_desc->ctrl.owner_opcode = opcode; wmb(); - writel(cpu_to_be32(ring->doorbell_qpn), ring->bf.uar->map + MLX4_SEND_DOORBELL); + writel(cpu_to_be32(ring->doorbell_qpn), + ((u8 *)ring->bf.uar->map) + MLX4_SEND_DOORBELL); } - return 0; + return (0); tx_drop: *mbp = NULL; m_freem(mb); - return EINVAL; + return (err); } static int @@ -1015,13 +996,16 @@ mlx4_en_transmit_locked(struct ifnet *dev, int tx_ind, struct mbuf *m) } enqueued = 0; - if (m != NULL) { - if ((err = drbr_enqueue(dev, ring->br, m)) != 0) - return (err); - } + if (m != NULL) + /* + * If we can't insert mbuf into drbr, try to xmit anyway. + * We keep the error we got so we could return that after xmit. + */ + err = drbr_enqueue(dev, ring->br, m); + /* Process the queue */ while ((next = drbr_peek(dev, ring->br)) != NULL) { - if ((err = mlx4_en_xmit(dev, tx_ind, &next)) != 0) { + if (mlx4_en_xmit(priv, tx_ind, &next) != 0) { if (next == NULL) { drbr_advance(dev, ring->br); } else { @@ -1072,10 +1056,14 @@ mlx4_en_transmit(struct ifnet *dev, struct mbuf *m) int i, err = 0; /* Compute which queue to use */ - i = mlx4_en_select_queue(dev, m); + if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { + i = m->m_pkthdr.flowid % priv->tx_ring_num; + } + else { + i = mlx4_en_select_queue(dev, m); + } ring = priv->tx_ring[i]; - if (spin_trylock(&ring->tx_lock)) { err = mlx4_en_transmit_locked(dev, i, m); spin_unlock(&ring->tx_lock); diff --git a/sys/ofed/drivers/net/mlx4/main.c b/sys/ofed/drivers/net/mlx4/main.c index ddcd7d14a..217220a0f 100644 --- a/sys/ofed/drivers/net/mlx4/main.c +++ b/sys/ofed/drivers/net/mlx4/main.c @@ -33,7 +33,7 @@ * SOFTWARE. */ -#include +#include /* * kmod.h must be included before module.h since it includes (indirectly) sys/module.h * To use the FBSD macro sys/module.h should define MODULE_VERSION before linux/module does. @@ -57,9 +57,7 @@ #include "icm.h" #include "mlx4_stats.h" -MODULE_AUTHOR("Roland Dreier"); -MODULE_DESCRIPTION("Mellanox ConnectX HCA low-level driver"); -MODULE_LICENSE("Dual BSD/GPL"); +/* Mellanox ConnectX HCA low-level driver */ struct workqueue_struct *mlx4_wq; @@ -177,7 +175,7 @@ MODULE_PARM_DESC(enable_64b_cqe_eqe, #define PF_CONTEXT_BEHAVIOUR_MASK MLX4_FUNC_CAP_64B_EQE_CQE static char mlx4_version[] __devinitdata = - DRV_NAME ": Mellanox ConnectX core driver v" + DRV_NAME ": Mellanox ConnectX VPI driver v" DRV_VERSION " (" DRV_RELDATE ")\n"; static int log_num_mac = 7; @@ -608,7 +606,7 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap) if (dev_cap->min_page_sz > PAGE_SIZE) { mlx4_err(dev, "HCA minimum page size of %d bigger than " "kernel PAGE_SIZE of %d, aborting.\n", - dev_cap->min_page_sz, PAGE_SIZE); + dev_cap->min_page_sz, (int)PAGE_SIZE); return -ENODEV; } if (dev_cap->num_ports > MLX4_MAX_PORTS) { @@ -979,7 +977,7 @@ static int mlx4_slave_cap(struct mlx4_dev *dev) if (page_size > PAGE_SIZE) { mlx4_err(dev, "HCA minimum page size of %d bigger than " "kernel PAGE_SIZE of %d, aborting.\n", - page_size, PAGE_SIZE); + page_size, (int)PAGE_SIZE); return -ENODEV; } @@ -989,7 +987,7 @@ static int mlx4_slave_cap(struct mlx4_dev *dev) /* TODO: relax this assumption */ if (dev->caps.uar_page_size != PAGE_SIZE) { mlx4_err(dev, "UAR size:%d != kernel PAGE_SIZE of %d\n", - dev->caps.uar_page_size, PAGE_SIZE); + dev->caps.uar_page_size, (int)PAGE_SIZE); return -ENODEV; } @@ -1299,6 +1297,43 @@ static inline int ibta_mtu_to_int(enum ibta_mtu mtu) } } +static ssize_t +show_board(struct device *device, struct device_attribute *attr, + char *buf) +{ + struct mlx4_hca_info *info = container_of(attr, struct mlx4_hca_info, + board_attr); + struct mlx4_dev *mdev = info->dev; + + return sprintf(buf, "%.*s\n", MLX4_BOARD_ID_LEN, + mdev->board_id); +} + +static ssize_t +show_hca(struct device *device, struct device_attribute *attr, + char *buf) +{ + struct mlx4_hca_info *info = container_of(attr, struct mlx4_hca_info, + hca_attr); + struct mlx4_dev *mdev = info->dev; + + return sprintf(buf, "MT%d\n", mdev->pdev->device); +} + +static ssize_t +show_firmware_version(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct mlx4_hca_info *info = container_of(attr, struct mlx4_hca_info, + firmware_attr); + struct mlx4_dev *mdev = info->dev; + + return sprintf(buf, "%d.%d.%d\n", (int)(mdev->caps.fw_ver >> 32), + (int)(mdev->caps.fw_ver >> 16) & 0xffff, + (int)mdev->caps.fw_ver & 0xffff); +} + static ssize_t show_port_ib_mtu(struct device *dev, struct device_attribute *attr, char *buf) @@ -2941,6 +2976,30 @@ static void mlx4_enable_msi_x(struct mlx4_dev *dev) no_irq: dev->caps.num_comp_vectors = 0; dev->caps.comp_pool = 0; + return; +} + +static void +mlx4_init_hca_info(struct mlx4_dev *dev) +{ + struct mlx4_hca_info *info = &mlx4_priv(dev)->hca_info; + + info->dev = dev; + + info->firmware_attr = (struct device_attribute)__ATTR(fw_ver, S_IRUGO, + show_firmware_version, NULL); + if (device_create_file(&dev->pdev->dev, &info->firmware_attr)) + mlx4_err(dev, "Failed to add file firmware version"); + + info->hca_attr = (struct device_attribute)__ATTR(hca, S_IRUGO, show_hca, + NULL); + if (device_create_file(&dev->pdev->dev, &info->hca_attr)) + mlx4_err(dev, "Failed to add file hca type"); + + info->board_attr = (struct device_attribute)__ATTR(board_id, S_IRUGO, + show_board, NULL); + if (device_create_file(&dev->pdev->dev, &info->board_attr)) + mlx4_err(dev, "Failed to add file board id type"); } static int mlx4_init_port_info(struct mlx4_dev *dev, int port) @@ -2994,6 +3053,14 @@ static int mlx4_init_port_info(struct mlx4_dev *dev, int port) return err; } +static void +mlx4_cleanup_hca_info(struct mlx4_hca_info *info) +{ + device_remove_file(&info->dev->pdev->dev, &info->firmware_attr); + device_remove_file(&info->dev->pdev->dev, &info->board_attr); + device_remove_file(&info->dev->pdev->dev, &info->hca_attr); +} + static void mlx4_cleanup_port_info(struct mlx4_port_info *info) { if (info->port < 0) @@ -3351,6 +3418,7 @@ static int __mlx4_init_one(struct pci_dev *pdev, int pci_dev_data) goto err_steer; mlx4_init_quotas(dev); + mlx4_init_hca_info(dev); for (port = 1; port <= dev->caps.num_ports; port++) { err = mlx4_init_port_info(dev, port); @@ -3443,8 +3511,7 @@ static int __mlx4_init_one(struct pci_dev *pdev, int pci_dev_data) static int __devinit mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id) { - printk_once(KERN_INFO "%s", mlx4_version); - + device_set_desc(pdev->dev.bsddev, mlx4_version); return __mlx4_init_one(pdev, id->driver_data); } @@ -3464,6 +3531,7 @@ static void mlx4_remove_one(struct pci_dev *pdev) mlx4_stop_sense(dev); mlx4_unregister_device(dev); + mlx4_cleanup_hca_info(&priv->hca_info); for (p = 1; p <= dev->caps.num_ports; p++) { mlx4_cleanup_port_info(&priv->port[p]); mlx4_CLOSE_PORT(dev, p); diff --git a/sys/ofed/drivers/net/mlx4/mlx4.h b/sys/ofed/drivers/net/mlx4/mlx4.h index 47fde257c..bef2e0043 100644 --- a/sys/ofed/drivers/net/mlx4/mlx4.h +++ b/sys/ofed/drivers/net/mlx4/mlx4.h @@ -51,7 +51,7 @@ #define DRV_NAME "mlx4_core" #define PFX DRV_NAME ": " -#define DRV_VERSION "2.1" +#define DRV_VERSION "2.1.6" #define DRV_RELDATE __DATE__ #define DRV_STACK_NAME "Linux-MLNX_OFED" @@ -755,6 +755,13 @@ struct mlx4_set_port_rqp_calc_context { __be32 mcast; }; +struct mlx4_hca_info { + struct mlx4_dev *dev; + struct device_attribute firmware_attr; + struct device_attribute hca_attr; + struct device_attribute board_attr; +}; + struct mlx4_port_info { struct mlx4_dev *dev; int port; @@ -845,6 +852,7 @@ struct mlx4_priv { struct mlx4_uar driver_uar; void __iomem *kar; struct mlx4_port_info port[MLX4_MAX_PORTS + 1]; + struct mlx4_hca_info hca_info; struct mlx4_sense sense; struct mutex port_mutex; struct mlx4_msix_ctl msix_ctl; diff --git a/sys/ofed/drivers/net/mlx4/mlx4_en.h b/sys/ofed/drivers/net/mlx4/mlx4_en.h index b96563d2a..c776b0e51 100644 --- a/sys/ofed/drivers/net/mlx4/mlx4_en.h +++ b/sys/ofed/drivers/net/mlx4/mlx4_en.h @@ -59,8 +59,6 @@ #include "mlx4_stats.h" #define DRV_NAME "mlx4_en" -#define DRV_VERSION "2.1" -#define DRV_RELDATE __DATE__ #define MLX4_EN_MSG_LEVEL (NETIF_MSG_LINK | NETIF_MSG_IFDOWN) @@ -95,10 +93,6 @@ #define VLAN_MIN_VALUE 1 #define VLAN_MAX_VALUE 4094 -/* Typical TSO descriptor with 16 gather entries is 352 bytes... */ -#define MAX_DESC_SIZE 512 -#define MAX_DESC_TXBBS (MAX_DESC_SIZE / TXBB_SIZE) - /* * OS related constants and tunables */ @@ -113,26 +107,6 @@ enum mlx4_en_alloc_type { MLX4_EN_ALLOC_REPLACEMENT = 1, }; -/* Receive fragment sizes; we use at most 3 fragments (for 9600 byte MTU - * and 4K allocations) */ -#if MJUMPAGESIZE == 4096 -enum { - FRAG_SZ0 = MCLBYTES, - FRAG_SZ1 = MJUMPAGESIZE, - FRAG_SZ2 = MJUMPAGESIZE, -}; -#define MLX4_EN_MAX_RX_FRAGS 3 -#elif MJUMPAGESIZE == 8192 -enum { - FRAG_SZ0 = MCLBYTES, - FRAG_SZ1 = MJUMPAGESIZE, -}; -#define MLX4_EN_MAX_RX_FRAGS 2 -#elif MJUMPAGESIZE == 8192 -#else -#error "Unknown PAGE_SIZE" -#endif - /* Maximum ring sizes */ #define MLX4_EN_DEF_TX_QUEUE_SIZE 4096 @@ -154,7 +128,7 @@ enum { #define MLX4_EN_NUM_UP 1 #define MAX_TX_RINGS (MLX4_EN_MAX_TX_RING_P_UP * \ - MLX4_EN_NUM_UP) + MLX4_EN_NUM_UP) #define MLX4_EN_DEF_TX_RING_SIZE 1024 #define MLX4_EN_DEF_RX_RING_SIZE 1024 @@ -235,16 +209,10 @@ enum cq_type { #define ILLEGAL_MAC(addr) (addr == 0xffffffffffffULL || addr == 0x0) struct mlx4_en_tx_info { + bus_dmamap_t dma_map; struct mbuf *mb; u32 nr_txbb; u32 nr_bytes; - u8 linear; - u8 nr_segs; - u8 data_offset; - u8 inl; -#if 0 - u8 ts_requested; -#endif }; @@ -265,14 +233,22 @@ struct mlx4_en_tx_desc { #define MLX4_EN_USE_SRQ 0x01000000 -#define MLX4_EN_TX_BUDGET 64*4 //Compensate for no NAPI in freeBSD - might need some fine tunning in the future. #define MLX4_EN_RX_BUDGET 64 +#define MLX4_EN_TX_MAX_DESC_SIZE 512 /* bytes */ +#define MLX4_EN_TX_MAX_MBUF_SIZE 65536 /* bytes */ +#define MLX4_EN_TX_MAX_PAYLOAD_SIZE 65536 /* bytes */ +#define MLX4_EN_TX_MAX_MBUF_FRAGS \ + ((MLX4_EN_TX_MAX_DESC_SIZE - 128) / DS_SIZE_ALIGNMENT) /* units */ +#define MLX4_EN_TX_WQE_MAX_WQEBBS \ + (MLX4_EN_TX_MAX_DESC_SIZE / TXBB_SIZE) /* units */ + #define MLX4_EN_CX3_LOW_ID 0x1000 #define MLX4_EN_CX3_HIGH_ID 0x1005 struct mlx4_en_tx_ring { spinlock_t tx_lock; + bus_dma_tag_t dma_tag; struct mlx4_hwq_resources wqres; u32 size ; /* number of TXBBs */ u32 size_mask; @@ -282,11 +258,10 @@ struct mlx4_en_tx_ring { u32 cons; u32 buf_size; u32 doorbell_qpn; - void *buf; + u8 *buf; u16 poll_cnt; int blocked; struct mlx4_en_tx_info *tx_info; - u8 *bounce_buf; u8 queue_index; cpuset_t affinity_mask; struct buf_ring *br; @@ -300,13 +275,12 @@ struct mlx4_en_tx_ring { unsigned long packets; unsigned long tx_csum; unsigned long queue_stopped; + unsigned long oversized_packets; unsigned long wake_queue; struct mlx4_bf bf; bool bf_enabled; - struct netdev_queue *tx_queue; int hwtstamp_tx_type; spinlock_t comp_lock; - int full_size; int inline_thold; u64 watchdog_time; }; @@ -316,14 +290,21 @@ struct mlx4_en_rx_desc { struct mlx4_wqe_data_seg data[0]; }; -struct mlx4_en_rx_buf { - dma_addr_t dma; - struct page *page; - unsigned int page_offset; +struct mlx4_en_rx_mbuf { + bus_dmamap_t dma_map; + struct mbuf *mbuf; +}; + +struct mlx4_en_rx_spare { + bus_dmamap_t dma_map; + struct mbuf *mbuf; + u64 paddr_be; }; struct mlx4_en_rx_ring { struct mlx4_hwq_resources wqres; + bus_dma_tag_t dma_tag; + struct mlx4_en_rx_spare spare; u32 size ; /* number of Rx descs*/ u32 actual_size; u32 size_mask; @@ -339,8 +320,8 @@ struct mlx4_en_rx_ring { u32 rx_buf_size; u32 rx_mb_size; int qpn; - void *buf; - void *rx_info; + u8 *buf; + struct mlx4_en_rx_mbuf *mbuf; unsigned long errors; unsigned long bytes; unsigned long packets; @@ -400,6 +381,7 @@ struct mlx4_en_cq { #define MLX4_EN_OPCODE_ERROR 0x1e u32 tot_rx; u32 tot_tx; + u32 curr_poll_rx_cpu_id; #ifdef CONFIG_NET_RX_BUSY_POLL unsigned int state; @@ -524,12 +506,6 @@ struct en_port { u8 vport_num; }; -struct mlx4_en_frag_info { - u16 frag_size; - u16 frag_prefix_size; -}; - - struct mlx4_en_priv { struct mlx4_en_dev *mdev; struct mlx4_en_port_profile *prof; @@ -575,18 +551,14 @@ struct mlx4_en_priv { int cqe_factor; struct mlx4_en_rss_map rss_map; - __be32 ctrl_flags; u32 flags; u8 num_tx_rings_p_up; u32 tx_ring_num; u32 rx_ring_num; u32 rx_mb_size; - struct mlx4_en_frag_info frag_info[MLX4_EN_MAX_RX_FRAGS]; u16 rx_alloc_order; u32 rx_alloc_size; u32 rx_buf_size; - u16 num_frags; - u16 log_rx_info; struct mlx4_en_tx_ring **tx_ring; struct mlx4_en_rx_ring *rx_ring[MAX_RX_RINGS]; @@ -640,7 +612,6 @@ struct mlx4_en_priv { unsigned long last_ifq_jiffies; u64 if_counters_rx_errors; u64 if_counters_rx_no_buffer; - }; enum mlx4_en_wol { diff --git a/sys/ofed/drivers/net/mlx4/mlx4_stats.h b/sys/ofed/drivers/net/mlx4/mlx4_stats.h index 0270cefd0..ef0b1c0a6 100644 --- a/sys/ofed/drivers/net/mlx4/mlx4_stats.h +++ b/sys/ofed/drivers/net/mlx4/mlx4_stats.h @@ -124,6 +124,7 @@ struct mlx4_en_port_stats { unsigned long queue_stopped; unsigned long wake_queue; unsigned long tx_timeout; + unsigned long oversized_packets; unsigned long rx_alloc_failed; unsigned long rx_chksum_good; unsigned long rx_chksum_none; diff --git a/sys/ofed/drivers/net/mlx4/port.c b/sys/ofed/drivers/net/mlx4/port.c index 4d7aa0ba9..9ef63f7a2 100644 --- a/sys/ofed/drivers/net/mlx4/port.c +++ b/sys/ofed/drivers/net/mlx4/port.c @@ -1145,12 +1145,17 @@ int mlx4_get_module_info(struct mlx4_dev *dev, u8 port, u16 offset, size = MODULE_INFO_MAX_READ; inbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(inbox)) + if (IS_ERR(inbox)) { + mlx4_err(dev, + "mlx4_alloc_cmd_mailbox returned with error(%lx)", PTR_ERR(inbox)); return PTR_ERR(inbox); + } outbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(outbox)) { mlx4_free_cmd_mailbox(dev, inbox); + mlx4_err(dev, + "mlx4_alloc_cmd_mailbox returned with error(%lx)", PTR_ERR(outbox)); return PTR_ERR(outbox); } diff --git a/sys/ofed/include/linux/mlx4/cq.h b/sys/ofed/include/linux/mlx4/cq.h index ed6bec188..b39910a97 100644 --- a/sys/ofed/include/linux/mlx4/cq.h +++ b/sys/ofed/include/linux/mlx4/cq.h @@ -139,7 +139,7 @@ enum { }; static inline void mlx4_cq_arm(struct mlx4_cq *cq, u32 cmd, - void __iomem *uar_page, + u8 __iomem *uar_page, spinlock_t *doorbell_lock) { __be32 doorbell[2]; diff --git a/sys/ofed/include/linux/mlx4/device.h b/sys/ofed/include/linux/mlx4/device.h index 536c42113..aca4aefe9 100644 --- a/sys/ofed/include/linux/mlx4/device.h +++ b/sys/ofed/include/linux/mlx4/device.h @@ -413,6 +413,13 @@ enum { #define MSTR_SM_CHANGE_MASK (MLX4_EQ_PORT_INFO_MSTR_SM_SL_CHANGE_MASK | \ MLX4_EQ_PORT_INFO_MSTR_SM_LID_CHANGE_MASK) +enum mlx4_module_id { + MLX4_MODULE_ID_SFP = 0x3, + MLX4_MODULE_ID_QSFP = 0xC, + MLX4_MODULE_ID_QSFP_PLUS = 0xD, + MLX4_MODULE_ID_QSFP28 = 0x11, +}; + static inline u64 mlx4_fw_ver(u64 major, u64 minor, u64 subminor) { return (major << 32) | (minor << 16) | subminor; diff --git a/sys/ofed/include/linux/mlx4/doorbell.h b/sys/ofed/include/linux/mlx4/doorbell.h index f31bba270..6724e5ea2 100644 --- a/sys/ofed/include/linux/mlx4/doorbell.h +++ b/sys/ofed/include/linux/mlx4/doorbell.h @@ -77,7 +77,7 @@ static inline void mlx4_write64(__be32 val[2], void __iomem *dest, spin_lock_irqsave(doorbell_lock, flags); __raw_writel((__force u32) val[0], dest); - __raw_writel((__force u32) val[1], dest + 4); + __raw_writel((__force u32) val[1], (u8 *)dest + 4); spin_unlock_irqrestore(doorbell_lock, flags); } diff --git a/sys/ofed/include/linux/mlx4/qp.h b/sys/ofed/include/linux/mlx4/qp.h index 9b78167e5..c8b6cb108 100644 --- a/sys/ofed/include/linux/mlx4/qp.h +++ b/sys/ofed/include/linux/mlx4/qp.h @@ -39,6 +39,12 @@ #define MLX4_INVALID_LKEY 0x100 +#define DS_SIZE_ALIGNMENT 16 + +#define SET_BYTE_COUNT(byte_count) cpu_to_be32(byte_count) +#define SET_LSO_MSS(mss_hdr_size) cpu_to_be32(mss_hdr_size) +#define DS_BYTE_COUNT_MASK cpu_to_be32(0x7fffffff) + enum ib_m_qp_attr_mask { IB_M_EXT_CLASS_1 = 1 << 28, IB_M_EXT_CLASS_2 = 1 << 29, @@ -266,7 +272,9 @@ enum { /* param3 */ #define MLX4_FW_VER_WQE_CTRL_NEC mlx4_fw_ver(2, 2, 232) enum { + MLX4_WQE_CTRL_OWN = 1 << 31, MLX4_WQE_CTRL_NEC = 1 << 29, + MLX4_WQE_CTRL_RR = 1 << 6, MLX4_WQE_CTRL_FENCE = 1 << 6, MLX4_WQE_CTRL_CQ_UPDATE = 3 << 2, MLX4_WQE_CTRL_SOLICITED = 1 << 1, -- 2.45.0