From 7cffab455269974a2244b831bc3195412e4b1ad8 Mon Sep 17 00:00:00 2001 From: davidcs Date: Tue, 6 Mar 2018 23:28:12 +0000 Subject: [PATCH] MFC r329855 1. Added support to offline a port if is error recovery on successful. 2. Sysctls to enable/disable driver_state_dump and error_recovery. 3. Sysctl to control the delay between hw/fw reinitialization and restarting the fastpath. 4. Stop periodic stats retrieval if interface has IFF_DRV_RUNNING flag off. 5. Print contents of PEG_HALT_STATUS1 and PEG_HALT_STATUS2 on heartbeat failure. 6. Speed up slowpath shutdown during error recovery. 7. link_state update using atomic_store. 8. Added timestamp information on driver state and minidump captures. 9. Added support for Slowpath event logging 10.Added additional failure injection types to simulate failures. git-svn-id: svn://svn.freebsd.org/base/stable/9@330557 ccf9f872-aa2e-dd11-9fc8-001c23d0bc1f --- sys/dev/qlxgbe/ql_dbg.h | 26 ++-- sys/dev/qlxgbe/ql_def.h | 10 +- sys/dev/qlxgbe/ql_glbl.h | 7 + sys/dev/qlxgbe/ql_hw.c | 305 ++++++++++++++++++++++++++++++++----- sys/dev/qlxgbe/ql_hw.h | 38 +++-- sys/dev/qlxgbe/ql_inline.h | 11 +- sys/dev/qlxgbe/ql_ioctl.c | 208 +++++++++++++++++++++---- sys/dev/qlxgbe/ql_ioctl.h | 75 +++++++++ sys/dev/qlxgbe/ql_isr.c | 32 ++-- sys/dev/qlxgbe/ql_misc.c | 23 ++- sys/dev/qlxgbe/ql_os.c | 230 +++++++++++++++++++++------- sys/dev/qlxgbe/ql_os.h | 8 + sys/dev/qlxgbe/ql_ver.h | 2 +- 13 files changed, 805 insertions(+), 170 deletions(-) diff --git a/sys/dev/qlxgbe/ql_dbg.h b/sys/dev/qlxgbe/ql_dbg.h index 9e2675bb4..863a7c8f2 100644 --- a/sys/dev/qlxgbe/ql_dbg.h +++ b/sys/dev/qlxgbe/ql_dbg.h @@ -42,17 +42,21 @@ extern void ql_dump_buf16(qla_host_t *ha, const char *str, void *dbuf, extern void ql_dump_buf32(qla_host_t *ha, const char *str, void *dbuf, uint32_t len32); -#define INJCT_RX_RXB_INVAL 0x00001 -#define INJCT_RX_MP_NULL 0x00002 -#define INJCT_LRO_RXB_INVAL 0x00003 -#define INJCT_LRO_MP_NULL 0x00004 -#define INJCT_NUM_HNDLE_INVALID 0x00005 -#define INJCT_RDWR_INDREG_FAILURE 0x00006 -#define INJCT_RDWR_OFFCHIPMEM_FAILURE 0x00007 -#define INJCT_MBX_CMD_FAILURE 0x00008 -#define INJCT_HEARTBEAT_FAILURE 0x00009 -#define INJCT_TEMPERATURE_FAILURE 0x0000A -#define INJCT_M_GETCL_M_GETJCL_FAILURE 0x0000B +#define INJCT_RX_RXB_INVAL 0x00001 +#define INJCT_RX_MP_NULL 0x00002 +#define INJCT_LRO_RXB_INVAL 0x00003 +#define INJCT_LRO_MP_NULL 0x00004 +#define INJCT_NUM_HNDLE_INVALID 0x00005 +#define INJCT_RDWR_INDREG_FAILURE 0x00006 +#define INJCT_RDWR_OFFCHIPMEM_FAILURE 0x00007 +#define INJCT_MBX_CMD_FAILURE 0x00008 +#define INJCT_HEARTBEAT_FAILURE 0x00009 +#define INJCT_TEMPERATURE_FAILURE 0x0000A +#define INJCT_M_GETCL_M_GETJCL_FAILURE 0x0000B +#define INJCT_INV_CONT_OPCODE 0x0000C +#define INJCT_SGL_RCV_INV_DESC_COUNT 0x0000D +#define INJCT_SGL_LRO_INV_DESC_COUNT 0x0000E +#define INJCT_PEER_PORT_FAILURE_ERR_RECOVERY 0x0000F #ifdef QL_DBG diff --git a/sys/dev/qlxgbe/ql_def.h b/sys/dev/qlxgbe/ql_def.h index dacc6b19d..8915c0ea7 100644 --- a/sys/dev/qlxgbe/ql_def.h +++ b/sys/dev/qlxgbe/ql_def.h @@ -144,12 +144,12 @@ struct qla_host { volatile uint32_t qla_watchdog_paused; volatile uint32_t qla_initiate_recovery; volatile uint32_t qla_detach_active; + volatile uint32_t offline; device_t pci_dev; - uint16_t watchdog_ticks; + volatile uint16_t watchdog_ticks; uint8_t pci_func; - uint8_t resvd; /* ioctl related */ struct cdev *ioctl_dev; @@ -182,6 +182,7 @@ struct qla_host { /* hardware access lock */ + struct mtx sp_log_lock; struct mtx hw_lock; volatile uint32_t hw_lock_held; uint64_t hw_lock_failed; @@ -239,6 +240,9 @@ struct qla_host { volatile const char *qla_unlock; uint32_t dbg_level; uint32_t enable_minidump; + uint32_t enable_driverstate_dump; + uint32_t enable_error_recovery; + uint32_t ms_delay_after_init; uint8_t fw_ver_str[32]; @@ -273,4 +277,6 @@ typedef struct qla_host qla_host_t; ((((*(uint32_t *) mac1) == (*(uint32_t *) mac2) && \ (*(uint16_t *)(mac1 + 4)) == (*(uint16_t *)(mac2 + 4)))) ? 0 : 1) +#define QL_INITIATE_RECOVERY(ha) qla_set_error_recovery(ha) + #endif /* #ifndef _QL_DEF_H_ */ diff --git a/sys/dev/qlxgbe/ql_glbl.h b/sys/dev/qlxgbe/ql_glbl.h index f8c7c8b16..9a5bbb669 100644 --- a/sys/dev/qlxgbe/ql_glbl.h +++ b/sys/dev/qlxgbe/ql_glbl.h @@ -47,6 +47,7 @@ extern uint32_t ql_rcv_isr(qla_host_t *ha, uint32_t sds_idx, uint32_t count); extern int ql_alloc_dmabuf(qla_host_t *ha, qla_dma_t *dma_buf); extern void ql_free_dmabuf(qla_host_t *ha, qla_dma_t *dma_buf); extern int ql_get_mbuf(qla_host_t *ha, qla_rx_buf_t *rxb, struct mbuf *nmp); +extern void qla_set_error_recovery(qla_host_t *ha); /* * from ql_hw.c @@ -115,5 +116,11 @@ extern unsigned int ql83xx_minidump_len; extern void ql_alloc_drvr_state_buffer(qla_host_t *ha); extern void ql_free_drvr_state_buffer(qla_host_t *ha); extern void ql_capture_drvr_state(qla_host_t *ha); +extern void ql_sp_log(qla_host_t *ha, uint16_t fmtstr_idx, uint16_t num_params, + uint32_t param0, uint32_t param1, uint32_t param2, + uint32_t param3, uint32_t param4); +extern void ql_alloc_sp_log_buffer(qla_host_t *ha); +extern void ql_free_sp_log_buffer(qla_host_t *ha); + #endif /* #ifndef_QL_GLBL_H_ */ diff --git a/sys/dev/qlxgbe/ql_hw.c b/sys/dev/qlxgbe/ql_hw.c index 102fb337b..45aab1222 100644 --- a/sys/dev/qlxgbe/ql_hw.c +++ b/sys/dev/qlxgbe/ql_hw.c @@ -49,7 +49,7 @@ __FBSDID("$FreeBSD$"); static void qla_del_rcv_cntxt(qla_host_t *ha); static int qla_init_rcv_cntxt(qla_host_t *ha); -static void qla_del_xmt_cntxt(qla_host_t *ha); +static int qla_del_xmt_cntxt(qla_host_t *ha); static int qla_init_xmt_cntxt(qla_host_t *ha); static int qla_mbx_cmd(qla_host_t *ha, uint32_t *h_mbox, uint32_t n_hmbox, uint32_t *fw_mbox, uint32_t n_fwmbox, uint32_t no_pause); @@ -646,12 +646,119 @@ qlnx_add_hw_xmt_stats_sysctls(qla_host_t *ha) return; } +static void +qlnx_add_hw_mbx_cmpl_stats_sysctls(qla_host_t *ha) +{ + struct sysctl_ctx_list *ctx; + struct sysctl_oid_list *node_children; + + ctx = device_get_sysctl_ctx(ha->pci_dev); + node_children = SYSCTL_CHILDREN(device_get_sysctl_tree(ha->pci_dev)); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "mbx_completion_time_lt_200ms", + CTLFLAG_RD, &ha->hw.mbx_comp_msecs[0], + "mbx_completion_time_lt_200ms"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "mbx_completion_time_200ms_400ms", + CTLFLAG_RD, &ha->hw.mbx_comp_msecs[1], + "mbx_completion_time_200ms_400ms"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "mbx_completion_time_400ms_600ms", + CTLFLAG_RD, &ha->hw.mbx_comp_msecs[2], + "mbx_completion_time_400ms_600ms"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "mbx_completion_time_600ms_800ms", + CTLFLAG_RD, &ha->hw.mbx_comp_msecs[3], + "mbx_completion_time_600ms_800ms"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "mbx_completion_time_800ms_1000ms", + CTLFLAG_RD, &ha->hw.mbx_comp_msecs[4], + "mbx_completion_time_800ms_1000ms"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "mbx_completion_time_1000ms_1200ms", + CTLFLAG_RD, &ha->hw.mbx_comp_msecs[5], + "mbx_completion_time_1000ms_1200ms"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "mbx_completion_time_1200ms_1400ms", + CTLFLAG_RD, &ha->hw.mbx_comp_msecs[6], + "mbx_completion_time_1200ms_1400ms"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "mbx_completion_time_1400ms_1600ms", + CTLFLAG_RD, &ha->hw.mbx_comp_msecs[7], + "mbx_completion_time_1400ms_1600ms"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "mbx_completion_time_1600ms_1800ms", + CTLFLAG_RD, &ha->hw.mbx_comp_msecs[8], + "mbx_completion_time_1600ms_1800ms"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "mbx_completion_time_1800ms_2000ms", + CTLFLAG_RD, &ha->hw.mbx_comp_msecs[9], + "mbx_completion_time_1800ms_2000ms"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "mbx_completion_time_2000ms_2200ms", + CTLFLAG_RD, &ha->hw.mbx_comp_msecs[10], + "mbx_completion_time_2000ms_2200ms"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "mbx_completion_time_2200ms_2400ms", + CTLFLAG_RD, &ha->hw.mbx_comp_msecs[11], + "mbx_completion_time_2200ms_2400ms"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "mbx_completion_time_2400ms_2600ms", + CTLFLAG_RD, &ha->hw.mbx_comp_msecs[12], + "mbx_completion_time_2400ms_2600ms"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "mbx_completion_time_2600ms_2800ms", + CTLFLAG_RD, &ha->hw.mbx_comp_msecs[13], + "mbx_completion_time_2600ms_2800ms"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "mbx_completion_time_2800ms_3000ms", + CTLFLAG_RD, &ha->hw.mbx_comp_msecs[14], + "mbx_completion_time_2800ms_3000ms"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "mbx_completion_time_3000ms_4000ms", + CTLFLAG_RD, &ha->hw.mbx_comp_msecs[15], + "mbx_completion_time_3000ms_4000ms"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "mbx_completion_time_4000ms_5000ms", + CTLFLAG_RD, &ha->hw.mbx_comp_msecs[16], + "mbx_completion_time_4000ms_5000ms"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "mbx_completion_host_mbx_cntrl_timeout", + CTLFLAG_RD, &ha->hw.mbx_comp_msecs[17], + "mbx_completion_host_mbx_cntrl_timeout"); + + SYSCTL_ADD_QUAD(ctx, node_children, + OID_AUTO, "mbx_completion_fw_mbx_cntrl_timeout", + CTLFLAG_RD, &ha->hw.mbx_comp_msecs[18], + "mbx_completion_fw_mbx_cntrl_timeout"); + return; +} + static void qlnx_add_hw_stats_sysctls(qla_host_t *ha) { qlnx_add_hw_mac_stats_sysctls(ha); qlnx_add_hw_rcv_stats_sysctls(ha); qlnx_add_hw_xmt_stats_sysctls(ha); + qlnx_add_hw_mbx_cmpl_stats_sysctls(ha); return; } @@ -918,6 +1025,30 @@ ql_hw_add_sysctls(qla_host_t *ha) "\t Any change requires ifconfig down/up to take effect\n" "\t Note that LRO may be turned off/on via ifconfig\n"); + SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "sp_log_index", CTLFLAG_RW, &ha->hw.sp_log_index, + ha->hw.sp_log_index, "sp_log_index"); + + SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "sp_log_stop", CTLFLAG_RW, &ha->hw.sp_log_stop, + ha->hw.sp_log_stop, "sp_log_stop"); + + ha->hw.sp_log_stop_events = 0; + + SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "sp_log_stop_events", CTLFLAG_RW, + &ha->hw.sp_log_stop_events, + ha->hw.sp_log_stop_events, "Slow path event log is stopped" + " when OR of the following events occur \n" + "\t 0x01 : Heart beat Failure\n" + "\t 0x02 : Temperature Failure\n" + "\t 0x04 : HW Initialization Failure\n" + "\t 0x08 : Interface Initialization Failure\n" + "\t 0x10 : Error Recovery Failure\n"); + ha->hw.mdump_active = 0; SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), @@ -998,13 +1129,13 @@ ql_hw_link_status(qla_host_t *ha) device_printf(ha->pci_dev, "link Down\n"); } - if (ha->hw.flags.fduplex) { + if (ha->hw.fduplex) { device_printf(ha->pci_dev, "Full Duplex\n"); } else { device_printf(ha->pci_dev, "Half Duplex\n"); } - if (ha->hw.flags.autoneg) { + if (ha->hw.autoneg) { device_printf(ha->pci_dev, "Auto Negotiation Enabled\n"); } else { device_printf(ha->pci_dev, "Auto Negotiation Disabled\n"); @@ -1255,19 +1386,39 @@ qla_mbx_cmd(qla_host_t *ha, uint32_t *h_mbox, uint32_t n_hmbox, uint32_t i; uint32_t data; int ret = 0; + uint64_t start_usecs; + uint64_t end_usecs; + uint64_t msecs_200; - if (QL_ERR_INJECT(ha, INJCT_MBX_CMD_FAILURE)) { + ql_sp_log(ha, 0, 5, no_pause, h_mbox[0], h_mbox[1], h_mbox[2], h_mbox[3]); + + if (ha->offline || ha->qla_initiate_recovery) { + ql_sp_log(ha, 1, 2, ha->offline, ha->qla_initiate_recovery, 0, 0, 0); + goto exit_qla_mbx_cmd; + } + + if (((ha->err_inject & 0xFFFF) == INJCT_MBX_CMD_FAILURE) && + (((ha->err_inject & ~0xFFFF) == ((h_mbox[0] & 0xFFFF) << 16))|| + !(ha->err_inject & ~0xFFFF))) { ret = -3; - ha->qla_initiate_recovery = 1; + QL_INITIATE_RECOVERY(ha); goto exit_qla_mbx_cmd; } + start_usecs = qla_get_usec_timestamp(); + if (no_pause) i = 1000; else i = Q8_MBX_MSEC_DELAY; while (i) { + + if (ha->qla_initiate_recovery) { + ql_sp_log(ha, 2, 1, ha->qla_initiate_recovery, 0, 0, 0, 0); + return (-1); + } + data = READ_REG32(ha, Q8_HOST_MBOX_CNTRL); if (data == 0) break; @@ -1282,8 +1433,10 @@ qla_mbx_cmd(qla_host_t *ha, uint32_t *h_mbox, uint32_t n_hmbox, if (i == 0) { device_printf(ha->pci_dev, "%s: host_mbx_cntrl 0x%08x\n", __func__, data); + ql_sp_log(ha, 3, 1, data, 0, 0, 0, 0); ret = -1; - ha->qla_initiate_recovery = 1; + ha->hw.mbx_comp_msecs[(Q8_MBX_COMP_MSECS - 2)]++; + QL_INITIATE_RECOVERY(ha); goto exit_qla_mbx_cmd; } @@ -1297,6 +1450,12 @@ qla_mbx_cmd(qla_host_t *ha, uint32_t *h_mbox, uint32_t n_hmbox, i = Q8_MBX_MSEC_DELAY; while (i) { + + if (ha->qla_initiate_recovery) { + ql_sp_log(ha, 4, 1, ha->qla_initiate_recovery, 0, 0, 0, 0); + return (-1); + } + data = READ_REG32(ha, Q8_FW_MBOX_CNTRL); if ((data & 0x3) == 1) { @@ -1314,18 +1473,44 @@ qla_mbx_cmd(qla_host_t *ha, uint32_t *h_mbox, uint32_t n_hmbox, if (i == 0) { device_printf(ha->pci_dev, "%s: fw_mbx_cntrl 0x%08x\n", __func__, data); + ql_sp_log(ha, 5, 1, data, 0, 0, 0, 0); ret = -2; - ha->qla_initiate_recovery = 1; + ha->hw.mbx_comp_msecs[(Q8_MBX_COMP_MSECS - 1)]++; + QL_INITIATE_RECOVERY(ha); goto exit_qla_mbx_cmd; } for (i = 0; i < n_fwmbox; i++) { + + if (ha->qla_initiate_recovery) { + ql_sp_log(ha, 6, 1, ha->qla_initiate_recovery, 0, 0, 0, 0); + return (-1); + } + *fw_mbox++ = READ_REG32(ha, (Q8_FW_MBOX0 + (i << 2))); } WRITE_REG32(ha, Q8_FW_MBOX_CNTRL, 0x0); WRITE_REG32(ha, ha->hw.mbx_intr_mask_offset, 0x0); + end_usecs = qla_get_usec_timestamp(); + + if (end_usecs > start_usecs) { + msecs_200 = (end_usecs - start_usecs)/(1000 * 200); + + if (msecs_200 < 15) + ha->hw.mbx_comp_msecs[msecs_200]++; + else if (msecs_200 < 20) + ha->hw.mbx_comp_msecs[15]++; + else { + device_printf(ha->pci_dev, "%s: [%ld, %ld] %ld\n", __func__, + start_usecs, end_usecs, msecs_200); + ha->hw.mbx_comp_msecs[16]++; + } + } + ql_sp_log(ha, 7, 5, fw_mbox[0], fw_mbox[1], fw_mbox[2], fw_mbox[3], fw_mbox[4]); + + exit_qla_mbx_cmd: return (ret); } @@ -1401,7 +1586,8 @@ qla_config_intr_cntxt(qla_host_t *ha, uint32_t start_idx, uint32_t num_intrs, if (qla_mbx_cmd(ha, (uint32_t *)c_intr, (sizeof (q80_config_intr_t) >> 2), ha->hw.mbox, (sizeof (q80_config_intr_rsp_t) >> 2), 0)) { - device_printf(dev, "%s: failed0\n", __func__); + device_printf(dev, "%s: %s failed0\n", __func__, + (create ? "create" : "delete")); return (-1); } @@ -1410,8 +1596,8 @@ qla_config_intr_cntxt(qla_host_t *ha, uint32_t start_idx, uint32_t num_intrs, err = Q8_MBX_RSP_STATUS(c_intr_rsp->regcnt_status); if (err) { - device_printf(dev, "%s: failed1 [0x%08x, %d]\n", __func__, err, - c_intr_rsp->nentries); + device_printf(dev, "%s: %s failed1 [0x%08x, %d]\n", __func__, + (create ? "create" : "delete"), err, c_intr_rsp->nentries); for (i = 0; i < c_intr_rsp->nentries; i++) { device_printf(dev, "%s: [%d]:[0x%x 0x%x 0x%x]\n", @@ -2015,7 +2201,8 @@ ql_get_stats(qla_host_t *ha) cmd |= ((ha->pci_func & 0x1) << 16); - if (ha->qla_watchdog_pause) + if (ha->qla_watchdog_pause || (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) || + ha->offline) goto ql_get_stats_exit; if (qla_get_hw_stats(ha, cmd, sizeof (q80_get_stats_rsp_t)) == 0) { @@ -2032,7 +2219,8 @@ ql_get_stats(qla_host_t *ha) // cmd |= Q8_GET_STATS_CMD_CLEAR; cmd |= (ha->hw.rcv_cntxt_id << 16); - if (ha->qla_watchdog_pause) + if (ha->qla_watchdog_pause || (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) || + ha->offline) goto ql_get_stats_exit; if (qla_get_hw_stats(ha, cmd, sizeof (q80_get_stats_rsp_t)) == 0) { @@ -2043,13 +2231,18 @@ ql_get_stats(qla_host_t *ha) __func__, ha->hw.mbox[0]); } - if (ha->qla_watchdog_pause) + if (ha->qla_watchdog_pause || (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) || + ha->offline) goto ql_get_stats_exit; /* * Get XMT Statistics */ - for (i = 0 ; ((i < ha->hw.num_tx_rings) && (!ha->qla_watchdog_pause)); - i++) { + for (i = 0 ; (i < ha->hw.num_tx_rings); i++) { + if (ha->qla_watchdog_pause || + (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) || + ha->offline) + goto ql_get_stats_exit; + cmd = Q8_GET_STATS_CMD_XMT | Q8_GET_STATS_CMD_TYPE_CNTXT; // cmd |= Q8_GET_STATS_CMD_CLEAR; cmd |= (ha->hw.tx_cntxt[i].tx_cntxt_id << 16); @@ -2679,7 +2872,8 @@ ql_del_hw_if(qla_host_t *ha) qla_del_rcv_cntxt(ha); - qla_del_xmt_cntxt(ha); + if(qla_del_xmt_cntxt(ha)) + goto ql_del_hw_if_exit; if (ha->hw.flags.init_intr_cnxt) { for (i = 0; i < ha->hw.num_sds_rings; ) { @@ -2688,7 +2882,9 @@ ql_del_hw_if(qla_host_t *ha) num_msix = Q8_MAX_INTR_VECTORS; else num_msix = ha->hw.num_sds_rings - i; - qla_config_intr_cntxt(ha, i, num_msix, 0); + + if (qla_config_intr_cntxt(ha, i, num_msix, 0)) + break; i += num_msix; } @@ -2696,6 +2892,7 @@ ql_del_hw_if(qla_host_t *ha) ha->hw.flags.init_intr_cnxt = 0; } +ql_del_hw_if_exit: if (ha->hw.enable_soft_lro) { qla_drain_soft_lro(ha); qla_free_soft_lro(ha); @@ -3328,19 +3525,22 @@ qla_del_xmt_cntxt_i(qla_host_t *ha, uint32_t txr_idx) return (0); } -static void +static int qla_del_xmt_cntxt(qla_host_t *ha) { uint32_t i; + int ret = 0; if (!ha->hw.flags.init_tx_cnxt) - return; + return (ret); for (i = 0; i < ha->hw.num_tx_rings; i++) { - if (qla_del_xmt_cntxt_i(ha, i)) + if ((ret = qla_del_xmt_cntxt_i(ha, i)) != 0) break; } ha->hw.flags.init_tx_cnxt = 0; + + return (ret); } static int @@ -3350,8 +3550,10 @@ qla_init_xmt_cntxt(qla_host_t *ha) for (i = 0; i < ha->hw.num_tx_rings; i++) { if (qla_init_xmt_cntxt_i(ha, i) != 0) { - for (j = 0; j < i; j++) - qla_del_xmt_cntxt_i(ha, j); + for (j = 0; j < i; j++) { + if (qla_del_xmt_cntxt_i(ha, j)) + break; + } return (-1); } } @@ -3627,21 +3829,22 @@ ql_hw_tx_done_locked(qla_host_t *ha, uint32_t txr_idx) void ql_update_link_state(qla_host_t *ha) { - uint32_t link_state; + uint32_t link_state = 0; uint32_t prev_link_state; - if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) { - ha->hw.link_up = 0; - return; - } - link_state = READ_REG32(ha, Q8_LINK_STATE); - prev_link_state = ha->hw.link_up; - if (ha->pci_func == 0) - ha->hw.link_up = (((link_state & 0xF) == 1)? 1 : 0); - else - ha->hw.link_up = ((((link_state >> 4)& 0xF) == 1)? 1 : 0); + if (ha->ifp->if_drv_flags & IFF_DRV_RUNNING) { + link_state = READ_REG32(ha, Q8_LINK_STATE); + + if (ha->pci_func == 0) { + link_state = (((link_state & 0xF) == 1)? 1 : 0); + } else { + link_state = ((((link_state >> 4)& 0xF) == 1)? 1 : 0); + } + } + + atomic_store_rel_8(&ha->hw.link_up, (uint8_t)link_state); if (prev_link_state != ha->hw.link_up) { if (ha->hw.link_up) { @@ -3669,8 +3872,14 @@ ql_hw_check_health(qla_host_t *ha) if (((val & 0xFFFF) == 2) || ((val & 0xFFFF) == 3) || (QL_ERR_INJECT(ha, INJCT_TEMPERATURE_FAILURE))) { - device_printf(ha->pci_dev, "%s: Temperature Alert [0x%08x]\n", - __func__, val); + device_printf(ha->pci_dev, "%s: Temperature Alert" + " at ts_usecs %ld ts_reg = 0x%08x\n", + __func__, qla_get_usec_timestamp(), val); + + if (ha->hw.sp_log_stop_events & Q8_SP_LOG_STOP_TEMP_FAILURE) + ha->hw.sp_log_stop = -1; + + QL_INITIATE_RECOVERY(ha); return -1; } @@ -3691,9 +3900,25 @@ ql_hw_check_health(qla_host_t *ha) __func__, val); if (ha->hw.hbeat_failure < 2) /* we ignore the first failure */ return 0; - else - device_printf(ha->pci_dev, "%s: Heartbeat Failue [0x%08x]\n", - __func__, val); + else { + uint32_t peg_halt_status1; + uint32_t peg_halt_status2; + + peg_halt_status1 = READ_REG32(ha, Q8_PEG_HALT_STATUS1); + peg_halt_status2 = READ_REG32(ha, Q8_PEG_HALT_STATUS2); + + device_printf(ha->pci_dev, + "%s: Heartbeat Failue at ts_usecs = %ld " + "fw_heart_beat = 0x%08x " + "peg_halt_status1 = 0x%08x " + "peg_halt_status2 = 0x%08x\n", + __func__, qla_get_usec_timestamp(), val, + peg_halt_status1, peg_halt_status2); + + if (ha->hw.sp_log_stop_events & Q8_SP_LOG_STOP_HBEAT_FAILURE) + ha->hw.sp_log_stop = -1; + } + QL_INITIATE_RECOVERY(ha); return -1; } @@ -4429,8 +4654,8 @@ ql_minidump(qla_host_t *ha) if (ha->hw.mdump_done) return; - - ha->hw.mdump_start_seq_index = ql_stop_sequence(ha); + ha->hw.mdump_usec_ts = qla_get_usec_timestamp(); + ha->hw.mdump_start_seq_index = ql_stop_sequence(ha); bzero(ha->hw.mdump_buffer, ha->hw.mdump_buffer_size); bzero(ha->hw.mdump_template, ha->hw.mdump_template_size); diff --git a/sys/dev/qlxgbe/ql_hw.h b/sys/dev/qlxgbe/ql_hw.h index e8f53c864..f59af522d 100644 --- a/sys/dev/qlxgbe/ql_hw.h +++ b/sys/dev/qlxgbe/ql_hw.h @@ -1600,26 +1600,26 @@ typedef struct _qla_hw { uint32_t unicast_mac :1, bcast_mac :1, - loopback_mode :2, init_tx_cnxt :1, init_rx_cnxt :1, init_intr_cnxt :1, - fduplex :1, - autoneg :1, fdt_valid :1; } flags; - uint16_t link_speed; - uint16_t cable_length; - uint32_t cable_oui; - uint8_t link_up; - uint8_t module_type; - uint8_t link_faults; + volatile uint16_t link_speed; + volatile uint16_t cable_length; + volatile uint32_t cable_oui; + volatile uint8_t link_up; + volatile uint8_t module_type; + volatile uint8_t link_faults; + volatile uint8_t loopback_mode; + volatile uint8_t fduplex; + volatile uint8_t autoneg; - uint8_t mac_rcv_mode; + volatile uint8_t mac_rcv_mode; - uint32_t max_mtu; + volatile uint32_t max_mtu; uint8_t mac_addr[ETHER_ADDR_LEN]; @@ -1703,9 +1703,25 @@ typedef struct _qla_hw { uint32_t mdump_buffer_size; void *mdump_template; uint32_t mdump_template_size; + uint64_t mdump_usec_ts; +#define Q8_MBX_COMP_MSECS (19) + uint64_t mbx_comp_msecs[Q8_MBX_COMP_MSECS]; /* driver state related */ void *drvr_state; + + /* slow path trace */ + uint32_t sp_log_stop_events; +#define Q8_SP_LOG_STOP_HBEAT_FAILURE 0x001 +#define Q8_SP_LOG_STOP_TEMP_FAILURE 0x002 +#define Q8_SP_LOG_STOP_HW_INIT_FAILURE 0x004 +#define Q8_SP_LOG_STOP_IF_START_FAILURE 0x008 +#define Q8_SP_LOG_STOP_ERR_RECOVERY_FAILURE 0x010 + + uint32_t sp_log_stop; + uint32_t sp_log_index; + uint32_t sp_log_num_entries; + void *sp_log; } qla_hw_t; #define QL_UPDATE_RDS_PRODUCER_INDEX(ha, prod_reg, val) \ diff --git a/sys/dev/qlxgbe/ql_inline.h b/sys/dev/qlxgbe/ql_inline.h index 986855a73..41093de21 100644 --- a/sys/dev/qlxgbe/ql_inline.h +++ b/sys/dev/qlxgbe/ql_inline.h @@ -166,7 +166,7 @@ qla_lock(qla_host_t *ha, const char *str, uint32_t timeout_ms, while (1) { mtx_lock(&ha->hw_lock); - if (ha->qla_detach_active) { + if (ha->qla_detach_active || ha->offline) { mtx_unlock(&ha->hw_lock); break; } @@ -191,7 +191,10 @@ qla_lock(qla_host_t *ha, const char *str, uint32_t timeout_ms, } } - //device_printf(ha->pci_dev, "%s: %s ret = %d\n", __func__, str,ret); +// if (!ha->enable_error_recovery) +// device_printf(ha->pci_dev, "%s: %s ret = %d\n", __func__, +// str,ret); + return (ret); } @@ -202,7 +205,9 @@ qla_unlock(qla_host_t *ha, const char *str) ha->hw_lock_held = 0; ha->qla_unlock = str; mtx_unlock(&ha->hw_lock); - //device_printf(ha->pci_dev, "%s: %s\n", __func__, str); + +// if (!ha->enable_error_recovery) +// device_printf(ha->pci_dev, "%s: %s\n", __func__, str); return; } diff --git a/sys/dev/qlxgbe/ql_ioctl.c b/sys/dev/qlxgbe/ql_ioctl.c index 7675d5482..377c28ecb 100644 --- a/sys/dev/qlxgbe/ql_ioctl.c +++ b/sys/dev/qlxgbe/ql_ioctl.c @@ -42,6 +42,7 @@ __FBSDID("$FreeBSD$"); #include "ql_ver.h" #include "ql_dbg.h" +static int ql_slowpath_log(qla_host_t *ha, qla_sp_log_t *log); static int ql_drvr_state(qla_host_t *ha, qla_driver_state_t *drvr_state); static uint32_t ql_drvr_state_size(qla_host_t *ha); static int ql_eioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, @@ -224,6 +225,7 @@ ql_eioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, case QLA_RD_FW_DUMP: if (ha->hw.mdump_init == 0) { + device_printf(pci_dev, "%s: minidump not initialized\n", __func__); rval = EINVAL; break; } @@ -233,45 +235,85 @@ ql_eioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, if ((fw_dump->minidump == NULL) || (fw_dump->minidump_size != (ha->hw.mdump_buffer_size + ha->hw.mdump_template_size))) { + device_printf(pci_dev, + "%s: minidump buffer [%p] size = [%d, %d] invalid\n", __func__, + fw_dump->minidump, fw_dump->minidump_size, + (ha->hw.mdump_buffer_size + ha->hw.mdump_template_size)); rval = EINVAL; break; } - if (QLA_LOCK(ha, __func__, QLA_LOCK_DEFAULT_MS_TIMEOUT, 0) == 0) { - if (!ha->hw.mdump_done) - ha->qla_initiate_recovery = 1; - QLA_UNLOCK(ha, __func__); - } else { + if ((ha->pci_func & 0x1)) { + device_printf(pci_dev, "%s: mindump allowed only on Port0\n", __func__); rval = ENXIO; break; } + + fw_dump->saved = 1; + + if (ha->offline) { + + if (ha->enable_minidump) + ql_minidump(ha); + + fw_dump->saved = 0; + fw_dump->usec_ts = ha->hw.mdump_usec_ts; + + if (!ha->hw.mdump_done) { + device_printf(pci_dev, + "%s: port offline minidump failed\n", __func__); + rval = ENXIO; + break; + } + } else { + + if (QLA_LOCK(ha, __func__, QLA_LOCK_DEFAULT_MS_TIMEOUT, 0) == 0) { + if (!ha->hw.mdump_done) { + fw_dump->saved = 0; + QL_INITIATE_RECOVERY(ha); + device_printf(pci_dev, "%s: recovery initiated " + " to trigger minidump\n", + __func__); + } + QLA_UNLOCK(ha, __func__); + } else { + device_printf(pci_dev, "%s: QLA_LOCK() failed0\n", __func__); + rval = ENXIO; + break; + } #define QLNX_DUMP_WAIT_SECS 30 - count = QLNX_DUMP_WAIT_SECS * 1000; + count = QLNX_DUMP_WAIT_SECS * 1000; - while (count) { - if (ha->hw.mdump_done) - break; - qla_mdelay(__func__, 100); - count -= 100; - } + while (count) { + if (ha->hw.mdump_done) + break; + qla_mdelay(__func__, 100); + count -= 100; + } - if (!ha->hw.mdump_done) { - rval = ENXIO; - break; - } + if (!ha->hw.mdump_done) { + device_printf(pci_dev, + "%s: port not offline minidump failed\n", __func__); + rval = ENXIO; + break; + } + fw_dump->usec_ts = ha->hw.mdump_usec_ts; - if (QLA_LOCK(ha, __func__, QLA_LOCK_DEFAULT_MS_TIMEOUT, 0) == 0) { - ha->hw.mdump_done = 0; - QLA_UNLOCK(ha, __func__); - } else { - rval = ENXIO; - break; + if (QLA_LOCK(ha, __func__, QLA_LOCK_DEFAULT_MS_TIMEOUT, 0) == 0) { + ha->hw.mdump_done = 0; + QLA_UNLOCK(ha, __func__); + } else { + device_printf(pci_dev, "%s: QLA_LOCK() failed1\n", __func__); + rval = ENXIO; + break; + } } if ((rval = copyout(ha->hw.mdump_template, fw_dump->minidump, ha->hw.mdump_template_size))) { + device_printf(pci_dev, "%s: template copyout failed\n", __func__); rval = ENXIO; break; } @@ -279,14 +321,20 @@ ql_eioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, if ((rval = copyout(ha->hw.mdump_buffer, ((uint8_t *)fw_dump->minidump + ha->hw.mdump_template_size), - ha->hw.mdump_buffer_size))) + ha->hw.mdump_buffer_size))) { + device_printf(pci_dev, "%s: minidump copyout failed\n", __func__); rval = ENXIO; + } break; case QLA_RD_DRVR_STATE: rval = ql_drvr_state(ha, (qla_driver_state_t *)data); break; + case QLA_RD_SLOWPATH_LOG: + rval = ql_slowpath_log(ha, (qla_sp_log_t *)data); + break; + case QLA_RD_PCI_IDS: pci_ids = (qla_rd_pci_ids_t *)data; pci_ids->ven_id = pci_get_vendor(pci_dev); @@ -304,12 +352,12 @@ ql_eioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, } + static int ql_drvr_state(qla_host_t *ha, qla_driver_state_t *state) { int rval = 0; uint32_t drvr_state_size; - qla_drvr_state_hdr_t *hdr; drvr_state_size = ql_drvr_state_size(ha); @@ -324,10 +372,7 @@ ql_drvr_state(qla_host_t *ha, qla_driver_state_t *state) if (ha->hw.drvr_state == NULL) return (ENOMEM); - hdr = ha->hw.drvr_state; - - if (!hdr->drvr_version_major) - ql_capture_drvr_state(ha); + ql_capture_drvr_state(ha); rval = copyout(ha->hw.drvr_state, state->buffer, drvr_state_size); @@ -416,21 +461,25 @@ ql_capture_drvr_state(qla_host_t *ha) { uint8_t *state_buffer; uint8_t *ptr; - uint32_t drvr_state_size; qla_drvr_state_hdr_t *hdr; uint32_t size; int i; - drvr_state_size = ql_drvr_state_size(ha); - state_buffer = ha->hw.drvr_state; if (state_buffer == NULL) return; - - bzero(state_buffer, drvr_state_size); hdr = (qla_drvr_state_hdr_t *)state_buffer; + + hdr->saved = 0; + + if (hdr->drvr_version_major) { + hdr->saved = 1; + return; + } + + hdr->usec_ts = qla_get_usec_timestamp(); hdr->drvr_version_major = QLA_VERSION_MAJOR; hdr->drvr_version_minor = QLA_VERSION_MINOR; @@ -512,6 +561,9 @@ ql_alloc_drvr_state_buffer(qla_host_t *ha) ha->hw.drvr_state = malloc(drvr_state_size, M_QLA83XXBUF, M_NOWAIT); + if (ha->hw.drvr_state != NULL) + bzero(ha->hw.drvr_state, drvr_state_size); + return; } @@ -523,3 +575,93 @@ ql_free_drvr_state_buffer(qla_host_t *ha) return; } +void +ql_sp_log(qla_host_t *ha, uint16_t fmtstr_idx, uint16_t num_params, + uint32_t param0, uint32_t param1, uint32_t param2, uint32_t param3, + uint32_t param4) +{ + qla_sp_log_entry_t *sp_e, *sp_log; + + if (((sp_log = ha->hw.sp_log) == NULL) || ha->hw.sp_log_stop) + return; + + mtx_lock(&ha->sp_log_lock); + + sp_e = &sp_log[ha->hw.sp_log_index]; + + bzero(sp_e, sizeof (qla_sp_log_entry_t)); + + sp_e->fmtstr_idx = fmtstr_idx; + sp_e->num_params = num_params; + + sp_e->usec_ts = qla_get_usec_timestamp(); + + sp_e->params[0] = param0; + sp_e->params[1] = param1; + sp_e->params[2] = param2; + sp_e->params[3] = param3; + sp_e->params[4] = param4; + + ha->hw.sp_log_index = (ha->hw.sp_log_index + 1) & (NUM_LOG_ENTRIES - 1); + + if (ha->hw.sp_log_num_entries < NUM_LOG_ENTRIES) + ha->hw.sp_log_num_entries++; + + mtx_unlock(&ha->sp_log_lock); + + return; +} + +void +ql_alloc_sp_log_buffer(qla_host_t *ha) +{ + uint32_t size; + + size = (sizeof(qla_sp_log_entry_t)) * NUM_LOG_ENTRIES; + + ha->hw.sp_log = malloc(size, M_QLA83XXBUF, M_NOWAIT); + + if (ha->hw.sp_log != NULL) + bzero(ha->hw.sp_log, size); + + ha->hw.sp_log_index = 0; + ha->hw.sp_log_num_entries = 0; + + return; +} + +void +ql_free_sp_log_buffer(qla_host_t *ha) +{ + if (ha->hw.sp_log != NULL) + free(ha->hw.sp_log, M_QLA83XXBUF); + return; +} + +static int +ql_slowpath_log(qla_host_t *ha, qla_sp_log_t *log) +{ + int rval = 0; + uint32_t size; + + if ((ha->hw.sp_log == NULL) || (log->buffer == NULL)) + return (EINVAL); + + size = (sizeof(qla_sp_log_entry_t) * NUM_LOG_ENTRIES); + + mtx_lock(&ha->sp_log_lock); + + rval = copyout(ha->hw.sp_log, log->buffer, size); + + if (!rval) { + log->next_idx = ha->hw.sp_log_index; + log->num_entries = ha->hw.sp_log_num_entries; + } + device_printf(ha->pci_dev, + "%s: exit [rval = %d][%p, next_idx = %d, %d entries, %d bytes]\n", + __func__, rval, log->buffer, log->next_idx, log->num_entries, size); + mtx_unlock(&ha->sp_log_lock); + + return (rval); +} + diff --git a/sys/dev/qlxgbe/ql_ioctl.h b/sys/dev/qlxgbe/ql_ioctl.h index f985958a7..3a8150b63 100644 --- a/sys/dev/qlxgbe/ql_ioctl.h +++ b/sys/dev/qlxgbe/ql_ioctl.h @@ -73,6 +73,17 @@ struct qla_rd_pci_ids { }; typedef struct qla_rd_pci_ids qla_rd_pci_ids_t; +#define NUM_LOG_ENTRY_PARAMS 5 +#define NUM_LOG_ENTRIES 512 + +struct qla_sp_log_entry { + uint32_t fmtstr_idx; + uint32_t num_params; + uint64_t usec_ts; + uint32_t params[NUM_LOG_ENTRY_PARAMS]; +}; +typedef struct qla_sp_log_entry qla_sp_log_entry_t; + /* * structure encapsulating the value to read/write from/to offchip (MS) memory */ @@ -88,6 +99,8 @@ typedef struct qla_offchip_mem_val qla_offchip_mem_val_t; struct qla_rd_fw_dump { uint16_t pci_func; + uint16_t saved; + uint64_t usec_ts; uint32_t minidump_size; void *minidump; }; @@ -122,6 +135,8 @@ struct qla_drvr_state_hdr { uint32_t drvr_version_build; uint8_t mac_addr[ETHER_ADDR_LEN]; + uint16_t saved; + uint64_t usec_ts; uint16_t link_speed; uint16_t cable_length; uint32_t cable_oui; @@ -159,6 +174,13 @@ struct qla_driver_state { }; typedef struct qla_driver_state qla_driver_state_t; +struct qla_sp_log { + uint32_t next_idx; /* index of next entry in slowpath trace log */ + uint32_t num_entries; /* number of entries in slowpath trace log */ + void *buffer; +}; +typedef struct qla_sp_log qla_sp_log_t; + /* * Read/Write Register */ @@ -204,5 +226,58 @@ typedef struct qla_driver_state qla_driver_state_t; */ #define QLA_RD_DRVR_STATE _IOWR('q', 9, qla_driver_state_t) +/* + * Read Slowpath Log + */ +#define QLA_RD_SLOWPATH_LOG _IOWR('q', 10, qla_sp_log_t) + +/* + * Format Strings For Slowpath Trace Logs + */ +#define SP_TLOG_FMT_STR_0 \ + "qla_mbx_cmd [%ld]: enter no_pause = %d [0x%08x 0x%08x 0x%08x 0x%08x]\n" + +#define SP_TLOG_FMT_STR_1 \ + "qla_mbx_cmd [%ld]: offline = 0x%08x qla_initiate_recovery = 0x%08x exit1\n" + +#define SP_TLOG_FMT_STR_2 \ + "qla_mbx_cmd [%ld]: qla_initiate_recovery = 0x%08x exit2\n" + +#define SP_TLOG_FMT_STR_3 \ + "qla_mbx_cmd [%ld]: timeout exit3 [host_mbx_cntrl = 0x%08x]\n" + +#define SP_TLOG_FMT_STR_4 \ + "qla_mbx_cmd [%ld]: qla_initiate_recovery = 0x%08x exit4\n" + +#define SP_TLOG_FMT_STR_5 \ + "qla_mbx_cmd [%ld]: timeout exit5 [fw_mbx_cntrl = 0x%08x]\n" + +#define SP_TLOG_FMT_STR_6 \ + "qla_mbx_cmd [%ld]: qla_initiate_recovery = 0x%08x exit6\n" + +#define SP_TLOG_FMT_STR_7 \ + "qla_mbx_cmd [%ld]: exit [0x%08x 0x%08x 0x%08x 0x%08x 0x%08x]\n" + +#define SP_TLOG_FMT_STR_8 \ + "qla_ioctl [%ld]: SIOCSIFADDR if_drv_flags = 0x%08x [0x%08x] ipv4 = 0x%08x\n" + +#define SP_TLOG_FMT_STR_9 \ + "qla_ioctl [%ld]: SIOCSIFMTU if_drv_flags = 0x%08x [0x%08x] max_frame_size = 0x%08x if_mtu = 0x%08x\n" + +#define SP_TLOG_FMT_STR_10 \ + "qla_ioctl [%ld]: SIOCSIFFLAGS if_drv_flags = 0x%08x [0x%08x] ha->if_flags = 0x%08x ifp->if_flags = 0x%08x\n" + +#define SP_TLOG_FMT_STR_11 \ + "qla_ioctl [%ld]: SIOCSIFCAP if_drv_flags = 0x%08x [0x%08x] mask = 0x%08x ifp->if_capenable = 0x%08x\n" + +#define SP_TLOG_FMT_STR_12 \ + "qla_set_multi [%ld]: if_drv_flags = 0x%08x [0x%08x] add_multi = 0x%08x mcnt = 0x%08x\n" + +#define SP_TLOG_FMT_STR_13 \ + "qla_stop [%ld]: \n" + +#define SP_TLOG_FMT_STR_14 \ + "qla_init_locked [%ld]: \n" + #endif /* #ifndef _QL_IOCTL_H_ */ diff --git a/sys/dev/qlxgbe/ql_isr.c b/sys/dev/qlxgbe/ql_isr.c index 79bc3b56b..d147df3e6 100644 --- a/sys/dev/qlxgbe/ql_isr.c +++ b/sys/dev/qlxgbe/ql_isr.c @@ -49,7 +49,7 @@ static void qla_rcv_error(qla_host_t *ha) { ha->stop_rcv = 1; - ha->qla_initiate_recovery = 1; + QL_INITIATE_RECOVERY(ha); } @@ -389,7 +389,7 @@ qla_rcv_cont_sds(qla_host_t *ha, uint32_t sds_idx, uint32_t comp_idx, opcode = Q8_STAT_DESC_OPCODE((sdesc->data[1])); - if (!opcode) { + if (!opcode || QL_ERR_INJECT(ha, INJCT_INV_CONT_OPCODE)) { device_printf(ha->pci_dev, "%s: opcode=0 %p %p\n", __func__, (void *)sdesc->data[0], (void *)sdesc->data[1]); @@ -558,8 +558,9 @@ ql_rcv_isr(qla_host_t *ha, uint32_t sds_idx, uint32_t count) sdesc0 = (q80_stat_desc_t *) &hw->sds[sds_idx].sds_ring_base[c_idx]; - if (Q8_STAT_DESC_OPCODE((sdesc0->data[1])) != - Q8_STAT_DESC_OPCODE_CONT) { + if ((Q8_STAT_DESC_OPCODE((sdesc0->data[1])) != + Q8_STAT_DESC_OPCODE_CONT) || + QL_ERR_INJECT(ha, INJCT_SGL_RCV_INV_DESC_COUNT)) { desc_count = 0; break; } @@ -620,8 +621,9 @@ ql_rcv_isr(qla_host_t *ha, uint32_t sds_idx, uint32_t count) sdesc0 = (q80_stat_desc_t *) &hw->sds[sds_idx].sds_ring_base[c_idx]; - if (Q8_STAT_DESC_OPCODE((sdesc0->data[1])) != - Q8_STAT_DESC_OPCODE_CONT) { + if ((Q8_STAT_DESC_OPCODE((sdesc0->data[1])) != + Q8_STAT_DESC_OPCODE_CONT) || + QL_ERR_INJECT(ha, INJCT_SGL_LRO_INV_DESC_COUNT)) { desc_count = 0; break; } @@ -822,7 +824,13 @@ ql_mbx_isr(void *arg) data = READ_REG32(ha, (Q8_FW_MBOX0 + 12)); prev_link_state = ha->hw.link_up; - ha->hw.link_up = (((data & 0xFF) == 0) ? 0 : 1); + + data = (((data & 0xFF) == 0) ? 0 : 1); + atomic_store_rel_8(&ha->hw.link_up, (uint8_t)data); + + device_printf(ha->pci_dev, + "%s: AEN[0x8001] data = 0x%08x, prev_link_state = 0x%08x\n", + __func__, data, prev_link_state); if (prev_link_state != ha->hw.link_up) { if (ha->hw.link_up) @@ -833,17 +841,18 @@ ql_mbx_isr(void *arg) ha->hw.module_type = ((data >> 8) & 0xFF); - ha->hw.flags.fduplex = (((data & 0xFF0000) == 0) ? 0 : 1); - ha->hw.flags.autoneg = (((data & 0xFF000000) == 0) ? 0 : 1); + ha->hw.fduplex = (((data & 0xFF0000) == 0) ? 0 : 1); + ha->hw.autoneg = (((data & 0xFF000000) == 0) ? 0 : 1); data = READ_REG32(ha, (Q8_FW_MBOX0 + 16)); - ha->hw.flags.loopback_mode = data & 0x03; + ha->hw.loopback_mode = data & 0x03; ha->hw.link_faults = (data >> 3) & 0xFF; break; case 0x8100: + device_printf(ha->pci_dev, "%s: AEN[0x%08x]\n", __func__, data); ha->hw.imd_compl=1; break; @@ -854,6 +863,9 @@ ql_mbx_isr(void *arg) ha->hw.aen_mb2 = READ_REG32(ha, (Q8_FW_MBOX0 + 8)); ha->hw.aen_mb3 = READ_REG32(ha, (Q8_FW_MBOX0 + 12)); ha->hw.aen_mb4 = READ_REG32(ha, (Q8_FW_MBOX0 + 16)); + device_printf(ha->pci_dev, "%s: AEN[0x%08x 0x%08x 0x%08x 0%08x 0x%08x]\n", + __func__, data, ha->hw.aen_mb1, ha->hw.aen_mb2, + ha->hw.aen_mb3, ha->hw.aen_mb4); break; case 0x8110: diff --git a/sys/dev/qlxgbe/ql_misc.c b/sys/dev/qlxgbe/ql_misc.c index 74d126905..185282008 100644 --- a/sys/dev/qlxgbe/ql_misc.c +++ b/sys/dev/qlxgbe/ql_misc.c @@ -77,7 +77,7 @@ ql_rdwr_indreg32(qla_host_t *ha, uint32_t addr, uint32_t *val, uint32_t rd) if (!count || QL_ERR_INJECT(ha, INJCT_RDWR_INDREG_FAILURE)) { device_printf(ha->pci_dev, "%s: [0x%08x, 0x%08x, %d] failed\n", __func__, addr, *val, rd); - ha->qla_initiate_recovery = 1; + QL_INITIATE_RECOVERY(ha); return -1; } @@ -210,7 +210,7 @@ exit_ql_rdwr_offchip_mem: (uint32_t)(addr), val->data_lo, val->data_hi, val->data_ulo, val->data_uhi, rd, step); - ha->qla_initiate_recovery = 1; + QL_INITIATE_RECOVERY(ha); return (-1); } @@ -773,12 +773,13 @@ ql_init_hw(qla_host_t *ha) } qla_mdelay(__func__, 100); } - return (-1); + ret = -1; + goto ql_init_hw_exit; } val = READ_REG32(ha, Q8_CMDPEG_STATE); - if (!cold || (val != 0xFF01)) { + if (!cold || (val != 0xFF01) || ha->qla_initiate_recovery) { ret = qla_init_from_flash(ha); qla_mdelay(__func__, 100); } @@ -794,6 +795,13 @@ qla_init_exit: ha->hw.flags.fdt_valid = 1; } +ql_init_hw_exit: + + if (ret) { + if (ha->hw.sp_log_stop_events & Q8_SP_LOG_STOP_HW_INIT_FAILURE) + ha->hw.sp_log_stop = -1; + } + return (ret); } @@ -1280,6 +1288,7 @@ qla_ld_fw_init(qla_host_t *ha) hdr = (q8_tmplt_hdr_t *)ql83xx_resetseq; + device_printf(ha->pci_dev, "%s: reset sequence\n", __func__); if (qla_tmplt_16bit_checksum(ha, (uint16_t *)ql83xx_resetseq, (uint32_t)hdr->size)) { device_printf(ha->pci_dev, "%s: reset seq checksum failed\n", @@ -1290,7 +1299,7 @@ qla_ld_fw_init(qla_host_t *ha) buf = ql83xx_resetseq + hdr->stop_seq_off; -// device_printf(ha->pci_dev, "%s: stop sequence\n", __func__); + device_printf(ha->pci_dev, "%s: stop sequence\n", __func__); if (qla_tmplt_execute(ha, buf, index , &end_idx, hdr->nentries)) { device_printf(ha->pci_dev, "%s: stop seq failed\n", __func__); return -1; @@ -1300,7 +1309,7 @@ qla_ld_fw_init(qla_host_t *ha) buf = ql83xx_resetseq + hdr->init_seq_off; -// device_printf(ha->pci_dev, "%s: init sequence\n", __func__); + device_printf(ha->pci_dev, "%s: init sequence\n", __func__); if (qla_tmplt_execute(ha, buf, index , &end_idx, hdr->nentries)) { device_printf(ha->pci_dev, "%s: init seq failed\n", __func__); return -1; @@ -1322,7 +1331,7 @@ qla_ld_fw_init(qla_host_t *ha) index = end_idx; buf = ql83xx_resetseq + hdr->start_seq_off; -// device_printf(ha->pci_dev, "%s: start sequence\n", __func__); + device_printf(ha->pci_dev, "%s: start sequence\n", __func__); if (qla_tmplt_execute(ha, buf, index , &end_idx, hdr->nentries)) { device_printf(ha->pci_dev, "%s: init seq failed\n", __func__); return -1; diff --git a/sys/dev/qlxgbe/ql_os.c b/sys/dev/qlxgbe/ql_os.c index 0accee7bf..d4f3c2948 100644 --- a/sys/dev/qlxgbe/ql_os.c +++ b/sys/dev/qlxgbe/ql_os.c @@ -190,7 +190,31 @@ qla_add_sysctls(qla_host_t *ha) SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "enable_minidump", CTLFLAG_RW, &ha->enable_minidump, ha->enable_minidump, - "Minidump retrival is enabled only when this is set"); + "Minidump retrival prior to error recovery " + "is enabled only when this is set"); + + ha->enable_driverstate_dump = 1; + SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "enable_driverstate_dump", CTLFLAG_RW, + &ha->enable_driverstate_dump, ha->enable_driverstate_dump, + "Driver State retrival prior to error recovery " + "is enabled only when this is set"); + + ha->enable_error_recovery = 1; + SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "enable_error_recovery", CTLFLAG_RW, + &ha->enable_error_recovery, ha->enable_error_recovery, + "when set error recovery is enabled on fatal errors " + "otherwise the port is turned offline"); + + ha->ms_delay_after_init = 1000; + SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "ms_delay_after_init", CTLFLAG_RW, + &ha->ms_delay_after_init, ha->ms_delay_after_init, + "millisecond delay after hw_init"); ha->std_replenish = QL_STD_REPLENISH_THRES; SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), @@ -249,49 +273,47 @@ qla_watchdog(void *arg) ha->qla_watchdog_exited = 0; if (!ha->qla_watchdog_pause) { - if (ql_hw_check_health(ha) || ha->qla_initiate_recovery || - (ha->msg_from_peer == QL_PEER_MSG_RESET)) { + if (!ha->offline && + (ql_hw_check_health(ha) || ha->qla_initiate_recovery || + (ha->msg_from_peer == QL_PEER_MSG_RESET))) { - if (!(ha->dbg_level & 0x8000)) { + ifp->if_drv_flags &= ~IFF_DRV_RUNNING; + ql_update_link_state(ha); + + if (ha->enable_error_recovery) { ha->qla_watchdog_paused = 1; ha->qla_watchdog_pause = 1; - ha->qla_initiate_recovery = 0; ha->err_inject = 0; device_printf(ha->pci_dev, "%s: taskqueue_enqueue(err_task) \n", __func__); taskqueue_enqueue(ha->err_tq, &ha->err_task); - return; + } else { + if (ifp != NULL) + ifp->if_drv_flags &= ~IFF_DRV_RUNNING; + ha->offline = 1; } + return; - } else if (ha->qla_interface_up) { - - ha->watchdog_ticks++; + } else { + if (ha->qla_interface_up) { - if (ha->watchdog_ticks > 1000) - ha->watchdog_ticks = 0; + ha->watchdog_ticks++; - if (!ha->watchdog_ticks && QL_RUNNING(ifp)) { - taskqueue_enqueue(ha->stats_tq, &ha->stats_task); - } + if (ha->watchdog_ticks > 1000) + ha->watchdog_ticks = 0; - if (ha->async_event) { - taskqueue_enqueue(ha->async_event_tq, - &ha->async_event_task); - } + if (!ha->watchdog_ticks && QL_RUNNING(ifp)) { + taskqueue_enqueue(ha->stats_tq, + &ha->stats_task); + } -#if 0 - for (i = 0; ((i < ha->hw.num_sds_rings) && - !ha->watchdog_ticks); i++) { - qla_tx_fp_t *fp = &ha->tx_fp[i]; + if (ha->async_event) { + taskqueue_enqueue(ha->async_event_tq, + &ha->async_event_task); + } - if (fp->fp_taskqueue != NULL) - taskqueue_enqueue(fp->fp_taskqueue, - &fp->fp_task); } -#endif - ha->qla_watchdog_paused = 0; - } else { ha->qla_watchdog_paused = 0; } } else { @@ -345,6 +367,7 @@ qla_pci_attach(device_t dev) ha->reg_rid); mtx_init(&ha->hw_lock, "qla83xx_hw_lock", MTX_NETWORK_LOCK, MTX_DEF); + mtx_init(&ha->sp_log_lock, "qla83xx_sp_log_lock", MTX_NETWORK_LOCK, MTX_DEF); ha->flags.lock_init = 1; qla_add_sysctls(ha); @@ -493,6 +516,7 @@ qla_pci_attach(device_t dev) goto qla_pci_attach_err; } ql_alloc_drvr_state_buffer(ha); + ql_alloc_sp_log_buffer(ha); /* create the o.s ethernet interface */ qla_init_ifnet(dev, ha); @@ -538,6 +562,7 @@ qla_pci_attach_err: if (ha->flags.lock_init) { mtx_destroy(&ha->hw_lock); + mtx_destroy(&ha->sp_log_lock); } QL_DPRINT2(ha, (dev, "%s: exit ENXIO\n", __func__)); @@ -576,6 +601,7 @@ qla_pci_detach(device_t dev) if (ha->flags.lock_init) { mtx_destroy(&ha->hw_lock); + mtx_destroy(&ha->sp_log_lock); } QL_DPRINT2(ha, (dev, "%s: exit\n", __func__)); @@ -617,17 +643,17 @@ qla_release(qla_host_t *ha) dev = ha->pci_dev; if (ha->async_event_tq) { - taskqueue_drain(ha->async_event_tq, &ha->async_event_task); + taskqueue_drain_all(ha->async_event_tq); taskqueue_free(ha->async_event_tq); } if (ha->err_tq) { - taskqueue_drain(ha->err_tq, &ha->err_task); + taskqueue_drain_all(ha->err_tq); taskqueue_free(ha->err_tq); } if (ha->stats_tq) { - taskqueue_drain(ha->stats_tq, &ha->stats_task); + taskqueue_drain_all(ha->stats_tq); taskqueue_free(ha->stats_tq); } @@ -647,6 +673,7 @@ qla_release(qla_host_t *ha) ether_ifdetach(ha->ifp); ql_free_drvr_state_buffer(ha); + ql_free_sp_log_buffer(ha); ql_free_dma(ha); qla_free_parent_dma_tag(ha); @@ -677,10 +704,6 @@ qla_release(qla_host_t *ha) if (ha->msix_count) pci_release_msi(dev); -// if (ha->flags.lock_init) { -// mtx_destroy(&ha->hw_lock); -// } - if (ha->pci_reg) (void) bus_release_resource(dev, SYS_RES_MEMORY, ha->reg_rid, ha->pci_reg); @@ -874,6 +897,7 @@ qla_init_ifnet(device_t dev, qla_host_t *ha) ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO4 | + IFCAP_TSO6 | IFCAP_JUMBO_MTU | IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | @@ -902,6 +926,8 @@ qla_init_locked(qla_host_t *ha) { struct ifnet *ifp = ha->ifp; + ql_sp_log(ha, 14, 0, 0, 0, 0, 0, 0); + qla_stop(ha); if (qla_alloc_xmt_bufs(ha) != 0) @@ -921,13 +947,17 @@ qla_init_locked(qla_host_t *ha) if (ql_init_hw_if(ha) == 0) { ifp = ha->ifp; ifp->if_drv_flags |= IFF_DRV_RUNNING; - ha->qla_watchdog_pause = 0; ha->hw_vlan_tx_frames = 0; ha->tx_tso_frames = 0; ha->qla_interface_up = 1; ql_update_link_state(ha); + } else { + if (ha->hw.sp_log_stop_events & Q8_SP_LOG_STOP_IF_START_FAILURE) + ha->hw.sp_log_stop = -1; } + ha->qla_watchdog_pause = 0; + return; } @@ -981,6 +1011,10 @@ qla_set_multi(qla_host_t *ha, uint32_t add_multi) QLA_LOCK_NO_SLEEP) != 0) return (-1); + ql_sp_log(ha, 12, 4, ifp->if_drv_flags, + (ifp->if_drv_flags & IFF_DRV_RUNNING), + add_multi, (uint32_t)mcnt, 0); + if (ifp->if_drv_flags & IFF_DRV_RUNNING) { if (!add_multi) { @@ -1011,6 +1045,8 @@ qla_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) qla_host_t *ha; ha = (qla_host_t *)ifp->if_softc; + if (ha->offline || ha->qla_initiate_recovery) + return (ret); switch (cmd) { case SIOCSIFADDR: @@ -1027,6 +1063,10 @@ qla_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) ifp->if_flags |= IFF_UP; + ql_sp_log(ha, 8, 3, ifp->if_drv_flags, + (ifp->if_drv_flags & IFF_DRV_RUNNING), + ntohl(IA_SIN(ifa)->sin_addr.s_addr), 0, 0); + if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { qla_init_locked(ha); } @@ -1060,6 +1100,10 @@ qla_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) ha->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; + ql_sp_log(ha, 9, 4, ifp->if_drv_flags, + (ifp->if_drv_flags & IFF_DRV_RUNNING), + ha->max_frame_size, ifp->if_mtu, 0); + if (ifp->if_drv_flags & IFF_DRV_RUNNING) { qla_init_locked(ha); } @@ -1085,6 +1129,10 @@ qla_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) if (ret) break; + ql_sp_log(ha, 10, 4, ifp->if_drv_flags, + (ifp->if_drv_flags & IFF_DRV_RUNNING), + ha->if_flags, ifp->if_flags, 0); + if (ifp->if_flags & IFF_UP) { ha->max_frame_size = ifp->if_mtu + @@ -1160,6 +1208,10 @@ qla_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) if (ret) break; + ql_sp_log(ha, 11, 4, ifp->if_drv_flags, + (ifp->if_drv_flags & IFF_DRV_RUNNING), + mask, ifp->if_capenable, 0); + qla_init_locked(ha); QLA_UNLOCK(ha, __func__); @@ -1531,7 +1583,7 @@ qla_destroy_fp_taskqueues(qla_host_t *ha) qla_tx_fp_t *fp = &ha->tx_fp[i]; if (fp->fp_taskqueue != NULL) { - taskqueue_drain(fp->fp_taskqueue, &fp->fp_task); + taskqueue_drain_all(fp->fp_taskqueue); taskqueue_free(fp->fp_taskqueue); fp->fp_taskqueue = NULL; } @@ -1548,7 +1600,7 @@ qla_drain_fp_taskqueues(qla_host_t *ha) qla_tx_fp_t *fp = &ha->tx_fp[i]; if (fp->fp_taskqueue != NULL) { - taskqueue_drain(fp->fp_taskqueue, &fp->fp_task); + taskqueue_drain_all(fp->fp_taskqueue); } } return; @@ -1633,6 +1685,8 @@ qla_stop(qla_host_t *ha) device_t dev; int i = 0; + ql_sp_log(ha, 13, 0, 0, 0, 0, 0, 0); + dev = ha->pci_dev; ifp->if_drv_flags &= ~IFF_DRV_RUNNING; @@ -2041,27 +2095,45 @@ qla_send_msg_to_peer(qla_host_t *ha, uint32_t msg_to_peer) } } +void +qla_set_error_recovery(qla_host_t *ha) +{ + struct ifnet *ifp = ha->ifp; + + if (!cold && ha->enable_error_recovery) { + if (ifp) + ifp->if_drv_flags &= ~IFF_DRV_RUNNING; + ha->qla_initiate_recovery = 1; + } else + ha->offline = 1; + return; +} + static void qla_error_recovery(void *context, int pending) { qla_host_t *ha = context; - uint32_t msecs_100 = 100; + uint32_t msecs_100 = 400; struct ifnet *ifp = ha->ifp; int i = 0; -device_printf(ha->pci_dev, "%s: \n", __func__); + device_printf(ha->pci_dev, "%s: enter\n", __func__); ha->hw.imd_compl = 1; + taskqueue_drain_all(ha->stats_tq); + taskqueue_drain_all(ha->async_event_tq); + if (QLA_LOCK(ha, __func__, -1, 0) != 0) return; -device_printf(ha->pci_dev, "%s: enter\n", __func__); + device_printf(ha->pci_dev, "%s: ts_usecs = %ld start\n", + __func__, qla_get_usec_timestamp()); if (ha->qla_interface_up) { qla_mdelay(__func__, 300); - ifp->if_drv_flags &= ~IFF_DRV_RUNNING; + //ifp->if_drv_flags &= ~IFF_DRV_RUNNING; for (i = 0; i < ha->hw.num_sds_rings; i++) { qla_tx_fp_t *fp; @@ -2078,7 +2150,6 @@ device_printf(ha->pci_dev, "%s: enter\n", __func__); } } - qla_drain_fp_taskqueues(ha); if ((ha->pci_func & 0x1) == 0) { @@ -2096,21 +2167,32 @@ device_printf(ha->pci_dev, "%s: enter\n", __func__); if (ha->enable_minidump) ql_minidump(ha); - (void) ql_init_hw(ha); + if (ha->enable_driverstate_dump) + ql_capture_drvr_state(ha); + if (ql_init_hw(ha)) { + device_printf(ha->pci_dev, + "%s: ts_usecs = %ld exit: ql_init_hw failed\n", + __func__, qla_get_usec_timestamp()); + ha->offline = 1; + goto qla_error_recovery_exit; + } + if (ha->qla_interface_up) { qla_free_xmt_bufs(ha); qla_free_rcv_bufs(ha); } - qla_send_msg_to_peer(ha, QL_PEER_MSG_ACK); + if (!QL_ERR_INJECT(ha, INJCT_PEER_PORT_FAILURE_ERR_RECOVERY)) + qla_send_msg_to_peer(ha, QL_PEER_MSG_ACK); } else { if (ha->msg_from_peer == QL_PEER_MSG_RESET) { ha->msg_from_peer = 0; - qla_send_msg_to_peer(ha, QL_PEER_MSG_ACK); + if (!QL_ERR_INJECT(ha, INJCT_PEER_PORT_FAILURE_ERR_RECOVERY)) + qla_send_msg_to_peer(ha, QL_PEER_MSG_ACK); } else { qla_send_msg_to_peer(ha, QL_PEER_MSG_RESET); } @@ -2119,9 +2201,24 @@ device_printf(ha->pci_dev, "%s: enter\n", __func__); qla_mdelay(__func__, 100); ha->msg_from_peer = 0; - (void) ql_init_hw(ha); + if (ha->enable_driverstate_dump) + ql_capture_drvr_state(ha); - qla_mdelay(__func__, 1000); + if (msecs_100 == 0) { + device_printf(ha->pci_dev, + "%s: ts_usecs = %ld exit: QL_PEER_MSG_ACK not received\n", + __func__, qla_get_usec_timestamp()); + ha->offline = 1; + goto qla_error_recovery_exit; + } + + if (ql_init_hw(ha)) { + device_printf(ha->pci_dev, + "%s: ts_usecs = %ld exit: ql_init_hw failed\n", + __func__, qla_get_usec_timestamp()); + ha->offline = 1; + goto qla_error_recovery_exit; + } if (ha->qla_interface_up) { qla_free_xmt_bufs(ha); @@ -2129,14 +2226,22 @@ device_printf(ha->pci_dev, "%s: enter\n", __func__); } } + qla_mdelay(__func__, ha->ms_delay_after_init); + + *((uint32_t *)&ha->hw.flags) = 0; + ha->qla_initiate_recovery = 0; + if (ha->qla_interface_up) { if (qla_alloc_xmt_bufs(ha) != 0) { + ha->offline = 1; goto qla_error_recovery_exit; } + qla_confirm_9kb_enable(ha); if (qla_alloc_rcv_bufs(ha) != 0) { + ha->offline = 1; goto qla_error_recovery_exit; } @@ -2146,18 +2251,38 @@ device_printf(ha->pci_dev, "%s: enter\n", __func__); ifp = ha->ifp; ifp->if_drv_flags |= IFF_DRV_RUNNING; ha->qla_watchdog_pause = 0; + ql_update_link_state(ha); + } else { + ha->offline = 1; + + if (ha->hw.sp_log_stop_events & + Q8_SP_LOG_STOP_IF_START_FAILURE) + ha->hw.sp_log_stop = -1; } - } else + } else { ha->qla_watchdog_pause = 0; + } qla_error_recovery_exit: -device_printf(ha->pci_dev, "%s: exit\n", __func__); + if (ha->offline ) { + device_printf(ha->pci_dev, "%s: ts_usecs = %ld port offline\n", + __func__, qla_get_usec_timestamp()); + if (ha->hw.sp_log_stop_events & + Q8_SP_LOG_STOP_ERR_RECOVERY_FAILURE) + ha->hw.sp_log_stop = -1; + } + QLA_UNLOCK(ha, __func__); - callout_reset(&ha->tx_callout, QLA_WATCHDOG_CALLOUT_TICKS, - qla_watchdog, ha); + if (!ha->offline) + callout_reset(&ha->tx_callout, QLA_WATCHDOG_CALLOUT_TICKS, + qla_watchdog, ha); + + device_printf(ha->pci_dev, + "%s: ts_usecs = %ld exit\n", + __func__, qla_get_usec_timestamp()); return; } @@ -2187,6 +2312,7 @@ qla_stats(void *context, int pending) ha = context; ql_get_stats(ha); + return; } diff --git a/sys/dev/qlxgbe/ql_os.h b/sys/dev/qlxgbe/ql_os.h index e8000cd52..443f3e250 100644 --- a/sys/dev/qlxgbe/ql_os.h +++ b/sys/dev/qlxgbe/ql_os.h @@ -119,6 +119,14 @@ static __inline int qla_sec_to_hz(int sec) return (tvtohz(&t)); } +static __inline uint64_t qla_get_usec_timestamp(void) +{ + struct timeval tv; + + microuptime(&tv); + + return ((uint64_t)(((uint64_t)tv.tv_sec) * 1000000 + tv.tv_usec)); +} #define qla_host_to_le16(x) htole16(x) #define qla_host_to_le32(x) htole32(x) diff --git a/sys/dev/qlxgbe/ql_ver.h b/sys/dev/qlxgbe/ql_ver.h index 24a9d30d1..9887ac6c4 100644 --- a/sys/dev/qlxgbe/ql_ver.h +++ b/sys/dev/qlxgbe/ql_ver.h @@ -36,6 +36,6 @@ #define QLA_VERSION_MAJOR 3 #define QLA_VERSION_MINOR 10 -#define QLA_VERSION_BUILD 35 +#define QLA_VERSION_BUILD 36 #endif /* #ifndef _QL_VER_H_ */ -- 2.45.0