2 * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 #include <linux/kernel.h>
29 #include <linux/module.h>
30 #include <linux/random.h>
31 #include <linux/vmalloc.h>
32 #include <dev/mlx5/driver.h>
33 #include <dev/mlx5/mlx5_ifc.h>
34 #include "mlx5_core.h"
36 #define MLX5_HEALTH_POLL_INTERVAL (2 * HZ)
39 static DEFINE_SPINLOCK(health_lock);
40 static LIST_HEAD(health_list);
41 static struct work_struct health_work;
43 static void health_care(struct work_struct *work)
45 struct mlx5_core_health *health, *n;
46 struct mlx5_core_dev *dev;
47 struct mlx5_priv *priv;
50 spin_lock_irq(&health_lock);
51 list_splice_init(&health_list, &tlist);
53 spin_unlock_irq(&health_lock);
55 list_for_each_entry_safe(health, n, &tlist, list) {
56 priv = container_of(health, struct mlx5_priv, health);
57 dev = container_of(priv, struct mlx5_core_dev, priv);
58 mlx5_core_warn(dev, "handling bad device here\n");
60 spin_lock_irq(&health_lock);
61 list_del_init(&health->list);
62 spin_unlock_irq(&health_lock);
66 static const char *hsynd_str(u8 synd)
69 case MLX5_HEALTH_SYNDR_FW_ERR:
70 return "firmware internal error";
71 case MLX5_HEALTH_SYNDR_IRISC_ERR:
72 return "irisc not responding";
73 case MLX5_HEALTH_SYNDR_CRC_ERR:
74 return "firmware CRC error";
75 case MLX5_HEALTH_SYNDR_FETCH_PCI_ERR:
76 return "ICM fetch PCI error";
77 case MLX5_HEALTH_SYNDR_HW_FTL_ERR:
78 return "HW fatal error\n";
79 case MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR:
80 return "async EQ buffer overrun";
81 case MLX5_HEALTH_SYNDR_EQ_ERR:
83 case MLX5_HEALTH_SYNDR_FFSER_ERR:
86 return "unrecognized error";
90 static u16 read_be16(__be16 __iomem *p)
92 return swab16(readl((__force u16 __iomem *) p));
95 static u32 read_be32(__be32 __iomem *p)
97 return swab32(readl((__force u32 __iomem *) p));
100 static void print_health_info(struct mlx5_core_dev *dev)
102 struct mlx5_core_health *health = &dev->priv.health;
103 struct mlx5_health_buffer __iomem *h = health->health;
106 for (i = 0; i < ARRAY_SIZE(h->assert_var); i++)
107 printf("mlx5_core: INFO: ""assert_var[%d] 0x%08x\n", i, read_be32(h->assert_var + i));
109 printf("mlx5_core: INFO: ""assert_exit_ptr 0x%08x\n", read_be32(&h->assert_exit_ptr));
110 printf("mlx5_core: INFO: ""assert_callra 0x%08x\n", read_be32(&h->assert_callra));
111 printf("mlx5_core: INFO: ""fw_ver 0x%08x\n", read_be32(&h->fw_ver));
112 printf("mlx5_core: INFO: ""hw_id 0x%08x\n", read_be32(&h->hw_id));
113 printf("mlx5_core: INFO: ""irisc_index %d\n", readb(&h->irisc_index));
114 printf("mlx5_core: INFO: ""synd 0x%x: %s\n", readb(&h->synd), hsynd_str(readb(&h->synd)));
115 printf("mlx5_core: INFO: ""ext_sync 0x%04x\n", read_be16(&h->ext_sync));
118 static void poll_health(unsigned long data)
120 struct mlx5_core_dev *dev = (struct mlx5_core_dev *)data;
121 struct mlx5_core_health *health = &dev->priv.health;
125 if (dev->state != MLX5_DEVICE_STATE_UP)
128 count = ioread32be(health->health_counter);
129 if (count == health->prev)
130 ++health->miss_counter;
132 health->miss_counter = 0;
134 health->prev = count;
135 if (health->miss_counter == MAX_MISSES) {
136 mlx5_core_err(dev, "device's health compromised\n");
137 print_health_info(dev);
138 spin_lock_irq(&health_lock);
139 list_add_tail(&health->list, &health_list);
140 spin_unlock_irq(&health_lock);
142 if (!queue_work(mlx5_core_wq, &health_work))
143 mlx5_core_warn(dev, "failed to queue health work\n");
145 get_random_bytes(&next, sizeof(next));
147 next += jiffies + MLX5_HEALTH_POLL_INTERVAL;
148 mod_timer(&health->timer, next);
152 void mlx5_start_health_poll(struct mlx5_core_dev *dev)
154 struct mlx5_core_health *health = &dev->priv.health;
156 INIT_LIST_HEAD(&health->list);
157 init_timer(&health->timer);
158 health->health = &dev->iseg->health;
159 health->health_counter = &dev->iseg->health_counter;
161 setup_timer(&health->timer, poll_health, (unsigned long)dev);
162 mod_timer(&health->timer,
163 round_jiffies(jiffies + MLX5_HEALTH_POLL_INTERVAL));
166 void mlx5_stop_health_poll(struct mlx5_core_dev *dev)
168 struct mlx5_core_health *health = &dev->priv.health;
170 del_timer_sync(&health->timer);
172 spin_lock_irq(&health_lock);
173 if (!list_empty(&health->list))
174 list_del_init(&health->list);
175 spin_unlock_irq(&health_lock);
178 void mlx5_health_cleanup(void)
182 void __init mlx5_health_init(void)
185 INIT_WORK(&health_work, health_care);