4 * Copyright (c) 2015-2017 Amazon.com, Inc. or its affiliates.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of copyright holder nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 #include <sys/cdefs.h>
38 __FBSDID("$FreeBSD$");
40 #include <sys/param.h>
41 #include <sys/systm.h>
44 #include <sys/condvar.h>
45 #include <sys/endian.h>
46 #include <sys/kernel.h>
47 #include <sys/kthread.h>
48 #include <sys/malloc.h>
50 #include <sys/module.h>
54 #include <sys/socket.h>
55 #include <sys/sockio.h>
56 #include <sys/sysctl.h>
57 #include <sys/taskqueue.h>
58 #include <sys/eventhandler.h>
59 #include <sys/types.h>
60 #include <sys/timetc.h>
61 #include <sys/cdefs.h>
63 #include <machine/atomic.h>
64 #include <machine/bus.h>
65 #include <machine/in_cksum.h>
66 #include <machine/pcpu.h>
67 #include <machine/resource.h>
70 #include <net/ethernet.h>
72 #include <net/if_var.h>
73 #include <net/if_arp.h>
74 #include <net/if_dl.h>
75 #include <net/if_media.h>
77 #include <net/if_types.h>
78 #include <net/if_vlan_var.h>
80 #include <netinet/in_systm.h>
81 #include <netinet/in.h>
82 #include <netinet/if_ether.h>
83 #include <netinet/ip.h>
84 #include <netinet/ip6.h>
85 #include <netinet/tcp.h>
86 #include <netinet/tcp_lro.h>
87 #include <netinet/udp.h>
89 #include <dev/led/led.h>
90 #include <dev/pci/pcivar.h>
91 #include <dev/pci/pcireg.h>
93 extern struct ena_bus_space ebs;
96 #define ENA_ALERT (1 << 0) /* Alerts are providing more error info. */
97 #define ENA_WARNING (1 << 1) /* Driver output is more error sensitive. */
98 #define ENA_INFO (1 << 2) /* Provides additional driver info. */
99 #define ENA_DBG (1 << 3) /* Driver output for debugging. */
100 /* Detailed info that will be printed with ENA_INFO or ENA_DEBUG flag. */
101 #define ENA_TXPTH (1 << 4) /* Allows TX path tracing. */
102 #define ENA_RXPTH (1 << 5) /* Allows RX path tracing. */
103 #define ENA_RSC (1 << 6) /* Goes with TXPTH or RXPTH, free/alloc res. */
104 #define ENA_IOQ (1 << 7) /* Detailed info about IO queues. */
105 #define ENA_ADMQ (1 << 8) /* Detailed info about admin queue. */
107 #ifndef ENA_DEBUG_LEVEL
108 #define ENA_DEBUG_LEVEL (ENA_ALERT | ENA_WARNING)
112 #define ena_trace_raw(level, fmt, args...) \
114 if (((level) & ENA_DEBUG_LEVEL) != (level)) \
116 printf(fmt, ##args); \
119 #define ena_trace(level, fmt, args...) \
120 ena_trace_raw(level, "%s() [TID:%d]: " \
121 fmt " \n", __func__, curthread->td_tid, ##args)
123 #else /* ENA_TRACE */
124 #define ena_trace_raw(...)
125 #define ena_trace(...)
126 #endif /* ENA_TRACE */
128 #define ena_trc_dbg(format, arg...) ena_trace(ENA_DBG, format, ##arg)
129 #define ena_trc_info(format, arg...) ena_trace(ENA_INFO, format, ##arg)
130 #define ena_trc_warn(format, arg...) ena_trace(ENA_WARNING, format, ##arg)
131 #define ena_trc_err(format, arg...) ena_trace(ENA_ALERT, format, ##arg)
133 #define unlikely(x) __predict_false(x)
134 #define likely(x) __predict_true(x)
137 #define ____cacheline_aligned __aligned(CACHE_LINE_SIZE)
139 #define MAX_ERRNO 4095
140 #define IS_ERR_VALUE(x) unlikely((x) <= (unsigned long)MAX_ERRNO)
142 #define WARN_ON(condition) \
144 int __ret_warn_on = !!(condition); \
145 if (unlikely(__ret_warn_on)) \
146 printf("%s %s", __FUNCTION__, __FILE__); \
147 unlikely(__ret_warn_on); \
150 #define ENA_ASSERT(cond, format, arg...) \
152 if (unlikely(!(cond))) { \
154 "Assert failed on %s:%s:%d:" format, \
155 __FILE__, __func__, __LINE__, ##arg); \
160 #define ENA_WARN(cond, format, arg...) \
162 if (unlikely((cond))) { \
163 ena_trc_warn(format, ##arg); \
167 static inline long IS_ERR(const void *ptr)
169 return IS_ERR_VALUE((unsigned long)ptr);
172 static inline void *ERR_PTR(long error)
174 return (void *)error;
177 static inline long PTR_ERR(const void *ptr)
182 #define GENMASK(h, l) (((1U << ((h) - (l) + 1)) - 1) << (l))
183 #define GENMASK_ULL(h, l) (((~0ULL) << (l)) & (~0ULL >> (64 - 1 - (h))))
184 #define BIT(x) (1 << (x))
186 #define ENA_ABORT() BUG()
187 #define BUG() panic("ENA BUG")
193 #define ENA_COM_FAULT EFAULT
194 #define ENA_COM_INVAL EINVAL
195 #define ENA_COM_NO_MEM ENOMEM
196 #define ENA_COM_NO_SPACE ENOSPC
197 #define ENA_COM_TRY_AGAIN -1
198 #define ENA_COM_NO_DEVICE ENODEV
199 #define ENA_COM_PERMISSION EPERM
200 #define ENA_COM_TIMER_EXPIRED ETIMEDOUT
202 #define ENA_MSLEEP(x) pause_sbt("ena", SBT_1MS * (x), SBT_1MS, 0)
203 #define ENA_UDELAY(x) DELAY(x)
204 #define ENA_GET_SYSTEM_TIMEOUT(timeout_us) \
205 ((long)cputick2usec(cpu_ticks()) + (timeout_us))
206 #define ENA_TIME_EXPIRE(timeout) ((timeout) < (long)cputick2usec(cpu_ticks()))
207 #define ENA_MIGHT_SLEEP()
209 #define min_t(type, _x, _y) ((type)(_x) < (type)(_y) ? (type)(_x) : (type)(_y))
210 #define max_t(type, _x, _y) ((type)(_x) > (type)(_y) ? (type)(_x) : (type)(_y))
212 #define ENA_MIN32(x,y) MIN(x, y)
213 #define ENA_MIN16(x,y) MIN(x, y)
214 #define ENA_MIN8(x,y) MIN(x, y)
216 #define ENA_MAX32(x,y) MAX(x, y)
217 #define ENA_MAX16(x,y) MAX(x, y)
218 #define ENA_MAX8(x,y) MAX(x, y)
220 /* Spinlock related methods */
221 #define ena_spinlock_t struct mtx
222 #define ENA_SPINLOCK_INIT(spinlock) \
223 mtx_init(&(spinlock), "ena_spin", NULL, MTX_SPIN)
224 #define ENA_SPINLOCK_DESTROY(spinlock) \
226 if (mtx_initialized(&(spinlock))) \
227 mtx_destroy(&(spinlock)); \
229 #define ENA_SPINLOCK_LOCK(spinlock, flags) \
232 mtx_lock_spin(&(spinlock)); \
234 #define ENA_SPINLOCK_UNLOCK(spinlock, flags) \
237 mtx_unlock_spin(&(spinlock)); \
241 /* Wait queue related methods */
242 #define ena_wait_event_t struct { struct cv wq; struct mtx mtx; }
243 #define ENA_WAIT_EVENT_INIT(waitqueue) \
245 cv_init(&((waitqueue).wq), "cv"); \
246 mtx_init(&((waitqueue).mtx), "wq", NULL, MTX_DEF); \
248 #define ENA_WAIT_EVENT_DESTROY(waitqueue) \
250 cv_destroy(&((waitqueue).wq)); \
251 mtx_destroy(&((waitqueue).mtx)); \
253 #define ENA_WAIT_EVENT_CLEAR(waitqueue) \
254 cv_init(&((waitqueue).wq), (waitqueue).wq.cv_description)
255 #define ENA_WAIT_EVENT_WAIT(waitqueue, timeout_us) \
257 mtx_lock(&((waitqueue).mtx)); \
258 cv_timedwait(&((waitqueue).wq), &((waitqueue).mtx), \
259 timeout_us * hz / 1000 / 1000 ); \
260 mtx_unlock(&((waitqueue).mtx)); \
262 #define ENA_WAIT_EVENT_SIGNAL(waitqueue) cv_broadcast(&((waitqueue).wq))
264 #define dma_addr_t bus_addr_t
275 bus_dma_segment_t seg;
280 bus_space_handle_t reg_bar_h;
281 bus_space_tag_t reg_bar_t;
282 bus_space_handle_t mem_bar_h;
283 bus_space_tag_t mem_bar_t;
286 typedef uint32_t ena_atomic32_t;
288 void ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg,
290 int ena_dma_alloc(device_t dmadev, bus_size_t size, ena_mem_handle_t *dma,
293 #define ENA_MEM_ALLOC(dmadev, size) malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO)
294 #define ENA_MEM_ALLOC_NODE(dmadev, size, virt, node, dev_node) (virt = NULL)
295 #define ENA_MEM_FREE(dmadev, ptr) free(ptr, M_DEVBUF)
296 #define ENA_MEM_ALLOC_COHERENT_NODE(dmadev, size, virt, phys, handle, node, \
303 #define ENA_MEM_ALLOC_COHERENT(dmadev, size, virt, phys, dma) \
305 ena_dma_alloc((dmadev), (size), &(dma), 0); \
306 (virt) = (void *)(dma).vaddr; \
307 (phys) = (dma).paddr; \
310 #define ENA_MEM_FREE_COHERENT(dmadev, size, virt, phys, dma) \
313 bus_dmamap_unload((dma).tag, (dma).map); \
314 bus_dmamem_free((dma).tag, (virt), (dma).map); \
315 bus_dma_tag_destroy((dma).tag); \
320 /* Register R/W methods */
321 #define ENA_REG_WRITE32(bus, value, offset) \
323 ((struct ena_bus*)bus)->reg_bar_t, \
324 ((struct ena_bus*)bus)->reg_bar_h, \
325 (bus_size_t)(offset), (value))
327 #define ENA_REG_READ32(bus, offset) \
329 ((struct ena_bus*)bus)->reg_bar_t, \
330 ((struct ena_bus*)bus)->reg_bar_h, \
331 (bus_size_t)(offset))
333 #define time_after(a,b) ((long)((unsigned long)(b) - (unsigned long)(a)) < 0)
335 #define VLAN_HLEN sizeof(struct ether_vlan_header)
336 #define CSUM_OFFLOAD (CSUM_IP|CSUM_TCP|CSUM_UDP)
338 #if defined(__i386__) || defined(__amd64__)
340 void prefetch(void *x)
342 __asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x));
348 /* DMA buffers access */
349 #define dma_unmap_addr(p, name) ((p)->dma->name)
350 #define dma_unmap_addr_set(p, name, v) (((p)->dma->name) = (v))
351 #define dma_unmap_len(p, name) ((p)->name)
352 #define dma_unmap_len_set(p, name, v) (((p)->name) = (v))
354 #define memcpy_toio memcpy
356 #define ATOMIC32_INC(I32_PTR) atomic_add_int(I32_PTR, 1)
357 #define ATOMIC32_DEC(I32_PTR) atomic_add_int(I32_PTR, -1)
358 #define ATOMIC32_READ(I32_PTR) atomic_load_acq_int(I32_PTR)
359 #define ATOMIC32_SET(I32_PTR, VAL) atomic_store_rel_int(I32_PTR, VAL)
361 #define barrier() __asm__ __volatile__("": : :"memory")
362 #define ACCESS_ONCE(x) (*(volatile __typeof(x) *)&(x))
363 #define READ_ONCE(x) ({ \
366 __var = ACCESS_ONCE(x); \
371 #include "ena_common_defs.h"
372 #include "ena_admin_defs.h"
373 #include "ena_eth_io_defs.h"
374 #include "ena_regs_defs.h"
376 #endif /* ENA_PLAT_H_ */