2 * Copyright (c) 2010-2011 Solarflare Communications, Inc.
5 * This software was developed in part by Philip Paeps under contract for
6 * Solarflare Communications, Inc.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39 #include <sys/param.h>
41 #include <sys/endian.h>
43 #include <sys/malloc.h>
45 #include <sys/mutex.h>
46 #include <sys/rwlock.h>
48 #include <sys/systm.h>
50 #include <machine/bus.h>
51 #include <machine/endian.h>
53 #define EFSYS_HAS_UINT64 1
54 #if defined(__x86_64__)
55 #define EFSYS_USE_UINT64 1
57 #define EFSYS_USE_UINT64 0
59 #if _BYTE_ORDER == _BIG_ENDIAN
60 #define EFSYS_IS_BIG_ENDIAN 1
61 #define EFSYS_IS_LITTLE_ENDIAN 0
62 #elif _BYTE_ORDER == _LITTLE_ENDIAN
63 #define EFSYS_IS_BIG_ENDIAN 0
64 #define EFSYS_IS_LITTLE_ENDIAN 1
66 #include "efx_types.h"
68 /* Common code requires this */
69 #if __FreeBSD_version < 800068
70 #define memmove(d, s, l) bcopy(s, d, l)
73 /* FreeBSD equivalents of Solaris things */
86 #define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0)
90 #define P2ROUNDUP(x, align) (-(-(x) & -(align)))
94 #define ISP2(x) (((x) & ((x) - 1)) == 0)
97 #if defined(__x86_64__)
98 #if !defined(bus_space_read_stream_8)
99 #define bus_space_read_stream_8(t, h, o) \
100 bus_space_read_8((t), (h), (o))
101 #define bus_space_write_stream_8(t, h, o, v) \
102 bus_space_write_8((t), (h), (o), (v))
106 #define ENOTACTIVE EINVAL
108 /* Memory type to use on FreeBSD */
109 MALLOC_DECLARE(M_SFXGE);
111 /* Machine dependend prefetch wrappers */
112 #if defined(__i386__) || defined(__amd64__)
114 prefetch_read_many(void *addr)
124 prefetch_read_once(void *addr)
132 #elif defined(__sparc64__)
134 prefetch_read_many(void *addr)
144 prefetch_read_once(void *addr)
154 prefetch_read_many(void *addr)
160 prefetch_read_once(void *addr)
166 #if defined(__i386__) || defined(__amd64__)
171 sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map,
172 struct mbuf *m, bus_dma_segment_t *seg)
174 #if defined(__i386__) || defined(__amd64__)
175 seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t));
176 seg->ds_len = m->m_len;
180 bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0);
184 /* Modifiers used for DOS builds */
188 /* Modifiers used for Windows builds */
191 #define __in_ecount(_n)
192 #define __in_ecount_opt(_n)
193 #define __in_bcount(_n)
194 #define __in_bcount_opt(_n)
198 #define __out_ecount(_n)
199 #define __out_ecount_opt(_n)
200 #define __out_bcount(_n)
201 #define __out_bcount_opt(_n)
207 #define __inout_ecount(_n)
208 #define __inout_ecount_opt(_n)
209 #define __inout_bcount(_n)
210 #define __inout_bcount_opt(_n)
211 #define __inout_bcount_full_opt(_n)
213 #define __deref_out_bcount_opt(n)
215 #define __checkReturn
217 #define __drv_when(_p, _c)
219 /* Code inclusion options */
222 #define EFSYS_OPT_NAMES 1
224 #define EFSYS_OPT_FALCON 0
225 #define EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0
226 #define EFSYS_OPT_SIENA 1
228 #define EFSYS_OPT_CHECK_REG 1
230 #define EFSYS_OPT_CHECK_REG 0
233 #define EFSYS_OPT_MCDI 1
235 #define EFSYS_OPT_MAC_FALCON_GMAC 0
236 #define EFSYS_OPT_MAC_FALCON_XMAC 0
237 #define EFSYS_OPT_MAC_STATS 1
239 #define EFSYS_OPT_LOOPBACK 0
241 #define EFSYS_OPT_MON_NULL 0
242 #define EFSYS_OPT_MON_LM87 0
243 #define EFSYS_OPT_MON_MAX6647 0
244 #define EFSYS_OPT_MON_SIENA 0
245 #define EFSYS_OPT_MON_STATS 0
247 #define EFSYS_OPT_PHY_NULL 0
248 #define EFSYS_OPT_PHY_QT2022C2 0
249 #define EFSYS_OPT_PHY_SFX7101 0
250 #define EFSYS_OPT_PHY_TXC43128 0
251 #define EFSYS_OPT_PHY_PM8358 0
252 #define EFSYS_OPT_PHY_SFT9001 0
253 #define EFSYS_OPT_PHY_QT2025C 0
254 #define EFSYS_OPT_PHY_STATS 1
255 #define EFSYS_OPT_PHY_PROPS 0
256 #define EFSYS_OPT_PHY_BIST 1
257 #define EFSYS_OPT_PHY_LED_CONTROL 1
258 #define EFSYS_OPT_PHY_FLAGS 0
260 #define EFSYS_OPT_VPD 1
261 #define EFSYS_OPT_NVRAM 1
262 #define EFSYS_OPT_NVRAM_FALCON_BOOTROM 0
263 #define EFSYS_OPT_NVRAM_SFT9001 0
264 #define EFSYS_OPT_NVRAM_SFX7101 0
265 #define EFSYS_OPT_BOOTCFG 0
267 #define EFSYS_OPT_PCIE_TUNE 0
268 #define EFSYS_OPT_DIAG 0
269 #define EFSYS_OPT_WOL 1
270 #define EFSYS_OPT_RX_SCALE 1
271 #define EFSYS_OPT_QSTATS 1
272 #define EFSYS_OPT_FILTER 0
273 #define EFSYS_OPT_RX_SCATTER 0
274 #define EFSYS_OPT_RX_HDR_SPLIT 0
276 #define EFSYS_OPT_EV_PREFETCH 0
278 #define EFSYS_OPT_DECODE_INTR_FATAL 1
282 typedef struct __efsys_identifier_s efsys_identifier_t;
286 #ifndef KDTRACE_HOOKS
288 #define EFSYS_PROBE(_name)
290 #define EFSYS_PROBE1(_name, _type1, _arg1)
292 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2)
294 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \
297 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
298 _type3, _arg3, _type4, _arg4)
300 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
301 _type3, _arg3, _type4, _arg4, _type5, _arg5)
303 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
304 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
307 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
308 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
309 _type6, _arg6, _type7, _arg7)
311 #else /* KDTRACE_HOOKS */
313 #define EFSYS_PROBE(_name) \
316 #define EFSYS_PROBE1(_name, _type1, _arg1) \
317 DTRACE_PROBE1(_name, _type1, _arg1)
319 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) \
320 DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2)
322 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \
324 DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2, \
327 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
328 _type3, _arg3, _type4, _arg4) \
329 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
330 _type3, _arg3, _type4, _arg4)
333 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
334 _type3, _arg3, _type4, _arg4, _type5, _arg5) \
335 DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
336 _type3, _arg3, _type4, _arg4, _type5, _arg5)
338 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
339 _type3, _arg3, _type4, _arg4, _type5, _arg5) \
340 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
341 _type3, _arg3, _type4, _arg4)
345 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
346 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
348 DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
349 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
352 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
353 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
355 EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
356 _type3, _arg3, _type4, _arg4, _type5, _arg5)
360 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
361 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
362 _type6, _arg6, _type7, _arg7) \
363 DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
364 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
365 _type6, _arg6, _type7, _arg7)
367 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
368 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
369 _type6, _arg6, _type7, _arg7) \
370 EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
371 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
375 #endif /* KDTRACE_HOOKS */
379 typedef uint64_t efsys_dma_addr_t;
381 typedef struct efsys_mem_s {
382 bus_dma_tag_t esm_tag;
383 bus_dmamap_t esm_map;
385 efsys_dma_addr_t esm_addr;
389 #define EFSYS_MEM_ZERO(_esmp, _size) \
391 (void) memset((_esmp)->esm_base, 0, (_size)); \
393 _NOTE(CONSTANTCONDITION) \
396 #define EFSYS_MEM_READD(_esmp, _offset, _edp) \
400 _NOTE(CONSTANTCONDITION) \
401 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
402 ("not power of 2 aligned")); \
404 addr = (void *)((_esmp)->esm_base + (_offset)); \
406 (_edp)->ed_u32[0] = *addr; \
408 EFSYS_PROBE2(mem_readd, unsigned int, (_offset), \
409 uint32_t, (_edp)->ed_u32[0]); \
411 _NOTE(CONSTANTCONDITION) \
414 #if defined(__x86_64__)
415 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \
419 _NOTE(CONSTANTCONDITION) \
420 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
421 ("not power of 2 aligned")); \
423 addr = (void *)((_esmp)->esm_base + (_offset)); \
425 (_eqp)->eq_u64[0] = *addr; \
427 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \
428 uint32_t, (_eqp)->eq_u32[1], \
429 uint32_t, (_eqp)->eq_u32[0]); \
431 _NOTE(CONSTANTCONDITION) \
434 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \
438 _NOTE(CONSTANTCONDITION) \
439 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
440 ("not power of 2 aligned")); \
442 addr = (void *)((_esmp)->esm_base + (_offset)); \
444 (_eqp)->eq_u32[0] = *addr++; \
445 (_eqp)->eq_u32[1] = *addr; \
447 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \
448 uint32_t, (_eqp)->eq_u32[1], \
449 uint32_t, (_eqp)->eq_u32[0]); \
451 _NOTE(CONSTANTCONDITION) \
455 #if defined(__x86_64__)
456 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \
460 _NOTE(CONSTANTCONDITION) \
461 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
462 ("not power of 2 aligned")); \
464 addr = (void *)((_esmp)->esm_base + (_offset)); \
466 (_eop)->eo_u64[0] = *addr++; \
467 (_eop)->eo_u64[1] = *addr; \
469 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \
470 uint32_t, (_eop)->eo_u32[3], \
471 uint32_t, (_eop)->eo_u32[2], \
472 uint32_t, (_eop)->eo_u32[1], \
473 uint32_t, (_eop)->eo_u32[0]); \
475 _NOTE(CONSTANTCONDITION) \
478 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \
482 _NOTE(CONSTANTCONDITION) \
483 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
484 ("not power of 2 aligned")); \
486 addr = (void *)((_esmp)->esm_base + (_offset)); \
488 (_eop)->eo_u32[0] = *addr++; \
489 (_eop)->eo_u32[1] = *addr++; \
490 (_eop)->eo_u32[2] = *addr++; \
491 (_eop)->eo_u32[3] = *addr; \
493 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \
494 uint32_t, (_eop)->eo_u32[3], \
495 uint32_t, (_eop)->eo_u32[2], \
496 uint32_t, (_eop)->eo_u32[1], \
497 uint32_t, (_eop)->eo_u32[0]); \
499 _NOTE(CONSTANTCONDITION) \
503 #define EFSYS_MEM_WRITED(_esmp, _offset, _edp) \
507 _NOTE(CONSTANTCONDITION) \
508 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
509 ("not power of 2 aligned")); \
511 EFSYS_PROBE2(mem_writed, unsigned int, (_offset), \
512 uint32_t, (_edp)->ed_u32[0]); \
514 addr = (void *)((_esmp)->esm_base + (_offset)); \
516 *addr = (_edp)->ed_u32[0]; \
518 _NOTE(CONSTANTCONDITION) \
521 #if defined(__x86_64__)
522 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \
526 _NOTE(CONSTANTCONDITION) \
527 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
528 ("not power of 2 aligned")); \
530 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \
531 uint32_t, (_eqp)->eq_u32[1], \
532 uint32_t, (_eqp)->eq_u32[0]); \
534 addr = (void *)((_esmp)->esm_base + (_offset)); \
536 *addr = (_eqp)->eq_u64[0]; \
538 _NOTE(CONSTANTCONDITION) \
542 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \
546 _NOTE(CONSTANTCONDITION) \
547 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
548 ("not power of 2 aligned")); \
550 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \
551 uint32_t, (_eqp)->eq_u32[1], \
552 uint32_t, (_eqp)->eq_u32[0]); \
554 addr = (void *)((_esmp)->esm_base + (_offset)); \
556 *addr++ = (_eqp)->eq_u32[0]; \
557 *addr = (_eqp)->eq_u32[1]; \
559 _NOTE(CONSTANTCONDITION) \
563 #if defined(__x86_64__)
564 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \
568 _NOTE(CONSTANTCONDITION) \
569 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
570 ("not power of 2 aligned")); \
572 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \
573 uint32_t, (_eop)->eo_u32[3], \
574 uint32_t, (_eop)->eo_u32[2], \
575 uint32_t, (_eop)->eo_u32[1], \
576 uint32_t, (_eop)->eo_u32[0]); \
578 addr = (void *)((_esmp)->esm_base + (_offset)); \
580 *addr++ = (_eop)->eo_u64[0]; \
581 *addr = (_eop)->eo_u64[1]; \
583 _NOTE(CONSTANTCONDITION) \
586 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \
590 _NOTE(CONSTANTCONDITION) \
591 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
592 ("not power of 2 aligned")); \
594 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \
595 uint32_t, (_eop)->eo_u32[3], \
596 uint32_t, (_eop)->eo_u32[2], \
597 uint32_t, (_eop)->eo_u32[1], \
598 uint32_t, (_eop)->eo_u32[0]); \
600 addr = (void *)((_esmp)->esm_base + (_offset)); \
602 *addr++ = (_eop)->eo_u32[0]; \
603 *addr++ = (_eop)->eo_u32[1]; \
604 *addr++ = (_eop)->eo_u32[2]; \
605 *addr = (_eop)->eo_u32[3]; \
607 _NOTE(CONSTANTCONDITION) \
611 #define EFSYS_MEM_ADDR(_esmp) \
616 #define SFXGE_LOCK_NAME_MAX 16
618 typedef struct efsys_bar_s {
620 char esb_lock_name[SFXGE_LOCK_NAME_MAX];
621 bus_space_tag_t esb_tag;
622 bus_space_handle_t esb_handle;
624 struct resource *esb_res;
627 #define SFXGE_BAR_LOCK_INIT(_esbp, _ifname) \
629 snprintf((_esbp)->esb_lock_name, \
630 sizeof((_esbp)->esb_lock_name), \
631 "%s:bar", (_ifname)); \
632 mtx_init(&(_esbp)->esb_lock, (_esbp)->esb_lock_name, \
634 _NOTE(CONSTANTCONDITION) \
636 #define SFXGE_BAR_LOCK_DESTROY(_esbp) \
637 mtx_destroy(&(_esbp)->esb_lock)
638 #define SFXGE_BAR_LOCK(_esbp) \
639 mtx_lock(&(_esbp)->esb_lock)
640 #define SFXGE_BAR_UNLOCK(_esbp) \
641 mtx_unlock(&(_esbp)->esb_lock)
643 #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock) \
645 _NOTE(CONSTANTCONDITION) \
646 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
647 ("not power of 2 aligned")); \
649 _NOTE(CONSTANTCONDITION) \
651 SFXGE_BAR_LOCK(_esbp); \
653 (_edp)->ed_u32[0] = bus_space_read_stream_4( \
654 (_esbp)->esb_tag, (_esbp)->esb_handle, \
657 EFSYS_PROBE2(bar_readd, unsigned int, (_offset), \
658 uint32_t, (_edp)->ed_u32[0]); \
660 _NOTE(CONSTANTCONDITION) \
662 SFXGE_BAR_UNLOCK(_esbp); \
663 _NOTE(CONSTANTCONDITION) \
666 #if defined(__x86_64__)
667 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \
669 _NOTE(CONSTANTCONDITION) \
670 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
671 ("not power of 2 aligned")); \
673 SFXGE_BAR_LOCK(_esbp); \
675 (_eqp)->eq_u64[0] = bus_space_read_stream_8( \
676 (_esbp)->esb_tag, (_esbp)->esb_handle, \
679 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \
680 uint32_t, (_eqp)->eq_u32[1], \
681 uint32_t, (_eqp)->eq_u32[0]); \
683 SFXGE_BAR_UNLOCK(_esbp); \
684 _NOTE(CONSTANTCONDITION) \
687 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \
689 _NOTE(CONSTANTCONDITION) \
690 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
691 ("not power of 2 aligned")); \
693 _NOTE(CONSTANTCONDITION) \
695 SFXGE_BAR_LOCK(_esbp); \
697 (_eop)->eo_u64[0] = bus_space_read_stream_8( \
698 (_esbp)->esb_tag, (_esbp)->esb_handle, \
700 (_eop)->eo_u64[1] = bus_space_read_stream_8( \
701 (_esbp)->esb_tag, (_esbp)->esb_handle, \
704 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \
705 uint32_t, (_eop)->eo_u32[3], \
706 uint32_t, (_eop)->eo_u32[2], \
707 uint32_t, (_eop)->eo_u32[1], \
708 uint32_t, (_eop)->eo_u32[0]); \
710 _NOTE(CONSTANTCONDITION) \
712 SFXGE_BAR_UNLOCK(_esbp); \
713 _NOTE(CONSTANTCONDITION) \
717 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \
719 _NOTE(CONSTANTCONDITION) \
720 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
721 ("not power of 2 aligned")); \
723 SFXGE_BAR_LOCK(_esbp); \
725 (_eqp)->eq_u32[0] = bus_space_read_stream_4( \
726 (_esbp)->esb_tag, (_esbp)->esb_handle, \
728 (_eqp)->eq_u32[1] = bus_space_read_stream_4( \
729 (_esbp)->esb_tag, (_esbp)->esb_handle, \
732 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \
733 uint32_t, (_eqp)->eq_u32[1], \
734 uint32_t, (_eqp)->eq_u32[0]); \
736 SFXGE_BAR_UNLOCK(_esbp); \
737 _NOTE(CONSTANTCONDITION) \
740 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \
742 _NOTE(CONSTANTCONDITION) \
743 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
744 ("not power of 2 aligned")); \
746 _NOTE(CONSTANTCONDITION) \
748 SFXGE_BAR_LOCK(_esbp); \
750 (_eop)->eo_u32[0] = bus_space_read_stream_4( \
751 (_esbp)->esb_tag, (_esbp)->esb_handle, \
753 (_eop)->eo_u32[1] = bus_space_read_stream_4( \
754 (_esbp)->esb_tag, (_esbp)->esb_handle, \
756 (_eop)->eo_u32[2] = bus_space_read_stream_4( \
757 (_esbp)->esb_tag, (_esbp)->esb_handle, \
759 (_eop)->eo_u32[3] = bus_space_read_stream_4( \
760 (_esbp)->esb_tag, (_esbp)->esb_handle, \
763 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \
764 uint32_t, (_eop)->eo_u32[3], \
765 uint32_t, (_eop)->eo_u32[2], \
766 uint32_t, (_eop)->eo_u32[1], \
767 uint32_t, (_eop)->eo_u32[0]); \
769 _NOTE(CONSTANTCONDITION) \
771 SFXGE_BAR_UNLOCK(_esbp); \
772 _NOTE(CONSTANTCONDITION) \
776 #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock) \
778 _NOTE(CONSTANTCONDITION) \
779 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
780 ("not power of 2 aligned")); \
782 _NOTE(CONSTANTCONDITION) \
784 SFXGE_BAR_LOCK(_esbp); \
786 EFSYS_PROBE2(bar_writed, unsigned int, (_offset), \
787 uint32_t, (_edp)->ed_u32[0]); \
789 bus_space_write_stream_4((_esbp)->esb_tag, \
790 (_esbp)->esb_handle, \
791 (_offset), (_edp)->ed_u32[0]); \
793 _NOTE(CONSTANTCONDITION) \
795 SFXGE_BAR_UNLOCK(_esbp); \
796 _NOTE(CONSTANTCONDITION) \
799 #if defined(__x86_64__)
800 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \
802 _NOTE(CONSTANTCONDITION) \
803 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
804 ("not power of 2 aligned")); \
806 SFXGE_BAR_LOCK(_esbp); \
808 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \
809 uint32_t, (_eqp)->eq_u32[1], \
810 uint32_t, (_eqp)->eq_u32[0]); \
812 bus_space_write_stream_8((_esbp)->esb_tag, \
813 (_esbp)->esb_handle, \
814 (_offset), (_eqp)->eq_u64[0]); \
816 SFXGE_BAR_UNLOCK(_esbp); \
817 _NOTE(CONSTANTCONDITION) \
820 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \
822 _NOTE(CONSTANTCONDITION) \
823 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
824 ("not power of 2 aligned")); \
826 SFXGE_BAR_LOCK(_esbp); \
828 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \
829 uint32_t, (_eqp)->eq_u32[1], \
830 uint32_t, (_eqp)->eq_u32[0]); \
832 bus_space_write_stream_4((_esbp)->esb_tag, \
833 (_esbp)->esb_handle, \
834 (_offset), (_eqp)->eq_u32[0]); \
835 bus_space_write_stream_4((_esbp)->esb_tag, \
836 (_esbp)->esb_handle, \
837 (_offset) + 4, (_eqp)->eq_u32[1]); \
839 SFXGE_BAR_UNLOCK(_esbp); \
840 _NOTE(CONSTANTCONDITION) \
844 #if defined(__x86_64__)
845 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \
847 _NOTE(CONSTANTCONDITION) \
848 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
849 ("not power of 2 aligned")); \
851 _NOTE(CONSTANTCONDITION) \
853 SFXGE_BAR_LOCK(_esbp); \
855 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \
856 uint32_t, (_eop)->eo_u32[3], \
857 uint32_t, (_eop)->eo_u32[2], \
858 uint32_t, (_eop)->eo_u32[1], \
859 uint32_t, (_eop)->eo_u32[0]); \
861 bus_space_write_stream_8((_esbp)->esb_tag, \
862 (_esbp)->esb_handle, \
863 (_offset), (_eop)->eo_u64[0]); \
864 bus_space_write_stream_8((_esbp)->esb_tag, \
865 (_esbp)->esb_handle, \
866 (_offset) + 8, (_eop)->eo_u64[1]); \
868 _NOTE(CONSTANTCONDITION) \
870 SFXGE_BAR_UNLOCK(_esbp); \
871 _NOTE(CONSTANTCONDITION) \
875 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \
877 _NOTE(CONSTANTCONDITION) \
878 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
879 ("not power of 2 aligned")); \
881 _NOTE(CONSTANTCONDITION) \
883 SFXGE_BAR_LOCK(_esbp); \
885 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \
886 uint32_t, (_eop)->eo_u32[3], \
887 uint32_t, (_eop)->eo_u32[2], \
888 uint32_t, (_eop)->eo_u32[1], \
889 uint32_t, (_eop)->eo_u32[0]); \
891 bus_space_write_stream_4((_esbp)->esb_tag, \
892 (_esbp)->esb_handle, \
893 (_offset), (_eop)->eo_u32[0]); \
894 bus_space_write_stream_4((_esbp)->esb_tag, \
895 (_esbp)->esb_handle, \
896 (_offset) + 4, (_eop)->eo_u32[1]); \
897 bus_space_write_stream_4((_esbp)->esb_tag, \
898 (_esbp)->esb_handle, \
899 (_offset) + 8, (_eop)->eo_u32[2]); \
900 bus_space_write_stream_4((_esbp)->esb_tag, \
901 (_esbp)->esb_handle, \
902 (_offset) + 12, (_eop)->eo_u32[3]); \
904 _NOTE(CONSTANTCONDITION) \
906 SFXGE_BAR_UNLOCK(_esbp); \
907 _NOTE(CONSTANTCONDITION) \
913 #define EFSYS_SPIN(_us) \
916 _NOTE(CONSTANTCONDITION) \
919 #define EFSYS_SLEEP EFSYS_SPIN
923 #define EFSYS_MEM_READ_BARRIER() rmb()
924 #define EFSYS_PIO_WRITE_BARRIER()
928 typedef clock_t efsys_timestamp_t;
930 #define EFSYS_TIMESTAMP(_usp) \
935 *(_usp) = now * hz / 1000000; \
936 _NOTE(CONSTANTCONDITION) \
941 #define EFSYS_KMEM_ALLOC(_esip, _size, _p) \
945 * The macro is used in non-sleepable contexts, for \
946 * example, holding a mutex. \
948 (_p) = malloc((_size), M_SFXGE, M_NOWAIT|M_ZERO); \
949 _NOTE(CONSTANTCONDITION) \
952 #define EFSYS_KMEM_FREE(_esip, _size, _p) \
956 free((_p), M_SFXGE); \
957 _NOTE(CONSTANTCONDITION) \
962 typedef struct efsys_lock_s {
964 char lock_name[SFXGE_LOCK_NAME_MAX];
967 #define SFXGE_EFSYS_LOCK_INIT(_eslp, _ifname, _label) \
969 efsys_lock_t *__eslp = (_eslp); \
971 snprintf((__eslp)->lock_name, \
972 sizeof((__eslp)->lock_name), \
973 "%s:%s", (_ifname), (_label)); \
974 mtx_init(&(__eslp)->lock, (__eslp)->lock_name, \
977 #define SFXGE_EFSYS_LOCK_DESTROY(_eslp) \
978 mtx_destroy(&(_eslp)->lock)
979 #define SFXGE_EFSYS_LOCK(_eslp) \
980 mtx_lock(&(_eslp)->lock)
981 #define SFXGE_EFSYS_UNLOCK(_eslp) \
982 mtx_unlock(&(_eslp)->lock)
983 #define SFXGE_EFSYS_LOCK_ASSERT_OWNED(_eslp) \
984 mtx_assert(&(_eslp)->lock, MA_OWNED)
986 #define EFSYS_LOCK_MAGIC 0x000010c4
988 #define EFSYS_LOCK(_lockp, _state) \
990 SFXGE_EFSYS_LOCK(_lockp); \
991 (_state) = EFSYS_LOCK_MAGIC; \
992 _NOTE(CONSTANTCONDITION) \
995 #define EFSYS_UNLOCK(_lockp, _state) \
997 if ((_state) != EFSYS_LOCK_MAGIC) \
998 KASSERT(B_FALSE, ("not locked")); \
999 SFXGE_EFSYS_UNLOCK(_lockp); \
1000 _NOTE(CONSTANTCONDITION) \
1005 #define EFSYS_PREEMPT_DISABLE(_state) \
1007 (_state) = (_state); \
1009 _NOTE(CONSTANTCONDITION) \
1012 #define EFSYS_PREEMPT_ENABLE(_state) \
1014 (_state) = (_state); \
1015 critical_exit(_state); \
1016 _NOTE(CONSTANTCONDITION) \
1021 typedef uint64_t efsys_stat_t;
1023 #define EFSYS_STAT_INCR(_knp, _delta) \
1025 *(_knp) += (_delta); \
1026 _NOTE(CONSTANTCONDITION) \
1029 #define EFSYS_STAT_DECR(_knp, _delta) \
1031 *(_knp) -= (_delta); \
1032 _NOTE(CONSTANTCONDITION) \
1035 #define EFSYS_STAT_SET(_knp, _val) \
1038 _NOTE(CONSTANTCONDITION) \
1041 #define EFSYS_STAT_SET_QWORD(_knp, _valp) \
1043 *(_knp) = le64toh((_valp)->eq_u64[0]); \
1044 _NOTE(CONSTANTCONDITION) \
1047 #define EFSYS_STAT_SET_DWORD(_knp, _valp) \
1049 *(_knp) = le32toh((_valp)->ed_u32[0]); \
1050 _NOTE(CONSTANTCONDITION) \
1053 #define EFSYS_STAT_INCR_QWORD(_knp, _valp) \
1055 *(_knp) += le64toh((_valp)->eq_u64[0]); \
1056 _NOTE(CONSTANTCONDITION) \
1059 #define EFSYS_STAT_SUBR_QWORD(_knp, _valp) \
1061 *(_knp) -= le64toh((_valp)->eq_u64[0]); \
1062 _NOTE(CONSTANTCONDITION) \
1067 extern void sfxge_err(efsys_identifier_t *, unsigned int,
1068 uint32_t, uint32_t);
1070 #if EFSYS_OPT_DECODE_INTR_FATAL
1071 #define EFSYS_ERR(_esip, _code, _dword0, _dword1) \
1073 sfxge_err((_esip), (_code), (_dword0), (_dword1)); \
1074 _NOTE(CONSTANTCONDITION) \
1080 #define EFSYS_ASSERT(_exp) do { \
1085 #define EFSYS_ASSERT3(_x, _op, _y, _t) do { \
1086 const _t __x = (_t)(_x); \
1087 const _t __y = (_t)(_y); \
1088 if (!(__x _op __y)) \
1089 panic("assertion failed at %s:%u", __FILE__, __LINE__); \
1092 #define EFSYS_ASSERT3U(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uint64_t)
1093 #define EFSYS_ASSERT3S(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, int64_t)
1094 #define EFSYS_ASSERT3P(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uintptr_t)
1100 #endif /* _SYS_EFSYS_H */