2 * Copyright (c) 2010-2011 Solarflare Communications, Inc.
5 * This software was developed in part by Philip Paeps under contract for
6 * Solarflare Communications, Inc.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39 #include <sys/param.h>
41 #include <sys/endian.h>
43 #include <sys/malloc.h>
45 #include <sys/mutex.h>
46 #include <sys/rwlock.h>
48 #include <sys/systm.h>
50 #include <machine/bus.h>
51 #include <machine/endian.h>
53 #define EFSYS_HAS_UINT64 1
54 #define EFSYS_USE_UINT64 0
55 #if _BYTE_ORDER == _BIG_ENDIAN
56 #define EFSYS_IS_BIG_ENDIAN 1
57 #define EFSYS_IS_LITTLE_ENDIAN 0
58 #elif _BYTE_ORDER == _LITTLE_ENDIAN
59 #define EFSYS_IS_BIG_ENDIAN 0
60 #define EFSYS_IS_LITTLE_ENDIAN 1
62 #include "efx_types.h"
64 /* Common code requires this */
65 #if __FreeBSD_version < 800068
66 #define memmove(d, s, l) bcopy(s, d, l)
69 /* FreeBSD equivalents of Solaris things */
82 #define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0)
86 #define P2ROUNDUP(x, align) (-(-(x) & -(align)))
90 #define ISP2(x) (((x) & ((x) - 1)) == 0)
93 #define ENOTACTIVE EINVAL
95 /* Memory type to use on FreeBSD */
96 MALLOC_DECLARE(M_SFXGE);
98 /* Machine dependend prefetch wrappers */
99 #if defined(__i386__) || defined(__amd64__)
101 prefetch_read_many(void *addr)
111 prefetch_read_once(void *addr)
119 #elif defined(__sparc64__)
121 prefetch_read_many(void *addr)
131 prefetch_read_once(void *addr)
141 prefetch_read_many(void *addr)
147 prefetch_read_once(void *addr)
153 #if defined(__i386__) || defined(__amd64__)
158 sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map,
159 struct mbuf *m, bus_dma_segment_t *seg)
161 #if defined(__i386__) || defined(__amd64__)
162 seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t));
163 seg->ds_len = m->m_len;
167 bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0);
171 /* Modifiers used for DOS builds */
175 /* Modifiers used for Windows builds */
178 #define __in_ecount(_n)
179 #define __in_ecount_opt(_n)
180 #define __in_bcount(_n)
181 #define __in_bcount_opt(_n)
185 #define __out_ecount(_n)
186 #define __out_ecount_opt(_n)
187 #define __out_bcount(_n)
188 #define __out_bcount_opt(_n)
194 #define __inout_ecount(_n)
195 #define __inout_ecount_opt(_n)
196 #define __inout_bcount(_n)
197 #define __inout_bcount_opt(_n)
198 #define __inout_bcount_full_opt(_n)
200 #define __deref_out_bcount_opt(n)
202 #define __checkReturn
204 #define __drv_when(_p, _c)
206 /* Code inclusion options */
209 #define EFSYS_OPT_NAMES 1
211 #define EFSYS_OPT_FALCON 0
212 #define EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0
213 #define EFSYS_OPT_SIENA 1
215 #define EFSYS_OPT_CHECK_REG 1
217 #define EFSYS_OPT_CHECK_REG 0
220 #define EFSYS_OPT_MCDI 1
222 #define EFSYS_OPT_MAC_FALCON_GMAC 0
223 #define EFSYS_OPT_MAC_FALCON_XMAC 0
224 #define EFSYS_OPT_MAC_STATS 1
226 #define EFSYS_OPT_LOOPBACK 0
228 #define EFSYS_OPT_MON_NULL 0
229 #define EFSYS_OPT_MON_LM87 0
230 #define EFSYS_OPT_MON_MAX6647 0
231 #define EFSYS_OPT_MON_SIENA 0
232 #define EFSYS_OPT_MON_STATS 0
234 #define EFSYS_OPT_PHY_NULL 0
235 #define EFSYS_OPT_PHY_QT2022C2 0
236 #define EFSYS_OPT_PHY_SFX7101 0
237 #define EFSYS_OPT_PHY_TXC43128 0
238 #define EFSYS_OPT_PHY_PM8358 0
239 #define EFSYS_OPT_PHY_SFT9001 0
240 #define EFSYS_OPT_PHY_QT2025C 0
241 #define EFSYS_OPT_PHY_STATS 1
242 #define EFSYS_OPT_PHY_PROPS 0
243 #define EFSYS_OPT_PHY_BIST 1
244 #define EFSYS_OPT_PHY_LED_CONTROL 1
245 #define EFSYS_OPT_PHY_FLAGS 0
247 #define EFSYS_OPT_VPD 1
248 #define EFSYS_OPT_NVRAM 1
249 #define EFSYS_OPT_NVRAM_FALCON_BOOTROM 0
250 #define EFSYS_OPT_NVRAM_SFT9001 0
251 #define EFSYS_OPT_NVRAM_SFX7101 0
252 #define EFSYS_OPT_BOOTCFG 0
254 #define EFSYS_OPT_PCIE_TUNE 0
255 #define EFSYS_OPT_DIAG 0
256 #define EFSYS_OPT_WOL 1
257 #define EFSYS_OPT_RX_SCALE 1
258 #define EFSYS_OPT_QSTATS 1
259 #define EFSYS_OPT_FILTER 0
260 #define EFSYS_OPT_RX_SCATTER 0
261 #define EFSYS_OPT_RX_HDR_SPLIT 0
263 #define EFSYS_OPT_EV_PREFETCH 0
265 #define EFSYS_OPT_DECODE_INTR_FATAL 1
269 typedef struct __efsys_identifier_s efsys_identifier_t;
273 #ifndef KDTRACE_HOOKS
275 #define EFSYS_PROBE(_name)
277 #define EFSYS_PROBE1(_name, _type1, _arg1)
279 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2)
281 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \
284 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
285 _type3, _arg3, _type4, _arg4)
287 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
288 _type3, _arg3, _type4, _arg4, _type5, _arg5)
290 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
291 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
294 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
295 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
296 _type6, _arg6, _type7, _arg7)
298 #else /* KDTRACE_HOOKS */
300 #define EFSYS_PROBE(_name) \
303 #define EFSYS_PROBE1(_name, _type1, _arg1) \
304 DTRACE_PROBE1(_name, _type1, _arg1)
306 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) \
307 DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2)
309 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \
311 DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2, \
314 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
315 _type3, _arg3, _type4, _arg4) \
316 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
317 _type3, _arg3, _type4, _arg4)
320 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
321 _type3, _arg3, _type4, _arg4, _type5, _arg5) \
322 DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
323 _type3, _arg3, _type4, _arg4, _type5, _arg5)
325 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
326 _type3, _arg3, _type4, _arg4, _type5, _arg5) \
327 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
328 _type3, _arg3, _type4, _arg4)
332 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
333 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
335 DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
336 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
339 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
340 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
342 EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
343 _type3, _arg3, _type4, _arg4, _type5, _arg5)
347 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
348 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
349 _type6, _arg6, _type7, _arg7) \
350 DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
351 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
352 _type6, _arg6, _type7, _arg7)
354 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
355 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
356 _type6, _arg6, _type7, _arg7) \
357 EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
358 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
362 #endif /* KDTRACE_HOOKS */
366 typedef uint64_t efsys_dma_addr_t;
368 typedef struct efsys_mem_s {
369 bus_dma_tag_t esm_tag;
370 bus_dmamap_t esm_map;
372 efsys_dma_addr_t esm_addr;
377 #define EFSYS_MEM_ZERO(_esmp, _size) \
379 (void) memset((_esmp)->esm_base, 0, (_size)); \
381 _NOTE(CONSTANTCONDITION) \
384 #define EFSYS_MEM_READD(_esmp, _offset, _edp) \
388 _NOTE(CONSTANTCONDITION) \
389 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
390 ("not power of 2 aligned")); \
392 addr = (void *)((_esmp)->esm_base + (_offset)); \
394 (_edp)->ed_u32[0] = *addr; \
396 EFSYS_PROBE2(mem_readd, unsigned int, (_offset), \
397 uint32_t, (_edp)->ed_u32[0]); \
399 _NOTE(CONSTANTCONDITION) \
402 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \
406 _NOTE(CONSTANTCONDITION) \
407 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
408 ("not power of 2 aligned")); \
410 addr = (void *)((_esmp)->esm_base + (_offset)); \
412 (_eqp)->eq_u32[0] = *addr++; \
413 (_eqp)->eq_u32[1] = *addr; \
415 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \
416 uint32_t, (_eqp)->eq_u32[1], \
417 uint32_t, (_eqp)->eq_u32[0]); \
419 _NOTE(CONSTANTCONDITION) \
422 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \
426 _NOTE(CONSTANTCONDITION) \
427 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
428 ("not power of 2 aligned")); \
430 addr = (void *)((_esmp)->esm_base + (_offset)); \
432 (_eop)->eo_u32[0] = *addr++; \
433 (_eop)->eo_u32[1] = *addr++; \
434 (_eop)->eo_u32[2] = *addr++; \
435 (_eop)->eo_u32[3] = *addr; \
437 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \
438 uint32_t, (_eop)->eo_u32[3], \
439 uint32_t, (_eop)->eo_u32[2], \
440 uint32_t, (_eop)->eo_u32[1], \
441 uint32_t, (_eop)->eo_u32[0]); \
443 _NOTE(CONSTANTCONDITION) \
446 #define EFSYS_MEM_WRITED(_esmp, _offset, _edp) \
450 _NOTE(CONSTANTCONDITION) \
451 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
452 ("not power of 2 aligned")); \
454 EFSYS_PROBE2(mem_writed, unsigned int, (_offset), \
455 uint32_t, (_edp)->ed_u32[0]); \
457 addr = (void *)((_esmp)->esm_base + (_offset)); \
459 *addr = (_edp)->ed_u32[0]; \
461 _NOTE(CONSTANTCONDITION) \
464 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \
468 _NOTE(CONSTANTCONDITION) \
469 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
470 ("not power of 2 aligned")); \
472 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \
473 uint32_t, (_eqp)->eq_u32[1], \
474 uint32_t, (_eqp)->eq_u32[0]); \
476 addr = (void *)((_esmp)->esm_base + (_offset)); \
478 *addr++ = (_eqp)->eq_u32[0]; \
479 *addr = (_eqp)->eq_u32[1]; \
481 _NOTE(CONSTANTCONDITION) \
484 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \
488 _NOTE(CONSTANTCONDITION) \
489 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
490 ("not power of 2 aligned")); \
492 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \
493 uint32_t, (_eop)->eo_u32[3], \
494 uint32_t, (_eop)->eo_u32[2], \
495 uint32_t, (_eop)->eo_u32[1], \
496 uint32_t, (_eop)->eo_u32[0]); \
498 addr = (void *)((_esmp)->esm_base + (_offset)); \
500 *addr++ = (_eop)->eo_u32[0]; \
501 *addr++ = (_eop)->eo_u32[1]; \
502 *addr++ = (_eop)->eo_u32[2]; \
503 *addr = (_eop)->eo_u32[3]; \
505 _NOTE(CONSTANTCONDITION) \
508 #define EFSYS_MEM_ADDR(_esmp) \
513 typedef struct efsys_bar_s {
515 bus_space_tag_t esb_tag;
516 bus_space_handle_t esb_handle;
518 struct resource *esb_res;
521 #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock) \
523 _NOTE(CONSTANTCONDITION) \
524 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
525 ("not power of 2 aligned")); \
527 _NOTE(CONSTANTCONDITION) \
529 mtx_lock(&((_esbp)->esb_lock)); \
531 (_edp)->ed_u32[0] = bus_space_read_4((_esbp)->esb_tag, \
532 (_esbp)->esb_handle, (_offset)); \
534 EFSYS_PROBE2(bar_readd, unsigned int, (_offset), \
535 uint32_t, (_edp)->ed_u32[0]); \
537 _NOTE(CONSTANTCONDITION) \
539 mtx_unlock(&((_esbp)->esb_lock)); \
540 _NOTE(CONSTANTCONDITION) \
543 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \
545 _NOTE(CONSTANTCONDITION) \
546 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
547 ("not power of 2 aligned")); \
549 mtx_lock(&((_esbp)->esb_lock)); \
551 (_eqp)->eq_u32[0] = bus_space_read_4((_esbp)->esb_tag, \
552 (_esbp)->esb_handle, (_offset)); \
553 (_eqp)->eq_u32[1] = bus_space_read_4((_esbp)->esb_tag, \
554 (_esbp)->esb_handle, (_offset+4)); \
556 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \
557 uint32_t, (_eqp)->eq_u32[1], \
558 uint32_t, (_eqp)->eq_u32[0]); \
560 mtx_unlock(&((_esbp)->esb_lock)); \
561 _NOTE(CONSTANTCONDITION) \
564 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \
566 _NOTE(CONSTANTCONDITION) \
567 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
568 ("not power of 2 aligned")); \
570 _NOTE(CONSTANTCONDITION) \
572 mtx_lock(&((_esbp)->esb_lock)); \
574 (_eop)->eo_u32[0] = bus_space_read_4((_esbp)->esb_tag, \
575 (_esbp)->esb_handle, (_offset)); \
576 (_eop)->eo_u32[1] = bus_space_read_4((_esbp)->esb_tag, \
577 (_esbp)->esb_handle, (_offset+4)); \
578 (_eop)->eo_u32[2] = bus_space_read_4((_esbp)->esb_tag, \
579 (_esbp)->esb_handle, (_offset+8)); \
580 (_eop)->eo_u32[3] = bus_space_read_4((_esbp)->esb_tag, \
581 (_esbp)->esb_handle, (_offset+12)); \
583 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \
584 uint32_t, (_eop)->eo_u32[3], \
585 uint32_t, (_eop)->eo_u32[2], \
586 uint32_t, (_eop)->eo_u32[1], \
587 uint32_t, (_eop)->eo_u32[0]); \
589 _NOTE(CONSTANTCONDITION) \
591 mtx_unlock(&((_esbp)->esb_lock)); \
592 _NOTE(CONSTANTCONDITION) \
595 #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock) \
597 _NOTE(CONSTANTCONDITION) \
598 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
599 ("not power of 2 aligned")); \
601 _NOTE(CONSTANTCONDITION) \
603 mtx_lock(&((_esbp)->esb_lock)); \
605 EFSYS_PROBE2(bar_writed, unsigned int, (_offset), \
606 uint32_t, (_edp)->ed_u32[0]); \
608 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
609 (_offset), (_edp)->ed_u32[0]); \
611 _NOTE(CONSTANTCONDITION) \
613 mtx_unlock(&((_esbp)->esb_lock)); \
614 _NOTE(CONSTANTCONDITION) \
617 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \
619 _NOTE(CONSTANTCONDITION) \
620 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
621 ("not power of 2 aligned")); \
623 mtx_lock(&((_esbp)->esb_lock)); \
625 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \
626 uint32_t, (_eqp)->eq_u32[1], \
627 uint32_t, (_eqp)->eq_u32[0]); \
629 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
630 (_offset), (_eqp)->eq_u32[0]); \
631 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
632 (_offset+4), (_eqp)->eq_u32[1]); \
634 mtx_unlock(&((_esbp)->esb_lock)); \
635 _NOTE(CONSTANTCONDITION) \
638 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \
640 _NOTE(CONSTANTCONDITION) \
641 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
642 ("not power of 2 aligned")); \
644 _NOTE(CONSTANTCONDITION) \
646 mtx_lock(&((_esbp)->esb_lock)); \
648 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \
649 uint32_t, (_eop)->eo_u32[3], \
650 uint32_t, (_eop)->eo_u32[2], \
651 uint32_t, (_eop)->eo_u32[1], \
652 uint32_t, (_eop)->eo_u32[0]); \
654 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
655 (_offset), (_eop)->eo_u32[0]); \
656 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
657 (_offset+4), (_eop)->eo_u32[1]); \
658 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
659 (_offset+8), (_eop)->eo_u32[2]); \
660 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
661 (_offset+12), (_eop)->eo_u32[3]); \
663 _NOTE(CONSTANTCONDITION) \
665 mtx_unlock(&((_esbp)->esb_lock)); \
666 _NOTE(CONSTANTCONDITION) \
671 #define EFSYS_SPIN(_us) \
674 _NOTE(CONSTANTCONDITION) \
677 #define EFSYS_SLEEP EFSYS_SPIN
681 /* Strict ordering guaranteed by devacc.devacc_attr_dataorder */
682 #define EFSYS_MEM_READ_BARRIER()
683 #define EFSYS_PIO_WRITE_BARRIER()
687 typedef clock_t efsys_timestamp_t;
689 #define EFSYS_TIMESTAMP(_usp) \
694 *(_usp) = now * hz / 1000000; \
695 _NOTE(CONSTANTCONDITION) \
700 #define EFSYS_KMEM_ALLOC(_esip, _size, _p) \
703 (_p) = malloc((_size), M_SFXGE, M_WAITOK|M_ZERO); \
704 _NOTE(CONSTANTCONDITION) \
707 #define EFSYS_KMEM_FREE(_esip, _size, _p) \
711 free((_p), M_SFXGE); \
712 _NOTE(CONSTANTCONDITION) \
717 typedef struct mtx efsys_lock_t;
719 #define EFSYS_LOCK_MAGIC 0x000010c4
721 #define EFSYS_LOCK(_lockp, _state) \
724 (_state) = EFSYS_LOCK_MAGIC; \
725 _NOTE(CONSTANTCONDITION) \
728 #define EFSYS_UNLOCK(_lockp, _state) \
730 if ((_state) != EFSYS_LOCK_MAGIC) \
731 KASSERT(B_FALSE, ("not locked")); \
732 mtx_unlock(_lockp); \
733 _NOTE(CONSTANTCONDITION) \
738 #define EFSYS_PREEMPT_DISABLE(_state) \
740 (_state) = (_state); \
742 _NOTE(CONSTANTCONDITION) \
745 #define EFSYS_PREEMPT_ENABLE(_state) \
747 (_state) = (_state); \
748 critical_exit(_state); \
749 _NOTE(CONSTANTCONDITION) \
754 typedef uint64_t efsys_stat_t;
756 #define EFSYS_STAT_INCR(_knp, _delta) \
758 *(_knp) += (_delta); \
759 _NOTE(CONSTANTCONDITION) \
762 #define EFSYS_STAT_DECR(_knp, _delta) \
764 *(_knp) -= (_delta); \
765 _NOTE(CONSTANTCONDITION) \
768 #define EFSYS_STAT_SET(_knp, _val) \
771 _NOTE(CONSTANTCONDITION) \
774 #define EFSYS_STAT_SET_QWORD(_knp, _valp) \
776 *(_knp) = le64toh((_valp)->eq_u64[0]); \
777 _NOTE(CONSTANTCONDITION) \
780 #define EFSYS_STAT_SET_DWORD(_knp, _valp) \
782 *(_knp) = le32toh((_valp)->ed_u32[0]); \
783 _NOTE(CONSTANTCONDITION) \
786 #define EFSYS_STAT_INCR_QWORD(_knp, _valp) \
788 *(_knp) += le64toh((_valp)->eq_u64[0]); \
789 _NOTE(CONSTANTCONDITION) \
792 #define EFSYS_STAT_SUBR_QWORD(_knp, _valp) \
794 *(_knp) -= le64toh((_valp)->eq_u64[0]); \
795 _NOTE(CONSTANTCONDITION) \
800 extern void sfxge_err(efsys_identifier_t *, unsigned int,
803 #if EFSYS_OPT_DECODE_INTR_FATAL
804 #define EFSYS_ERR(_esip, _code, _dword0, _dword1) \
806 sfxge_err((_esip), (_code), (_dword0), (_dword1)); \
807 _NOTE(CONSTANTCONDITION) \
813 #define EFSYS_ASSERT(_exp) do { \
818 #define EFSYS_ASSERT3(_x, _op, _y, _t) do { \
819 const _t __x = (_t)(_x); \
820 const _t __y = (_t)(_y); \
821 if (!(__x _op __y)) \
822 panic("assertion failed at %s:%u", __FILE__, __LINE__); \
825 #define EFSYS_ASSERT3U(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uint64_t)
826 #define EFSYS_ASSERT3S(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, int64_t)
827 #define EFSYS_ASSERT3P(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uintptr_t)
833 #endif /* _SYS_EFSYS_H */