2 * Copyright (c) 2010-2015 Solarflare Communications Inc.
5 * This software was developed in part by Philip Paeps under contract for
6 * Solarflare Communications, Inc.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are met:
11 * 1. Redistributions of source code must retain the above copyright notice,
12 * this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright notice,
14 * this list of conditions and the following disclaimer in the documentation
15 * and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
19 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
21 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
22 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
23 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
24 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
25 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
27 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 * The views and conclusions contained in the software and documentation are
30 * those of the authors and should not be interpreted as representing official
31 * policies, either expressed or implied, of the FreeBSD Project.
43 #include <sys/param.h>
45 #include <sys/endian.h>
47 #include <sys/malloc.h>
49 #include <sys/mutex.h>
50 #include <sys/rwlock.h>
52 #include <sys/systm.h>
54 #include <machine/bus.h>
55 #include <machine/endian.h>
57 #define EFSYS_HAS_UINT64 1
58 #if defined(__x86_64__)
59 #define EFSYS_USE_UINT64 1
61 #define EFSYS_USE_UINT64 0
63 #define EFSYS_HAS_SSE2_M128 0
64 #if _BYTE_ORDER == _BIG_ENDIAN
65 #define EFSYS_IS_BIG_ENDIAN 1
66 #define EFSYS_IS_LITTLE_ENDIAN 0
67 #elif _BYTE_ORDER == _LITTLE_ENDIAN
68 #define EFSYS_IS_BIG_ENDIAN 0
69 #define EFSYS_IS_LITTLE_ENDIAN 1
71 #include "efx_types.h"
73 /* Common code requires this */
74 #if __FreeBSD_version < 800068
75 #define memmove(d, s, l) bcopy(s, d, l)
78 /* FreeBSD equivalents of Solaris things */
91 #define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0)
95 #define P2ROUNDUP(x, align) (-(-(x) & -(align)))
99 #define P2ALIGN(_x, _a) ((_x) & -(_a))
103 #define ISP2(x) (((x) & ((x) - 1)) == 0)
106 #if defined(__x86_64__) && __FreeBSD_version >= 1000000
108 #define SFXGE_USE_BUS_SPACE_8 1
110 #if !defined(bus_space_read_stream_8)
112 #define bus_space_read_stream_8(t, h, o) \
113 bus_space_read_8((t), (h), (o))
115 #define bus_space_write_stream_8(t, h, o, v) \
116 bus_space_write_8((t), (h), (o), (v))
122 #define ENOTACTIVE EINVAL
124 /* Memory type to use on FreeBSD */
125 MALLOC_DECLARE(M_SFXGE);
127 /* Machine dependend prefetch wrappers */
128 #if defined(__i386__) || defined(__amd64__)
130 prefetch_read_many(void *addr)
140 prefetch_read_once(void *addr)
148 #elif defined(__sparc64__)
150 prefetch_read_many(void *addr)
160 prefetch_read_once(void *addr)
170 prefetch_read_many(void *addr)
176 prefetch_read_once(void *addr)
182 #if defined(__i386__) || defined(__amd64__)
187 sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map,
188 struct mbuf *m, bus_dma_segment_t *seg)
190 #if defined(__i386__) || defined(__amd64__)
191 seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t));
192 seg->ds_len = m->m_len;
196 bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0);
200 /* Modifiers used for Windows builds */
203 #define __in_ecount(_n)
204 #define __in_ecount_opt(_n)
205 #define __in_bcount(_n)
206 #define __in_bcount_opt(_n)
210 #define __out_ecount(_n)
211 #define __out_ecount_opt(_n)
212 #define __out_bcount(_n)
213 #define __out_bcount_opt(_n)
219 #define __inout_ecount(_n)
220 #define __inout_ecount_opt(_n)
221 #define __inout_bcount(_n)
222 #define __inout_bcount_opt(_n)
223 #define __inout_bcount_full_opt(_n)
225 #define __deref_out_bcount_opt(n)
227 #define __checkReturn
228 #define __success(_x)
230 #define __drv_when(_p, _c)
232 /* Code inclusion options */
235 #define EFSYS_OPT_NAMES 1
237 #define EFSYS_OPT_FALCON 0
238 #define EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0
239 #define EFSYS_OPT_SIENA 1
240 #define EFSYS_OPT_HUNTINGTON 1
241 #define EFSYS_OPT_MEDFORD 0
243 #define EFSYS_OPT_CHECK_REG 1
245 #define EFSYS_OPT_CHECK_REG 0
248 #define EFSYS_OPT_MCDI 1
249 #define EFSYS_OPT_MCDI_LOGGING 0
250 #define EFSYS_OPT_MCDI_PROXY_AUTH 0
252 #define EFSYS_OPT_MAC_FALCON_GMAC 0
253 #define EFSYS_OPT_MAC_FALCON_XMAC 0
254 #define EFSYS_OPT_MAC_STATS 1
256 #define EFSYS_OPT_LOOPBACK 0
258 #define EFSYS_OPT_MON_NULL 0
259 #define EFSYS_OPT_MON_LM87 0
260 #define EFSYS_OPT_MON_MAX6647 0
261 #define EFSYS_OPT_MON_MCDI 0
262 #define EFSYS_OPT_MON_STATS 0
264 #define EFSYS_OPT_PHY_NULL 0
265 #define EFSYS_OPT_PHY_QT2022C2 0
266 #define EFSYS_OPT_PHY_SFX7101 0
267 #define EFSYS_OPT_PHY_TXC43128 0
268 #define EFSYS_OPT_PHY_SFT9001 0
269 #define EFSYS_OPT_PHY_QT2025C 0
270 #define EFSYS_OPT_PHY_STATS 1
271 #define EFSYS_OPT_PHY_PROPS 0
272 #define EFSYS_OPT_PHY_BIST 0
273 #define EFSYS_OPT_BIST 1
274 #define EFSYS_OPT_PHY_LED_CONTROL 1
275 #define EFSYS_OPT_PHY_FLAGS 0
277 #define EFSYS_OPT_VPD 1
278 #define EFSYS_OPT_NVRAM 1
279 #define EFSYS_OPT_NVRAM_FALCON_BOOTROM 0
280 #define EFSYS_OPT_NVRAM_SFT9001 0
281 #define EFSYS_OPT_NVRAM_SFX7101 0
282 #define EFSYS_OPT_BOOTCFG 0
284 #define EFSYS_OPT_PCIE_TUNE 0
285 #define EFSYS_OPT_DIAG 0
286 #define EFSYS_OPT_WOL 1
287 #define EFSYS_OPT_RX_SCALE 1
288 #define EFSYS_OPT_QSTATS 1
289 #define EFSYS_OPT_FILTER 1
290 #define EFSYS_OPT_RX_SCATTER 0
291 #define EFSYS_OPT_RX_HDR_SPLIT 0
293 #define EFSYS_OPT_EV_PREFETCH 0
295 #define EFSYS_OPT_DECODE_INTR_FATAL 1
299 typedef struct __efsys_identifier_s efsys_identifier_t;
305 #define EFSYS_PROBE(_name)
307 #define EFSYS_PROBE1(_name, _type1, _arg1)
309 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2)
311 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \
314 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
315 _type3, _arg3, _type4, _arg4)
317 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
318 _type3, _arg3, _type4, _arg4, _type5, _arg5)
320 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
321 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
324 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
325 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
326 _type6, _arg6, _type7, _arg7)
328 #else /* DTRACE_PROBE */
330 #define EFSYS_PROBE(_name) \
333 #define EFSYS_PROBE1(_name, _type1, _arg1) \
334 DTRACE_PROBE1(_name, _type1, _arg1)
336 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) \
337 DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2)
339 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \
341 DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2, \
344 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
345 _type3, _arg3, _type4, _arg4) \
346 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
347 _type3, _arg3, _type4, _arg4)
350 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
351 _type3, _arg3, _type4, _arg4, _type5, _arg5) \
352 DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
353 _type3, _arg3, _type4, _arg4, _type5, _arg5)
355 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
356 _type3, _arg3, _type4, _arg4, _type5, _arg5) \
357 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
358 _type3, _arg3, _type4, _arg4)
362 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
363 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
365 DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
366 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
369 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
370 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
372 EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
373 _type3, _arg3, _type4, _arg4, _type5, _arg5)
377 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
378 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
379 _type6, _arg6, _type7, _arg7) \
380 DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
381 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
382 _type6, _arg6, _type7, _arg7)
384 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
385 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
386 _type6, _arg6, _type7, _arg7) \
387 EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
388 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
392 #endif /* DTRACE_PROBE */
396 typedef uint64_t efsys_dma_addr_t;
398 typedef struct efsys_mem_s {
399 bus_dma_tag_t esm_tag;
400 bus_dmamap_t esm_map;
402 efsys_dma_addr_t esm_addr;
406 #define EFSYS_MEM_ZERO(_esmp, _size) \
408 (void) memset((_esmp)->esm_base, 0, (_size)); \
410 _NOTE(CONSTANTCONDITION) \
413 #define EFSYS_MEM_READD(_esmp, _offset, _edp) \
417 _NOTE(CONSTANTCONDITION) \
418 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
419 ("not power of 2 aligned")); \
421 addr = (void *)((_esmp)->esm_base + (_offset)); \
423 (_edp)->ed_u32[0] = *addr; \
425 EFSYS_PROBE2(mem_readd, unsigned int, (_offset), \
426 uint32_t, (_edp)->ed_u32[0]); \
428 _NOTE(CONSTANTCONDITION) \
431 #if defined(__x86_64__)
432 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \
436 _NOTE(CONSTANTCONDITION) \
437 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
438 ("not power of 2 aligned")); \
440 addr = (void *)((_esmp)->esm_base + (_offset)); \
442 (_eqp)->eq_u64[0] = *addr; \
444 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \
445 uint32_t, (_eqp)->eq_u32[1], \
446 uint32_t, (_eqp)->eq_u32[0]); \
448 _NOTE(CONSTANTCONDITION) \
451 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \
455 _NOTE(CONSTANTCONDITION) \
456 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
457 ("not power of 2 aligned")); \
459 addr = (void *)((_esmp)->esm_base + (_offset)); \
461 (_eqp)->eq_u32[0] = *addr++; \
462 (_eqp)->eq_u32[1] = *addr; \
464 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \
465 uint32_t, (_eqp)->eq_u32[1], \
466 uint32_t, (_eqp)->eq_u32[0]); \
468 _NOTE(CONSTANTCONDITION) \
472 #if defined(__x86_64__)
473 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \
477 _NOTE(CONSTANTCONDITION) \
478 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
479 ("not power of 2 aligned")); \
481 addr = (void *)((_esmp)->esm_base + (_offset)); \
483 (_eop)->eo_u64[0] = *addr++; \
484 (_eop)->eo_u64[1] = *addr; \
486 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \
487 uint32_t, (_eop)->eo_u32[3], \
488 uint32_t, (_eop)->eo_u32[2], \
489 uint32_t, (_eop)->eo_u32[1], \
490 uint32_t, (_eop)->eo_u32[0]); \
492 _NOTE(CONSTANTCONDITION) \
495 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \
499 _NOTE(CONSTANTCONDITION) \
500 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
501 ("not power of 2 aligned")); \
503 addr = (void *)((_esmp)->esm_base + (_offset)); \
505 (_eop)->eo_u32[0] = *addr++; \
506 (_eop)->eo_u32[1] = *addr++; \
507 (_eop)->eo_u32[2] = *addr++; \
508 (_eop)->eo_u32[3] = *addr; \
510 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \
511 uint32_t, (_eop)->eo_u32[3], \
512 uint32_t, (_eop)->eo_u32[2], \
513 uint32_t, (_eop)->eo_u32[1], \
514 uint32_t, (_eop)->eo_u32[0]); \
516 _NOTE(CONSTANTCONDITION) \
520 #define EFSYS_MEM_WRITED(_esmp, _offset, _edp) \
524 _NOTE(CONSTANTCONDITION) \
525 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
526 ("not power of 2 aligned")); \
528 EFSYS_PROBE2(mem_writed, unsigned int, (_offset), \
529 uint32_t, (_edp)->ed_u32[0]); \
531 addr = (void *)((_esmp)->esm_base + (_offset)); \
533 *addr = (_edp)->ed_u32[0]; \
535 _NOTE(CONSTANTCONDITION) \
538 #if defined(__x86_64__)
539 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \
543 _NOTE(CONSTANTCONDITION) \
544 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
545 ("not power of 2 aligned")); \
547 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \
548 uint32_t, (_eqp)->eq_u32[1], \
549 uint32_t, (_eqp)->eq_u32[0]); \
551 addr = (void *)((_esmp)->esm_base + (_offset)); \
553 *addr = (_eqp)->eq_u64[0]; \
555 _NOTE(CONSTANTCONDITION) \
559 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \
563 _NOTE(CONSTANTCONDITION) \
564 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
565 ("not power of 2 aligned")); \
567 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \
568 uint32_t, (_eqp)->eq_u32[1], \
569 uint32_t, (_eqp)->eq_u32[0]); \
571 addr = (void *)((_esmp)->esm_base + (_offset)); \
573 *addr++ = (_eqp)->eq_u32[0]; \
574 *addr = (_eqp)->eq_u32[1]; \
576 _NOTE(CONSTANTCONDITION) \
580 #if defined(__x86_64__)
581 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \
585 _NOTE(CONSTANTCONDITION) \
586 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
587 ("not power of 2 aligned")); \
589 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \
590 uint32_t, (_eop)->eo_u32[3], \
591 uint32_t, (_eop)->eo_u32[2], \
592 uint32_t, (_eop)->eo_u32[1], \
593 uint32_t, (_eop)->eo_u32[0]); \
595 addr = (void *)((_esmp)->esm_base + (_offset)); \
597 *addr++ = (_eop)->eo_u64[0]; \
598 *addr = (_eop)->eo_u64[1]; \
600 _NOTE(CONSTANTCONDITION) \
603 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \
607 _NOTE(CONSTANTCONDITION) \
608 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
609 ("not power of 2 aligned")); \
611 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \
612 uint32_t, (_eop)->eo_u32[3], \
613 uint32_t, (_eop)->eo_u32[2], \
614 uint32_t, (_eop)->eo_u32[1], \
615 uint32_t, (_eop)->eo_u32[0]); \
617 addr = (void *)((_esmp)->esm_base + (_offset)); \
619 *addr++ = (_eop)->eo_u32[0]; \
620 *addr++ = (_eop)->eo_u32[1]; \
621 *addr++ = (_eop)->eo_u32[2]; \
622 *addr = (_eop)->eo_u32[3]; \
624 _NOTE(CONSTANTCONDITION) \
628 #define EFSYS_MEM_ADDR(_esmp) \
631 #define EFSYS_MEM_IS_NULL(_esmp) \
632 ((_esmp)->esm_base == NULL)
636 #define SFXGE_LOCK_NAME_MAX 16
638 typedef struct efsys_bar_s {
640 char esb_lock_name[SFXGE_LOCK_NAME_MAX];
641 bus_space_tag_t esb_tag;
642 bus_space_handle_t esb_handle;
644 struct resource *esb_res;
647 #define SFXGE_BAR_LOCK_INIT(_esbp, _ifname) \
649 snprintf((_esbp)->esb_lock_name, \
650 sizeof((_esbp)->esb_lock_name), \
651 "%s:bar", (_ifname)); \
652 mtx_init(&(_esbp)->esb_lock, (_esbp)->esb_lock_name, \
654 _NOTE(CONSTANTCONDITION) \
656 #define SFXGE_BAR_LOCK_DESTROY(_esbp) \
657 mtx_destroy(&(_esbp)->esb_lock)
658 #define SFXGE_BAR_LOCK(_esbp) \
659 mtx_lock(&(_esbp)->esb_lock)
660 #define SFXGE_BAR_UNLOCK(_esbp) \
661 mtx_unlock(&(_esbp)->esb_lock)
663 #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock) \
665 _NOTE(CONSTANTCONDITION) \
666 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
667 ("not power of 2 aligned")); \
669 _NOTE(CONSTANTCONDITION) \
671 SFXGE_BAR_LOCK(_esbp); \
673 (_edp)->ed_u32[0] = bus_space_read_stream_4( \
674 (_esbp)->esb_tag, (_esbp)->esb_handle, \
677 EFSYS_PROBE2(bar_readd, unsigned int, (_offset), \
678 uint32_t, (_edp)->ed_u32[0]); \
680 _NOTE(CONSTANTCONDITION) \
682 SFXGE_BAR_UNLOCK(_esbp); \
683 _NOTE(CONSTANTCONDITION) \
686 #if defined(SFXGE_USE_BUS_SPACE_8)
687 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \
689 _NOTE(CONSTANTCONDITION) \
690 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
691 ("not power of 2 aligned")); \
693 SFXGE_BAR_LOCK(_esbp); \
695 (_eqp)->eq_u64[0] = bus_space_read_stream_8( \
696 (_esbp)->esb_tag, (_esbp)->esb_handle, \
699 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \
700 uint32_t, (_eqp)->eq_u32[1], \
701 uint32_t, (_eqp)->eq_u32[0]); \
703 SFXGE_BAR_UNLOCK(_esbp); \
704 _NOTE(CONSTANTCONDITION) \
707 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \
709 _NOTE(CONSTANTCONDITION) \
710 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
711 ("not power of 2 aligned")); \
713 _NOTE(CONSTANTCONDITION) \
715 SFXGE_BAR_LOCK(_esbp); \
717 (_eop)->eo_u64[0] = bus_space_read_stream_8( \
718 (_esbp)->esb_tag, (_esbp)->esb_handle, \
720 (_eop)->eo_u64[1] = bus_space_read_stream_8( \
721 (_esbp)->esb_tag, (_esbp)->esb_handle, \
724 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \
725 uint32_t, (_eop)->eo_u32[3], \
726 uint32_t, (_eop)->eo_u32[2], \
727 uint32_t, (_eop)->eo_u32[1], \
728 uint32_t, (_eop)->eo_u32[0]); \
730 _NOTE(CONSTANTCONDITION) \
732 SFXGE_BAR_UNLOCK(_esbp); \
733 _NOTE(CONSTANTCONDITION) \
737 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \
739 _NOTE(CONSTANTCONDITION) \
740 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
741 ("not power of 2 aligned")); \
743 SFXGE_BAR_LOCK(_esbp); \
745 (_eqp)->eq_u32[0] = bus_space_read_stream_4( \
746 (_esbp)->esb_tag, (_esbp)->esb_handle, \
748 (_eqp)->eq_u32[1] = bus_space_read_stream_4( \
749 (_esbp)->esb_tag, (_esbp)->esb_handle, \
752 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \
753 uint32_t, (_eqp)->eq_u32[1], \
754 uint32_t, (_eqp)->eq_u32[0]); \
756 SFXGE_BAR_UNLOCK(_esbp); \
757 _NOTE(CONSTANTCONDITION) \
760 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \
762 _NOTE(CONSTANTCONDITION) \
763 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
764 ("not power of 2 aligned")); \
766 _NOTE(CONSTANTCONDITION) \
768 SFXGE_BAR_LOCK(_esbp); \
770 (_eop)->eo_u32[0] = bus_space_read_stream_4( \
771 (_esbp)->esb_tag, (_esbp)->esb_handle, \
773 (_eop)->eo_u32[1] = bus_space_read_stream_4( \
774 (_esbp)->esb_tag, (_esbp)->esb_handle, \
776 (_eop)->eo_u32[2] = bus_space_read_stream_4( \
777 (_esbp)->esb_tag, (_esbp)->esb_handle, \
779 (_eop)->eo_u32[3] = bus_space_read_stream_4( \
780 (_esbp)->esb_tag, (_esbp)->esb_handle, \
783 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \
784 uint32_t, (_eop)->eo_u32[3], \
785 uint32_t, (_eop)->eo_u32[2], \
786 uint32_t, (_eop)->eo_u32[1], \
787 uint32_t, (_eop)->eo_u32[0]); \
789 _NOTE(CONSTANTCONDITION) \
791 SFXGE_BAR_UNLOCK(_esbp); \
792 _NOTE(CONSTANTCONDITION) \
796 #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock) \
798 _NOTE(CONSTANTCONDITION) \
799 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
800 ("not power of 2 aligned")); \
802 _NOTE(CONSTANTCONDITION) \
804 SFXGE_BAR_LOCK(_esbp); \
806 EFSYS_PROBE2(bar_writed, unsigned int, (_offset), \
807 uint32_t, (_edp)->ed_u32[0]); \
810 * Make sure that previous writes to the dword have \
811 * been done. It should be cheaper than barrier just \
812 * after the write below. \
814 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
815 (_offset), sizeof (efx_dword_t), \
816 BUS_SPACE_BARRIER_WRITE); \
817 bus_space_write_stream_4((_esbp)->esb_tag, \
818 (_esbp)->esb_handle, \
819 (_offset), (_edp)->ed_u32[0]); \
821 _NOTE(CONSTANTCONDITION) \
823 SFXGE_BAR_UNLOCK(_esbp); \
824 _NOTE(CONSTANTCONDITION) \
827 #if defined(SFXGE_USE_BUS_SPACE_8)
828 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \
830 _NOTE(CONSTANTCONDITION) \
831 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
832 ("not power of 2 aligned")); \
834 SFXGE_BAR_LOCK(_esbp); \
836 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \
837 uint32_t, (_eqp)->eq_u32[1], \
838 uint32_t, (_eqp)->eq_u32[0]); \
841 * Make sure that previous writes to the qword have \
842 * been done. It should be cheaper than barrier just \
843 * after the write below. \
845 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
846 (_offset), sizeof (efx_qword_t), \
847 BUS_SPACE_BARRIER_WRITE); \
848 bus_space_write_stream_8((_esbp)->esb_tag, \
849 (_esbp)->esb_handle, \
850 (_offset), (_eqp)->eq_u64[0]); \
852 SFXGE_BAR_UNLOCK(_esbp); \
853 _NOTE(CONSTANTCONDITION) \
856 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \
858 _NOTE(CONSTANTCONDITION) \
859 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
860 ("not power of 2 aligned")); \
862 SFXGE_BAR_LOCK(_esbp); \
864 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \
865 uint32_t, (_eqp)->eq_u32[1], \
866 uint32_t, (_eqp)->eq_u32[0]); \
869 * Make sure that previous writes to the qword have \
870 * been done. It should be cheaper than barrier just \
871 * after the last write below. \
873 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
874 (_offset), sizeof (efx_qword_t), \
875 BUS_SPACE_BARRIER_WRITE); \
876 bus_space_write_stream_4((_esbp)->esb_tag, \
877 (_esbp)->esb_handle, \
878 (_offset), (_eqp)->eq_u32[0]); \
880 * It should be guaranteed that the last dword comes \
881 * the last, so barrier entire qword to be sure that \
882 * neither above nor below writes are reordered. \
884 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
885 (_offset), sizeof (efx_qword_t), \
886 BUS_SPACE_BARRIER_WRITE); \
887 bus_space_write_stream_4((_esbp)->esb_tag, \
888 (_esbp)->esb_handle, \
889 (_offset) + 4, (_eqp)->eq_u32[1]); \
891 SFXGE_BAR_UNLOCK(_esbp); \
892 _NOTE(CONSTANTCONDITION) \
897 * Guarantees 64bit aligned 64bit writes to write combined BAR mapping
898 * (required by PIO hardware)
900 #define EFSYS_BAR_WC_WRITEQ(_esbp, _offset, _eqp) \
902 _NOTE(CONSTANTCONDITION) \
903 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
904 ("not power of 2 aligned")); \
908 /* FIXME: Perform a 64-bit write */ \
909 KASSERT(0, ("not implemented")); \
911 _NOTE(CONSTANTCONDITION) \
914 #if defined(SFXGE_USE_BUS_SPACE_8)
915 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \
917 _NOTE(CONSTANTCONDITION) \
918 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
919 ("not power of 2 aligned")); \
921 _NOTE(CONSTANTCONDITION) \
923 SFXGE_BAR_LOCK(_esbp); \
925 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \
926 uint32_t, (_eop)->eo_u32[3], \
927 uint32_t, (_eop)->eo_u32[2], \
928 uint32_t, (_eop)->eo_u32[1], \
929 uint32_t, (_eop)->eo_u32[0]); \
932 * Make sure that previous writes to the oword have \
933 * been done. It should be cheaper than barrier just \
934 * after the last write below. \
936 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
937 (_offset), sizeof (efx_oword_t), \
938 BUS_SPACE_BARRIER_WRITE); \
939 bus_space_write_stream_8((_esbp)->esb_tag, \
940 (_esbp)->esb_handle, \
941 (_offset), (_eop)->eo_u64[0]); \
943 * It should be guaranteed that the last qword comes \
944 * the last, so barrier entire oword to be sure that \
945 * neither above nor below writes are reordered. \
947 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
948 (_offset), sizeof (efx_oword_t), \
949 BUS_SPACE_BARRIER_WRITE); \
950 bus_space_write_stream_8((_esbp)->esb_tag, \
951 (_esbp)->esb_handle, \
952 (_offset) + 8, (_eop)->eo_u64[1]); \
954 _NOTE(CONSTANTCONDITION) \
956 SFXGE_BAR_UNLOCK(_esbp); \
957 _NOTE(CONSTANTCONDITION) \
961 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \
963 _NOTE(CONSTANTCONDITION) \
964 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
965 ("not power of 2 aligned")); \
967 _NOTE(CONSTANTCONDITION) \
969 SFXGE_BAR_LOCK(_esbp); \
971 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \
972 uint32_t, (_eop)->eo_u32[3], \
973 uint32_t, (_eop)->eo_u32[2], \
974 uint32_t, (_eop)->eo_u32[1], \
975 uint32_t, (_eop)->eo_u32[0]); \
978 * Make sure that previous writes to the oword have \
979 * been done. It should be cheaper than barrier just \
980 * after the last write below. \
982 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
983 (_offset), sizeof (efx_oword_t), \
984 BUS_SPACE_BARRIER_WRITE); \
985 bus_space_write_stream_4((_esbp)->esb_tag, \
986 (_esbp)->esb_handle, \
987 (_offset), (_eop)->eo_u32[0]); \
988 bus_space_write_stream_4((_esbp)->esb_tag, \
989 (_esbp)->esb_handle, \
990 (_offset) + 4, (_eop)->eo_u32[1]); \
991 bus_space_write_stream_4((_esbp)->esb_tag, \
992 (_esbp)->esb_handle, \
993 (_offset) + 8, (_eop)->eo_u32[2]); \
995 * It should be guaranteed that the last dword comes \
996 * the last, so barrier entire oword to be sure that \
997 * neither above nor below writes are reordered. \
999 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
1000 (_offset), sizeof (efx_oword_t), \
1001 BUS_SPACE_BARRIER_WRITE); \
1002 bus_space_write_stream_4((_esbp)->esb_tag, \
1003 (_esbp)->esb_handle, \
1004 (_offset) + 12, (_eop)->eo_u32[3]); \
1006 _NOTE(CONSTANTCONDITION) \
1008 SFXGE_BAR_UNLOCK(_esbp); \
1009 _NOTE(CONSTANTCONDITION) \
1013 /* Use the standard octo-word write for doorbell writes */
1014 #define EFSYS_BAR_DOORBELL_WRITEO(_esbp, _offset, _eop) \
1016 EFSYS_BAR_WRITEO((_esbp), (_offset), (_eop), B_FALSE); \
1017 _NOTE(CONSTANTCONDITION) \
1022 #define EFSYS_SPIN(_us) \
1025 _NOTE(CONSTANTCONDITION) \
1028 #define EFSYS_SLEEP EFSYS_SPIN
1032 #define EFSYS_MEM_READ_BARRIER() rmb()
1033 #define EFSYS_PIO_WRITE_BARRIER()
1036 #define EFSYS_DMA_SYNC_FOR_KERNEL(_esmp, _offset, _size) \
1038 bus_dmamap_sync((_esmp)->esm_tag, \
1040 BUS_DMASYNC_POSTREAD); \
1041 _NOTE(CONSTANTCONDITION) \
1044 #define EFSYS_DMA_SYNC_FOR_DEVICE(_esmp, _offset, _size) \
1046 bus_dmamap_sync((_esmp)->esm_tag, \
1048 BUS_DMASYNC_PREWRITE); \
1049 _NOTE(CONSTANTCONDITION) \
1054 typedef clock_t efsys_timestamp_t;
1056 #define EFSYS_TIMESTAMP(_usp) \
1061 *(_usp) = now * hz / 1000000; \
1062 _NOTE(CONSTANTCONDITION) \
1067 #define EFSYS_KMEM_ALLOC(_esip, _size, _p) \
1069 (_esip) = (_esip); \
1071 * The macro is used in non-sleepable contexts, for \
1072 * example, holding a mutex. \
1074 (_p) = malloc((_size), M_SFXGE, M_NOWAIT|M_ZERO); \
1075 _NOTE(CONSTANTCONDITION) \
1078 #define EFSYS_KMEM_FREE(_esip, _size, _p) \
1082 free((_p), M_SFXGE); \
1083 _NOTE(CONSTANTCONDITION) \
1088 typedef struct efsys_lock_s {
1090 char lock_name[SFXGE_LOCK_NAME_MAX];
1093 #define SFXGE_EFSYS_LOCK_INIT(_eslp, _ifname, _label) \
1095 efsys_lock_t *__eslp = (_eslp); \
1097 snprintf((__eslp)->lock_name, \
1098 sizeof((__eslp)->lock_name), \
1099 "%s:%s", (_ifname), (_label)); \
1100 mtx_init(&(__eslp)->lock, (__eslp)->lock_name, \
1103 #define SFXGE_EFSYS_LOCK_DESTROY(_eslp) \
1104 mtx_destroy(&(_eslp)->lock)
1105 #define SFXGE_EFSYS_LOCK(_eslp) \
1106 mtx_lock(&(_eslp)->lock)
1107 #define SFXGE_EFSYS_UNLOCK(_eslp) \
1108 mtx_unlock(&(_eslp)->lock)
1109 #define SFXGE_EFSYS_LOCK_ASSERT_OWNED(_eslp) \
1110 mtx_assert(&(_eslp)->lock, MA_OWNED)
1112 #define EFSYS_LOCK_MAGIC 0x000010c4
1114 #define EFSYS_LOCK(_lockp, _state) \
1116 SFXGE_EFSYS_LOCK(_lockp); \
1117 (_state) = EFSYS_LOCK_MAGIC; \
1118 _NOTE(CONSTANTCONDITION) \
1121 #define EFSYS_UNLOCK(_lockp, _state) \
1123 if ((_state) != EFSYS_LOCK_MAGIC) \
1124 KASSERT(B_FALSE, ("not locked")); \
1125 SFXGE_EFSYS_UNLOCK(_lockp); \
1126 _NOTE(CONSTANTCONDITION) \
1131 #define EFSYS_PREEMPT_DISABLE(_state) \
1133 (_state) = (_state); \
1135 _NOTE(CONSTANTCONDITION) \
1138 #define EFSYS_PREEMPT_ENABLE(_state) \
1140 (_state) = (_state); \
1141 critical_exit(_state); \
1142 _NOTE(CONSTANTCONDITION) \
1147 typedef uint64_t efsys_stat_t;
1149 #define EFSYS_STAT_INCR(_knp, _delta) \
1151 *(_knp) += (_delta); \
1152 _NOTE(CONSTANTCONDITION) \
1155 #define EFSYS_STAT_DECR(_knp, _delta) \
1157 *(_knp) -= (_delta); \
1158 _NOTE(CONSTANTCONDITION) \
1161 #define EFSYS_STAT_SET(_knp, _val) \
1164 _NOTE(CONSTANTCONDITION) \
1167 #define EFSYS_STAT_SET_QWORD(_knp, _valp) \
1169 *(_knp) = le64toh((_valp)->eq_u64[0]); \
1170 _NOTE(CONSTANTCONDITION) \
1173 #define EFSYS_STAT_SET_DWORD(_knp, _valp) \
1175 *(_knp) = le32toh((_valp)->ed_u32[0]); \
1176 _NOTE(CONSTANTCONDITION) \
1179 #define EFSYS_STAT_INCR_QWORD(_knp, _valp) \
1181 *(_knp) += le64toh((_valp)->eq_u64[0]); \
1182 _NOTE(CONSTANTCONDITION) \
1185 #define EFSYS_STAT_SUBR_QWORD(_knp, _valp) \
1187 *(_knp) -= le64toh((_valp)->eq_u64[0]); \
1188 _NOTE(CONSTANTCONDITION) \
1193 extern void sfxge_err(efsys_identifier_t *, unsigned int,
1194 uint32_t, uint32_t);
1196 #if EFSYS_OPT_DECODE_INTR_FATAL
1197 #define EFSYS_ERR(_esip, _code, _dword0, _dword1) \
1199 sfxge_err((_esip), (_code), (_dword0), (_dword1)); \
1200 _NOTE(CONSTANTCONDITION) \
1206 #define EFSYS_ASSERT(_exp) do { \
1208 panic("%s", #_exp); \
1211 #define EFSYS_ASSERT3(_x, _op, _y, _t) do { \
1212 const _t __x = (_t)(_x); \
1213 const _t __y = (_t)(_y); \
1214 if (!(__x _op __y)) \
1215 panic("assertion failed at %s:%u", __FILE__, __LINE__); \
1218 #define EFSYS_ASSERT3U(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uint64_t)
1219 #define EFSYS_ASSERT3S(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, int64_t)
1220 #define EFSYS_ASSERT3P(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uintptr_t)
1224 #define EFSYS_HAS_ROTL_DWORD 0
1230 #endif /* _SYS_EFSYS_H */