2 * Copyright (c) 2010-2015 Solarflare Communications Inc.
5 * This software was developed in part by Philip Paeps under contract for
6 * Solarflare Communications, Inc.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are met:
11 * 1. Redistributions of source code must retain the above copyright notice,
12 * this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright notice,
14 * this list of conditions and the following disclaimer in the documentation
15 * and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
19 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
21 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
22 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
23 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
24 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
25 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
27 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 * The views and conclusions contained in the software and documentation are
30 * those of the authors and should not be interpreted as representing official
31 * policies, either expressed or implied, of the FreeBSD Project.
43 #include <sys/param.h>
45 #include <sys/endian.h>
47 #include <sys/malloc.h>
49 #include <sys/mutex.h>
50 #include <sys/rwlock.h>
52 #include <sys/systm.h>
54 #include <machine/bus.h>
55 #include <machine/endian.h>
57 #define EFSYS_HAS_UINT64 1
58 #if defined(__x86_64__)
59 #define EFSYS_USE_UINT64 1
61 #define EFSYS_USE_UINT64 0
63 #define EFSYS_HAS_SSE2_M128 0
64 #if _BYTE_ORDER == _BIG_ENDIAN
65 #define EFSYS_IS_BIG_ENDIAN 1
66 #define EFSYS_IS_LITTLE_ENDIAN 0
67 #elif _BYTE_ORDER == _LITTLE_ENDIAN
68 #define EFSYS_IS_BIG_ENDIAN 0
69 #define EFSYS_IS_LITTLE_ENDIAN 1
71 #include "efx_types.h"
73 /* Common code requires this */
74 #if __FreeBSD_version < 800068
75 #define memmove(d, s, l) bcopy(s, d, l)
78 /* FreeBSD equivalents of Solaris things */
91 #define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0)
95 #define P2ROUNDUP(x, align) (-(-(x) & -(align)))
99 #define P2ALIGN(_x, _a) ((_x) & -(_a))
103 #define ISP2(x) (((x) & ((x) - 1)) == 0)
106 #if defined(__x86_64__) && __FreeBSD_version >= 1000000
108 #define SFXGE_USE_BUS_SPACE_8 1
110 #if !defined(bus_space_read_stream_8)
112 #define bus_space_read_stream_8(t, h, o) \
113 bus_space_read_8((t), (h), (o))
115 #define bus_space_write_stream_8(t, h, o, v) \
116 bus_space_write_8((t), (h), (o), (v))
122 #define ENOTACTIVE EINVAL
124 /* Memory type to use on FreeBSD */
125 MALLOC_DECLARE(M_SFXGE);
127 /* Machine dependend prefetch wrappers */
128 #if defined(__i386__) || defined(__amd64__)
130 prefetch_read_many(void *addr)
140 prefetch_read_once(void *addr)
148 #elif defined(__sparc64__)
150 prefetch_read_many(void *addr)
160 prefetch_read_once(void *addr)
170 prefetch_read_many(void *addr)
176 prefetch_read_once(void *addr)
182 #if defined(__i386__) || defined(__amd64__)
187 sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map,
188 struct mbuf *m, bus_dma_segment_t *seg)
190 #if defined(__i386__) || defined(__amd64__)
191 seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t));
192 seg->ds_len = m->m_len;
196 bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0);
200 /* Modifiers used for Windows builds */
203 #define __in_ecount(_n)
204 #define __in_ecount_opt(_n)
205 #define __in_bcount(_n)
206 #define __in_bcount_opt(_n)
210 #define __out_ecount(_n)
211 #define __out_ecount_opt(_n)
212 #define __out_bcount(_n)
213 #define __out_bcount_opt(_n)
219 #define __inout_ecount(_n)
220 #define __inout_ecount_opt(_n)
221 #define __inout_bcount(_n)
222 #define __inout_bcount_opt(_n)
223 #define __inout_bcount_full_opt(_n)
225 #define __deref_out_bcount_opt(n)
227 #define __checkReturn
229 #define __drv_when(_p, _c)
231 /* Code inclusion options */
234 #define EFSYS_OPT_NAMES 1
236 #define EFSYS_OPT_FALCON 0
237 #define EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0
238 #define EFSYS_OPT_SIENA 1
239 #define EFSYS_OPT_HUNTINGTON 1
241 #define EFSYS_OPT_CHECK_REG 1
243 #define EFSYS_OPT_CHECK_REG 0
246 #define EFSYS_OPT_MCDI 1
248 #define EFSYS_OPT_MAC_FALCON_GMAC 0
249 #define EFSYS_OPT_MAC_FALCON_XMAC 0
250 #define EFSYS_OPT_MAC_STATS 1
252 #define EFSYS_OPT_LOOPBACK 0
254 #define EFSYS_OPT_MON_NULL 0
255 #define EFSYS_OPT_MON_LM87 0
256 #define EFSYS_OPT_MON_MAX6647 0
257 #define EFSYS_OPT_MON_MCDI 0
258 #define EFSYS_OPT_MON_STATS 0
260 #define EFSYS_OPT_PHY_NULL 0
261 #define EFSYS_OPT_PHY_QT2022C2 0
262 #define EFSYS_OPT_PHY_SFX7101 0
263 #define EFSYS_OPT_PHY_TXC43128 0
264 #define EFSYS_OPT_PHY_SFT9001 0
265 #define EFSYS_OPT_PHY_QT2025C 0
266 #define EFSYS_OPT_PHY_STATS 1
267 #define EFSYS_OPT_PHY_PROPS 0
268 #define EFSYS_OPT_PHY_BIST 0
269 #define EFSYS_OPT_BIST 1
270 #define EFSYS_OPT_PHY_LED_CONTROL 1
271 #define EFSYS_OPT_PHY_FLAGS 0
273 #define EFSYS_OPT_VPD 1
274 #define EFSYS_OPT_NVRAM 1
275 #define EFSYS_OPT_NVRAM_FALCON_BOOTROM 0
276 #define EFSYS_OPT_NVRAM_SFT9001 0
277 #define EFSYS_OPT_NVRAM_SFX7101 0
278 #define EFSYS_OPT_BOOTCFG 0
280 #define EFSYS_OPT_PCIE_TUNE 0
281 #define EFSYS_OPT_DIAG 0
282 #define EFSYS_OPT_WOL 1
283 #define EFSYS_OPT_RX_SCALE 1
284 #define EFSYS_OPT_QSTATS 1
285 #define EFSYS_OPT_FILTER 1
286 #define EFSYS_OPT_MCAST_FILTER_LIST 1
287 #define EFSYS_OPT_RX_SCATTER 0
288 #define EFSYS_OPT_RX_HDR_SPLIT 0
290 #define EFSYS_OPT_EV_PREFETCH 0
292 #define EFSYS_OPT_DECODE_INTR_FATAL 1
296 typedef struct __efsys_identifier_s efsys_identifier_t;
300 #ifndef KDTRACE_HOOKS
302 #define EFSYS_PROBE(_name)
304 #define EFSYS_PROBE1(_name, _type1, _arg1)
306 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2)
308 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \
311 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
312 _type3, _arg3, _type4, _arg4)
314 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
315 _type3, _arg3, _type4, _arg4, _type5, _arg5)
317 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
318 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
321 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
322 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
323 _type6, _arg6, _type7, _arg7)
325 #else /* KDTRACE_HOOKS */
327 #define EFSYS_PROBE(_name) \
330 #define EFSYS_PROBE1(_name, _type1, _arg1) \
331 DTRACE_PROBE1(_name, _type1, _arg1)
333 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) \
334 DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2)
336 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \
338 DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2, \
341 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
342 _type3, _arg3, _type4, _arg4) \
343 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
344 _type3, _arg3, _type4, _arg4)
347 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
348 _type3, _arg3, _type4, _arg4, _type5, _arg5) \
349 DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
350 _type3, _arg3, _type4, _arg4, _type5, _arg5)
352 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
353 _type3, _arg3, _type4, _arg4, _type5, _arg5) \
354 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
355 _type3, _arg3, _type4, _arg4)
359 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
360 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
362 DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
363 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
366 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
367 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
369 EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
370 _type3, _arg3, _type4, _arg4, _type5, _arg5)
374 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
375 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
376 _type6, _arg6, _type7, _arg7) \
377 DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
378 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
379 _type6, _arg6, _type7, _arg7)
381 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
382 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
383 _type6, _arg6, _type7, _arg7) \
384 EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
385 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
389 #endif /* KDTRACE_HOOKS */
393 typedef uint64_t efsys_dma_addr_t;
395 typedef struct efsys_mem_s {
396 bus_dma_tag_t esm_tag;
397 bus_dmamap_t esm_map;
399 efsys_dma_addr_t esm_addr;
403 #define EFSYS_MEM_ZERO(_esmp, _size) \
405 (void) memset((_esmp)->esm_base, 0, (_size)); \
407 _NOTE(CONSTANTCONDITION) \
410 #define EFSYS_MEM_READD(_esmp, _offset, _edp) \
414 _NOTE(CONSTANTCONDITION) \
415 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
416 ("not power of 2 aligned")); \
418 addr = (void *)((_esmp)->esm_base + (_offset)); \
420 (_edp)->ed_u32[0] = *addr; \
422 EFSYS_PROBE2(mem_readd, unsigned int, (_offset), \
423 uint32_t, (_edp)->ed_u32[0]); \
425 _NOTE(CONSTANTCONDITION) \
428 #if defined(__x86_64__)
429 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \
433 _NOTE(CONSTANTCONDITION) \
434 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
435 ("not power of 2 aligned")); \
437 addr = (void *)((_esmp)->esm_base + (_offset)); \
439 (_eqp)->eq_u64[0] = *addr; \
441 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \
442 uint32_t, (_eqp)->eq_u32[1], \
443 uint32_t, (_eqp)->eq_u32[0]); \
445 _NOTE(CONSTANTCONDITION) \
448 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \
452 _NOTE(CONSTANTCONDITION) \
453 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
454 ("not power of 2 aligned")); \
456 addr = (void *)((_esmp)->esm_base + (_offset)); \
458 (_eqp)->eq_u32[0] = *addr++; \
459 (_eqp)->eq_u32[1] = *addr; \
461 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \
462 uint32_t, (_eqp)->eq_u32[1], \
463 uint32_t, (_eqp)->eq_u32[0]); \
465 _NOTE(CONSTANTCONDITION) \
469 #if defined(__x86_64__)
470 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \
474 _NOTE(CONSTANTCONDITION) \
475 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
476 ("not power of 2 aligned")); \
478 addr = (void *)((_esmp)->esm_base + (_offset)); \
480 (_eop)->eo_u64[0] = *addr++; \
481 (_eop)->eo_u64[1] = *addr; \
483 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \
484 uint32_t, (_eop)->eo_u32[3], \
485 uint32_t, (_eop)->eo_u32[2], \
486 uint32_t, (_eop)->eo_u32[1], \
487 uint32_t, (_eop)->eo_u32[0]); \
489 _NOTE(CONSTANTCONDITION) \
492 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \
496 _NOTE(CONSTANTCONDITION) \
497 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
498 ("not power of 2 aligned")); \
500 addr = (void *)((_esmp)->esm_base + (_offset)); \
502 (_eop)->eo_u32[0] = *addr++; \
503 (_eop)->eo_u32[1] = *addr++; \
504 (_eop)->eo_u32[2] = *addr++; \
505 (_eop)->eo_u32[3] = *addr; \
507 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \
508 uint32_t, (_eop)->eo_u32[3], \
509 uint32_t, (_eop)->eo_u32[2], \
510 uint32_t, (_eop)->eo_u32[1], \
511 uint32_t, (_eop)->eo_u32[0]); \
513 _NOTE(CONSTANTCONDITION) \
517 #define EFSYS_MEM_WRITED(_esmp, _offset, _edp) \
521 _NOTE(CONSTANTCONDITION) \
522 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
523 ("not power of 2 aligned")); \
525 EFSYS_PROBE2(mem_writed, unsigned int, (_offset), \
526 uint32_t, (_edp)->ed_u32[0]); \
528 addr = (void *)((_esmp)->esm_base + (_offset)); \
530 *addr = (_edp)->ed_u32[0]; \
532 _NOTE(CONSTANTCONDITION) \
535 #if defined(__x86_64__)
536 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \
540 _NOTE(CONSTANTCONDITION) \
541 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
542 ("not power of 2 aligned")); \
544 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \
545 uint32_t, (_eqp)->eq_u32[1], \
546 uint32_t, (_eqp)->eq_u32[0]); \
548 addr = (void *)((_esmp)->esm_base + (_offset)); \
550 *addr = (_eqp)->eq_u64[0]; \
552 _NOTE(CONSTANTCONDITION) \
556 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \
560 _NOTE(CONSTANTCONDITION) \
561 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
562 ("not power of 2 aligned")); \
564 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \
565 uint32_t, (_eqp)->eq_u32[1], \
566 uint32_t, (_eqp)->eq_u32[0]); \
568 addr = (void *)((_esmp)->esm_base + (_offset)); \
570 *addr++ = (_eqp)->eq_u32[0]; \
571 *addr = (_eqp)->eq_u32[1]; \
573 _NOTE(CONSTANTCONDITION) \
577 #if defined(__x86_64__)
578 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \
582 _NOTE(CONSTANTCONDITION) \
583 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
584 ("not power of 2 aligned")); \
586 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \
587 uint32_t, (_eop)->eo_u32[3], \
588 uint32_t, (_eop)->eo_u32[2], \
589 uint32_t, (_eop)->eo_u32[1], \
590 uint32_t, (_eop)->eo_u32[0]); \
592 addr = (void *)((_esmp)->esm_base + (_offset)); \
594 *addr++ = (_eop)->eo_u64[0]; \
595 *addr = (_eop)->eo_u64[1]; \
597 _NOTE(CONSTANTCONDITION) \
600 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \
604 _NOTE(CONSTANTCONDITION) \
605 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
606 ("not power of 2 aligned")); \
608 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \
609 uint32_t, (_eop)->eo_u32[3], \
610 uint32_t, (_eop)->eo_u32[2], \
611 uint32_t, (_eop)->eo_u32[1], \
612 uint32_t, (_eop)->eo_u32[0]); \
614 addr = (void *)((_esmp)->esm_base + (_offset)); \
616 *addr++ = (_eop)->eo_u32[0]; \
617 *addr++ = (_eop)->eo_u32[1]; \
618 *addr++ = (_eop)->eo_u32[2]; \
619 *addr = (_eop)->eo_u32[3]; \
621 _NOTE(CONSTANTCONDITION) \
625 #define EFSYS_MEM_ADDR(_esmp) \
628 #define EFSYS_MEM_IS_NULL(_esmp) \
629 ((_esmp)->esm_base == NULL)
633 #define SFXGE_LOCK_NAME_MAX 16
635 typedef struct efsys_bar_s {
637 char esb_lock_name[SFXGE_LOCK_NAME_MAX];
638 bus_space_tag_t esb_tag;
639 bus_space_handle_t esb_handle;
641 struct resource *esb_res;
644 #define SFXGE_BAR_LOCK_INIT(_esbp, _ifname) \
646 snprintf((_esbp)->esb_lock_name, \
647 sizeof((_esbp)->esb_lock_name), \
648 "%s:bar", (_ifname)); \
649 mtx_init(&(_esbp)->esb_lock, (_esbp)->esb_lock_name, \
651 _NOTE(CONSTANTCONDITION) \
653 #define SFXGE_BAR_LOCK_DESTROY(_esbp) \
654 mtx_destroy(&(_esbp)->esb_lock)
655 #define SFXGE_BAR_LOCK(_esbp) \
656 mtx_lock(&(_esbp)->esb_lock)
657 #define SFXGE_BAR_UNLOCK(_esbp) \
658 mtx_unlock(&(_esbp)->esb_lock)
660 #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock) \
662 _NOTE(CONSTANTCONDITION) \
663 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
664 ("not power of 2 aligned")); \
666 _NOTE(CONSTANTCONDITION) \
668 SFXGE_BAR_LOCK(_esbp); \
670 (_edp)->ed_u32[0] = bus_space_read_stream_4( \
671 (_esbp)->esb_tag, (_esbp)->esb_handle, \
674 EFSYS_PROBE2(bar_readd, unsigned int, (_offset), \
675 uint32_t, (_edp)->ed_u32[0]); \
677 _NOTE(CONSTANTCONDITION) \
679 SFXGE_BAR_UNLOCK(_esbp); \
680 _NOTE(CONSTANTCONDITION) \
683 #if defined(SFXGE_USE_BUS_SPACE_8)
684 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \
686 _NOTE(CONSTANTCONDITION) \
687 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
688 ("not power of 2 aligned")); \
690 SFXGE_BAR_LOCK(_esbp); \
692 (_eqp)->eq_u64[0] = bus_space_read_stream_8( \
693 (_esbp)->esb_tag, (_esbp)->esb_handle, \
696 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \
697 uint32_t, (_eqp)->eq_u32[1], \
698 uint32_t, (_eqp)->eq_u32[0]); \
700 SFXGE_BAR_UNLOCK(_esbp); \
701 _NOTE(CONSTANTCONDITION) \
704 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \
706 _NOTE(CONSTANTCONDITION) \
707 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
708 ("not power of 2 aligned")); \
710 _NOTE(CONSTANTCONDITION) \
712 SFXGE_BAR_LOCK(_esbp); \
714 (_eop)->eo_u64[0] = bus_space_read_stream_8( \
715 (_esbp)->esb_tag, (_esbp)->esb_handle, \
717 (_eop)->eo_u64[1] = bus_space_read_stream_8( \
718 (_esbp)->esb_tag, (_esbp)->esb_handle, \
721 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \
722 uint32_t, (_eop)->eo_u32[3], \
723 uint32_t, (_eop)->eo_u32[2], \
724 uint32_t, (_eop)->eo_u32[1], \
725 uint32_t, (_eop)->eo_u32[0]); \
727 _NOTE(CONSTANTCONDITION) \
729 SFXGE_BAR_UNLOCK(_esbp); \
730 _NOTE(CONSTANTCONDITION) \
734 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \
736 _NOTE(CONSTANTCONDITION) \
737 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
738 ("not power of 2 aligned")); \
740 SFXGE_BAR_LOCK(_esbp); \
742 (_eqp)->eq_u32[0] = bus_space_read_stream_4( \
743 (_esbp)->esb_tag, (_esbp)->esb_handle, \
745 (_eqp)->eq_u32[1] = bus_space_read_stream_4( \
746 (_esbp)->esb_tag, (_esbp)->esb_handle, \
749 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \
750 uint32_t, (_eqp)->eq_u32[1], \
751 uint32_t, (_eqp)->eq_u32[0]); \
753 SFXGE_BAR_UNLOCK(_esbp); \
754 _NOTE(CONSTANTCONDITION) \
757 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \
759 _NOTE(CONSTANTCONDITION) \
760 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
761 ("not power of 2 aligned")); \
763 _NOTE(CONSTANTCONDITION) \
765 SFXGE_BAR_LOCK(_esbp); \
767 (_eop)->eo_u32[0] = bus_space_read_stream_4( \
768 (_esbp)->esb_tag, (_esbp)->esb_handle, \
770 (_eop)->eo_u32[1] = bus_space_read_stream_4( \
771 (_esbp)->esb_tag, (_esbp)->esb_handle, \
773 (_eop)->eo_u32[2] = bus_space_read_stream_4( \
774 (_esbp)->esb_tag, (_esbp)->esb_handle, \
776 (_eop)->eo_u32[3] = bus_space_read_stream_4( \
777 (_esbp)->esb_tag, (_esbp)->esb_handle, \
780 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \
781 uint32_t, (_eop)->eo_u32[3], \
782 uint32_t, (_eop)->eo_u32[2], \
783 uint32_t, (_eop)->eo_u32[1], \
784 uint32_t, (_eop)->eo_u32[0]); \
786 _NOTE(CONSTANTCONDITION) \
788 SFXGE_BAR_UNLOCK(_esbp); \
789 _NOTE(CONSTANTCONDITION) \
793 #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock) \
795 _NOTE(CONSTANTCONDITION) \
796 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
797 ("not power of 2 aligned")); \
799 _NOTE(CONSTANTCONDITION) \
801 SFXGE_BAR_LOCK(_esbp); \
803 EFSYS_PROBE2(bar_writed, unsigned int, (_offset), \
804 uint32_t, (_edp)->ed_u32[0]); \
807 * Make sure that previous writes to the dword have \
808 * been done. It should be cheaper than barrier just \
809 * after the write below. \
811 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
812 (_offset), sizeof (efx_dword_t), \
813 BUS_SPACE_BARRIER_WRITE); \
814 bus_space_write_stream_4((_esbp)->esb_tag, \
815 (_esbp)->esb_handle, \
816 (_offset), (_edp)->ed_u32[0]); \
818 _NOTE(CONSTANTCONDITION) \
820 SFXGE_BAR_UNLOCK(_esbp); \
821 _NOTE(CONSTANTCONDITION) \
824 #if defined(SFXGE_USE_BUS_SPACE_8)
825 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \
827 _NOTE(CONSTANTCONDITION) \
828 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
829 ("not power of 2 aligned")); \
831 SFXGE_BAR_LOCK(_esbp); \
833 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \
834 uint32_t, (_eqp)->eq_u32[1], \
835 uint32_t, (_eqp)->eq_u32[0]); \
838 * Make sure that previous writes to the qword have \
839 * been done. It should be cheaper than barrier just \
840 * after the write below. \
842 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
843 (_offset), sizeof (efx_qword_t), \
844 BUS_SPACE_BARRIER_WRITE); \
845 bus_space_write_stream_8((_esbp)->esb_tag, \
846 (_esbp)->esb_handle, \
847 (_offset), (_eqp)->eq_u64[0]); \
849 SFXGE_BAR_UNLOCK(_esbp); \
850 _NOTE(CONSTANTCONDITION) \
853 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \
855 _NOTE(CONSTANTCONDITION) \
856 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
857 ("not power of 2 aligned")); \
859 SFXGE_BAR_LOCK(_esbp); \
861 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \
862 uint32_t, (_eqp)->eq_u32[1], \
863 uint32_t, (_eqp)->eq_u32[0]); \
866 * Make sure that previous writes to the qword have \
867 * been done. It should be cheaper than barrier just \
868 * after the last write below. \
870 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
871 (_offset), sizeof (efx_qword_t), \
872 BUS_SPACE_BARRIER_WRITE); \
873 bus_space_write_stream_4((_esbp)->esb_tag, \
874 (_esbp)->esb_handle, \
875 (_offset), (_eqp)->eq_u32[0]); \
877 * It should be guaranteed that the last dword comes \
878 * the last, so barrier entire qword to be sure that \
879 * neither above nor below writes are reordered. \
881 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
882 (_offset), sizeof (efx_qword_t), \
883 BUS_SPACE_BARRIER_WRITE); \
884 bus_space_write_stream_4((_esbp)->esb_tag, \
885 (_esbp)->esb_handle, \
886 (_offset) + 4, (_eqp)->eq_u32[1]); \
888 SFXGE_BAR_UNLOCK(_esbp); \
889 _NOTE(CONSTANTCONDITION) \
894 * Guarantees 64bit aligned 64bit writes to write combined BAR mapping
895 * (required by PIO hardware)
897 #define EFSYS_BAR_WC_WRITEQ(_esbp, _offset, _eqp) \
899 _NOTE(CONSTANTCONDITION) \
900 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
901 ("not power of 2 aligned")); \
905 /* FIXME: Perform a 64-bit write */ \
906 KASSERT(0, ("not implemented")); \
908 _NOTE(CONSTANTCONDITION) \
911 #if defined(SFXGE_USE_BUS_SPACE_8)
912 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \
914 _NOTE(CONSTANTCONDITION) \
915 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
916 ("not power of 2 aligned")); \
918 _NOTE(CONSTANTCONDITION) \
920 SFXGE_BAR_LOCK(_esbp); \
922 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \
923 uint32_t, (_eop)->eo_u32[3], \
924 uint32_t, (_eop)->eo_u32[2], \
925 uint32_t, (_eop)->eo_u32[1], \
926 uint32_t, (_eop)->eo_u32[0]); \
929 * Make sure that previous writes to the oword have \
930 * been done. It should be cheaper than barrier just \
931 * after the last write below. \
933 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
934 (_offset), sizeof (efx_oword_t), \
935 BUS_SPACE_BARRIER_WRITE); \
936 bus_space_write_stream_8((_esbp)->esb_tag, \
937 (_esbp)->esb_handle, \
938 (_offset), (_eop)->eo_u64[0]); \
940 * It should be guaranteed that the last qword comes \
941 * the last, so barrier entire oword to be sure that \
942 * neither above nor below writes are reordered. \
944 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
945 (_offset), sizeof (efx_oword_t), \
946 BUS_SPACE_BARRIER_WRITE); \
947 bus_space_write_stream_8((_esbp)->esb_tag, \
948 (_esbp)->esb_handle, \
949 (_offset) + 8, (_eop)->eo_u64[1]); \
951 _NOTE(CONSTANTCONDITION) \
953 SFXGE_BAR_UNLOCK(_esbp); \
954 _NOTE(CONSTANTCONDITION) \
958 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \
960 _NOTE(CONSTANTCONDITION) \
961 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
962 ("not power of 2 aligned")); \
964 _NOTE(CONSTANTCONDITION) \
966 SFXGE_BAR_LOCK(_esbp); \
968 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \
969 uint32_t, (_eop)->eo_u32[3], \
970 uint32_t, (_eop)->eo_u32[2], \
971 uint32_t, (_eop)->eo_u32[1], \
972 uint32_t, (_eop)->eo_u32[0]); \
975 * Make sure that previous writes to the oword have \
976 * been done. It should be cheaper than barrier just \
977 * after the last write below. \
979 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
980 (_offset), sizeof (efx_oword_t), \
981 BUS_SPACE_BARRIER_WRITE); \
982 bus_space_write_stream_4((_esbp)->esb_tag, \
983 (_esbp)->esb_handle, \
984 (_offset), (_eop)->eo_u32[0]); \
985 bus_space_write_stream_4((_esbp)->esb_tag, \
986 (_esbp)->esb_handle, \
987 (_offset) + 4, (_eop)->eo_u32[1]); \
988 bus_space_write_stream_4((_esbp)->esb_tag, \
989 (_esbp)->esb_handle, \
990 (_offset) + 8, (_eop)->eo_u32[2]); \
992 * It should be guaranteed that the last dword comes \
993 * the last, so barrier entire oword to be sure that \
994 * neither above nor below writes are reordered. \
996 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
997 (_offset), sizeof (efx_oword_t), \
998 BUS_SPACE_BARRIER_WRITE); \
999 bus_space_write_stream_4((_esbp)->esb_tag, \
1000 (_esbp)->esb_handle, \
1001 (_offset) + 12, (_eop)->eo_u32[3]); \
1003 _NOTE(CONSTANTCONDITION) \
1005 SFXGE_BAR_UNLOCK(_esbp); \
1006 _NOTE(CONSTANTCONDITION) \
1010 /* Use the standard octo-word write for doorbell writes */
1011 #define EFSYS_BAR_DOORBELL_WRITEO(_esbp, _offset, _eop) \
1013 EFSYS_BAR_WRITEO((_esbp), (_offset), (_eop), B_FALSE); \
1014 _NOTE(CONSTANTCONDITION) \
1019 #define EFSYS_SPIN(_us) \
1022 _NOTE(CONSTANTCONDITION) \
1025 #define EFSYS_SLEEP EFSYS_SPIN
1029 #define EFSYS_MEM_READ_BARRIER() rmb()
1030 #define EFSYS_PIO_WRITE_BARRIER()
1033 #define EFSYS_DMA_SYNC_FOR_KERNEL(_esmp, _offset, _size) \
1035 bus_dmamap_sync((_esmp)->esm_tag, \
1037 BUS_DMASYNC_POSTREAD); \
1038 _NOTE(CONSTANTCONDITION) \
1041 #define EFSYS_DMA_SYNC_FOR_DEVICE(_esmp, _offset, _size) \
1043 bus_dmamap_sync((_esmp)->esm_tag, \
1045 BUS_DMASYNC_PREWRITE); \
1046 _NOTE(CONSTANTCONDITION) \
1051 typedef clock_t efsys_timestamp_t;
1053 #define EFSYS_TIMESTAMP(_usp) \
1058 *(_usp) = now * hz / 1000000; \
1059 _NOTE(CONSTANTCONDITION) \
1064 #define EFSYS_KMEM_ALLOC(_esip, _size, _p) \
1066 (_esip) = (_esip); \
1068 * The macro is used in non-sleepable contexts, for \
1069 * example, holding a mutex. \
1071 (_p) = malloc((_size), M_SFXGE, M_NOWAIT|M_ZERO); \
1072 _NOTE(CONSTANTCONDITION) \
1075 #define EFSYS_KMEM_FREE(_esip, _size, _p) \
1079 free((_p), M_SFXGE); \
1080 _NOTE(CONSTANTCONDITION) \
1085 typedef struct efsys_lock_s {
1087 char lock_name[SFXGE_LOCK_NAME_MAX];
1090 #define SFXGE_EFSYS_LOCK_INIT(_eslp, _ifname, _label) \
1092 efsys_lock_t *__eslp = (_eslp); \
1094 snprintf((__eslp)->lock_name, \
1095 sizeof((__eslp)->lock_name), \
1096 "%s:%s", (_ifname), (_label)); \
1097 mtx_init(&(__eslp)->lock, (__eslp)->lock_name, \
1100 #define SFXGE_EFSYS_LOCK_DESTROY(_eslp) \
1101 mtx_destroy(&(_eslp)->lock)
1102 #define SFXGE_EFSYS_LOCK(_eslp) \
1103 mtx_lock(&(_eslp)->lock)
1104 #define SFXGE_EFSYS_UNLOCK(_eslp) \
1105 mtx_unlock(&(_eslp)->lock)
1106 #define SFXGE_EFSYS_LOCK_ASSERT_OWNED(_eslp) \
1107 mtx_assert(&(_eslp)->lock, MA_OWNED)
1109 #define EFSYS_LOCK_MAGIC 0x000010c4
1111 #define EFSYS_LOCK(_lockp, _state) \
1113 SFXGE_EFSYS_LOCK(_lockp); \
1114 (_state) = EFSYS_LOCK_MAGIC; \
1115 _NOTE(CONSTANTCONDITION) \
1118 #define EFSYS_UNLOCK(_lockp, _state) \
1120 if ((_state) != EFSYS_LOCK_MAGIC) \
1121 KASSERT(B_FALSE, ("not locked")); \
1122 SFXGE_EFSYS_UNLOCK(_lockp); \
1123 _NOTE(CONSTANTCONDITION) \
1128 #define EFSYS_PREEMPT_DISABLE(_state) \
1130 (_state) = (_state); \
1132 _NOTE(CONSTANTCONDITION) \
1135 #define EFSYS_PREEMPT_ENABLE(_state) \
1137 (_state) = (_state); \
1138 critical_exit(_state); \
1139 _NOTE(CONSTANTCONDITION) \
1144 typedef uint64_t efsys_stat_t;
1146 #define EFSYS_STAT_INCR(_knp, _delta) \
1148 *(_knp) += (_delta); \
1149 _NOTE(CONSTANTCONDITION) \
1152 #define EFSYS_STAT_DECR(_knp, _delta) \
1154 *(_knp) -= (_delta); \
1155 _NOTE(CONSTANTCONDITION) \
1158 #define EFSYS_STAT_SET(_knp, _val) \
1161 _NOTE(CONSTANTCONDITION) \
1164 #define EFSYS_STAT_SET_QWORD(_knp, _valp) \
1166 *(_knp) = le64toh((_valp)->eq_u64[0]); \
1167 _NOTE(CONSTANTCONDITION) \
1170 #define EFSYS_STAT_SET_DWORD(_knp, _valp) \
1172 *(_knp) = le32toh((_valp)->ed_u32[0]); \
1173 _NOTE(CONSTANTCONDITION) \
1176 #define EFSYS_STAT_INCR_QWORD(_knp, _valp) \
1178 *(_knp) += le64toh((_valp)->eq_u64[0]); \
1179 _NOTE(CONSTANTCONDITION) \
1182 #define EFSYS_STAT_SUBR_QWORD(_knp, _valp) \
1184 *(_knp) -= le64toh((_valp)->eq_u64[0]); \
1185 _NOTE(CONSTANTCONDITION) \
1190 extern void sfxge_err(efsys_identifier_t *, unsigned int,
1191 uint32_t, uint32_t);
1193 #if EFSYS_OPT_DECODE_INTR_FATAL
1194 #define EFSYS_ERR(_esip, _code, _dword0, _dword1) \
1196 sfxge_err((_esip), (_code), (_dword0), (_dword1)); \
1197 _NOTE(CONSTANTCONDITION) \
1203 #define EFSYS_ASSERT(_exp) do { \
1205 panic("%s", #_exp); \
1208 #define EFSYS_ASSERT3(_x, _op, _y, _t) do { \
1209 const _t __x = (_t)(_x); \
1210 const _t __y = (_t)(_y); \
1211 if (!(__x _op __y)) \
1212 panic("assertion failed at %s:%u", __FILE__, __LINE__); \
1215 #define EFSYS_ASSERT3U(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uint64_t)
1216 #define EFSYS_ASSERT3S(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, int64_t)
1217 #define EFSYS_ASSERT3P(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uintptr_t)
1221 #define EFSYS_HAS_ROTL_DWORD 0
1227 #endif /* _SYS_EFSYS_H */