2 * Copyright (c) 2010-2015 Solarflare Communications Inc.
5 * This software was developed in part by Philip Paeps under contract for
6 * Solarflare Communications, Inc.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are met:
11 * 1. Redistributions of source code must retain the above copyright notice,
12 * this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright notice,
14 * this list of conditions and the following disclaimer in the documentation
15 * and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
19 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
21 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
22 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
23 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
24 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
25 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
27 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 * The views and conclusions contained in the software and documentation are
30 * those of the authors and should not be interpreted as representing official
31 * policies, either expressed or implied, of the FreeBSD Project.
43 #include <sys/param.h>
45 #include <sys/endian.h>
47 #include <sys/malloc.h>
49 #include <sys/mutex.h>
50 #include <sys/rwlock.h>
52 #include <sys/systm.h>
54 #include <machine/bus.h>
55 #include <machine/endian.h>
57 #define EFSYS_HAS_UINT64 1
58 #if defined(__x86_64__)
59 #define EFSYS_USE_UINT64 1
61 #define EFSYS_USE_UINT64 0
63 #define EFSYS_HAS_SSE2_M128 0
64 #if _BYTE_ORDER == _BIG_ENDIAN
65 #define EFSYS_IS_BIG_ENDIAN 1
66 #define EFSYS_IS_LITTLE_ENDIAN 0
67 #elif _BYTE_ORDER == _LITTLE_ENDIAN
68 #define EFSYS_IS_BIG_ENDIAN 0
69 #define EFSYS_IS_LITTLE_ENDIAN 1
71 #include "efx_types.h"
73 /* Common code requires this */
74 #if __FreeBSD_version < 800068
75 #define memmove(d, s, l) bcopy(s, d, l)
78 /* FreeBSD equivalents of Solaris things */
91 #define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0)
95 #define P2ROUNDUP(x, align) (-(-(x) & -(align)))
99 #define P2ALIGN(_x, _a) ((_x) & -(_a))
103 #define ISP2(x) (((x) & ((x) - 1)) == 0)
106 #if defined(__x86_64__) && __FreeBSD_version >= 1000000
108 #define SFXGE_USE_BUS_SPACE_8 1
110 #if !defined(bus_space_read_stream_8)
112 #define bus_space_read_stream_8(t, h, o) \
113 bus_space_read_8((t), (h), (o))
115 #define bus_space_write_stream_8(t, h, o, v) \
116 bus_space_write_8((t), (h), (o), (v))
122 #define ENOTACTIVE EINVAL
124 /* Memory type to use on FreeBSD */
125 MALLOC_DECLARE(M_SFXGE);
127 /* Machine dependend prefetch wrappers */
128 #if defined(__i386__) || defined(__amd64__)
130 prefetch_read_many(void *addr)
140 prefetch_read_once(void *addr)
148 #elif defined(__sparc64__)
150 prefetch_read_many(void *addr)
160 prefetch_read_once(void *addr)
170 prefetch_read_many(void *addr)
176 prefetch_read_once(void *addr)
182 #if defined(__i386__) || defined(__amd64__)
187 sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map,
188 struct mbuf *m, bus_dma_segment_t *seg)
190 #if defined(__i386__) || defined(__amd64__)
191 seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t));
192 seg->ds_len = m->m_len;
196 bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0);
200 /* Modifiers used for Windows builds */
203 #define __in_ecount(_n)
204 #define __in_ecount_opt(_n)
205 #define __in_bcount(_n)
206 #define __in_bcount_opt(_n)
210 #define __out_ecount(_n)
211 #define __out_ecount_opt(_n)
212 #define __out_bcount(_n)
213 #define __out_bcount_opt(_n)
219 #define __inout_ecount(_n)
220 #define __inout_ecount_opt(_n)
221 #define __inout_bcount(_n)
222 #define __inout_bcount_opt(_n)
223 #define __inout_bcount_full_opt(_n)
225 #define __deref_out_bcount_opt(n)
227 #define __checkReturn
228 #define __success(_x)
230 #define __drv_when(_p, _c)
232 /* Code inclusion options */
235 #define EFSYS_OPT_NAMES 1
237 #define EFSYS_OPT_FALCON 0
238 #define EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0
239 #define EFSYS_OPT_SIENA 1
240 #define EFSYS_OPT_HUNTINGTON 1
241 #define EFSYS_OPT_MEDFORD 0
243 #define EFSYS_OPT_CHECK_REG 1
245 #define EFSYS_OPT_CHECK_REG 0
248 #define EFSYS_OPT_MCDI 1
249 #define EFSYS_OPT_MCDI_LOGGING 0
250 #define EFSYS_OPT_MCDI_PROXY_AUTH 0
252 #define EFSYS_OPT_MAC_FALCON_GMAC 0
253 #define EFSYS_OPT_MAC_FALCON_XMAC 0
254 #define EFSYS_OPT_MAC_STATS 1
256 #define EFSYS_OPT_LOOPBACK 0
258 #define EFSYS_OPT_MON_NULL 0
259 #define EFSYS_OPT_MON_LM87 0
260 #define EFSYS_OPT_MON_MAX6647 0
261 #define EFSYS_OPT_MON_MCDI 0
262 #define EFSYS_OPT_MON_STATS 0
264 #define EFSYS_OPT_PHY_NULL 0
265 #define EFSYS_OPT_PHY_QT2022C2 0
266 #define EFSYS_OPT_PHY_SFX7101 0
267 #define EFSYS_OPT_PHY_TXC43128 0
268 #define EFSYS_OPT_PHY_SFT9001 0
269 #define EFSYS_OPT_PHY_QT2025C 0
270 #define EFSYS_OPT_PHY_STATS 1
271 #define EFSYS_OPT_PHY_PROPS 0
272 #define EFSYS_OPT_PHY_BIST 0
273 #define EFSYS_OPT_BIST 1
274 #define EFSYS_OPT_PHY_LED_CONTROL 1
275 #define EFSYS_OPT_PHY_FLAGS 0
277 #define EFSYS_OPT_VPD 1
278 #define EFSYS_OPT_NVRAM 1
279 #define EFSYS_OPT_NVRAM_FALCON_BOOTROM 0
280 #define EFSYS_OPT_NVRAM_SFT9001 0
281 #define EFSYS_OPT_NVRAM_SFX7101 0
282 #define EFSYS_OPT_BOOTCFG 0
284 #define EFSYS_OPT_PCIE_TUNE 0
285 #define EFSYS_OPT_DIAG 0
286 #define EFSYS_OPT_WOL 1
287 #define EFSYS_OPT_RX_SCALE 1
288 #define EFSYS_OPT_QSTATS 1
289 #define EFSYS_OPT_FILTER 1
290 #define EFSYS_OPT_MCAST_FILTER_LIST 1
291 #define EFSYS_OPT_RX_SCATTER 0
292 #define EFSYS_OPT_RX_HDR_SPLIT 0
294 #define EFSYS_OPT_EV_PREFETCH 0
296 #define EFSYS_OPT_DECODE_INTR_FATAL 1
300 typedef struct __efsys_identifier_s efsys_identifier_t;
306 #define EFSYS_PROBE(_name)
308 #define EFSYS_PROBE1(_name, _type1, _arg1)
310 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2)
312 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \
315 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
316 _type3, _arg3, _type4, _arg4)
318 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
319 _type3, _arg3, _type4, _arg4, _type5, _arg5)
321 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
322 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
325 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
326 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
327 _type6, _arg6, _type7, _arg7)
329 #else /* DTRACE_PROBE */
331 #define EFSYS_PROBE(_name) \
334 #define EFSYS_PROBE1(_name, _type1, _arg1) \
335 DTRACE_PROBE1(_name, _type1, _arg1)
337 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) \
338 DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2)
340 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \
342 DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2, \
345 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
346 _type3, _arg3, _type4, _arg4) \
347 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
348 _type3, _arg3, _type4, _arg4)
351 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
352 _type3, _arg3, _type4, _arg4, _type5, _arg5) \
353 DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
354 _type3, _arg3, _type4, _arg4, _type5, _arg5)
356 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
357 _type3, _arg3, _type4, _arg4, _type5, _arg5) \
358 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
359 _type3, _arg3, _type4, _arg4)
363 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
364 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
366 DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
367 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
370 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
371 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
373 EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
374 _type3, _arg3, _type4, _arg4, _type5, _arg5)
378 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
379 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
380 _type6, _arg6, _type7, _arg7) \
381 DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
382 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
383 _type6, _arg6, _type7, _arg7)
385 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
386 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
387 _type6, _arg6, _type7, _arg7) \
388 EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
389 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
393 #endif /* DTRACE_PROBE */
397 typedef uint64_t efsys_dma_addr_t;
399 typedef struct efsys_mem_s {
400 bus_dma_tag_t esm_tag;
401 bus_dmamap_t esm_map;
403 efsys_dma_addr_t esm_addr;
407 #define EFSYS_MEM_ZERO(_esmp, _size) \
409 (void) memset((_esmp)->esm_base, 0, (_size)); \
411 _NOTE(CONSTANTCONDITION) \
414 #define EFSYS_MEM_READD(_esmp, _offset, _edp) \
418 _NOTE(CONSTANTCONDITION) \
419 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
420 ("not power of 2 aligned")); \
422 addr = (void *)((_esmp)->esm_base + (_offset)); \
424 (_edp)->ed_u32[0] = *addr; \
426 EFSYS_PROBE2(mem_readd, unsigned int, (_offset), \
427 uint32_t, (_edp)->ed_u32[0]); \
429 _NOTE(CONSTANTCONDITION) \
432 #if defined(__x86_64__)
433 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \
437 _NOTE(CONSTANTCONDITION) \
438 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
439 ("not power of 2 aligned")); \
441 addr = (void *)((_esmp)->esm_base + (_offset)); \
443 (_eqp)->eq_u64[0] = *addr; \
445 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \
446 uint32_t, (_eqp)->eq_u32[1], \
447 uint32_t, (_eqp)->eq_u32[0]); \
449 _NOTE(CONSTANTCONDITION) \
452 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \
456 _NOTE(CONSTANTCONDITION) \
457 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
458 ("not power of 2 aligned")); \
460 addr = (void *)((_esmp)->esm_base + (_offset)); \
462 (_eqp)->eq_u32[0] = *addr++; \
463 (_eqp)->eq_u32[1] = *addr; \
465 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \
466 uint32_t, (_eqp)->eq_u32[1], \
467 uint32_t, (_eqp)->eq_u32[0]); \
469 _NOTE(CONSTANTCONDITION) \
473 #if defined(__x86_64__)
474 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \
478 _NOTE(CONSTANTCONDITION) \
479 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
480 ("not power of 2 aligned")); \
482 addr = (void *)((_esmp)->esm_base + (_offset)); \
484 (_eop)->eo_u64[0] = *addr++; \
485 (_eop)->eo_u64[1] = *addr; \
487 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \
488 uint32_t, (_eop)->eo_u32[3], \
489 uint32_t, (_eop)->eo_u32[2], \
490 uint32_t, (_eop)->eo_u32[1], \
491 uint32_t, (_eop)->eo_u32[0]); \
493 _NOTE(CONSTANTCONDITION) \
496 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \
500 _NOTE(CONSTANTCONDITION) \
501 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
502 ("not power of 2 aligned")); \
504 addr = (void *)((_esmp)->esm_base + (_offset)); \
506 (_eop)->eo_u32[0] = *addr++; \
507 (_eop)->eo_u32[1] = *addr++; \
508 (_eop)->eo_u32[2] = *addr++; \
509 (_eop)->eo_u32[3] = *addr; \
511 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \
512 uint32_t, (_eop)->eo_u32[3], \
513 uint32_t, (_eop)->eo_u32[2], \
514 uint32_t, (_eop)->eo_u32[1], \
515 uint32_t, (_eop)->eo_u32[0]); \
517 _NOTE(CONSTANTCONDITION) \
521 #define EFSYS_MEM_WRITED(_esmp, _offset, _edp) \
525 _NOTE(CONSTANTCONDITION) \
526 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
527 ("not power of 2 aligned")); \
529 EFSYS_PROBE2(mem_writed, unsigned int, (_offset), \
530 uint32_t, (_edp)->ed_u32[0]); \
532 addr = (void *)((_esmp)->esm_base + (_offset)); \
534 *addr = (_edp)->ed_u32[0]; \
536 _NOTE(CONSTANTCONDITION) \
539 #if defined(__x86_64__)
540 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \
544 _NOTE(CONSTANTCONDITION) \
545 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
546 ("not power of 2 aligned")); \
548 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \
549 uint32_t, (_eqp)->eq_u32[1], \
550 uint32_t, (_eqp)->eq_u32[0]); \
552 addr = (void *)((_esmp)->esm_base + (_offset)); \
554 *addr = (_eqp)->eq_u64[0]; \
556 _NOTE(CONSTANTCONDITION) \
560 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \
564 _NOTE(CONSTANTCONDITION) \
565 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
566 ("not power of 2 aligned")); \
568 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \
569 uint32_t, (_eqp)->eq_u32[1], \
570 uint32_t, (_eqp)->eq_u32[0]); \
572 addr = (void *)((_esmp)->esm_base + (_offset)); \
574 *addr++ = (_eqp)->eq_u32[0]; \
575 *addr = (_eqp)->eq_u32[1]; \
577 _NOTE(CONSTANTCONDITION) \
581 #if defined(__x86_64__)
582 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \
586 _NOTE(CONSTANTCONDITION) \
587 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
588 ("not power of 2 aligned")); \
590 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \
591 uint32_t, (_eop)->eo_u32[3], \
592 uint32_t, (_eop)->eo_u32[2], \
593 uint32_t, (_eop)->eo_u32[1], \
594 uint32_t, (_eop)->eo_u32[0]); \
596 addr = (void *)((_esmp)->esm_base + (_offset)); \
598 *addr++ = (_eop)->eo_u64[0]; \
599 *addr = (_eop)->eo_u64[1]; \
601 _NOTE(CONSTANTCONDITION) \
604 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \
608 _NOTE(CONSTANTCONDITION) \
609 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
610 ("not power of 2 aligned")); \
612 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \
613 uint32_t, (_eop)->eo_u32[3], \
614 uint32_t, (_eop)->eo_u32[2], \
615 uint32_t, (_eop)->eo_u32[1], \
616 uint32_t, (_eop)->eo_u32[0]); \
618 addr = (void *)((_esmp)->esm_base + (_offset)); \
620 *addr++ = (_eop)->eo_u32[0]; \
621 *addr++ = (_eop)->eo_u32[1]; \
622 *addr++ = (_eop)->eo_u32[2]; \
623 *addr = (_eop)->eo_u32[3]; \
625 _NOTE(CONSTANTCONDITION) \
629 #define EFSYS_MEM_ADDR(_esmp) \
632 #define EFSYS_MEM_IS_NULL(_esmp) \
633 ((_esmp)->esm_base == NULL)
637 #define SFXGE_LOCK_NAME_MAX 16
639 typedef struct efsys_bar_s {
641 char esb_lock_name[SFXGE_LOCK_NAME_MAX];
642 bus_space_tag_t esb_tag;
643 bus_space_handle_t esb_handle;
645 struct resource *esb_res;
648 #define SFXGE_BAR_LOCK_INIT(_esbp, _ifname) \
650 snprintf((_esbp)->esb_lock_name, \
651 sizeof((_esbp)->esb_lock_name), \
652 "%s:bar", (_ifname)); \
653 mtx_init(&(_esbp)->esb_lock, (_esbp)->esb_lock_name, \
655 _NOTE(CONSTANTCONDITION) \
657 #define SFXGE_BAR_LOCK_DESTROY(_esbp) \
658 mtx_destroy(&(_esbp)->esb_lock)
659 #define SFXGE_BAR_LOCK(_esbp) \
660 mtx_lock(&(_esbp)->esb_lock)
661 #define SFXGE_BAR_UNLOCK(_esbp) \
662 mtx_unlock(&(_esbp)->esb_lock)
664 #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock) \
666 _NOTE(CONSTANTCONDITION) \
667 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
668 ("not power of 2 aligned")); \
670 _NOTE(CONSTANTCONDITION) \
672 SFXGE_BAR_LOCK(_esbp); \
674 (_edp)->ed_u32[0] = bus_space_read_stream_4( \
675 (_esbp)->esb_tag, (_esbp)->esb_handle, \
678 EFSYS_PROBE2(bar_readd, unsigned int, (_offset), \
679 uint32_t, (_edp)->ed_u32[0]); \
681 _NOTE(CONSTANTCONDITION) \
683 SFXGE_BAR_UNLOCK(_esbp); \
684 _NOTE(CONSTANTCONDITION) \
687 #if defined(SFXGE_USE_BUS_SPACE_8)
688 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \
690 _NOTE(CONSTANTCONDITION) \
691 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
692 ("not power of 2 aligned")); \
694 SFXGE_BAR_LOCK(_esbp); \
696 (_eqp)->eq_u64[0] = bus_space_read_stream_8( \
697 (_esbp)->esb_tag, (_esbp)->esb_handle, \
700 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \
701 uint32_t, (_eqp)->eq_u32[1], \
702 uint32_t, (_eqp)->eq_u32[0]); \
704 SFXGE_BAR_UNLOCK(_esbp); \
705 _NOTE(CONSTANTCONDITION) \
708 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \
710 _NOTE(CONSTANTCONDITION) \
711 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
712 ("not power of 2 aligned")); \
714 _NOTE(CONSTANTCONDITION) \
716 SFXGE_BAR_LOCK(_esbp); \
718 (_eop)->eo_u64[0] = bus_space_read_stream_8( \
719 (_esbp)->esb_tag, (_esbp)->esb_handle, \
721 (_eop)->eo_u64[1] = bus_space_read_stream_8( \
722 (_esbp)->esb_tag, (_esbp)->esb_handle, \
725 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \
726 uint32_t, (_eop)->eo_u32[3], \
727 uint32_t, (_eop)->eo_u32[2], \
728 uint32_t, (_eop)->eo_u32[1], \
729 uint32_t, (_eop)->eo_u32[0]); \
731 _NOTE(CONSTANTCONDITION) \
733 SFXGE_BAR_UNLOCK(_esbp); \
734 _NOTE(CONSTANTCONDITION) \
738 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \
740 _NOTE(CONSTANTCONDITION) \
741 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
742 ("not power of 2 aligned")); \
744 SFXGE_BAR_LOCK(_esbp); \
746 (_eqp)->eq_u32[0] = bus_space_read_stream_4( \
747 (_esbp)->esb_tag, (_esbp)->esb_handle, \
749 (_eqp)->eq_u32[1] = bus_space_read_stream_4( \
750 (_esbp)->esb_tag, (_esbp)->esb_handle, \
753 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \
754 uint32_t, (_eqp)->eq_u32[1], \
755 uint32_t, (_eqp)->eq_u32[0]); \
757 SFXGE_BAR_UNLOCK(_esbp); \
758 _NOTE(CONSTANTCONDITION) \
761 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \
763 _NOTE(CONSTANTCONDITION) \
764 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
765 ("not power of 2 aligned")); \
767 _NOTE(CONSTANTCONDITION) \
769 SFXGE_BAR_LOCK(_esbp); \
771 (_eop)->eo_u32[0] = bus_space_read_stream_4( \
772 (_esbp)->esb_tag, (_esbp)->esb_handle, \
774 (_eop)->eo_u32[1] = bus_space_read_stream_4( \
775 (_esbp)->esb_tag, (_esbp)->esb_handle, \
777 (_eop)->eo_u32[2] = bus_space_read_stream_4( \
778 (_esbp)->esb_tag, (_esbp)->esb_handle, \
780 (_eop)->eo_u32[3] = bus_space_read_stream_4( \
781 (_esbp)->esb_tag, (_esbp)->esb_handle, \
784 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \
785 uint32_t, (_eop)->eo_u32[3], \
786 uint32_t, (_eop)->eo_u32[2], \
787 uint32_t, (_eop)->eo_u32[1], \
788 uint32_t, (_eop)->eo_u32[0]); \
790 _NOTE(CONSTANTCONDITION) \
792 SFXGE_BAR_UNLOCK(_esbp); \
793 _NOTE(CONSTANTCONDITION) \
797 #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock) \
799 _NOTE(CONSTANTCONDITION) \
800 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
801 ("not power of 2 aligned")); \
803 _NOTE(CONSTANTCONDITION) \
805 SFXGE_BAR_LOCK(_esbp); \
807 EFSYS_PROBE2(bar_writed, unsigned int, (_offset), \
808 uint32_t, (_edp)->ed_u32[0]); \
811 * Make sure that previous writes to the dword have \
812 * been done. It should be cheaper than barrier just \
813 * after the write below. \
815 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
816 (_offset), sizeof (efx_dword_t), \
817 BUS_SPACE_BARRIER_WRITE); \
818 bus_space_write_stream_4((_esbp)->esb_tag, \
819 (_esbp)->esb_handle, \
820 (_offset), (_edp)->ed_u32[0]); \
822 _NOTE(CONSTANTCONDITION) \
824 SFXGE_BAR_UNLOCK(_esbp); \
825 _NOTE(CONSTANTCONDITION) \
828 #if defined(SFXGE_USE_BUS_SPACE_8)
829 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \
831 _NOTE(CONSTANTCONDITION) \
832 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
833 ("not power of 2 aligned")); \
835 SFXGE_BAR_LOCK(_esbp); \
837 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \
838 uint32_t, (_eqp)->eq_u32[1], \
839 uint32_t, (_eqp)->eq_u32[0]); \
842 * Make sure that previous writes to the qword have \
843 * been done. It should be cheaper than barrier just \
844 * after the write below. \
846 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
847 (_offset), sizeof (efx_qword_t), \
848 BUS_SPACE_BARRIER_WRITE); \
849 bus_space_write_stream_8((_esbp)->esb_tag, \
850 (_esbp)->esb_handle, \
851 (_offset), (_eqp)->eq_u64[0]); \
853 SFXGE_BAR_UNLOCK(_esbp); \
854 _NOTE(CONSTANTCONDITION) \
857 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \
859 _NOTE(CONSTANTCONDITION) \
860 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
861 ("not power of 2 aligned")); \
863 SFXGE_BAR_LOCK(_esbp); \
865 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \
866 uint32_t, (_eqp)->eq_u32[1], \
867 uint32_t, (_eqp)->eq_u32[0]); \
870 * Make sure that previous writes to the qword have \
871 * been done. It should be cheaper than barrier just \
872 * after the last write below. \
874 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
875 (_offset), sizeof (efx_qword_t), \
876 BUS_SPACE_BARRIER_WRITE); \
877 bus_space_write_stream_4((_esbp)->esb_tag, \
878 (_esbp)->esb_handle, \
879 (_offset), (_eqp)->eq_u32[0]); \
881 * It should be guaranteed that the last dword comes \
882 * the last, so barrier entire qword to be sure that \
883 * neither above nor below writes are reordered. \
885 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
886 (_offset), sizeof (efx_qword_t), \
887 BUS_SPACE_BARRIER_WRITE); \
888 bus_space_write_stream_4((_esbp)->esb_tag, \
889 (_esbp)->esb_handle, \
890 (_offset) + 4, (_eqp)->eq_u32[1]); \
892 SFXGE_BAR_UNLOCK(_esbp); \
893 _NOTE(CONSTANTCONDITION) \
898 * Guarantees 64bit aligned 64bit writes to write combined BAR mapping
899 * (required by PIO hardware)
901 #define EFSYS_BAR_WC_WRITEQ(_esbp, _offset, _eqp) \
903 _NOTE(CONSTANTCONDITION) \
904 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
905 ("not power of 2 aligned")); \
909 /* FIXME: Perform a 64-bit write */ \
910 KASSERT(0, ("not implemented")); \
912 _NOTE(CONSTANTCONDITION) \
915 #if defined(SFXGE_USE_BUS_SPACE_8)
916 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \
918 _NOTE(CONSTANTCONDITION) \
919 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
920 ("not power of 2 aligned")); \
922 _NOTE(CONSTANTCONDITION) \
924 SFXGE_BAR_LOCK(_esbp); \
926 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \
927 uint32_t, (_eop)->eo_u32[3], \
928 uint32_t, (_eop)->eo_u32[2], \
929 uint32_t, (_eop)->eo_u32[1], \
930 uint32_t, (_eop)->eo_u32[0]); \
933 * Make sure that previous writes to the oword have \
934 * been done. It should be cheaper than barrier just \
935 * after the last write below. \
937 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
938 (_offset), sizeof (efx_oword_t), \
939 BUS_SPACE_BARRIER_WRITE); \
940 bus_space_write_stream_8((_esbp)->esb_tag, \
941 (_esbp)->esb_handle, \
942 (_offset), (_eop)->eo_u64[0]); \
944 * It should be guaranteed that the last qword comes \
945 * the last, so barrier entire oword to be sure that \
946 * neither above nor below writes are reordered. \
948 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
949 (_offset), sizeof (efx_oword_t), \
950 BUS_SPACE_BARRIER_WRITE); \
951 bus_space_write_stream_8((_esbp)->esb_tag, \
952 (_esbp)->esb_handle, \
953 (_offset) + 8, (_eop)->eo_u64[1]); \
955 _NOTE(CONSTANTCONDITION) \
957 SFXGE_BAR_UNLOCK(_esbp); \
958 _NOTE(CONSTANTCONDITION) \
962 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \
964 _NOTE(CONSTANTCONDITION) \
965 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
966 ("not power of 2 aligned")); \
968 _NOTE(CONSTANTCONDITION) \
970 SFXGE_BAR_LOCK(_esbp); \
972 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \
973 uint32_t, (_eop)->eo_u32[3], \
974 uint32_t, (_eop)->eo_u32[2], \
975 uint32_t, (_eop)->eo_u32[1], \
976 uint32_t, (_eop)->eo_u32[0]); \
979 * Make sure that previous writes to the oword have \
980 * been done. It should be cheaper than barrier just \
981 * after the last write below. \
983 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
984 (_offset), sizeof (efx_oword_t), \
985 BUS_SPACE_BARRIER_WRITE); \
986 bus_space_write_stream_4((_esbp)->esb_tag, \
987 (_esbp)->esb_handle, \
988 (_offset), (_eop)->eo_u32[0]); \
989 bus_space_write_stream_4((_esbp)->esb_tag, \
990 (_esbp)->esb_handle, \
991 (_offset) + 4, (_eop)->eo_u32[1]); \
992 bus_space_write_stream_4((_esbp)->esb_tag, \
993 (_esbp)->esb_handle, \
994 (_offset) + 8, (_eop)->eo_u32[2]); \
996 * It should be guaranteed that the last dword comes \
997 * the last, so barrier entire oword to be sure that \
998 * neither above nor below writes are reordered. \
1000 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
1001 (_offset), sizeof (efx_oword_t), \
1002 BUS_SPACE_BARRIER_WRITE); \
1003 bus_space_write_stream_4((_esbp)->esb_tag, \
1004 (_esbp)->esb_handle, \
1005 (_offset) + 12, (_eop)->eo_u32[3]); \
1007 _NOTE(CONSTANTCONDITION) \
1009 SFXGE_BAR_UNLOCK(_esbp); \
1010 _NOTE(CONSTANTCONDITION) \
1014 /* Use the standard octo-word write for doorbell writes */
1015 #define EFSYS_BAR_DOORBELL_WRITEO(_esbp, _offset, _eop) \
1017 EFSYS_BAR_WRITEO((_esbp), (_offset), (_eop), B_FALSE); \
1018 _NOTE(CONSTANTCONDITION) \
1023 #define EFSYS_SPIN(_us) \
1026 _NOTE(CONSTANTCONDITION) \
1029 #define EFSYS_SLEEP EFSYS_SPIN
1033 #define EFSYS_MEM_READ_BARRIER() rmb()
1034 #define EFSYS_PIO_WRITE_BARRIER()
1037 #define EFSYS_DMA_SYNC_FOR_KERNEL(_esmp, _offset, _size) \
1039 bus_dmamap_sync((_esmp)->esm_tag, \
1041 BUS_DMASYNC_POSTREAD); \
1042 _NOTE(CONSTANTCONDITION) \
1045 #define EFSYS_DMA_SYNC_FOR_DEVICE(_esmp, _offset, _size) \
1047 bus_dmamap_sync((_esmp)->esm_tag, \
1049 BUS_DMASYNC_PREWRITE); \
1050 _NOTE(CONSTANTCONDITION) \
1055 typedef clock_t efsys_timestamp_t;
1057 #define EFSYS_TIMESTAMP(_usp) \
1062 *(_usp) = now * hz / 1000000; \
1063 _NOTE(CONSTANTCONDITION) \
1068 #define EFSYS_KMEM_ALLOC(_esip, _size, _p) \
1070 (_esip) = (_esip); \
1072 * The macro is used in non-sleepable contexts, for \
1073 * example, holding a mutex. \
1075 (_p) = malloc((_size), M_SFXGE, M_NOWAIT|M_ZERO); \
1076 _NOTE(CONSTANTCONDITION) \
1079 #define EFSYS_KMEM_FREE(_esip, _size, _p) \
1083 free((_p), M_SFXGE); \
1084 _NOTE(CONSTANTCONDITION) \
1089 typedef struct efsys_lock_s {
1091 char lock_name[SFXGE_LOCK_NAME_MAX];
1094 #define SFXGE_EFSYS_LOCK_INIT(_eslp, _ifname, _label) \
1096 efsys_lock_t *__eslp = (_eslp); \
1098 snprintf((__eslp)->lock_name, \
1099 sizeof((__eslp)->lock_name), \
1100 "%s:%s", (_ifname), (_label)); \
1101 mtx_init(&(__eslp)->lock, (__eslp)->lock_name, \
1104 #define SFXGE_EFSYS_LOCK_DESTROY(_eslp) \
1105 mtx_destroy(&(_eslp)->lock)
1106 #define SFXGE_EFSYS_LOCK(_eslp) \
1107 mtx_lock(&(_eslp)->lock)
1108 #define SFXGE_EFSYS_UNLOCK(_eslp) \
1109 mtx_unlock(&(_eslp)->lock)
1110 #define SFXGE_EFSYS_LOCK_ASSERT_OWNED(_eslp) \
1111 mtx_assert(&(_eslp)->lock, MA_OWNED)
1113 #define EFSYS_LOCK_MAGIC 0x000010c4
1115 #define EFSYS_LOCK(_lockp, _state) \
1117 SFXGE_EFSYS_LOCK(_lockp); \
1118 (_state) = EFSYS_LOCK_MAGIC; \
1119 _NOTE(CONSTANTCONDITION) \
1122 #define EFSYS_UNLOCK(_lockp, _state) \
1124 if ((_state) != EFSYS_LOCK_MAGIC) \
1125 KASSERT(B_FALSE, ("not locked")); \
1126 SFXGE_EFSYS_UNLOCK(_lockp); \
1127 _NOTE(CONSTANTCONDITION) \
1132 #define EFSYS_PREEMPT_DISABLE(_state) \
1134 (_state) = (_state); \
1136 _NOTE(CONSTANTCONDITION) \
1139 #define EFSYS_PREEMPT_ENABLE(_state) \
1141 (_state) = (_state); \
1142 critical_exit(_state); \
1143 _NOTE(CONSTANTCONDITION) \
1148 typedef uint64_t efsys_stat_t;
1150 #define EFSYS_STAT_INCR(_knp, _delta) \
1152 *(_knp) += (_delta); \
1153 _NOTE(CONSTANTCONDITION) \
1156 #define EFSYS_STAT_DECR(_knp, _delta) \
1158 *(_knp) -= (_delta); \
1159 _NOTE(CONSTANTCONDITION) \
1162 #define EFSYS_STAT_SET(_knp, _val) \
1165 _NOTE(CONSTANTCONDITION) \
1168 #define EFSYS_STAT_SET_QWORD(_knp, _valp) \
1170 *(_knp) = le64toh((_valp)->eq_u64[0]); \
1171 _NOTE(CONSTANTCONDITION) \
1174 #define EFSYS_STAT_SET_DWORD(_knp, _valp) \
1176 *(_knp) = le32toh((_valp)->ed_u32[0]); \
1177 _NOTE(CONSTANTCONDITION) \
1180 #define EFSYS_STAT_INCR_QWORD(_knp, _valp) \
1182 *(_knp) += le64toh((_valp)->eq_u64[0]); \
1183 _NOTE(CONSTANTCONDITION) \
1186 #define EFSYS_STAT_SUBR_QWORD(_knp, _valp) \
1188 *(_knp) -= le64toh((_valp)->eq_u64[0]); \
1189 _NOTE(CONSTANTCONDITION) \
1194 extern void sfxge_err(efsys_identifier_t *, unsigned int,
1195 uint32_t, uint32_t);
1197 #if EFSYS_OPT_DECODE_INTR_FATAL
1198 #define EFSYS_ERR(_esip, _code, _dword0, _dword1) \
1200 sfxge_err((_esip), (_code), (_dword0), (_dword1)); \
1201 _NOTE(CONSTANTCONDITION) \
1207 #define EFSYS_ASSERT(_exp) do { \
1209 panic("%s", #_exp); \
1212 #define EFSYS_ASSERT3(_x, _op, _y, _t) do { \
1213 const _t __x = (_t)(_x); \
1214 const _t __y = (_t)(_y); \
1215 if (!(__x _op __y)) \
1216 panic("assertion failed at %s:%u", __FILE__, __LINE__); \
1219 #define EFSYS_ASSERT3U(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uint64_t)
1220 #define EFSYS_ASSERT3S(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, int64_t)
1221 #define EFSYS_ASSERT3P(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uintptr_t)
1225 #define EFSYS_HAS_ROTL_DWORD 0
1231 #endif /* _SYS_EFSYS_H */