2 * Copyright (c) 2010-2015 Solarflare Communications Inc.
5 * This software was developed in part by Philip Paeps under contract for
6 * Solarflare Communications, Inc.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are met:
11 * 1. Redistributions of source code must retain the above copyright notice,
12 * this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright notice,
14 * this list of conditions and the following disclaimer in the documentation
15 * and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
19 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
21 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
22 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
23 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
24 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
25 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
27 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 * The views and conclusions contained in the software and documentation are
30 * those of the authors and should not be interpreted as representing official
31 * policies, either expressed or implied, of the FreeBSD Project.
43 #include <sys/param.h>
45 #include <sys/endian.h>
47 #include <sys/malloc.h>
49 #include <sys/mutex.h>
50 #include <sys/rwlock.h>
52 #include <sys/systm.h>
54 #include <machine/bus.h>
55 #include <machine/endian.h>
57 #define EFSYS_HAS_UINT64 1
58 #if defined(__x86_64__)
59 #define EFSYS_USE_UINT64 1
61 #define EFSYS_USE_UINT64 0
63 #define EFSYS_HAS_SSE2_M128 0
64 #if _BYTE_ORDER == _BIG_ENDIAN
65 #define EFSYS_IS_BIG_ENDIAN 1
66 #define EFSYS_IS_LITTLE_ENDIAN 0
67 #elif _BYTE_ORDER == _LITTLE_ENDIAN
68 #define EFSYS_IS_BIG_ENDIAN 0
69 #define EFSYS_IS_LITTLE_ENDIAN 1
71 #include "efx_types.h"
73 /* Common code requires this */
74 #if __FreeBSD_version < 800068
75 #define memmove(d, s, l) bcopy(s, d, l)
78 /* FreeBSD equivalents of Solaris things */
91 #define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0)
95 #define P2ROUNDUP(x, align) (-(-(x) & -(align)))
99 #define P2ALIGN(_x, _a) ((_x) & -(_a))
103 #define ISP2(x) (((x) & ((x) - 1)) == 0)
106 #if defined(__x86_64__) && __FreeBSD_version >= 1000000
108 #define SFXGE_USE_BUS_SPACE_8 1
110 #if !defined(bus_space_read_stream_8)
112 #define bus_space_read_stream_8(t, h, o) \
113 bus_space_read_8((t), (h), (o))
115 #define bus_space_write_stream_8(t, h, o, v) \
116 bus_space_write_8((t), (h), (o), (v))
122 #define ENOTACTIVE EINVAL
124 /* Memory type to use on FreeBSD */
125 MALLOC_DECLARE(M_SFXGE);
127 /* Machine dependend prefetch wrappers */
128 #if defined(__i386__) || defined(__amd64__)
130 prefetch_read_many(void *addr)
140 prefetch_read_once(void *addr)
148 #elif defined(__sparc64__)
150 prefetch_read_many(void *addr)
160 prefetch_read_once(void *addr)
170 prefetch_read_many(void *addr)
176 prefetch_read_once(void *addr)
182 #if defined(__i386__) || defined(__amd64__)
187 sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map,
188 struct mbuf *m, bus_dma_segment_t *seg)
190 #if defined(__i386__) || defined(__amd64__)
191 seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t));
192 seg->ds_len = m->m_len;
196 bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0);
200 /* Modifiers used for Windows builds */
203 #define __in_ecount(_n)
204 #define __in_ecount_opt(_n)
205 #define __in_bcount(_n)
206 #define __in_bcount_opt(_n)
210 #define __out_ecount(_n)
211 #define __out_ecount_opt(_n)
212 #define __out_bcount(_n)
213 #define __out_bcount_opt(_n)
214 #define __out_bcount_part(_n, _l)
215 #define __out_bcount_part_opt(_n, _l)
221 #define __inout_ecount(_n)
222 #define __inout_ecount_opt(_n)
223 #define __inout_bcount(_n)
224 #define __inout_bcount_opt(_n)
225 #define __inout_bcount_full_opt(_n)
227 #define __deref_out_bcount_opt(n)
229 #define __checkReturn
230 #define __success(_x)
232 #define __drv_when(_p, _c)
234 /* Code inclusion options */
237 #define EFSYS_OPT_NAMES 1
239 #define EFSYS_OPT_FALCON 0
240 #define EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0
241 #define EFSYS_OPT_SIENA 1
242 #define EFSYS_OPT_HUNTINGTON 1
243 #define EFSYS_OPT_MEDFORD 0
245 #define EFSYS_OPT_CHECK_REG 1
247 #define EFSYS_OPT_CHECK_REG 0
250 #define EFSYS_OPT_MCDI 1
251 #define EFSYS_OPT_MCDI_LOGGING 0
252 #define EFSYS_OPT_MCDI_PROXY_AUTH 0
254 #define EFSYS_OPT_MAC_FALCON_GMAC 0
255 #define EFSYS_OPT_MAC_FALCON_XMAC 0
256 #define EFSYS_OPT_MAC_STATS 1
258 #define EFSYS_OPT_LOOPBACK 0
260 #define EFSYS_OPT_MON_NULL 0
261 #define EFSYS_OPT_MON_LM87 0
262 #define EFSYS_OPT_MON_MAX6647 0
263 #define EFSYS_OPT_MON_MCDI 0
264 #define EFSYS_OPT_MON_STATS 0
266 #define EFSYS_OPT_PHY_NULL 0
267 #define EFSYS_OPT_PHY_QT2022C2 0
268 #define EFSYS_OPT_PHY_SFX7101 0
269 #define EFSYS_OPT_PHY_TXC43128 0
270 #define EFSYS_OPT_PHY_SFT9001 0
271 #define EFSYS_OPT_PHY_QT2025C 0
272 #define EFSYS_OPT_PHY_STATS 1
273 #define EFSYS_OPT_PHY_PROPS 0
274 #define EFSYS_OPT_PHY_BIST 0
275 #define EFSYS_OPT_BIST 1
276 #define EFSYS_OPT_PHY_LED_CONTROL 1
277 #define EFSYS_OPT_PHY_FLAGS 0
279 #define EFSYS_OPT_VPD 1
280 #define EFSYS_OPT_NVRAM 1
281 #define EFSYS_OPT_NVRAM_FALCON_BOOTROM 0
282 #define EFSYS_OPT_NVRAM_SFT9001 0
283 #define EFSYS_OPT_NVRAM_SFX7101 0
284 #define EFSYS_OPT_BOOTCFG 0
286 #define EFSYS_OPT_PCIE_TUNE 0
287 #define EFSYS_OPT_DIAG 0
288 #define EFSYS_OPT_WOL 1
289 #define EFSYS_OPT_RX_SCALE 1
290 #define EFSYS_OPT_QSTATS 1
291 #define EFSYS_OPT_FILTER 1
292 #define EFSYS_OPT_RX_SCATTER 0
294 #define EFSYS_OPT_EV_PREFETCH 0
296 #define EFSYS_OPT_DECODE_INTR_FATAL 1
298 #define EFSYS_OPT_LICENSING 0
302 typedef struct __efsys_identifier_s efsys_identifier_t;
306 #ifndef KDTRACE_HOOKS
308 #define EFSYS_PROBE(_name)
310 #define EFSYS_PROBE1(_name, _type1, _arg1)
312 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2)
314 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \
317 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
318 _type3, _arg3, _type4, _arg4)
320 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
321 _type3, _arg3, _type4, _arg4, _type5, _arg5)
323 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
324 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
327 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
328 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
329 _type6, _arg6, _type7, _arg7)
331 #else /* KDTRACE_HOOKS */
333 #define EFSYS_PROBE(_name) \
336 #define EFSYS_PROBE1(_name, _type1, _arg1) \
337 DTRACE_PROBE1(_name, _type1, _arg1)
339 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) \
340 DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2)
342 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \
344 DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2, \
347 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
348 _type3, _arg3, _type4, _arg4) \
349 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
350 _type3, _arg3, _type4, _arg4)
353 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
354 _type3, _arg3, _type4, _arg4, _type5, _arg5) \
355 DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
356 _type3, _arg3, _type4, _arg4, _type5, _arg5)
358 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
359 _type3, _arg3, _type4, _arg4, _type5, _arg5) \
360 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \
361 _type3, _arg3, _type4, _arg4)
365 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
366 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
368 DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
369 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
372 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
373 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
375 EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \
376 _type3, _arg3, _type4, _arg4, _type5, _arg5)
380 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
381 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
382 _type6, _arg6, _type7, _arg7) \
383 DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
384 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
385 _type6, _arg6, _type7, _arg7)
387 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \
388 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
389 _type6, _arg6, _type7, _arg7) \
390 EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \
391 _type3, _arg3, _type4, _arg4, _type5, _arg5, \
395 #endif /* KDTRACE_HOOKS */
399 typedef uint64_t efsys_dma_addr_t;
401 typedef struct efsys_mem_s {
402 bus_dma_tag_t esm_tag;
403 bus_dmamap_t esm_map;
405 efsys_dma_addr_t esm_addr;
409 #define EFSYS_MEM_ZERO(_esmp, _size) \
411 (void) memset((_esmp)->esm_base, 0, (_size)); \
413 _NOTE(CONSTANTCONDITION) \
416 #define EFSYS_MEM_READD(_esmp, _offset, _edp) \
420 _NOTE(CONSTANTCONDITION) \
421 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
422 ("not power of 2 aligned")); \
424 addr = (void *)((_esmp)->esm_base + (_offset)); \
426 (_edp)->ed_u32[0] = *addr; \
428 EFSYS_PROBE2(mem_readd, unsigned int, (_offset), \
429 uint32_t, (_edp)->ed_u32[0]); \
431 _NOTE(CONSTANTCONDITION) \
434 #if defined(__x86_64__)
435 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \
439 _NOTE(CONSTANTCONDITION) \
440 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
441 ("not power of 2 aligned")); \
443 addr = (void *)((_esmp)->esm_base + (_offset)); \
445 (_eqp)->eq_u64[0] = *addr; \
447 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \
448 uint32_t, (_eqp)->eq_u32[1], \
449 uint32_t, (_eqp)->eq_u32[0]); \
451 _NOTE(CONSTANTCONDITION) \
454 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \
458 _NOTE(CONSTANTCONDITION) \
459 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
460 ("not power of 2 aligned")); \
462 addr = (void *)((_esmp)->esm_base + (_offset)); \
464 (_eqp)->eq_u32[0] = *addr++; \
465 (_eqp)->eq_u32[1] = *addr; \
467 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \
468 uint32_t, (_eqp)->eq_u32[1], \
469 uint32_t, (_eqp)->eq_u32[0]); \
471 _NOTE(CONSTANTCONDITION) \
475 #if defined(__x86_64__)
476 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \
480 _NOTE(CONSTANTCONDITION) \
481 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
482 ("not power of 2 aligned")); \
484 addr = (void *)((_esmp)->esm_base + (_offset)); \
486 (_eop)->eo_u64[0] = *addr++; \
487 (_eop)->eo_u64[1] = *addr; \
489 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \
490 uint32_t, (_eop)->eo_u32[3], \
491 uint32_t, (_eop)->eo_u32[2], \
492 uint32_t, (_eop)->eo_u32[1], \
493 uint32_t, (_eop)->eo_u32[0]); \
495 _NOTE(CONSTANTCONDITION) \
498 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \
502 _NOTE(CONSTANTCONDITION) \
503 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
504 ("not power of 2 aligned")); \
506 addr = (void *)((_esmp)->esm_base + (_offset)); \
508 (_eop)->eo_u32[0] = *addr++; \
509 (_eop)->eo_u32[1] = *addr++; \
510 (_eop)->eo_u32[2] = *addr++; \
511 (_eop)->eo_u32[3] = *addr; \
513 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \
514 uint32_t, (_eop)->eo_u32[3], \
515 uint32_t, (_eop)->eo_u32[2], \
516 uint32_t, (_eop)->eo_u32[1], \
517 uint32_t, (_eop)->eo_u32[0]); \
519 _NOTE(CONSTANTCONDITION) \
523 #define EFSYS_MEM_WRITED(_esmp, _offset, _edp) \
527 _NOTE(CONSTANTCONDITION) \
528 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
529 ("not power of 2 aligned")); \
531 EFSYS_PROBE2(mem_writed, unsigned int, (_offset), \
532 uint32_t, (_edp)->ed_u32[0]); \
534 addr = (void *)((_esmp)->esm_base + (_offset)); \
536 *addr = (_edp)->ed_u32[0]; \
538 _NOTE(CONSTANTCONDITION) \
541 #if defined(__x86_64__)
542 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \
546 _NOTE(CONSTANTCONDITION) \
547 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
548 ("not power of 2 aligned")); \
550 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \
551 uint32_t, (_eqp)->eq_u32[1], \
552 uint32_t, (_eqp)->eq_u32[0]); \
554 addr = (void *)((_esmp)->esm_base + (_offset)); \
556 *addr = (_eqp)->eq_u64[0]; \
558 _NOTE(CONSTANTCONDITION) \
562 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \
566 _NOTE(CONSTANTCONDITION) \
567 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
568 ("not power of 2 aligned")); \
570 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \
571 uint32_t, (_eqp)->eq_u32[1], \
572 uint32_t, (_eqp)->eq_u32[0]); \
574 addr = (void *)((_esmp)->esm_base + (_offset)); \
576 *addr++ = (_eqp)->eq_u32[0]; \
577 *addr = (_eqp)->eq_u32[1]; \
579 _NOTE(CONSTANTCONDITION) \
583 #if defined(__x86_64__)
584 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \
588 _NOTE(CONSTANTCONDITION) \
589 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
590 ("not power of 2 aligned")); \
592 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \
593 uint32_t, (_eop)->eo_u32[3], \
594 uint32_t, (_eop)->eo_u32[2], \
595 uint32_t, (_eop)->eo_u32[1], \
596 uint32_t, (_eop)->eo_u32[0]); \
598 addr = (void *)((_esmp)->esm_base + (_offset)); \
600 *addr++ = (_eop)->eo_u64[0]; \
601 *addr = (_eop)->eo_u64[1]; \
603 _NOTE(CONSTANTCONDITION) \
606 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \
610 _NOTE(CONSTANTCONDITION) \
611 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
612 ("not power of 2 aligned")); \
614 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \
615 uint32_t, (_eop)->eo_u32[3], \
616 uint32_t, (_eop)->eo_u32[2], \
617 uint32_t, (_eop)->eo_u32[1], \
618 uint32_t, (_eop)->eo_u32[0]); \
620 addr = (void *)((_esmp)->esm_base + (_offset)); \
622 *addr++ = (_eop)->eo_u32[0]; \
623 *addr++ = (_eop)->eo_u32[1]; \
624 *addr++ = (_eop)->eo_u32[2]; \
625 *addr = (_eop)->eo_u32[3]; \
627 _NOTE(CONSTANTCONDITION) \
631 #define EFSYS_MEM_ADDR(_esmp) \
634 #define EFSYS_MEM_IS_NULL(_esmp) \
635 ((_esmp)->esm_base == NULL)
639 #define SFXGE_LOCK_NAME_MAX 16
641 typedef struct efsys_bar_s {
643 char esb_lock_name[SFXGE_LOCK_NAME_MAX];
644 bus_space_tag_t esb_tag;
645 bus_space_handle_t esb_handle;
647 struct resource *esb_res;
650 #define SFXGE_BAR_LOCK_INIT(_esbp, _ifname) \
652 snprintf((_esbp)->esb_lock_name, \
653 sizeof((_esbp)->esb_lock_name), \
654 "%s:bar", (_ifname)); \
655 mtx_init(&(_esbp)->esb_lock, (_esbp)->esb_lock_name, \
657 _NOTE(CONSTANTCONDITION) \
659 #define SFXGE_BAR_LOCK_DESTROY(_esbp) \
660 mtx_destroy(&(_esbp)->esb_lock)
661 #define SFXGE_BAR_LOCK(_esbp) \
662 mtx_lock(&(_esbp)->esb_lock)
663 #define SFXGE_BAR_UNLOCK(_esbp) \
664 mtx_unlock(&(_esbp)->esb_lock)
666 #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock) \
668 _NOTE(CONSTANTCONDITION) \
669 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
670 ("not power of 2 aligned")); \
672 _NOTE(CONSTANTCONDITION) \
674 SFXGE_BAR_LOCK(_esbp); \
676 (_edp)->ed_u32[0] = bus_space_read_stream_4( \
677 (_esbp)->esb_tag, (_esbp)->esb_handle, \
680 EFSYS_PROBE2(bar_readd, unsigned int, (_offset), \
681 uint32_t, (_edp)->ed_u32[0]); \
683 _NOTE(CONSTANTCONDITION) \
685 SFXGE_BAR_UNLOCK(_esbp); \
686 _NOTE(CONSTANTCONDITION) \
689 #if defined(SFXGE_USE_BUS_SPACE_8)
690 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \
692 _NOTE(CONSTANTCONDITION) \
693 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
694 ("not power of 2 aligned")); \
696 SFXGE_BAR_LOCK(_esbp); \
698 (_eqp)->eq_u64[0] = bus_space_read_stream_8( \
699 (_esbp)->esb_tag, (_esbp)->esb_handle, \
702 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \
703 uint32_t, (_eqp)->eq_u32[1], \
704 uint32_t, (_eqp)->eq_u32[0]); \
706 SFXGE_BAR_UNLOCK(_esbp); \
707 _NOTE(CONSTANTCONDITION) \
710 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \
712 _NOTE(CONSTANTCONDITION) \
713 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
714 ("not power of 2 aligned")); \
716 _NOTE(CONSTANTCONDITION) \
718 SFXGE_BAR_LOCK(_esbp); \
720 (_eop)->eo_u64[0] = bus_space_read_stream_8( \
721 (_esbp)->esb_tag, (_esbp)->esb_handle, \
723 (_eop)->eo_u64[1] = bus_space_read_stream_8( \
724 (_esbp)->esb_tag, (_esbp)->esb_handle, \
727 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \
728 uint32_t, (_eop)->eo_u32[3], \
729 uint32_t, (_eop)->eo_u32[2], \
730 uint32_t, (_eop)->eo_u32[1], \
731 uint32_t, (_eop)->eo_u32[0]); \
733 _NOTE(CONSTANTCONDITION) \
735 SFXGE_BAR_UNLOCK(_esbp); \
736 _NOTE(CONSTANTCONDITION) \
740 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \
742 _NOTE(CONSTANTCONDITION) \
743 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
744 ("not power of 2 aligned")); \
746 SFXGE_BAR_LOCK(_esbp); \
748 (_eqp)->eq_u32[0] = bus_space_read_stream_4( \
749 (_esbp)->esb_tag, (_esbp)->esb_handle, \
751 (_eqp)->eq_u32[1] = bus_space_read_stream_4( \
752 (_esbp)->esb_tag, (_esbp)->esb_handle, \
755 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \
756 uint32_t, (_eqp)->eq_u32[1], \
757 uint32_t, (_eqp)->eq_u32[0]); \
759 SFXGE_BAR_UNLOCK(_esbp); \
760 _NOTE(CONSTANTCONDITION) \
763 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \
765 _NOTE(CONSTANTCONDITION) \
766 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
767 ("not power of 2 aligned")); \
769 _NOTE(CONSTANTCONDITION) \
771 SFXGE_BAR_LOCK(_esbp); \
773 (_eop)->eo_u32[0] = bus_space_read_stream_4( \
774 (_esbp)->esb_tag, (_esbp)->esb_handle, \
776 (_eop)->eo_u32[1] = bus_space_read_stream_4( \
777 (_esbp)->esb_tag, (_esbp)->esb_handle, \
779 (_eop)->eo_u32[2] = bus_space_read_stream_4( \
780 (_esbp)->esb_tag, (_esbp)->esb_handle, \
782 (_eop)->eo_u32[3] = bus_space_read_stream_4( \
783 (_esbp)->esb_tag, (_esbp)->esb_handle, \
786 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \
787 uint32_t, (_eop)->eo_u32[3], \
788 uint32_t, (_eop)->eo_u32[2], \
789 uint32_t, (_eop)->eo_u32[1], \
790 uint32_t, (_eop)->eo_u32[0]); \
792 _NOTE(CONSTANTCONDITION) \
794 SFXGE_BAR_UNLOCK(_esbp); \
795 _NOTE(CONSTANTCONDITION) \
799 #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock) \
801 _NOTE(CONSTANTCONDITION) \
802 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \
803 ("not power of 2 aligned")); \
805 _NOTE(CONSTANTCONDITION) \
807 SFXGE_BAR_LOCK(_esbp); \
809 EFSYS_PROBE2(bar_writed, unsigned int, (_offset), \
810 uint32_t, (_edp)->ed_u32[0]); \
813 * Make sure that previous writes to the dword have \
814 * been done. It should be cheaper than barrier just \
815 * after the write below. \
817 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
818 (_offset), sizeof (efx_dword_t), \
819 BUS_SPACE_BARRIER_WRITE); \
820 bus_space_write_stream_4((_esbp)->esb_tag, \
821 (_esbp)->esb_handle, \
822 (_offset), (_edp)->ed_u32[0]); \
824 _NOTE(CONSTANTCONDITION) \
826 SFXGE_BAR_UNLOCK(_esbp); \
827 _NOTE(CONSTANTCONDITION) \
830 #if defined(SFXGE_USE_BUS_SPACE_8)
831 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \
833 _NOTE(CONSTANTCONDITION) \
834 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
835 ("not power of 2 aligned")); \
837 SFXGE_BAR_LOCK(_esbp); \
839 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \
840 uint32_t, (_eqp)->eq_u32[1], \
841 uint32_t, (_eqp)->eq_u32[0]); \
844 * Make sure that previous writes to the qword have \
845 * been done. It should be cheaper than barrier just \
846 * after the write below. \
848 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
849 (_offset), sizeof (efx_qword_t), \
850 BUS_SPACE_BARRIER_WRITE); \
851 bus_space_write_stream_8((_esbp)->esb_tag, \
852 (_esbp)->esb_handle, \
853 (_offset), (_eqp)->eq_u64[0]); \
855 SFXGE_BAR_UNLOCK(_esbp); \
856 _NOTE(CONSTANTCONDITION) \
859 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \
861 _NOTE(CONSTANTCONDITION) \
862 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
863 ("not power of 2 aligned")); \
865 SFXGE_BAR_LOCK(_esbp); \
867 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \
868 uint32_t, (_eqp)->eq_u32[1], \
869 uint32_t, (_eqp)->eq_u32[0]); \
872 * Make sure that previous writes to the qword have \
873 * been done. It should be cheaper than barrier just \
874 * after the last write below. \
876 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
877 (_offset), sizeof (efx_qword_t), \
878 BUS_SPACE_BARRIER_WRITE); \
879 bus_space_write_stream_4((_esbp)->esb_tag, \
880 (_esbp)->esb_handle, \
881 (_offset), (_eqp)->eq_u32[0]); \
883 * It should be guaranteed that the last dword comes \
884 * the last, so barrier entire qword to be sure that \
885 * neither above nor below writes are reordered. \
887 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
888 (_offset), sizeof (efx_qword_t), \
889 BUS_SPACE_BARRIER_WRITE); \
890 bus_space_write_stream_4((_esbp)->esb_tag, \
891 (_esbp)->esb_handle, \
892 (_offset) + 4, (_eqp)->eq_u32[1]); \
894 SFXGE_BAR_UNLOCK(_esbp); \
895 _NOTE(CONSTANTCONDITION) \
900 * Guarantees 64bit aligned 64bit writes to write combined BAR mapping
901 * (required by PIO hardware)
903 #define EFSYS_BAR_WC_WRITEQ(_esbp, _offset, _eqp) \
905 _NOTE(CONSTANTCONDITION) \
906 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \
907 ("not power of 2 aligned")); \
911 /* FIXME: Perform a 64-bit write */ \
912 KASSERT(0, ("not implemented")); \
914 _NOTE(CONSTANTCONDITION) \
917 #if defined(SFXGE_USE_BUS_SPACE_8)
918 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \
920 _NOTE(CONSTANTCONDITION) \
921 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
922 ("not power of 2 aligned")); \
924 _NOTE(CONSTANTCONDITION) \
926 SFXGE_BAR_LOCK(_esbp); \
928 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \
929 uint32_t, (_eop)->eo_u32[3], \
930 uint32_t, (_eop)->eo_u32[2], \
931 uint32_t, (_eop)->eo_u32[1], \
932 uint32_t, (_eop)->eo_u32[0]); \
935 * Make sure that previous writes to the oword have \
936 * been done. It should be cheaper than barrier just \
937 * after the last write below. \
939 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
940 (_offset), sizeof (efx_oword_t), \
941 BUS_SPACE_BARRIER_WRITE); \
942 bus_space_write_stream_8((_esbp)->esb_tag, \
943 (_esbp)->esb_handle, \
944 (_offset), (_eop)->eo_u64[0]); \
946 * It should be guaranteed that the last qword comes \
947 * the last, so barrier entire oword to be sure that \
948 * neither above nor below writes are reordered. \
950 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
951 (_offset), sizeof (efx_oword_t), \
952 BUS_SPACE_BARRIER_WRITE); \
953 bus_space_write_stream_8((_esbp)->esb_tag, \
954 (_esbp)->esb_handle, \
955 (_offset) + 8, (_eop)->eo_u64[1]); \
957 _NOTE(CONSTANTCONDITION) \
959 SFXGE_BAR_UNLOCK(_esbp); \
960 _NOTE(CONSTANTCONDITION) \
964 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \
966 _NOTE(CONSTANTCONDITION) \
967 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \
968 ("not power of 2 aligned")); \
970 _NOTE(CONSTANTCONDITION) \
972 SFXGE_BAR_LOCK(_esbp); \
974 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \
975 uint32_t, (_eop)->eo_u32[3], \
976 uint32_t, (_eop)->eo_u32[2], \
977 uint32_t, (_eop)->eo_u32[1], \
978 uint32_t, (_eop)->eo_u32[0]); \
981 * Make sure that previous writes to the oword have \
982 * been done. It should be cheaper than barrier just \
983 * after the last write below. \
985 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
986 (_offset), sizeof (efx_oword_t), \
987 BUS_SPACE_BARRIER_WRITE); \
988 bus_space_write_stream_4((_esbp)->esb_tag, \
989 (_esbp)->esb_handle, \
990 (_offset), (_eop)->eo_u32[0]); \
991 bus_space_write_stream_4((_esbp)->esb_tag, \
992 (_esbp)->esb_handle, \
993 (_offset) + 4, (_eop)->eo_u32[1]); \
994 bus_space_write_stream_4((_esbp)->esb_tag, \
995 (_esbp)->esb_handle, \
996 (_offset) + 8, (_eop)->eo_u32[2]); \
998 * It should be guaranteed that the last dword comes \
999 * the last, so barrier entire oword to be sure that \
1000 * neither above nor below writes are reordered. \
1002 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\
1003 (_offset), sizeof (efx_oword_t), \
1004 BUS_SPACE_BARRIER_WRITE); \
1005 bus_space_write_stream_4((_esbp)->esb_tag, \
1006 (_esbp)->esb_handle, \
1007 (_offset) + 12, (_eop)->eo_u32[3]); \
1009 _NOTE(CONSTANTCONDITION) \
1011 SFXGE_BAR_UNLOCK(_esbp); \
1012 _NOTE(CONSTANTCONDITION) \
1016 /* Use the standard octo-word write for doorbell writes */
1017 #define EFSYS_BAR_DOORBELL_WRITEO(_esbp, _offset, _eop) \
1019 EFSYS_BAR_WRITEO((_esbp), (_offset), (_eop), B_FALSE); \
1020 _NOTE(CONSTANTCONDITION) \
1025 #define EFSYS_SPIN(_us) \
1028 _NOTE(CONSTANTCONDITION) \
1031 #define EFSYS_SLEEP EFSYS_SPIN
1035 #define EFSYS_MEM_READ_BARRIER() rmb()
1036 #define EFSYS_PIO_WRITE_BARRIER()
1039 #define EFSYS_DMA_SYNC_FOR_KERNEL(_esmp, _offset, _size) \
1041 bus_dmamap_sync((_esmp)->esm_tag, \
1043 BUS_DMASYNC_POSTREAD); \
1044 _NOTE(CONSTANTCONDITION) \
1047 #define EFSYS_DMA_SYNC_FOR_DEVICE(_esmp, _offset, _size) \
1049 bus_dmamap_sync((_esmp)->esm_tag, \
1051 BUS_DMASYNC_PREWRITE); \
1052 _NOTE(CONSTANTCONDITION) \
1057 typedef clock_t efsys_timestamp_t;
1059 #define EFSYS_TIMESTAMP(_usp) \
1064 *(_usp) = now * hz / 1000000; \
1065 _NOTE(CONSTANTCONDITION) \
1070 #define EFSYS_KMEM_ALLOC(_esip, _size, _p) \
1072 (_esip) = (_esip); \
1074 * The macro is used in non-sleepable contexts, for \
1075 * example, holding a mutex. \
1077 (_p) = malloc((_size), M_SFXGE, M_NOWAIT|M_ZERO); \
1078 _NOTE(CONSTANTCONDITION) \
1081 #define EFSYS_KMEM_FREE(_esip, _size, _p) \
1085 free((_p), M_SFXGE); \
1086 _NOTE(CONSTANTCONDITION) \
1091 typedef struct efsys_lock_s {
1093 char lock_name[SFXGE_LOCK_NAME_MAX];
1096 #define SFXGE_EFSYS_LOCK_INIT(_eslp, _ifname, _label) \
1098 efsys_lock_t *__eslp = (_eslp); \
1100 snprintf((__eslp)->lock_name, \
1101 sizeof((__eslp)->lock_name), \
1102 "%s:%s", (_ifname), (_label)); \
1103 mtx_init(&(__eslp)->lock, (__eslp)->lock_name, \
1106 #define SFXGE_EFSYS_LOCK_DESTROY(_eslp) \
1107 mtx_destroy(&(_eslp)->lock)
1108 #define SFXGE_EFSYS_LOCK(_eslp) \
1109 mtx_lock(&(_eslp)->lock)
1110 #define SFXGE_EFSYS_UNLOCK(_eslp) \
1111 mtx_unlock(&(_eslp)->lock)
1112 #define SFXGE_EFSYS_LOCK_ASSERT_OWNED(_eslp) \
1113 mtx_assert(&(_eslp)->lock, MA_OWNED)
1115 #define EFSYS_LOCK_MAGIC 0x000010c4
1117 #define EFSYS_LOCK(_lockp, _state) \
1119 SFXGE_EFSYS_LOCK(_lockp); \
1120 (_state) = EFSYS_LOCK_MAGIC; \
1121 _NOTE(CONSTANTCONDITION) \
1124 #define EFSYS_UNLOCK(_lockp, _state) \
1126 if ((_state) != EFSYS_LOCK_MAGIC) \
1127 KASSERT(B_FALSE, ("not locked")); \
1128 SFXGE_EFSYS_UNLOCK(_lockp); \
1129 _NOTE(CONSTANTCONDITION) \
1134 #define EFSYS_PREEMPT_DISABLE(_state) \
1136 (_state) = (_state); \
1138 _NOTE(CONSTANTCONDITION) \
1141 #define EFSYS_PREEMPT_ENABLE(_state) \
1143 (_state) = (_state); \
1144 critical_exit(_state); \
1145 _NOTE(CONSTANTCONDITION) \
1150 typedef uint64_t efsys_stat_t;
1152 #define EFSYS_STAT_INCR(_knp, _delta) \
1154 *(_knp) += (_delta); \
1155 _NOTE(CONSTANTCONDITION) \
1158 #define EFSYS_STAT_DECR(_knp, _delta) \
1160 *(_knp) -= (_delta); \
1161 _NOTE(CONSTANTCONDITION) \
1164 #define EFSYS_STAT_SET(_knp, _val) \
1167 _NOTE(CONSTANTCONDITION) \
1170 #define EFSYS_STAT_SET_QWORD(_knp, _valp) \
1172 *(_knp) = le64toh((_valp)->eq_u64[0]); \
1173 _NOTE(CONSTANTCONDITION) \
1176 #define EFSYS_STAT_SET_DWORD(_knp, _valp) \
1178 *(_knp) = le32toh((_valp)->ed_u32[0]); \
1179 _NOTE(CONSTANTCONDITION) \
1182 #define EFSYS_STAT_INCR_QWORD(_knp, _valp) \
1184 *(_knp) += le64toh((_valp)->eq_u64[0]); \
1185 _NOTE(CONSTANTCONDITION) \
1188 #define EFSYS_STAT_SUBR_QWORD(_knp, _valp) \
1190 *(_knp) -= le64toh((_valp)->eq_u64[0]); \
1191 _NOTE(CONSTANTCONDITION) \
1196 extern void sfxge_err(efsys_identifier_t *, unsigned int,
1197 uint32_t, uint32_t);
1199 #if EFSYS_OPT_DECODE_INTR_FATAL
1200 #define EFSYS_ERR(_esip, _code, _dword0, _dword1) \
1202 sfxge_err((_esip), (_code), (_dword0), (_dword1)); \
1203 _NOTE(CONSTANTCONDITION) \
1209 #define EFSYS_ASSERT(_exp) do { \
1211 panic("%s", #_exp); \
1214 #define EFSYS_ASSERT3(_x, _op, _y, _t) do { \
1215 const _t __x = (_t)(_x); \
1216 const _t __y = (_t)(_y); \
1217 if (!(__x _op __y)) \
1218 panic("assertion failed at %s:%u", __FILE__, __LINE__); \
1221 #define EFSYS_ASSERT3U(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uint64_t)
1222 #define EFSYS_ASSERT3S(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, int64_t)
1223 #define EFSYS_ASSERT3P(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uintptr_t)
1227 #define EFSYS_HAS_ROTL_DWORD 0
1233 #endif /* _SYS_EFSYS_H */