sys/dev/sfxge/common/efsys.h

   1 /*-
   2  * Copyright (c) 2010-2011 Solarflare Communications, Inc.
   3  * All rights reserved.
   4  *
   5  * This software was developed in part by Philip Paeps under contract for
   6  * Solarflare Communications, Inc.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  * $FreeBSD$
  30  */
  31
  32 #ifndef _SYS_EFSYS_H
  33 #define _SYS_EFSYS_H
  34
  35 #ifdef  __cplusplus
  36 extern "C" {
  37 #endif
  38
  39 #include <sys/param.h>
  40 #include <sys/bus.h>
  41 #include <sys/endian.h>
  42 #include <sys/lock.h>
  43 #include <sys/malloc.h>
  44 #include <sys/mbuf.h>
  45 #include <sys/mutex.h>
  46 #include <sys/rwlock.h>
  47 #include <sys/sdt.h>
  48 #include <sys/systm.h>
  49
  50 #include <machine/bus.h>
  51 #include <machine/endian.h>
  52
  53 #define EFSYS_HAS_UINT64 1
  54 #if defined(__x86_64__)
  55 #define EFSYS_USE_UINT64 1
  56 #else
  57 #define EFSYS_USE_UINT64 0
  58 #endif
  59 #if _BYTE_ORDER == _BIG_ENDIAN
  60 #define EFSYS_IS_BIG_ENDIAN 1
  61 #define EFSYS_IS_LITTLE_ENDIAN 0
  62 #elif _BYTE_ORDER == _LITTLE_ENDIAN
  63 #define EFSYS_IS_BIG_ENDIAN 0
  64 #define EFSYS_IS_LITTLE_ENDIAN 1
  65 #endif
  66 #include "efx_types.h"
  67
  68 /* Common code requires this */
  69 #if __FreeBSD_version < 800068
  70 #define memmove(d, s, l) bcopy(s, d, l)
  71 #endif
  72
  73 /* FreeBSD equivalents of Solaris things */
  74 #ifndef _NOTE
  75 #define _NOTE(s)
  76 #endif
  77
  78 #ifndef B_FALSE
  79 #define B_FALSE FALSE
  80 #endif
  81 #ifndef B_TRUE
  82 #define B_TRUE  TRUE
  83 #endif
  84
  85 #ifndef IS_P2ALIGNED
  86 #define IS_P2ALIGNED(v, a)      ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0)
  87 #endif
  88
  89 #ifndef P2ROUNDUP
  90 #define P2ROUNDUP(x, align)     (-(-(x) & -(align)))
  91 #endif
  92
  93 #ifndef IS2P
  94 #define ISP2(x)                 (((x) & ((x) - 1)) == 0)
  95 #endif
  96
  97 #define ENOTACTIVE EINVAL
  98
  99 /* Memory type to use on FreeBSD */
 100 MALLOC_DECLARE(M_SFXGE);
 101
 102 /* Machine dependend prefetch wrappers */
 103 #if defined(__i386__) || defined(__amd64__)
 104 static __inline void
 105 prefetch_read_many(void *addr)
 106 {
 107
 108         __asm__(
 109             "prefetcht0 (%0)"
 110             :
 111             : "r" (addr));
 112 }
 113
 114 static __inline void
 115 prefetch_read_once(void *addr)
 116 {
 117
 118         __asm__(
 119             "prefetchnta (%0)"
 120             :
 121             : "r" (addr));
 122 }
 123 #elif defined(__sparc64__)
 124 static __inline void
 125 prefetch_read_many(void *addr)
 126 {
 127
 128         __asm__(
 129             "prefetch [%0], 0"
 130             :
 131             : "r" (addr));
 132 }
 133
 134 static __inline void
 135 prefetch_read_once(void *addr)
 136 {
 137
 138         __asm__(
 139             "prefetch [%0], 1"
 140             :
 141             : "r" (addr));
 142 }
 143 #else
 144 static __inline void
 145 prefetch_read_many(void *addr)
 146 {
 147
 148 }
 149
 150 static __inline void
 151 prefetch_read_once(void *addr)
 152 {
 153
 154 }
 155 #endif
 156
 157 #if defined(__i386__) || defined(__amd64__)
 158 #include <vm/vm.h>
 159 #include <vm/pmap.h>
 160 #endif
 161 static __inline void
 162 sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map,
 163     struct mbuf *m, bus_dma_segment_t *seg)
 164 {
 165 #if defined(__i386__) || defined(__amd64__)
 166         seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t));
 167         seg->ds_len = m->m_len;
 168 #else
 169         int nsegstmp;
 170
 171         bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0);
 172 #endif
 173 }
 174
 175 /* Modifiers used for DOS builds */
 176 #define __cs
 177 #define __far
 178
 179 /* Modifiers used for Windows builds */
 180 #define __in
 181 #define __in_opt
 182 #define __in_ecount(_n)
 183 #define __in_ecount_opt(_n)
 184 #define __in_bcount(_n)
 185 #define __in_bcount_opt(_n)
 186
 187 #define __out
 188 #define __out_opt
 189 #define __out_ecount(_n)
 190 #define __out_ecount_opt(_n)
 191 #define __out_bcount(_n)
 192 #define __out_bcount_opt(_n)
 193
 194 #define __deref_out
 195
 196 #define __inout
 197 #define __inout_opt
 198 #define __inout_ecount(_n)
 199 #define __inout_ecount_opt(_n)
 200 #define __inout_bcount(_n)
 201 #define __inout_bcount_opt(_n)
 202 #define __inout_bcount_full_opt(_n)
 203
 204 #define __deref_out_bcount_opt(n)
 205
 206 #define __checkReturn
 207
 208 #define __drv_when(_p, _c)
 209
 210 /* Code inclusion options */
 211
 212
 213 #define EFSYS_OPT_NAMES 1
 214
 215 #define EFSYS_OPT_FALCON 0
 216 #define EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0
 217 #define EFSYS_OPT_SIENA 1
 218 #ifdef DEBUG
 219 #define EFSYS_OPT_CHECK_REG 1
 220 #else
 221 #define EFSYS_OPT_CHECK_REG 0
 222 #endif
 223
 224 #define EFSYS_OPT_MCDI 1
 225
 226 #define EFSYS_OPT_MAC_FALCON_GMAC 0
 227 #define EFSYS_OPT_MAC_FALCON_XMAC 0
 228 #define EFSYS_OPT_MAC_STATS 1
 229
 230 #define EFSYS_OPT_LOOPBACK 0
 231
 232 #define EFSYS_OPT_MON_NULL 0
 233 #define EFSYS_OPT_MON_LM87 0
 234 #define EFSYS_OPT_MON_MAX6647 0
 235 #define EFSYS_OPT_MON_SIENA 0
 236 #define EFSYS_OPT_MON_STATS 0
 237
 238 #define EFSYS_OPT_PHY_NULL 0
 239 #define EFSYS_OPT_PHY_QT2022C2 0
 240 #define EFSYS_OPT_PHY_SFX7101 0
 241 #define EFSYS_OPT_PHY_TXC43128 0
 242 #define EFSYS_OPT_PHY_PM8358 0
 243 #define EFSYS_OPT_PHY_SFT9001 0
 244 #define EFSYS_OPT_PHY_QT2025C 0
 245 #define EFSYS_OPT_PHY_STATS 1
 246 #define EFSYS_OPT_PHY_PROPS 0
 247 #define EFSYS_OPT_PHY_BIST 1
 248 #define EFSYS_OPT_PHY_LED_CONTROL 1
 249 #define EFSYS_OPT_PHY_FLAGS 0
 250
 251 #define EFSYS_OPT_VPD 1
 252 #define EFSYS_OPT_NVRAM 1
 253 #define EFSYS_OPT_NVRAM_FALCON_BOOTROM 0
 254 #define EFSYS_OPT_NVRAM_SFT9001 0
 255 #define EFSYS_OPT_NVRAM_SFX7101 0
 256 #define EFSYS_OPT_BOOTCFG 0
 257
 258 #define EFSYS_OPT_PCIE_TUNE 0
 259 #define EFSYS_OPT_DIAG 0
 260 #define EFSYS_OPT_WOL 1
 261 #define EFSYS_OPT_RX_SCALE 1
 262 #define EFSYS_OPT_QSTATS 1
 263 #define EFSYS_OPT_FILTER 0
 264 #define EFSYS_OPT_RX_SCATTER 0
 265 #define EFSYS_OPT_RX_HDR_SPLIT 0
 266
 267 #define EFSYS_OPT_EV_PREFETCH 0
 268
 269 #define EFSYS_OPT_DECODE_INTR_FATAL 1
 270
 271 /* ID */
 272
 273 typedef struct __efsys_identifier_s     efsys_identifier_t;
 274
 275 /* PROBE */
 276
 277 #ifndef KDTRACE_HOOKS
 278
 279 #define EFSYS_PROBE(_name)
 280
 281 #define EFSYS_PROBE1(_name, _type1, _arg1)
 282
 283 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2)
 284
 285 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2,               \
 286             _type3, _arg3)
 287
 288 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2,               \
 289             _type3, _arg3, _type4, _arg4)
 290
 291 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,               \
 292             _type3, _arg3, _type4, _arg4, _type5, _arg5)
 293
 294 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,               \
 295             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 296             _type6, _arg6)
 297
 298 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2,               \
 299             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 300             _type6, _arg6, _type7, _arg7)
 301
 302 #else /* KDTRACE_HOOKS */
 303
 304 #define EFSYS_PROBE(_name)                                              \
 305         DTRACE_PROBE(_name)
 306
 307 #define EFSYS_PROBE1(_name, _type1, _arg1)                              \
 308         DTRACE_PROBE1(_name, _type1, _arg1)
 309
 310 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2)               \
 311         DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2)
 312
 313 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2,               \
 314             _type3, _arg3)                                              \
 315         DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2,              \
 316             _type3, _arg3)
 317
 318 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2,               \
 319             _type3, _arg3, _type4, _arg4)                               \
 320         DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2,              \
 321             _type3, _arg3, _type4, _arg4)
 322
 323 #ifdef DTRACE_PROBE5
 324 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,               \
 325             _type3, _arg3, _type4, _arg4, _type5, _arg5)                \
 326         DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2,              \
 327             _type3, _arg3, _type4, _arg4, _type5, _arg5)
 328 #else
 329 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,               \
 330             _type3, _arg3, _type4, _arg4, _type5, _arg5)                \
 331         DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2,              \
 332             _type3, _arg3, _type4, _arg4)
 333 #endif
 334
 335 #ifdef DTRACE_PROBE6
 336 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,               \
 337             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 338             _type6, _arg6)                                              \
 339         DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2,              \
 340             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 341             _type6, _arg6)
 342 #else
 343 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,               \
 344             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 345             _type6, _arg6)                                              \
 346         EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,               \
 347             _type3, _arg3, _type4, _arg4, _type5, _arg5)
 348 #endif
 349
 350 #ifdef DTRACE_PROBE7
 351 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2,               \
 352             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 353             _type6, _arg6, _type7, _arg7)                               \
 354         DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2,              \
 355             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 356             _type6, _arg6, _type7, _arg7)
 357 #else
 358 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2,               \
 359             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 360             _type6, _arg6, _type7, _arg7)                               \
 361         EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,               \
 362             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 363             _type6, _arg6)
 364 #endif
 365
 366 #endif /* KDTRACE_HOOKS */
 367
 368 /* DMA */
 369
 370 typedef uint64_t                efsys_dma_addr_t;
 371
 372 typedef struct efsys_mem_s {
 373         bus_dma_tag_t           esm_tag;
 374         bus_dmamap_t            esm_map;
 375         caddr_t                 esm_base;
 376         efsys_dma_addr_t        esm_addr;
 377 } efsys_mem_t;
 378
 379
 380 #define EFSYS_MEM_ZERO(_esmp, _size)                                    \
 381         do {                                                            \
 382                 (void) memset((_esmp)->esm_base, 0, (_size));           \
 383                                                                         \
 384         _NOTE(CONSTANTCONDITION)                                        \
 385         } while (B_FALSE)
 386
 387 #define EFSYS_MEM_READD(_esmp, _offset, _edp)                           \
 388         do {                                                            \
 389                 uint32_t *addr;                                         \
 390                                                                         \
 391                 _NOTE(CONSTANTCONDITION)                                \
 392                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),    \
 393                     ("not power of 2 aligned"));                        \
 394                                                                         \
 395                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 396                                                                         \
 397                 (_edp)->ed_u32[0] = *addr;                              \
 398                                                                         \
 399                 EFSYS_PROBE2(mem_readd, unsigned int, (_offset),        \
 400                     uint32_t, (_edp)->ed_u32[0]);                       \
 401                                                                         \
 402         _NOTE(CONSTANTCONDITION)                                        \
 403         } while (B_FALSE)
 404
 405 #if defined(__x86_64__)
 406 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp)                           \
 407         do {                                                            \
 408                 uint64_t *addr;                                         \
 409                                                                         \
 410                 _NOTE(CONSTANTCONDITION)                                \
 411                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),    \
 412                     ("not power of 2 aligned"));                        \
 413                                                                         \
 414                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 415                                                                         \
 416                 (_eqp)->eq_u64[0] = *addr;                              \
 417                                                                         \
 418                 EFSYS_PROBE3(mem_readq, unsigned int, (_offset),        \
 419                     uint32_t, (_eqp)->eq_u32[1],                        \
 420                     uint32_t, (_eqp)->eq_u32[0]);                       \
 421                                                                         \
 422         _NOTE(CONSTANTCONDITION)                                        \
 423         } while (B_FALSE)
 424 #else
 425 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp)                           \
 426         do {                                                            \
 427                 uint32_t *addr;                                         \
 428                                                                         \
 429                 _NOTE(CONSTANTCONDITION)                                \
 430                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),    \
 431                     ("not power of 2 aligned"));                        \
 432                                                                         \
 433                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 434                                                                         \
 435                 (_eqp)->eq_u32[0] = *addr++;                            \
 436                 (_eqp)->eq_u32[1] = *addr;                              \
 437                                                                         \
 438                 EFSYS_PROBE3(mem_readq, unsigned int, (_offset),        \
 439                     uint32_t, (_eqp)->eq_u32[1],                        \
 440                     uint32_t, (_eqp)->eq_u32[0]);                       \
 441                                                                         \
 442         _NOTE(CONSTANTCONDITION)                                        \
 443         } while (B_FALSE)
 444 #endif
 445
 446 #if defined(__x86_64__)
 447 #define EFSYS_MEM_READO(_esmp, _offset, _eop)                           \
 448         do {                                                            \
 449                 uint64_t *addr;                                         \
 450                                                                         \
 451                 _NOTE(CONSTANTCONDITION)                                \
 452                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),    \
 453                     ("not power of 2 aligned"));                        \
 454                                                                         \
 455                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 456                                                                         \
 457                 (_eop)->eo_u64[0] = *addr++;                            \
 458                 (_eop)->eo_u64[1] = *addr;                              \
 459                                                                         \
 460                 EFSYS_PROBE5(mem_reado, unsigned int, (_offset),        \
 461                     uint32_t, (_eop)->eo_u32[3],                        \
 462                     uint32_t, (_eop)->eo_u32[2],                        \
 463                     uint32_t, (_eop)->eo_u32[1],                        \
 464                     uint32_t, (_eop)->eo_u32[0]);                       \
 465                                                                         \
 466         _NOTE(CONSTANTCONDITION)                                        \
 467         } while (B_FALSE)
 468 #else
 469 #define EFSYS_MEM_READO(_esmp, _offset, _eop)                           \
 470         do {                                                            \
 471                 uint32_t *addr;                                         \
 472                                                                         \
 473                 _NOTE(CONSTANTCONDITION)                                \
 474                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),    \
 475                     ("not power of 2 aligned"));                        \
 476                                                                         \
 477                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 478                                                                         \
 479                 (_eop)->eo_u32[0] = *addr++;                            \
 480                 (_eop)->eo_u32[1] = *addr++;                            \
 481                 (_eop)->eo_u32[2] = *addr++;                            \
 482                 (_eop)->eo_u32[3] = *addr;                              \
 483                                                                         \
 484                 EFSYS_PROBE5(mem_reado, unsigned int, (_offset),        \
 485                     uint32_t, (_eop)->eo_u32[3],                        \
 486                     uint32_t, (_eop)->eo_u32[2],                        \
 487                     uint32_t, (_eop)->eo_u32[1],                        \
 488                     uint32_t, (_eop)->eo_u32[0]);                       \
 489                                                                         \
 490         _NOTE(CONSTANTCONDITION)                                        \
 491         } while (B_FALSE)
 492 #endif
 493
 494 #define EFSYS_MEM_WRITED(_esmp, _offset, _edp)                          \
 495         do {                                                            \
 496                 uint32_t *addr;                                         \
 497                                                                         \
 498                 _NOTE(CONSTANTCONDITION)                                \
 499                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),    \
 500                     ("not power of 2 aligned"));                        \
 501                                                                         \
 502                 EFSYS_PROBE2(mem_writed, unsigned int, (_offset),       \
 503                     uint32_t, (_edp)->ed_u32[0]);                       \
 504                                                                         \
 505                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 506                                                                         \
 507                 *addr = (_edp)->ed_u32[0];                              \
 508                                                                         \
 509         _NOTE(CONSTANTCONDITION)                                        \
 510         } while (B_FALSE)
 511
 512 #if defined(__x86_64__)
 513 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp)                          \
 514         do {                                                            \
 515                 uint64_t *addr;                                         \
 516                                                                         \
 517                 _NOTE(CONSTANTCONDITION)                                \
 518                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),    \
 519                     ("not power of 2 aligned"));                        \
 520                                                                         \
 521                 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset),       \
 522                     uint32_t, (_eqp)->eq_u32[1],                        \
 523                     uint32_t, (_eqp)->eq_u32[0]);                       \
 524                                                                         \
 525                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 526                                                                         \
 527                 *addr   = (_eqp)->eq_u64[0];                            \
 528                                                                         \
 529         _NOTE(CONSTANTCONDITION)                                        \
 530         } while (B_FALSE)
 531
 532 #else
 533 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp)                          \
 534         do {                                                            \
 535                 uint32_t *addr;                                         \
 536                                                                         \
 537                 _NOTE(CONSTANTCONDITION)                                \
 538                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),    \
 539                     ("not power of 2 aligned"));                        \
 540                                                                         \
 541                 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset),       \
 542                     uint32_t, (_eqp)->eq_u32[1],                        \
 543                     uint32_t, (_eqp)->eq_u32[0]);                       \
 544                                                                         \
 545                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 546                                                                         \
 547                 *addr++ = (_eqp)->eq_u32[0];                            \
 548                 *addr   = (_eqp)->eq_u32[1];                            \
 549                                                                         \
 550         _NOTE(CONSTANTCONDITION)                                        \
 551         } while (B_FALSE)
 552 #endif
 553
 554 #if defined(__x86_64__)
 555 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop)                          \
 556         do {                                                            \
 557                 uint64_t *addr;                                         \
 558                                                                         \
 559                 _NOTE(CONSTANTCONDITION)                                \
 560                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),    \
 561                     ("not power of 2 aligned"));                        \
 562                                                                         \
 563                 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset),       \
 564                     uint32_t, (_eop)->eo_u32[3],                        \
 565                     uint32_t, (_eop)->eo_u32[2],                        \
 566                     uint32_t, (_eop)->eo_u32[1],                        \
 567                     uint32_t, (_eop)->eo_u32[0]);                       \
 568                                                                         \
 569                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 570                                                                         \
 571                 *addr++ = (_eop)->eo_u64[0];                            \
 572                 *addr   = (_eop)->eo_u64[1];                            \
 573                                                                         \
 574         _NOTE(CONSTANTCONDITION)                                        \
 575         } while (B_FALSE)
 576 #else
 577 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop)                          \
 578         do {                                                            \
 579                 uint32_t *addr;                                         \
 580                                                                         \
 581                 _NOTE(CONSTANTCONDITION)                                \
 582                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),    \
 583                     ("not power of 2 aligned"));                        \
 584                                                                         \
 585                 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset),       \
 586                     uint32_t, (_eop)->eo_u32[3],                        \
 587                     uint32_t, (_eop)->eo_u32[2],                        \
 588                     uint32_t, (_eop)->eo_u32[1],                        \
 589                     uint32_t, (_eop)->eo_u32[0]);                       \
 590                                                                         \
 591                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 592                                                                         \
 593                 *addr++ = (_eop)->eo_u32[0];                            \
 594                 *addr++ = (_eop)->eo_u32[1];                            \
 595                 *addr++ = (_eop)->eo_u32[2];                            \
 596                 *addr   = (_eop)->eo_u32[3];                            \
 597                                                                         \
 598         _NOTE(CONSTANTCONDITION)                                        \
 599         } while (B_FALSE)
 600 #endif
 601
 602 #define EFSYS_MEM_ADDR(_esmp)                                           \
 603         ((_esmp)->esm_addr)
 604
 605 /* BAR */
 606
 607 #define SFXGE_LOCK_NAME_MAX     16
 608
 609 typedef struct efsys_bar_s {
 610         struct mtx              esb_lock;
 611         char                    esb_lock_name[SFXGE_LOCK_NAME_MAX];
 612         bus_space_tag_t         esb_tag;
 613         bus_space_handle_t      esb_handle;
 614         int                     esb_rid;
 615         struct resource         *esb_res;
 616 } efsys_bar_t;
 617
 618 #define SFXGE_BAR_LOCK_INIT(_esbp, _ifname)                             \
 619         do {                                                            \
 620                 snprintf((_esbp)->esb_lock_name,                        \
 621                          sizeof((_esbp)->esb_lock_name),                \
 622                          "%s:bar", (_ifname));                          \
 623                 mtx_init(&(_esbp)->esb_lock, (_esbp)->esb_lock_name,    \
 624                          NULL, MTX_DEF);                                \
 625         _NOTE(CONSTANTCONDITION)                                        \
 626         } while (B_FALSE)
 627 #define SFXGE_BAR_LOCK_DESTROY(_esbp)                                   \
 628         mtx_destroy(&(_esbp)->esb_lock)
 629 #define SFXGE_BAR_LOCK(_esbp)                                           \
 630         mtx_lock(&(_esbp)->esb_lock)
 631 #define SFXGE_BAR_UNLOCK(_esbp)                                         \
 632         mtx_unlock(&(_esbp)->esb_lock)
 633
 634 #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock)                    \
 635         do {                                                            \
 636                 _NOTE(CONSTANTCONDITION)                                \
 637                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),    \
 638                     ("not power of 2 aligned"));                        \
 639                                                                         \
 640                 _NOTE(CONSTANTCONDITION)                                \
 641                 if (_lock)                                              \
 642                         SFXGE_BAR_LOCK(_esbp);                          \
 643                                                                         \
 644                 (_edp)->ed_u32[0] = bus_space_read_4((_esbp)->esb_tag,  \
 645                     (_esbp)->esb_handle, (_offset));                    \
 646                                                                         \
 647                 EFSYS_PROBE2(bar_readd, unsigned int, (_offset),        \
 648                     uint32_t, (_edp)->ed_u32[0]);                       \
 649                                                                         \
 650                 _NOTE(CONSTANTCONDITION)                                \
 651                 if (_lock)                                              \
 652                         SFXGE_BAR_UNLOCK(_esbp);                        \
 653         _NOTE(CONSTANTCONDITION)                                        \
 654         } while (B_FALSE)
 655
 656 #if defined(__x86_64__)
 657 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp)                           \
 658         do {                                                            \
 659                 _NOTE(CONSTANTCONDITION)                                \
 660                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),    \
 661                     ("not power of 2 aligned"));                        \
 662                                                                         \
 663                 SFXGE_BAR_LOCK(_esbp);                                  \
 664                                                                         \
 665                 (_eqp)->eq_u64[0] = bus_space_read_8((_esbp)->esb_tag,  \
 666                     (_esbp)->esb_handle, (_offset));                    \
 667                                                                         \
 668                 EFSYS_PROBE3(bar_readq, unsigned int, (_offset),        \
 669                     uint32_t, (_eqp)->eq_u32[1],                        \
 670                     uint32_t, (_eqp)->eq_u32[0]);                       \
 671                                                                         \
 672                 mtx_unlock(&((_esbp)->esb_lock));                       \
 673         _NOTE(CONSTANTCONDITION)                                        \
 674         } while (B_FALSE)
 675
 676 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock)                    \
 677         do {                                                            \
 678                 _NOTE(CONSTANTCONDITION)                                \
 679                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),    \
 680                     ("not power of 2 aligned"));                        \
 681                                                                         \
 682                 _NOTE(CONSTANTCONDITION)                                \
 683                 if (_lock)                                              \
 684                         mtx_lock(&((_esbp)->esb_lock));                 \
 685                                                                         \
 686                 (_eop)->eo_u64[0] = bus_space_read_8((_esbp)->esb_tag,  \
 687                     (_esbp)->esb_handle, (_offset));                    \
 688                 (_eop)->eo_u64[1] = bus_space_read_8((_esbp)->esb_tag,  \
 689                     (_esbp)->esb_handle, (_offset+8));                  \
 690                                                                         \
 691                 EFSYS_PROBE5(bar_reado, unsigned int, (_offset),        \
 692                     uint32_t, (_eop)->eo_u32[3],                        \
 693                     uint32_t, (_eop)->eo_u32[2],                        \
 694                     uint32_t, (_eop)->eo_u32[1],                        \
 695                     uint32_t, (_eop)->eo_u32[0]);                       \
 696                                                                         \
 697                 _NOTE(CONSTANTCONDITION)                                \
 698                 if (_lock)                                              \
 699                         mtx_unlock(&((_esbp)->esb_lock));               \
 700         _NOTE(CONSTANTCONDITION)                                        \
 701         } while (B_FALSE)
 702
 703 #else
 704 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp)                           \
 705         do {                                                            \
 706                 _NOTE(CONSTANTCONDITION)                                \
 707                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),    \
 708                     ("not power of 2 aligned"));                        \
 709                                                                         \
 710                 mtx_lock(&((_esbp)->esb_lock));                         \
 711                                                                         \
 712                 (_eqp)->eq_u32[0] = bus_space_read_4((_esbp)->esb_tag,  \
 713                     (_esbp)->esb_handle, (_offset));                    \
 714                 (_eqp)->eq_u32[1] = bus_space_read_4((_esbp)->esb_tag,  \
 715                     (_esbp)->esb_handle, (_offset+4));                  \
 716                                                                         \
 717                 EFSYS_PROBE3(bar_readq, unsigned int, (_offset),        \
 718                     uint32_t, (_eqp)->eq_u32[1],                        \
 719                     uint32_t, (_eqp)->eq_u32[0]);                       \
 720                                                                         \
 721                 SFXGE_BAR_UNLOCK(_esbp);                                \
 722         _NOTE(CONSTANTCONDITION)                                        \
 723         } while (B_FALSE)
 724
 725 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock)                    \
 726         do {                                                            \
 727                 _NOTE(CONSTANTCONDITION)                                \
 728                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),    \
 729                     ("not power of 2 aligned"));                        \
 730                                                                         \
 731                 _NOTE(CONSTANTCONDITION)                                \
 732                 if (_lock)                                              \
 733                         SFXGE_BAR_LOCK(_esbp);                          \
 734                                                                         \
 735                 (_eop)->eo_u32[0] = bus_space_read_4((_esbp)->esb_tag,  \
 736                     (_esbp)->esb_handle, (_offset));                    \
 737                 (_eop)->eo_u32[1] = bus_space_read_4((_esbp)->esb_tag,  \
 738                     (_esbp)->esb_handle, (_offset+4));                  \
 739                 (_eop)->eo_u32[2] = bus_space_read_4((_esbp)->esb_tag,  \
 740                     (_esbp)->esb_handle, (_offset+8));                  \
 741                 (_eop)->eo_u32[3] = bus_space_read_4((_esbp)->esb_tag,  \
 742                     (_esbp)->esb_handle, (_offset+12));                 \
 743                                                                         \
 744                 EFSYS_PROBE5(bar_reado, unsigned int, (_offset),        \
 745                     uint32_t, (_eop)->eo_u32[3],                        \
 746                     uint32_t, (_eop)->eo_u32[2],                        \
 747                     uint32_t, (_eop)->eo_u32[1],                        \
 748                     uint32_t, (_eop)->eo_u32[0]);                       \
 749                                                                         \
 750                 _NOTE(CONSTANTCONDITION)                                \
 751                 if (_lock)                                              \
 752                         SFXGE_BAR_UNLOCK(_esbp);                        \
 753         _NOTE(CONSTANTCONDITION)                                        \
 754         } while (B_FALSE)
 755 #endif
 756
 757 #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock)                   \
 758         do {                                                            \
 759                 _NOTE(CONSTANTCONDITION)                                \
 760                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),    \
 761                     ("not power of 2 aligned"));                        \
 762                                                                         \
 763                 _NOTE(CONSTANTCONDITION)                                \
 764                 if (_lock)                                              \
 765                         SFXGE_BAR_LOCK(_esbp);                  \
 766                                                                         \
 767                 EFSYS_PROBE2(bar_writed, unsigned int, (_offset),       \
 768                     uint32_t, (_edp)->ed_u32[0]);                       \
 769                                                                         \
 770                 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
 771                     (_offset), (_edp)->ed_u32[0]);                      \
 772                                                                         \
 773                 _NOTE(CONSTANTCONDITION)                                \
 774                 if (_lock)                                              \
 775                         SFXGE_BAR_UNLOCK(_esbp);                \
 776         _NOTE(CONSTANTCONDITION)                                        \
 777         } while (B_FALSE)
 778
 779 #if defined(__x86_64__)
 780 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp)                          \
 781         do {                                                            \
 782                 _NOTE(CONSTANTCONDITION)                                \
 783                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),    \
 784                     ("not power of 2 aligned"));                        \
 785                                                                         \
 786                 SFXGE_BAR_LOCK(_esbp);                                  \
 787                                                                         \
 788                 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset),       \
 789                     uint32_t, (_eqp)->eq_u32[1],                        \
 790                     uint32_t, (_eqp)->eq_u32[0]);                       \
 791                                                                         \
 792                 bus_space_write_8((_esbp)->esb_tag, (_esbp)->esb_handle,\
 793                     (_offset), (_eqp)->eq_u64[0]);                      \
 794                                                                         \
 795                 mtx_unlock(&((_esbp)->esb_lock));                       \
 796         _NOTE(CONSTANTCONDITION)                                        \
 797         } while (B_FALSE)
 798 #else
 799 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp)                          \
 800         do {                                                            \
 801                 _NOTE(CONSTANTCONDITION)                                \
 802                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),    \
 803                     ("not power of 2 aligned"));                        \
 804                                                                         \
 805                 mtx_lock(&((_esbp)->esb_lock));                         \
 806                                                                         \
 807                 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset),       \
 808                     uint32_t, (_eqp)->eq_u32[1],                        \
 809                     uint32_t, (_eqp)->eq_u32[0]);                       \
 810                                                                         \
 811                 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
 812                     (_offset), (_eqp)->eq_u32[0]);                      \
 813                 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
 814                     (_offset+4), (_eqp)->eq_u32[1]);                    \
 815                                                                         \
 816                 SFXGE_BAR_UNLOCK(_esbp);                                \
 817         _NOTE(CONSTANTCONDITION)                                        \
 818         } while (B_FALSE)
 819 #endif
 820
 821 #if defined(__x86_64__)
 822 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock)                   \
 823         do {                                                            \
 824                 _NOTE(CONSTANTCONDITION)                                \
 825                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),    \
 826                     ("not power of 2 aligned"));                        \
 827                                                                         \
 828                 _NOTE(CONSTANTCONDITION)                                \
 829                 if (_lock)                                              \
 830                         SFXGE_BAR_LOCK(_esbp);                          \
 831                                                                         \
 832                 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset),       \
 833                     uint32_t, (_eop)->eo_u32[3],                        \
 834                     uint32_t, (_eop)->eo_u32[2],                        \
 835                     uint32_t, (_eop)->eo_u32[1],                        \
 836                     uint32_t, (_eop)->eo_u32[0]);                       \
 837                                                                         \
 838                 bus_space_write_8((_esbp)->esb_tag, (_esbp)->esb_handle,\
 839                     (_offset), (_eop)->eo_u64[0]);                      \
 840                 bus_space_write_8((_esbp)->esb_tag, (_esbp)->esb_handle,\
 841                     (_offset+8), (_eop)->eo_u64[1]);                    \
 842                                                                         \
 843                 _NOTE(CONSTANTCONDITION)                                \
 844                 if (_lock)                                              \
 845                         mtx_unlock(&((_esbp)->esb_lock));               \
 846         _NOTE(CONSTANTCONDITION)                                        \
 847         } while (B_FALSE)
 848
 849 #else
 850 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock)                   \
 851         do {                                                            \
 852                 _NOTE(CONSTANTCONDITION)                                \
 853                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),    \
 854                     ("not power of 2 aligned"));                        \
 855                                                                         \
 856                 _NOTE(CONSTANTCONDITION)                                \
 857                 if (_lock)                                              \
 858                         mtx_lock(&((_esbp)->esb_lock));                 \
 859                                                                         \
 860                 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset),       \
 861                     uint32_t, (_eop)->eo_u32[3],                        \
 862                     uint32_t, (_eop)->eo_u32[2],                        \
 863                     uint32_t, (_eop)->eo_u32[1],                        \
 864                     uint32_t, (_eop)->eo_u32[0]);                       \
 865                                                                         \
 866                 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
 867                     (_offset), (_eop)->eo_u32[0]);                      \
 868                 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
 869                     (_offset+4), (_eop)->eo_u32[1]);                    \
 870                 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
 871                     (_offset+8), (_eop)->eo_u32[2]);                    \
 872                 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
 873                     (_offset+12), (_eop)->eo_u32[3]);                   \
 874                                                                         \
 875                 _NOTE(CONSTANTCONDITION)                                \
 876                 if (_lock)                                              \
 877                         SFXGE_BAR_UNLOCK(_esbp);                        \
 878         _NOTE(CONSTANTCONDITION)                                        \
 879         } while (B_FALSE)
 880 #endif
 881
 882 /* SPIN */
 883
 884 #define EFSYS_SPIN(_us)                                                 \
 885         do {                                                            \
 886                 DELAY(_us);                                             \
 887         _NOTE(CONSTANTCONDITION)                                        \
 888         } while (B_FALSE)
 889
 890 #define EFSYS_SLEEP     EFSYS_SPIN
 891
 892 /* BARRIERS */
 893
 894 #define EFSYS_MEM_READ_BARRIER()        rmb()
 895 #define EFSYS_PIO_WRITE_BARRIER()
 896
 897 /* TIMESTAMP */
 898
 899 typedef clock_t efsys_timestamp_t;
 900
 901 #define EFSYS_TIMESTAMP(_usp)                                           \
 902         do {                                                            \
 903                 clock_t now;                                            \
 904                                                                         \
 905                 now = ticks;                                            \
 906                 *(_usp) = now * hz / 1000000;                           \
 907         _NOTE(CONSTANTCONDITION)                                        \
 908         } while (B_FALSE)
 909
 910 /* KMEM */
 911
 912 #define EFSYS_KMEM_ALLOC(_esip, _size, _p)                              \
 913         do {                                                            \
 914                 (_esip) = (_esip);                                      \
 915                 (_p) = malloc((_size), M_SFXGE, M_WAITOK|M_ZERO);       \
 916         _NOTE(CONSTANTCONDITION)                                        \
 917         } while (B_FALSE)
 918
 919 #define EFSYS_KMEM_FREE(_esip, _size, _p)                               \
 920         do {                                                            \
 921                 (void) (_esip);                                         \
 922                 (void) (_size);                                         \
 923                 free((_p), M_SFXGE);                                    \
 924         _NOTE(CONSTANTCONDITION)                                        \
 925         } while (B_FALSE)
 926
 927 /* LOCK */
 928
 929 typedef struct efsys_lock_s {
 930         struct mtx      lock;
 931         char            lock_name[SFXGE_LOCK_NAME_MAX];
 932 } efsys_lock_t;
 933
 934 #define SFXGE_EFSYS_LOCK_INIT(_eslp, _ifname, _label)                   \
 935         do {                                                            \
 936                 efsys_lock_t *__eslp = (_eslp);                         \
 937                                                                         \
 938                 snprintf((__eslp)->lock_name,                           \
 939                          sizeof((__eslp)->lock_name),                   \
 940                          "%s:%s", (_ifname), (_label));                 \
 941                 mtx_init(&(__eslp)->lock, (__eslp)->lock_name,          \
 942                          NULL, MTX_DEF);                                \
 943         } while (B_FALSE)
 944 #define SFXGE_EFSYS_LOCK_DESTROY(_eslp)                                 \
 945         mtx_destroy(&(_eslp)->lock)
 946 #define SFXGE_EFSYS_LOCK(_eslp)                                         \
 947         mtx_lock(&(_eslp)->lock)
 948 #define SFXGE_EFSYS_UNLOCK(_eslp)                                       \
 949         mtx_unlock(&(_eslp)->lock)
 950 #define SFXGE_EFSYS_LOCK_ASSERT_OWNED(_eslp)                            \
 951         mtx_assert(&(_eslp)->lock, MA_OWNED)
 952
 953 #define EFSYS_LOCK_MAGIC        0x000010c4
 954
 955 #define EFSYS_LOCK(_lockp, _state)                                      \
 956         do {                                                            \
 957                 SFXGE_EFSYS_LOCK(_lockp);                               \
 958                 (_state) = EFSYS_LOCK_MAGIC;                            \
 959         _NOTE(CONSTANTCONDITION)                                        \
 960         } while (B_FALSE)
 961
 962 #define EFSYS_UNLOCK(_lockp, _state)                                    \
 963         do {                                                            \
 964                 if ((_state) != EFSYS_LOCK_MAGIC)                       \
 965                         KASSERT(B_FALSE, ("not locked"));               \
 966                 SFXGE_EFSYS_UNLOCK(_lockp);                             \
 967         _NOTE(CONSTANTCONDITION)                                        \
 968         } while (B_FALSE)
 969
 970 /* PREEMPT */
 971
 972 #define EFSYS_PREEMPT_DISABLE(_state)                                   \
 973         do {                                                            \
 974                 (_state) = (_state);                                    \
 975                 critical_enter();                                       \
 976         _NOTE(CONSTANTCONDITION)                                        \
 977         } while (B_FALSE)
 978
 979 #define EFSYS_PREEMPT_ENABLE(_state)                                    \
 980         do {                                                            \
 981                 (_state) = (_state);                                    \
 982                 critical_exit(_state);                                  \
 983         _NOTE(CONSTANTCONDITION)                                        \
 984         } while (B_FALSE)
 985
 986 /* STAT */
 987
 988 typedef uint64_t                efsys_stat_t;
 989
 990 #define EFSYS_STAT_INCR(_knp, _delta)                                   \
 991         do {                                                            \
 992                 *(_knp) += (_delta);                                    \
 993         _NOTE(CONSTANTCONDITION)                                        \
 994         } while (B_FALSE)
 995
 996 #define EFSYS_STAT_DECR(_knp, _delta)                                   \
 997         do {                                                            \
 998                 *(_knp) -= (_delta);                                    \
 999         _NOTE(CONSTANTCONDITION)                                        \
1000         } while (B_FALSE)
1001
1002 #define EFSYS_STAT_SET(_knp, _val)                                      \
1003         do {                                                            \
1004                 *(_knp) = (_val);                                       \
1005         _NOTE(CONSTANTCONDITION)                                        \
1006         } while (B_FALSE)
1007
1008 #define EFSYS_STAT_SET_QWORD(_knp, _valp)                               \
1009         do {                                                            \
1010                 *(_knp) = le64toh((_valp)->eq_u64[0]);                  \
1011         _NOTE(CONSTANTCONDITION)                                        \
1012         } while (B_FALSE)
1013
1014 #define EFSYS_STAT_SET_DWORD(_knp, _valp)                               \
1015         do {                                                            \
1016                 *(_knp) = le32toh((_valp)->ed_u32[0]);                  \
1017         _NOTE(CONSTANTCONDITION)                                        \
1018         } while (B_FALSE)
1019
1020 #define EFSYS_STAT_INCR_QWORD(_knp, _valp)                              \
1021         do {                                                            \
1022                 *(_knp) += le64toh((_valp)->eq_u64[0]);                 \
1023         _NOTE(CONSTANTCONDITION)                                        \
1024         } while (B_FALSE)
1025
1026 #define EFSYS_STAT_SUBR_QWORD(_knp, _valp)                              \
1027         do {                                                            \
1028                 *(_knp) -= le64toh((_valp)->eq_u64[0]);                 \
1029         _NOTE(CONSTANTCONDITION)                                        \
1030         } while (B_FALSE)
1031
1032 /* ERR */
1033
1034 extern void     sfxge_err(efsys_identifier_t *, unsigned int,
1035                     uint32_t, uint32_t);
1036
1037 #if EFSYS_OPT_DECODE_INTR_FATAL
1038 #define EFSYS_ERR(_esip, _code, _dword0, _dword1)                       \
1039         do {                                                            \
1040                 sfxge_err((_esip), (_code), (_dword0), (_dword1));      \
1041         _NOTE(CONSTANTCONDITION)                                        \
1042         } while (B_FALSE)
1043 #endif
1044
1045 /* ASSERT */
1046
1047 #define EFSYS_ASSERT(_exp) do {                                         \
1048         if (!(_exp))                                                    \
1049                 panic(#_exp);                                           \
1050         } while (0)
1051
1052 #define EFSYS_ASSERT3(_x, _op, _y, _t) do {                             \
1053         const _t __x = (_t)(_x);                                        \
1054         const _t __y = (_t)(_y);                                        \
1055         if (!(__x _op __y))                                             \
1056                 panic("assertion failed at %s:%u", __FILE__, __LINE__); \
1057         } while(0)
1058
1059 #define EFSYS_ASSERT3U(_x, _op, _y)     EFSYS_ASSERT3(_x, _op, _y, uint64_t)
1060 #define EFSYS_ASSERT3S(_x, _op, _y)     EFSYS_ASSERT3(_x, _op, _y, int64_t)
1061 #define EFSYS_ASSERT3P(_x, _op, _y)     EFSYS_ASSERT3(_x, _op, _y, uintptr_t)
1062
1063 #ifdef  __cplusplus
1064 }
1065 #endif
1066
1067 #endif  /* _SYS_EFSYS_H */