sys/dev/sfxge/common/efsys.h

   1 /*-
   2  * Copyright (c) 2010-2011 Solarflare Communications, Inc.
   3  * All rights reserved.
   4  *
   5  * This software was developed in part by Philip Paeps under contract for
   6  * Solarflare Communications, Inc.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  * $FreeBSD$
  30  */
  31
  32 #ifndef _SYS_EFSYS_H
  33 #define _SYS_EFSYS_H
  34
  35 #ifdef  __cplusplus
  36 extern "C" {
  37 #endif
  38
  39 #include <sys/param.h>
  40 #include <sys/bus.h>
  41 #include <sys/endian.h>
  42 #include <sys/lock.h>
  43 #include <sys/malloc.h>
  44 #include <sys/mbuf.h>
  45 #include <sys/mutex.h>
  46 #include <sys/rwlock.h>
  47 #include <sys/sdt.h>
  48 #include <sys/systm.h>
  49
  50 #include <machine/bus.h>
  51 #include <machine/endian.h>
  52
  53 #define EFSYS_HAS_UINT64 1
  54 #if defined(__x86_64__)
  55 #define EFSYS_USE_UINT64 1
  56 #else
  57 #define EFSYS_USE_UINT64 0
  58 #endif
  59 #if _BYTE_ORDER == _BIG_ENDIAN
  60 #define EFSYS_IS_BIG_ENDIAN 1
  61 #define EFSYS_IS_LITTLE_ENDIAN 0
  62 #elif _BYTE_ORDER == _LITTLE_ENDIAN
  63 #define EFSYS_IS_BIG_ENDIAN 0
  64 #define EFSYS_IS_LITTLE_ENDIAN 1
  65 #endif
  66 #include "efx_types.h"
  67
  68 /* Common code requires this */
  69 #if __FreeBSD_version < 800068
  70 #define memmove(d, s, l) bcopy(s, d, l)
  71 #endif
  72
  73 /* FreeBSD equivalents of Solaris things */
  74 #ifndef _NOTE
  75 #define _NOTE(s)
  76 #endif
  77
  78 #ifndef B_FALSE
  79 #define B_FALSE FALSE
  80 #endif
  81 #ifndef B_TRUE
  82 #define B_TRUE  TRUE
  83 #endif
  84
  85 #ifndef IS_P2ALIGNED
  86 #define IS_P2ALIGNED(v, a)      ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0)
  87 #endif
  88
  89 #ifndef P2ROUNDUP
  90 #define P2ROUNDUP(x, align)     (-(-(x) & -(align)))
  91 #endif
  92
  93 #ifndef IS2P
  94 #define ISP2(x)                 (((x) & ((x) - 1)) == 0)
  95 #endif
  96
  97 #if defined(__x86_64__)
  98 #if !defined(bus_space_read_stream_8)
  99 #define bus_space_read_stream_8(t, h, o)                                \
 100         bus_space_read_8((t), (h), (o))
 101 #define bus_space_write_stream_8(t, h, o, v)                            \
 102         bus_space_write_8((t), (h), (o), (v))
 103 #endif
 104 #endif
 105
 106 #define ENOTACTIVE EINVAL
 107
 108 /* Memory type to use on FreeBSD */
 109 MALLOC_DECLARE(M_SFXGE);
 110
 111 /* Machine dependend prefetch wrappers */
 112 #if defined(__i386__) || defined(__amd64__)
 113 static __inline void
 114 prefetch_read_many(void *addr)
 115 {
 116
 117         __asm__(
 118             "prefetcht0 (%0)"
 119             :
 120             : "r" (addr));
 121 }
 122
 123 static __inline void
 124 prefetch_read_once(void *addr)
 125 {
 126
 127         __asm__(
 128             "prefetchnta (%0)"
 129             :
 130             : "r" (addr));
 131 }
 132 #elif defined(__sparc64__)
 133 static __inline void
 134 prefetch_read_many(void *addr)
 135 {
 136
 137         __asm__(
 138             "prefetch [%0], 0"
 139             :
 140             : "r" (addr));
 141 }
 142
 143 static __inline void
 144 prefetch_read_once(void *addr)
 145 {
 146
 147         __asm__(
 148             "prefetch [%0], 1"
 149             :
 150             : "r" (addr));
 151 }
 152 #else
 153 static __inline void
 154 prefetch_read_many(void *addr)
 155 {
 156
 157 }
 158
 159 static __inline void
 160 prefetch_read_once(void *addr)
 161 {
 162
 163 }
 164 #endif
 165
 166 #if defined(__i386__) || defined(__amd64__)
 167 #include <vm/vm.h>
 168 #include <vm/pmap.h>
 169 #endif
 170 static __inline void
 171 sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map,
 172     struct mbuf *m, bus_dma_segment_t *seg)
 173 {
 174 #if defined(__i386__) || defined(__amd64__)
 175         seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t));
 176         seg->ds_len = m->m_len;
 177 #else
 178         int nsegstmp;
 179
 180         bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0);
 181 #endif
 182 }
 183
 184 /* Modifiers used for DOS builds */
 185 #define __cs
 186 #define __far
 187
 188 /* Modifiers used for Windows builds */
 189 #define __in
 190 #define __in_opt
 191 #define __in_ecount(_n)
 192 #define __in_ecount_opt(_n)
 193 #define __in_bcount(_n)
 194 #define __in_bcount_opt(_n)
 195
 196 #define __out
 197 #define __out_opt
 198 #define __out_ecount(_n)
 199 #define __out_ecount_opt(_n)
 200 #define __out_bcount(_n)
 201 #define __out_bcount_opt(_n)
 202
 203 #define __deref_out
 204
 205 #define __inout
 206 #define __inout_opt
 207 #define __inout_ecount(_n)
 208 #define __inout_ecount_opt(_n)
 209 #define __inout_bcount(_n)
 210 #define __inout_bcount_opt(_n)
 211 #define __inout_bcount_full_opt(_n)
 212
 213 #define __deref_out_bcount_opt(n)
 214
 215 #define __checkReturn
 216
 217 #define __drv_when(_p, _c)
 218
 219 /* Code inclusion options */
 220
 221
 222 #define EFSYS_OPT_NAMES 1
 223
 224 #define EFSYS_OPT_FALCON 0
 225 #define EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0
 226 #define EFSYS_OPT_SIENA 1
 227 #ifdef DEBUG
 228 #define EFSYS_OPT_CHECK_REG 1
 229 #else
 230 #define EFSYS_OPT_CHECK_REG 0
 231 #endif
 232
 233 #define EFSYS_OPT_MCDI 1
 234
 235 #define EFSYS_OPT_MAC_FALCON_GMAC 0
 236 #define EFSYS_OPT_MAC_FALCON_XMAC 0
 237 #define EFSYS_OPT_MAC_STATS 1
 238
 239 #define EFSYS_OPT_LOOPBACK 0
 240
 241 #define EFSYS_OPT_MON_NULL 0
 242 #define EFSYS_OPT_MON_LM87 0
 243 #define EFSYS_OPT_MON_MAX6647 0
 244 #define EFSYS_OPT_MON_SIENA 0
 245 #define EFSYS_OPT_MON_STATS 0
 246
 247 #define EFSYS_OPT_PHY_NULL 0
 248 #define EFSYS_OPT_PHY_QT2022C2 0
 249 #define EFSYS_OPT_PHY_SFX7101 0
 250 #define EFSYS_OPT_PHY_TXC43128 0
 251 #define EFSYS_OPT_PHY_PM8358 0
 252 #define EFSYS_OPT_PHY_SFT9001 0
 253 #define EFSYS_OPT_PHY_QT2025C 0
 254 #define EFSYS_OPT_PHY_STATS 1
 255 #define EFSYS_OPT_PHY_PROPS 0
 256 #define EFSYS_OPT_PHY_BIST 1
 257 #define EFSYS_OPT_PHY_LED_CONTROL 1
 258 #define EFSYS_OPT_PHY_FLAGS 0
 259
 260 #define EFSYS_OPT_VPD 1
 261 #define EFSYS_OPT_NVRAM 1
 262 #define EFSYS_OPT_NVRAM_FALCON_BOOTROM 0
 263 #define EFSYS_OPT_NVRAM_SFT9001 0
 264 #define EFSYS_OPT_NVRAM_SFX7101 0
 265 #define EFSYS_OPT_BOOTCFG 0
 266
 267 #define EFSYS_OPT_PCIE_TUNE 0
 268 #define EFSYS_OPT_DIAG 0
 269 #define EFSYS_OPT_WOL 1
 270 #define EFSYS_OPT_RX_SCALE 1
 271 #define EFSYS_OPT_QSTATS 1
 272 #define EFSYS_OPT_FILTER 0
 273 #define EFSYS_OPT_RX_SCATTER 0
 274 #define EFSYS_OPT_RX_HDR_SPLIT 0
 275
 276 #define EFSYS_OPT_EV_PREFETCH 0
 277
 278 #define EFSYS_OPT_DECODE_INTR_FATAL 1
 279
 280 /* ID */
 281
 282 typedef struct __efsys_identifier_s     efsys_identifier_t;
 283
 284 /* PROBE */
 285
 286 #ifndef KDTRACE_HOOKS
 287
 288 #define EFSYS_PROBE(_name)
 289
 290 #define EFSYS_PROBE1(_name, _type1, _arg1)
 291
 292 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2)
 293
 294 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2,               \
 295             _type3, _arg3)
 296
 297 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2,               \
 298             _type3, _arg3, _type4, _arg4)
 299
 300 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,               \
 301             _type3, _arg3, _type4, _arg4, _type5, _arg5)
 302
 303 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,               \
 304             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 305             _type6, _arg6)
 306
 307 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2,               \
 308             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 309             _type6, _arg6, _type7, _arg7)
 310
 311 #else /* KDTRACE_HOOKS */
 312
 313 #define EFSYS_PROBE(_name)                                              \
 314         DTRACE_PROBE(_name)
 315
 316 #define EFSYS_PROBE1(_name, _type1, _arg1)                              \
 317         DTRACE_PROBE1(_name, _type1, _arg1)
 318
 319 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2)               \
 320         DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2)
 321
 322 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2,               \
 323             _type3, _arg3)                                              \
 324         DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2,              \
 325             _type3, _arg3)
 326
 327 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2,               \
 328             _type3, _arg3, _type4, _arg4)                               \
 329         DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2,              \
 330             _type3, _arg3, _type4, _arg4)
 331
 332 #ifdef DTRACE_PROBE5
 333 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,               \
 334             _type3, _arg3, _type4, _arg4, _type5, _arg5)                \
 335         DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2,              \
 336             _type3, _arg3, _type4, _arg4, _type5, _arg5)
 337 #else
 338 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,               \
 339             _type3, _arg3, _type4, _arg4, _type5, _arg5)                \
 340         DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2,              \
 341             _type3, _arg3, _type4, _arg4)
 342 #endif
 343
 344 #ifdef DTRACE_PROBE6
 345 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,               \
 346             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 347             _type6, _arg6)                                              \
 348         DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2,              \
 349             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 350             _type6, _arg6)
 351 #else
 352 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,               \
 353             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 354             _type6, _arg6)                                              \
 355         EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,               \
 356             _type3, _arg3, _type4, _arg4, _type5, _arg5)
 357 #endif
 358
 359 #ifdef DTRACE_PROBE7
 360 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2,               \
 361             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 362             _type6, _arg6, _type7, _arg7)                               \
 363         DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2,              \
 364             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 365             _type6, _arg6, _type7, _arg7)
 366 #else
 367 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2,               \
 368             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 369             _type6, _arg6, _type7, _arg7)                               \
 370         EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,               \
 371             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 372             _type6, _arg6)
 373 #endif
 374
 375 #endif /* KDTRACE_HOOKS */
 376
 377 /* DMA */
 378
 379 typedef uint64_t                efsys_dma_addr_t;
 380
 381 typedef struct efsys_mem_s {
 382         bus_dma_tag_t           esm_tag;
 383         bus_dmamap_t            esm_map;
 384         caddr_t                 esm_base;
 385         efsys_dma_addr_t        esm_addr;
 386 } efsys_mem_t;
 387
 388
 389 #define EFSYS_MEM_ZERO(_esmp, _size)                                    \
 390         do {                                                            \
 391                 (void) memset((_esmp)->esm_base, 0, (_size));           \
 392                                                                         \
 393         _NOTE(CONSTANTCONDITION)                                        \
 394         } while (B_FALSE)
 395
 396 #define EFSYS_MEM_READD(_esmp, _offset, _edp)                           \
 397         do {                                                            \
 398                 uint32_t *addr;                                         \
 399                                                                         \
 400                 _NOTE(CONSTANTCONDITION)                                \
 401                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),    \
 402                     ("not power of 2 aligned"));                        \
 403                                                                         \
 404                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 405                                                                         \
 406                 (_edp)->ed_u32[0] = *addr;                              \
 407                                                                         \
 408                 EFSYS_PROBE2(mem_readd, unsigned int, (_offset),        \
 409                     uint32_t, (_edp)->ed_u32[0]);                       \
 410                                                                         \
 411         _NOTE(CONSTANTCONDITION)                                        \
 412         } while (B_FALSE)
 413
 414 #if defined(__x86_64__)
 415 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp)                           \
 416         do {                                                            \
 417                 uint64_t *addr;                                         \
 418                                                                         \
 419                 _NOTE(CONSTANTCONDITION)                                \
 420                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),    \
 421                     ("not power of 2 aligned"));                        \
 422                                                                         \
 423                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 424                                                                         \
 425                 (_eqp)->eq_u64[0] = *addr;                              \
 426                                                                         \
 427                 EFSYS_PROBE3(mem_readq, unsigned int, (_offset),        \
 428                     uint32_t, (_eqp)->eq_u32[1],                        \
 429                     uint32_t, (_eqp)->eq_u32[0]);                       \
 430                                                                         \
 431         _NOTE(CONSTANTCONDITION)                                        \
 432         } while (B_FALSE)
 433 #else
 434 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp)                           \
 435         do {                                                            \
 436                 uint32_t *addr;                                         \
 437                                                                         \
 438                 _NOTE(CONSTANTCONDITION)                                \
 439                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),    \
 440                     ("not power of 2 aligned"));                        \
 441                                                                         \
 442                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 443                                                                         \
 444                 (_eqp)->eq_u32[0] = *addr++;                            \
 445                 (_eqp)->eq_u32[1] = *addr;                              \
 446                                                                         \
 447                 EFSYS_PROBE3(mem_readq, unsigned int, (_offset),        \
 448                     uint32_t, (_eqp)->eq_u32[1],                        \
 449                     uint32_t, (_eqp)->eq_u32[0]);                       \
 450                                                                         \
 451         _NOTE(CONSTANTCONDITION)                                        \
 452         } while (B_FALSE)
 453 #endif
 454
 455 #if defined(__x86_64__)
 456 #define EFSYS_MEM_READO(_esmp, _offset, _eop)                           \
 457         do {                                                            \
 458                 uint64_t *addr;                                         \
 459                                                                         \
 460                 _NOTE(CONSTANTCONDITION)                                \
 461                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),    \
 462                     ("not power of 2 aligned"));                        \
 463                                                                         \
 464                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 465                                                                         \
 466                 (_eop)->eo_u64[0] = *addr++;                            \
 467                 (_eop)->eo_u64[1] = *addr;                              \
 468                                                                         \
 469                 EFSYS_PROBE5(mem_reado, unsigned int, (_offset),        \
 470                     uint32_t, (_eop)->eo_u32[3],                        \
 471                     uint32_t, (_eop)->eo_u32[2],                        \
 472                     uint32_t, (_eop)->eo_u32[1],                        \
 473                     uint32_t, (_eop)->eo_u32[0]);                       \
 474                                                                         \
 475         _NOTE(CONSTANTCONDITION)                                        \
 476         } while (B_FALSE)
 477 #else
 478 #define EFSYS_MEM_READO(_esmp, _offset, _eop)                           \
 479         do {                                                            \
 480                 uint32_t *addr;                                         \
 481                                                                         \
 482                 _NOTE(CONSTANTCONDITION)                                \
 483                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),    \
 484                     ("not power of 2 aligned"));                        \
 485                                                                         \
 486                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 487                                                                         \
 488                 (_eop)->eo_u32[0] = *addr++;                            \
 489                 (_eop)->eo_u32[1] = *addr++;                            \
 490                 (_eop)->eo_u32[2] = *addr++;                            \
 491                 (_eop)->eo_u32[3] = *addr;                              \
 492                                                                         \
 493                 EFSYS_PROBE5(mem_reado, unsigned int, (_offset),        \
 494                     uint32_t, (_eop)->eo_u32[3],                        \
 495                     uint32_t, (_eop)->eo_u32[2],                        \
 496                     uint32_t, (_eop)->eo_u32[1],                        \
 497                     uint32_t, (_eop)->eo_u32[0]);                       \
 498                                                                         \
 499         _NOTE(CONSTANTCONDITION)                                        \
 500         } while (B_FALSE)
 501 #endif
 502
 503 #define EFSYS_MEM_WRITED(_esmp, _offset, _edp)                          \
 504         do {                                                            \
 505                 uint32_t *addr;                                         \
 506                                                                         \
 507                 _NOTE(CONSTANTCONDITION)                                \
 508                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),    \
 509                     ("not power of 2 aligned"));                        \
 510                                                                         \
 511                 EFSYS_PROBE2(mem_writed, unsigned int, (_offset),       \
 512                     uint32_t, (_edp)->ed_u32[0]);                       \
 513                                                                         \
 514                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 515                                                                         \
 516                 *addr = (_edp)->ed_u32[0];                              \
 517                                                                         \
 518         _NOTE(CONSTANTCONDITION)                                        \
 519         } while (B_FALSE)
 520
 521 #if defined(__x86_64__)
 522 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp)                          \
 523         do {                                                            \
 524                 uint64_t *addr;                                         \
 525                                                                         \
 526                 _NOTE(CONSTANTCONDITION)                                \
 527                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),    \
 528                     ("not power of 2 aligned"));                        \
 529                                                                         \
 530                 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset),       \
 531                     uint32_t, (_eqp)->eq_u32[1],                        \
 532                     uint32_t, (_eqp)->eq_u32[0]);                       \
 533                                                                         \
 534                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 535                                                                         \
 536                 *addr   = (_eqp)->eq_u64[0];                            \
 537                                                                         \
 538         _NOTE(CONSTANTCONDITION)                                        \
 539         } while (B_FALSE)
 540
 541 #else
 542 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp)                          \
 543         do {                                                            \
 544                 uint32_t *addr;                                         \
 545                                                                         \
 546                 _NOTE(CONSTANTCONDITION)                                \
 547                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),    \
 548                     ("not power of 2 aligned"));                        \
 549                                                                         \
 550                 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset),       \
 551                     uint32_t, (_eqp)->eq_u32[1],                        \
 552                     uint32_t, (_eqp)->eq_u32[0]);                       \
 553                                                                         \
 554                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 555                                                                         \
 556                 *addr++ = (_eqp)->eq_u32[0];                            \
 557                 *addr   = (_eqp)->eq_u32[1];                            \
 558                                                                         \
 559         _NOTE(CONSTANTCONDITION)                                        \
 560         } while (B_FALSE)
 561 #endif
 562
 563 #if defined(__x86_64__)
 564 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop)                          \
 565         do {                                                            \
 566                 uint64_t *addr;                                         \
 567                                                                         \
 568                 _NOTE(CONSTANTCONDITION)                                \
 569                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),    \
 570                     ("not power of 2 aligned"));                        \
 571                                                                         \
 572                 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset),       \
 573                     uint32_t, (_eop)->eo_u32[3],                        \
 574                     uint32_t, (_eop)->eo_u32[2],                        \
 575                     uint32_t, (_eop)->eo_u32[1],                        \
 576                     uint32_t, (_eop)->eo_u32[0]);                       \
 577                                                                         \
 578                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 579                                                                         \
 580                 *addr++ = (_eop)->eo_u64[0];                            \
 581                 *addr   = (_eop)->eo_u64[1];                            \
 582                                                                         \
 583         _NOTE(CONSTANTCONDITION)                                        \
 584         } while (B_FALSE)
 585 #else
 586 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop)                          \
 587         do {                                                            \
 588                 uint32_t *addr;                                         \
 589                                                                         \
 590                 _NOTE(CONSTANTCONDITION)                                \
 591                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),    \
 592                     ("not power of 2 aligned"));                        \
 593                                                                         \
 594                 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset),       \
 595                     uint32_t, (_eop)->eo_u32[3],                        \
 596                     uint32_t, (_eop)->eo_u32[2],                        \
 597                     uint32_t, (_eop)->eo_u32[1],                        \
 598                     uint32_t, (_eop)->eo_u32[0]);                       \
 599                                                                         \
 600                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 601                                                                         \
 602                 *addr++ = (_eop)->eo_u32[0];                            \
 603                 *addr++ = (_eop)->eo_u32[1];                            \
 604                 *addr++ = (_eop)->eo_u32[2];                            \
 605                 *addr   = (_eop)->eo_u32[3];                            \
 606                                                                         \
 607         _NOTE(CONSTANTCONDITION)                                        \
 608         } while (B_FALSE)
 609 #endif
 610
 611 #define EFSYS_MEM_ADDR(_esmp)                                           \
 612         ((_esmp)->esm_addr)
 613
 614 /* BAR */
 615
 616 #define SFXGE_LOCK_NAME_MAX     16
 617
 618 typedef struct efsys_bar_s {
 619         struct mtx              esb_lock;
 620         char                    esb_lock_name[SFXGE_LOCK_NAME_MAX];
 621         bus_space_tag_t         esb_tag;
 622         bus_space_handle_t      esb_handle;
 623         int                     esb_rid;
 624         struct resource         *esb_res;
 625 } efsys_bar_t;
 626
 627 #define SFXGE_BAR_LOCK_INIT(_esbp, _ifname)                             \
 628         do {                                                            \
 629                 snprintf((_esbp)->esb_lock_name,                        \
 630                          sizeof((_esbp)->esb_lock_name),                \
 631                          "%s:bar", (_ifname));                          \
 632                 mtx_init(&(_esbp)->esb_lock, (_esbp)->esb_lock_name,    \
 633                          NULL, MTX_DEF);                                \
 634         _NOTE(CONSTANTCONDITION)                                        \
 635         } while (B_FALSE)
 636 #define SFXGE_BAR_LOCK_DESTROY(_esbp)                                   \
 637         mtx_destroy(&(_esbp)->esb_lock)
 638 #define SFXGE_BAR_LOCK(_esbp)                                           \
 639         mtx_lock(&(_esbp)->esb_lock)
 640 #define SFXGE_BAR_UNLOCK(_esbp)                                         \
 641         mtx_unlock(&(_esbp)->esb_lock)
 642
 643 #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock)                    \
 644         do {                                                            \
 645                 _NOTE(CONSTANTCONDITION)                                \
 646                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),    \
 647                     ("not power of 2 aligned"));                        \
 648                                                                         \
 649                 _NOTE(CONSTANTCONDITION)                                \
 650                 if (_lock)                                              \
 651                         SFXGE_BAR_LOCK(_esbp);                          \
 652                                                                         \
 653                 (_edp)->ed_u32[0] = bus_space_read_stream_4(            \
 654                     (_esbp)->esb_tag, (_esbp)->esb_handle,              \
 655                     (_offset));                                         \
 656                                                                         \
 657                 EFSYS_PROBE2(bar_readd, unsigned int, (_offset),        \
 658                     uint32_t, (_edp)->ed_u32[0]);                       \
 659                                                                         \
 660                 _NOTE(CONSTANTCONDITION)                                \
 661                 if (_lock)                                              \
 662                         SFXGE_BAR_UNLOCK(_esbp);                        \
 663         _NOTE(CONSTANTCONDITION)                                        \
 664         } while (B_FALSE)
 665
 666 #if defined(__x86_64__)
 667 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp)                           \
 668         do {                                                            \
 669                 _NOTE(CONSTANTCONDITION)                                \
 670                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),    \
 671                     ("not power of 2 aligned"));                        \
 672                                                                         \
 673                 SFXGE_BAR_LOCK(_esbp);                                  \
 674                                                                         \
 675                 (_eqp)->eq_u64[0] = bus_space_read_stream_8(            \
 676                     (_esbp)->esb_tag, (_esbp)->esb_handle,              \
 677                     (_offset));                                         \
 678                                                                         \
 679                 EFSYS_PROBE3(bar_readq, unsigned int, (_offset),        \
 680                     uint32_t, (_eqp)->eq_u32[1],                        \
 681                     uint32_t, (_eqp)->eq_u32[0]);                       \
 682                                                                         \
 683                 SFXGE_BAR_UNLOCK(_esbp);                                \
 684         _NOTE(CONSTANTCONDITION)                                        \
 685         } while (B_FALSE)
 686
 687 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock)                    \
 688         do {                                                            \
 689                 _NOTE(CONSTANTCONDITION)                                \
 690                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),    \
 691                     ("not power of 2 aligned"));                        \
 692                                                                         \
 693                 _NOTE(CONSTANTCONDITION)                                \
 694                 if (_lock)                                              \
 695                         SFXGE_BAR_LOCK(_esbp);                          \
 696                                                                         \
 697                 (_eop)->eo_u64[0] = bus_space_read_stream_8(            \
 698                     (_esbp)->esb_tag, (_esbp)->esb_handle,              \
 699                     (_offset));                                         \
 700                 (_eop)->eo_u64[1] = bus_space_read_stream_8(            \
 701                     (_esbp)->esb_tag, (_esbp)->esb_handle,              \
 702                     (_offset) + 8);                                     \
 703                                                                         \
 704                 EFSYS_PROBE5(bar_reado, unsigned int, (_offset),        \
 705                     uint32_t, (_eop)->eo_u32[3],                        \
 706                     uint32_t, (_eop)->eo_u32[2],                        \
 707                     uint32_t, (_eop)->eo_u32[1],                        \
 708                     uint32_t, (_eop)->eo_u32[0]);                       \
 709                                                                         \
 710                 _NOTE(CONSTANTCONDITION)                                \
 711                 if (_lock)                                              \
 712                         SFXGE_BAR_UNLOCK(_esbp);                        \
 713         _NOTE(CONSTANTCONDITION)                                        \
 714         } while (B_FALSE)
 715
 716 #else
 717 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp)                           \
 718         do {                                                            \
 719                 _NOTE(CONSTANTCONDITION)                                \
 720                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),    \
 721                     ("not power of 2 aligned"));                        \
 722                                                                         \
 723                 SFXGE_BAR_LOCK(_esbp);                                  \
 724                                                                         \
 725                 (_eqp)->eq_u32[0] = bus_space_read_stream_4(            \
 726                     (_esbp)->esb_tag, (_esbp)->esb_handle,              \
 727                     (_offset));                                         \
 728                 (_eqp)->eq_u32[1] = bus_space_read_stream_4(            \
 729                     (_esbp)->esb_tag, (_esbp)->esb_handle,              \
 730                     (_offset) + 4);                                     \
 731                                                                         \
 732                 EFSYS_PROBE3(bar_readq, unsigned int, (_offset),        \
 733                     uint32_t, (_eqp)->eq_u32[1],                        \
 734                     uint32_t, (_eqp)->eq_u32[0]);                       \
 735                                                                         \
 736                 SFXGE_BAR_UNLOCK(_esbp);                                \
 737         _NOTE(CONSTANTCONDITION)                                        \
 738         } while (B_FALSE)
 739
 740 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock)                    \
 741         do {                                                            \
 742                 _NOTE(CONSTANTCONDITION)                                \
 743                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),    \
 744                     ("not power of 2 aligned"));                        \
 745                                                                         \
 746                 _NOTE(CONSTANTCONDITION)                                \
 747                 if (_lock)                                              \
 748                         SFXGE_BAR_LOCK(_esbp);                          \
 749                                                                         \
 750                 (_eop)->eo_u32[0] = bus_space_read_stream_4(            \
 751                     (_esbp)->esb_tag, (_esbp)->esb_handle,              \
 752                     (_offset));                                         \
 753                 (_eop)->eo_u32[1] = bus_space_read_stream_4(            \
 754                     (_esbp)->esb_tag, (_esbp)->esb_handle,              \
 755                     (_offset) + 4);                                     \
 756                 (_eop)->eo_u32[2] = bus_space_read_stream_4(            \
 757                     (_esbp)->esb_tag, (_esbp)->esb_handle,              \
 758                     (_offset) + 8);                                     \
 759                 (_eop)->eo_u32[3] = bus_space_read_stream_4(            \
 760                     (_esbp)->esb_tag, (_esbp)->esb_handle,              \
 761                     (_offset) + 12);                                    \
 762                                                                         \
 763                 EFSYS_PROBE5(bar_reado, unsigned int, (_offset),        \
 764                     uint32_t, (_eop)->eo_u32[3],                        \
 765                     uint32_t, (_eop)->eo_u32[2],                        \
 766                     uint32_t, (_eop)->eo_u32[1],                        \
 767                     uint32_t, (_eop)->eo_u32[0]);                       \
 768                                                                         \
 769                 _NOTE(CONSTANTCONDITION)                                \
 770                 if (_lock)                                              \
 771                         SFXGE_BAR_UNLOCK(_esbp);                        \
 772         _NOTE(CONSTANTCONDITION)                                        \
 773         } while (B_FALSE)
 774 #endif
 775
 776 #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock)                   \
 777         do {                                                            \
 778                 _NOTE(CONSTANTCONDITION)                                \
 779                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),    \
 780                     ("not power of 2 aligned"));                        \
 781                                                                         \
 782                 _NOTE(CONSTANTCONDITION)                                \
 783                 if (_lock)                                              \
 784                         SFXGE_BAR_LOCK(_esbp);                          \
 785                                                                         \
 786                 EFSYS_PROBE2(bar_writed, unsigned int, (_offset),       \
 787                     uint32_t, (_edp)->ed_u32[0]);                       \
 788                                                                         \
 789                 bus_space_write_stream_4((_esbp)->esb_tag,              \
 790                     (_esbp)->esb_handle,                                \
 791                     (_offset), (_edp)->ed_u32[0]);                      \
 792                                                                         \
 793                 _NOTE(CONSTANTCONDITION)                                \
 794                 if (_lock)                                              \
 795                         SFXGE_BAR_UNLOCK(_esbp);                        \
 796         _NOTE(CONSTANTCONDITION)                                        \
 797         } while (B_FALSE)
 798
 799 #if defined(__x86_64__)
 800 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp)                          \
 801         do {                                                            \
 802                 _NOTE(CONSTANTCONDITION)                                \
 803                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),    \
 804                     ("not power of 2 aligned"));                        \
 805                                                                         \
 806                 SFXGE_BAR_LOCK(_esbp);                                  \
 807                                                                         \
 808                 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset),       \
 809                     uint32_t, (_eqp)->eq_u32[1],                        \
 810                     uint32_t, (_eqp)->eq_u32[0]);                       \
 811                                                                         \
 812                 bus_space_write_stream_8((_esbp)->esb_tag,              \
 813                     (_esbp)->esb_handle,                                \
 814                     (_offset), (_eqp)->eq_u64[0]);                      \
 815                                                                         \
 816                 SFXGE_BAR_UNLOCK(_esbp);                                \
 817         _NOTE(CONSTANTCONDITION)                                        \
 818         } while (B_FALSE)
 819 #else
 820 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp)                          \
 821         do {                                                            \
 822                 _NOTE(CONSTANTCONDITION)                                \
 823                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),    \
 824                     ("not power of 2 aligned"));                        \
 825                                                                         \
 826                 SFXGE_BAR_LOCK(_esbp);                                  \
 827                                                                         \
 828                 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset),       \
 829                     uint32_t, (_eqp)->eq_u32[1],                        \
 830                     uint32_t, (_eqp)->eq_u32[0]);                       \
 831                                                                         \
 832                 bus_space_write_stream_4((_esbp)->esb_tag,              \
 833                     (_esbp)->esb_handle,                                \
 834                     (_offset), (_eqp)->eq_u32[0]);                      \
 835                 bus_space_write_stream_4((_esbp)->esb_tag,              \
 836                     (_esbp)->esb_handle,                                \
 837                     (_offset) + 4, (_eqp)->eq_u32[1]);                  \
 838                                                                         \
 839                 SFXGE_BAR_UNLOCK(_esbp);                                \
 840         _NOTE(CONSTANTCONDITION)                                        \
 841         } while (B_FALSE)
 842 #endif
 843
 844 #if defined(__x86_64__)
 845 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock)                   \
 846         do {                                                            \
 847                 _NOTE(CONSTANTCONDITION)                                \
 848                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),    \
 849                     ("not power of 2 aligned"));                        \
 850                                                                         \
 851                 _NOTE(CONSTANTCONDITION)                                \
 852                 if (_lock)                                              \
 853                         SFXGE_BAR_LOCK(_esbp);                          \
 854                                                                         \
 855                 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset),       \
 856                     uint32_t, (_eop)->eo_u32[3],                        \
 857                     uint32_t, (_eop)->eo_u32[2],                        \
 858                     uint32_t, (_eop)->eo_u32[1],                        \
 859                     uint32_t, (_eop)->eo_u32[0]);                       \
 860                                                                         \
 861                 bus_space_write_stream_8((_esbp)->esb_tag,              \
 862                     (_esbp)->esb_handle,                                \
 863                     (_offset), (_eop)->eo_u64[0]);                      \
 864                 bus_space_write_stream_8((_esbp)->esb_tag,              \
 865                     (_esbp)->esb_handle,                                \
 866                     (_offset) + 8, (_eop)->eo_u64[1]);                  \
 867                                                                         \
 868                 _NOTE(CONSTANTCONDITION)                                \
 869                 if (_lock)                                              \
 870                         SFXGE_BAR_UNLOCK(_esbp);                        \
 871         _NOTE(CONSTANTCONDITION)                                        \
 872         } while (B_FALSE)
 873
 874 #else
 875 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock)                   \
 876         do {                                                            \
 877                 _NOTE(CONSTANTCONDITION)                                \
 878                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),    \
 879                     ("not power of 2 aligned"));                        \
 880                                                                         \
 881                 _NOTE(CONSTANTCONDITION)                                \
 882                 if (_lock)                                              \
 883                         SFXGE_BAR_LOCK(_esbp);                          \
 884                                                                         \
 885                 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset),       \
 886                     uint32_t, (_eop)->eo_u32[3],                        \
 887                     uint32_t, (_eop)->eo_u32[2],                        \
 888                     uint32_t, (_eop)->eo_u32[1],                        \
 889                     uint32_t, (_eop)->eo_u32[0]);                       \
 890                                                                         \
 891                 bus_space_write_stream_4((_esbp)->esb_tag,              \
 892                     (_esbp)->esb_handle,                                \
 893                     (_offset), (_eop)->eo_u32[0]);                      \
 894                 bus_space_write_stream_4((_esbp)->esb_tag,              \
 895                     (_esbp)->esb_handle,                                \
 896                     (_offset) + 4, (_eop)->eo_u32[1]);                  \
 897                 bus_space_write_stream_4((_esbp)->esb_tag,              \
 898                     (_esbp)->esb_handle,                                \
 899                     (_offset) + 8, (_eop)->eo_u32[2]);                  \
 900                 bus_space_write_stream_4((_esbp)->esb_tag,              \
 901                     (_esbp)->esb_handle,                                \
 902                     (_offset) + 12, (_eop)->eo_u32[3]);                 \
 903                                                                         \
 904                 _NOTE(CONSTANTCONDITION)                                \
 905                 if (_lock)                                              \
 906                         SFXGE_BAR_UNLOCK(_esbp);                        \
 907         _NOTE(CONSTANTCONDITION)                                        \
 908         } while (B_FALSE)
 909 #endif
 910
 911 /* SPIN */
 912
 913 #define EFSYS_SPIN(_us)                                                 \
 914         do {                                                            \
 915                 DELAY(_us);                                             \
 916         _NOTE(CONSTANTCONDITION)                                        \
 917         } while (B_FALSE)
 918
 919 #define EFSYS_SLEEP     EFSYS_SPIN
 920
 921 /* BARRIERS */
 922
 923 #define EFSYS_MEM_READ_BARRIER()        rmb()
 924 #define EFSYS_PIO_WRITE_BARRIER()
 925
 926 /* TIMESTAMP */
 927
 928 typedef clock_t efsys_timestamp_t;
 929
 930 #define EFSYS_TIMESTAMP(_usp)                                           \
 931         do {                                                            \
 932                 clock_t now;                                            \
 933                                                                         \
 934                 now = ticks;                                            \
 935                 *(_usp) = now * hz / 1000000;                           \
 936         _NOTE(CONSTANTCONDITION)                                        \
 937         } while (B_FALSE)
 938
 939 /* KMEM */
 940
 941 #define EFSYS_KMEM_ALLOC(_esip, _size, _p)                              \
 942         do {                                                            \
 943                 (_esip) = (_esip);                                      \
 944                 (_p) = malloc((_size), M_SFXGE, M_WAITOK|M_ZERO);       \
 945         _NOTE(CONSTANTCONDITION)                                        \
 946         } while (B_FALSE)
 947
 948 #define EFSYS_KMEM_FREE(_esip, _size, _p)                               \
 949         do {                                                            \
 950                 (void) (_esip);                                         \
 951                 (void) (_size);                                         \
 952                 free((_p), M_SFXGE);                                    \
 953         _NOTE(CONSTANTCONDITION)                                        \
 954         } while (B_FALSE)
 955
 956 /* LOCK */
 957
 958 typedef struct efsys_lock_s {
 959         struct mtx      lock;
 960         char            lock_name[SFXGE_LOCK_NAME_MAX];
 961 } efsys_lock_t;
 962
 963 #define SFXGE_EFSYS_LOCK_INIT(_eslp, _ifname, _label)                   \
 964         do {                                                            \
 965                 efsys_lock_t *__eslp = (_eslp);                         \
 966                                                                         \
 967                 snprintf((__eslp)->lock_name,                           \
 968                          sizeof((__eslp)->lock_name),                   \
 969                          "%s:%s", (_ifname), (_label));                 \
 970                 mtx_init(&(__eslp)->lock, (__eslp)->lock_name,          \
 971                          NULL, MTX_DEF);                                \
 972         } while (B_FALSE)
 973 #define SFXGE_EFSYS_LOCK_DESTROY(_eslp)                                 \
 974         mtx_destroy(&(_eslp)->lock)
 975 #define SFXGE_EFSYS_LOCK(_eslp)                                         \
 976         mtx_lock(&(_eslp)->lock)
 977 #define SFXGE_EFSYS_UNLOCK(_eslp)                                       \
 978         mtx_unlock(&(_eslp)->lock)
 979 #define SFXGE_EFSYS_LOCK_ASSERT_OWNED(_eslp)                            \
 980         mtx_assert(&(_eslp)->lock, MA_OWNED)
 981
 982 #define EFSYS_LOCK_MAGIC        0x000010c4
 983
 984 #define EFSYS_LOCK(_lockp, _state)                                      \
 985         do {                                                            \
 986                 SFXGE_EFSYS_LOCK(_lockp);                               \
 987                 (_state) = EFSYS_LOCK_MAGIC;                            \
 988         _NOTE(CONSTANTCONDITION)                                        \
 989         } while (B_FALSE)
 990
 991 #define EFSYS_UNLOCK(_lockp, _state)                                    \
 992         do {                                                            \
 993                 if ((_state) != EFSYS_LOCK_MAGIC)                       \
 994                         KASSERT(B_FALSE, ("not locked"));               \
 995                 SFXGE_EFSYS_UNLOCK(_lockp);                             \
 996         _NOTE(CONSTANTCONDITION)                                        \
 997         } while (B_FALSE)
 998
 999 /* PREEMPT */
1000
1001 #define EFSYS_PREEMPT_DISABLE(_state)                                   \
1002         do {                                                            \
1003                 (_state) = (_state);                                    \
1004                 critical_enter();                                       \
1005         _NOTE(CONSTANTCONDITION)                                        \
1006         } while (B_FALSE)
1007
1008 #define EFSYS_PREEMPT_ENABLE(_state)                                    \
1009         do {                                                            \
1010                 (_state) = (_state);                                    \
1011                 critical_exit(_state);                                  \
1012         _NOTE(CONSTANTCONDITION)                                        \
1013         } while (B_FALSE)
1014
1015 /* STAT */
1016
1017 typedef uint64_t                efsys_stat_t;
1018
1019 #define EFSYS_STAT_INCR(_knp, _delta)                                   \
1020         do {                                                            \
1021                 *(_knp) += (_delta);                                    \
1022         _NOTE(CONSTANTCONDITION)                                        \
1023         } while (B_FALSE)
1024
1025 #define EFSYS_STAT_DECR(_knp, _delta)                                   \
1026         do {                                                            \
1027                 *(_knp) -= (_delta);                                    \
1028         _NOTE(CONSTANTCONDITION)                                        \
1029         } while (B_FALSE)
1030
1031 #define EFSYS_STAT_SET(_knp, _val)                                      \
1032         do {                                                            \
1033                 *(_knp) = (_val);                                       \
1034         _NOTE(CONSTANTCONDITION)                                        \
1035         } while (B_FALSE)
1036
1037 #define EFSYS_STAT_SET_QWORD(_knp, _valp)                               \
1038         do {                                                            \
1039                 *(_knp) = le64toh((_valp)->eq_u64[0]);                  \
1040         _NOTE(CONSTANTCONDITION)                                        \
1041         } while (B_FALSE)
1042
1043 #define EFSYS_STAT_SET_DWORD(_knp, _valp)                               \
1044         do {                                                            \
1045                 *(_knp) = le32toh((_valp)->ed_u32[0]);                  \
1046         _NOTE(CONSTANTCONDITION)                                        \
1047         } while (B_FALSE)
1048
1049 #define EFSYS_STAT_INCR_QWORD(_knp, _valp)                              \
1050         do {                                                            \
1051                 *(_knp) += le64toh((_valp)->eq_u64[0]);                 \
1052         _NOTE(CONSTANTCONDITION)                                        \
1053         } while (B_FALSE)
1054
1055 #define EFSYS_STAT_SUBR_QWORD(_knp, _valp)                              \
1056         do {                                                            \
1057                 *(_knp) -= le64toh((_valp)->eq_u64[0]);                 \
1058         _NOTE(CONSTANTCONDITION)                                        \
1059         } while (B_FALSE)
1060
1061 /* ERR */
1062
1063 extern void     sfxge_err(efsys_identifier_t *, unsigned int,
1064                     uint32_t, uint32_t);
1065
1066 #if EFSYS_OPT_DECODE_INTR_FATAL
1067 #define EFSYS_ERR(_esip, _code, _dword0, _dword1)                       \
1068         do {                                                            \
1069                 sfxge_err((_esip), (_code), (_dword0), (_dword1));      \
1070         _NOTE(CONSTANTCONDITION)                                        \
1071         } while (B_FALSE)
1072 #endif
1073
1074 /* ASSERT */
1075
1076 #define EFSYS_ASSERT(_exp) do {                                         \
1077         if (!(_exp))                                                    \
1078                 panic(#_exp);                                           \
1079         } while (0)
1080
1081 #define EFSYS_ASSERT3(_x, _op, _y, _t) do {                             \
1082         const _t __x = (_t)(_x);                                        \
1083         const _t __y = (_t)(_y);                                        \
1084         if (!(__x _op __y))                                             \
1085                 panic("assertion failed at %s:%u", __FILE__, __LINE__); \
1086         } while(0)
1087
1088 #define EFSYS_ASSERT3U(_x, _op, _y)     EFSYS_ASSERT3(_x, _op, _y, uint64_t)
1089 #define EFSYS_ASSERT3S(_x, _op, _y)     EFSYS_ASSERT3(_x, _op, _y, int64_t)
1090 #define EFSYS_ASSERT3P(_x, _op, _y)     EFSYS_ASSERT3(_x, _op, _y, uintptr_t)
1091
1092 #ifdef  __cplusplus
1093 }
1094 #endif
1095
1096 #endif  /* _SYS_EFSYS_H */