sys/dev/sfxge/common/efsys.h

   1 /*-
   2  * Copyright (c) 2010-2011 Solarflare Communications, Inc.
   3  * All rights reserved.
   4  *
   5  * This software was developed in part by Philip Paeps under contract for
   6  * Solarflare Communications, Inc.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  * $FreeBSD$
  30  */
  31
  32 #ifndef _SYS_EFSYS_H
  33 #define _SYS_EFSYS_H
  34
  35 #ifdef  __cplusplus
  36 extern "C" {
  37 #endif
  38
  39 #include <sys/param.h>
  40 #include <sys/bus.h>
  41 #include <sys/endian.h>
  42 #include <sys/lock.h>
  43 #include <sys/malloc.h>
  44 #include <sys/mbuf.h>
  45 #include <sys/mutex.h>
  46 #include <sys/rwlock.h>
  47 #include <sys/sdt.h>
  48 #include <sys/systm.h>
  49
  50 #include <machine/bus.h>
  51 #include <machine/endian.h>
  52
  53 #define EFSYS_HAS_UINT64 1
  54 #define EFSYS_USE_UINT64 0
  55 #if _BYTE_ORDER == _BIG_ENDIAN
  56 #define EFSYS_IS_BIG_ENDIAN 1
  57 #define EFSYS_IS_LITTLE_ENDIAN 0
  58 #elif _BYTE_ORDER == _LITTLE_ENDIAN
  59 #define EFSYS_IS_BIG_ENDIAN 0
  60 #define EFSYS_IS_LITTLE_ENDIAN 1
  61 #endif
  62 #include "efx_types.h"
  63
  64 /* Common code requires this */
  65 #if __FreeBSD_version < 800068
  66 #define memmove(d, s, l) bcopy(s, d, l)
  67 #endif
  68
  69 /* FreeBSD equivalents of Solaris things */
  70 #ifndef _NOTE
  71 #define _NOTE(s)
  72 #endif
  73
  74 #ifndef B_FALSE
  75 #define B_FALSE FALSE
  76 #endif
  77 #ifndef B_TRUE
  78 #define B_TRUE TRUE
  79 #endif
  80
  81 #ifndef IS_P2ALIGNED
  82 #define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0)
  83 #endif
  84
  85 #ifndef P2ROUNDUP
  86 #define P2ROUNDUP(x, align)             (-(-(x) & -(align)))
  87 #endif
  88
  89 #ifndef IS2P
  90 #define ISP2(x)         (((x) & ((x) - 1)) == 0)
  91 #endif
  92
  93 #define ENOTACTIVE EINVAL
  94
  95 /* Memory type to use on FreeBSD */
  96 MALLOC_DECLARE(M_SFXGE);
  97
  98 /* Machine dependend prefetch wrappers */
  99 #if defined(__i386__) || defined(__amd64__)
 100 static __inline void
 101 prefetch_read_many(void *addr)
 102 {
 103
 104         __asm__(
 105             "prefetcht0 (%0)"
 106             :
 107             : "r" (addr));
 108 }
 109
 110 static __inline void
 111 prefetch_read_once(void *addr)
 112 {
 113
 114         __asm__(
 115             "prefetchnta (%0)"
 116             :
 117             : "r" (addr));
 118 }
 119 #elif defined(__sparc64__)
 120 static __inline void
 121 prefetch_read_many(void *addr)
 122 {
 123
 124         __asm__(
 125             "prefetch [%0], 0"
 126             :
 127             : "r" (addr));
 128 }
 129
 130 static __inline void
 131 prefetch_read_once(void *addr)
 132 {
 133
 134         __asm__(
 135             "prefetch [%0], 1"
 136             :
 137             : "r" (addr));
 138 }
 139 #else
 140 static __inline void
 141 prefetch_read_many(void *addr)
 142 {
 143
 144 }
 145
 146 static __inline void
 147 prefetch_read_once(void *addr)
 148 {
 149
 150 }
 151 #endif
 152
 153 #if defined(__i386__) || defined(__amd64__)
 154 #include <vm/vm.h>
 155 #include <vm/pmap.h>
 156 #endif
 157 static __inline void
 158 sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map,
 159     struct mbuf *m, bus_dma_segment_t *seg)
 160 {
 161 #if defined(__i386__) || defined(__amd64__)
 162         seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t));
 163         seg->ds_len = m->m_len;
 164 #else
 165         int nsegstmp;
 166
 167         bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0);
 168 #endif
 169 }
 170
 171 /* Modifiers used for DOS builds */
 172 #define __cs
 173 #define __far
 174
 175 /* Modifiers used for Windows builds */
 176 #define __in
 177 #define __in_opt
 178 #define __in_ecount(_n)
 179 #define __in_ecount_opt(_n)
 180 #define __in_bcount(_n)
 181 #define __in_bcount_opt(_n)
 182
 183 #define __out
 184 #define __out_opt
 185 #define __out_ecount(_n)
 186 #define __out_ecount_opt(_n)
 187 #define __out_bcount(_n)
 188 #define __out_bcount_opt(_n)
 189
 190 #define __deref_out
 191
 192 #define __inout
 193 #define __inout_opt
 194 #define __inout_ecount(_n)
 195 #define __inout_ecount_opt(_n)
 196 #define __inout_bcount(_n)
 197 #define __inout_bcount_opt(_n)
 198 #define __inout_bcount_full_opt(_n)
 199
 200 #define __deref_out_bcount_opt(n)
 201
 202 #define __checkReturn
 203
 204 #define __drv_when(_p, _c)
 205
 206 /* Code inclusion options */
 207
 208
 209 #define EFSYS_OPT_NAMES 1
 210
 211 #define EFSYS_OPT_FALCON 0
 212 #define EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0
 213 #define EFSYS_OPT_SIENA 1
 214 #ifdef DEBUG
 215 #define EFSYS_OPT_CHECK_REG 1
 216 #else
 217 #define EFSYS_OPT_CHECK_REG 0
 218 #endif
 219
 220 #define EFSYS_OPT_MCDI 1
 221
 222 #define EFSYS_OPT_MAC_FALCON_GMAC 0
 223 #define EFSYS_OPT_MAC_FALCON_XMAC 0
 224 #define EFSYS_OPT_MAC_STATS 1
 225
 226 #define EFSYS_OPT_LOOPBACK 0
 227
 228 #define EFSYS_OPT_MON_NULL 0
 229 #define EFSYS_OPT_MON_LM87 0
 230 #define EFSYS_OPT_MON_MAX6647 0
 231 #define EFSYS_OPT_MON_SIENA 0
 232 #define EFSYS_OPT_MON_STATS 0
 233
 234 #define EFSYS_OPT_PHY_NULL 0
 235 #define EFSYS_OPT_PHY_QT2022C2 0
 236 #define EFSYS_OPT_PHY_SFX7101 0
 237 #define EFSYS_OPT_PHY_TXC43128 0
 238 #define EFSYS_OPT_PHY_PM8358 0
 239 #define EFSYS_OPT_PHY_SFT9001 0
 240 #define EFSYS_OPT_PHY_QT2025C 0
 241 #define EFSYS_OPT_PHY_STATS 1
 242 #define EFSYS_OPT_PHY_PROPS 0
 243 #define EFSYS_OPT_PHY_BIST 1
 244 #define EFSYS_OPT_PHY_LED_CONTROL 1
 245 #define EFSYS_OPT_PHY_FLAGS 0
 246
 247 #define EFSYS_OPT_VPD 1
 248 #define EFSYS_OPT_NVRAM 1
 249 #define EFSYS_OPT_NVRAM_FALCON_BOOTROM 0
 250 #define EFSYS_OPT_NVRAM_SFT9001 0
 251 #define EFSYS_OPT_NVRAM_SFX7101 0
 252 #define EFSYS_OPT_BOOTCFG 0
 253
 254 #define EFSYS_OPT_PCIE_TUNE 0
 255 #define EFSYS_OPT_DIAG 0
 256 #define EFSYS_OPT_WOL 1
 257 #define EFSYS_OPT_RX_SCALE 1
 258 #define EFSYS_OPT_QSTATS 1
 259 #define EFSYS_OPT_FILTER 0
 260 #define EFSYS_OPT_RX_SCATTER 0
 261 #define EFSYS_OPT_RX_HDR_SPLIT 0
 262
 263 #define EFSYS_OPT_EV_PREFETCH 0
 264
 265 #define EFSYS_OPT_DECODE_INTR_FATAL 1
 266
 267 /* ID */
 268
 269 typedef struct __efsys_identifier_s     efsys_identifier_t;
 270
 271 /* PROBE */
 272
 273 #ifndef KDTRACE_HOOKS
 274
 275 #define EFSYS_PROBE(_name)
 276
 277 #define EFSYS_PROBE1(_name, _type1, _arg1)
 278
 279 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2)
 280
 281 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2,               \
 282             _type3, _arg3)
 283
 284 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2,               \
 285             _type3, _arg3, _type4, _arg4)
 286
 287 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,               \
 288             _type3, _arg3, _type4, _arg4, _type5, _arg5)
 289
 290 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,               \
 291             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 292             _type6, _arg6)
 293
 294 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2,               \
 295             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 296             _type6, _arg6, _type7, _arg7)
 297
 298 #else /* KDTRACE_HOOKS */
 299
 300 #define EFSYS_PROBE(_name)                                              \
 301         DTRACE_PROBE(_name)
 302
 303 #define EFSYS_PROBE1(_name, _type1, _arg1)                              \
 304         DTRACE_PROBE1(_name, _type1, _arg1)
 305
 306 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2)               \
 307         DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2)
 308
 309 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2,               \
 310             _type3, _arg3)                                              \
 311         DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2,              \
 312             _type3, _arg3)
 313
 314 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2,               \
 315             _type3, _arg3, _type4, _arg4)                               \
 316         DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2,              \
 317             _type3, _arg3, _type4, _arg4)
 318
 319 #ifdef DTRACE_PROBE5
 320 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,               \
 321             _type3, _arg3, _type4, _arg4, _type5, _arg5)                \
 322         DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2,              \
 323             _type3, _arg3, _type4, _arg4, _type5, _arg5)
 324 #else
 325 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,               \
 326             _type3, _arg3, _type4, _arg4, _type5, _arg5)                \
 327         DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2,              \
 328             _type3, _arg3, _type4, _arg4)
 329 #endif
 330
 331 #ifdef DTRACE_PROBE6
 332 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,               \
 333             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 334             _type6, _arg6)                                              \
 335         DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2,              \
 336             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 337             _type6, _arg6)
 338 #else
 339 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,               \
 340             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 341             _type6, _arg6)                                              \
 342         EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,               \
 343             _type3, _arg3, _type4, _arg4, _type5, _arg5)
 344 #endif
 345
 346 #ifdef DTRACE_PROBE7
 347 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2,               \
 348             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 349             _type6, _arg6, _type7, _arg7)                               \
 350         DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2,              \
 351             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 352             _type6, _arg6, _type7, _arg7)
 353 #else
 354 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2,               \
 355             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 356             _type6, _arg6, _type7, _arg7)                               \
 357         EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,               \
 358             _type3, _arg3, _type4, _arg4, _type5, _arg5,                \
 359             _type6, _arg6)
 360 #endif
 361
 362 #endif /* KDTRACE_HOOKS */
 363
 364 /* DMA */
 365
 366 typedef uint64_t                efsys_dma_addr_t;
 367
 368 typedef struct efsys_mem_s {
 369         bus_dma_tag_t           esm_tag;
 370         bus_dmamap_t            esm_map;
 371         caddr_t                 esm_base;
 372         efsys_dma_addr_t        esm_addr;
 373         size_t                  esm_size;
 374 } efsys_mem_t;
 375
 376
 377 #define EFSYS_MEM_ZERO(_esmp, _size)                                    \
 378         do {                                                            \
 379                 (void) memset((_esmp)->esm_base, 0, (_size));           \
 380                                                                         \
 381         _NOTE(CONSTANTCONDITION)                                        \
 382         } while (B_FALSE)
 383
 384 #define EFSYS_MEM_READD(_esmp, _offset, _edp)                           \
 385         do {                                                            \
 386                 uint32_t *addr;                                         \
 387                                                                         \
 388                 _NOTE(CONSTANTCONDITION)                                \
 389                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),    \
 390                     ("not power of 2 aligned"));                        \
 391                                                                         \
 392                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 393                                                                         \
 394                 (_edp)->ed_u32[0] = *addr;                              \
 395                                                                         \
 396                 EFSYS_PROBE2(mem_readd, unsigned int, (_offset),        \
 397                     uint32_t, (_edp)->ed_u32[0]);                       \
 398                                                                         \
 399         _NOTE(CONSTANTCONDITION)                                        \
 400         } while (B_FALSE)
 401
 402 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp)                           \
 403         do {                                                            \
 404                 uint32_t *addr;                                         \
 405                                                                         \
 406                 _NOTE(CONSTANTCONDITION)                                \
 407                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),    \
 408                     ("not power of 2 aligned"));                        \
 409                                                                         \
 410                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 411                                                                         \
 412                 (_eqp)->eq_u32[0] = *addr++;                            \
 413                 (_eqp)->eq_u32[1] = *addr;                              \
 414                                                                         \
 415                 EFSYS_PROBE3(mem_readq, unsigned int, (_offset),        \
 416                     uint32_t, (_eqp)->eq_u32[1],                        \
 417                     uint32_t, (_eqp)->eq_u32[0]);                       \
 418                                                                         \
 419         _NOTE(CONSTANTCONDITION)                                        \
 420         } while (B_FALSE)
 421
 422 #define EFSYS_MEM_READO(_esmp, _offset, _eop)                           \
 423         do {                                                            \
 424                 uint32_t *addr;                                         \
 425                                                                         \
 426                 _NOTE(CONSTANTCONDITION)                                \
 427                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),    \
 428                     ("not power of 2 aligned"));                        \
 429                                                                         \
 430                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 431                                                                         \
 432                 (_eop)->eo_u32[0] = *addr++;                            \
 433                 (_eop)->eo_u32[1] = *addr++;                            \
 434                 (_eop)->eo_u32[2] = *addr++;                            \
 435                 (_eop)->eo_u32[3] = *addr;                              \
 436                                                                         \
 437                 EFSYS_PROBE5(mem_reado, unsigned int, (_offset),        \
 438                     uint32_t, (_eop)->eo_u32[3],                        \
 439                     uint32_t, (_eop)->eo_u32[2],                        \
 440                     uint32_t, (_eop)->eo_u32[1],                        \
 441                     uint32_t, (_eop)->eo_u32[0]);                       \
 442                                                                         \
 443         _NOTE(CONSTANTCONDITION)                                        \
 444         } while (B_FALSE)
 445
 446 #define EFSYS_MEM_WRITED(_esmp, _offset, _edp)                          \
 447         do {                                                            \
 448                 uint32_t *addr;                                         \
 449                                                                         \
 450                 _NOTE(CONSTANTCONDITION)                                \
 451                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),    \
 452                     ("not power of 2 aligned"));                        \
 453                                                                         \
 454                 EFSYS_PROBE2(mem_writed, unsigned int, (_offset),       \
 455                     uint32_t, (_edp)->ed_u32[0]);                       \
 456                                                                         \
 457                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 458                                                                         \
 459                 *addr = (_edp)->ed_u32[0];                              \
 460                                                                         \
 461         _NOTE(CONSTANTCONDITION)                                        \
 462         } while (B_FALSE)
 463
 464 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp)                          \
 465         do {                                                            \
 466                 uint32_t *addr;                                         \
 467                                                                         \
 468                 _NOTE(CONSTANTCONDITION)                                \
 469                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),    \
 470                     ("not power of 2 aligned"));                        \
 471                                                                         \
 472                 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset),       \
 473                     uint32_t, (_eqp)->eq_u32[1],                        \
 474                     uint32_t, (_eqp)->eq_u32[0]);                       \
 475                                                                         \
 476                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 477                                                                         \
 478                 *addr++ = (_eqp)->eq_u32[0];                            \
 479                 *addr   = (_eqp)->eq_u32[1];                            \
 480                                                                         \
 481         _NOTE(CONSTANTCONDITION)                                        \
 482         } while (B_FALSE)
 483
 484 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop)                          \
 485         do {                                                            \
 486                 uint32_t *addr;                                         \
 487                                                                         \
 488                 _NOTE(CONSTANTCONDITION)                                \
 489                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),    \
 490                     ("not power of 2 aligned"));                        \
 491                                                                         \
 492                 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset),       \
 493                     uint32_t, (_eop)->eo_u32[3],                        \
 494                     uint32_t, (_eop)->eo_u32[2],                        \
 495                     uint32_t, (_eop)->eo_u32[1],                        \
 496                     uint32_t, (_eop)->eo_u32[0]);                       \
 497                                                                         \
 498                 addr = (void *)((_esmp)->esm_base + (_offset));         \
 499                                                                         \
 500                 *addr++ = (_eop)->eo_u32[0];                            \
 501                 *addr++ = (_eop)->eo_u32[1];                            \
 502                 *addr++ = (_eop)->eo_u32[2];                            \
 503                 *addr   = (_eop)->eo_u32[3];                            \
 504                                                                         \
 505         _NOTE(CONSTANTCONDITION)                                        \
 506         } while (B_FALSE)
 507
 508 #define EFSYS_MEM_ADDR(_esmp)                                           \
 509         ((_esmp)->esm_addr)
 510
 511 /* BAR */
 512
 513 typedef struct efsys_bar_s {
 514         struct mtx              esb_lock;
 515         bus_space_tag_t         esb_tag;
 516         bus_space_handle_t      esb_handle;
 517         int                     esb_rid;
 518         struct resource         *esb_res;
 519 } efsys_bar_t;
 520
 521 #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock)                    \
 522         do {                                                            \
 523                 _NOTE(CONSTANTCONDITION)                                \
 524                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),    \
 525                     ("not power of 2 aligned"));                        \
 526                                                                         \
 527                 _NOTE(CONSTANTCONDITION)                                \
 528                 if (_lock)                                              \
 529                         mtx_lock(&((_esbp)->esb_lock));                 \
 530                                                                         \
 531                 (_edp)->ed_u32[0] = bus_space_read_4((_esbp)->esb_tag,  \
 532                     (_esbp)->esb_handle, (_offset));                    \
 533                                                                         \
 534                 EFSYS_PROBE2(bar_readd, unsigned int, (_offset),        \
 535                     uint32_t, (_edp)->ed_u32[0]);                       \
 536                                                                         \
 537                 _NOTE(CONSTANTCONDITION)                                \
 538                 if (_lock)                                              \
 539                         mtx_unlock(&((_esbp)->esb_lock));               \
 540         _NOTE(CONSTANTCONDITION)                                        \
 541         } while (B_FALSE)
 542
 543 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp)                           \
 544         do {                                                            \
 545                 _NOTE(CONSTANTCONDITION)                                \
 546                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),    \
 547                     ("not power of 2 aligned"));                        \
 548                                                                         \
 549                 mtx_lock(&((_esbp)->esb_lock));                         \
 550                                                                         \
 551                 (_eqp)->eq_u32[0] = bus_space_read_4((_esbp)->esb_tag,  \
 552                     (_esbp)->esb_handle, (_offset));                    \
 553                 (_eqp)->eq_u32[1] = bus_space_read_4((_esbp)->esb_tag,  \
 554                     (_esbp)->esb_handle, (_offset+4));                  \
 555                                                                         \
 556                 EFSYS_PROBE3(bar_readq, unsigned int, (_offset),        \
 557                     uint32_t, (_eqp)->eq_u32[1],                        \
 558                     uint32_t, (_eqp)->eq_u32[0]);                       \
 559                                                                         \
 560                 mtx_unlock(&((_esbp)->esb_lock));                       \
 561         _NOTE(CONSTANTCONDITION)                                        \
 562         } while (B_FALSE)
 563
 564 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock)                    \
 565         do {                                                            \
 566                 _NOTE(CONSTANTCONDITION)                                \
 567                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),    \
 568                     ("not power of 2 aligned"));                        \
 569                                                                         \
 570                 _NOTE(CONSTANTCONDITION)                                \
 571                 if (_lock)                                              \
 572                         mtx_lock(&((_esbp)->esb_lock));                 \
 573                                                                         \
 574                 (_eop)->eo_u32[0] = bus_space_read_4((_esbp)->esb_tag,  \
 575                     (_esbp)->esb_handle, (_offset));                    \
 576                 (_eop)->eo_u32[1] = bus_space_read_4((_esbp)->esb_tag,  \
 577                     (_esbp)->esb_handle, (_offset+4));                  \
 578                 (_eop)->eo_u32[2] = bus_space_read_4((_esbp)->esb_tag,  \
 579                     (_esbp)->esb_handle, (_offset+8));                  \
 580                 (_eop)->eo_u32[3] = bus_space_read_4((_esbp)->esb_tag,  \
 581                     (_esbp)->esb_handle, (_offset+12));                 \
 582                                                                         \
 583                 EFSYS_PROBE5(bar_reado, unsigned int, (_offset),        \
 584                     uint32_t, (_eop)->eo_u32[3],                        \
 585                     uint32_t, (_eop)->eo_u32[2],                        \
 586                     uint32_t, (_eop)->eo_u32[1],                        \
 587                     uint32_t, (_eop)->eo_u32[0]);                       \
 588                                                                         \
 589                 _NOTE(CONSTANTCONDITION)                                \
 590                 if (_lock)                                              \
 591                         mtx_unlock(&((_esbp)->esb_lock));               \
 592         _NOTE(CONSTANTCONDITION)                                        \
 593         } while (B_FALSE)
 594
 595 #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock)                   \
 596         do {                                                            \
 597                 _NOTE(CONSTANTCONDITION)                                \
 598                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)),    \
 599                     ("not power of 2 aligned"));                        \
 600                                                                         \
 601                 _NOTE(CONSTANTCONDITION)                                \
 602                 if (_lock)                                              \
 603                         mtx_lock(&((_esbp)->esb_lock));                 \
 604                                                                         \
 605                 EFSYS_PROBE2(bar_writed, unsigned int, (_offset),       \
 606                     uint32_t, (_edp)->ed_u32[0]);                       \
 607                                                                         \
 608                 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
 609                     (_offset), (_edp)->ed_u32[0]);                      \
 610                                                                         \
 611                 _NOTE(CONSTANTCONDITION)                                \
 612                 if (_lock)                                              \
 613                         mtx_unlock(&((_esbp)->esb_lock));               \
 614         _NOTE(CONSTANTCONDITION)                                        \
 615         } while (B_FALSE)
 616
 617 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp)                          \
 618         do {                                                            \
 619                 _NOTE(CONSTANTCONDITION)                                \
 620                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)),    \
 621                     ("not power of 2 aligned"));                        \
 622                                                                         \
 623                 mtx_lock(&((_esbp)->esb_lock));                         \
 624                                                                         \
 625                 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset),       \
 626                     uint32_t, (_eqp)->eq_u32[1],                        \
 627                     uint32_t, (_eqp)->eq_u32[0]);                       \
 628                                                                         \
 629                 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
 630                     (_offset), (_eqp)->eq_u32[0]);                      \
 631                 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
 632                     (_offset+4), (_eqp)->eq_u32[1]);                    \
 633                                                                         \
 634                 mtx_unlock(&((_esbp)->esb_lock));                       \
 635         _NOTE(CONSTANTCONDITION)                                        \
 636         } while (B_FALSE)
 637
 638 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock)                   \
 639         do {                                                            \
 640                 _NOTE(CONSTANTCONDITION)                                \
 641                 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)),    \
 642                     ("not power of 2 aligned"));                        \
 643                                                                         \
 644                 _NOTE(CONSTANTCONDITION)                                \
 645                 if (_lock)                                              \
 646                         mtx_lock(&((_esbp)->esb_lock));                 \
 647                                                                         \
 648                 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset),       \
 649                     uint32_t, (_eop)->eo_u32[3],                        \
 650                     uint32_t, (_eop)->eo_u32[2],                        \
 651                     uint32_t, (_eop)->eo_u32[1],                        \
 652                     uint32_t, (_eop)->eo_u32[0]);                       \
 653                                                                         \
 654                 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
 655                     (_offset), (_eop)->eo_u32[0]);                      \
 656                 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
 657                     (_offset+4), (_eop)->eo_u32[1]);                    \
 658                 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
 659                     (_offset+8), (_eop)->eo_u32[2]);                    \
 660                 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\
 661                     (_offset+12), (_eop)->eo_u32[3]);                   \
 662                                                                         \
 663                 _NOTE(CONSTANTCONDITION)                                \
 664                 if (_lock)                                              \
 665                         mtx_unlock(&((_esbp)->esb_lock));               \
 666         _NOTE(CONSTANTCONDITION)                                        \
 667         } while (B_FALSE)
 668
 669 /* SPIN */
 670
 671 #define EFSYS_SPIN(_us)                                                 \
 672         do {                                                            \
 673                 DELAY(_us);                                             \
 674         _NOTE(CONSTANTCONDITION)                                        \
 675         } while (B_FALSE)
 676
 677 #define EFSYS_SLEEP     EFSYS_SPIN
 678
 679 /* BARRIERS */
 680
 681 /* Strict ordering guaranteed by devacc.devacc_attr_dataorder */
 682 #define EFSYS_MEM_READ_BARRIER()
 683 #define EFSYS_PIO_WRITE_BARRIER()
 684
 685 /* TIMESTAMP */
 686
 687 typedef clock_t efsys_timestamp_t;
 688
 689 #define EFSYS_TIMESTAMP(_usp)                                           \
 690         do {                                                            \
 691                 clock_t now;                                            \
 692                                                                         \
 693                 now = ticks;                                            \
 694                 *(_usp) = now * hz / 1000000;                           \
 695         _NOTE(CONSTANTCONDITION)                                        \
 696         } while (B_FALSE)
 697
 698 /* KMEM */
 699
 700 #define EFSYS_KMEM_ALLOC(_esip, _size, _p)                              \
 701         do {                                                            \
 702                 (_esip) = (_esip);                                      \
 703                 (_p) = malloc((_size), M_SFXGE, M_WAITOK|M_ZERO);       \
 704         _NOTE(CONSTANTCONDITION)                                        \
 705         } while (B_FALSE)
 706
 707 #define EFSYS_KMEM_FREE(_esip, _size, _p)                               \
 708         do {                                                            \
 709                 (void) (_esip);                                         \
 710                 (void) (_size);                                         \
 711                 free((_p), M_SFXGE);                                    \
 712         _NOTE(CONSTANTCONDITION)                                        \
 713         } while (B_FALSE)
 714
 715 /* LOCK */
 716
 717 typedef struct mtx      efsys_lock_t;
 718
 719 #define EFSYS_LOCK_MAGIC        0x000010c4
 720
 721 #define EFSYS_LOCK(_lockp, _state)                                      \
 722         do {                                                            \
 723                 mtx_lock(_lockp);                                       \
 724                 (_state) = EFSYS_LOCK_MAGIC;                            \
 725         _NOTE(CONSTANTCONDITION)                                        \
 726         } while (B_FALSE)
 727
 728 #define EFSYS_UNLOCK(_lockp, _state)                                    \
 729         do {                                                            \
 730                 if ((_state) != EFSYS_LOCK_MAGIC)                       \
 731                         KASSERT(B_FALSE, ("not locked"));               \
 732                 mtx_unlock(_lockp);                                     \
 733         _NOTE(CONSTANTCONDITION)                                        \
 734         } while (B_FALSE)
 735
 736 /* PREEMPT */
 737
 738 #define EFSYS_PREEMPT_DISABLE(_state)                                   \
 739         do {                                                            \
 740                 (_state) = (_state);                                    \
 741                 critical_enter();                                       \
 742         _NOTE(CONSTANTCONDITION)                                        \
 743         } while (B_FALSE)
 744
 745 #define EFSYS_PREEMPT_ENABLE(_state)                                    \
 746         do {                                                            \
 747                 (_state) = (_state);                                    \
 748                 critical_exit(_state);                                  \
 749         _NOTE(CONSTANTCONDITION)                                        \
 750         } while (B_FALSE)
 751
 752 /* STAT */
 753
 754 typedef uint64_t                efsys_stat_t;
 755
 756 #define EFSYS_STAT_INCR(_knp, _delta)                                   \
 757         do {                                                            \
 758                 *(_knp) += (_delta);                                    \
 759         _NOTE(CONSTANTCONDITION)                                        \
 760         } while (B_FALSE)
 761
 762 #define EFSYS_STAT_DECR(_knp, _delta)                                   \
 763         do {                                                            \
 764                 *(_knp) -= (_delta);                                    \
 765         _NOTE(CONSTANTCONDITION)                                        \
 766         } while (B_FALSE)
 767
 768 #define EFSYS_STAT_SET(_knp, _val)                                      \
 769         do {                                                            \
 770                 *(_knp) = (_val);                                       \
 771         _NOTE(CONSTANTCONDITION)                                        \
 772         } while (B_FALSE)
 773
 774 #define EFSYS_STAT_SET_QWORD(_knp, _valp)                               \
 775         do {                                                            \
 776                 *(_knp) = le64toh((_valp)->eq_u64[0]);                  \
 777         _NOTE(CONSTANTCONDITION)                                        \
 778         } while (B_FALSE)
 779
 780 #define EFSYS_STAT_SET_DWORD(_knp, _valp)                               \
 781         do {                                                            \
 782                 *(_knp) = le32toh((_valp)->ed_u32[0]);                  \
 783         _NOTE(CONSTANTCONDITION)                                        \
 784         } while (B_FALSE)
 785
 786 #define EFSYS_STAT_INCR_QWORD(_knp, _valp)                              \
 787         do {                                                            \
 788                 *(_knp) += le64toh((_valp)->eq_u64[0]);                 \
 789         _NOTE(CONSTANTCONDITION)                                        \
 790         } while (B_FALSE)
 791
 792 #define EFSYS_STAT_SUBR_QWORD(_knp, _valp)                              \
 793         do {                                                            \
 794                 *(_knp) -= le64toh((_valp)->eq_u64[0]);                 \
 795         _NOTE(CONSTANTCONDITION)                                        \
 796         } while (B_FALSE)
 797
 798 /* ERR */
 799
 800 extern void     sfxge_err(efsys_identifier_t *, unsigned int,
 801                     uint32_t, uint32_t);
 802
 803 #if EFSYS_OPT_DECODE_INTR_FATAL
 804 #define EFSYS_ERR(_esip, _code, _dword0, _dword1)                       \
 805         do {                                                            \
 806                 sfxge_err((_esip), (_code), (_dword0), (_dword1));      \
 807         _NOTE(CONSTANTCONDITION)                                        \
 808         } while (B_FALSE)
 809 #endif
 810
 811 /* ASSERT */
 812
 813 #define EFSYS_ASSERT(_exp) do {                                         \
 814         if (!(_exp))                                                    \
 815                 panic(#_exp);                                           \
 816         } while (0)
 817
 818 #define EFSYS_ASSERT3(_x, _op, _y, _t) do {                             \
 819         const _t __x = (_t)(_x);                                        \
 820         const _t __y = (_t)(_y);                                        \
 821         if (!(__x _op __y))                                             \
 822                 panic("assertion failed at %s:%u", __FILE__, __LINE__); \
 823         } while(0)
 824
 825 #define EFSYS_ASSERT3U(_x, _op, _y)     EFSYS_ASSERT3(_x, _op, _y, uint64_t)
 826 #define EFSYS_ASSERT3S(_x, _op, _y)     EFSYS_ASSERT3(_x, _op, _y, int64_t)
 827 #define EFSYS_ASSERT3P(_x, _op, _y)     EFSYS_ASSERT3(_x, _op, _y, uintptr_t)
 828
 829 #ifdef  __cplusplus
 830 }
 831 #endif
 832
 833 #endif  /* _SYS_EFSYS_H */