sys/dev/sfxge/common/efx_types.h

   1 /*-
   2  * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
  14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  16  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  23  * SUCH DAMAGE.
  24  *
  25  * Ackowledgement to Fen Systems Ltd.
  26  *
  27  * $FreeBSD$
  28  */
  29
  30 #ifndef _SYS_EFX_TYPES_H
  31 #define _SYS_EFX_TYPES_H
  32
  33 #include "efsys.h"
  34
  35 #ifdef  __cplusplus
  36 extern "C" {
  37 #endif
  38
  39 /*
  40  * Bitfield access
  41  *
  42  * Solarflare NICs make extensive use of bitfields up to 128 bits
  43  * wide.  Since there is no native 128-bit datatype on most systems,
  44  * and since 64-bit datatypes are inefficient on 32-bit systems and
  45  * vice versa, we wrap accesses in a way that uses the most efficient
  46  * datatype.
  47  *
  48  * The NICs are PCI devices and therefore little-endian.  Since most
  49  * of the quantities that we deal with are DMAed to/from host memory,
  50  * we define    our datatypes (efx_oword_t, efx_qword_t and efx_dword_t)
  51  * to be little-endian.
  52  *
  53  * In the less common case of using PIO for individual register
  54  * writes, we construct the little-endian datatype in host memory and
  55  * then use non-swapping register access primitives, rather than
  56  * constructing a native-endian datatype and relying on implicit
  57  * byte-swapping.  (We use a similar strategy for register reads.)
  58  */
  59
  60 /*
  61  * NOTE: Field definitions here and elsewhere are done in terms of a lowest
  62  *       bit number (LBN) and a width.
  63  */
  64
  65 #define EFX_DUMMY_FIELD_LBN 0
  66 #define EFX_DUMMY_FIELD_WIDTH 0
  67
  68 #define EFX_BYTE_0_LBN 0
  69 #define EFX_BYTE_0_WIDTH 8
  70
  71 #define EFX_BYTE_1_LBN 8
  72 #define EFX_BYTE_1_WIDTH 8
  73
  74 #define EFX_BYTE_2_LBN 16
  75 #define EFX_BYTE_2_WIDTH 8
  76
  77 #define EFX_BYTE_3_LBN 24
  78 #define EFX_BYTE_3_WIDTH 8
  79
  80 #define EFX_BYTE_4_LBN 32
  81 #define EFX_BYTE_4_WIDTH 8
  82
  83 #define EFX_BYTE_5_LBN 40
  84 #define EFX_BYTE_5_WIDTH 8
  85
  86 #define EFX_BYTE_6_LBN 48
  87 #define EFX_BYTE_6_WIDTH 8
  88
  89 #define EFX_BYTE_7_LBN 56
  90 #define EFX_BYTE_7_WIDTH 8
  91
  92 #define EFX_WORD_0_LBN 0
  93 #define EFX_WORD_0_WIDTH 16
  94
  95 #define EFX_WORD_1_LBN 16
  96 #define EFX_WORD_1_WIDTH 16
  97
  98 #define EFX_WORD_2_LBN 32
  99 #define EFX_WORD_2_WIDTH 16
 100
 101 #define EFX_WORD_3_LBN 48
 102 #define EFX_WORD_3_WIDTH 16
 103
 104 #define EFX_DWORD_0_LBN 0
 105 #define EFX_DWORD_0_WIDTH 32
 106
 107 #define EFX_DWORD_1_LBN 32
 108 #define EFX_DWORD_1_WIDTH 32
 109
 110 #define EFX_DWORD_2_LBN 64
 111 #define EFX_DWORD_2_WIDTH 32
 112
 113 #define EFX_DWORD_3_LBN 96
 114 #define EFX_DWORD_3_WIDTH 32
 115
 116 #define EFX_QWORD_0_LBN 0
 117 #define EFX_QWORD_0_WIDTH 64
 118
 119 #define EFX_QWORD_1_LBN 64
 120 #define EFX_QWORD_1_WIDTH 64
 121
 122 /* Specified attribute (i.e. LBN ow WIDTH) of the specified field */
 123 #define EFX_VAL(_field, _attribute)                                     \
 124         _field ## _ ## _attribute
 125
 126 /* Lowest bit number of the specified field */
 127 #define EFX_LOW_BIT(_field)                                             \
 128         EFX_VAL(_field, LBN)
 129
 130 /* Width of the specified field */
 131 #define EFX_WIDTH(_field)                                               \
 132         EFX_VAL(_field, WIDTH)
 133
 134 /* Highest bit number of the specified field */
 135 #define EFX_HIGH_BIT(_field)                                            \
 136         (EFX_LOW_BIT(_field) + EFX_WIDTH(_field) - 1)
 137
 138 /*
 139  * 64-bit mask equal in width to the specified field.
 140  *
 141  * For example, a field with width 5 would have a mask of 0x000000000000001f.
 142  */
 143 #define EFX_MASK64(_field)                                              \
 144         ((EFX_WIDTH(_field) == 64) ? ~((uint64_t)0) :                   \
 145             (((((uint64_t)1) << EFX_WIDTH(_field))) - 1))
 146 /*
 147  * 32-bit mask equal in width to the specified field.
 148  *
 149  * For example, a field with width 5 would have a mask of 0x0000001f.
 150  */
 151 #define EFX_MASK32(_field)                                              \
 152         ((EFX_WIDTH(_field) == 32) ? ~((uint32_t)0) :                   \
 153             (((((uint32_t)1) << EFX_WIDTH(_field))) - 1))
 154
 155 /*
 156  * 16-bit mask equal in width to the specified field.
 157  *
 158  * For example, a field with width 5 would have a mask of 0x001f.
 159  */
 160 #define EFX_MASK16(_field)                                              \
 161         ((EFX_WIDTH(_field) == 16) ? 0xffffu :                          \
 162             (uint16_t)((1 << EFX_WIDTH(_field)) - 1))
 163
 164 /*
 165  * 8-bit mask equal in width to the specified field.
 166  *
 167  * For example, a field with width 5 would have a mask of 0x1f.
 168  */
 169 #define EFX_MASK8(_field)                                               \
 170         ((uint8_t)((1 << EFX_WIDTH(_field)) - 1))
 171
 172 #pragma pack(1)
 173
 174 /*
 175  * A byte (i.e. 8-bit) datatype
 176  */
 177 typedef union efx_byte_u {
 178         uint8_t eb_u8[1];
 179 } efx_byte_t;
 180
 181 /*
 182  * A word (i.e. 16-bit) datatype
 183  *
 184  * This datatype is defined to be little-endian.
 185  */
 186 typedef union efx_word_u {
 187         efx_byte_t ew_byte[2];
 188         uint16_t ew_u16[1];
 189         uint8_t ew_u8[2];
 190 } efx_word_t;
 191
 192 /*
 193  * A doubleword (i.e. 32-bit) datatype
 194  *
 195  * This datatype is defined to be little-endian.
 196  */
 197 typedef union efx_dword_u {
 198         efx_byte_t ed_byte[4];
 199         efx_word_t ed_word[2];
 200         uint32_t ed_u32[1];
 201         uint16_t ed_u16[2];
 202         uint8_t ed_u8[4];
 203 } efx_dword_t;
 204
 205 /*
 206  * A quadword (i.e. 64-bit) datatype
 207  *
 208  * This datatype is defined to be little-endian.
 209  */
 210 typedef union efx_qword_u {
 211         efx_byte_t eq_byte[8];
 212         efx_word_t eq_word[4];
 213         efx_dword_t eq_dword[2];
 214 #if EFSYS_HAS_UINT64
 215         uint64_t eq_u64[1];
 216 #endif
 217         uint32_t eq_u32[2];
 218         uint16_t eq_u16[4];
 219         uint8_t eq_u8[8];
 220 } efx_qword_t;
 221
 222 /*
 223  * An octword (i.e. 128-bit) datatype
 224  *
 225  * This datatype is defined to be little-endian.
 226  */
 227 typedef union efx_oword_u {
 228         efx_byte_t eo_byte[16];
 229         efx_word_t eo_word[8];
 230         efx_dword_t eo_dword[4];
 231         efx_qword_t eo_qword[2];
 232 #if EFSYS_HAS_UINT64
 233         uint64_t eo_u64[2];
 234 #endif
 235         uint32_t eo_u32[4];
 236         uint16_t eo_u16[8];
 237         uint8_t eo_u8[16];
 238 } efx_oword_t;
 239
 240 #pragma pack()
 241
 242 #define __SWAP16(_x)                            \
 243         ((((_x) & 0xff) << 8) |                 \
 244         (((_x) >> 8) & 0xff))
 245
 246 #define __SWAP32(_x)                            \
 247         ((__SWAP16((_x) & 0xffff) << 16) |      \
 248         __SWAP16(((_x) >> 16) & 0xffff))
 249
 250 #define __SWAP64(_x)                            \
 251         ((__SWAP32((_x) & 0xffffffff) << 32) |  \
 252         __SWAP32(((_x) >> 32) & 0xffffffff))
 253
 254 #define __NOSWAP16(_x)          (_x)
 255 #define __NOSWAP32(_x)          (_x)
 256 #define __NOSWAP64(_x)          (_x)
 257
 258 #if EFSYS_IS_BIG_ENDIAN
 259
 260 #define __CPU_TO_LE_16(_x)      (uint16_t)__SWAP16(_x)
 261 #define __LE_TO_CPU_16(_x)      (uint16_t)__SWAP16(_x)
 262 #define __CPU_TO_BE_16(_x)      (uint16_t)__NOSWAP16(_x)
 263 #define __BE_TO_CPU_16(_x)      (uint16_t)__NOSWAP16(_x)
 264
 265 #define __CPU_TO_LE_32(_x)      (uint32_t)__SWAP32(_x)
 266 #define __LE_TO_CPU_32(_x)      (uint32_t)__SWAP32(_x)
 267 #define __CPU_TO_BE_32(_x)      (uint32_t)__NOSWAP32(_x)
 268 #define __BE_TO_CPU_32(_x)      (uint32_t)__NOSWAP32(_x)
 269
 270 #define __CPU_TO_LE_64(_x)      (uint64_t)__SWAP64(_x)
 271 #define __LE_TO_CPU_64(_x)      (uint64_t)__SWAP64(_x)
 272 #define __CPU_TO_BE_64(_x)      (uint64_t)__NOSWAP64(_x)
 273 #define __BE_TO_CPU_64(_x)      (uint64_t)__NOSWAP64(_x)
 274
 275 #elif EFSYS_IS_LITTLE_ENDIAN
 276
 277 #define __CPU_TO_LE_16(_x)      (uint16_t)__NOSWAP16(_x)
 278 #define __LE_TO_CPU_16(_x)      (uint16_t)__NOSWAP16(_x)
 279 #define __CPU_TO_BE_16(_x)      (uint16_t)__SWAP16(_x)
 280 #define __BE_TO_CPU_16(_x)      (uint16_t)__SWAP16(_x)
 281
 282 #define __CPU_TO_LE_32(_x)      (uint32_t)__NOSWAP32(_x)
 283 #define __LE_TO_CPU_32(_x)      (uint32_t)__NOSWAP32(_x)
 284 #define __CPU_TO_BE_32(_x)      (uint32_t)__SWAP32(_x)
 285 #define __BE_TO_CPU_32(_x)      (uint32_t)__SWAP32(_x)
 286
 287 #define __CPU_TO_LE_64(_x)      (uint64_t)__NOSWAP64(_x)
 288 #define __LE_TO_CPU_64(_x)      (uint64_t)__NOSWAP64(_x)
 289 #define __CPU_TO_BE_64(_x)      (uint64_t)__SWAP64(_x)
 290 #define __BE_TO_CPU_64(_x)      (uint64_t)__SWAP64(_x)
 291
 292 #else
 293
 294 #error "Neither of EFSYS_IS_{BIG,LITTLE}_ENDIAN is set"
 295
 296 #endif
 297
 298 #define __NATIVE_8(_x)  (uint8_t)(_x)
 299
 300 /* Format string for printing an efx_byte_t */
 301 #define EFX_BYTE_FMT "0x%02x"
 302
 303 /* Format string for printing an efx_word_t */
 304 #define EFX_WORD_FMT "0x%04x"
 305
 306 /* Format string for printing an efx_dword_t */
 307 #define EFX_DWORD_FMT "0x%08x"
 308
 309 /* Format string for printing an efx_qword_t */
 310 #define EFX_QWORD_FMT "0x%08x:%08x"
 311
 312 /* Format string for printing an efx_oword_t */
 313 #define EFX_OWORD_FMT "0x%08x:%08x:%08x:%08x"
 314
 315 /* Parameters for printing an efx_byte_t */
 316 #define EFX_BYTE_VAL(_byte)                                     \
 317         ((unsigned int)__NATIVE_8((_byte).eb_u8[0]))
 318
 319 /* Parameters for printing an efx_word_t */
 320 #define EFX_WORD_VAL(_word)                                     \
 321         ((unsigned int)__LE_TO_CPU_16((_word).ew_u16[0]))
 322
 323 /* Parameters for printing an efx_dword_t */
 324 #define EFX_DWORD_VAL(_dword)                                   \
 325         ((unsigned int)__LE_TO_CPU_32((_dword).ed_u32[0]))
 326
 327 /* Parameters for printing an efx_qword_t */
 328 #define EFX_QWORD_VAL(_qword)                                   \
 329         ((unsigned int)__LE_TO_CPU_32((_qword).eq_u32[1])),     \
 330         ((unsigned int)__LE_TO_CPU_32((_qword).eq_u32[0]))
 331
 332 /* Parameters for printing an efx_oword_t */
 333 #define EFX_OWORD_VAL(_oword)                                   \
 334         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[3])),     \
 335         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[2])),     \
 336         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[1])),     \
 337         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[0]))
 338
 339 /*
 340  * Stop lint complaining about some shifts.
 341  */
 342 #ifdef  __lint
 343 extern int fix_lint;
 344 #define FIX_LINT(_x)    (_x + fix_lint)
 345 #else
 346 #define FIX_LINT(_x)    (_x)
 347 #endif
 348
 349 /*
 350  * Extract bit field portion [low,high) from the native-endian element
 351  * which contains bits [min,max).
 352  *
 353  * For example, suppose "element" represents the high 32 bits of a
 354  * 64-bit value, and we wish to extract the bits belonging to the bit
 355  * field occupying bits 28-45 of this 64-bit value.
 356  *
 357  * Then EFX_EXTRACT(_element, 32, 63, 28, 45) would give
 358  *
 359  *   (_element) << 4
 360  *
 361  * The result will contain the relevant bits filled in in the range
 362  * [0,high-low), with garbage in bits [high-low+1,...).
 363  */
 364 #define EFX_EXTRACT_NATIVE(_element, _min, _max, _low, _high)           \
 365         ((FIX_LINT(_low > _max) || FIX_LINT(_high < _min)) ?            \
 366                 0U :                                                    \
 367                 ((_low > _min) ?                                        \
 368                         ((_element) >> (_low - _min)) :                 \
 369                         ((_element) << (_min - _low))))
 370
 371 /*
 372  * Extract bit field portion [low,high) from the 64-bit little-endian
 373  * element which contains bits [min,max)
 374  */
 375 #define EFX_EXTRACT64(_element, _min, _max, _low, _high)                \
 376         EFX_EXTRACT_NATIVE(__LE_TO_CPU_64(_element), _min, _max, _low, _high)
 377
 378 /*
 379  * Extract bit field portion [low,high) from the 32-bit little-endian
 380  * element which contains bits [min,max)
 381  */
 382 #define EFX_EXTRACT32(_element, _min, _max, _low, _high)                \
 383         EFX_EXTRACT_NATIVE(__LE_TO_CPU_32(_element), _min, _max, _low, _high)
 384
 385 /*
 386  * Extract bit field portion [low,high) from the 16-bit little-endian
 387  * element which contains bits [min,max)
 388  */
 389 #define EFX_EXTRACT16(_element, _min, _max, _low, _high)                \
 390         EFX_EXTRACT_NATIVE(__LE_TO_CPU_16(_element), _min, _max, _low, _high)
 391
 392 /*
 393  * Extract bit field portion [low,high) from the 8-bit
 394  * element which contains bits [min,max)
 395  */
 396 #define EFX_EXTRACT8(_element, _min, _max, _low, _high)                 \
 397         EFX_EXTRACT_NATIVE(__NATIVE_8(_element), _min, _max, _low, _high)
 398
 399 #define EFX_EXTRACT_OWORD64(_oword, _low, _high)                        \
 400         (EFX_EXTRACT64((_oword).eo_u64[0], FIX_LINT(0), FIX_LINT(63),   \
 401             _low, _high) |                                              \
 402         EFX_EXTRACT64((_oword).eo_u64[1], FIX_LINT(64), FIX_LINT(127),  \
 403             _low, _high))
 404
 405 #define EFX_EXTRACT_OWORD32(_oword, _low, _high)                        \
 406         (EFX_EXTRACT32((_oword).eo_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 407             _low, _high) |                                              \
 408         EFX_EXTRACT32((_oword).eo_u32[1], FIX_LINT(32), FIX_LINT(63),   \
 409             _low, _high) |                                              \
 410         EFX_EXTRACT32((_oword).eo_u32[2], FIX_LINT(64), FIX_LINT(95),   \
 411             _low, _high) |                                              \
 412         EFX_EXTRACT32((_oword).eo_u32[3], FIX_LINT(96), FIX_LINT(127),  \
 413             _low, _high))
 414
 415 #define EFX_EXTRACT_QWORD64(_qword, _low, _high)                        \
 416         (EFX_EXTRACT64((_qword).eq_u64[0], FIX_LINT(0), FIX_LINT(63),   \
 417             _low, _high))
 418
 419 #define EFX_EXTRACT_QWORD32(_qword, _low, _high)                        \
 420         (EFX_EXTRACT32((_qword).eq_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 421             _low, _high) |                                              \
 422         EFX_EXTRACT32((_qword).eq_u32[1], FIX_LINT(32), FIX_LINT(63),   \
 423             _low, _high))
 424
 425 #define EFX_EXTRACT_DWORD(_dword, _low, _high)                          \
 426         (EFX_EXTRACT32((_dword).ed_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 427             _low, _high))
 428
 429 #define EFX_EXTRACT_WORD(_word, _low, _high)                            \
 430         (EFX_EXTRACT16((_word).ew_u16[0], FIX_LINT(0), FIX_LINT(15),    \
 431             _low, _high))
 432
 433 #define EFX_EXTRACT_BYTE(_byte, _low, _high)                            \
 434         (EFX_EXTRACT8((_byte).eb_u8[0], FIX_LINT(0), FIX_LINT(7),       \
 435             _low, _high))
 436
 437
 438 #define EFX_OWORD_FIELD64(_oword, _field)                               \
 439         ((uint32_t)EFX_EXTRACT_OWORD64(_oword, EFX_LOW_BIT(_field),     \
 440             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 441
 442 #define EFX_OWORD_FIELD32(_oword, _field)                               \
 443         (EFX_EXTRACT_OWORD32(_oword, EFX_LOW_BIT(_field),               \
 444             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 445
 446 #define EFX_QWORD_FIELD64(_qword, _field)                               \
 447         ((uint32_t)EFX_EXTRACT_QWORD64(_qword, EFX_LOW_BIT(_field),     \
 448             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 449
 450 #define EFX_QWORD_FIELD32(_qword, _field)                               \
 451         (EFX_EXTRACT_QWORD32(_qword, EFX_LOW_BIT(_field),               \
 452             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 453
 454 #define EFX_DWORD_FIELD(_dword, _field)                                 \
 455         (EFX_EXTRACT_DWORD(_dword, EFX_LOW_BIT(_field),                 \
 456             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 457
 458 #define EFX_WORD_FIELD(_word, _field)                                   \
 459         (EFX_EXTRACT_WORD(_word, EFX_LOW_BIT(_field),                   \
 460             EFX_HIGH_BIT(_field)) & EFX_MASK16(_field))
 461
 462 #define EFX_BYTE_FIELD(_byte, _field)                                   \
 463         (EFX_EXTRACT_BYTE(_byte, EFX_LOW_BIT(_field),                   \
 464             EFX_HIGH_BIT(_field)) & EFX_MASK8(_field))
 465
 466
 467 #define EFX_OWORD_IS_EQUAL64(_oword_a, _oword_b)                        \
 468         ((_oword_a).eo_u64[0] == (_oword_b).eo_u64[0] &&                \
 469             (_oword_a).eo_u64[1] == (_oword_b).eo_u64[1])
 470
 471 #define EFX_OWORD_IS_EQUAL32(_oword_a, _oword_b)                        \
 472         ((_oword_a).eo_u32[0] == (_oword_b).eo_u32[0] &&                \
 473             (_oword_a).eo_u32[1] == (_oword_b).eo_u32[1] &&             \
 474             (_oword_a).eo_u32[2] == (_oword_b).eo_u32[2] &&             \
 475             (_oword_a).eo_u32[3] == (_oword_b).eo_u32[3])
 476
 477 #define EFX_QWORD_IS_EQUAL64(_qword_a, _qword_b)                        \
 478         ((_qword_a).eq_u64[0] == (_qword_b).eq_u64[0])
 479
 480 #define EFX_QWORD_IS_EQUAL32(_qword_a, _qword_b)                        \
 481         ((_qword_a).eq_u32[0] == (_qword_b).eq_u32[0] &&                \
 482             (_qword_a).eq_u32[1] == (_qword_b).eq_u32[1])
 483
 484 #define EFX_DWORD_IS_EQUAL(_dword_a, _dword_b)                          \
 485         ((_dword_a).ed_u32[0] == (_dword_b).ed_u32[0])
 486
 487 #define EFX_WORD_IS_EQUAL(_word_a, _word_b)                             \
 488         ((_word_a).ew_u16[0] == (_word_b).ew_u16[0])
 489
 490 #define EFX_BYTE_IS_EQUAL(_byte_a, _byte_b)                             \
 491         ((_byte_a).eb_u8[0] == (_byte_b).eb_u8[0])
 492
 493
 494 #define EFX_OWORD_IS_ZERO64(_oword)                                     \
 495         (((_oword).eo_u64[0] |                                          \
 496             (_oword).eo_u64[1]) == 0)
 497
 498 #define EFX_OWORD_IS_ZERO32(_oword)                                     \
 499         (((_oword).eo_u32[0] |                                          \
 500             (_oword).eo_u32[1] |                                        \
 501             (_oword).eo_u32[2] |                                        \
 502             (_oword).eo_u32[3]) == 0)
 503
 504 #define EFX_QWORD_IS_ZERO64(_qword)                                     \
 505         (((_qword).eq_u64[0]) == 0)
 506
 507 #define EFX_QWORD_IS_ZERO32(_qword)                                     \
 508         (((_qword).eq_u32[0] |                                          \
 509             (_qword).eq_u32[1]) == 0)
 510
 511 #define EFX_DWORD_IS_ZERO(_dword)                                       \
 512         (((_dword).ed_u32[0]) == 0)
 513
 514 #define EFX_WORD_IS_ZERO(_word)                                         \
 515         (((_word).ew_u16[0]) == 0)
 516
 517 #define EFX_BYTE_IS_ZERO(_byte)                                         \
 518         (((_byte).eb_u8[0]) == 0)
 519
 520
 521 #define EFX_OWORD_IS_SET64(_oword)                                      \
 522         (((_oword).eo_u64[0] &                                          \
 523             (_oword).eo_u64[1]) == ~((uint64_t)0))
 524
 525 #define EFX_OWORD_IS_SET32(_oword)                                      \
 526         (((_oword).eo_u32[0] &                                          \
 527             (_oword).eo_u32[1] &                                        \
 528             (_oword).eo_u32[2] &                                        \
 529             (_oword).eo_u32[3]) == ~((uint32_t)0))
 530
 531 #define EFX_QWORD_IS_SET64(_qword)                                      \
 532         (((_qword).eq_u64[0]) == ~((uint32_t)0))
 533
 534 #define EFX_QWORD_IS_SET32(_qword)                                      \
 535         (((_qword).eq_u32[0] &                                          \
 536             (_qword).eq_u32[1]) == ~((uint32_t)0))
 537
 538 #define EFX_DWORD_IS_SET(_dword)                                        \
 539         ((_dword).ed_u32[0] == ~((uint32_t)0))
 540
 541 #define EFX_WORD_IS_SET(_word)                                          \
 542         ((_word).ew_u16[0] == ~((uint16_t)0))
 543
 544 #define EFX_BYTE_IS_SET(_byte)                                          \
 545         ((_byte).eb_u8[0] == ~((uint8_t)0))
 546
 547 /*
 548  * Construct bit field portion
 549  *
 550  * Creates the portion of the bit field [low,high) that lies within
 551  * the range [min,max).
 552  */
 553
 554 #define EFX_INSERT_NATIVE64(_min, _max, _low, _high, _value)            \
 555         (((_low > _max) || (_high < _min)) ?                            \
 556                 0U :                                                    \
 557                 ((_low > _min) ?                                        \
 558                         (((uint64_t)(_value)) << (_low - _min)) :       \
 559                         (((uint64_t)(_value)) >> (_min - _low))))
 560
 561 #define EFX_INSERT_NATIVE32(_min, _max, _low, _high, _value)            \
 562         (((_low > _max) || (_high < _min)) ?                            \
 563                 0U :                                                    \
 564                 ((_low > _min) ?                                        \
 565                         (((uint32_t)(_value)) << (_low - _min)) :       \
 566                         (((uint32_t)(_value)) >> (_min - _low))))
 567
 568 #define EFX_INSERT_NATIVE16(_min, _max, _low, _high, _value)            \
 569         (((_low > _max) || (_high < _min)) ?                            \
 570                 0U :                                                    \
 571                 (uint16_t)((_low > _min) ?                              \
 572                                 ((_value) << (_low - _min)) :           \
 573                                 ((_value) >> (_min - _low))))
 574
 575 #define EFX_INSERT_NATIVE8(_min, _max, _low, _high, _value)             \
 576         (((_low > _max) || (_high < _min)) ?                            \
 577                 0U :                                                    \
 578                 (uint8_t)((_low > _min) ?                               \
 579                                 ((_value) << (_low - _min)) :   \
 580                                 ((_value) >> (_min - _low))))
 581
 582 /*
 583  * Construct bit field portion
 584  *
 585  * Creates the portion of the named bit field that lies within the
 586  * range [min,max).
 587  */
 588 #define EFX_INSERT_FIELD_NATIVE64(_min, _max, _field, _value)           \
 589         EFX_INSERT_NATIVE64(_min, _max, EFX_LOW_BIT(_field),            \
 590             EFX_HIGH_BIT(_field), _value)
 591
 592 #define EFX_INSERT_FIELD_NATIVE32(_min, _max, _field, _value)           \
 593         EFX_INSERT_NATIVE32(_min, _max, EFX_LOW_BIT(_field),            \
 594             EFX_HIGH_BIT(_field), _value)
 595
 596 #define EFX_INSERT_FIELD_NATIVE16(_min, _max, _field, _value)           \
 597         EFX_INSERT_NATIVE16(_min, _max, EFX_LOW_BIT(_field),            \
 598             EFX_HIGH_BIT(_field), _value)
 599
 600 #define EFX_INSERT_FIELD_NATIVE8(_min, _max, _field, _value)            \
 601         EFX_INSERT_NATIVE8(_min, _max, EFX_LOW_BIT(_field),             \
 602             EFX_HIGH_BIT(_field), _value)
 603
 604 /*
 605  * Construct bit field
 606  *
 607  * Creates the portion of the named bit fields that lie within the
 608  * range [min,max).
 609  */
 610 #define EFX_INSERT_FIELDS64(_min, _max,                                 \
 611             _field1, _value1, _field2, _value2, _field3, _value3,       \
 612             _field4, _value4, _field5, _value5, _field6, _value6,       \
 613             _field7, _value7, _field8, _value8, _field9, _value9,       \
 614             _field10, _value10)                                         \
 615         __CPU_TO_LE_64(                                                 \
 616             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field1, _value1) |   \
 617             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field2, _value2) |   \
 618             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field3, _value3) |   \
 619             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field4, _value4) |   \
 620             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field5, _value5) |   \
 621             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field6, _value6) |   \
 622             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field7, _value7) |   \
 623             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field8, _value8) |   \
 624             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field9, _value9) |   \
 625             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field10, _value10))
 626
 627 #define EFX_INSERT_FIELDS32(_min, _max,                                 \
 628             _field1, _value1, _field2, _value2, _field3, _value3,       \
 629             _field4, _value4, _field5, _value5, _field6, _value6,       \
 630             _field7, _value7, _field8, _value8, _field9, _value9,       \
 631             _field10, _value10)                                         \
 632         __CPU_TO_LE_32(                                                 \
 633             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field1, _value1) |   \
 634             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field2, _value2) |   \
 635             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field3, _value3) |   \
 636             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field4, _value4) |   \
 637             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field5, _value5) |   \
 638             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field6, _value6) |   \
 639             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field7, _value7) |   \
 640             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field8, _value8) |   \
 641             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field9, _value9) |   \
 642             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field10, _value10))
 643
 644 #define EFX_INSERT_FIELDS16(_min, _max,                                 \
 645             _field1, _value1, _field2, _value2, _field3, _value3,       \
 646             _field4, _value4, _field5, _value5, _field6, _value6,       \
 647             _field7, _value7, _field8, _value8, _field9, _value9,       \
 648             _field10, _value10)                                         \
 649         __CPU_TO_LE_16(                                                 \
 650             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field1, _value1) |   \
 651             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field2, _value2) |   \
 652             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field3, _value3) |   \
 653             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field4, _value4) |   \
 654             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field5, _value5) |   \
 655             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field6, _value6) |   \
 656             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field7, _value7) |   \
 657             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field8, _value8) |   \
 658             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field9, _value9) |   \
 659             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field10, _value10))
 660
 661 #define EFX_INSERT_FIELDS8(_min, _max,                                  \
 662             _field1, _value1, _field2, _value2, _field3, _value3,       \
 663             _field4, _value4, _field5, _value5, _field6, _value6,       \
 664             _field7, _value7, _field8, _value8, _field9, _value9,       \
 665             _field10, _value10)                                         \
 666         __NATIVE_8(                                                     \
 667             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field1, _value1) |    \
 668             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field2, _value2) |    \
 669             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field3, _value3) |    \
 670             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field4, _value4) |    \
 671             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field5, _value5) |    \
 672             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field6, _value6) |    \
 673             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field7, _value7) |    \
 674             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field8, _value8) |    \
 675             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field9, _value9) |    \
 676             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field10, _value10))
 677
 678 #define EFX_POPULATE_OWORD64(_oword,                                    \
 679             _field1, _value1, _field2, _value2, _field3, _value3,       \
 680             _field4, _value4, _field5, _value5, _field6, _value6,       \
 681             _field7, _value7, _field8, _value8, _field9, _value9,       \
 682             _field10, _value10)                                         \
 683         do {                                                            \
 684                 _NOTE(CONSTANTCONDITION)                                \
 685                 (_oword).eo_u64[0] = EFX_INSERT_FIELDS64(0, 63,         \
 686                     _field1, _value1, _field2, _value2,                 \
 687                     _field3, _value3, _field4, _value4,                 \
 688                     _field5, _value5, _field6, _value6,                 \
 689                     _field7, _value7, _field8, _value8,                 \
 690                     _field9, _value9, _field10, _value10);              \
 691                 _NOTE(CONSTANTCONDITION)                                \
 692                 (_oword).eo_u64[1] = EFX_INSERT_FIELDS64(64, 127,       \
 693                     _field1, _value1, _field2, _value2,                 \
 694                     _field3, _value3, _field4, _value4,                 \
 695                     _field5, _value5, _field6, _value6,                 \
 696                     _field7, _value7, _field8, _value8,                 \
 697                     _field9, _value9, _field10, _value10);              \
 698         _NOTE(CONSTANTCONDITION)                                        \
 699         } while (B_FALSE)
 700
 701 #define EFX_POPULATE_OWORD32(_oword,                                    \
 702             _field1, _value1, _field2, _value2, _field3, _value3,       \
 703             _field4, _value4, _field5, _value5, _field6, _value6,       \
 704             _field7, _value7, _field8, _value8, _field9, _value9,       \
 705             _field10, _value10)                                         \
 706         do {                                                            \
 707                 _NOTE(CONSTANTCONDITION)                                \
 708                 (_oword).eo_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 709                     _field1, _value1, _field2, _value2,                 \
 710                     _field3, _value3, _field4, _value4,                 \
 711                     _field5, _value5, _field6, _value6,                 \
 712                     _field7, _value7, _field8, _value8,                 \
 713                     _field9, _value9, _field10, _value10);              \
 714                 _NOTE(CONSTANTCONDITION)                                \
 715                 (_oword).eo_u32[1] = EFX_INSERT_FIELDS32(32, 63,        \
 716                     _field1, _value1, _field2, _value2,                 \
 717                     _field3, _value3, _field4, _value4,                 \
 718                     _field5, _value5, _field6, _value6,                 \
 719                     _field7, _value7, _field8, _value8,                 \
 720                     _field9, _value9, _field10, _value10);              \
 721                 _NOTE(CONSTANTCONDITION)                                \
 722                 (_oword).eo_u32[2] = EFX_INSERT_FIELDS32(64, 95,        \
 723                     _field1, _value1, _field2, _value2,                 \
 724                     _field3, _value3, _field4, _value4,                 \
 725                     _field5, _value5, _field6, _value6,                 \
 726                     _field7, _value7, _field8, _value8,                 \
 727                     _field9, _value9, _field10, _value10);              \
 728                 _NOTE(CONSTANTCONDITION)                                \
 729                 (_oword).eo_u32[3] = EFX_INSERT_FIELDS32(96, 127,       \
 730                     _field1, _value1, _field2, _value2,                 \
 731                     _field3, _value3, _field4, _value4,                 \
 732                     _field5, _value5, _field6, _value6,                 \
 733                     _field7, _value7, _field8, _value8,                 \
 734                     _field9, _value9, _field10, _value10);              \
 735         _NOTE(CONSTANTCONDITION)                                        \
 736         } while (B_FALSE)
 737
 738 #define EFX_POPULATE_QWORD64(_qword,                                    \
 739             _field1, _value1, _field2, _value2, _field3, _value3,       \
 740             _field4, _value4, _field5, _value5, _field6, _value6,       \
 741             _field7, _value7, _field8, _value8, _field9, _value9,       \
 742             _field10, _value10)                                         \
 743         do {                                                            \
 744                 _NOTE(CONSTANTCONDITION)                                \
 745                 (_qword).eq_u64[0] = EFX_INSERT_FIELDS64(0, 63,         \
 746                     _field1, _value1, _field2, _value2,                 \
 747                     _field3, _value3, _field4, _value4,                 \
 748                     _field5, _value5, _field6, _value6,                 \
 749                     _field7, _value7, _field8, _value8,                 \
 750                     _field9, _value9, _field10, _value10);              \
 751         _NOTE(CONSTANTCONDITION)                                        \
 752         } while (B_FALSE)
 753
 754 #define EFX_POPULATE_QWORD32(_qword,                                    \
 755             _field1, _value1, _field2, _value2, _field3, _value3,       \
 756             _field4, _value4, _field5, _value5, _field6, _value6,       \
 757             _field7, _value7, _field8, _value8, _field9, _value9,       \
 758             _field10, _value10)                                         \
 759         do {                                                            \
 760                 _NOTE(CONSTANTCONDITION)                                \
 761                 (_qword).eq_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 762                     _field1, _value1, _field2, _value2,                 \
 763                     _field3, _value3, _field4, _value4,                 \
 764                     _field5, _value5, _field6, _value6,                 \
 765                     _field7, _value7, _field8, _value8,                 \
 766                     _field9, _value9, _field10, _value10);              \
 767                 _NOTE(CONSTANTCONDITION)                                \
 768                 (_qword).eq_u32[1] = EFX_INSERT_FIELDS32(32, 63,        \
 769                     _field1, _value1, _field2, _value2,                 \
 770                     _field3, _value3, _field4, _value4,                 \
 771                     _field5, _value5, _field6, _value6,                 \
 772                     _field7, _value7, _field8, _value8,                 \
 773                     _field9, _value9, _field10, _value10);              \
 774         _NOTE(CONSTANTCONDITION)                                        \
 775         } while (B_FALSE)
 776
 777 #define EFX_POPULATE_DWORD(_dword,                                      \
 778             _field1, _value1, _field2, _value2, _field3, _value3,       \
 779             _field4, _value4, _field5, _value5, _field6, _value6,       \
 780             _field7, _value7, _field8, _value8, _field9, _value9,       \
 781             _field10, _value10)                                         \
 782         do {                                                            \
 783                 _NOTE(CONSTANTCONDITION)                                \
 784                 (_dword).ed_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 785                     _field1, _value1, _field2, _value2,                 \
 786                     _field3, _value3, _field4, _value4,                 \
 787                     _field5, _value5, _field6, _value6,                 \
 788                     _field7, _value7, _field8, _value8,                 \
 789                     _field9, _value9, _field10, _value10);              \
 790         _NOTE(CONSTANTCONDITION)                                        \
 791         } while (B_FALSE)
 792
 793 #define EFX_POPULATE_WORD(_word,                                        \
 794             _field1, _value1, _field2, _value2, _field3, _value3,       \
 795             _field4, _value4, _field5, _value5, _field6, _value6,       \
 796             _field7, _value7, _field8, _value8, _field9, _value9,       \
 797             _field10, _value10)                                         \
 798         do {                                                            \
 799                 _NOTE(CONSTANTCONDITION)                                \
 800                 (_word).ew_u16[0] = EFX_INSERT_FIELDS16(0, 15,          \
 801                     _field1, _value1, _field2, _value2,                 \
 802                     _field3, _value3, _field4, _value4,                 \
 803                     _field5, _value5, _field6, _value6,                 \
 804                     _field7, _value7, _field8, _value8,                 \
 805                     _field9, _value9, _field10, _value10);              \
 806         _NOTE(CONSTANTCONDITION)                                        \
 807         } while (B_FALSE)
 808
 809 #define EFX_POPULATE_BYTE(_byte,                                        \
 810             _field1, _value1, _field2, _value2, _field3, _value3,       \
 811             _field4, _value4, _field5, _value5, _field6, _value6,       \
 812             _field7, _value7, _field8, _value8, _field9, _value9,       \
 813             _field10, _value10)                                         \
 814         do {                                                            \
 815                 _NOTE(CONSTANTCONDITION)                                \
 816                 (_byte).eb_u8[0] = EFX_INSERT_FIELDS8(0, 7,             \
 817                     _field1, _value1, _field2, _value2,                 \
 818                     _field3, _value3, _field4, _value4,                 \
 819                     _field5, _value5, _field6, _value6,                 \
 820                     _field7, _value7, _field8, _value8,                 \
 821                     _field9, _value9, _field10, _value10);              \
 822         _NOTE(CONSTANTCONDITION)                                        \
 823         } while (B_FALSE)
 824
 825 /* Populate an octword field with various numbers of arguments */
 826 #define EFX_POPULATE_OWORD_10 EFX_POPULATE_OWORD
 827
 828 #define EFX_POPULATE_OWORD_9(_oword,                                    \
 829             _field1, _value1, _field2, _value2, _field3, _value3,       \
 830             _field4, _value4, _field5, _value5, _field6, _value6,       \
 831             _field7, _value7, _field8, _value8, _field9, _value9)       \
 832         EFX_POPULATE_OWORD_10(_oword, EFX_DUMMY_FIELD, 0,               \
 833             _field1, _value1, _field2, _value2, _field3, _value3,       \
 834             _field4, _value4, _field5, _value5, _field6, _value6,       \
 835             _field7, _value7, _field8, _value8, _field9, _value9)
 836
 837 #define EFX_POPULATE_OWORD_8(_oword,                                    \
 838             _field1, _value1, _field2, _value2, _field3, _value3,       \
 839             _field4, _value4, _field5, _value5, _field6, _value6,       \
 840             _field7, _value7, _field8, _value8)                         \
 841         EFX_POPULATE_OWORD_9(_oword, EFX_DUMMY_FIELD, 0,                \
 842             _field1, _value1, _field2, _value2, _field3, _value3,       \
 843             _field4, _value4, _field5, _value5, _field6, _value6,       \
 844             _field7, _value7, _field8, _value8)
 845
 846 #define EFX_POPULATE_OWORD_7(_oword,                                    \
 847             _field1, _value1, _field2, _value2, _field3, _value3,       \
 848             _field4, _value4, _field5, _value5, _field6, _value6,       \
 849             _field7, _value7)                                           \
 850         EFX_POPULATE_OWORD_8(_oword, EFX_DUMMY_FIELD, 0,                \
 851             _field1, _value1, _field2, _value2, _field3, _value3,       \
 852             _field4, _value4, _field5, _value5, _field6, _value6,       \
 853             _field7, _value7)
 854
 855 #define EFX_POPULATE_OWORD_6(_oword,                                    \
 856             _field1, _value1, _field2, _value2, _field3, _value3,       \
 857             _field4, _value4, _field5, _value5, _field6, _value6)       \
 858         EFX_POPULATE_OWORD_7(_oword, EFX_DUMMY_FIELD, 0,                \
 859             _field1, _value1, _field2, _value2, _field3, _value3,       \
 860             _field4, _value4, _field5, _value5, _field6, _value6)
 861
 862 #define EFX_POPULATE_OWORD_5(_oword,                                    \
 863             _field1, _value1, _field2, _value2, _field3, _value3,       \
 864             _field4, _value4, _field5, _value5)                         \
 865         EFX_POPULATE_OWORD_6(_oword, EFX_DUMMY_FIELD, 0,                \
 866             _field1, _value1, _field2, _value2, _field3, _value3,       \
 867             _field4, _value4, _field5, _value5)
 868
 869 #define EFX_POPULATE_OWORD_4(_oword,                                    \
 870             _field1, _value1, _field2, _value2, _field3, _value3,       \
 871             _field4, _value4)                                           \
 872         EFX_POPULATE_OWORD_5(_oword, EFX_DUMMY_FIELD, 0,                \
 873             _field1, _value1, _field2, _value2, _field3, _value3,       \
 874             _field4, _value4)
 875
 876 #define EFX_POPULATE_OWORD_3(_oword,                                    \
 877             _field1, _value1, _field2, _value2, _field3, _value3)       \
 878         EFX_POPULATE_OWORD_4(_oword, EFX_DUMMY_FIELD, 0,                \
 879             _field1, _value1, _field2, _value2, _field3, _value3)
 880
 881 #define EFX_POPULATE_OWORD_2(_oword,                                    \
 882             _field1, _value1, _field2, _value2)                         \
 883         EFX_POPULATE_OWORD_3(_oword, EFX_DUMMY_FIELD, 0,                \
 884             _field1, _value1, _field2, _value2)
 885
 886 #define EFX_POPULATE_OWORD_1(_oword,                                    \
 887             _field1, _value1)                                           \
 888         EFX_POPULATE_OWORD_2(_oword, EFX_DUMMY_FIELD, 0,                \
 889             _field1, _value1)
 890
 891 #define EFX_ZERO_OWORD(_oword)                                          \
 892         EFX_POPULATE_OWORD_1(_oword, EFX_DUMMY_FIELD, 0)
 893
 894 #define EFX_SET_OWORD64(_oword)                                         \
 895         EFX_POPULATE_OWORD_2(_oword,                                    \
 896             EFX_QWORD_0, (uint64_t)-1, EFX_QWORD_1, (uint64_t)-1)
 897
 898 #define EFX_SET_OWORD32(_oword)                                         \
 899         EFX_POPULATE_OWORD_4(_oword,                                    \
 900             EFX_DWORD_0, 0xffffffff, EFX_DWORD_1, 0xffffffff,           \
 901             EFX_DWORD_2, 0xffffffff, EFX_DWORD_3, 0xffffffff)
 902
 903 /* Populate a quadword field with various numbers of arguments */
 904 #define EFX_POPULATE_QWORD_10 EFX_POPULATE_QWORD
 905
 906 #define EFX_POPULATE_QWORD_9(_qword,                                    \
 907             _field1, _value1, _field2, _value2, _field3, _value3,       \
 908             _field4, _value4, _field5, _value5, _field6, _value6,       \
 909             _field7, _value7, _field8, _value8, _field9, _value9)       \
 910         EFX_POPULATE_QWORD_10(_qword, EFX_DUMMY_FIELD, 0,               \
 911             _field1, _value1, _field2, _value2, _field3, _value3,       \
 912             _field4, _value4, _field5, _value5, _field6, _value6,       \
 913             _field7, _value7, _field8, _value8, _field9, _value9)
 914
 915 #define EFX_POPULATE_QWORD_8(_qword,                                    \
 916             _field1, _value1, _field2, _value2, _field3, _value3,       \
 917             _field4, _value4, _field5, _value5, _field6, _value6,       \
 918             _field7, _value7, _field8, _value8)                         \
 919         EFX_POPULATE_QWORD_9(_qword, EFX_DUMMY_FIELD, 0,                \
 920             _field1, _value1, _field2, _value2, _field3, _value3,       \
 921             _field4, _value4, _field5, _value5, _field6, _value6,       \
 922             _field7, _value7, _field8, _value8)
 923
 924 #define EFX_POPULATE_QWORD_7(_qword,                                    \
 925             _field1, _value1, _field2, _value2, _field3, _value3,       \
 926             _field4, _value4, _field5, _value5, _field6, _value6,       \
 927             _field7, _value7)                                           \
 928         EFX_POPULATE_QWORD_8(_qword, EFX_DUMMY_FIELD, 0,                \
 929             _field1, _value1, _field2, _value2, _field3, _value3,       \
 930             _field4, _value4, _field5, _value5, _field6, _value6,       \
 931             _field7, _value7)
 932
 933 #define EFX_POPULATE_QWORD_6(_qword,                                    \
 934             _field1, _value1, _field2, _value2, _field3, _value3,       \
 935             _field4, _value4, _field5, _value5, _field6, _value6)       \
 936         EFX_POPULATE_QWORD_7(_qword, EFX_DUMMY_FIELD, 0,                \
 937             _field1, _value1, _field2, _value2, _field3, _value3,       \
 938             _field4, _value4, _field5, _value5, _field6, _value6)
 939
 940 #define EFX_POPULATE_QWORD_5(_qword,                                    \
 941             _field1, _value1, _field2, _value2, _field3, _value3,       \
 942             _field4, _value4, _field5, _value5)                         \
 943         EFX_POPULATE_QWORD_6(_qword, EFX_DUMMY_FIELD, 0,                \
 944             _field1, _value1, _field2, _value2, _field3, _value3,       \
 945             _field4, _value4, _field5, _value5)
 946
 947 #define EFX_POPULATE_QWORD_4(_qword,                                    \
 948             _field1, _value1, _field2, _value2, _field3, _value3,       \
 949             _field4, _value4)                                           \
 950         EFX_POPULATE_QWORD_5(_qword, EFX_DUMMY_FIELD, 0,                \
 951             _field1, _value1, _field2, _value2, _field3, _value3,       \
 952             _field4, _value4)
 953
 954 #define EFX_POPULATE_QWORD_3(_qword,                                    \
 955             _field1, _value1, _field2, _value2, _field3, _value3)       \
 956         EFX_POPULATE_QWORD_4(_qword, EFX_DUMMY_FIELD, 0,                \
 957             _field1, _value1, _field2, _value2, _field3, _value3)
 958
 959 #define EFX_POPULATE_QWORD_2(_qword,                                    \
 960             _field1, _value1, _field2, _value2)                         \
 961         EFX_POPULATE_QWORD_3(_qword, EFX_DUMMY_FIELD, 0,                \
 962             _field1, _value1, _field2, _value2)
 963
 964 #define EFX_POPULATE_QWORD_1(_qword,                                    \
 965             _field1, _value1)                                           \
 966         EFX_POPULATE_QWORD_2(_qword, EFX_DUMMY_FIELD, 0,                \
 967             _field1, _value1)
 968
 969 #define EFX_ZERO_QWORD(_qword)                                          \
 970         EFX_POPULATE_QWORD_1(_qword, EFX_DUMMY_FIELD, 0)
 971
 972 #define EFX_SET_QWORD64(_qword)                                         \
 973         EFX_POPULATE_QWORD_1(_qword,                                    \
 974             EFX_QWORD_0, (uint64_t)-1)
 975
 976 #define EFX_SET_QWORD32(_qword)                                         \
 977         EFX_POPULATE_QWORD_2(_qword,                                    \
 978             EFX_DWORD_0, 0xffffffff, EFX_DWORD_1, 0xffffffff)
 979
 980 /* Populate a dword field with various numbers of arguments */
 981 #define EFX_POPULATE_DWORD_10 EFX_POPULATE_DWORD
 982
 983 #define EFX_POPULATE_DWORD_9(_dword,                                    \
 984             _field1, _value1, _field2, _value2, _field3, _value3,       \
 985             _field4, _value4, _field5, _value5, _field6, _value6,       \
 986             _field7, _value7, _field8, _value8, _field9, _value9)       \
 987         EFX_POPULATE_DWORD_10(_dword, EFX_DUMMY_FIELD, 0,               \
 988             _field1, _value1, _field2, _value2, _field3, _value3,       \
 989             _field4, _value4, _field5, _value5, _field6, _value6,       \
 990             _field7, _value7, _field8, _value8, _field9, _value9)
 991
 992 #define EFX_POPULATE_DWORD_8(_dword,                                    \
 993             _field1, _value1, _field2, _value2, _field3, _value3,       \
 994             _field4, _value4, _field5, _value5, _field6, _value6,       \
 995             _field7, _value7, _field8, _value8)                         \
 996         EFX_POPULATE_DWORD_9(_dword, EFX_DUMMY_FIELD, 0,                \
 997             _field1, _value1, _field2, _value2, _field3, _value3,       \
 998             _field4, _value4, _field5, _value5, _field6, _value6,       \
 999             _field7, _value7, _field8, _value8)
1000
1001 #define EFX_POPULATE_DWORD_7(_dword,                                    \
1002             _field1, _value1, _field2, _value2, _field3, _value3,       \
1003             _field4, _value4, _field5, _value5, _field6, _value6,       \
1004             _field7, _value7)                                           \
1005         EFX_POPULATE_DWORD_8(_dword, EFX_DUMMY_FIELD, 0,                \
1006             _field1, _value1, _field2, _value2, _field3, _value3,       \
1007             _field4, _value4, _field5, _value5, _field6, _value6,       \
1008             _field7, _value7)
1009
1010 #define EFX_POPULATE_DWORD_6(_dword,                                    \
1011             _field1, _value1, _field2, _value2, _field3, _value3,       \
1012             _field4, _value4, _field5, _value5, _field6, _value6)       \
1013         EFX_POPULATE_DWORD_7(_dword, EFX_DUMMY_FIELD, 0,                \
1014             _field1, _value1, _field2, _value2, _field3, _value3,       \
1015             _field4, _value4, _field5, _value5, _field6, _value6)
1016
1017 #define EFX_POPULATE_DWORD_5(_dword,                                    \
1018             _field1, _value1, _field2, _value2, _field3, _value3,       \
1019             _field4, _value4, _field5, _value5)                         \
1020         EFX_POPULATE_DWORD_6(_dword, EFX_DUMMY_FIELD, 0,                \
1021             _field1, _value1, _field2, _value2, _field3, _value3,       \
1022             _field4, _value4, _field5, _value5)
1023
1024 #define EFX_POPULATE_DWORD_4(_dword,                                    \
1025             _field1, _value1, _field2, _value2, _field3, _value3,       \
1026             _field4, _value4)                                           \
1027         EFX_POPULATE_DWORD_5(_dword, EFX_DUMMY_FIELD, 0,                \
1028             _field1, _value1, _field2, _value2, _field3, _value3,       \
1029             _field4, _value4)
1030
1031 #define EFX_POPULATE_DWORD_3(_dword,                                    \
1032             _field1, _value1, _field2, _value2, _field3, _value3)       \
1033         EFX_POPULATE_DWORD_4(_dword, EFX_DUMMY_FIELD, 0,                \
1034             _field1, _value1, _field2, _value2, _field3, _value3)
1035
1036 #define EFX_POPULATE_DWORD_2(_dword,                                    \
1037             _field1, _value1, _field2, _value2)                         \
1038         EFX_POPULATE_DWORD_3(_dword, EFX_DUMMY_FIELD, 0,                \
1039             _field1, _value1, _field2, _value2)
1040
1041 #define EFX_POPULATE_DWORD_1(_dword,                                    \
1042             _field1, _value1)                                           \
1043         EFX_POPULATE_DWORD_2(_dword, EFX_DUMMY_FIELD, 0,                \
1044             _field1, _value1)
1045
1046 #define EFX_ZERO_DWORD(_dword)                                          \
1047         EFX_POPULATE_DWORD_1(_dword, EFX_DUMMY_FIELD, 0)
1048
1049 #define EFX_SET_DWORD(_dword)                                           \
1050         EFX_POPULATE_DWORD_1(_dword,                                    \
1051             EFX_DWORD_0, 0xffffffff)
1052
1053 /* Populate a word field with various numbers of arguments */
1054 #define EFX_POPULATE_WORD_10 EFX_POPULATE_WORD
1055
1056 #define EFX_POPULATE_WORD_9(_word,                                      \
1057             _field1, _value1, _field2, _value2, _field3, _value3,       \
1058             _field4, _value4, _field5, _value5, _field6, _value6,       \
1059             _field7, _value7, _field8, _value8, _field9, _value9)       \
1060         EFX_POPULATE_WORD_10(_word, EFX_DUMMY_FIELD, 0,                 \
1061             _field1, _value1, _field2, _value2, _field3, _value3,       \
1062             _field4, _value4, _field5, _value5, _field6, _value6,       \
1063             _field7, _value7, _field8, _value8, _field9, _value9)
1064
1065 #define EFX_POPULATE_WORD_8(_word,                                      \
1066             _field1, _value1, _field2, _value2, _field3, _value3,       \
1067             _field4, _value4, _field5, _value5, _field6, _value6,       \
1068             _field7, _value7, _field8, _value8)                         \
1069         EFX_POPULATE_WORD_9(_word, EFX_DUMMY_FIELD, 0,                  \
1070             _field1, _value1, _field2, _value2, _field3, _value3,       \
1071             _field4, _value4, _field5, _value5, _field6, _value6,       \
1072             _field7, _value7, _field8, _value8)
1073
1074 #define EFX_POPULATE_WORD_7(_word,                                      \
1075             _field1, _value1, _field2, _value2, _field3, _value3,       \
1076             _field4, _value4, _field5, _value5, _field6, _value6,       \
1077             _field7, _value7)                                           \
1078         EFX_POPULATE_WORD_8(_word, EFX_DUMMY_FIELD, 0,                  \
1079             _field1, _value1, _field2, _value2, _field3, _value3,       \
1080             _field4, _value4, _field5, _value5, _field6, _value6,       \
1081             _field7, _value7)
1082
1083 #define EFX_POPULATE_WORD_6(_word,                                      \
1084             _field1, _value1, _field2, _value2, _field3, _value3,       \
1085             _field4, _value4, _field5, _value5, _field6, _value6)       \
1086         EFX_POPULATE_WORD_7(_word, EFX_DUMMY_FIELD, 0,                  \
1087             _field1, _value1, _field2, _value2, _field3, _value3,       \
1088             _field4, _value4, _field5, _value5, _field6, _value6)
1089
1090 #define EFX_POPULATE_WORD_5(_word,                                      \
1091             _field1, _value1, _field2, _value2, _field3, _value3,       \
1092             _field4, _value4, _field5, _value5)                         \
1093         EFX_POPULATE_WORD_6(_word, EFX_DUMMY_FIELD, 0,                  \
1094             _field1, _value1, _field2, _value2, _field3, _value3,       \
1095             _field4, _value4, _field5, _value5)
1096
1097 #define EFX_POPULATE_WORD_4(_word,                                      \
1098             _field1, _value1, _field2, _value2, _field3, _value3,       \
1099             _field4, _value4)                                           \
1100         EFX_POPULATE_WORD_5(_word, EFX_DUMMY_FIELD, 0,                  \
1101             _field1, _value1, _field2, _value2, _field3, _value3,       \
1102             _field4, _value4)
1103
1104 #define EFX_POPULATE_WORD_3(_word,                                      \
1105             _field1, _value1, _field2, _value2, _field3, _value3)       \
1106         EFX_POPULATE_WORD_4(_word, EFX_DUMMY_FIELD, 0,                  \
1107             _field1, _value1, _field2, _value2, _field3, _value3)
1108
1109 #define EFX_POPULATE_WORD_2(_word,                                      \
1110             _field1, _value1, _field2, _value2)                         \
1111         EFX_POPULATE_WORD_3(_word, EFX_DUMMY_FIELD, 0,                  \
1112             _field1, _value1, _field2, _value2)
1113
1114 #define EFX_POPULATE_WORD_1(_word,                                      \
1115             _field1, _value1)                                           \
1116         EFX_POPULATE_WORD_2(_word, EFX_DUMMY_FIELD, 0,                  \
1117             _field1, _value1)
1118
1119 #define EFX_ZERO_WORD(_word)                                            \
1120         EFX_POPULATE_WORD_1(_word, EFX_DUMMY_FIELD, 0)
1121
1122 #define EFX_SET_WORD(_word)                                             \
1123         EFX_POPULATE_WORD_1(_word,                                      \
1124             EFX_WORD_0, 0xffff)
1125
1126 /* Populate a byte field with various numbers of arguments */
1127 #define EFX_POPULATE_BYTE_10 EFX_POPULATE_BYTE
1128
1129 #define EFX_POPULATE_BYTE_9(_byte,                                      \
1130             _field1, _value1, _field2, _value2, _field3, _value3,       \
1131             _field4, _value4, _field5, _value5, _field6, _value6,       \
1132             _field7, _value7, _field8, _value8, _field9, _value9)       \
1133         EFX_POPULATE_BYTE_10(_byte, EFX_DUMMY_FIELD, 0,                 \
1134             _field1, _value1, _field2, _value2, _field3, _value3,       \
1135             _field4, _value4, _field5, _value5, _field6, _value6,       \
1136             _field7, _value7, _field8, _value8, _field9, _value9)
1137
1138 #define EFX_POPULATE_BYTE_8(_byte,                                      \
1139             _field1, _value1, _field2, _value2, _field3, _value3,       \
1140             _field4, _value4, _field5, _value5, _field6, _value6,       \
1141             _field7, _value7, _field8, _value8)                         \
1142         EFX_POPULATE_BYTE_9(_byte, EFX_DUMMY_FIELD, 0,                  \
1143             _field1, _value1, _field2, _value2, _field3, _value3,       \
1144             _field4, _value4, _field5, _value5, _field6, _value6,       \
1145             _field7, _value7, _field8, _value8)
1146
1147 #define EFX_POPULATE_BYTE_7(_byte,                                      \
1148             _field1, _value1, _field2, _value2, _field3, _value3,       \
1149             _field4, _value4, _field5, _value5, _field6, _value6,       \
1150             _field7, _value7)                                           \
1151         EFX_POPULATE_BYTE_8(_byte, EFX_DUMMY_FIELD, 0,                  \
1152             _field1, _value1, _field2, _value2, _field3, _value3,       \
1153             _field4, _value4, _field5, _value5, _field6, _value6,       \
1154             _field7, _value7)
1155
1156 #define EFX_POPULATE_BYTE_6(_byte,                                      \
1157             _field1, _value1, _field2, _value2, _field3, _value3,       \
1158             _field4, _value4, _field5, _value5, _field6, _value6)       \
1159         EFX_POPULATE_BYTE_7(_byte, EFX_DUMMY_FIELD, 0,                  \
1160             _field1, _value1, _field2, _value2, _field3, _value3,       \
1161             _field4, _value4, _field5, _value5, _field6, _value6)
1162
1163 #define EFX_POPULATE_BYTE_5(_byte,                                      \
1164             _field1, _value1, _field2, _value2, _field3, _value3,       \
1165             _field4, _value4, _field5, _value5)                         \
1166         EFX_POPULATE_BYTE_6(_byte, EFX_DUMMY_FIELD, 0,                  \
1167             _field1, _value1, _field2, _value2, _field3, _value3,       \
1168             _field4, _value4, _field5, _value5)
1169
1170 #define EFX_POPULATE_BYTE_4(_byte,                                      \
1171             _field1, _value1, _field2, _value2, _field3, _value3,       \
1172             _field4, _value4)                                           \
1173         EFX_POPULATE_BYTE_5(_byte, EFX_DUMMY_FIELD, 0,                  \
1174             _field1, _value1, _field2, _value2, _field3, _value3,       \
1175             _field4, _value4)
1176
1177 #define EFX_POPULATE_BYTE_3(_byte,                                      \
1178             _field1, _value1, _field2, _value2, _field3, _value3)       \
1179         EFX_POPULATE_BYTE_4(_byte, EFX_DUMMY_FIELD, 0,                  \
1180             _field1, _value1, _field2, _value2, _field3, _value3)
1181
1182 #define EFX_POPULATE_BYTE_2(_byte,                                      \
1183             _field1, _value1, _field2, _value2)                         \
1184         EFX_POPULATE_BYTE_3(_byte, EFX_DUMMY_FIELD, 0,                  \
1185             _field1, _value1, _field2, _value2)
1186
1187 #define EFX_POPULATE_BYTE_1(_byte,                                      \
1188             _field1, _value1)                                           \
1189         EFX_POPULATE_BYTE_2(_byte, EFX_DUMMY_FIELD, 0,                  \
1190             _field1, _value1)
1191
1192 #define EFX_ZERO_BYTE(_byte)                                            \
1193         EFX_POPULATE_BYTE_1(_byte, EFX_DUMMY_FIELD, 0)
1194
1195 #define EFX_SET_BYTE(_byte)                                             \
1196         EFX_POPULATE_BYTE_1(_byte,                                      \
1197             EFX_BYTE_0, 0xff)
1198
1199 /*
1200  * Modify a named field within an already-populated structure.  Used
1201  * for read-modify-write operations.
1202  */
1203
1204 #define EFX_INSERT_FIELD64(_min, _max, _field, _value)                  \
1205         __CPU_TO_LE_64(EFX_INSERT_FIELD_NATIVE64(_min, _max, _field, _value))
1206
1207 #define EFX_INSERT_FIELD32(_min, _max, _field, _value)                  \
1208         __CPU_TO_LE_32(EFX_INSERT_FIELD_NATIVE32(_min, _max, _field, _value))
1209
1210 #define EFX_INSERT_FIELD16(_min, _max, _field, _value)                  \
1211         __CPU_TO_LE_16(EFX_INSERT_FIELD_NATIVE16(_min, _max, _field, _value))
1212
1213 #define EFX_INSERT_FIELD8(_min, _max, _field, _value)                   \
1214         __NATIVE_8(EFX_INSERT_FIELD_NATIVE8(_min, _max, _field, _value))
1215
1216 #define EFX_INPLACE_MASK64(_min, _max, _field)                          \
1217         EFX_INSERT_FIELD64(_min, _max, _field, EFX_MASK64(_field))
1218
1219 #define EFX_INPLACE_MASK32(_min, _max, _field)                          \
1220         EFX_INSERT_FIELD32(_min, _max, _field, EFX_MASK32(_field))
1221
1222 #define EFX_INPLACE_MASK16(_min, _max, _field)                          \
1223         EFX_INSERT_FIELD16(_min, _max, _field, EFX_MASK16(_field))
1224
1225 #define EFX_INPLACE_MASK8(_min, _max, _field)                           \
1226         EFX_INSERT_FIELD8(_min, _max, _field, EFX_MASK8(_field))
1227
1228 #define EFX_SET_OWORD_FIELD64(_oword, _field, _value)                   \
1229         do {                                                            \
1230                 _NOTE(CONSTANTCONDITION)                                \
1231                 (_oword).eo_u64[0] = (((_oword).eo_u64[0] &             \
1232                     ~EFX_INPLACE_MASK64(0, 63, _field)) |               \
1233                     EFX_INSERT_FIELD64(0, 63, _field, _value));         \
1234                 _NOTE(CONSTANTCONDITION)                                \
1235                 (_oword).eo_u64[1] = (((_oword).eo_u64[1] &             \
1236                     ~EFX_INPLACE_MASK64(64, 127, _field)) |             \
1237                     EFX_INSERT_FIELD64(64, 127, _field, _value));       \
1238         _NOTE(CONSTANTCONDITION)                                        \
1239         } while (B_FALSE)
1240
1241 #define EFX_SET_OWORD_FIELD32(_oword, _field, _value)                   \
1242         do {                                                            \
1243                 _NOTE(CONSTANTCONDITION)                                \
1244                 (_oword).eo_u32[0] = (((_oword).eo_u32[0] &             \
1245                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1246                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1247                 _NOTE(CONSTANTCONDITION)                                \
1248                 (_oword).eo_u32[1] = (((_oword).eo_u32[1] &             \
1249                     ~EFX_INPLACE_MASK32(32, 63, _field)) |              \
1250                     EFX_INSERT_FIELD32(32, 63, _field, _value));        \
1251                 _NOTE(CONSTANTCONDITION)                                \
1252                 (_oword).eo_u32[2] = (((_oword).eo_u32[2] &             \
1253                     ~EFX_INPLACE_MASK32(64, 95, _field)) |              \
1254                     EFX_INSERT_FIELD32(64, 95, _field, _value));        \
1255                 _NOTE(CONSTANTCONDITION)                                \
1256                 (_oword).eo_u32[3] = (((_oword).eo_u32[3] &             \
1257                     ~EFX_INPLACE_MASK32(96, 127, _field)) |             \
1258                     EFX_INSERT_FIELD32(96, 127, _field, _value));       \
1259         _NOTE(CONSTANTCONDITION)                                        \
1260         } while (B_FALSE)
1261
1262 #define EFX_SET_QWORD_FIELD64(_qword, _field, _value)                   \
1263         do {                                                            \
1264                 _NOTE(CONSTANTCONDITION)                                \
1265                 (_qword).eq_u64[0] = (((_qword).eq_u64[0] &             \
1266                     ~EFX_INPLACE_MASK64(0, 63, _field)) |               \
1267                     EFX_INSERT_FIELD64(0, 63, _field, _value));         \
1268         _NOTE(CONSTANTCONDITION)                                        \
1269         } while (B_FALSE)
1270
1271 #define EFX_SET_QWORD_FIELD32(_qword, _field, _value)                   \
1272         do {                                                            \
1273                 _NOTE(CONSTANTCONDITION)                                \
1274                 (_qword).eq_u32[0] = (((_qword).eq_u32[0] &             \
1275                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1276                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1277                 _NOTE(CONSTANTCONDITION)                                \
1278                 (_qword).eq_u32[1] = (((_qword).eq_u32[1] &             \
1279                     ~EFX_INPLACE_MASK32(32, 63, _field)) |              \
1280                     EFX_INSERT_FIELD32(32, 63, _field, _value));        \
1281         _NOTE(CONSTANTCONDITION)                                        \
1282         } while (B_FALSE)
1283
1284 #define EFX_SET_DWORD_FIELD(_dword, _field, _value)                     \
1285         do {                                                            \
1286                 _NOTE(CONSTANTCONDITION)                                \
1287                 (_dword).ed_u32[0] = (((_dword).ed_u32[0] &             \
1288                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1289                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1290         _NOTE(CONSTANTCONDITION)                                        \
1291         } while (B_FALSE)
1292
1293 #define EFX_SET_WORD_FIELD(_word, _field, _value)                       \
1294         do {                                                            \
1295                 _NOTE(CONSTANTCONDITION)                                \
1296                 (_word).ew_u16[0] = (((_word).ew_u16[0] &               \
1297                     ~EFX_INPLACE_MASK16(0, 15, _field)) |               \
1298                     EFX_INSERT_FIELD16(0, 15, _field, _value));         \
1299         _NOTE(CONSTANTCONDITION)                                        \
1300         } while (B_FALSE)
1301
1302 #define EFX_SET_BYTE_FIELD(_byte, _field, _value)                       \
1303         do {                                                            \
1304                 _NOTE(CONSTANTCONDITION)                                \
1305                 (_byte).eb_u8[0] = (((_byte).eb_u8[0] &                 \
1306                     ~EFX_INPLACE_MASK8(0, 7, _field)) |                 \
1307                     EFX_INSERT_FIELD8(0, 7, _field, _value));           \
1308         _NOTE(CONSTANTCONDITION)                                        \
1309         } while (B_FALSE)
1310
1311 /*
1312  * Set or clear a numbered bit within an octword.
1313  */
1314
1315 #define EFX_SHIFT64(_bit, _base)                                        \
1316         (((_bit) >= (_base) && (_bit) < (_base) + 64) ?                 \
1317                 ((uint64_t)1 << ((_bit) - (_base))) :                   \
1318                 0U)
1319
1320 #define EFX_SHIFT32(_bit, _base)                                        \
1321         (((_bit) >= (_base) && (_bit) < (_base) + 32) ?                 \
1322                 ((uint32_t)1 << ((_bit) - (_base))) :                   \
1323                 0U)
1324
1325 #define EFX_SHIFT16(_bit, _base)                                        \
1326         (((_bit) >= (_base) && (_bit) < (_base) + 16) ?                 \
1327                 (uint16_t)(1 << ((_bit) - (_base))) :                   \
1328                 0U)
1329
1330 #define EFX_SHIFT8(_bit, _base)                                         \
1331         (((_bit) >= (_base) && (_bit) < (_base) + 8) ?                  \
1332                 (uint8_t)(1 << ((_bit) - (_base))) :                    \
1333                 0U)
1334
1335 #define EFX_SET_OWORD_BIT64(_oword, _bit)                               \
1336         do {                                                            \
1337                 _NOTE(CONSTANTCONDITION)                                \
1338                 (_oword).eo_u64[0] |=                                   \
1339                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)));     \
1340                 (_oword).eo_u64[1] |=                                   \
1341                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(64)));    \
1342         _NOTE(CONSTANTCONDITION)                                        \
1343         } while (B_FALSE)
1344
1345 #define EFX_SET_OWORD_BIT32(_oword, _bit)                               \
1346         do {                                                            \
1347                 _NOTE(CONSTANTCONDITION)                                \
1348                 (_oword).eo_u32[0] |=                                   \
1349                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1350                 (_oword).eo_u32[1] |=                                   \
1351                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)));    \
1352                 (_oword).eo_u32[2] |=                                   \
1353                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(64)));    \
1354                 (_oword).eo_u32[3] |=                                   \
1355                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(96)));    \
1356         _NOTE(CONSTANTCONDITION)                                        \
1357         } while (B_FALSE)
1358
1359 #define EFX_CLEAR_OWORD_BIT64(_oword, _bit)                             \
1360         do {                                                            \
1361                 _NOTE(CONSTANTCONDITION)                                \
1362                 (_oword).eo_u64[0] &=                                   \
1363                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(0)));    \
1364                 (_oword).eo_u64[1] &=                                   \
1365                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(64)));   \
1366         _NOTE(CONSTANTCONDITION)                                        \
1367         } while (B_FALSE)
1368
1369 #define EFX_CLEAR_OWORD_BIT32(_oword, _bit)                             \
1370         do {                                                            \
1371                 _NOTE(CONSTANTCONDITION)                                \
1372                 (_oword).eo_u32[0] &=                                   \
1373                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1374                 (_oword).eo_u32[1] &=                                   \
1375                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(32)));   \
1376                 (_oword).eo_u32[2] &=                                   \
1377                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(64)));   \
1378                 (_oword).eo_u32[3] &=                                   \
1379                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(96)));   \
1380         _NOTE(CONSTANTCONDITION)                                        \
1381         } while (B_FALSE)
1382
1383 #define EFX_SET_QWORD_BIT64(_qword, _bit)                               \
1384         do {                                                            \
1385                 _NOTE(CONSTANTCONDITION)                                \
1386                 (_qword).eq_u64[0] |=                                   \
1387                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)));     \
1388         _NOTE(CONSTANTCONDITION)                                        \
1389         } while (B_FALSE)
1390
1391 #define EFX_SET_QWORD_BIT32(_qword, _bit)                               \
1392         do {                                                            \
1393                 _NOTE(CONSTANTCONDITION)                                \
1394                 (_qword).eq_u32[0] |=                                   \
1395                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1396                 (_qword).eq_u32[1] |=                                   \
1397                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)));    \
1398         _NOTE(CONSTANTCONDITION)                                        \
1399         } while (B_FALSE)
1400
1401 #define EFX_CLEAR_QWORD_BIT64(_qword, _bit)                             \
1402         do {                                                            \
1403                 _NOTE(CONSTANTCONDITION)                                \
1404                 (_qword).eq_u64[0] &=                                   \
1405                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(0)));    \
1406         _NOTE(CONSTANTCONDITION)                                        \
1407         } while (B_FALSE)
1408
1409 #define EFX_CLEAR_QWORD_BIT32(_qword, _bit)                             \
1410         do {                                                            \
1411                 _NOTE(CONSTANTCONDITION)                                \
1412                 (_qword).eq_u32[0] &=                                   \
1413                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1414                 (_qword).eq_u32[1] &=                                   \
1415                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(32)));   \
1416         _NOTE(CONSTANTCONDITION)                                        \
1417         } while (B_FALSE)
1418
1419 #define EFX_SET_DWORD_BIT(_dword, _bit)                                 \
1420         do {                                                            \
1421                 (_dword).ed_u32[0] |=                                   \
1422                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1423         _NOTE(CONSTANTCONDITION)                                        \
1424         } while (B_FALSE)
1425
1426 #define EFX_CLEAR_DWORD_BIT(_dword, _bit)                               \
1427         do {                                                            \
1428                 (_dword).ed_u32[0] &=                                   \
1429                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1430         _NOTE(CONSTANTCONDITION)                                        \
1431         } while (B_FALSE)
1432
1433 #define EFX_SET_WORD_BIT(_word, _bit)                                   \
1434         do {                                                            \
1435                 (_word).ew_u16[0] |=                                    \
1436                     __CPU_TO_LE_16(EFX_SHIFT16(_bit, FIX_LINT(0)));     \
1437         _NOTE(CONSTANTCONDITION)                                        \
1438         } while (B_FALSE)
1439
1440 #define EFX_CLEAR_WORD_BIT(_word, _bit)                                 \
1441         do {                                                            \
1442                 (_word).ew_u32[0] &=                                    \
1443                     __CPU_TO_LE_16(~EFX_SHIFT16(_bit, FIX_LINT(0)));    \
1444         _NOTE(CONSTANTCONDITION)                                        \
1445         } while (B_FALSE)
1446
1447 #define EFX_SET_BYTE_BIT(_byte, _bit)                                   \
1448         do {                                                            \
1449                 (_byte).eb_u8[0] |=                                     \
1450                     __NATIVE_8(EFX_SHIFT8(_bit, FIX_LINT(0)));          \
1451         _NOTE(CONSTANTCONDITION)                                        \
1452         } while (B_FALSE)
1453
1454 #define EFX_CLEAR_BYTE_BIT(_byte, _bit)                                 \
1455         do {                                                            \
1456                 (_byte).eb_u8[0] &=                                     \
1457                     __NATIVE_8(~EFX_SHIFT8(_bit, FIX_LINT(0)));         \
1458         _NOTE(CONSTANTCONDITION)                                        \
1459         } while (B_FALSE)
1460
1461 #define EFX_OR_OWORD64(_oword1, _oword2)                                \
1462         do {                                                            \
1463                 (_oword1).eo_u64[0] |= (_oword2).eo_u64[0];             \
1464                 (_oword1).eo_u64[1] |= (_oword2).eo_u64[1];             \
1465         _NOTE(CONSTANTCONDITION)                                        \
1466         } while (B_FALSE)
1467
1468 #define EFX_OR_OWORD32(_oword1, _oword2)                                \
1469         do {                                                            \
1470                 (_oword1).eo_u32[0] |= (_oword2).eo_u32[0];             \
1471                 (_oword1).eo_u32[1] |= (_oword2).eo_u32[1];             \
1472                 (_oword1).eo_u32[2] |= (_oword2).eo_u32[2];             \
1473                 (_oword1).eo_u32[3] |= (_oword2).eo_u32[3];             \
1474         _NOTE(CONSTANTCONDITION)                                        \
1475         } while (B_FALSE)
1476
1477 #define EFX_AND_OWORD64(_oword1, _oword2)                               \
1478         do {                                                            \
1479                 (_oword1).eo_u64[0] &= (_oword2).eo_u64[0];             \
1480                 (_oword1).eo_u64[1] &= (_oword2).eo_u64[1];             \
1481         _NOTE(CONSTANTCONDITION)                                        \
1482         } while (B_FALSE)
1483
1484 #define EFX_AND_OWORD32(_oword1, _oword2)                               \
1485         do {                                                            \
1486                 (_oword1).eo_u32[0] &= (_oword2).eo_u32[0];             \
1487                 (_oword1).eo_u32[1] &= (_oword2).eo_u32[1];             \
1488                 (_oword1).eo_u32[2] &= (_oword2).eo_u32[2];             \
1489                 (_oword1).eo_u32[3] &= (_oword2).eo_u32[3];             \
1490         _NOTE(CONSTANTCONDITION)                                        \
1491         } while (B_FALSE)
1492
1493 #define EFX_OR_QWORD64(_qword1, _qword2)                                \
1494         do {                                                            \
1495                 (_qword1).eq_u64[0] |= (_qword2).eq_u64[0];             \
1496         _NOTE(CONSTANTCONDITION)                                        \
1497         } while (B_FALSE)
1498
1499 #define EFX_OR_QWORD32(_qword1, _qword2)                                \
1500         do {                                                            \
1501                 (_qword1).eq_u32[0] |= (_qword2).eq_u32[0];             \
1502                 (_qword1).eq_u32[1] |= (_qword2).eq_u32[1];             \
1503         _NOTE(CONSTANTCONDITION)                                        \
1504         } while (B_FALSE)
1505
1506 #define EFX_AND_QWORD64(_qword1, _qword2)                               \
1507         do {                                                            \
1508                 (_qword1).eq_u64[0] &= (_qword2).eq_u64[0];             \
1509         _NOTE(CONSTANTCONDITION)                                        \
1510         } while (B_FALSE)
1511
1512 #define EFX_AND_QWORD32(_qword1, _qword2)                               \
1513         do {                                                            \
1514                 (_qword1).eq_u32[0] &= (_qword2).eq_u32[0];             \
1515                 (_qword1).eq_u32[1] &= (_qword2).eq_u32[1];             \
1516         _NOTE(CONSTANTCONDITION)                                        \
1517         } while (B_FALSE)
1518
1519 #define EFX_OR_DWORD(_dword1, _dword2)                                  \
1520         do {                                                            \
1521                 (_dword1).ed_u32[0] |= (_dword2).ed_u32[0];             \
1522         _NOTE(CONSTANTCONDITION)                                        \
1523         } while (B_FALSE)
1524
1525 #define EFX_AND_DWORD(_dword1, _dword2)                                 \
1526         do {                                                            \
1527                 (_dword1).ed_u32[0] &= (_dword2).ed_u32[0];             \
1528         _NOTE(CONSTANTCONDITION)                                        \
1529         } while (B_FALSE)
1530
1531 #define EFX_OR_WORD(_word1, _word2)                                     \
1532         do {                                                            \
1533                 (_word1).ew_u16[0] |= (_word2).ew_u16[0];               \
1534         _NOTE(CONSTANTCONDITION)                                        \
1535         } while (B_FALSE)
1536
1537 #define EFX_AND_WORD(_word1, _word2)                                    \
1538         do {                                                            \
1539                 (_word1).ew_u16[0] &= (_word2).ew_u16[0];               \
1540         _NOTE(CONSTANTCONDITION)                                        \
1541         } while (B_FALSE)
1542
1543 #define EFX_OR_BYTE(_byte1, _byte2)                                     \
1544         do {                                                            \
1545                 (_byte1).eb_u8[0] &= (_byte2).eb_u8[0];                 \
1546         _NOTE(CONSTANTCONDITION)                                        \
1547         } while (B_FALSE)
1548
1549 #define EFX_AND_BYTE(_byte1, _byte2)                                    \
1550         do {                                                            \
1551                 (_byte1).eb_u8[0] &= (_byte2).eb_u8[0];                 \
1552         _NOTE(CONSTANTCONDITION)                                        \
1553         } while (B_FALSE)
1554
1555 #if EFSYS_USE_UINT64
1556 #define EFX_OWORD_FIELD         EFX_OWORD_FIELD64
1557 #define EFX_QWORD_FIELD         EFX_QWORD_FIELD64
1558 #define EFX_OWORD_IS_EQUAL      EFX_OWORD_IS_EQUAL64
1559 #define EFX_QWORD_IS_EQUAL      EFX_QWORD_IS_EQUAL64
1560 #define EFX_OWORD_IS_ZERO       EFX_OWORD_IS_ZERO64
1561 #define EFX_QWORD_IS_ZERO       EFX_QWORD_IS_ZERO64
1562 #define EFX_OWORD_IS_SET        EFX_OWORD_IS_SET64
1563 #define EFX_QWORD_IS_SET        EFX_QWORD_IS_SET64
1564 #define EFX_POPULATE_OWORD      EFX_POPULATE_OWORD64
1565 #define EFX_POPULATE_QWORD      EFX_POPULATE_QWORD64
1566 #define EFX_SET_OWORD           EFX_SET_OWORD64
1567 #define EFX_SET_QWORD           EFX_SET_QWORD64
1568 #define EFX_SET_OWORD_FIELD     EFX_SET_OWORD_FIELD64
1569 #define EFX_SET_QWORD_FIELD     EFX_SET_QWORD_FIELD64
1570 #define EFX_SET_OWORD_BIT       EFX_SET_OWORD_BIT64
1571 #define EFX_CLEAR_OWORD_BIT     EFX_CLEAR_OWORD_BIT64
1572 #define EFX_SET_QWORD_BIT       EFX_SET_QWORD_BIT64
1573 #define EFX_CLEAR_QWORD_BIT     EFX_CLEAR_QWORD_BIT64
1574 #define EFX_OR_OWORD            EFX_OR_OWORD64
1575 #define EFX_AND_OWORD           EFX_AND_OWORD64
1576 #define EFX_OR_QWORD            EFX_OR_QWORD64
1577 #define EFX_AND_QWORD           EFX_OR_QWORD64
1578 #else
1579 #define EFX_OWORD_FIELD         EFX_OWORD_FIELD32
1580 #define EFX_QWORD_FIELD         EFX_QWORD_FIELD32
1581 #define EFX_OWORD_IS_EQUAL      EFX_OWORD_IS_EQUAL32
1582 #define EFX_QWORD_IS_EQUAL      EFX_QWORD_IS_EQUAL32
1583 #define EFX_OWORD_IS_ZERO       EFX_OWORD_IS_ZERO32
1584 #define EFX_QWORD_IS_ZERO       EFX_QWORD_IS_ZERO32
1585 #define EFX_OWORD_IS_SET        EFX_OWORD_IS_SET32
1586 #define EFX_QWORD_IS_SET        EFX_QWORD_IS_SET32
1587 #define EFX_POPULATE_OWORD      EFX_POPULATE_OWORD32
1588 #define EFX_POPULATE_QWORD      EFX_POPULATE_QWORD32
1589 #define EFX_SET_OWORD           EFX_SET_OWORD32
1590 #define EFX_SET_QWORD           EFX_SET_QWORD32
1591 #define EFX_SET_OWORD_FIELD     EFX_SET_OWORD_FIELD32
1592 #define EFX_SET_QWORD_FIELD     EFX_SET_QWORD_FIELD32
1593 #define EFX_SET_OWORD_BIT       EFX_SET_OWORD_BIT32
1594 #define EFX_CLEAR_OWORD_BIT     EFX_CLEAR_OWORD_BIT32
1595 #define EFX_SET_QWORD_BIT       EFX_SET_QWORD_BIT32
1596 #define EFX_CLEAR_QWORD_BIT     EFX_CLEAR_QWORD_BIT32
1597 #define EFX_OR_OWORD            EFX_OR_OWORD32
1598 #define EFX_AND_OWORD           EFX_AND_OWORD32
1599 #define EFX_OR_QWORD            EFX_OR_QWORD32
1600 #define EFX_AND_QWORD           EFX_OR_QWORD32
1601 #endif
1602
1603 #ifdef  __cplusplus
1604 }
1605 #endif
1606
1607 #endif  /* _SYS_EFX_TYPES_H */