sys/dev/sfxge/common/efx_types.h

   1 /*-
   2  * Copyright (c) 2007-2015 Solarflare Communications Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *
   8  * 1. Redistributions of source code must retain the above copyright notice,
   9  *    this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright notice,
  11  *    this list of conditions and the following disclaimer in the documentation
  12  *    and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * The views and conclusions contained in the software and documentation are
  27  * those of the authors and should not be interpreted as representing official
  28  * policies, either expressed or implied, of the FreeBSD Project.
  29  *
  30  * Ackowledgement to Fen Systems Ltd.
  31  *
  32  * $FreeBSD$
  33  */
  34
  35 #ifndef _SYS_EFX_TYPES_H
  36 #define _SYS_EFX_TYPES_H
  37
  38 #include "efsys.h"
  39
  40 #ifdef  __cplusplus
  41 extern "C" {
  42 #endif
  43
  44 /*
  45  * Bitfield access
  46  *
  47  * Solarflare NICs make extensive use of bitfields up to 128 bits
  48  * wide.  Since there is no native 128-bit datatype on most systems,
  49  * and since 64-bit datatypes are inefficient on 32-bit systems and
  50  * vice versa, we wrap accesses in a way that uses the most efficient
  51  * datatype.
  52  *
  53  * The NICs are PCI devices and therefore little-endian.  Since most
  54  * of the quantities that we deal with are DMAed to/from host memory,
  55  * we define    our datatypes (efx_oword_t, efx_qword_t and efx_dword_t)
  56  * to be little-endian.
  57  *
  58  * In the less common case of using PIO for individual register
  59  * writes, we construct the little-endian datatype in host memory and
  60  * then use non-swapping register access primitives, rather than
  61  * constructing a native-endian datatype and relying on implicit
  62  * byte-swapping.  (We use a similar strategy for register reads.)
  63  */
  64
  65 /*
  66  * NOTE: Field definitions here and elsewhere are done in terms of a lowest
  67  *       bit number (LBN) and a width.
  68  */
  69
  70 #define EFX_DUMMY_FIELD_LBN 0
  71 #define EFX_DUMMY_FIELD_WIDTH 0
  72
  73 #define EFX_BYTE_0_LBN 0
  74 #define EFX_BYTE_0_WIDTH 8
  75
  76 #define EFX_BYTE_1_LBN 8
  77 #define EFX_BYTE_1_WIDTH 8
  78
  79 #define EFX_BYTE_2_LBN 16
  80 #define EFX_BYTE_2_WIDTH 8
  81
  82 #define EFX_BYTE_3_LBN 24
  83 #define EFX_BYTE_3_WIDTH 8
  84
  85 #define EFX_BYTE_4_LBN 32
  86 #define EFX_BYTE_4_WIDTH 8
  87
  88 #define EFX_BYTE_5_LBN 40
  89 #define EFX_BYTE_5_WIDTH 8
  90
  91 #define EFX_BYTE_6_LBN 48
  92 #define EFX_BYTE_6_WIDTH 8
  93
  94 #define EFX_BYTE_7_LBN 56
  95 #define EFX_BYTE_7_WIDTH 8
  96
  97 #define EFX_WORD_0_LBN 0
  98 #define EFX_WORD_0_WIDTH 16
  99
 100 #define EFX_WORD_1_LBN 16
 101 #define EFX_WORD_1_WIDTH 16
 102
 103 #define EFX_WORD_2_LBN 32
 104 #define EFX_WORD_2_WIDTH 16
 105
 106 #define EFX_WORD_3_LBN 48
 107 #define EFX_WORD_3_WIDTH 16
 108
 109 #define EFX_DWORD_0_LBN 0
 110 #define EFX_DWORD_0_WIDTH 32
 111
 112 #define EFX_DWORD_1_LBN 32
 113 #define EFX_DWORD_1_WIDTH 32
 114
 115 #define EFX_DWORD_2_LBN 64
 116 #define EFX_DWORD_2_WIDTH 32
 117
 118 #define EFX_DWORD_3_LBN 96
 119 #define EFX_DWORD_3_WIDTH 32
 120
 121 /* There are intentionally no EFX_QWORD_0 or EFX_QWORD_1 field definitions
 122  * here as the implementaion of EFX_QWORD_FIELD and EFX_OWORD_FIELD do not
 123  * support field widths larger than 32 bits.
 124  */
 125
 126 /* Specified attribute (i.e. LBN ow WIDTH) of the specified field */
 127 #define EFX_VAL(_field, _attribute)                                     \
 128         _field ## _ ## _attribute
 129
 130 /* Lowest bit number of the specified field */
 131 #define EFX_LOW_BIT(_field)                                             \
 132         EFX_VAL(_field, LBN)
 133
 134 /* Width of the specified field */
 135 #define EFX_WIDTH(_field)                                               \
 136         EFX_VAL(_field, WIDTH)
 137
 138 /* Highest bit number of the specified field */
 139 #define EFX_HIGH_BIT(_field)                                            \
 140         (EFX_LOW_BIT(_field) + EFX_WIDTH(_field) - 1)
 141
 142 /*
 143  * 64-bit mask equal in width to the specified field.
 144  *
 145  * For example, a field with width 5 would have a mask of 0x000000000000001f.
 146  */
 147 #define EFX_MASK64(_field)                                              \
 148         ((EFX_WIDTH(_field) == 64) ? ~((uint64_t)0) :                   \
 149             (((((uint64_t)1) << EFX_WIDTH(_field))) - 1))
 150 /*
 151  * 32-bit mask equal in width to the specified field.
 152  *
 153  * For example, a field with width 5 would have a mask of 0x0000001f.
 154  */
 155 #define EFX_MASK32(_field)                                              \
 156         ((EFX_WIDTH(_field) == 32) ? ~((uint32_t)0) :                   \
 157             (((((uint32_t)1) << EFX_WIDTH(_field))) - 1))
 158
 159 /*
 160  * 16-bit mask equal in width to the specified field.
 161  *
 162  * For example, a field with width 5 would have a mask of 0x001f.
 163  */
 164 #define EFX_MASK16(_field)                                              \
 165         ((EFX_WIDTH(_field) == 16) ? 0xffffu :                          \
 166             (uint16_t)((1 << EFX_WIDTH(_field)) - 1))
 167
 168 /*
 169  * 8-bit mask equal in width to the specified field.
 170  *
 171  * For example, a field with width 5 would have a mask of 0x1f.
 172  */
 173 #define EFX_MASK8(_field)                                               \
 174         ((uint8_t)((1 << EFX_WIDTH(_field)) - 1))
 175
 176 #pragma pack(1)
 177
 178 /*
 179  * A byte (i.e. 8-bit) datatype
 180  */
 181 typedef union efx_byte_u {
 182         uint8_t eb_u8[1];
 183 } efx_byte_t;
 184
 185 /*
 186  * A word (i.e. 16-bit) datatype
 187  *
 188  * This datatype is defined to be little-endian.
 189  */
 190 typedef union efx_word_u {
 191         efx_byte_t ew_byte[2];
 192         uint16_t ew_u16[1];
 193         uint8_t ew_u8[2];
 194 } efx_word_t;
 195
 196 /*
 197  * A doubleword (i.e. 32-bit) datatype
 198  *
 199  * This datatype is defined to be little-endian.
 200  */
 201 typedef union efx_dword_u {
 202         efx_byte_t ed_byte[4];
 203         efx_word_t ed_word[2];
 204         uint32_t ed_u32[1];
 205         uint16_t ed_u16[2];
 206         uint8_t ed_u8[4];
 207 } efx_dword_t;
 208
 209 /*
 210  * A quadword (i.e. 64-bit) datatype
 211  *
 212  * This datatype is defined to be little-endian.
 213  */
 214 typedef union efx_qword_u {
 215         efx_byte_t eq_byte[8];
 216         efx_word_t eq_word[4];
 217         efx_dword_t eq_dword[2];
 218 #if EFSYS_HAS_UINT64
 219         uint64_t eq_u64[1];
 220 #endif
 221         uint32_t eq_u32[2];
 222         uint16_t eq_u16[4];
 223         uint8_t eq_u8[8];
 224 } efx_qword_t;
 225
 226 /*
 227  * An octword (i.e. 128-bit) datatype
 228  *
 229  * This datatype is defined to be little-endian.
 230  */
 231 typedef union efx_oword_u {
 232         efx_byte_t eo_byte[16];
 233         efx_word_t eo_word[8];
 234         efx_dword_t eo_dword[4];
 235         efx_qword_t eo_qword[2];
 236 #if EFSYS_HAS_SSE2_M128
 237         __m128i eo_u128[1];
 238 #endif
 239 #if EFSYS_HAS_UINT64
 240         uint64_t eo_u64[2];
 241 #endif
 242         uint32_t eo_u32[4];
 243         uint16_t eo_u16[8];
 244         uint8_t eo_u8[16];
 245 } efx_oword_t;
 246
 247 #pragma pack()
 248
 249 #define __SWAP16(_x)                            \
 250         ((((_x) & 0xff) << 8) |                 \
 251         (((_x) >> 8) & 0xff))
 252
 253 #define __SWAP32(_x)                            \
 254         ((__SWAP16((_x) & 0xffff) << 16) |      \
 255         __SWAP16(((_x) >> 16) & 0xffff))
 256
 257 #define __SWAP64(_x)                            \
 258         ((__SWAP32((_x) & 0xffffffff) << 32) |  \
 259         __SWAP32(((_x) >> 32) & 0xffffffff))
 260
 261 #define __NOSWAP16(_x)          (_x)
 262 #define __NOSWAP32(_x)          (_x)
 263 #define __NOSWAP64(_x)          (_x)
 264
 265 #if EFSYS_IS_BIG_ENDIAN
 266
 267 #define __CPU_TO_LE_16(_x)      (uint16_t)__SWAP16(_x)
 268 #define __LE_TO_CPU_16(_x)      (uint16_t)__SWAP16(_x)
 269 #define __CPU_TO_BE_16(_x)      (uint16_t)__NOSWAP16(_x)
 270 #define __BE_TO_CPU_16(_x)      (uint16_t)__NOSWAP16(_x)
 271
 272 #define __CPU_TO_LE_32(_x)      (uint32_t)__SWAP32(_x)
 273 #define __LE_TO_CPU_32(_x)      (uint32_t)__SWAP32(_x)
 274 #define __CPU_TO_BE_32(_x)      (uint32_t)__NOSWAP32(_x)
 275 #define __BE_TO_CPU_32(_x)      (uint32_t)__NOSWAP32(_x)
 276
 277 #define __CPU_TO_LE_64(_x)      (uint64_t)__SWAP64(_x)
 278 #define __LE_TO_CPU_64(_x)      (uint64_t)__SWAP64(_x)
 279 #define __CPU_TO_BE_64(_x)      (uint64_t)__NOSWAP64(_x)
 280 #define __BE_TO_CPU_64(_x)      (uint64_t)__NOSWAP64(_x)
 281
 282 #elif EFSYS_IS_LITTLE_ENDIAN
 283
 284 #define __CPU_TO_LE_16(_x)      (uint16_t)__NOSWAP16(_x)
 285 #define __LE_TO_CPU_16(_x)      (uint16_t)__NOSWAP16(_x)
 286 #define __CPU_TO_BE_16(_x)      (uint16_t)__SWAP16(_x)
 287 #define __BE_TO_CPU_16(_x)      (uint16_t)__SWAP16(_x)
 288
 289 #define __CPU_TO_LE_32(_x)      (uint32_t)__NOSWAP32(_x)
 290 #define __LE_TO_CPU_32(_x)      (uint32_t)__NOSWAP32(_x)
 291 #define __CPU_TO_BE_32(_x)      (uint32_t)__SWAP32(_x)
 292 #define __BE_TO_CPU_32(_x)      (uint32_t)__SWAP32(_x)
 293
 294 #define __CPU_TO_LE_64(_x)      (uint64_t)__NOSWAP64(_x)
 295 #define __LE_TO_CPU_64(_x)      (uint64_t)__NOSWAP64(_x)
 296 #define __CPU_TO_BE_64(_x)      (uint64_t)__SWAP64(_x)
 297 #define __BE_TO_CPU_64(_x)      (uint64_t)__SWAP64(_x)
 298
 299 #else
 300
 301 #error "Neither of EFSYS_IS_{BIG,LITTLE}_ENDIAN is set"
 302
 303 #endif
 304
 305 #define __NATIVE_8(_x)  (uint8_t)(_x)
 306
 307 /* Format string for printing an efx_byte_t */
 308 #define EFX_BYTE_FMT "0x%02x"
 309
 310 /* Format string for printing an efx_word_t */
 311 #define EFX_WORD_FMT "0x%04x"
 312
 313 /* Format string for printing an efx_dword_t */
 314 #define EFX_DWORD_FMT "0x%08x"
 315
 316 /* Format string for printing an efx_qword_t */
 317 #define EFX_QWORD_FMT "0x%08x:%08x"
 318
 319 /* Format string for printing an efx_oword_t */
 320 #define EFX_OWORD_FMT "0x%08x:%08x:%08x:%08x"
 321
 322 /* Parameters for printing an efx_byte_t */
 323 #define EFX_BYTE_VAL(_byte)                                     \
 324         ((unsigned int)__NATIVE_8((_byte).eb_u8[0]))
 325
 326 /* Parameters for printing an efx_word_t */
 327 #define EFX_WORD_VAL(_word)                                     \
 328         ((unsigned int)__LE_TO_CPU_16((_word).ew_u16[0]))
 329
 330 /* Parameters for printing an efx_dword_t */
 331 #define EFX_DWORD_VAL(_dword)                                   \
 332         ((unsigned int)__LE_TO_CPU_32((_dword).ed_u32[0]))
 333
 334 /* Parameters for printing an efx_qword_t */
 335 #define EFX_QWORD_VAL(_qword)                                   \
 336         ((unsigned int)__LE_TO_CPU_32((_qword).eq_u32[1])),     \
 337         ((unsigned int)__LE_TO_CPU_32((_qword).eq_u32[0]))
 338
 339 /* Parameters for printing an efx_oword_t */
 340 #define EFX_OWORD_VAL(_oword)                                   \
 341         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[3])),     \
 342         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[2])),     \
 343         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[1])),     \
 344         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[0]))
 345
 346 /*
 347  * Stop lint complaining about some shifts.
 348  */
 349 #ifdef  __lint
 350 extern int fix_lint;
 351 #define FIX_LINT(_x)    (_x + fix_lint)
 352 #else
 353 #define FIX_LINT(_x)    (_x)
 354 #endif
 355
 356 /*
 357  * Extract bit field portion [low,high) from the native-endian element
 358  * which contains bits [min,max).
 359  *
 360  * For example, suppose "element" represents the high 32 bits of a
 361  * 64-bit value, and we wish to extract the bits belonging to the bit
 362  * field occupying bits 28-45 of this 64-bit value.
 363  *
 364  * Then EFX_EXTRACT(_element, 32, 63, 28, 45) would give
 365  *
 366  *   (_element) << 4
 367  *
 368  * The result will contain the relevant bits filled in in the range
 369  * [0,high-low), with garbage in bits [high-low+1,...).
 370  */
 371 #define EFX_EXTRACT_NATIVE(_element, _min, _max, _low, _high)           \
 372         ((FIX_LINT(_low > _max) || FIX_LINT(_high < _min)) ?            \
 373                 0U :                                                    \
 374                 ((_low > _min) ?                                        \
 375                         ((_element) >> (_low - _min)) :                 \
 376                         ((_element) << (_min - _low))))
 377
 378 /*
 379  * Extract bit field portion [low,high) from the 64-bit little-endian
 380  * element which contains bits [min,max)
 381  */
 382 #define EFX_EXTRACT64(_element, _min, _max, _low, _high)                \
 383         EFX_EXTRACT_NATIVE(__LE_TO_CPU_64(_element), _min, _max, _low, _high)
 384
 385 /*
 386  * Extract bit field portion [low,high) from the 32-bit little-endian
 387  * element which contains bits [min,max)
 388  */
 389 #define EFX_EXTRACT32(_element, _min, _max, _low, _high)                \
 390         EFX_EXTRACT_NATIVE(__LE_TO_CPU_32(_element), _min, _max, _low, _high)
 391
 392 /*
 393  * Extract bit field portion [low,high) from the 16-bit little-endian
 394  * element which contains bits [min,max)
 395  */
 396 #define EFX_EXTRACT16(_element, _min, _max, _low, _high)                \
 397         EFX_EXTRACT_NATIVE(__LE_TO_CPU_16(_element), _min, _max, _low, _high)
 398
 399 /*
 400  * Extract bit field portion [low,high) from the 8-bit
 401  * element which contains bits [min,max)
 402  */
 403 #define EFX_EXTRACT8(_element, _min, _max, _low, _high)                 \
 404         EFX_EXTRACT_NATIVE(__NATIVE_8(_element), _min, _max, _low, _high)
 405
 406 #define EFX_EXTRACT_OWORD64(_oword, _low, _high)                        \
 407         (EFX_EXTRACT64((_oword).eo_u64[0], FIX_LINT(0), FIX_LINT(63),   \
 408             _low, _high) |                                              \
 409         EFX_EXTRACT64((_oword).eo_u64[1], FIX_LINT(64), FIX_LINT(127),  \
 410             _low, _high))
 411
 412 #define EFX_EXTRACT_OWORD32(_oword, _low, _high)                        \
 413         (EFX_EXTRACT32((_oword).eo_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 414             _low, _high) |                                              \
 415         EFX_EXTRACT32((_oword).eo_u32[1], FIX_LINT(32), FIX_LINT(63),   \
 416             _low, _high) |                                              \
 417         EFX_EXTRACT32((_oword).eo_u32[2], FIX_LINT(64), FIX_LINT(95),   \
 418             _low, _high) |                                              \
 419         EFX_EXTRACT32((_oword).eo_u32[3], FIX_LINT(96), FIX_LINT(127),  \
 420             _low, _high))
 421
 422 #define EFX_EXTRACT_QWORD64(_qword, _low, _high)                        \
 423         (EFX_EXTRACT64((_qword).eq_u64[0], FIX_LINT(0), FIX_LINT(63),   \
 424             _low, _high))
 425
 426 #define EFX_EXTRACT_QWORD32(_qword, _low, _high)                        \
 427         (EFX_EXTRACT32((_qword).eq_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 428             _low, _high) |                                              \
 429         EFX_EXTRACT32((_qword).eq_u32[1], FIX_LINT(32), FIX_LINT(63),   \
 430             _low, _high))
 431
 432 #define EFX_EXTRACT_DWORD(_dword, _low, _high)                          \
 433         (EFX_EXTRACT32((_dword).ed_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 434             _low, _high))
 435
 436 #define EFX_EXTRACT_WORD(_word, _low, _high)                            \
 437         (EFX_EXTRACT16((_word).ew_u16[0], FIX_LINT(0), FIX_LINT(15),    \
 438             _low, _high))
 439
 440 #define EFX_EXTRACT_BYTE(_byte, _low, _high)                            \
 441         (EFX_EXTRACT8((_byte).eb_u8[0], FIX_LINT(0), FIX_LINT(7),       \
 442             _low, _high))
 443
 444
 445 #define EFX_OWORD_FIELD64(_oword, _field)                               \
 446         ((uint32_t)EFX_EXTRACT_OWORD64(_oword, EFX_LOW_BIT(_field),     \
 447             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 448
 449 #define EFX_OWORD_FIELD32(_oword, _field)                               \
 450         (EFX_EXTRACT_OWORD32(_oword, EFX_LOW_BIT(_field),               \
 451             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 452
 453 #define EFX_QWORD_FIELD64(_qword, _field)                               \
 454         ((uint32_t)EFX_EXTRACT_QWORD64(_qword, EFX_LOW_BIT(_field),     \
 455             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 456
 457 #define EFX_QWORD_FIELD32(_qword, _field)                               \
 458         (EFX_EXTRACT_QWORD32(_qword, EFX_LOW_BIT(_field),               \
 459             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 460
 461 #define EFX_DWORD_FIELD(_dword, _field)                                 \
 462         (EFX_EXTRACT_DWORD(_dword, EFX_LOW_BIT(_field),                 \
 463             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 464
 465 #define EFX_WORD_FIELD(_word, _field)                                   \
 466         (EFX_EXTRACT_WORD(_word, EFX_LOW_BIT(_field),                   \
 467             EFX_HIGH_BIT(_field)) & EFX_MASK16(_field))
 468
 469 #define EFX_BYTE_FIELD(_byte, _field)                                   \
 470         (EFX_EXTRACT_BYTE(_byte, EFX_LOW_BIT(_field),                   \
 471             EFX_HIGH_BIT(_field)) & EFX_MASK8(_field))
 472
 473
 474 #define EFX_OWORD_IS_EQUAL64(_oword_a, _oword_b)                        \
 475         ((_oword_a).eo_u64[0] == (_oword_b).eo_u64[0] &&                \
 476             (_oword_a).eo_u64[1] == (_oword_b).eo_u64[1])
 477
 478 #define EFX_OWORD_IS_EQUAL32(_oword_a, _oword_b)                        \
 479         ((_oword_a).eo_u32[0] == (_oword_b).eo_u32[0] &&                \
 480             (_oword_a).eo_u32[1] == (_oword_b).eo_u32[1] &&             \
 481             (_oword_a).eo_u32[2] == (_oword_b).eo_u32[2] &&             \
 482             (_oword_a).eo_u32[3] == (_oword_b).eo_u32[3])
 483
 484 #define EFX_QWORD_IS_EQUAL64(_qword_a, _qword_b)                        \
 485         ((_qword_a).eq_u64[0] == (_qword_b).eq_u64[0])
 486
 487 #define EFX_QWORD_IS_EQUAL32(_qword_a, _qword_b)                        \
 488         ((_qword_a).eq_u32[0] == (_qword_b).eq_u32[0] &&                \
 489             (_qword_a).eq_u32[1] == (_qword_b).eq_u32[1])
 490
 491 #define EFX_DWORD_IS_EQUAL(_dword_a, _dword_b)                          \
 492         ((_dword_a).ed_u32[0] == (_dword_b).ed_u32[0])
 493
 494 #define EFX_WORD_IS_EQUAL(_word_a, _word_b)                             \
 495         ((_word_a).ew_u16[0] == (_word_b).ew_u16[0])
 496
 497 #define EFX_BYTE_IS_EQUAL(_byte_a, _byte_b)                             \
 498         ((_byte_a).eb_u8[0] == (_byte_b).eb_u8[0])
 499
 500
 501 #define EFX_OWORD_IS_ZERO64(_oword)                                     \
 502         (((_oword).eo_u64[0] |                                          \
 503             (_oword).eo_u64[1]) == 0)
 504
 505 #define EFX_OWORD_IS_ZERO32(_oword)                                     \
 506         (((_oword).eo_u32[0] |                                          \
 507             (_oword).eo_u32[1] |                                        \
 508             (_oword).eo_u32[2] |                                        \
 509             (_oword).eo_u32[3]) == 0)
 510
 511 #define EFX_QWORD_IS_ZERO64(_qword)                                     \
 512         (((_qword).eq_u64[0]) == 0)
 513
 514 #define EFX_QWORD_IS_ZERO32(_qword)                                     \
 515         (((_qword).eq_u32[0] |                                          \
 516             (_qword).eq_u32[1]) == 0)
 517
 518 #define EFX_DWORD_IS_ZERO(_dword)                                       \
 519         (((_dword).ed_u32[0]) == 0)
 520
 521 #define EFX_WORD_IS_ZERO(_word)                                         \
 522         (((_word).ew_u16[0]) == 0)
 523
 524 #define EFX_BYTE_IS_ZERO(_byte)                                         \
 525         (((_byte).eb_u8[0]) == 0)
 526
 527
 528 #define EFX_OWORD_IS_SET64(_oword)                                      \
 529         (((_oword).eo_u64[0] &                                          \
 530             (_oword).eo_u64[1]) == ~((uint64_t)0))
 531
 532 #define EFX_OWORD_IS_SET32(_oword)                                      \
 533         (((_oword).eo_u32[0] &                                          \
 534             (_oword).eo_u32[1] &                                        \
 535             (_oword).eo_u32[2] &                                        \
 536             (_oword).eo_u32[3]) == ~((uint32_t)0))
 537
 538 #define EFX_QWORD_IS_SET64(_qword)                                      \
 539         (((_qword).eq_u64[0]) == ~((uint32_t)0))
 540
 541 #define EFX_QWORD_IS_SET32(_qword)                                      \
 542         (((_qword).eq_u32[0] &                                          \
 543             (_qword).eq_u32[1]) == ~((uint32_t)0))
 544
 545 #define EFX_DWORD_IS_SET(_dword)                                        \
 546         ((_dword).ed_u32[0] == ~((uint32_t)0))
 547
 548 #define EFX_WORD_IS_SET(_word)                                          \
 549         ((_word).ew_u16[0] == ~((uint16_t)0))
 550
 551 #define EFX_BYTE_IS_SET(_byte)                                          \
 552         ((_byte).eb_u8[0] == ~((uint8_t)0))
 553
 554 /*
 555  * Construct bit field portion
 556  *
 557  * Creates the portion of the bit field [low,high) that lies within
 558  * the range [min,max).
 559  */
 560
 561 #define EFX_INSERT_NATIVE64(_min, _max, _low, _high, _value)            \
 562         (((_low > _max) || (_high < _min)) ?                            \
 563                 0U :                                                    \
 564                 ((_low > _min) ?                                        \
 565                         (((uint64_t)(_value)) << (_low - _min)) :       \
 566                         (((uint64_t)(_value)) >> (_min - _low))))
 567
 568 #define EFX_INSERT_NATIVE32(_min, _max, _low, _high, _value)            \
 569         (((_low > _max) || (_high < _min)) ?                            \
 570                 0U :                                                    \
 571                 ((_low > _min) ?                                        \
 572                         (((uint32_t)(_value)) << (_low - _min)) :       \
 573                         (((uint32_t)(_value)) >> (_min - _low))))
 574
 575 #define EFX_INSERT_NATIVE16(_min, _max, _low, _high, _value)            \
 576         (((_low > _max) || (_high < _min)) ?                            \
 577                 0U :                                                    \
 578                 (uint16_t)((_low > _min) ?                              \
 579                                 ((_value) << (_low - _min)) :           \
 580                                 ((_value) >> (_min - _low))))
 581
 582 #define EFX_INSERT_NATIVE8(_min, _max, _low, _high, _value)             \
 583         (((_low > _max) || (_high < _min)) ?                            \
 584                 0U :                                                    \
 585                 (uint8_t)((_low > _min) ?                               \
 586                                 ((_value) << (_low - _min)) :   \
 587                                 ((_value) >> (_min - _low))))
 588
 589 /*
 590  * Construct bit field portion
 591  *
 592  * Creates the portion of the named bit field that lies within the
 593  * range [min,max).
 594  */
 595 #define EFX_INSERT_FIELD_NATIVE64(_min, _max, _field, _value)           \
 596         EFX_INSERT_NATIVE64(_min, _max, EFX_LOW_BIT(_field),            \
 597             EFX_HIGH_BIT(_field), _value)
 598
 599 #define EFX_INSERT_FIELD_NATIVE32(_min, _max, _field, _value)           \
 600         EFX_INSERT_NATIVE32(_min, _max, EFX_LOW_BIT(_field),            \
 601             EFX_HIGH_BIT(_field), _value)
 602
 603 #define EFX_INSERT_FIELD_NATIVE16(_min, _max, _field, _value)           \
 604         EFX_INSERT_NATIVE16(_min, _max, EFX_LOW_BIT(_field),            \
 605             EFX_HIGH_BIT(_field), _value)
 606
 607 #define EFX_INSERT_FIELD_NATIVE8(_min, _max, _field, _value)            \
 608         EFX_INSERT_NATIVE8(_min, _max, EFX_LOW_BIT(_field),             \
 609             EFX_HIGH_BIT(_field), _value)
 610
 611 /*
 612  * Construct bit field
 613  *
 614  * Creates the portion of the named bit fields that lie within the
 615  * range [min,max).
 616  */
 617 #define EFX_INSERT_FIELDS64(_min, _max,                                 \
 618             _field1, _value1, _field2, _value2, _field3, _value3,       \
 619             _field4, _value4, _field5, _value5, _field6, _value6,       \
 620             _field7, _value7, _field8, _value8, _field9, _value9,       \
 621             _field10, _value10)                                         \
 622         __CPU_TO_LE_64(                                                 \
 623             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field1, _value1) |   \
 624             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field2, _value2) |   \
 625             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field3, _value3) |   \
 626             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field4, _value4) |   \
 627             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field5, _value5) |   \
 628             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field6, _value6) |   \
 629             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field7, _value7) |   \
 630             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field8, _value8) |   \
 631             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field9, _value9) |   \
 632             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field10, _value10))
 633
 634 #define EFX_INSERT_FIELDS32(_min, _max,                                 \
 635             _field1, _value1, _field2, _value2, _field3, _value3,       \
 636             _field4, _value4, _field5, _value5, _field6, _value6,       \
 637             _field7, _value7, _field8, _value8, _field9, _value9,       \
 638             _field10, _value10)                                         \
 639         __CPU_TO_LE_32(                                                 \
 640             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field1, _value1) |   \
 641             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field2, _value2) |   \
 642             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field3, _value3) |   \
 643             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field4, _value4) |   \
 644             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field5, _value5) |   \
 645             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field6, _value6) |   \
 646             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field7, _value7) |   \
 647             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field8, _value8) |   \
 648             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field9, _value9) |   \
 649             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field10, _value10))
 650
 651 #define EFX_INSERT_FIELDS16(_min, _max,                                 \
 652             _field1, _value1, _field2, _value2, _field3, _value3,       \
 653             _field4, _value4, _field5, _value5, _field6, _value6,       \
 654             _field7, _value7, _field8, _value8, _field9, _value9,       \
 655             _field10, _value10)                                         \
 656         __CPU_TO_LE_16(                                                 \
 657             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field1, _value1) |   \
 658             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field2, _value2) |   \
 659             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field3, _value3) |   \
 660             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field4, _value4) |   \
 661             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field5, _value5) |   \
 662             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field6, _value6) |   \
 663             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field7, _value7) |   \
 664             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field8, _value8) |   \
 665             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field9, _value9) |   \
 666             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field10, _value10))
 667
 668 #define EFX_INSERT_FIELDS8(_min, _max,                                  \
 669             _field1, _value1, _field2, _value2, _field3, _value3,       \
 670             _field4, _value4, _field5, _value5, _field6, _value6,       \
 671             _field7, _value7, _field8, _value8, _field9, _value9,       \
 672             _field10, _value10)                                         \
 673         __NATIVE_8(                                                     \
 674             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field1, _value1) |    \
 675             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field2, _value2) |    \
 676             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field3, _value3) |    \
 677             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field4, _value4) |    \
 678             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field5, _value5) |    \
 679             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field6, _value6) |    \
 680             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field7, _value7) |    \
 681             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field8, _value8) |    \
 682             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field9, _value9) |    \
 683             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field10, _value10))
 684
 685 #define EFX_POPULATE_OWORD64(_oword,                                    \
 686             _field1, _value1, _field2, _value2, _field3, _value3,       \
 687             _field4, _value4, _field5, _value5, _field6, _value6,       \
 688             _field7, _value7, _field8, _value8, _field9, _value9,       \
 689             _field10, _value10)                                         \
 690         do {                                                            \
 691                 _NOTE(CONSTANTCONDITION)                                \
 692                 (_oword).eo_u64[0] = EFX_INSERT_FIELDS64(0, 63,         \
 693                     _field1, _value1, _field2, _value2,                 \
 694                     _field3, _value3, _field4, _value4,                 \
 695                     _field5, _value5, _field6, _value6,                 \
 696                     _field7, _value7, _field8, _value8,                 \
 697                     _field9, _value9, _field10, _value10);              \
 698                 _NOTE(CONSTANTCONDITION)                                \
 699                 (_oword).eo_u64[1] = EFX_INSERT_FIELDS64(64, 127,       \
 700                     _field1, _value1, _field2, _value2,                 \
 701                     _field3, _value3, _field4, _value4,                 \
 702                     _field5, _value5, _field6, _value6,                 \
 703                     _field7, _value7, _field8, _value8,                 \
 704                     _field9, _value9, _field10, _value10);              \
 705         _NOTE(CONSTANTCONDITION)                                        \
 706         } while (B_FALSE)
 707
 708 #define EFX_POPULATE_OWORD32(_oword,                                    \
 709             _field1, _value1, _field2, _value2, _field3, _value3,       \
 710             _field4, _value4, _field5, _value5, _field6, _value6,       \
 711             _field7, _value7, _field8, _value8, _field9, _value9,       \
 712             _field10, _value10)                                         \
 713         do {                                                            \
 714                 _NOTE(CONSTANTCONDITION)                                \
 715                 (_oword).eo_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 716                     _field1, _value1, _field2, _value2,                 \
 717                     _field3, _value3, _field4, _value4,                 \
 718                     _field5, _value5, _field6, _value6,                 \
 719                     _field7, _value7, _field8, _value8,                 \
 720                     _field9, _value9, _field10, _value10);              \
 721                 _NOTE(CONSTANTCONDITION)                                \
 722                 (_oword).eo_u32[1] = EFX_INSERT_FIELDS32(32, 63,        \
 723                     _field1, _value1, _field2, _value2,                 \
 724                     _field3, _value3, _field4, _value4,                 \
 725                     _field5, _value5, _field6, _value6,                 \
 726                     _field7, _value7, _field8, _value8,                 \
 727                     _field9, _value9, _field10, _value10);              \
 728                 _NOTE(CONSTANTCONDITION)                                \
 729                 (_oword).eo_u32[2] = EFX_INSERT_FIELDS32(64, 95,        \
 730                     _field1, _value1, _field2, _value2,                 \
 731                     _field3, _value3, _field4, _value4,                 \
 732                     _field5, _value5, _field6, _value6,                 \
 733                     _field7, _value7, _field8, _value8,                 \
 734                     _field9, _value9, _field10, _value10);              \
 735                 _NOTE(CONSTANTCONDITION)                                \
 736                 (_oword).eo_u32[3] = EFX_INSERT_FIELDS32(96, 127,       \
 737                     _field1, _value1, _field2, _value2,                 \
 738                     _field3, _value3, _field4, _value4,                 \
 739                     _field5, _value5, _field6, _value6,                 \
 740                     _field7, _value7, _field8, _value8,                 \
 741                     _field9, _value9, _field10, _value10);              \
 742         _NOTE(CONSTANTCONDITION)                                        \
 743         } while (B_FALSE)
 744
 745 #define EFX_POPULATE_QWORD64(_qword,                                    \
 746             _field1, _value1, _field2, _value2, _field3, _value3,       \
 747             _field4, _value4, _field5, _value5, _field6, _value6,       \
 748             _field7, _value7, _field8, _value8, _field9, _value9,       \
 749             _field10, _value10)                                         \
 750         do {                                                            \
 751                 _NOTE(CONSTANTCONDITION)                                \
 752                 (_qword).eq_u64[0] = EFX_INSERT_FIELDS64(0, 63,         \
 753                     _field1, _value1, _field2, _value2,                 \
 754                     _field3, _value3, _field4, _value4,                 \
 755                     _field5, _value5, _field6, _value6,                 \
 756                     _field7, _value7, _field8, _value8,                 \
 757                     _field9, _value9, _field10, _value10);              \
 758         _NOTE(CONSTANTCONDITION)                                        \
 759         } while (B_FALSE)
 760
 761 #define EFX_POPULATE_QWORD32(_qword,                                    \
 762             _field1, _value1, _field2, _value2, _field3, _value3,       \
 763             _field4, _value4, _field5, _value5, _field6, _value6,       \
 764             _field7, _value7, _field8, _value8, _field9, _value9,       \
 765             _field10, _value10)                                         \
 766         do {                                                            \
 767                 _NOTE(CONSTANTCONDITION)                                \
 768                 (_qword).eq_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 769                     _field1, _value1, _field2, _value2,                 \
 770                     _field3, _value3, _field4, _value4,                 \
 771                     _field5, _value5, _field6, _value6,                 \
 772                     _field7, _value7, _field8, _value8,                 \
 773                     _field9, _value9, _field10, _value10);              \
 774                 _NOTE(CONSTANTCONDITION)                                \
 775                 (_qword).eq_u32[1] = EFX_INSERT_FIELDS32(32, 63,        \
 776                     _field1, _value1, _field2, _value2,                 \
 777                     _field3, _value3, _field4, _value4,                 \
 778                     _field5, _value5, _field6, _value6,                 \
 779                     _field7, _value7, _field8, _value8,                 \
 780                     _field9, _value9, _field10, _value10);              \
 781         _NOTE(CONSTANTCONDITION)                                        \
 782         } while (B_FALSE)
 783
 784 #define EFX_POPULATE_DWORD(_dword,                                      \
 785             _field1, _value1, _field2, _value2, _field3, _value3,       \
 786             _field4, _value4, _field5, _value5, _field6, _value6,       \
 787             _field7, _value7, _field8, _value8, _field9, _value9,       \
 788             _field10, _value10)                                         \
 789         do {                                                            \
 790                 _NOTE(CONSTANTCONDITION)                                \
 791                 (_dword).ed_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 792                     _field1, _value1, _field2, _value2,                 \
 793                     _field3, _value3, _field4, _value4,                 \
 794                     _field5, _value5, _field6, _value6,                 \
 795                     _field7, _value7, _field8, _value8,                 \
 796                     _field9, _value9, _field10, _value10);              \
 797         _NOTE(CONSTANTCONDITION)                                        \
 798         } while (B_FALSE)
 799
 800 #define EFX_POPULATE_WORD(_word,                                        \
 801             _field1, _value1, _field2, _value2, _field3, _value3,       \
 802             _field4, _value4, _field5, _value5, _field6, _value6,       \
 803             _field7, _value7, _field8, _value8, _field9, _value9,       \
 804             _field10, _value10)                                         \
 805         do {                                                            \
 806                 _NOTE(CONSTANTCONDITION)                                \
 807                 (_word).ew_u16[0] = EFX_INSERT_FIELDS16(0, 15,          \
 808                     _field1, _value1, _field2, _value2,                 \
 809                     _field3, _value3, _field4, _value4,                 \
 810                     _field5, _value5, _field6, _value6,                 \
 811                     _field7, _value7, _field8, _value8,                 \
 812                     _field9, _value9, _field10, _value10);              \
 813         _NOTE(CONSTANTCONDITION)                                        \
 814         } while (B_FALSE)
 815
 816 #define EFX_POPULATE_BYTE(_byte,                                        \
 817             _field1, _value1, _field2, _value2, _field3, _value3,       \
 818             _field4, _value4, _field5, _value5, _field6, _value6,       \
 819             _field7, _value7, _field8, _value8, _field9, _value9,       \
 820             _field10, _value10)                                         \
 821         do {                                                            \
 822                 _NOTE(CONSTANTCONDITION)                                \
 823                 (_byte).eb_u8[0] = EFX_INSERT_FIELDS8(0, 7,             \
 824                     _field1, _value1, _field2, _value2,                 \
 825                     _field3, _value3, _field4, _value4,                 \
 826                     _field5, _value5, _field6, _value6,                 \
 827                     _field7, _value7, _field8, _value8,                 \
 828                     _field9, _value9, _field10, _value10);              \
 829         _NOTE(CONSTANTCONDITION)                                        \
 830         } while (B_FALSE)
 831
 832 /* Populate an octword field with various numbers of arguments */
 833 #define EFX_POPULATE_OWORD_10 EFX_POPULATE_OWORD
 834
 835 #define EFX_POPULATE_OWORD_9(_oword,                                    \
 836             _field1, _value1, _field2, _value2, _field3, _value3,       \
 837             _field4, _value4, _field5, _value5, _field6, _value6,       \
 838             _field7, _value7, _field8, _value8, _field9, _value9)       \
 839         EFX_POPULATE_OWORD_10(_oword, EFX_DUMMY_FIELD, 0,               \
 840             _field1, _value1, _field2, _value2, _field3, _value3,       \
 841             _field4, _value4, _field5, _value5, _field6, _value6,       \
 842             _field7, _value7, _field8, _value8, _field9, _value9)
 843
 844 #define EFX_POPULATE_OWORD_8(_oword,                                    \
 845             _field1, _value1, _field2, _value2, _field3, _value3,       \
 846             _field4, _value4, _field5, _value5, _field6, _value6,       \
 847             _field7, _value7, _field8, _value8)                         \
 848         EFX_POPULATE_OWORD_9(_oword, EFX_DUMMY_FIELD, 0,                \
 849             _field1, _value1, _field2, _value2, _field3, _value3,       \
 850             _field4, _value4, _field5, _value5, _field6, _value6,       \
 851             _field7, _value7, _field8, _value8)
 852
 853 #define EFX_POPULATE_OWORD_7(_oword,                                    \
 854             _field1, _value1, _field2, _value2, _field3, _value3,       \
 855             _field4, _value4, _field5, _value5, _field6, _value6,       \
 856             _field7, _value7)                                           \
 857         EFX_POPULATE_OWORD_8(_oword, EFX_DUMMY_FIELD, 0,                \
 858             _field1, _value1, _field2, _value2, _field3, _value3,       \
 859             _field4, _value4, _field5, _value5, _field6, _value6,       \
 860             _field7, _value7)
 861
 862 #define EFX_POPULATE_OWORD_6(_oword,                                    \
 863             _field1, _value1, _field2, _value2, _field3, _value3,       \
 864             _field4, _value4, _field5, _value5, _field6, _value6)       \
 865         EFX_POPULATE_OWORD_7(_oword, EFX_DUMMY_FIELD, 0,                \
 866             _field1, _value1, _field2, _value2, _field3, _value3,       \
 867             _field4, _value4, _field5, _value5, _field6, _value6)
 868
 869 #define EFX_POPULATE_OWORD_5(_oword,                                    \
 870             _field1, _value1, _field2, _value2, _field3, _value3,       \
 871             _field4, _value4, _field5, _value5)                         \
 872         EFX_POPULATE_OWORD_6(_oword, EFX_DUMMY_FIELD, 0,                \
 873             _field1, _value1, _field2, _value2, _field3, _value3,       \
 874             _field4, _value4, _field5, _value5)
 875
 876 #define EFX_POPULATE_OWORD_4(_oword,                                    \
 877             _field1, _value1, _field2, _value2, _field3, _value3,       \
 878             _field4, _value4)                                           \
 879         EFX_POPULATE_OWORD_5(_oword, EFX_DUMMY_FIELD, 0,                \
 880             _field1, _value1, _field2, _value2, _field3, _value3,       \
 881             _field4, _value4)
 882
 883 #define EFX_POPULATE_OWORD_3(_oword,                                    \
 884             _field1, _value1, _field2, _value2, _field3, _value3)       \
 885         EFX_POPULATE_OWORD_4(_oword, EFX_DUMMY_FIELD, 0,                \
 886             _field1, _value1, _field2, _value2, _field3, _value3)
 887
 888 #define EFX_POPULATE_OWORD_2(_oword,                                    \
 889             _field1, _value1, _field2, _value2)                         \
 890         EFX_POPULATE_OWORD_3(_oword, EFX_DUMMY_FIELD, 0,                \
 891             _field1, _value1, _field2, _value2)
 892
 893 #define EFX_POPULATE_OWORD_1(_oword,                                    \
 894             _field1, _value1)                                           \
 895         EFX_POPULATE_OWORD_2(_oword, EFX_DUMMY_FIELD, 0,                \
 896             _field1, _value1)
 897
 898 #define EFX_ZERO_OWORD(_oword)                                          \
 899         EFX_POPULATE_OWORD_1(_oword, EFX_DUMMY_FIELD, 0)
 900
 901 #define EFX_SET_OWORD(_oword)                                           \
 902         EFX_POPULATE_OWORD_4(_oword,                                    \
 903             EFX_DWORD_0, 0xffffffff, EFX_DWORD_1, 0xffffffff,           \
 904             EFX_DWORD_2, 0xffffffff, EFX_DWORD_3, 0xffffffff)
 905
 906 /* Populate a quadword field with various numbers of arguments */
 907 #define EFX_POPULATE_QWORD_10 EFX_POPULATE_QWORD
 908
 909 #define EFX_POPULATE_QWORD_9(_qword,                                    \
 910             _field1, _value1, _field2, _value2, _field3, _value3,       \
 911             _field4, _value4, _field5, _value5, _field6, _value6,       \
 912             _field7, _value7, _field8, _value8, _field9, _value9)       \
 913         EFX_POPULATE_QWORD_10(_qword, EFX_DUMMY_FIELD, 0,               \
 914             _field1, _value1, _field2, _value2, _field3, _value3,       \
 915             _field4, _value4, _field5, _value5, _field6, _value6,       \
 916             _field7, _value7, _field8, _value8, _field9, _value9)
 917
 918 #define EFX_POPULATE_QWORD_8(_qword,                                    \
 919             _field1, _value1, _field2, _value2, _field3, _value3,       \
 920             _field4, _value4, _field5, _value5, _field6, _value6,       \
 921             _field7, _value7, _field8, _value8)                         \
 922         EFX_POPULATE_QWORD_9(_qword, EFX_DUMMY_FIELD, 0,                \
 923             _field1, _value1, _field2, _value2, _field3, _value3,       \
 924             _field4, _value4, _field5, _value5, _field6, _value6,       \
 925             _field7, _value7, _field8, _value8)
 926
 927 #define EFX_POPULATE_QWORD_7(_qword,                                    \
 928             _field1, _value1, _field2, _value2, _field3, _value3,       \
 929             _field4, _value4, _field5, _value5, _field6, _value6,       \
 930             _field7, _value7)                                           \
 931         EFX_POPULATE_QWORD_8(_qword, EFX_DUMMY_FIELD, 0,                \
 932             _field1, _value1, _field2, _value2, _field3, _value3,       \
 933             _field4, _value4, _field5, _value5, _field6, _value6,       \
 934             _field7, _value7)
 935
 936 #define EFX_POPULATE_QWORD_6(_qword,                                    \
 937             _field1, _value1, _field2, _value2, _field3, _value3,       \
 938             _field4, _value4, _field5, _value5, _field6, _value6)       \
 939         EFX_POPULATE_QWORD_7(_qword, EFX_DUMMY_FIELD, 0,                \
 940             _field1, _value1, _field2, _value2, _field3, _value3,       \
 941             _field4, _value4, _field5, _value5, _field6, _value6)
 942
 943 #define EFX_POPULATE_QWORD_5(_qword,                                    \
 944             _field1, _value1, _field2, _value2, _field3, _value3,       \
 945             _field4, _value4, _field5, _value5)                         \
 946         EFX_POPULATE_QWORD_6(_qword, EFX_DUMMY_FIELD, 0,                \
 947             _field1, _value1, _field2, _value2, _field3, _value3,       \
 948             _field4, _value4, _field5, _value5)
 949
 950 #define EFX_POPULATE_QWORD_4(_qword,                                    \
 951             _field1, _value1, _field2, _value2, _field3, _value3,       \
 952             _field4, _value4)                                           \
 953         EFX_POPULATE_QWORD_5(_qword, EFX_DUMMY_FIELD, 0,                \
 954             _field1, _value1, _field2, _value2, _field3, _value3,       \
 955             _field4, _value4)
 956
 957 #define EFX_POPULATE_QWORD_3(_qword,                                    \
 958             _field1, _value1, _field2, _value2, _field3, _value3)       \
 959         EFX_POPULATE_QWORD_4(_qword, EFX_DUMMY_FIELD, 0,                \
 960             _field1, _value1, _field2, _value2, _field3, _value3)
 961
 962 #define EFX_POPULATE_QWORD_2(_qword,                                    \
 963             _field1, _value1, _field2, _value2)                         \
 964         EFX_POPULATE_QWORD_3(_qword, EFX_DUMMY_FIELD, 0,                \
 965             _field1, _value1, _field2, _value2)
 966
 967 #define EFX_POPULATE_QWORD_1(_qword,                                    \
 968             _field1, _value1)                                           \
 969         EFX_POPULATE_QWORD_2(_qword, EFX_DUMMY_FIELD, 0,                \
 970             _field1, _value1)
 971
 972 #define EFX_ZERO_QWORD(_qword)                                          \
 973         EFX_POPULATE_QWORD_1(_qword, EFX_DUMMY_FIELD, 0)
 974
 975 #define EFX_SET_QWORD(_qword)                                           \
 976         EFX_POPULATE_QWORD_2(_qword,                                    \
 977             EFX_DWORD_0, 0xffffffff, EFX_DWORD_1, 0xffffffff)
 978
 979 /* Populate a dword field with various numbers of arguments */
 980 #define EFX_POPULATE_DWORD_10 EFX_POPULATE_DWORD
 981
 982 #define EFX_POPULATE_DWORD_9(_dword,                                    \
 983             _field1, _value1, _field2, _value2, _field3, _value3,       \
 984             _field4, _value4, _field5, _value5, _field6, _value6,       \
 985             _field7, _value7, _field8, _value8, _field9, _value9)       \
 986         EFX_POPULATE_DWORD_10(_dword, EFX_DUMMY_FIELD, 0,               \
 987             _field1, _value1, _field2, _value2, _field3, _value3,       \
 988             _field4, _value4, _field5, _value5, _field6, _value6,       \
 989             _field7, _value7, _field8, _value8, _field9, _value9)
 990
 991 #define EFX_POPULATE_DWORD_8(_dword,                                    \
 992             _field1, _value1, _field2, _value2, _field3, _value3,       \
 993             _field4, _value4, _field5, _value5, _field6, _value6,       \
 994             _field7, _value7, _field8, _value8)                         \
 995         EFX_POPULATE_DWORD_9(_dword, EFX_DUMMY_FIELD, 0,                \
 996             _field1, _value1, _field2, _value2, _field3, _value3,       \
 997             _field4, _value4, _field5, _value5, _field6, _value6,       \
 998             _field7, _value7, _field8, _value8)
 999
1000 #define EFX_POPULATE_DWORD_7(_dword,                                    \
1001             _field1, _value1, _field2, _value2, _field3, _value3,       \
1002             _field4, _value4, _field5, _value5, _field6, _value6,       \
1003             _field7, _value7)                                           \
1004         EFX_POPULATE_DWORD_8(_dword, EFX_DUMMY_FIELD, 0,                \
1005             _field1, _value1, _field2, _value2, _field3, _value3,       \
1006             _field4, _value4, _field5, _value5, _field6, _value6,       \
1007             _field7, _value7)
1008
1009 #define EFX_POPULATE_DWORD_6(_dword,                                    \
1010             _field1, _value1, _field2, _value2, _field3, _value3,       \
1011             _field4, _value4, _field5, _value5, _field6, _value6)       \
1012         EFX_POPULATE_DWORD_7(_dword, EFX_DUMMY_FIELD, 0,                \
1013             _field1, _value1, _field2, _value2, _field3, _value3,       \
1014             _field4, _value4, _field5, _value5, _field6, _value6)
1015
1016 #define EFX_POPULATE_DWORD_5(_dword,                                    \
1017             _field1, _value1, _field2, _value2, _field3, _value3,       \
1018             _field4, _value4, _field5, _value5)                         \
1019         EFX_POPULATE_DWORD_6(_dword, EFX_DUMMY_FIELD, 0,                \
1020             _field1, _value1, _field2, _value2, _field3, _value3,       \
1021             _field4, _value4, _field5, _value5)
1022
1023 #define EFX_POPULATE_DWORD_4(_dword,                                    \
1024             _field1, _value1, _field2, _value2, _field3, _value3,       \
1025             _field4, _value4)                                           \
1026         EFX_POPULATE_DWORD_5(_dword, EFX_DUMMY_FIELD, 0,                \
1027             _field1, _value1, _field2, _value2, _field3, _value3,       \
1028             _field4, _value4)
1029
1030 #define EFX_POPULATE_DWORD_3(_dword,                                    \
1031             _field1, _value1, _field2, _value2, _field3, _value3)       \
1032         EFX_POPULATE_DWORD_4(_dword, EFX_DUMMY_FIELD, 0,                \
1033             _field1, _value1, _field2, _value2, _field3, _value3)
1034
1035 #define EFX_POPULATE_DWORD_2(_dword,                                    \
1036             _field1, _value1, _field2, _value2)                         \
1037         EFX_POPULATE_DWORD_3(_dword, EFX_DUMMY_FIELD, 0,                \
1038             _field1, _value1, _field2, _value2)
1039
1040 #define EFX_POPULATE_DWORD_1(_dword,                                    \
1041             _field1, _value1)                                           \
1042         EFX_POPULATE_DWORD_2(_dword, EFX_DUMMY_FIELD, 0,                \
1043             _field1, _value1)
1044
1045 #define EFX_ZERO_DWORD(_dword)                                          \
1046         EFX_POPULATE_DWORD_1(_dword, EFX_DUMMY_FIELD, 0)
1047
1048 #define EFX_SET_DWORD(_dword)                                           \
1049         EFX_POPULATE_DWORD_1(_dword,                                    \
1050             EFX_DWORD_0, 0xffffffff)
1051
1052 /* Populate a word field with various numbers of arguments */
1053 #define EFX_POPULATE_WORD_10 EFX_POPULATE_WORD
1054
1055 #define EFX_POPULATE_WORD_9(_word,                                      \
1056             _field1, _value1, _field2, _value2, _field3, _value3,       \
1057             _field4, _value4, _field5, _value5, _field6, _value6,       \
1058             _field7, _value7, _field8, _value8, _field9, _value9)       \
1059         EFX_POPULATE_WORD_10(_word, EFX_DUMMY_FIELD, 0,                 \
1060             _field1, _value1, _field2, _value2, _field3, _value3,       \
1061             _field4, _value4, _field5, _value5, _field6, _value6,       \
1062             _field7, _value7, _field8, _value8, _field9, _value9)
1063
1064 #define EFX_POPULATE_WORD_8(_word,                                      \
1065             _field1, _value1, _field2, _value2, _field3, _value3,       \
1066             _field4, _value4, _field5, _value5, _field6, _value6,       \
1067             _field7, _value7, _field8, _value8)                         \
1068         EFX_POPULATE_WORD_9(_word, EFX_DUMMY_FIELD, 0,                  \
1069             _field1, _value1, _field2, _value2, _field3, _value3,       \
1070             _field4, _value4, _field5, _value5, _field6, _value6,       \
1071             _field7, _value7, _field8, _value8)
1072
1073 #define EFX_POPULATE_WORD_7(_word,                                      \
1074             _field1, _value1, _field2, _value2, _field3, _value3,       \
1075             _field4, _value4, _field5, _value5, _field6, _value6,       \
1076             _field7, _value7)                                           \
1077         EFX_POPULATE_WORD_8(_word, EFX_DUMMY_FIELD, 0,                  \
1078             _field1, _value1, _field2, _value2, _field3, _value3,       \
1079             _field4, _value4, _field5, _value5, _field6, _value6,       \
1080             _field7, _value7)
1081
1082 #define EFX_POPULATE_WORD_6(_word,                                      \
1083             _field1, _value1, _field2, _value2, _field3, _value3,       \
1084             _field4, _value4, _field5, _value5, _field6, _value6)       \
1085         EFX_POPULATE_WORD_7(_word, EFX_DUMMY_FIELD, 0,                  \
1086             _field1, _value1, _field2, _value2, _field3, _value3,       \
1087             _field4, _value4, _field5, _value5, _field6, _value6)
1088
1089 #define EFX_POPULATE_WORD_5(_word,                                      \
1090             _field1, _value1, _field2, _value2, _field3, _value3,       \
1091             _field4, _value4, _field5, _value5)                         \
1092         EFX_POPULATE_WORD_6(_word, EFX_DUMMY_FIELD, 0,                  \
1093             _field1, _value1, _field2, _value2, _field3, _value3,       \
1094             _field4, _value4, _field5, _value5)
1095
1096 #define EFX_POPULATE_WORD_4(_word,                                      \
1097             _field1, _value1, _field2, _value2, _field3, _value3,       \
1098             _field4, _value4)                                           \
1099         EFX_POPULATE_WORD_5(_word, EFX_DUMMY_FIELD, 0,                  \
1100             _field1, _value1, _field2, _value2, _field3, _value3,       \
1101             _field4, _value4)
1102
1103 #define EFX_POPULATE_WORD_3(_word,                                      \
1104             _field1, _value1, _field2, _value2, _field3, _value3)       \
1105         EFX_POPULATE_WORD_4(_word, EFX_DUMMY_FIELD, 0,                  \
1106             _field1, _value1, _field2, _value2, _field3, _value3)
1107
1108 #define EFX_POPULATE_WORD_2(_word,                                      \
1109             _field1, _value1, _field2, _value2)                         \
1110         EFX_POPULATE_WORD_3(_word, EFX_DUMMY_FIELD, 0,                  \
1111             _field1, _value1, _field2, _value2)
1112
1113 #define EFX_POPULATE_WORD_1(_word,                                      \
1114             _field1, _value1)                                           \
1115         EFX_POPULATE_WORD_2(_word, EFX_DUMMY_FIELD, 0,                  \
1116             _field1, _value1)
1117
1118 #define EFX_ZERO_WORD(_word)                                            \
1119         EFX_POPULATE_WORD_1(_word, EFX_DUMMY_FIELD, 0)
1120
1121 #define EFX_SET_WORD(_word)                                             \
1122         EFX_POPULATE_WORD_1(_word,                                      \
1123             EFX_WORD_0, 0xffff)
1124
1125 /* Populate a byte field with various numbers of arguments */
1126 #define EFX_POPULATE_BYTE_10 EFX_POPULATE_BYTE
1127
1128 #define EFX_POPULATE_BYTE_9(_byte,                                      \
1129             _field1, _value1, _field2, _value2, _field3, _value3,       \
1130             _field4, _value4, _field5, _value5, _field6, _value6,       \
1131             _field7, _value7, _field8, _value8, _field9, _value9)       \
1132         EFX_POPULATE_BYTE_10(_byte, EFX_DUMMY_FIELD, 0,                 \
1133             _field1, _value1, _field2, _value2, _field3, _value3,       \
1134             _field4, _value4, _field5, _value5, _field6, _value6,       \
1135             _field7, _value7, _field8, _value8, _field9, _value9)
1136
1137 #define EFX_POPULATE_BYTE_8(_byte,                                      \
1138             _field1, _value1, _field2, _value2, _field3, _value3,       \
1139             _field4, _value4, _field5, _value5, _field6, _value6,       \
1140             _field7, _value7, _field8, _value8)                         \
1141         EFX_POPULATE_BYTE_9(_byte, EFX_DUMMY_FIELD, 0,                  \
1142             _field1, _value1, _field2, _value2, _field3, _value3,       \
1143             _field4, _value4, _field5, _value5, _field6, _value6,       \
1144             _field7, _value7, _field8, _value8)
1145
1146 #define EFX_POPULATE_BYTE_7(_byte,                                      \
1147             _field1, _value1, _field2, _value2, _field3, _value3,       \
1148             _field4, _value4, _field5, _value5, _field6, _value6,       \
1149             _field7, _value7)                                           \
1150         EFX_POPULATE_BYTE_8(_byte, EFX_DUMMY_FIELD, 0,                  \
1151             _field1, _value1, _field2, _value2, _field3, _value3,       \
1152             _field4, _value4, _field5, _value5, _field6, _value6,       \
1153             _field7, _value7)
1154
1155 #define EFX_POPULATE_BYTE_6(_byte,                                      \
1156             _field1, _value1, _field2, _value2, _field3, _value3,       \
1157             _field4, _value4, _field5, _value5, _field6, _value6)       \
1158         EFX_POPULATE_BYTE_7(_byte, EFX_DUMMY_FIELD, 0,                  \
1159             _field1, _value1, _field2, _value2, _field3, _value3,       \
1160             _field4, _value4, _field5, _value5, _field6, _value6)
1161
1162 #define EFX_POPULATE_BYTE_5(_byte,                                      \
1163             _field1, _value1, _field2, _value2, _field3, _value3,       \
1164             _field4, _value4, _field5, _value5)                         \
1165         EFX_POPULATE_BYTE_6(_byte, EFX_DUMMY_FIELD, 0,                  \
1166             _field1, _value1, _field2, _value2, _field3, _value3,       \
1167             _field4, _value4, _field5, _value5)
1168
1169 #define EFX_POPULATE_BYTE_4(_byte,                                      \
1170             _field1, _value1, _field2, _value2, _field3, _value3,       \
1171             _field4, _value4)                                           \
1172         EFX_POPULATE_BYTE_5(_byte, EFX_DUMMY_FIELD, 0,                  \
1173             _field1, _value1, _field2, _value2, _field3, _value3,       \
1174             _field4, _value4)
1175
1176 #define EFX_POPULATE_BYTE_3(_byte,                                      \
1177             _field1, _value1, _field2, _value2, _field3, _value3)       \
1178         EFX_POPULATE_BYTE_4(_byte, EFX_DUMMY_FIELD, 0,                  \
1179             _field1, _value1, _field2, _value2, _field3, _value3)
1180
1181 #define EFX_POPULATE_BYTE_2(_byte,                                      \
1182             _field1, _value1, _field2, _value2)                         \
1183         EFX_POPULATE_BYTE_3(_byte, EFX_DUMMY_FIELD, 0,                  \
1184             _field1, _value1, _field2, _value2)
1185
1186 #define EFX_POPULATE_BYTE_1(_byte,                                      \
1187             _field1, _value1)                                           \
1188         EFX_POPULATE_BYTE_2(_byte, EFX_DUMMY_FIELD, 0,                  \
1189             _field1, _value1)
1190
1191 #define EFX_ZERO_BYTE(_byte)                                            \
1192         EFX_POPULATE_BYTE_1(_byte, EFX_DUMMY_FIELD, 0)
1193
1194 #define EFX_SET_BYTE(_byte)                                             \
1195         EFX_POPULATE_BYTE_1(_byte,                                      \
1196             EFX_BYTE_0, 0xff)
1197
1198 /*
1199  * Modify a named field within an already-populated structure.  Used
1200  * for read-modify-write operations.
1201  */
1202
1203 #define EFX_INSERT_FIELD64(_min, _max, _field, _value)                  \
1204         __CPU_TO_LE_64(EFX_INSERT_FIELD_NATIVE64(_min, _max, _field, _value))
1205
1206 #define EFX_INSERT_FIELD32(_min, _max, _field, _value)                  \
1207         __CPU_TO_LE_32(EFX_INSERT_FIELD_NATIVE32(_min, _max, _field, _value))
1208
1209 #define EFX_INSERT_FIELD16(_min, _max, _field, _value)                  \
1210         __CPU_TO_LE_16(EFX_INSERT_FIELD_NATIVE16(_min, _max, _field, _value))
1211
1212 #define EFX_INSERT_FIELD8(_min, _max, _field, _value)                   \
1213         __NATIVE_8(EFX_INSERT_FIELD_NATIVE8(_min, _max, _field, _value))
1214
1215 #define EFX_INPLACE_MASK64(_min, _max, _field)                          \
1216         EFX_INSERT_FIELD64(_min, _max, _field, EFX_MASK64(_field))
1217
1218 #define EFX_INPLACE_MASK32(_min, _max, _field)                          \
1219         EFX_INSERT_FIELD32(_min, _max, _field, EFX_MASK32(_field))
1220
1221 #define EFX_INPLACE_MASK16(_min, _max, _field)                          \
1222         EFX_INSERT_FIELD16(_min, _max, _field, EFX_MASK16(_field))
1223
1224 #define EFX_INPLACE_MASK8(_min, _max, _field)                           \
1225         EFX_INSERT_FIELD8(_min, _max, _field, EFX_MASK8(_field))
1226
1227 #define EFX_SET_OWORD_FIELD64(_oword, _field, _value)                   \
1228         do {                                                            \
1229                 _NOTE(CONSTANTCONDITION)                                \
1230                 (_oword).eo_u64[0] = (((_oword).eo_u64[0] &             \
1231                     ~EFX_INPLACE_MASK64(0, 63, _field)) |               \
1232                     EFX_INSERT_FIELD64(0, 63, _field, _value));         \
1233                 _NOTE(CONSTANTCONDITION)                                \
1234                 (_oword).eo_u64[1] = (((_oword).eo_u64[1] &             \
1235                     ~EFX_INPLACE_MASK64(64, 127, _field)) |             \
1236                     EFX_INSERT_FIELD64(64, 127, _field, _value));       \
1237         _NOTE(CONSTANTCONDITION)                                        \
1238         } while (B_FALSE)
1239
1240 #define EFX_SET_OWORD_FIELD32(_oword, _field, _value)                   \
1241         do {                                                            \
1242                 _NOTE(CONSTANTCONDITION)                                \
1243                 (_oword).eo_u32[0] = (((_oword).eo_u32[0] &             \
1244                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1245                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1246                 _NOTE(CONSTANTCONDITION)                                \
1247                 (_oword).eo_u32[1] = (((_oword).eo_u32[1] &             \
1248                     ~EFX_INPLACE_MASK32(32, 63, _field)) |              \
1249                     EFX_INSERT_FIELD32(32, 63, _field, _value));        \
1250                 _NOTE(CONSTANTCONDITION)                                \
1251                 (_oword).eo_u32[2] = (((_oword).eo_u32[2] &             \
1252                     ~EFX_INPLACE_MASK32(64, 95, _field)) |              \
1253                     EFX_INSERT_FIELD32(64, 95, _field, _value));        \
1254                 _NOTE(CONSTANTCONDITION)                                \
1255                 (_oword).eo_u32[3] = (((_oword).eo_u32[3] &             \
1256                     ~EFX_INPLACE_MASK32(96, 127, _field)) |             \
1257                     EFX_INSERT_FIELD32(96, 127, _field, _value));       \
1258         _NOTE(CONSTANTCONDITION)                                        \
1259         } while (B_FALSE)
1260
1261 #define EFX_SET_QWORD_FIELD64(_qword, _field, _value)                   \
1262         do {                                                            \
1263                 _NOTE(CONSTANTCONDITION)                                \
1264                 (_qword).eq_u64[0] = (((_qword).eq_u64[0] &             \
1265                     ~EFX_INPLACE_MASK64(0, 63, _field)) |               \
1266                     EFX_INSERT_FIELD64(0, 63, _field, _value));         \
1267         _NOTE(CONSTANTCONDITION)                                        \
1268         } while (B_FALSE)
1269
1270 #define EFX_SET_QWORD_FIELD32(_qword, _field, _value)                   \
1271         do {                                                            \
1272                 _NOTE(CONSTANTCONDITION)                                \
1273                 (_qword).eq_u32[0] = (((_qword).eq_u32[0] &             \
1274                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1275                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1276                 _NOTE(CONSTANTCONDITION)                                \
1277                 (_qword).eq_u32[1] = (((_qword).eq_u32[1] &             \
1278                     ~EFX_INPLACE_MASK32(32, 63, _field)) |              \
1279                     EFX_INSERT_FIELD32(32, 63, _field, _value));        \
1280         _NOTE(CONSTANTCONDITION)                                        \
1281         } while (B_FALSE)
1282
1283 #define EFX_SET_DWORD_FIELD(_dword, _field, _value)                     \
1284         do {                                                            \
1285                 _NOTE(CONSTANTCONDITION)                                \
1286                 (_dword).ed_u32[0] = (((_dword).ed_u32[0] &             \
1287                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1288                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1289         _NOTE(CONSTANTCONDITION)                                        \
1290         } while (B_FALSE)
1291
1292 #define EFX_SET_WORD_FIELD(_word, _field, _value)                       \
1293         do {                                                            \
1294                 _NOTE(CONSTANTCONDITION)                                \
1295                 (_word).ew_u16[0] = (((_word).ew_u16[0] &               \
1296                     ~EFX_INPLACE_MASK16(0, 15, _field)) |               \
1297                     EFX_INSERT_FIELD16(0, 15, _field, _value));         \
1298         _NOTE(CONSTANTCONDITION)                                        \
1299         } while (B_FALSE)
1300
1301 #define EFX_SET_BYTE_FIELD(_byte, _field, _value)                       \
1302         do {                                                            \
1303                 _NOTE(CONSTANTCONDITION)                                \
1304                 (_byte).eb_u8[0] = (((_byte).eb_u8[0] &                 \
1305                     ~EFX_INPLACE_MASK8(0, 7, _field)) |                 \
1306                     EFX_INSERT_FIELD8(0, 7, _field, _value));           \
1307         _NOTE(CONSTANTCONDITION)                                        \
1308         } while (B_FALSE)
1309
1310 /*
1311  * Set or clear a numbered bit within an octword.
1312  */
1313
1314 #define EFX_SHIFT64(_bit, _base)                                        \
1315         (((_bit) >= (_base) && (_bit) < (_base) + 64) ?                 \
1316                 ((uint64_t)1 << ((_bit) - (_base))) :                   \
1317                 0U)
1318
1319 #define EFX_SHIFT32(_bit, _base)                                        \
1320         (((_bit) >= (_base) && (_bit) < (_base) + 32) ?                 \
1321                 ((uint32_t)1 << ((_bit) - (_base))) :                   \
1322                 0U)
1323
1324 #define EFX_SHIFT16(_bit, _base)                                        \
1325         (((_bit) >= (_base) && (_bit) < (_base) + 16) ?                 \
1326                 (uint16_t)(1 << ((_bit) - (_base))) :                   \
1327                 0U)
1328
1329 #define EFX_SHIFT8(_bit, _base)                                         \
1330         (((_bit) >= (_base) && (_bit) < (_base) + 8) ?                  \
1331                 (uint8_t)(1 << ((_bit) - (_base))) :                    \
1332                 0U)
1333
1334 #define EFX_SET_OWORD_BIT64(_oword, _bit)                               \
1335         do {                                                            \
1336                 _NOTE(CONSTANTCONDITION)                                \
1337                 (_oword).eo_u64[0] |=                                   \
1338                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)));     \
1339                 (_oword).eo_u64[1] |=                                   \
1340                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(64)));    \
1341         _NOTE(CONSTANTCONDITION)                                        \
1342         } while (B_FALSE)
1343
1344 #define EFX_SET_OWORD_BIT32(_oword, _bit)                               \
1345         do {                                                            \
1346                 _NOTE(CONSTANTCONDITION)                                \
1347                 (_oword).eo_u32[0] |=                                   \
1348                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1349                 (_oword).eo_u32[1] |=                                   \
1350                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)));    \
1351                 (_oword).eo_u32[2] |=                                   \
1352                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(64)));    \
1353                 (_oword).eo_u32[3] |=                                   \
1354                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(96)));    \
1355         _NOTE(CONSTANTCONDITION)                                        \
1356         } while (B_FALSE)
1357
1358 #define EFX_CLEAR_OWORD_BIT64(_oword, _bit)                             \
1359         do {                                                            \
1360                 _NOTE(CONSTANTCONDITION)                                \
1361                 (_oword).eo_u64[0] &=                                   \
1362                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(0)));    \
1363                 (_oword).eo_u64[1] &=                                   \
1364                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(64)));   \
1365         _NOTE(CONSTANTCONDITION)                                        \
1366         } while (B_FALSE)
1367
1368 #define EFX_CLEAR_OWORD_BIT32(_oword, _bit)                             \
1369         do {                                                            \
1370                 _NOTE(CONSTANTCONDITION)                                \
1371                 (_oword).eo_u32[0] &=                                   \
1372                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1373                 (_oword).eo_u32[1] &=                                   \
1374                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(32)));   \
1375                 (_oword).eo_u32[2] &=                                   \
1376                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(64)));   \
1377                 (_oword).eo_u32[3] &=                                   \
1378                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(96)));   \
1379         _NOTE(CONSTANTCONDITION)                                        \
1380         } while (B_FALSE)
1381
1382 #define EFX_TEST_OWORD_BIT64(_oword, _bit)                              \
1383         (((_oword).eo_u64[0] &                                          \
1384                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)))) ||  \
1385         ((_oword).eo_u64[1] &                                           \
1386                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(64)))))
1387
1388 #define EFX_TEST_OWORD_BIT32(_oword, _bit)                              \
1389         (((_oword).eo_u32[0] &                                          \
1390                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) ||  \
1391         ((_oword).eo_u32[1] &                                           \
1392                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)))) || \
1393         ((_oword).eo_u32[2] &                                           \
1394                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(64)))) || \
1395         ((_oword).eo_u32[3] &                                           \
1396                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(96)))))
1397
1398
1399 #define EFX_SET_QWORD_BIT64(_qword, _bit)                               \
1400         do {                                                            \
1401                 _NOTE(CONSTANTCONDITION)                                \
1402                 (_qword).eq_u64[0] |=                                   \
1403                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)));     \
1404         _NOTE(CONSTANTCONDITION)                                        \
1405         } while (B_FALSE)
1406
1407 #define EFX_SET_QWORD_BIT32(_qword, _bit)                               \
1408         do {                                                            \
1409                 _NOTE(CONSTANTCONDITION)                                \
1410                 (_qword).eq_u32[0] |=                                   \
1411                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1412                 (_qword).eq_u32[1] |=                                   \
1413                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)));    \
1414         _NOTE(CONSTANTCONDITION)                                        \
1415         } while (B_FALSE)
1416
1417 #define EFX_CLEAR_QWORD_BIT64(_qword, _bit)                             \
1418         do {                                                            \
1419                 _NOTE(CONSTANTCONDITION)                                \
1420                 (_qword).eq_u64[0] &=                                   \
1421                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(0)));    \
1422         _NOTE(CONSTANTCONDITION)                                        \
1423         } while (B_FALSE)
1424
1425 #define EFX_CLEAR_QWORD_BIT32(_qword, _bit)                             \
1426         do {                                                            \
1427                 _NOTE(CONSTANTCONDITION)                                \
1428                 (_qword).eq_u32[0] &=                                   \
1429                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1430                 (_qword).eq_u32[1] &=                                   \
1431                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(32)));   \
1432         _NOTE(CONSTANTCONDITION)                                        \
1433         } while (B_FALSE)
1434
1435 #define EFX_TEST_QWORD_BIT64(_qword, _bit)                              \
1436         (((_qword).eq_u64[0] &                                          \
1437                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)))) != 0)
1438
1439 #define EFX_TEST_QWORD_BIT32(_qword, _bit)                              \
1440         (((_qword).eq_u32[0] &                                          \
1441                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) ||  \
1442         ((_qword).eq_u32[1] &                                           \
1443                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)))))
1444
1445
1446 #define EFX_SET_DWORD_BIT(_dword, _bit)                                 \
1447         do {                                                            \
1448                 (_dword).ed_u32[0] |=                                   \
1449                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1450         _NOTE(CONSTANTCONDITION)                                        \
1451         } while (B_FALSE)
1452
1453 #define EFX_CLEAR_DWORD_BIT(_dword, _bit)                               \
1454         do {                                                            \
1455                 (_dword).ed_u32[0] &=                                   \
1456                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1457         _NOTE(CONSTANTCONDITION)                                        \
1458         } while (B_FALSE)
1459
1460 #define EFX_TEST_DWORD_BIT(_dword, _bit)                                \
1461         (((_dword).ed_u32[0] &                                          \
1462                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) != 0)
1463
1464
1465 #define EFX_SET_WORD_BIT(_word, _bit)                                   \
1466         do {                                                            \
1467                 (_word).ew_u16[0] |=                                    \
1468                     __CPU_TO_LE_16(EFX_SHIFT16(_bit, FIX_LINT(0)));     \
1469         _NOTE(CONSTANTCONDITION)                                        \
1470         } while (B_FALSE)
1471
1472 #define EFX_CLEAR_WORD_BIT(_word, _bit)                                 \
1473         do {                                                            \
1474                 (_word).ew_u32[0] &=                                    \
1475                     __CPU_TO_LE_16(~EFX_SHIFT16(_bit, FIX_LINT(0)));    \
1476         _NOTE(CONSTANTCONDITION)                                        \
1477         } while (B_FALSE)
1478
1479 #define EFX_TEST_WORD_BIT(_word, _bit)                                  \
1480         (((_word).ew_u16[0] &                                           \
1481                     __CPU_TO_LE_16(EFX_SHIFT16(_bit, FIX_LINT(0)))) != 0)
1482
1483
1484 #define EFX_SET_BYTE_BIT(_byte, _bit)                                   \
1485         do {                                                            \
1486                 (_byte).eb_u8[0] |=                                     \
1487                     __NATIVE_8(EFX_SHIFT8(_bit, FIX_LINT(0)));          \
1488         _NOTE(CONSTANTCONDITION)                                        \
1489         } while (B_FALSE)
1490
1491 #define EFX_CLEAR_BYTE_BIT(_byte, _bit)                                 \
1492         do {                                                            \
1493                 (_byte).eb_u8[0] &=                                     \
1494                     __NATIVE_8(~EFX_SHIFT8(_bit, FIX_LINT(0)));         \
1495         _NOTE(CONSTANTCONDITION)                                        \
1496         } while (B_FALSE)
1497
1498 #define EFX_TEST_BYTE_BIT(_byte, _bit)                                  \
1499         (((_byte).eb_u8[0] &                                            \
1500                     __NATIVE_8(EFX_SHIFT8(_bit, FIX_LINT(0)))) != 0)
1501
1502
1503 #define EFX_OR_OWORD64(_oword1, _oword2)                                \
1504         do {                                                            \
1505                 (_oword1).eo_u64[0] |= (_oword2).eo_u64[0];             \
1506                 (_oword1).eo_u64[1] |= (_oword2).eo_u64[1];             \
1507         _NOTE(CONSTANTCONDITION)                                        \
1508         } while (B_FALSE)
1509
1510 #define EFX_OR_OWORD32(_oword1, _oword2)                                \
1511         do {                                                            \
1512                 (_oword1).eo_u32[0] |= (_oword2).eo_u32[0];             \
1513                 (_oword1).eo_u32[1] |= (_oword2).eo_u32[1];             \
1514                 (_oword1).eo_u32[2] |= (_oword2).eo_u32[2];             \
1515                 (_oword1).eo_u32[3] |= (_oword2).eo_u32[3];             \
1516         _NOTE(CONSTANTCONDITION)                                        \
1517         } while (B_FALSE)
1518
1519 #define EFX_AND_OWORD64(_oword1, _oword2)                               \
1520         do {                                                            \
1521                 (_oword1).eo_u64[0] &= (_oword2).eo_u64[0];             \
1522                 (_oword1).eo_u64[1] &= (_oword2).eo_u64[1];             \
1523         _NOTE(CONSTANTCONDITION)                                        \
1524         } while (B_FALSE)
1525
1526 #define EFX_AND_OWORD32(_oword1, _oword2)                               \
1527         do {                                                            \
1528                 (_oword1).eo_u32[0] &= (_oword2).eo_u32[0];             \
1529                 (_oword1).eo_u32[1] &= (_oword2).eo_u32[1];             \
1530                 (_oword1).eo_u32[2] &= (_oword2).eo_u32[2];             \
1531                 (_oword1).eo_u32[3] &= (_oword2).eo_u32[3];             \
1532         _NOTE(CONSTANTCONDITION)                                        \
1533         } while (B_FALSE)
1534
1535 #define EFX_OR_QWORD64(_qword1, _qword2)                                \
1536         do {                                                            \
1537                 (_qword1).eq_u64[0] |= (_qword2).eq_u64[0];             \
1538         _NOTE(CONSTANTCONDITION)                                        \
1539         } while (B_FALSE)
1540
1541 #define EFX_OR_QWORD32(_qword1, _qword2)                                \
1542         do {                                                            \
1543                 (_qword1).eq_u32[0] |= (_qword2).eq_u32[0];             \
1544                 (_qword1).eq_u32[1] |= (_qword2).eq_u32[1];             \
1545         _NOTE(CONSTANTCONDITION)                                        \
1546         } while (B_FALSE)
1547
1548 #define EFX_AND_QWORD64(_qword1, _qword2)                               \
1549         do {                                                            \
1550                 (_qword1).eq_u64[0] &= (_qword2).eq_u64[0];             \
1551         _NOTE(CONSTANTCONDITION)                                        \
1552         } while (B_FALSE)
1553
1554 #define EFX_AND_QWORD32(_qword1, _qword2)                               \
1555         do {                                                            \
1556                 (_qword1).eq_u32[0] &= (_qword2).eq_u32[0];             \
1557                 (_qword1).eq_u32[1] &= (_qword2).eq_u32[1];             \
1558         _NOTE(CONSTANTCONDITION)                                        \
1559         } while (B_FALSE)
1560
1561 #define EFX_OR_DWORD(_dword1, _dword2)                                  \
1562         do {                                                            \
1563                 (_dword1).ed_u32[0] |= (_dword2).ed_u32[0];             \
1564         _NOTE(CONSTANTCONDITION)                                        \
1565         } while (B_FALSE)
1566
1567 #define EFX_AND_DWORD(_dword1, _dword2)                                 \
1568         do {                                                            \
1569                 (_dword1).ed_u32[0] &= (_dword2).ed_u32[0];             \
1570         _NOTE(CONSTANTCONDITION)                                        \
1571         } while (B_FALSE)
1572
1573 #define EFX_OR_WORD(_word1, _word2)                                     \
1574         do {                                                            \
1575                 (_word1).ew_u16[0] |= (_word2).ew_u16[0];               \
1576         _NOTE(CONSTANTCONDITION)                                        \
1577         } while (B_FALSE)
1578
1579 #define EFX_AND_WORD(_word1, _word2)                                    \
1580         do {                                                            \
1581                 (_word1).ew_u16[0] &= (_word2).ew_u16[0];               \
1582         _NOTE(CONSTANTCONDITION)                                        \
1583         } while (B_FALSE)
1584
1585 #define EFX_OR_BYTE(_byte1, _byte2)                                     \
1586         do {                                                            \
1587                 (_byte1).eb_u8[0] &= (_byte2).eb_u8[0];                 \
1588         _NOTE(CONSTANTCONDITION)                                        \
1589         } while (B_FALSE)
1590
1591 #define EFX_AND_BYTE(_byte1, _byte2)                                    \
1592         do {                                                            \
1593                 (_byte1).eb_u8[0] &= (_byte2).eb_u8[0];                 \
1594         _NOTE(CONSTANTCONDITION)                                        \
1595         } while (B_FALSE)
1596
1597 #if EFSYS_USE_UINT64
1598 #define EFX_OWORD_FIELD         EFX_OWORD_FIELD64
1599 #define EFX_QWORD_FIELD         EFX_QWORD_FIELD64
1600 #define EFX_OWORD_IS_EQUAL      EFX_OWORD_IS_EQUAL64
1601 #define EFX_QWORD_IS_EQUAL      EFX_QWORD_IS_EQUAL64
1602 #define EFX_OWORD_IS_ZERO       EFX_OWORD_IS_ZERO64
1603 #define EFX_QWORD_IS_ZERO       EFX_QWORD_IS_ZERO64
1604 #define EFX_OWORD_IS_SET        EFX_OWORD_IS_SET64
1605 #define EFX_QWORD_IS_SET        EFX_QWORD_IS_SET64
1606 #define EFX_POPULATE_OWORD      EFX_POPULATE_OWORD64
1607 #define EFX_POPULATE_QWORD      EFX_POPULATE_QWORD64
1608 #define EFX_SET_OWORD_FIELD     EFX_SET_OWORD_FIELD64
1609 #define EFX_SET_QWORD_FIELD     EFX_SET_QWORD_FIELD64
1610 #define EFX_SET_OWORD_BIT       EFX_SET_OWORD_BIT64
1611 #define EFX_CLEAR_OWORD_BIT     EFX_CLEAR_OWORD_BIT64
1612 #define EFX_TEST_OWORD_BIT      EFX_TEST_OWORD_BIT64
1613 #define EFX_SET_QWORD_BIT       EFX_SET_QWORD_BIT64
1614 #define EFX_CLEAR_QWORD_BIT     EFX_CLEAR_QWORD_BIT64
1615 #define EFX_TEST_QWORD_BIT      EFX_TEST_QWORD_BIT64
1616 #define EFX_OR_OWORD            EFX_OR_OWORD64
1617 #define EFX_AND_OWORD           EFX_AND_OWORD64
1618 #define EFX_OR_QWORD            EFX_OR_QWORD64
1619 #define EFX_AND_QWORD           EFX_AND_QWORD64
1620 #else
1621 #define EFX_OWORD_FIELD         EFX_OWORD_FIELD32
1622 #define EFX_QWORD_FIELD         EFX_QWORD_FIELD32
1623 #define EFX_OWORD_IS_EQUAL      EFX_OWORD_IS_EQUAL32
1624 #define EFX_QWORD_IS_EQUAL      EFX_QWORD_IS_EQUAL32
1625 #define EFX_OWORD_IS_ZERO       EFX_OWORD_IS_ZERO32
1626 #define EFX_QWORD_IS_ZERO       EFX_QWORD_IS_ZERO32
1627 #define EFX_OWORD_IS_SET        EFX_OWORD_IS_SET32
1628 #define EFX_QWORD_IS_SET        EFX_QWORD_IS_SET32
1629 #define EFX_POPULATE_OWORD      EFX_POPULATE_OWORD32
1630 #define EFX_POPULATE_QWORD      EFX_POPULATE_QWORD32
1631 #define EFX_SET_OWORD_FIELD     EFX_SET_OWORD_FIELD32
1632 #define EFX_SET_QWORD_FIELD     EFX_SET_QWORD_FIELD32
1633 #define EFX_SET_OWORD_BIT       EFX_SET_OWORD_BIT32
1634 #define EFX_CLEAR_OWORD_BIT     EFX_CLEAR_OWORD_BIT32
1635 #define EFX_TEST_OWORD_BIT      EFX_TEST_OWORD_BIT32
1636 #define EFX_SET_QWORD_BIT       EFX_SET_QWORD_BIT32
1637 #define EFX_CLEAR_QWORD_BIT     EFX_CLEAR_QWORD_BIT32
1638 #define EFX_TEST_QWORD_BIT      EFX_TEST_QWORD_BIT32
1639 #define EFX_OR_OWORD            EFX_OR_OWORD32
1640 #define EFX_AND_OWORD           EFX_AND_OWORD32
1641 #define EFX_OR_QWORD            EFX_OR_QWORD32
1642 #define EFX_AND_QWORD           EFX_AND_QWORD32
1643 #endif
1644
1645 #ifdef  __cplusplus
1646 }
1647 #endif
1648
1649 #endif  /* _SYS_EFX_TYPES_H */