sys/dev/sfxge/common/efx_types.h

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2007-2016 Solarflare Communications Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions are met:
   9  *
  10  * 1. Redistributions of source code must retain the above copyright notice,
  11  *    this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright notice,
  13  *    this list of conditions and the following disclaimer in the documentation
  14  *    and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  18  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  20  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  21  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  22  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  23  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  24  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  25  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  26  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  *
  28  * The views and conclusions contained in the software and documentation are
  29  * those of the authors and should not be interpreted as representing official
  30  * policies, either expressed or implied, of the FreeBSD Project.
  31  *
  32  * Ackowledgement to Fen Systems Ltd.
  33  *
  34  * $FreeBSD$
  35  */
  36
  37 #ifndef _SYS_EFX_TYPES_H
  38 #define _SYS_EFX_TYPES_H
  39
  40 #include "efsys.h"
  41
  42 #ifdef  __cplusplus
  43 extern "C" {
  44 #endif
  45
  46 /*
  47  * Bitfield access
  48  *
  49  * Solarflare NICs make extensive use of bitfields up to 128 bits
  50  * wide.  Since there is no native 128-bit datatype on most systems,
  51  * and since 64-bit datatypes are inefficient on 32-bit systems and
  52  * vice versa, we wrap accesses in a way that uses the most efficient
  53  * datatype.
  54  *
  55  * The NICs are PCI devices and therefore little-endian.  Since most
  56  * of the quantities that we deal with are DMAed to/from host memory,
  57  * we define    our datatypes (efx_oword_t, efx_qword_t and efx_dword_t)
  58  * to be little-endian.
  59  *
  60  * In the less common case of using PIO for individual register
  61  * writes, we construct the little-endian datatype in host memory and
  62  * then use non-swapping register access primitives, rather than
  63  * constructing a native-endian datatype and relying on implicit
  64  * byte-swapping.  (We use a similar strategy for register reads.)
  65  */
  66
  67 /*
  68  * NOTE: Field definitions here and elsewhere are done in terms of a lowest
  69  *       bit number (LBN) and a width.
  70  */
  71
  72 #define EFX_DUMMY_FIELD_LBN 0
  73 #define EFX_DUMMY_FIELD_WIDTH 0
  74
  75 #define EFX_BYTE_0_LBN 0
  76 #define EFX_BYTE_0_WIDTH 8
  77
  78 #define EFX_BYTE_1_LBN 8
  79 #define EFX_BYTE_1_WIDTH 8
  80
  81 #define EFX_BYTE_2_LBN 16
  82 #define EFX_BYTE_2_WIDTH 8
  83
  84 #define EFX_BYTE_3_LBN 24
  85 #define EFX_BYTE_3_WIDTH 8
  86
  87 #define EFX_BYTE_4_LBN 32
  88 #define EFX_BYTE_4_WIDTH 8
  89
  90 #define EFX_BYTE_5_LBN 40
  91 #define EFX_BYTE_5_WIDTH 8
  92
  93 #define EFX_BYTE_6_LBN 48
  94 #define EFX_BYTE_6_WIDTH 8
  95
  96 #define EFX_BYTE_7_LBN 56
  97 #define EFX_BYTE_7_WIDTH 8
  98
  99 #define EFX_WORD_0_LBN 0
 100 #define EFX_WORD_0_WIDTH 16
 101
 102 #define EFX_WORD_1_LBN 16
 103 #define EFX_WORD_1_WIDTH 16
 104
 105 #define EFX_WORD_2_LBN 32
 106 #define EFX_WORD_2_WIDTH 16
 107
 108 #define EFX_WORD_3_LBN 48
 109 #define EFX_WORD_3_WIDTH 16
 110
 111 #define EFX_DWORD_0_LBN 0
 112 #define EFX_DWORD_0_WIDTH 32
 113
 114 #define EFX_DWORD_1_LBN 32
 115 #define EFX_DWORD_1_WIDTH 32
 116
 117 #define EFX_DWORD_2_LBN 64
 118 #define EFX_DWORD_2_WIDTH 32
 119
 120 #define EFX_DWORD_3_LBN 96
 121 #define EFX_DWORD_3_WIDTH 32
 122
 123 /*
 124  * There are intentionally no EFX_QWORD_0 or EFX_QWORD_1 field definitions
 125  * here as the implementaion of EFX_QWORD_FIELD and EFX_OWORD_FIELD do not
 126  * support field widths larger than 32 bits.
 127  */
 128
 129 /* Specified attribute (i.e. LBN ow WIDTH) of the specified field */
 130 #define EFX_VAL(_field, _attribute)                                     \
 131         _field ## _ ## _attribute
 132
 133 /* Lowest bit number of the specified field */
 134 #define EFX_LOW_BIT(_field)                                             \
 135         EFX_VAL(_field, LBN)
 136
 137 /* Width of the specified field */
 138 #define EFX_WIDTH(_field)                                               \
 139         EFX_VAL(_field, WIDTH)
 140
 141 /* Highest bit number of the specified field */
 142 #define EFX_HIGH_BIT(_field)                                            \
 143         (EFX_LOW_BIT(_field) + EFX_WIDTH(_field) - 1)
 144
 145 /*
 146  * 64-bit mask equal in width to the specified field.
 147  *
 148  * For example, a field with width 5 would have a mask of 0x000000000000001f.
 149  */
 150 #define EFX_MASK64(_field)                                              \
 151         ((EFX_WIDTH(_field) == 64) ? ~((uint64_t)0) :                   \
 152             (((((uint64_t)1) << EFX_WIDTH(_field))) - 1))
 153 /*
 154  * 32-bit mask equal in width to the specified field.
 155  *
 156  * For example, a field with width 5 would have a mask of 0x0000001f.
 157  */
 158 #define EFX_MASK32(_field)                                              \
 159         ((EFX_WIDTH(_field) == 32) ? ~((uint32_t)0) :                   \
 160             (((((uint32_t)1) << EFX_WIDTH(_field))) - 1))
 161
 162 /*
 163  * 16-bit mask equal in width to the specified field.
 164  *
 165  * For example, a field with width 5 would have a mask of 0x001f.
 166  */
 167 #define EFX_MASK16(_field)                                              \
 168         ((EFX_WIDTH(_field) == 16) ? 0xffffu :                          \
 169             (uint16_t)((1 << EFX_WIDTH(_field)) - 1))
 170
 171 /*
 172  * 8-bit mask equal in width to the specified field.
 173  *
 174  * For example, a field with width 5 would have a mask of 0x1f.
 175  */
 176 #define EFX_MASK8(_field)                                               \
 177         ((uint8_t)((1 << EFX_WIDTH(_field)) - 1))
 178
 179 #pragma pack(1)
 180
 181 /*
 182  * A byte (i.e. 8-bit) datatype
 183  */
 184 typedef union efx_byte_u {
 185         uint8_t eb_u8[1];
 186 } efx_byte_t;
 187
 188 /*
 189  * A word (i.e. 16-bit) datatype
 190  *
 191  * This datatype is defined to be little-endian.
 192  */
 193 typedef union efx_word_u {
 194         efx_byte_t ew_byte[2];
 195         uint16_t ew_u16[1];
 196         uint8_t ew_u8[2];
 197 } efx_word_t;
 198
 199 /*
 200  * A doubleword (i.e. 32-bit) datatype
 201  *
 202  * This datatype is defined to be little-endian.
 203  */
 204 typedef union efx_dword_u {
 205         efx_byte_t ed_byte[4];
 206         efx_word_t ed_word[2];
 207         uint32_t ed_u32[1];
 208         uint16_t ed_u16[2];
 209         uint8_t ed_u8[4];
 210 } efx_dword_t;
 211
 212 /*
 213  * A quadword (i.e. 64-bit) datatype
 214  *
 215  * This datatype is defined to be little-endian.
 216  */
 217 typedef union efx_qword_u {
 218         efx_byte_t eq_byte[8];
 219         efx_word_t eq_word[4];
 220         efx_dword_t eq_dword[2];
 221 #if EFSYS_HAS_UINT64
 222         uint64_t eq_u64[1];
 223 #endif
 224         uint32_t eq_u32[2];
 225         uint16_t eq_u16[4];
 226         uint8_t eq_u8[8];
 227 } efx_qword_t;
 228
 229 /*
 230  * An octword (i.e. 128-bit) datatype
 231  *
 232  * This datatype is defined to be little-endian.
 233  */
 234 typedef union efx_oword_u {
 235         efx_byte_t eo_byte[16];
 236         efx_word_t eo_word[8];
 237         efx_dword_t eo_dword[4];
 238         efx_qword_t eo_qword[2];
 239 #if EFSYS_HAS_SSE2_M128
 240         __m128i eo_u128[1];
 241 #endif
 242 #if EFSYS_HAS_UINT64
 243         uint64_t eo_u64[2];
 244 #endif
 245         uint32_t eo_u32[4];
 246         uint16_t eo_u16[8];
 247         uint8_t eo_u8[16];
 248 } efx_oword_t;
 249
 250 #pragma pack()
 251
 252 #define __SWAP16(_x)                            \
 253         ((((_x) & 0xff) << 8) |                 \
 254         (((_x) >> 8) & 0xff))
 255
 256 #define __SWAP32(_x)                            \
 257         ((__SWAP16((_x) & 0xffff) << 16) |      \
 258         __SWAP16(((_x) >> 16) & 0xffff))
 259
 260 #define __SWAP64(_x)                            \
 261         ((__SWAP32((_x) & 0xffffffff) << 32) |  \
 262         __SWAP32(((_x) >> 32) & 0xffffffff))
 263
 264 #define __NOSWAP16(_x)          (_x)
 265 #define __NOSWAP32(_x)          (_x)
 266 #define __NOSWAP64(_x)          (_x)
 267
 268 #if EFSYS_IS_BIG_ENDIAN
 269
 270 #define __CPU_TO_LE_16(_x)      ((uint16_t)__SWAP16(_x))
 271 #define __LE_TO_CPU_16(_x)      ((uint16_t)__SWAP16(_x))
 272 #define __CPU_TO_BE_16(_x)      ((uint16_t)__NOSWAP16(_x))
 273 #define __BE_TO_CPU_16(_x)      ((uint16_t)__NOSWAP16(_x))
 274
 275 #define __CPU_TO_LE_32(_x)      ((uint32_t)__SWAP32(_x))
 276 #define __LE_TO_CPU_32(_x)      ((uint32_t)__SWAP32(_x))
 277 #define __CPU_TO_BE_32(_x)      ((uint32_t)__NOSWAP32(_x))
 278 #define __BE_TO_CPU_32(_x)      ((uint32_t)__NOSWAP32(_x))
 279
 280 #define __CPU_TO_LE_64(_x)      ((uint64_t)__SWAP64(_x))
 281 #define __LE_TO_CPU_64(_x)      ((uint64_t)__SWAP64(_x))
 282 #define __CPU_TO_BE_64(_x)      ((uint64_t)__NOSWAP64(_x))
 283 #define __BE_TO_CPU_64(_x)      ((uint64_t)__NOSWAP64(_x))
 284
 285 #elif EFSYS_IS_LITTLE_ENDIAN
 286
 287 #define __CPU_TO_LE_16(_x)      ((uint16_t)__NOSWAP16(_x))
 288 #define __LE_TO_CPU_16(_x)      ((uint16_t)__NOSWAP16(_x))
 289 #define __CPU_TO_BE_16(_x)      ((uint16_t)__SWAP16(_x))
 290 #define __BE_TO_CPU_16(_x)      ((uint16_t)__SWAP16(_x))
 291
 292 #define __CPU_TO_LE_32(_x)      ((uint32_t)__NOSWAP32(_x))
 293 #define __LE_TO_CPU_32(_x)      ((uint32_t)__NOSWAP32(_x))
 294 #define __CPU_TO_BE_32(_x)      ((uint32_t)__SWAP32(_x))
 295 #define __BE_TO_CPU_32(_x)      ((uint32_t)__SWAP32(_x))
 296
 297 #define __CPU_TO_LE_64(_x)      ((uint64_t)__NOSWAP64(_x))
 298 #define __LE_TO_CPU_64(_x)      ((uint64_t)__NOSWAP64(_x))
 299 #define __CPU_TO_BE_64(_x)      ((uint64_t)__SWAP64(_x))
 300 #define __BE_TO_CPU_64(_x)      ((uint64_t)__SWAP64(_x))
 301
 302 #else
 303
 304 #error "Neither of EFSYS_IS_{BIG,LITTLE}_ENDIAN is set"
 305
 306 #endif
 307
 308 #define __NATIVE_8(_x)  (uint8_t)(_x)
 309
 310 /* Format string for printing an efx_byte_t */
 311 #define EFX_BYTE_FMT "0x%02x"
 312
 313 /* Format string for printing an efx_word_t */
 314 #define EFX_WORD_FMT "0x%04x"
 315
 316 /* Format string for printing an efx_dword_t */
 317 #define EFX_DWORD_FMT "0x%08x"
 318
 319 /* Format string for printing an efx_qword_t */
 320 #define EFX_QWORD_FMT "0x%08x:%08x"
 321
 322 /* Format string for printing an efx_oword_t */
 323 #define EFX_OWORD_FMT "0x%08x:%08x:%08x:%08x"
 324
 325 /* Parameters for printing an efx_byte_t */
 326 #define EFX_BYTE_VAL(_byte)                                     \
 327         ((unsigned int)__NATIVE_8((_byte).eb_u8[0]))
 328
 329 /* Parameters for printing an efx_word_t */
 330 #define EFX_WORD_VAL(_word)                                     \
 331         ((unsigned int)__LE_TO_CPU_16((_word).ew_u16[0]))
 332
 333 /* Parameters for printing an efx_dword_t */
 334 #define EFX_DWORD_VAL(_dword)                                   \
 335         ((unsigned int)__LE_TO_CPU_32((_dword).ed_u32[0]))
 336
 337 /* Parameters for printing an efx_qword_t */
 338 #define EFX_QWORD_VAL(_qword)                                   \
 339         ((unsigned int)__LE_TO_CPU_32((_qword).eq_u32[1])),     \
 340         ((unsigned int)__LE_TO_CPU_32((_qword).eq_u32[0]))
 341
 342 /* Parameters for printing an efx_oword_t */
 343 #define EFX_OWORD_VAL(_oword)                                   \
 344         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[3])),     \
 345         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[2])),     \
 346         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[1])),     \
 347         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[0]))
 348
 349 /*
 350  * Stop lint complaining about some shifts.
 351  */
 352 #ifdef  __lint
 353 extern int fix_lint;
 354 #define FIX_LINT(_x)    (_x + fix_lint)
 355 #else
 356 #define FIX_LINT(_x)    (_x)
 357 #endif
 358
 359 /*
 360  * Saturation arithmetic subtract with minimum equal to zero.
 361  *
 362  * Use saturating arithmetic to ensure a non-negative result. This
 363  * avoids undefined behaviour (and compiler warnings) when used as a
 364  * shift count.
 365  */
 366 #define EFX_SSUB(_val, _sub) \
 367         ((_val) > (_sub) ? ((_val) - (_sub)) : 0)
 368
 369 /*
 370  * Extract bit field portion [low,high) from the native-endian element
 371  * which contains bits [min,max).
 372  *
 373  * For example, suppose "element" represents the high 32 bits of a
 374  * 64-bit value, and we wish to extract the bits belonging to the bit
 375  * field occupying bits 28-45 of this 64-bit value.
 376  *
 377  * Then EFX_EXTRACT(_element, 32, 63, 28, 45) would give
 378  *
 379  *   (_element) << 4
 380  *
 381  * The result will contain the relevant bits filled in in the range
 382  * [0,high-low), with garbage in bits [high-low+1,...).
 383  */
 384 #define EFX_EXTRACT_NATIVE(_element, _min, _max, _low, _high)           \
 385         ((FIX_LINT(_low > _max) || FIX_LINT(_high < _min)) ?            \
 386                 0U :                                                    \
 387                 ((_low > _min) ?                                        \
 388                         ((_element) >> EFX_SSUB(_low, _min)) :          \
 389                         ((_element) << EFX_SSUB(_min, _low))))
 390
 391 /*
 392  * Extract bit field portion [low,high) from the 64-bit little-endian
 393  * element which contains bits [min,max)
 394  */
 395 #define EFX_EXTRACT64(_element, _min, _max, _low, _high)                \
 396         EFX_EXTRACT_NATIVE(__LE_TO_CPU_64(_element), _min, _max, _low, _high)
 397
 398 /*
 399  * Extract bit field portion [low,high) from the 32-bit little-endian
 400  * element which contains bits [min,max)
 401  */
 402 #define EFX_EXTRACT32(_element, _min, _max, _low, _high)                \
 403         EFX_EXTRACT_NATIVE(__LE_TO_CPU_32(_element), _min, _max, _low, _high)
 404
 405 /*
 406  * Extract bit field portion [low,high) from the 16-bit little-endian
 407  * element which contains bits [min,max)
 408  */
 409 #define EFX_EXTRACT16(_element, _min, _max, _low, _high)                \
 410         EFX_EXTRACT_NATIVE(__LE_TO_CPU_16(_element), _min, _max, _low, _high)
 411
 412 /*
 413  * Extract bit field portion [low,high) from the 8-bit
 414  * element which contains bits [min,max)
 415  */
 416 #define EFX_EXTRACT8(_element, _min, _max, _low, _high)                 \
 417         EFX_EXTRACT_NATIVE(__NATIVE_8(_element), _min, _max, _low, _high)
 418
 419 #define EFX_EXTRACT_OWORD64(_oword, _low, _high)                        \
 420         (EFX_EXTRACT64((_oword).eo_u64[0], FIX_LINT(0), FIX_LINT(63),   \
 421             _low, _high) |                                              \
 422         EFX_EXTRACT64((_oword).eo_u64[1], FIX_LINT(64), FIX_LINT(127),  \
 423             _low, _high))
 424
 425 #define EFX_EXTRACT_OWORD32(_oword, _low, _high)                        \
 426         (EFX_EXTRACT32((_oword).eo_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 427             _low, _high) |                                              \
 428         EFX_EXTRACT32((_oword).eo_u32[1], FIX_LINT(32), FIX_LINT(63),   \
 429             _low, _high) |                                              \
 430         EFX_EXTRACT32((_oword).eo_u32[2], FIX_LINT(64), FIX_LINT(95),   \
 431             _low, _high) |                                              \
 432         EFX_EXTRACT32((_oword).eo_u32[3], FIX_LINT(96), FIX_LINT(127),  \
 433             _low, _high))
 434
 435 #define EFX_EXTRACT_QWORD64(_qword, _low, _high)                        \
 436         (EFX_EXTRACT64((_qword).eq_u64[0], FIX_LINT(0), FIX_LINT(63),   \
 437             _low, _high))
 438
 439 #define EFX_EXTRACT_QWORD32(_qword, _low, _high)                        \
 440         (EFX_EXTRACT32((_qword).eq_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 441             _low, _high) |                                              \
 442         EFX_EXTRACT32((_qword).eq_u32[1], FIX_LINT(32), FIX_LINT(63),   \
 443             _low, _high))
 444
 445 #define EFX_EXTRACT_DWORD(_dword, _low, _high)                          \
 446         (EFX_EXTRACT32((_dword).ed_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 447             _low, _high))
 448
 449 #define EFX_EXTRACT_WORD(_word, _low, _high)                            \
 450         (EFX_EXTRACT16((_word).ew_u16[0], FIX_LINT(0), FIX_LINT(15),    \
 451             _low, _high))
 452
 453 #define EFX_EXTRACT_BYTE(_byte, _low, _high)                            \
 454         (EFX_EXTRACT8((_byte).eb_u8[0], FIX_LINT(0), FIX_LINT(7),       \
 455             _low, _high))
 456
 457
 458 #define EFX_OWORD_FIELD64(_oword, _field)                               \
 459         ((uint32_t)EFX_EXTRACT_OWORD64(_oword, EFX_LOW_BIT(_field),     \
 460             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 461
 462 #define EFX_OWORD_FIELD32(_oword, _field)                               \
 463         (EFX_EXTRACT_OWORD32(_oword, EFX_LOW_BIT(_field),               \
 464             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 465
 466 #define EFX_QWORD_FIELD64(_qword, _field)                               \
 467         ((uint32_t)EFX_EXTRACT_QWORD64(_qword, EFX_LOW_BIT(_field),     \
 468             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 469
 470 #define EFX_QWORD_FIELD32(_qword, _field)                               \
 471         (EFX_EXTRACT_QWORD32(_qword, EFX_LOW_BIT(_field),               \
 472             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 473
 474 #define EFX_DWORD_FIELD(_dword, _field)                                 \
 475         (EFX_EXTRACT_DWORD(_dword, EFX_LOW_BIT(_field),                 \
 476             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 477
 478 #define EFX_WORD_FIELD(_word, _field)                                   \
 479         (EFX_EXTRACT_WORD(_word, EFX_LOW_BIT(_field),                   \
 480             EFX_HIGH_BIT(_field)) & EFX_MASK16(_field))
 481
 482 #define EFX_BYTE_FIELD(_byte, _field)                                   \
 483         (EFX_EXTRACT_BYTE(_byte, EFX_LOW_BIT(_field),                   \
 484             EFX_HIGH_BIT(_field)) & EFX_MASK8(_field))
 485
 486
 487 #define EFX_OWORD_IS_EQUAL64(_oword_a, _oword_b)                        \
 488         ((_oword_a).eo_u64[0] == (_oword_b).eo_u64[0] &&                \
 489             (_oword_a).eo_u64[1] == (_oword_b).eo_u64[1])
 490
 491 #define EFX_OWORD_IS_EQUAL32(_oword_a, _oword_b)                        \
 492         ((_oword_a).eo_u32[0] == (_oword_b).eo_u32[0] &&                \
 493             (_oword_a).eo_u32[1] == (_oword_b).eo_u32[1] &&             \
 494             (_oword_a).eo_u32[2] == (_oword_b).eo_u32[2] &&             \
 495             (_oword_a).eo_u32[3] == (_oword_b).eo_u32[3])
 496
 497 #define EFX_QWORD_IS_EQUAL64(_qword_a, _qword_b)                        \
 498         ((_qword_a).eq_u64[0] == (_qword_b).eq_u64[0])
 499
 500 #define EFX_QWORD_IS_EQUAL32(_qword_a, _qword_b)                        \
 501         ((_qword_a).eq_u32[0] == (_qword_b).eq_u32[0] &&                \
 502             (_qword_a).eq_u32[1] == (_qword_b).eq_u32[1])
 503
 504 #define EFX_DWORD_IS_EQUAL(_dword_a, _dword_b)                          \
 505         ((_dword_a).ed_u32[0] == (_dword_b).ed_u32[0])
 506
 507 #define EFX_WORD_IS_EQUAL(_word_a, _word_b)                             \
 508         ((_word_a).ew_u16[0] == (_word_b).ew_u16[0])
 509
 510 #define EFX_BYTE_IS_EQUAL(_byte_a, _byte_b)                             \
 511         ((_byte_a).eb_u8[0] == (_byte_b).eb_u8[0])
 512
 513
 514 #define EFX_OWORD_IS_ZERO64(_oword)                                     \
 515         (((_oword).eo_u64[0] |                                          \
 516             (_oword).eo_u64[1]) == 0)
 517
 518 #define EFX_OWORD_IS_ZERO32(_oword)                                     \
 519         (((_oword).eo_u32[0] |                                          \
 520             (_oword).eo_u32[1] |                                        \
 521             (_oword).eo_u32[2] |                                        \
 522             (_oword).eo_u32[3]) == 0)
 523
 524 #define EFX_QWORD_IS_ZERO64(_qword)                                     \
 525         (((_qword).eq_u64[0]) == 0)
 526
 527 #define EFX_QWORD_IS_ZERO32(_qword)                                     \
 528         (((_qword).eq_u32[0] |                                          \
 529             (_qword).eq_u32[1]) == 0)
 530
 531 #define EFX_DWORD_IS_ZERO(_dword)                                       \
 532         (((_dword).ed_u32[0]) == 0)
 533
 534 #define EFX_WORD_IS_ZERO(_word)                                         \
 535         (((_word).ew_u16[0]) == 0)
 536
 537 #define EFX_BYTE_IS_ZERO(_byte)                                         \
 538         (((_byte).eb_u8[0]) == 0)
 539
 540
 541 #define EFX_OWORD_IS_SET64(_oword)                                      \
 542         (((_oword).eo_u64[0] &                                          \
 543             (_oword).eo_u64[1]) == ~((uint64_t)0))
 544
 545 #define EFX_OWORD_IS_SET32(_oword)                                      \
 546         (((_oword).eo_u32[0] &                                          \
 547             (_oword).eo_u32[1] &                                        \
 548             (_oword).eo_u32[2] &                                        \
 549             (_oword).eo_u32[3]) == ~((uint32_t)0))
 550
 551 #define EFX_QWORD_IS_SET64(_qword)                                      \
 552         (((_qword).eq_u64[0]) == ~((uint64_t)0))
 553
 554 #define EFX_QWORD_IS_SET32(_qword)                                      \
 555         (((_qword).eq_u32[0] &                                          \
 556             (_qword).eq_u32[1]) == ~((uint32_t)0))
 557
 558 #define EFX_DWORD_IS_SET(_dword)                                        \
 559         ((_dword).ed_u32[0] == ~((uint32_t)0))
 560
 561 #define EFX_WORD_IS_SET(_word)                                          \
 562         ((_word).ew_u16[0] == ~((uint16_t)0))
 563
 564 #define EFX_BYTE_IS_SET(_byte)                                          \
 565         ((_byte).eb_u8[0] == ~((uint8_t)0))
 566
 567 /*
 568  * Construct bit field portion
 569  *
 570  * Creates the portion of the bit field [low,high) that lies within
 571  * the range [min,max).
 572  */
 573
 574 #define EFX_INSERT_NATIVE64(_min, _max, _low, _high, _value)            \
 575         (((_low > _max) || (_high < _min)) ?                            \
 576                 0U :                                                    \
 577                 ((_low > _min) ?                                        \
 578                         (((uint64_t)(_value)) << EFX_SSUB(_low, _min)) :\
 579                         (((uint64_t)(_value)) >> EFX_SSUB(_min, _low))))
 580
 581 #define EFX_INSERT_NATIVE32(_min, _max, _low, _high, _value)            \
 582         (((_low > _max) || (_high < _min)) ?                            \
 583                 0U :                                                    \
 584                 ((_low > _min) ?                                        \
 585                         (((uint32_t)(_value)) << EFX_SSUB(_low, _min)) :\
 586                         (((uint32_t)(_value)) >> EFX_SSUB(_min, _low))))
 587
 588 #define EFX_INSERT_NATIVE16(_min, _max, _low, _high, _value)            \
 589         (((_low > _max) || (_high < _min)) ?                            \
 590                 0U :                                                    \
 591                 (uint16_t)((_low > _min) ?                              \
 592                                 ((_value) << EFX_SSUB(_low, _min)) :    \
 593                                 ((_value) >> EFX_SSUB(_min, _low))))
 594
 595 #define EFX_INSERT_NATIVE8(_min, _max, _low, _high, _value)             \
 596         (((_low > _max) || (_high < _min)) ?                            \
 597                 0U :                                                    \
 598                 (uint8_t)((_low > _min) ?                               \
 599                                 ((_value) << EFX_SSUB(_low, _min)) :    \
 600                                 ((_value) >> EFX_SSUB(_min, _low))))
 601
 602 /*
 603  * Construct bit field portion
 604  *
 605  * Creates the portion of the named bit field that lies within the
 606  * range [min,max).
 607  */
 608 #define EFX_INSERT_FIELD_NATIVE64(_min, _max, _field, _value)           \
 609         EFX_INSERT_NATIVE64(_min, _max, EFX_LOW_BIT(_field),            \
 610             EFX_HIGH_BIT(_field), _value)
 611
 612 #define EFX_INSERT_FIELD_NATIVE32(_min, _max, _field, _value)           \
 613         EFX_INSERT_NATIVE32(_min, _max, EFX_LOW_BIT(_field),            \
 614             EFX_HIGH_BIT(_field), _value)
 615
 616 #define EFX_INSERT_FIELD_NATIVE16(_min, _max, _field, _value)           \
 617         EFX_INSERT_NATIVE16(_min, _max, EFX_LOW_BIT(_field),            \
 618             EFX_HIGH_BIT(_field), _value)
 619
 620 #define EFX_INSERT_FIELD_NATIVE8(_min, _max, _field, _value)            \
 621         EFX_INSERT_NATIVE8(_min, _max, EFX_LOW_BIT(_field),             \
 622             EFX_HIGH_BIT(_field), _value)
 623
 624 /*
 625  * Construct bit field
 626  *
 627  * Creates the portion of the named bit fields that lie within the
 628  * range [min,max).
 629  */
 630 #define EFX_INSERT_FIELDS64(_min, _max,                                 \
 631             _field1, _value1, _field2, _value2, _field3, _value3,       \
 632             _field4, _value4, _field5, _value5, _field6, _value6,       \
 633             _field7, _value7, _field8, _value8, _field9, _value9,       \
 634             _field10, _value10)                                         \
 635         __CPU_TO_LE_64(                                                 \
 636             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field1, _value1) |   \
 637             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field2, _value2) |   \
 638             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field3, _value3) |   \
 639             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field4, _value4) |   \
 640             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field5, _value5) |   \
 641             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field6, _value6) |   \
 642             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field7, _value7) |   \
 643             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field8, _value8) |   \
 644             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field9, _value9) |   \
 645             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field10, _value10))
 646
 647 #define EFX_INSERT_FIELDS32(_min, _max,                                 \
 648             _field1, _value1, _field2, _value2, _field3, _value3,       \
 649             _field4, _value4, _field5, _value5, _field6, _value6,       \
 650             _field7, _value7, _field8, _value8, _field9, _value9,       \
 651             _field10, _value10)                                         \
 652         __CPU_TO_LE_32(                                                 \
 653             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field1, _value1) |   \
 654             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field2, _value2) |   \
 655             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field3, _value3) |   \
 656             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field4, _value4) |   \
 657             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field5, _value5) |   \
 658             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field6, _value6) |   \
 659             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field7, _value7) |   \
 660             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field8, _value8) |   \
 661             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field9, _value9) |   \
 662             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field10, _value10))
 663
 664 #define EFX_INSERT_FIELDS16(_min, _max,                                 \
 665             _field1, _value1, _field2, _value2, _field3, _value3,       \
 666             _field4, _value4, _field5, _value5, _field6, _value6,       \
 667             _field7, _value7, _field8, _value8, _field9, _value9,       \
 668             _field10, _value10)                                         \
 669         __CPU_TO_LE_16(                                                 \
 670             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field1, _value1) |   \
 671             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field2, _value2) |   \
 672             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field3, _value3) |   \
 673             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field4, _value4) |   \
 674             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field5, _value5) |   \
 675             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field6, _value6) |   \
 676             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field7, _value7) |   \
 677             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field8, _value8) |   \
 678             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field9, _value9) |   \
 679             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field10, _value10))
 680
 681 #define EFX_INSERT_FIELDS8(_min, _max,                                  \
 682             _field1, _value1, _field2, _value2, _field3, _value3,       \
 683             _field4, _value4, _field5, _value5, _field6, _value6,       \
 684             _field7, _value7, _field8, _value8, _field9, _value9,       \
 685             _field10, _value10)                                         \
 686         __NATIVE_8(                                                     \
 687             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field1, _value1) |    \
 688             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field2, _value2) |    \
 689             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field3, _value3) |    \
 690             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field4, _value4) |    \
 691             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field5, _value5) |    \
 692             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field6, _value6) |    \
 693             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field7, _value7) |    \
 694             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field8, _value8) |    \
 695             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field9, _value9) |    \
 696             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field10, _value10))
 697
 698 #define EFX_POPULATE_OWORD64(_oword,                                    \
 699             _field1, _value1, _field2, _value2, _field3, _value3,       \
 700             _field4, _value4, _field5, _value5, _field6, _value6,       \
 701             _field7, _value7, _field8, _value8, _field9, _value9,       \
 702             _field10, _value10)                                         \
 703         do {                                                            \
 704                 _NOTE(CONSTANTCONDITION)                                \
 705                 (_oword).eo_u64[0] = EFX_INSERT_FIELDS64(0, 63,         \
 706                     _field1, _value1, _field2, _value2,                 \
 707                     _field3, _value3, _field4, _value4,                 \
 708                     _field5, _value5, _field6, _value6,                 \
 709                     _field7, _value7, _field8, _value8,                 \
 710                     _field9, _value9, _field10, _value10);              \
 711                 _NOTE(CONSTANTCONDITION)                                \
 712                 (_oword).eo_u64[1] = EFX_INSERT_FIELDS64(64, 127,       \
 713                     _field1, _value1, _field2, _value2,                 \
 714                     _field3, _value3, _field4, _value4,                 \
 715                     _field5, _value5, _field6, _value6,                 \
 716                     _field7, _value7, _field8, _value8,                 \
 717                     _field9, _value9, _field10, _value10);              \
 718         _NOTE(CONSTANTCONDITION)                                        \
 719         } while (B_FALSE)
 720
 721 #define EFX_POPULATE_OWORD32(_oword,                                    \
 722             _field1, _value1, _field2, _value2, _field3, _value3,       \
 723             _field4, _value4, _field5, _value5, _field6, _value6,       \
 724             _field7, _value7, _field8, _value8, _field9, _value9,       \
 725             _field10, _value10)                                         \
 726         do {                                                            \
 727                 _NOTE(CONSTANTCONDITION)                                \
 728                 (_oword).eo_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 729                     _field1, _value1, _field2, _value2,                 \
 730                     _field3, _value3, _field4, _value4,                 \
 731                     _field5, _value5, _field6, _value6,                 \
 732                     _field7, _value7, _field8, _value8,                 \
 733                     _field9, _value9, _field10, _value10);              \
 734                 _NOTE(CONSTANTCONDITION)                                \
 735                 (_oword).eo_u32[1] = EFX_INSERT_FIELDS32(32, 63,        \
 736                     _field1, _value1, _field2, _value2,                 \
 737                     _field3, _value3, _field4, _value4,                 \
 738                     _field5, _value5, _field6, _value6,                 \
 739                     _field7, _value7, _field8, _value8,                 \
 740                     _field9, _value9, _field10, _value10);              \
 741                 _NOTE(CONSTANTCONDITION)                                \
 742                 (_oword).eo_u32[2] = EFX_INSERT_FIELDS32(64, 95,        \
 743                     _field1, _value1, _field2, _value2,                 \
 744                     _field3, _value3, _field4, _value4,                 \
 745                     _field5, _value5, _field6, _value6,                 \
 746                     _field7, _value7, _field8, _value8,                 \
 747                     _field9, _value9, _field10, _value10);              \
 748                 _NOTE(CONSTANTCONDITION)                                \
 749                 (_oword).eo_u32[3] = EFX_INSERT_FIELDS32(96, 127,       \
 750                     _field1, _value1, _field2, _value2,                 \
 751                     _field3, _value3, _field4, _value4,                 \
 752                     _field5, _value5, _field6, _value6,                 \
 753                     _field7, _value7, _field8, _value8,                 \
 754                     _field9, _value9, _field10, _value10);              \
 755         _NOTE(CONSTANTCONDITION)                                        \
 756         } while (B_FALSE)
 757
 758 #define EFX_POPULATE_QWORD64(_qword,                                    \
 759             _field1, _value1, _field2, _value2, _field3, _value3,       \
 760             _field4, _value4, _field5, _value5, _field6, _value6,       \
 761             _field7, _value7, _field8, _value8, _field9, _value9,       \
 762             _field10, _value10)                                         \
 763         do {                                                            \
 764                 _NOTE(CONSTANTCONDITION)                                \
 765                 (_qword).eq_u64[0] = EFX_INSERT_FIELDS64(0, 63,         \
 766                     _field1, _value1, _field2, _value2,                 \
 767                     _field3, _value3, _field4, _value4,                 \
 768                     _field5, _value5, _field6, _value6,                 \
 769                     _field7, _value7, _field8, _value8,                 \
 770                     _field9, _value9, _field10, _value10);              \
 771         _NOTE(CONSTANTCONDITION)                                        \
 772         } while (B_FALSE)
 773
 774 #define EFX_POPULATE_QWORD32(_qword,                                    \
 775             _field1, _value1, _field2, _value2, _field3, _value3,       \
 776             _field4, _value4, _field5, _value5, _field6, _value6,       \
 777             _field7, _value7, _field8, _value8, _field9, _value9,       \
 778             _field10, _value10)                                         \
 779         do {                                                            \
 780                 _NOTE(CONSTANTCONDITION)                                \
 781                 (_qword).eq_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 782                     _field1, _value1, _field2, _value2,                 \
 783                     _field3, _value3, _field4, _value4,                 \
 784                     _field5, _value5, _field6, _value6,                 \
 785                     _field7, _value7, _field8, _value8,                 \
 786                     _field9, _value9, _field10, _value10);              \
 787                 _NOTE(CONSTANTCONDITION)                                \
 788                 (_qword).eq_u32[1] = EFX_INSERT_FIELDS32(32, 63,        \
 789                     _field1, _value1, _field2, _value2,                 \
 790                     _field3, _value3, _field4, _value4,                 \
 791                     _field5, _value5, _field6, _value6,                 \
 792                     _field7, _value7, _field8, _value8,                 \
 793                     _field9, _value9, _field10, _value10);              \
 794         _NOTE(CONSTANTCONDITION)                                        \
 795         } while (B_FALSE)
 796
 797 #define EFX_POPULATE_DWORD(_dword,                                      \
 798             _field1, _value1, _field2, _value2, _field3, _value3,       \
 799             _field4, _value4, _field5, _value5, _field6, _value6,       \
 800             _field7, _value7, _field8, _value8, _field9, _value9,       \
 801             _field10, _value10)                                         \
 802         do {                                                            \
 803                 _NOTE(CONSTANTCONDITION)                                \
 804                 (_dword).ed_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 805                     _field1, _value1, _field2, _value2,                 \
 806                     _field3, _value3, _field4, _value4,                 \
 807                     _field5, _value5, _field6, _value6,                 \
 808                     _field7, _value7, _field8, _value8,                 \
 809                     _field9, _value9, _field10, _value10);              \
 810         _NOTE(CONSTANTCONDITION)                                        \
 811         } while (B_FALSE)
 812
 813 #define EFX_POPULATE_WORD(_word,                                        \
 814             _field1, _value1, _field2, _value2, _field3, _value3,       \
 815             _field4, _value4, _field5, _value5, _field6, _value6,       \
 816             _field7, _value7, _field8, _value8, _field9, _value9,       \
 817             _field10, _value10)                                         \
 818         do {                                                            \
 819                 _NOTE(CONSTANTCONDITION)                                \
 820                 (_word).ew_u16[0] = EFX_INSERT_FIELDS16(0, 15,          \
 821                     _field1, _value1, _field2, _value2,                 \
 822                     _field3, _value3, _field4, _value4,                 \
 823                     _field5, _value5, _field6, _value6,                 \
 824                     _field7, _value7, _field8, _value8,                 \
 825                     _field9, _value9, _field10, _value10);              \
 826         _NOTE(CONSTANTCONDITION)                                        \
 827         } while (B_FALSE)
 828
 829 #define EFX_POPULATE_BYTE(_byte,                                        \
 830             _field1, _value1, _field2, _value2, _field3, _value3,       \
 831             _field4, _value4, _field5, _value5, _field6, _value6,       \
 832             _field7, _value7, _field8, _value8, _field9, _value9,       \
 833             _field10, _value10)                                         \
 834         do {                                                            \
 835                 _NOTE(CONSTANTCONDITION)                                \
 836                 (_byte).eb_u8[0] = EFX_INSERT_FIELDS8(0, 7,             \
 837                     _field1, _value1, _field2, _value2,                 \
 838                     _field3, _value3, _field4, _value4,                 \
 839                     _field5, _value5, _field6, _value6,                 \
 840                     _field7, _value7, _field8, _value8,                 \
 841                     _field9, _value9, _field10, _value10);              \
 842         _NOTE(CONSTANTCONDITION)                                        \
 843         } while (B_FALSE)
 844
 845 /* Populate an octword field with various numbers of arguments */
 846 #define EFX_POPULATE_OWORD_10 EFX_POPULATE_OWORD
 847
 848 #define EFX_POPULATE_OWORD_9(_oword,                                    \
 849             _field1, _value1, _field2, _value2, _field3, _value3,       \
 850             _field4, _value4, _field5, _value5, _field6, _value6,       \
 851             _field7, _value7, _field8, _value8, _field9, _value9)       \
 852         EFX_POPULATE_OWORD_10(_oword, EFX_DUMMY_FIELD, 0,               \
 853             _field1, _value1, _field2, _value2, _field3, _value3,       \
 854             _field4, _value4, _field5, _value5, _field6, _value6,       \
 855             _field7, _value7, _field8, _value8, _field9, _value9)
 856
 857 #define EFX_POPULATE_OWORD_8(_oword,                                    \
 858             _field1, _value1, _field2, _value2, _field3, _value3,       \
 859             _field4, _value4, _field5, _value5, _field6, _value6,       \
 860             _field7, _value7, _field8, _value8)                         \
 861         EFX_POPULATE_OWORD_9(_oword, EFX_DUMMY_FIELD, 0,                \
 862             _field1, _value1, _field2, _value2, _field3, _value3,       \
 863             _field4, _value4, _field5, _value5, _field6, _value6,       \
 864             _field7, _value7, _field8, _value8)
 865
 866 #define EFX_POPULATE_OWORD_7(_oword,                                    \
 867             _field1, _value1, _field2, _value2, _field3, _value3,       \
 868             _field4, _value4, _field5, _value5, _field6, _value6,       \
 869             _field7, _value7)                                           \
 870         EFX_POPULATE_OWORD_8(_oword, EFX_DUMMY_FIELD, 0,                \
 871             _field1, _value1, _field2, _value2, _field3, _value3,       \
 872             _field4, _value4, _field5, _value5, _field6, _value6,       \
 873             _field7, _value7)
 874
 875 #define EFX_POPULATE_OWORD_6(_oword,                                    \
 876             _field1, _value1, _field2, _value2, _field3, _value3,       \
 877             _field4, _value4, _field5, _value5, _field6, _value6)       \
 878         EFX_POPULATE_OWORD_7(_oword, EFX_DUMMY_FIELD, 0,                \
 879             _field1, _value1, _field2, _value2, _field3, _value3,       \
 880             _field4, _value4, _field5, _value5, _field6, _value6)
 881
 882 #define EFX_POPULATE_OWORD_5(_oword,                                    \
 883             _field1, _value1, _field2, _value2, _field3, _value3,       \
 884             _field4, _value4, _field5, _value5)                         \
 885         EFX_POPULATE_OWORD_6(_oword, EFX_DUMMY_FIELD, 0,                \
 886             _field1, _value1, _field2, _value2, _field3, _value3,       \
 887             _field4, _value4, _field5, _value5)
 888
 889 #define EFX_POPULATE_OWORD_4(_oword,                                    \
 890             _field1, _value1, _field2, _value2, _field3, _value3,       \
 891             _field4, _value4)                                           \
 892         EFX_POPULATE_OWORD_5(_oword, EFX_DUMMY_FIELD, 0,                \
 893             _field1, _value1, _field2, _value2, _field3, _value3,       \
 894             _field4, _value4)
 895
 896 #define EFX_POPULATE_OWORD_3(_oword,                                    \
 897             _field1, _value1, _field2, _value2, _field3, _value3)       \
 898         EFX_POPULATE_OWORD_4(_oword, EFX_DUMMY_FIELD, 0,                \
 899             _field1, _value1, _field2, _value2, _field3, _value3)
 900
 901 #define EFX_POPULATE_OWORD_2(_oword,                                    \
 902             _field1, _value1, _field2, _value2)                         \
 903         EFX_POPULATE_OWORD_3(_oword, EFX_DUMMY_FIELD, 0,                \
 904             _field1, _value1, _field2, _value2)
 905
 906 #define EFX_POPULATE_OWORD_1(_oword,                                    \
 907             _field1, _value1)                                           \
 908         EFX_POPULATE_OWORD_2(_oword, EFX_DUMMY_FIELD, 0,                \
 909             _field1, _value1)
 910
 911 #define EFX_ZERO_OWORD(_oword)                                          \
 912         EFX_POPULATE_OWORD_1(_oword, EFX_DUMMY_FIELD, 0)
 913
 914 #define EFX_SET_OWORD(_oword)                                           \
 915         EFX_POPULATE_OWORD_4(_oword,                                    \
 916             EFX_DWORD_0, 0xffffffff, EFX_DWORD_1, 0xffffffff,           \
 917             EFX_DWORD_2, 0xffffffff, EFX_DWORD_3, 0xffffffff)
 918
 919 /* Populate a quadword field with various numbers of arguments */
 920 #define EFX_POPULATE_QWORD_10 EFX_POPULATE_QWORD
 921
 922 #define EFX_POPULATE_QWORD_9(_qword,                                    \
 923             _field1, _value1, _field2, _value2, _field3, _value3,       \
 924             _field4, _value4, _field5, _value5, _field6, _value6,       \
 925             _field7, _value7, _field8, _value8, _field9, _value9)       \
 926         EFX_POPULATE_QWORD_10(_qword, EFX_DUMMY_FIELD, 0,               \
 927             _field1, _value1, _field2, _value2, _field3, _value3,       \
 928             _field4, _value4, _field5, _value5, _field6, _value6,       \
 929             _field7, _value7, _field8, _value8, _field9, _value9)
 930
 931 #define EFX_POPULATE_QWORD_8(_qword,                                    \
 932             _field1, _value1, _field2, _value2, _field3, _value3,       \
 933             _field4, _value4, _field5, _value5, _field6, _value6,       \
 934             _field7, _value7, _field8, _value8)                         \
 935         EFX_POPULATE_QWORD_9(_qword, EFX_DUMMY_FIELD, 0,                \
 936             _field1, _value1, _field2, _value2, _field3, _value3,       \
 937             _field4, _value4, _field5, _value5, _field6, _value6,       \
 938             _field7, _value7, _field8, _value8)
 939
 940 #define EFX_POPULATE_QWORD_7(_qword,                                    \
 941             _field1, _value1, _field2, _value2, _field3, _value3,       \
 942             _field4, _value4, _field5, _value5, _field6, _value6,       \
 943             _field7, _value7)                                           \
 944         EFX_POPULATE_QWORD_8(_qword, EFX_DUMMY_FIELD, 0,                \
 945             _field1, _value1, _field2, _value2, _field3, _value3,       \
 946             _field4, _value4, _field5, _value5, _field6, _value6,       \
 947             _field7, _value7)
 948
 949 #define EFX_POPULATE_QWORD_6(_qword,                                    \
 950             _field1, _value1, _field2, _value2, _field3, _value3,       \
 951             _field4, _value4, _field5, _value5, _field6, _value6)       \
 952         EFX_POPULATE_QWORD_7(_qword, EFX_DUMMY_FIELD, 0,                \
 953             _field1, _value1, _field2, _value2, _field3, _value3,       \
 954             _field4, _value4, _field5, _value5, _field6, _value6)
 955
 956 #define EFX_POPULATE_QWORD_5(_qword,                                    \
 957             _field1, _value1, _field2, _value2, _field3, _value3,       \
 958             _field4, _value4, _field5, _value5)                         \
 959         EFX_POPULATE_QWORD_6(_qword, EFX_DUMMY_FIELD, 0,                \
 960             _field1, _value1, _field2, _value2, _field3, _value3,       \
 961             _field4, _value4, _field5, _value5)
 962
 963 #define EFX_POPULATE_QWORD_4(_qword,                                    \
 964             _field1, _value1, _field2, _value2, _field3, _value3,       \
 965             _field4, _value4)                                           \
 966         EFX_POPULATE_QWORD_5(_qword, EFX_DUMMY_FIELD, 0,                \
 967             _field1, _value1, _field2, _value2, _field3, _value3,       \
 968             _field4, _value4)
 969
 970 #define EFX_POPULATE_QWORD_3(_qword,                                    \
 971             _field1, _value1, _field2, _value2, _field3, _value3)       \
 972         EFX_POPULATE_QWORD_4(_qword, EFX_DUMMY_FIELD, 0,                \
 973             _field1, _value1, _field2, _value2, _field3, _value3)
 974
 975 #define EFX_POPULATE_QWORD_2(_qword,                                    \
 976             _field1, _value1, _field2, _value2)                         \
 977         EFX_POPULATE_QWORD_3(_qword, EFX_DUMMY_FIELD, 0,                \
 978             _field1, _value1, _field2, _value2)
 979
 980 #define EFX_POPULATE_QWORD_1(_qword,                                    \
 981             _field1, _value1)                                           \
 982         EFX_POPULATE_QWORD_2(_qword, EFX_DUMMY_FIELD, 0,                \
 983             _field1, _value1)
 984
 985 #define EFX_ZERO_QWORD(_qword)                                          \
 986         EFX_POPULATE_QWORD_1(_qword, EFX_DUMMY_FIELD, 0)
 987
 988 #define EFX_SET_QWORD(_qword)                                           \
 989         EFX_POPULATE_QWORD_2(_qword,                                    \
 990             EFX_DWORD_0, 0xffffffff, EFX_DWORD_1, 0xffffffff)
 991
 992 /* Populate a dword field with various numbers of arguments */
 993 #define EFX_POPULATE_DWORD_10 EFX_POPULATE_DWORD
 994
 995 #define EFX_POPULATE_DWORD_9(_dword,                                    \
 996             _field1, _value1, _field2, _value2, _field3, _value3,       \
 997             _field4, _value4, _field5, _value5, _field6, _value6,       \
 998             _field7, _value7, _field8, _value8, _field9, _value9)       \
 999         EFX_POPULATE_DWORD_10(_dword, EFX_DUMMY_FIELD, 0,               \
1000             _field1, _value1, _field2, _value2, _field3, _value3,       \
1001             _field4, _value4, _field5, _value5, _field6, _value6,       \
1002             _field7, _value7, _field8, _value8, _field9, _value9)
1003
1004 #define EFX_POPULATE_DWORD_8(_dword,                                    \
1005             _field1, _value1, _field2, _value2, _field3, _value3,       \
1006             _field4, _value4, _field5, _value5, _field6, _value6,       \
1007             _field7, _value7, _field8, _value8)                         \
1008         EFX_POPULATE_DWORD_9(_dword, EFX_DUMMY_FIELD, 0,                \
1009             _field1, _value1, _field2, _value2, _field3, _value3,       \
1010             _field4, _value4, _field5, _value5, _field6, _value6,       \
1011             _field7, _value7, _field8, _value8)
1012
1013 #define EFX_POPULATE_DWORD_7(_dword,                                    \
1014             _field1, _value1, _field2, _value2, _field3, _value3,       \
1015             _field4, _value4, _field5, _value5, _field6, _value6,       \
1016             _field7, _value7)                                           \
1017         EFX_POPULATE_DWORD_8(_dword, EFX_DUMMY_FIELD, 0,                \
1018             _field1, _value1, _field2, _value2, _field3, _value3,       \
1019             _field4, _value4, _field5, _value5, _field6, _value6,       \
1020             _field7, _value7)
1021
1022 #define EFX_POPULATE_DWORD_6(_dword,                                    \
1023             _field1, _value1, _field2, _value2, _field3, _value3,       \
1024             _field4, _value4, _field5, _value5, _field6, _value6)       \
1025         EFX_POPULATE_DWORD_7(_dword, EFX_DUMMY_FIELD, 0,                \
1026             _field1, _value1, _field2, _value2, _field3, _value3,       \
1027             _field4, _value4, _field5, _value5, _field6, _value6)
1028
1029 #define EFX_POPULATE_DWORD_5(_dword,                                    \
1030             _field1, _value1, _field2, _value2, _field3, _value3,       \
1031             _field4, _value4, _field5, _value5)                         \
1032         EFX_POPULATE_DWORD_6(_dword, EFX_DUMMY_FIELD, 0,                \
1033             _field1, _value1, _field2, _value2, _field3, _value3,       \
1034             _field4, _value4, _field5, _value5)
1035
1036 #define EFX_POPULATE_DWORD_4(_dword,                                    \
1037             _field1, _value1, _field2, _value2, _field3, _value3,       \
1038             _field4, _value4)                                           \
1039         EFX_POPULATE_DWORD_5(_dword, EFX_DUMMY_FIELD, 0,                \
1040             _field1, _value1, _field2, _value2, _field3, _value3,       \
1041             _field4, _value4)
1042
1043 #define EFX_POPULATE_DWORD_3(_dword,                                    \
1044             _field1, _value1, _field2, _value2, _field3, _value3)       \
1045         EFX_POPULATE_DWORD_4(_dword, EFX_DUMMY_FIELD, 0,                \
1046             _field1, _value1, _field2, _value2, _field3, _value3)
1047
1048 #define EFX_POPULATE_DWORD_2(_dword,                                    \
1049             _field1, _value1, _field2, _value2)                         \
1050         EFX_POPULATE_DWORD_3(_dword, EFX_DUMMY_FIELD, 0,                \
1051             _field1, _value1, _field2, _value2)
1052
1053 #define EFX_POPULATE_DWORD_1(_dword,                                    \
1054             _field1, _value1)                                           \
1055         EFX_POPULATE_DWORD_2(_dword, EFX_DUMMY_FIELD, 0,                \
1056             _field1, _value1)
1057
1058 #define EFX_ZERO_DWORD(_dword)                                          \
1059         EFX_POPULATE_DWORD_1(_dword, EFX_DUMMY_FIELD, 0)
1060
1061 #define EFX_SET_DWORD(_dword)                                           \
1062         EFX_POPULATE_DWORD_1(_dword,                                    \
1063             EFX_DWORD_0, 0xffffffff)
1064
1065 /* Populate a word field with various numbers of arguments */
1066 #define EFX_POPULATE_WORD_10 EFX_POPULATE_WORD
1067
1068 #define EFX_POPULATE_WORD_9(_word,                                      \
1069             _field1, _value1, _field2, _value2, _field3, _value3,       \
1070             _field4, _value4, _field5, _value5, _field6, _value6,       \
1071             _field7, _value7, _field8, _value8, _field9, _value9)       \
1072         EFX_POPULATE_WORD_10(_word, EFX_DUMMY_FIELD, 0,                 \
1073             _field1, _value1, _field2, _value2, _field3, _value3,       \
1074             _field4, _value4, _field5, _value5, _field6, _value6,       \
1075             _field7, _value7, _field8, _value8, _field9, _value9)
1076
1077 #define EFX_POPULATE_WORD_8(_word,                                      \
1078             _field1, _value1, _field2, _value2, _field3, _value3,       \
1079             _field4, _value4, _field5, _value5, _field6, _value6,       \
1080             _field7, _value7, _field8, _value8)                         \
1081         EFX_POPULATE_WORD_9(_word, EFX_DUMMY_FIELD, 0,                  \
1082             _field1, _value1, _field2, _value2, _field3, _value3,       \
1083             _field4, _value4, _field5, _value5, _field6, _value6,       \
1084             _field7, _value7, _field8, _value8)
1085
1086 #define EFX_POPULATE_WORD_7(_word,                                      \
1087             _field1, _value1, _field2, _value2, _field3, _value3,       \
1088             _field4, _value4, _field5, _value5, _field6, _value6,       \
1089             _field7, _value7)                                           \
1090         EFX_POPULATE_WORD_8(_word, EFX_DUMMY_FIELD, 0,                  \
1091             _field1, _value1, _field2, _value2, _field3, _value3,       \
1092             _field4, _value4, _field5, _value5, _field6, _value6,       \
1093             _field7, _value7)
1094
1095 #define EFX_POPULATE_WORD_6(_word,                                      \
1096             _field1, _value1, _field2, _value2, _field3, _value3,       \
1097             _field4, _value4, _field5, _value5, _field6, _value6)       \
1098         EFX_POPULATE_WORD_7(_word, EFX_DUMMY_FIELD, 0,                  \
1099             _field1, _value1, _field2, _value2, _field3, _value3,       \
1100             _field4, _value4, _field5, _value5, _field6, _value6)
1101
1102 #define EFX_POPULATE_WORD_5(_word,                                      \
1103             _field1, _value1, _field2, _value2, _field3, _value3,       \
1104             _field4, _value4, _field5, _value5)                         \
1105         EFX_POPULATE_WORD_6(_word, EFX_DUMMY_FIELD, 0,                  \
1106             _field1, _value1, _field2, _value2, _field3, _value3,       \
1107             _field4, _value4, _field5, _value5)
1108
1109 #define EFX_POPULATE_WORD_4(_word,                                      \
1110             _field1, _value1, _field2, _value2, _field3, _value3,       \
1111             _field4, _value4)                                           \
1112         EFX_POPULATE_WORD_5(_word, EFX_DUMMY_FIELD, 0,                  \
1113             _field1, _value1, _field2, _value2, _field3, _value3,       \
1114             _field4, _value4)
1115
1116 #define EFX_POPULATE_WORD_3(_word,                                      \
1117             _field1, _value1, _field2, _value2, _field3, _value3)       \
1118         EFX_POPULATE_WORD_4(_word, EFX_DUMMY_FIELD, 0,                  \
1119             _field1, _value1, _field2, _value2, _field3, _value3)
1120
1121 #define EFX_POPULATE_WORD_2(_word,                                      \
1122             _field1, _value1, _field2, _value2)                         \
1123         EFX_POPULATE_WORD_3(_word, EFX_DUMMY_FIELD, 0,                  \
1124             _field1, _value1, _field2, _value2)
1125
1126 #define EFX_POPULATE_WORD_1(_word,                                      \
1127             _field1, _value1)                                           \
1128         EFX_POPULATE_WORD_2(_word, EFX_DUMMY_FIELD, 0,                  \
1129             _field1, _value1)
1130
1131 #define EFX_ZERO_WORD(_word)                                            \
1132         EFX_POPULATE_WORD_1(_word, EFX_DUMMY_FIELD, 0)
1133
1134 #define EFX_SET_WORD(_word)                                             \
1135         EFX_POPULATE_WORD_1(_word,                                      \
1136             EFX_WORD_0, 0xffff)
1137
1138 /* Populate a byte field with various numbers of arguments */
1139 #define EFX_POPULATE_BYTE_10 EFX_POPULATE_BYTE
1140
1141 #define EFX_POPULATE_BYTE_9(_byte,                                      \
1142             _field1, _value1, _field2, _value2, _field3, _value3,       \
1143             _field4, _value4, _field5, _value5, _field6, _value6,       \
1144             _field7, _value7, _field8, _value8, _field9, _value9)       \
1145         EFX_POPULATE_BYTE_10(_byte, EFX_DUMMY_FIELD, 0,                 \
1146             _field1, _value1, _field2, _value2, _field3, _value3,       \
1147             _field4, _value4, _field5, _value5, _field6, _value6,       \
1148             _field7, _value7, _field8, _value8, _field9, _value9)
1149
1150 #define EFX_POPULATE_BYTE_8(_byte,                                      \
1151             _field1, _value1, _field2, _value2, _field3, _value3,       \
1152             _field4, _value4, _field5, _value5, _field6, _value6,       \
1153             _field7, _value7, _field8, _value8)                         \
1154         EFX_POPULATE_BYTE_9(_byte, EFX_DUMMY_FIELD, 0,                  \
1155             _field1, _value1, _field2, _value2, _field3, _value3,       \
1156             _field4, _value4, _field5, _value5, _field6, _value6,       \
1157             _field7, _value7, _field8, _value8)
1158
1159 #define EFX_POPULATE_BYTE_7(_byte,                                      \
1160             _field1, _value1, _field2, _value2, _field3, _value3,       \
1161             _field4, _value4, _field5, _value5, _field6, _value6,       \
1162             _field7, _value7)                                           \
1163         EFX_POPULATE_BYTE_8(_byte, EFX_DUMMY_FIELD, 0,                  \
1164             _field1, _value1, _field2, _value2, _field3, _value3,       \
1165             _field4, _value4, _field5, _value5, _field6, _value6,       \
1166             _field7, _value7)
1167
1168 #define EFX_POPULATE_BYTE_6(_byte,                                      \
1169             _field1, _value1, _field2, _value2, _field3, _value3,       \
1170             _field4, _value4, _field5, _value5, _field6, _value6)       \
1171         EFX_POPULATE_BYTE_7(_byte, EFX_DUMMY_FIELD, 0,                  \
1172             _field1, _value1, _field2, _value2, _field3, _value3,       \
1173             _field4, _value4, _field5, _value5, _field6, _value6)
1174
1175 #define EFX_POPULATE_BYTE_5(_byte,                                      \
1176             _field1, _value1, _field2, _value2, _field3, _value3,       \
1177             _field4, _value4, _field5, _value5)                         \
1178         EFX_POPULATE_BYTE_6(_byte, EFX_DUMMY_FIELD, 0,                  \
1179             _field1, _value1, _field2, _value2, _field3, _value3,       \
1180             _field4, _value4, _field5, _value5)
1181
1182 #define EFX_POPULATE_BYTE_4(_byte,                                      \
1183             _field1, _value1, _field2, _value2, _field3, _value3,       \
1184             _field4, _value4)                                           \
1185         EFX_POPULATE_BYTE_5(_byte, EFX_DUMMY_FIELD, 0,                  \
1186             _field1, _value1, _field2, _value2, _field3, _value3,       \
1187             _field4, _value4)
1188
1189 #define EFX_POPULATE_BYTE_3(_byte,                                      \
1190             _field1, _value1, _field2, _value2, _field3, _value3)       \
1191         EFX_POPULATE_BYTE_4(_byte, EFX_DUMMY_FIELD, 0,                  \
1192             _field1, _value1, _field2, _value2, _field3, _value3)
1193
1194 #define EFX_POPULATE_BYTE_2(_byte,                                      \
1195             _field1, _value1, _field2, _value2)                         \
1196         EFX_POPULATE_BYTE_3(_byte, EFX_DUMMY_FIELD, 0,                  \
1197             _field1, _value1, _field2, _value2)
1198
1199 #define EFX_POPULATE_BYTE_1(_byte,                                      \
1200             _field1, _value1)                                           \
1201         EFX_POPULATE_BYTE_2(_byte, EFX_DUMMY_FIELD, 0,                  \
1202             _field1, _value1)
1203
1204 #define EFX_ZERO_BYTE(_byte)                                            \
1205         EFX_POPULATE_BYTE_1(_byte, EFX_DUMMY_FIELD, 0)
1206
1207 #define EFX_SET_BYTE(_byte)                                             \
1208         EFX_POPULATE_BYTE_1(_byte,                                      \
1209             EFX_BYTE_0, 0xff)
1210
1211 /*
1212  * Modify a named field within an already-populated structure.  Used
1213  * for read-modify-write operations.
1214  */
1215
1216 #define EFX_INSERT_FIELD64(_min, _max, _field, _value)                  \
1217         __CPU_TO_LE_64(EFX_INSERT_FIELD_NATIVE64(_min, _max, _field, _value))
1218
1219 #define EFX_INSERT_FIELD32(_min, _max, _field, _value)                  \
1220         __CPU_TO_LE_32(EFX_INSERT_FIELD_NATIVE32(_min, _max, _field, _value))
1221
1222 #define EFX_INSERT_FIELD16(_min, _max, _field, _value)                  \
1223         __CPU_TO_LE_16(EFX_INSERT_FIELD_NATIVE16(_min, _max, _field, _value))
1224
1225 #define EFX_INSERT_FIELD8(_min, _max, _field, _value)                   \
1226         __NATIVE_8(EFX_INSERT_FIELD_NATIVE8(_min, _max, _field, _value))
1227
1228 #define EFX_INPLACE_MASK64(_min, _max, _field)                          \
1229         EFX_INSERT_FIELD64(_min, _max, _field, EFX_MASK64(_field))
1230
1231 #define EFX_INPLACE_MASK32(_min, _max, _field)                          \
1232         EFX_INSERT_FIELD32(_min, _max, _field, EFX_MASK32(_field))
1233
1234 #define EFX_INPLACE_MASK16(_min, _max, _field)                          \
1235         EFX_INSERT_FIELD16(_min, _max, _field, EFX_MASK16(_field))
1236
1237 #define EFX_INPLACE_MASK8(_min, _max, _field)                           \
1238         EFX_INSERT_FIELD8(_min, _max, _field, EFX_MASK8(_field))
1239
1240 #define EFX_SET_OWORD_FIELD64(_oword, _field, _value)                   \
1241         do {                                                            \
1242                 _NOTE(CONSTANTCONDITION)                                \
1243                 (_oword).eo_u64[0] = (((_oword).eo_u64[0] &             \
1244                     ~EFX_INPLACE_MASK64(0, 63, _field)) |               \
1245                     EFX_INSERT_FIELD64(0, 63, _field, _value));         \
1246                 _NOTE(CONSTANTCONDITION)                                \
1247                 (_oword).eo_u64[1] = (((_oword).eo_u64[1] &             \
1248                     ~EFX_INPLACE_MASK64(64, 127, _field)) |             \
1249                     EFX_INSERT_FIELD64(64, 127, _field, _value));       \
1250         _NOTE(CONSTANTCONDITION)                                        \
1251         } while (B_FALSE)
1252
1253 #define EFX_SET_OWORD_FIELD32(_oword, _field, _value)                   \
1254         do {                                                            \
1255                 _NOTE(CONSTANTCONDITION)                                \
1256                 (_oword).eo_u32[0] = (((_oword).eo_u32[0] &             \
1257                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1258                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1259                 _NOTE(CONSTANTCONDITION)                                \
1260                 (_oword).eo_u32[1] = (((_oword).eo_u32[1] &             \
1261                     ~EFX_INPLACE_MASK32(32, 63, _field)) |              \
1262                     EFX_INSERT_FIELD32(32, 63, _field, _value));        \
1263                 _NOTE(CONSTANTCONDITION)                                \
1264                 (_oword).eo_u32[2] = (((_oword).eo_u32[2] &             \
1265                     ~EFX_INPLACE_MASK32(64, 95, _field)) |              \
1266                     EFX_INSERT_FIELD32(64, 95, _field, _value));        \
1267                 _NOTE(CONSTANTCONDITION)                                \
1268                 (_oword).eo_u32[3] = (((_oword).eo_u32[3] &             \
1269                     ~EFX_INPLACE_MASK32(96, 127, _field)) |             \
1270                     EFX_INSERT_FIELD32(96, 127, _field, _value));       \
1271         _NOTE(CONSTANTCONDITION)                                        \
1272         } while (B_FALSE)
1273
1274 #define EFX_SET_QWORD_FIELD64(_qword, _field, _value)                   \
1275         do {                                                            \
1276                 _NOTE(CONSTANTCONDITION)                                \
1277                 (_qword).eq_u64[0] = (((_qword).eq_u64[0] &             \
1278                     ~EFX_INPLACE_MASK64(0, 63, _field)) |               \
1279                     EFX_INSERT_FIELD64(0, 63, _field, _value));         \
1280         _NOTE(CONSTANTCONDITION)                                        \
1281         } while (B_FALSE)
1282
1283 #define EFX_SET_QWORD_FIELD32(_qword, _field, _value)                   \
1284         do {                                                            \
1285                 _NOTE(CONSTANTCONDITION)                                \
1286                 (_qword).eq_u32[0] = (((_qword).eq_u32[0] &             \
1287                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1288                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1289                 _NOTE(CONSTANTCONDITION)                                \
1290                 (_qword).eq_u32[1] = (((_qword).eq_u32[1] &             \
1291                     ~EFX_INPLACE_MASK32(32, 63, _field)) |              \
1292                     EFX_INSERT_FIELD32(32, 63, _field, _value));        \
1293         _NOTE(CONSTANTCONDITION)                                        \
1294         } while (B_FALSE)
1295
1296 #define EFX_SET_DWORD_FIELD(_dword, _field, _value)                     \
1297         do {                                                            \
1298                 _NOTE(CONSTANTCONDITION)                                \
1299                 (_dword).ed_u32[0] = (((_dword).ed_u32[0] &             \
1300                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1301                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1302         _NOTE(CONSTANTCONDITION)                                        \
1303         } while (B_FALSE)
1304
1305 #define EFX_SET_WORD_FIELD(_word, _field, _value)                       \
1306         do {                                                            \
1307                 _NOTE(CONSTANTCONDITION)                                \
1308                 (_word).ew_u16[0] = (((_word).ew_u16[0] &               \
1309                     ~EFX_INPLACE_MASK16(0, 15, _field)) |               \
1310                     EFX_INSERT_FIELD16(0, 15, _field, _value));         \
1311         _NOTE(CONSTANTCONDITION)                                        \
1312         } while (B_FALSE)
1313
1314 #define EFX_SET_BYTE_FIELD(_byte, _field, _value)                       \
1315         do {                                                            \
1316                 _NOTE(CONSTANTCONDITION)                                \
1317                 (_byte).eb_u8[0] = (((_byte).eb_u8[0] &                 \
1318                     ~EFX_INPLACE_MASK8(0, 7, _field)) |                 \
1319                     EFX_INSERT_FIELD8(0, 7, _field, _value));           \
1320         _NOTE(CONSTANTCONDITION)                                        \
1321         } while (B_FALSE)
1322
1323 /*
1324  * Set or clear a numbered bit within an octword.
1325  */
1326
1327 #define EFX_SHIFT64(_bit, _base)                                        \
1328         (((_bit) >= (_base) && (_bit) < (_base) + 64) ?                 \
1329                 ((uint64_t)1 << EFX_SSUB((_bit), (_base))) :            \
1330                 0U)
1331
1332 #define EFX_SHIFT32(_bit, _base)                                        \
1333         (((_bit) >= (_base) && (_bit) < (_base) + 32) ?                 \
1334                 ((uint32_t)1 << EFX_SSUB((_bit),(_base))) :             \
1335                 0U)
1336
1337 #define EFX_SHIFT16(_bit, _base)                                        \
1338         (((_bit) >= (_base) && (_bit) < (_base) + 16) ?                 \
1339                 (uint16_t)(1 << EFX_SSUB((_bit), (_base))) :            \
1340                 0U)
1341
1342 #define EFX_SHIFT8(_bit, _base)                                         \
1343         (((_bit) >= (_base) && (_bit) < (_base) + 8) ?                  \
1344                 (uint8_t)(1 << EFX_SSUB((_bit), (_base))) :             \
1345                 0U)
1346
1347 #define EFX_SET_OWORD_BIT64(_oword, _bit)                               \
1348         do {                                                            \
1349                 _NOTE(CONSTANTCONDITION)                                \
1350                 (_oword).eo_u64[0] |=                                   \
1351                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)));     \
1352                 (_oword).eo_u64[1] |=                                   \
1353                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(64)));    \
1354         _NOTE(CONSTANTCONDITION)                                        \
1355         } while (B_FALSE)
1356
1357 #define EFX_SET_OWORD_BIT32(_oword, _bit)                               \
1358         do {                                                            \
1359                 _NOTE(CONSTANTCONDITION)                                \
1360                 (_oword).eo_u32[0] |=                                   \
1361                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1362                 (_oword).eo_u32[1] |=                                   \
1363                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)));    \
1364                 (_oword).eo_u32[2] |=                                   \
1365                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(64)));    \
1366                 (_oword).eo_u32[3] |=                                   \
1367                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(96)));    \
1368         _NOTE(CONSTANTCONDITION)                                        \
1369         } while (B_FALSE)
1370
1371 #define EFX_CLEAR_OWORD_BIT64(_oword, _bit)                             \
1372         do {                                                            \
1373                 _NOTE(CONSTANTCONDITION)                                \
1374                 (_oword).eo_u64[0] &=                                   \
1375                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(0)));    \
1376                 (_oword).eo_u64[1] &=                                   \
1377                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(64)));   \
1378         _NOTE(CONSTANTCONDITION)                                        \
1379         } while (B_FALSE)
1380
1381 #define EFX_CLEAR_OWORD_BIT32(_oword, _bit)                             \
1382         do {                                                            \
1383                 _NOTE(CONSTANTCONDITION)                                \
1384                 (_oword).eo_u32[0] &=                                   \
1385                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1386                 (_oword).eo_u32[1] &=                                   \
1387                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(32)));   \
1388                 (_oword).eo_u32[2] &=                                   \
1389                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(64)));   \
1390                 (_oword).eo_u32[3] &=                                   \
1391                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(96)));   \
1392         _NOTE(CONSTANTCONDITION)                                        \
1393         } while (B_FALSE)
1394
1395 #define EFX_TEST_OWORD_BIT64(_oword, _bit)                              \
1396         (((_oword).eo_u64[0] &                                          \
1397                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)))) ||  \
1398         ((_oword).eo_u64[1] &                                           \
1399                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(64)))))
1400
1401 #define EFX_TEST_OWORD_BIT32(_oword, _bit)                              \
1402         (((_oword).eo_u32[0] &                                          \
1403                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) ||  \
1404         ((_oword).eo_u32[1] &                                           \
1405                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)))) || \
1406         ((_oword).eo_u32[2] &                                           \
1407                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(64)))) || \
1408         ((_oword).eo_u32[3] &                                           \
1409                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(96)))))
1410
1411
1412 #define EFX_SET_QWORD_BIT64(_qword, _bit)                               \
1413         do {                                                            \
1414                 _NOTE(CONSTANTCONDITION)                                \
1415                 (_qword).eq_u64[0] |=                                   \
1416                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)));     \
1417         _NOTE(CONSTANTCONDITION)                                        \
1418         } while (B_FALSE)
1419
1420 #define EFX_SET_QWORD_BIT32(_qword, _bit)                               \
1421         do {                                                            \
1422                 _NOTE(CONSTANTCONDITION)                                \
1423                 (_qword).eq_u32[0] |=                                   \
1424                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1425                 (_qword).eq_u32[1] |=                                   \
1426                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)));    \
1427         _NOTE(CONSTANTCONDITION)                                        \
1428         } while (B_FALSE)
1429
1430 #define EFX_CLEAR_QWORD_BIT64(_qword, _bit)                             \
1431         do {                                                            \
1432                 _NOTE(CONSTANTCONDITION)                                \
1433                 (_qword).eq_u64[0] &=                                   \
1434                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(0)));    \
1435         _NOTE(CONSTANTCONDITION)                                        \
1436         } while (B_FALSE)
1437
1438 #define EFX_CLEAR_QWORD_BIT32(_qword, _bit)                             \
1439         do {                                                            \
1440                 _NOTE(CONSTANTCONDITION)                                \
1441                 (_qword).eq_u32[0] &=                                   \
1442                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1443                 (_qword).eq_u32[1] &=                                   \
1444                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(32)));   \
1445         _NOTE(CONSTANTCONDITION)                                        \
1446         } while (B_FALSE)
1447
1448 #define EFX_TEST_QWORD_BIT64(_qword, _bit)                              \
1449         (((_qword).eq_u64[0] &                                          \
1450                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)))) != 0)
1451
1452 #define EFX_TEST_QWORD_BIT32(_qword, _bit)                              \
1453         (((_qword).eq_u32[0] &                                          \
1454                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) ||  \
1455         ((_qword).eq_u32[1] &                                           \
1456                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)))))
1457
1458
1459 #define EFX_SET_DWORD_BIT(_dword, _bit)                                 \
1460         do {                                                            \
1461                 (_dword).ed_u32[0] |=                                   \
1462                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1463         _NOTE(CONSTANTCONDITION)                                        \
1464         } while (B_FALSE)
1465
1466 #define EFX_CLEAR_DWORD_BIT(_dword, _bit)                               \
1467         do {                                                            \
1468                 (_dword).ed_u32[0] &=                                   \
1469                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1470         _NOTE(CONSTANTCONDITION)                                        \
1471         } while (B_FALSE)
1472
1473 #define EFX_TEST_DWORD_BIT(_dword, _bit)                                \
1474         (((_dword).ed_u32[0] &                                          \
1475                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) != 0)
1476
1477
1478 #define EFX_SET_WORD_BIT(_word, _bit)                                   \
1479         do {                                                            \
1480                 (_word).ew_u16[0] |=                                    \
1481                     __CPU_TO_LE_16(EFX_SHIFT16(_bit, FIX_LINT(0)));     \
1482         _NOTE(CONSTANTCONDITION)                                        \
1483         } while (B_FALSE)
1484
1485 #define EFX_CLEAR_WORD_BIT(_word, _bit)                                 \
1486         do {                                                            \
1487                 (_word).ew_u32[0] &=                                    \
1488                     __CPU_TO_LE_16(~EFX_SHIFT16(_bit, FIX_LINT(0)));    \
1489         _NOTE(CONSTANTCONDITION)                                        \
1490         } while (B_FALSE)
1491
1492 #define EFX_TEST_WORD_BIT(_word, _bit)                                  \
1493         (((_word).ew_u16[0] &                                           \
1494                     __CPU_TO_LE_16(EFX_SHIFT16(_bit, FIX_LINT(0)))) != 0)
1495
1496
1497 #define EFX_SET_BYTE_BIT(_byte, _bit)                                   \
1498         do {                                                            \
1499                 (_byte).eb_u8[0] |=                                     \
1500                     __NATIVE_8(EFX_SHIFT8(_bit, FIX_LINT(0)));          \
1501         _NOTE(CONSTANTCONDITION)                                        \
1502         } while (B_FALSE)
1503
1504 #define EFX_CLEAR_BYTE_BIT(_byte, _bit)                                 \
1505         do {                                                            \
1506                 (_byte).eb_u8[0] &=                                     \
1507                     __NATIVE_8(~EFX_SHIFT8(_bit, FIX_LINT(0)));         \
1508         _NOTE(CONSTANTCONDITION)                                        \
1509         } while (B_FALSE)
1510
1511 #define EFX_TEST_BYTE_BIT(_byte, _bit)                                  \
1512         (((_byte).eb_u8[0] &                                            \
1513                     __NATIVE_8(EFX_SHIFT8(_bit, FIX_LINT(0)))) != 0)
1514
1515
1516 #define EFX_OR_OWORD64(_oword1, _oword2)                                \
1517         do {                                                            \
1518                 (_oword1).eo_u64[0] |= (_oword2).eo_u64[0];             \
1519                 (_oword1).eo_u64[1] |= (_oword2).eo_u64[1];             \
1520         _NOTE(CONSTANTCONDITION)                                        \
1521         } while (B_FALSE)
1522
1523 #define EFX_OR_OWORD32(_oword1, _oword2)                                \
1524         do {                                                            \
1525                 (_oword1).eo_u32[0] |= (_oword2).eo_u32[0];             \
1526                 (_oword1).eo_u32[1] |= (_oword2).eo_u32[1];             \
1527                 (_oword1).eo_u32[2] |= (_oword2).eo_u32[2];             \
1528                 (_oword1).eo_u32[3] |= (_oword2).eo_u32[3];             \
1529         _NOTE(CONSTANTCONDITION)                                        \
1530         } while (B_FALSE)
1531
1532 #define EFX_AND_OWORD64(_oword1, _oword2)                               \
1533         do {                                                            \
1534                 (_oword1).eo_u64[0] &= (_oword2).eo_u64[0];             \
1535                 (_oword1).eo_u64[1] &= (_oword2).eo_u64[1];             \
1536         _NOTE(CONSTANTCONDITION)                                        \
1537         } while (B_FALSE)
1538
1539 #define EFX_AND_OWORD32(_oword1, _oword2)                               \
1540         do {                                                            \
1541                 (_oword1).eo_u32[0] &= (_oword2).eo_u32[0];             \
1542                 (_oword1).eo_u32[1] &= (_oword2).eo_u32[1];             \
1543                 (_oword1).eo_u32[2] &= (_oword2).eo_u32[2];             \
1544                 (_oword1).eo_u32[3] &= (_oword2).eo_u32[3];             \
1545         _NOTE(CONSTANTCONDITION)                                        \
1546         } while (B_FALSE)
1547
1548 #define EFX_OR_QWORD64(_qword1, _qword2)                                \
1549         do {                                                            \
1550                 (_qword1).eq_u64[0] |= (_qword2).eq_u64[0];             \
1551         _NOTE(CONSTANTCONDITION)                                        \
1552         } while (B_FALSE)
1553
1554 #define EFX_OR_QWORD32(_qword1, _qword2)                                \
1555         do {                                                            \
1556                 (_qword1).eq_u32[0] |= (_qword2).eq_u32[0];             \
1557                 (_qword1).eq_u32[1] |= (_qword2).eq_u32[1];             \
1558         _NOTE(CONSTANTCONDITION)                                        \
1559         } while (B_FALSE)
1560
1561 #define EFX_AND_QWORD64(_qword1, _qword2)                               \
1562         do {                                                            \
1563                 (_qword1).eq_u64[0] &= (_qword2).eq_u64[0];             \
1564         _NOTE(CONSTANTCONDITION)                                        \
1565         } while (B_FALSE)
1566
1567 #define EFX_AND_QWORD32(_qword1, _qword2)                               \
1568         do {                                                            \
1569                 (_qword1).eq_u32[0] &= (_qword2).eq_u32[0];             \
1570                 (_qword1).eq_u32[1] &= (_qword2).eq_u32[1];             \
1571         _NOTE(CONSTANTCONDITION)                                        \
1572         } while (B_FALSE)
1573
1574 #define EFX_OR_DWORD(_dword1, _dword2)                                  \
1575         do {                                                            \
1576                 (_dword1).ed_u32[0] |= (_dword2).ed_u32[0];             \
1577         _NOTE(CONSTANTCONDITION)                                        \
1578         } while (B_FALSE)
1579
1580 #define EFX_AND_DWORD(_dword1, _dword2)                                 \
1581         do {                                                            \
1582                 (_dword1).ed_u32[0] &= (_dword2).ed_u32[0];             \
1583         _NOTE(CONSTANTCONDITION)                                        \
1584         } while (B_FALSE)
1585
1586 #define EFX_OR_WORD(_word1, _word2)                                     \
1587         do {                                                            \
1588                 (_word1).ew_u16[0] |= (_word2).ew_u16[0];               \
1589         _NOTE(CONSTANTCONDITION)                                        \
1590         } while (B_FALSE)
1591
1592 #define EFX_AND_WORD(_word1, _word2)                                    \
1593         do {                                                            \
1594                 (_word1).ew_u16[0] &= (_word2).ew_u16[0];               \
1595         _NOTE(CONSTANTCONDITION)                                        \
1596         } while (B_FALSE)
1597
1598 #define EFX_OR_BYTE(_byte1, _byte2)                                     \
1599         do {                                                            \
1600                 (_byte1).eb_u8[0] |= (_byte2).eb_u8[0];                 \
1601         _NOTE(CONSTANTCONDITION)                                        \
1602         } while (B_FALSE)
1603
1604 #define EFX_AND_BYTE(_byte1, _byte2)                                    \
1605         do {                                                            \
1606                 (_byte1).eb_u8[0] &= (_byte2).eb_u8[0];                 \
1607         _NOTE(CONSTANTCONDITION)                                        \
1608         } while (B_FALSE)
1609
1610 #if EFSYS_USE_UINT64
1611 #define EFX_OWORD_FIELD         EFX_OWORD_FIELD64
1612 #define EFX_QWORD_FIELD         EFX_QWORD_FIELD64
1613 #define EFX_OWORD_IS_EQUAL      EFX_OWORD_IS_EQUAL64
1614 #define EFX_QWORD_IS_EQUAL      EFX_QWORD_IS_EQUAL64
1615 #define EFX_OWORD_IS_ZERO       EFX_OWORD_IS_ZERO64
1616 #define EFX_QWORD_IS_ZERO       EFX_QWORD_IS_ZERO64
1617 #define EFX_OWORD_IS_SET        EFX_OWORD_IS_SET64
1618 #define EFX_QWORD_IS_SET        EFX_QWORD_IS_SET64
1619 #define EFX_POPULATE_OWORD      EFX_POPULATE_OWORD64
1620 #define EFX_POPULATE_QWORD      EFX_POPULATE_QWORD64
1621 #define EFX_SET_OWORD_FIELD     EFX_SET_OWORD_FIELD64
1622 #define EFX_SET_QWORD_FIELD     EFX_SET_QWORD_FIELD64
1623 #define EFX_SET_OWORD_BIT       EFX_SET_OWORD_BIT64
1624 #define EFX_CLEAR_OWORD_BIT     EFX_CLEAR_OWORD_BIT64
1625 #define EFX_TEST_OWORD_BIT      EFX_TEST_OWORD_BIT64
1626 #define EFX_SET_QWORD_BIT       EFX_SET_QWORD_BIT64
1627 #define EFX_CLEAR_QWORD_BIT     EFX_CLEAR_QWORD_BIT64
1628 #define EFX_TEST_QWORD_BIT      EFX_TEST_QWORD_BIT64
1629 #define EFX_OR_OWORD            EFX_OR_OWORD64
1630 #define EFX_AND_OWORD           EFX_AND_OWORD64
1631 #define EFX_OR_QWORD            EFX_OR_QWORD64
1632 #define EFX_AND_QWORD           EFX_AND_QWORD64
1633 #else
1634 #define EFX_OWORD_FIELD         EFX_OWORD_FIELD32
1635 #define EFX_QWORD_FIELD         EFX_QWORD_FIELD32
1636 #define EFX_OWORD_IS_EQUAL      EFX_OWORD_IS_EQUAL32
1637 #define EFX_QWORD_IS_EQUAL      EFX_QWORD_IS_EQUAL32
1638 #define EFX_OWORD_IS_ZERO       EFX_OWORD_IS_ZERO32
1639 #define EFX_QWORD_IS_ZERO       EFX_QWORD_IS_ZERO32
1640 #define EFX_OWORD_IS_SET        EFX_OWORD_IS_SET32
1641 #define EFX_QWORD_IS_SET        EFX_QWORD_IS_SET32
1642 #define EFX_POPULATE_OWORD      EFX_POPULATE_OWORD32
1643 #define EFX_POPULATE_QWORD      EFX_POPULATE_QWORD32
1644 #define EFX_SET_OWORD_FIELD     EFX_SET_OWORD_FIELD32
1645 #define EFX_SET_QWORD_FIELD     EFX_SET_QWORD_FIELD32
1646 #define EFX_SET_OWORD_BIT       EFX_SET_OWORD_BIT32
1647 #define EFX_CLEAR_OWORD_BIT     EFX_CLEAR_OWORD_BIT32
1648 #define EFX_TEST_OWORD_BIT      EFX_TEST_OWORD_BIT32
1649 #define EFX_SET_QWORD_BIT       EFX_SET_QWORD_BIT32
1650 #define EFX_CLEAR_QWORD_BIT     EFX_CLEAR_QWORD_BIT32
1651 #define EFX_TEST_QWORD_BIT      EFX_TEST_QWORD_BIT32
1652 #define EFX_OR_OWORD            EFX_OR_OWORD32
1653 #define EFX_AND_OWORD           EFX_AND_OWORD32
1654 #define EFX_OR_QWORD            EFX_OR_QWORD32
1655 #define EFX_AND_QWORD           EFX_AND_QWORD32
1656 #endif
1657
1658 #ifdef  __cplusplus
1659 }
1660 #endif
1661
1662 #endif  /* _SYS_EFX_TYPES_H */