sys/dev/sfxge/common/efx_types.h

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2007-2016 Solarflare Communications Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions are met:
   9  *
  10  * 1. Redistributions of source code must retain the above copyright notice,
  11  *    this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright notice,
  13  *    this list of conditions and the following disclaimer in the documentation
  14  *    and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  18  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  20  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  21  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  22  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  23  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  24  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  25  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  26  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  *
  28  * The views and conclusions contained in the software and documentation are
  29  * those of the authors and should not be interpreted as representing official
  30  * policies, either expressed or implied, of the FreeBSD Project.
  31  *
  32  * Ackowledgement to Fen Systems Ltd.
  33  *
  34  * $FreeBSD$
  35  */
  36
  37 #ifndef _SYS_EFX_TYPES_H
  38 #define _SYS_EFX_TYPES_H
  39
  40 #include "efsys.h"
  41
  42 #ifdef  __cplusplus
  43 extern "C" {
  44 #endif
  45
  46 /*
  47  * Bitfield access
  48  *
  49  * Solarflare NICs make extensive use of bitfields up to 128 bits
  50  * wide.  Since there is no native 128-bit datatype on most systems,
  51  * and since 64-bit datatypes are inefficient on 32-bit systems and
  52  * vice versa, we wrap accesses in a way that uses the most efficient
  53  * datatype.
  54  *
  55  * The NICs are PCI devices and therefore little-endian.  Since most
  56  * of the quantities that we deal with are DMAed to/from host memory,
  57  * we define    our datatypes (efx_oword_t, efx_qword_t and efx_dword_t)
  58  * to be little-endian.
  59  *
  60  * In the less common case of using PIO for individual register
  61  * writes, we construct the little-endian datatype in host memory and
  62  * then use non-swapping register access primitives, rather than
  63  * constructing a native-endian datatype and relying on implicit
  64  * byte-swapping.  (We use a similar strategy for register reads.)
  65  */
  66
  67 /*
  68  * NOTE: Field definitions here and elsewhere are done in terms of a lowest
  69  *       bit number (LBN) and a width.
  70  */
  71
  72 #define EFX_DUMMY_FIELD_LBN 0
  73 #define EFX_DUMMY_FIELD_WIDTH 0
  74
  75 #define EFX_BYTE_0_LBN 0
  76 #define EFX_BYTE_0_WIDTH 8
  77
  78 #define EFX_BYTE_1_LBN 8
  79 #define EFX_BYTE_1_WIDTH 8
  80
  81 #define EFX_BYTE_2_LBN 16
  82 #define EFX_BYTE_2_WIDTH 8
  83
  84 #define EFX_BYTE_3_LBN 24
  85 #define EFX_BYTE_3_WIDTH 8
  86
  87 #define EFX_BYTE_4_LBN 32
  88 #define EFX_BYTE_4_WIDTH 8
  89
  90 #define EFX_BYTE_5_LBN 40
  91 #define EFX_BYTE_5_WIDTH 8
  92
  93 #define EFX_BYTE_6_LBN 48
  94 #define EFX_BYTE_6_WIDTH 8
  95
  96 #define EFX_BYTE_7_LBN 56
  97 #define EFX_BYTE_7_WIDTH 8
  98
  99 #define EFX_WORD_0_LBN 0
 100 #define EFX_WORD_0_WIDTH 16
 101
 102 #define EFX_WORD_1_LBN 16
 103 #define EFX_WORD_1_WIDTH 16
 104
 105 #define EFX_WORD_2_LBN 32
 106 #define EFX_WORD_2_WIDTH 16
 107
 108 #define EFX_WORD_3_LBN 48
 109 #define EFX_WORD_3_WIDTH 16
 110
 111 #define EFX_DWORD_0_LBN 0
 112 #define EFX_DWORD_0_WIDTH 32
 113
 114 #define EFX_DWORD_1_LBN 32
 115 #define EFX_DWORD_1_WIDTH 32
 116
 117 #define EFX_DWORD_2_LBN 64
 118 #define EFX_DWORD_2_WIDTH 32
 119
 120 #define EFX_DWORD_3_LBN 96
 121 #define EFX_DWORD_3_WIDTH 32
 122
 123 /* There are intentionally no EFX_QWORD_0 or EFX_QWORD_1 field definitions
 124  * here as the implementaion of EFX_QWORD_FIELD and EFX_OWORD_FIELD do not
 125  * support field widths larger than 32 bits.
 126  */
 127
 128 /* Specified attribute (i.e. LBN ow WIDTH) of the specified field */
 129 #define EFX_VAL(_field, _attribute)                                     \
 130         _field ## _ ## _attribute
 131
 132 /* Lowest bit number of the specified field */
 133 #define EFX_LOW_BIT(_field)                                             \
 134         EFX_VAL(_field, LBN)
 135
 136 /* Width of the specified field */
 137 #define EFX_WIDTH(_field)                                               \
 138         EFX_VAL(_field, WIDTH)
 139
 140 /* Highest bit number of the specified field */
 141 #define EFX_HIGH_BIT(_field)                                            \
 142         (EFX_LOW_BIT(_field) + EFX_WIDTH(_field) - 1)
 143
 144 /*
 145  * 64-bit mask equal in width to the specified field.
 146  *
 147  * For example, a field with width 5 would have a mask of 0x000000000000001f.
 148  */
 149 #define EFX_MASK64(_field)                                              \
 150         ((EFX_WIDTH(_field) == 64) ? ~((uint64_t)0) :                   \
 151             (((((uint64_t)1) << EFX_WIDTH(_field))) - 1))
 152 /*
 153  * 32-bit mask equal in width to the specified field.
 154  *
 155  * For example, a field with width 5 would have a mask of 0x0000001f.
 156  */
 157 #define EFX_MASK32(_field)                                              \
 158         ((EFX_WIDTH(_field) == 32) ? ~((uint32_t)0) :                   \
 159             (((((uint32_t)1) << EFX_WIDTH(_field))) - 1))
 160
 161 /*
 162  * 16-bit mask equal in width to the specified field.
 163  *
 164  * For example, a field with width 5 would have a mask of 0x001f.
 165  */
 166 #define EFX_MASK16(_field)                                              \
 167         ((EFX_WIDTH(_field) == 16) ? 0xffffu :                          \
 168             (uint16_t)((1 << EFX_WIDTH(_field)) - 1))
 169
 170 /*
 171  * 8-bit mask equal in width to the specified field.
 172  *
 173  * For example, a field with width 5 would have a mask of 0x1f.
 174  */
 175 #define EFX_MASK8(_field)                                               \
 176         ((uint8_t)((1 << EFX_WIDTH(_field)) - 1))
 177
 178 #pragma pack(1)
 179
 180 /*
 181  * A byte (i.e. 8-bit) datatype
 182  */
 183 typedef union efx_byte_u {
 184         uint8_t eb_u8[1];
 185 } efx_byte_t;
 186
 187 /*
 188  * A word (i.e. 16-bit) datatype
 189  *
 190  * This datatype is defined to be little-endian.
 191  */
 192 typedef union efx_word_u {
 193         efx_byte_t ew_byte[2];
 194         uint16_t ew_u16[1];
 195         uint8_t ew_u8[2];
 196 } efx_word_t;
 197
 198 /*
 199  * A doubleword (i.e. 32-bit) datatype
 200  *
 201  * This datatype is defined to be little-endian.
 202  */
 203 typedef union efx_dword_u {
 204         efx_byte_t ed_byte[4];
 205         efx_word_t ed_word[2];
 206         uint32_t ed_u32[1];
 207         uint16_t ed_u16[2];
 208         uint8_t ed_u8[4];
 209 } efx_dword_t;
 210
 211 /*
 212  * A quadword (i.e. 64-bit) datatype
 213  *
 214  * This datatype is defined to be little-endian.
 215  */
 216 typedef union efx_qword_u {
 217         efx_byte_t eq_byte[8];
 218         efx_word_t eq_word[4];
 219         efx_dword_t eq_dword[2];
 220 #if EFSYS_HAS_UINT64
 221         uint64_t eq_u64[1];
 222 #endif
 223         uint32_t eq_u32[2];
 224         uint16_t eq_u16[4];
 225         uint8_t eq_u8[8];
 226 } efx_qword_t;
 227
 228 /*
 229  * An octword (i.e. 128-bit) datatype
 230  *
 231  * This datatype is defined to be little-endian.
 232  */
 233 typedef union efx_oword_u {
 234         efx_byte_t eo_byte[16];
 235         efx_word_t eo_word[8];
 236         efx_dword_t eo_dword[4];
 237         efx_qword_t eo_qword[2];
 238 #if EFSYS_HAS_SSE2_M128
 239         __m128i eo_u128[1];
 240 #endif
 241 #if EFSYS_HAS_UINT64
 242         uint64_t eo_u64[2];
 243 #endif
 244         uint32_t eo_u32[4];
 245         uint16_t eo_u16[8];
 246         uint8_t eo_u8[16];
 247 } efx_oword_t;
 248
 249 #pragma pack()
 250
 251 #define __SWAP16(_x)                            \
 252         ((((_x) & 0xff) << 8) |                 \
 253         (((_x) >> 8) & 0xff))
 254
 255 #define __SWAP32(_x)                            \
 256         ((__SWAP16((_x) & 0xffff) << 16) |      \
 257         __SWAP16(((_x) >> 16) & 0xffff))
 258
 259 #define __SWAP64(_x)                            \
 260         ((__SWAP32((_x) & 0xffffffff) << 32) |  \
 261         __SWAP32(((_x) >> 32) & 0xffffffff))
 262
 263 #define __NOSWAP16(_x)          (_x)
 264 #define __NOSWAP32(_x)          (_x)
 265 #define __NOSWAP64(_x)          (_x)
 266
 267 #if EFSYS_IS_BIG_ENDIAN
 268
 269 #define __CPU_TO_LE_16(_x)      ((uint16_t)__SWAP16(_x))
 270 #define __LE_TO_CPU_16(_x)      ((uint16_t)__SWAP16(_x))
 271 #define __CPU_TO_BE_16(_x)      ((uint16_t)__NOSWAP16(_x))
 272 #define __BE_TO_CPU_16(_x)      ((uint16_t)__NOSWAP16(_x))
 273
 274 #define __CPU_TO_LE_32(_x)      ((uint32_t)__SWAP32(_x))
 275 #define __LE_TO_CPU_32(_x)      ((uint32_t)__SWAP32(_x))
 276 #define __CPU_TO_BE_32(_x)      ((uint32_t)__NOSWAP32(_x))
 277 #define __BE_TO_CPU_32(_x)      ((uint32_t)__NOSWAP32(_x))
 278
 279 #define __CPU_TO_LE_64(_x)      ((uint64_t)__SWAP64(_x))
 280 #define __LE_TO_CPU_64(_x)      ((uint64_t)__SWAP64(_x))
 281 #define __CPU_TO_BE_64(_x)      ((uint64_t)__NOSWAP64(_x))
 282 #define __BE_TO_CPU_64(_x)      ((uint64_t)__NOSWAP64(_x))
 283
 284 #elif EFSYS_IS_LITTLE_ENDIAN
 285
 286 #define __CPU_TO_LE_16(_x)      ((uint16_t)__NOSWAP16(_x))
 287 #define __LE_TO_CPU_16(_x)      ((uint16_t)__NOSWAP16(_x))
 288 #define __CPU_TO_BE_16(_x)      ((uint16_t)__SWAP16(_x))
 289 #define __BE_TO_CPU_16(_x)      ((uint16_t)__SWAP16(_x))
 290
 291 #define __CPU_TO_LE_32(_x)      ((uint32_t)__NOSWAP32(_x))
 292 #define __LE_TO_CPU_32(_x)      ((uint32_t)__NOSWAP32(_x))
 293 #define __CPU_TO_BE_32(_x)      ((uint32_t)__SWAP32(_x))
 294 #define __BE_TO_CPU_32(_x)      ((uint32_t)__SWAP32(_x))
 295
 296 #define __CPU_TO_LE_64(_x)      ((uint64_t)__NOSWAP64(_x))
 297 #define __LE_TO_CPU_64(_x)      ((uint64_t)__NOSWAP64(_x))
 298 #define __CPU_TO_BE_64(_x)      ((uint64_t)__SWAP64(_x))
 299 #define __BE_TO_CPU_64(_x)      ((uint64_t)__SWAP64(_x))
 300
 301 #else
 302
 303 #error "Neither of EFSYS_IS_{BIG,LITTLE}_ENDIAN is set"
 304
 305 #endif
 306
 307 #define __NATIVE_8(_x)  (uint8_t)(_x)
 308
 309 /* Format string for printing an efx_byte_t */
 310 #define EFX_BYTE_FMT "0x%02x"
 311
 312 /* Format string for printing an efx_word_t */
 313 #define EFX_WORD_FMT "0x%04x"
 314
 315 /* Format string for printing an efx_dword_t */
 316 #define EFX_DWORD_FMT "0x%08x"
 317
 318 /* Format string for printing an efx_qword_t */
 319 #define EFX_QWORD_FMT "0x%08x:%08x"
 320
 321 /* Format string for printing an efx_oword_t */
 322 #define EFX_OWORD_FMT "0x%08x:%08x:%08x:%08x"
 323
 324 /* Parameters for printing an efx_byte_t */
 325 #define EFX_BYTE_VAL(_byte)                                     \
 326         ((unsigned int)__NATIVE_8((_byte).eb_u8[0]))
 327
 328 /* Parameters for printing an efx_word_t */
 329 #define EFX_WORD_VAL(_word)                                     \
 330         ((unsigned int)__LE_TO_CPU_16((_word).ew_u16[0]))
 331
 332 /* Parameters for printing an efx_dword_t */
 333 #define EFX_DWORD_VAL(_dword)                                   \
 334         ((unsigned int)__LE_TO_CPU_32((_dword).ed_u32[0]))
 335
 336 /* Parameters for printing an efx_qword_t */
 337 #define EFX_QWORD_VAL(_qword)                                   \
 338         ((unsigned int)__LE_TO_CPU_32((_qword).eq_u32[1])),     \
 339         ((unsigned int)__LE_TO_CPU_32((_qword).eq_u32[0]))
 340
 341 /* Parameters for printing an efx_oword_t */
 342 #define EFX_OWORD_VAL(_oword)                                   \
 343         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[3])),     \
 344         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[2])),     \
 345         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[1])),     \
 346         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[0]))
 347
 348 /*
 349  * Stop lint complaining about some shifts.
 350  */
 351 #ifdef  __lint
 352 extern int fix_lint;
 353 #define FIX_LINT(_x)    (_x + fix_lint)
 354 #else
 355 #define FIX_LINT(_x)    (_x)
 356 #endif
 357
 358 /*
 359  * Extract bit field portion [low,high) from the native-endian element
 360  * which contains bits [min,max).
 361  *
 362  * For example, suppose "element" represents the high 32 bits of a
 363  * 64-bit value, and we wish to extract the bits belonging to the bit
 364  * field occupying bits 28-45 of this 64-bit value.
 365  *
 366  * Then EFX_EXTRACT(_element, 32, 63, 28, 45) would give
 367  *
 368  *   (_element) << 4
 369  *
 370  * The result will contain the relevant bits filled in in the range
 371  * [0,high-low), with garbage in bits [high-low+1,...).
 372  */
 373 #define EFX_EXTRACT_NATIVE(_element, _min, _max, _low, _high)           \
 374         ((FIX_LINT(_low > _max) || FIX_LINT(_high < _min)) ?            \
 375                 0U :                                                    \
 376                 ((_low > _min) ?                                        \
 377                         ((_element) >> (_low - _min)) :                 \
 378                         ((_element) << (_min - _low))))
 379
 380 /*
 381  * Extract bit field portion [low,high) from the 64-bit little-endian
 382  * element which contains bits [min,max)
 383  */
 384 #define EFX_EXTRACT64(_element, _min, _max, _low, _high)                \
 385         EFX_EXTRACT_NATIVE(__LE_TO_CPU_64(_element), _min, _max, _low, _high)
 386
 387 /*
 388  * Extract bit field portion [low,high) from the 32-bit little-endian
 389  * element which contains bits [min,max)
 390  */
 391 #define EFX_EXTRACT32(_element, _min, _max, _low, _high)                \
 392         EFX_EXTRACT_NATIVE(__LE_TO_CPU_32(_element), _min, _max, _low, _high)
 393
 394 /*
 395  * Extract bit field portion [low,high) from the 16-bit little-endian
 396  * element which contains bits [min,max)
 397  */
 398 #define EFX_EXTRACT16(_element, _min, _max, _low, _high)                \
 399         EFX_EXTRACT_NATIVE(__LE_TO_CPU_16(_element), _min, _max, _low, _high)
 400
 401 /*
 402  * Extract bit field portion [low,high) from the 8-bit
 403  * element which contains bits [min,max)
 404  */
 405 #define EFX_EXTRACT8(_element, _min, _max, _low, _high)                 \
 406         EFX_EXTRACT_NATIVE(__NATIVE_8(_element), _min, _max, _low, _high)
 407
 408 #define EFX_EXTRACT_OWORD64(_oword, _low, _high)                        \
 409         (EFX_EXTRACT64((_oword).eo_u64[0], FIX_LINT(0), FIX_LINT(63),   \
 410             _low, _high) |                                              \
 411         EFX_EXTRACT64((_oword).eo_u64[1], FIX_LINT(64), FIX_LINT(127),  \
 412             _low, _high))
 413
 414 #define EFX_EXTRACT_OWORD32(_oword, _low, _high)                        \
 415         (EFX_EXTRACT32((_oword).eo_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 416             _low, _high) |                                              \
 417         EFX_EXTRACT32((_oword).eo_u32[1], FIX_LINT(32), FIX_LINT(63),   \
 418             _low, _high) |                                              \
 419         EFX_EXTRACT32((_oword).eo_u32[2], FIX_LINT(64), FIX_LINT(95),   \
 420             _low, _high) |                                              \
 421         EFX_EXTRACT32((_oword).eo_u32[3], FIX_LINT(96), FIX_LINT(127),  \
 422             _low, _high))
 423
 424 #define EFX_EXTRACT_QWORD64(_qword, _low, _high)                        \
 425         (EFX_EXTRACT64((_qword).eq_u64[0], FIX_LINT(0), FIX_LINT(63),   \
 426             _low, _high))
 427
 428 #define EFX_EXTRACT_QWORD32(_qword, _low, _high)                        \
 429         (EFX_EXTRACT32((_qword).eq_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 430             _low, _high) |                                              \
 431         EFX_EXTRACT32((_qword).eq_u32[1], FIX_LINT(32), FIX_LINT(63),   \
 432             _low, _high))
 433
 434 #define EFX_EXTRACT_DWORD(_dword, _low, _high)                          \
 435         (EFX_EXTRACT32((_dword).ed_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 436             _low, _high))
 437
 438 #define EFX_EXTRACT_WORD(_word, _low, _high)                            \
 439         (EFX_EXTRACT16((_word).ew_u16[0], FIX_LINT(0), FIX_LINT(15),    \
 440             _low, _high))
 441
 442 #define EFX_EXTRACT_BYTE(_byte, _low, _high)                            \
 443         (EFX_EXTRACT8((_byte).eb_u8[0], FIX_LINT(0), FIX_LINT(7),       \
 444             _low, _high))
 445
 446
 447 #define EFX_OWORD_FIELD64(_oword, _field)                               \
 448         ((uint32_t)EFX_EXTRACT_OWORD64(_oword, EFX_LOW_BIT(_field),     \
 449             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 450
 451 #define EFX_OWORD_FIELD32(_oword, _field)                               \
 452         (EFX_EXTRACT_OWORD32(_oword, EFX_LOW_BIT(_field),               \
 453             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 454
 455 #define EFX_QWORD_FIELD64(_qword, _field)                               \
 456         ((uint32_t)EFX_EXTRACT_QWORD64(_qword, EFX_LOW_BIT(_field),     \
 457             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 458
 459 #define EFX_QWORD_FIELD32(_qword, _field)                               \
 460         (EFX_EXTRACT_QWORD32(_qword, EFX_LOW_BIT(_field),               \
 461             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 462
 463 #define EFX_DWORD_FIELD(_dword, _field)                                 \
 464         (EFX_EXTRACT_DWORD(_dword, EFX_LOW_BIT(_field),                 \
 465             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 466
 467 #define EFX_WORD_FIELD(_word, _field)                                   \
 468         (EFX_EXTRACT_WORD(_word, EFX_LOW_BIT(_field),                   \
 469             EFX_HIGH_BIT(_field)) & EFX_MASK16(_field))
 470
 471 #define EFX_BYTE_FIELD(_byte, _field)                                   \
 472         (EFX_EXTRACT_BYTE(_byte, EFX_LOW_BIT(_field),                   \
 473             EFX_HIGH_BIT(_field)) & EFX_MASK8(_field))
 474
 475
 476 #define EFX_OWORD_IS_EQUAL64(_oword_a, _oword_b)                        \
 477         ((_oword_a).eo_u64[0] == (_oword_b).eo_u64[0] &&                \
 478             (_oword_a).eo_u64[1] == (_oword_b).eo_u64[1])
 479
 480 #define EFX_OWORD_IS_EQUAL32(_oword_a, _oword_b)                        \
 481         ((_oword_a).eo_u32[0] == (_oword_b).eo_u32[0] &&                \
 482             (_oword_a).eo_u32[1] == (_oword_b).eo_u32[1] &&             \
 483             (_oword_a).eo_u32[2] == (_oword_b).eo_u32[2] &&             \
 484             (_oword_a).eo_u32[3] == (_oword_b).eo_u32[3])
 485
 486 #define EFX_QWORD_IS_EQUAL64(_qword_a, _qword_b)                        \
 487         ((_qword_a).eq_u64[0] == (_qword_b).eq_u64[0])
 488
 489 #define EFX_QWORD_IS_EQUAL32(_qword_a, _qword_b)                        \
 490         ((_qword_a).eq_u32[0] == (_qword_b).eq_u32[0] &&                \
 491             (_qword_a).eq_u32[1] == (_qword_b).eq_u32[1])
 492
 493 #define EFX_DWORD_IS_EQUAL(_dword_a, _dword_b)                          \
 494         ((_dword_a).ed_u32[0] == (_dword_b).ed_u32[0])
 495
 496 #define EFX_WORD_IS_EQUAL(_word_a, _word_b)                             \
 497         ((_word_a).ew_u16[0] == (_word_b).ew_u16[0])
 498
 499 #define EFX_BYTE_IS_EQUAL(_byte_a, _byte_b)                             \
 500         ((_byte_a).eb_u8[0] == (_byte_b).eb_u8[0])
 501
 502
 503 #define EFX_OWORD_IS_ZERO64(_oword)                                     \
 504         (((_oword).eo_u64[0] |                                          \
 505             (_oword).eo_u64[1]) == 0)
 506
 507 #define EFX_OWORD_IS_ZERO32(_oword)                                     \
 508         (((_oword).eo_u32[0] |                                          \
 509             (_oword).eo_u32[1] |                                        \
 510             (_oword).eo_u32[2] |                                        \
 511             (_oword).eo_u32[3]) == 0)
 512
 513 #define EFX_QWORD_IS_ZERO64(_qword)                                     \
 514         (((_qword).eq_u64[0]) == 0)
 515
 516 #define EFX_QWORD_IS_ZERO32(_qword)                                     \
 517         (((_qword).eq_u32[0] |                                          \
 518             (_qword).eq_u32[1]) == 0)
 519
 520 #define EFX_DWORD_IS_ZERO(_dword)                                       \
 521         (((_dword).ed_u32[0]) == 0)
 522
 523 #define EFX_WORD_IS_ZERO(_word)                                         \
 524         (((_word).ew_u16[0]) == 0)
 525
 526 #define EFX_BYTE_IS_ZERO(_byte)                                         \
 527         (((_byte).eb_u8[0]) == 0)
 528
 529
 530 #define EFX_OWORD_IS_SET64(_oword)                                      \
 531         (((_oword).eo_u64[0] &                                          \
 532             (_oword).eo_u64[1]) == ~((uint64_t)0))
 533
 534 #define EFX_OWORD_IS_SET32(_oword)                                      \
 535         (((_oword).eo_u32[0] &                                          \
 536             (_oword).eo_u32[1] &                                        \
 537             (_oword).eo_u32[2] &                                        \
 538             (_oword).eo_u32[3]) == ~((uint32_t)0))
 539
 540 #define EFX_QWORD_IS_SET64(_qword)                                      \
 541         (((_qword).eq_u64[0]) == ~((uint64_t)0))
 542
 543 #define EFX_QWORD_IS_SET32(_qword)                                      \
 544         (((_qword).eq_u32[0] &                                          \
 545             (_qword).eq_u32[1]) == ~((uint32_t)0))
 546
 547 #define EFX_DWORD_IS_SET(_dword)                                        \
 548         ((_dword).ed_u32[0] == ~((uint32_t)0))
 549
 550 #define EFX_WORD_IS_SET(_word)                                          \
 551         ((_word).ew_u16[0] == ~((uint16_t)0))
 552
 553 #define EFX_BYTE_IS_SET(_byte)                                          \
 554         ((_byte).eb_u8[0] == ~((uint8_t)0))
 555
 556 /*
 557  * Construct bit field portion
 558  *
 559  * Creates the portion of the bit field [low,high) that lies within
 560  * the range [min,max).
 561  */
 562
 563 #define EFX_INSERT_NATIVE64(_min, _max, _low, _high, _value)            \
 564         (((_low > _max) || (_high < _min)) ?                            \
 565                 0U :                                                    \
 566                 ((_low > _min) ?                                        \
 567                         (((uint64_t)(_value)) << (_low - _min)) :       \
 568                         (((uint64_t)(_value)) >> (_min - _low))))
 569
 570 #define EFX_INSERT_NATIVE32(_min, _max, _low, _high, _value)            \
 571         (((_low > _max) || (_high < _min)) ?                            \
 572                 0U :                                                    \
 573                 ((_low > _min) ?                                        \
 574                         (((uint32_t)(_value)) << (_low - _min)) :       \
 575                         (((uint32_t)(_value)) >> (_min - _low))))
 576
 577 #define EFX_INSERT_NATIVE16(_min, _max, _low, _high, _value)            \
 578         (((_low > _max) || (_high < _min)) ?                            \
 579                 0U :                                                    \
 580                 (uint16_t)((_low > _min) ?                              \
 581                                 ((_value) << (_low - _min)) :           \
 582                                 ((_value) >> (_min - _low))))
 583
 584 #define EFX_INSERT_NATIVE8(_min, _max, _low, _high, _value)             \
 585         (((_low > _max) || (_high < _min)) ?                            \
 586                 0U :                                                    \
 587                 (uint8_t)((_low > _min) ?                               \
 588                                 ((_value) << (_low - _min)) :   \
 589                                 ((_value) >> (_min - _low))))
 590
 591 /*
 592  * Construct bit field portion
 593  *
 594  * Creates the portion of the named bit field that lies within the
 595  * range [min,max).
 596  */
 597 #define EFX_INSERT_FIELD_NATIVE64(_min, _max, _field, _value)           \
 598         EFX_INSERT_NATIVE64(_min, _max, EFX_LOW_BIT(_field),            \
 599             EFX_HIGH_BIT(_field), _value)
 600
 601 #define EFX_INSERT_FIELD_NATIVE32(_min, _max, _field, _value)           \
 602         EFX_INSERT_NATIVE32(_min, _max, EFX_LOW_BIT(_field),            \
 603             EFX_HIGH_BIT(_field), _value)
 604
 605 #define EFX_INSERT_FIELD_NATIVE16(_min, _max, _field, _value)           \
 606         EFX_INSERT_NATIVE16(_min, _max, EFX_LOW_BIT(_field),            \
 607             EFX_HIGH_BIT(_field), _value)
 608
 609 #define EFX_INSERT_FIELD_NATIVE8(_min, _max, _field, _value)            \
 610         EFX_INSERT_NATIVE8(_min, _max, EFX_LOW_BIT(_field),             \
 611             EFX_HIGH_BIT(_field), _value)
 612
 613 /*
 614  * Construct bit field
 615  *
 616  * Creates the portion of the named bit fields that lie within the
 617  * range [min,max).
 618  */
 619 #define EFX_INSERT_FIELDS64(_min, _max,                                 \
 620             _field1, _value1, _field2, _value2, _field3, _value3,       \
 621             _field4, _value4, _field5, _value5, _field6, _value6,       \
 622             _field7, _value7, _field8, _value8, _field9, _value9,       \
 623             _field10, _value10)                                         \
 624         __CPU_TO_LE_64(                                                 \
 625             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field1, _value1) |   \
 626             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field2, _value2) |   \
 627             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field3, _value3) |   \
 628             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field4, _value4) |   \
 629             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field5, _value5) |   \
 630             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field6, _value6) |   \
 631             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field7, _value7) |   \
 632             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field8, _value8) |   \
 633             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field9, _value9) |   \
 634             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field10, _value10))
 635
 636 #define EFX_INSERT_FIELDS32(_min, _max,                                 \
 637             _field1, _value1, _field2, _value2, _field3, _value3,       \
 638             _field4, _value4, _field5, _value5, _field6, _value6,       \
 639             _field7, _value7, _field8, _value8, _field9, _value9,       \
 640             _field10, _value10)                                         \
 641         __CPU_TO_LE_32(                                                 \
 642             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field1, _value1) |   \
 643             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field2, _value2) |   \
 644             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field3, _value3) |   \
 645             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field4, _value4) |   \
 646             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field5, _value5) |   \
 647             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field6, _value6) |   \
 648             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field7, _value7) |   \
 649             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field8, _value8) |   \
 650             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field9, _value9) |   \
 651             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field10, _value10))
 652
 653 #define EFX_INSERT_FIELDS16(_min, _max,                                 \
 654             _field1, _value1, _field2, _value2, _field3, _value3,       \
 655             _field4, _value4, _field5, _value5, _field6, _value6,       \
 656             _field7, _value7, _field8, _value8, _field9, _value9,       \
 657             _field10, _value10)                                         \
 658         __CPU_TO_LE_16(                                                 \
 659             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field1, _value1) |   \
 660             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field2, _value2) |   \
 661             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field3, _value3) |   \
 662             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field4, _value4) |   \
 663             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field5, _value5) |   \
 664             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field6, _value6) |   \
 665             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field7, _value7) |   \
 666             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field8, _value8) |   \
 667             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field9, _value9) |   \
 668             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field10, _value10))
 669
 670 #define EFX_INSERT_FIELDS8(_min, _max,                                  \
 671             _field1, _value1, _field2, _value2, _field3, _value3,       \
 672             _field4, _value4, _field5, _value5, _field6, _value6,       \
 673             _field7, _value7, _field8, _value8, _field9, _value9,       \
 674             _field10, _value10)                                         \
 675         __NATIVE_8(                                                     \
 676             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field1, _value1) |    \
 677             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field2, _value2) |    \
 678             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field3, _value3) |    \
 679             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field4, _value4) |    \
 680             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field5, _value5) |    \
 681             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field6, _value6) |    \
 682             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field7, _value7) |    \
 683             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field8, _value8) |    \
 684             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field9, _value9) |    \
 685             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field10, _value10))
 686
 687 #define EFX_POPULATE_OWORD64(_oword,                                    \
 688             _field1, _value1, _field2, _value2, _field3, _value3,       \
 689             _field4, _value4, _field5, _value5, _field6, _value6,       \
 690             _field7, _value7, _field8, _value8, _field9, _value9,       \
 691             _field10, _value10)                                         \
 692         do {                                                            \
 693                 _NOTE(CONSTANTCONDITION)                                \
 694                 (_oword).eo_u64[0] = EFX_INSERT_FIELDS64(0, 63,         \
 695                     _field1, _value1, _field2, _value2,                 \
 696                     _field3, _value3, _field4, _value4,                 \
 697                     _field5, _value5, _field6, _value6,                 \
 698                     _field7, _value7, _field8, _value8,                 \
 699                     _field9, _value9, _field10, _value10);              \
 700                 _NOTE(CONSTANTCONDITION)                                \
 701                 (_oword).eo_u64[1] = EFX_INSERT_FIELDS64(64, 127,       \
 702                     _field1, _value1, _field2, _value2,                 \
 703                     _field3, _value3, _field4, _value4,                 \
 704                     _field5, _value5, _field6, _value6,                 \
 705                     _field7, _value7, _field8, _value8,                 \
 706                     _field9, _value9, _field10, _value10);              \
 707         _NOTE(CONSTANTCONDITION)                                        \
 708         } while (B_FALSE)
 709
 710 #define EFX_POPULATE_OWORD32(_oword,                                    \
 711             _field1, _value1, _field2, _value2, _field3, _value3,       \
 712             _field4, _value4, _field5, _value5, _field6, _value6,       \
 713             _field7, _value7, _field8, _value8, _field9, _value9,       \
 714             _field10, _value10)                                         \
 715         do {                                                            \
 716                 _NOTE(CONSTANTCONDITION)                                \
 717                 (_oword).eo_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 718                     _field1, _value1, _field2, _value2,                 \
 719                     _field3, _value3, _field4, _value4,                 \
 720                     _field5, _value5, _field6, _value6,                 \
 721                     _field7, _value7, _field8, _value8,                 \
 722                     _field9, _value9, _field10, _value10);              \
 723                 _NOTE(CONSTANTCONDITION)                                \
 724                 (_oword).eo_u32[1] = EFX_INSERT_FIELDS32(32, 63,        \
 725                     _field1, _value1, _field2, _value2,                 \
 726                     _field3, _value3, _field4, _value4,                 \
 727                     _field5, _value5, _field6, _value6,                 \
 728                     _field7, _value7, _field8, _value8,                 \
 729                     _field9, _value9, _field10, _value10);              \
 730                 _NOTE(CONSTANTCONDITION)                                \
 731                 (_oword).eo_u32[2] = EFX_INSERT_FIELDS32(64, 95,        \
 732                     _field1, _value1, _field2, _value2,                 \
 733                     _field3, _value3, _field4, _value4,                 \
 734                     _field5, _value5, _field6, _value6,                 \
 735                     _field7, _value7, _field8, _value8,                 \
 736                     _field9, _value9, _field10, _value10);              \
 737                 _NOTE(CONSTANTCONDITION)                                \
 738                 (_oword).eo_u32[3] = EFX_INSERT_FIELDS32(96, 127,       \
 739                     _field1, _value1, _field2, _value2,                 \
 740                     _field3, _value3, _field4, _value4,                 \
 741                     _field5, _value5, _field6, _value6,                 \
 742                     _field7, _value7, _field8, _value8,                 \
 743                     _field9, _value9, _field10, _value10);              \
 744         _NOTE(CONSTANTCONDITION)                                        \
 745         } while (B_FALSE)
 746
 747 #define EFX_POPULATE_QWORD64(_qword,                                    \
 748             _field1, _value1, _field2, _value2, _field3, _value3,       \
 749             _field4, _value4, _field5, _value5, _field6, _value6,       \
 750             _field7, _value7, _field8, _value8, _field9, _value9,       \
 751             _field10, _value10)                                         \
 752         do {                                                            \
 753                 _NOTE(CONSTANTCONDITION)                                \
 754                 (_qword).eq_u64[0] = EFX_INSERT_FIELDS64(0, 63,         \
 755                     _field1, _value1, _field2, _value2,                 \
 756                     _field3, _value3, _field4, _value4,                 \
 757                     _field5, _value5, _field6, _value6,                 \
 758                     _field7, _value7, _field8, _value8,                 \
 759                     _field9, _value9, _field10, _value10);              \
 760         _NOTE(CONSTANTCONDITION)                                        \
 761         } while (B_FALSE)
 762
 763 #define EFX_POPULATE_QWORD32(_qword,                                    \
 764             _field1, _value1, _field2, _value2, _field3, _value3,       \
 765             _field4, _value4, _field5, _value5, _field6, _value6,       \
 766             _field7, _value7, _field8, _value8, _field9, _value9,       \
 767             _field10, _value10)                                         \
 768         do {                                                            \
 769                 _NOTE(CONSTANTCONDITION)                                \
 770                 (_qword).eq_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 771                     _field1, _value1, _field2, _value2,                 \
 772                     _field3, _value3, _field4, _value4,                 \
 773                     _field5, _value5, _field6, _value6,                 \
 774                     _field7, _value7, _field8, _value8,                 \
 775                     _field9, _value9, _field10, _value10);              \
 776                 _NOTE(CONSTANTCONDITION)                                \
 777                 (_qword).eq_u32[1] = EFX_INSERT_FIELDS32(32, 63,        \
 778                     _field1, _value1, _field2, _value2,                 \
 779                     _field3, _value3, _field4, _value4,                 \
 780                     _field5, _value5, _field6, _value6,                 \
 781                     _field7, _value7, _field8, _value8,                 \
 782                     _field9, _value9, _field10, _value10);              \
 783         _NOTE(CONSTANTCONDITION)                                        \
 784         } while (B_FALSE)
 785
 786 #define EFX_POPULATE_DWORD(_dword,                                      \
 787             _field1, _value1, _field2, _value2, _field3, _value3,       \
 788             _field4, _value4, _field5, _value5, _field6, _value6,       \
 789             _field7, _value7, _field8, _value8, _field9, _value9,       \
 790             _field10, _value10)                                         \
 791         do {                                                            \
 792                 _NOTE(CONSTANTCONDITION)                                \
 793                 (_dword).ed_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 794                     _field1, _value1, _field2, _value2,                 \
 795                     _field3, _value3, _field4, _value4,                 \
 796                     _field5, _value5, _field6, _value6,                 \
 797                     _field7, _value7, _field8, _value8,                 \
 798                     _field9, _value9, _field10, _value10);              \
 799         _NOTE(CONSTANTCONDITION)                                        \
 800         } while (B_FALSE)
 801
 802 #define EFX_POPULATE_WORD(_word,                                        \
 803             _field1, _value1, _field2, _value2, _field3, _value3,       \
 804             _field4, _value4, _field5, _value5, _field6, _value6,       \
 805             _field7, _value7, _field8, _value8, _field9, _value9,       \
 806             _field10, _value10)                                         \
 807         do {                                                            \
 808                 _NOTE(CONSTANTCONDITION)                                \
 809                 (_word).ew_u16[0] = EFX_INSERT_FIELDS16(0, 15,          \
 810                     _field1, _value1, _field2, _value2,                 \
 811                     _field3, _value3, _field4, _value4,                 \
 812                     _field5, _value5, _field6, _value6,                 \
 813                     _field7, _value7, _field8, _value8,                 \
 814                     _field9, _value9, _field10, _value10);              \
 815         _NOTE(CONSTANTCONDITION)                                        \
 816         } while (B_FALSE)
 817
 818 #define EFX_POPULATE_BYTE(_byte,                                        \
 819             _field1, _value1, _field2, _value2, _field3, _value3,       \
 820             _field4, _value4, _field5, _value5, _field6, _value6,       \
 821             _field7, _value7, _field8, _value8, _field9, _value9,       \
 822             _field10, _value10)                                         \
 823         do {                                                            \
 824                 _NOTE(CONSTANTCONDITION)                                \
 825                 (_byte).eb_u8[0] = EFX_INSERT_FIELDS8(0, 7,             \
 826                     _field1, _value1, _field2, _value2,                 \
 827                     _field3, _value3, _field4, _value4,                 \
 828                     _field5, _value5, _field6, _value6,                 \
 829                     _field7, _value7, _field8, _value8,                 \
 830                     _field9, _value9, _field10, _value10);              \
 831         _NOTE(CONSTANTCONDITION)                                        \
 832         } while (B_FALSE)
 833
 834 /* Populate an octword field with various numbers of arguments */
 835 #define EFX_POPULATE_OWORD_10 EFX_POPULATE_OWORD
 836
 837 #define EFX_POPULATE_OWORD_9(_oword,                                    \
 838             _field1, _value1, _field2, _value2, _field3, _value3,       \
 839             _field4, _value4, _field5, _value5, _field6, _value6,       \
 840             _field7, _value7, _field8, _value8, _field9, _value9)       \
 841         EFX_POPULATE_OWORD_10(_oword, EFX_DUMMY_FIELD, 0,               \
 842             _field1, _value1, _field2, _value2, _field3, _value3,       \
 843             _field4, _value4, _field5, _value5, _field6, _value6,       \
 844             _field7, _value7, _field8, _value8, _field9, _value9)
 845
 846 #define EFX_POPULATE_OWORD_8(_oword,                                    \
 847             _field1, _value1, _field2, _value2, _field3, _value3,       \
 848             _field4, _value4, _field5, _value5, _field6, _value6,       \
 849             _field7, _value7, _field8, _value8)                         \
 850         EFX_POPULATE_OWORD_9(_oword, EFX_DUMMY_FIELD, 0,                \
 851             _field1, _value1, _field2, _value2, _field3, _value3,       \
 852             _field4, _value4, _field5, _value5, _field6, _value6,       \
 853             _field7, _value7, _field8, _value8)
 854
 855 #define EFX_POPULATE_OWORD_7(_oword,                                    \
 856             _field1, _value1, _field2, _value2, _field3, _value3,       \
 857             _field4, _value4, _field5, _value5, _field6, _value6,       \
 858             _field7, _value7)                                           \
 859         EFX_POPULATE_OWORD_8(_oword, EFX_DUMMY_FIELD, 0,                \
 860             _field1, _value1, _field2, _value2, _field3, _value3,       \
 861             _field4, _value4, _field5, _value5, _field6, _value6,       \
 862             _field7, _value7)
 863
 864 #define EFX_POPULATE_OWORD_6(_oword,                                    \
 865             _field1, _value1, _field2, _value2, _field3, _value3,       \
 866             _field4, _value4, _field5, _value5, _field6, _value6)       \
 867         EFX_POPULATE_OWORD_7(_oword, EFX_DUMMY_FIELD, 0,                \
 868             _field1, _value1, _field2, _value2, _field3, _value3,       \
 869             _field4, _value4, _field5, _value5, _field6, _value6)
 870
 871 #define EFX_POPULATE_OWORD_5(_oword,                                    \
 872             _field1, _value1, _field2, _value2, _field3, _value3,       \
 873             _field4, _value4, _field5, _value5)                         \
 874         EFX_POPULATE_OWORD_6(_oword, EFX_DUMMY_FIELD, 0,                \
 875             _field1, _value1, _field2, _value2, _field3, _value3,       \
 876             _field4, _value4, _field5, _value5)
 877
 878 #define EFX_POPULATE_OWORD_4(_oword,                                    \
 879             _field1, _value1, _field2, _value2, _field3, _value3,       \
 880             _field4, _value4)                                           \
 881         EFX_POPULATE_OWORD_5(_oword, EFX_DUMMY_FIELD, 0,                \
 882             _field1, _value1, _field2, _value2, _field3, _value3,       \
 883             _field4, _value4)
 884
 885 #define EFX_POPULATE_OWORD_3(_oword,                                    \
 886             _field1, _value1, _field2, _value2, _field3, _value3)       \
 887         EFX_POPULATE_OWORD_4(_oword, EFX_DUMMY_FIELD, 0,                \
 888             _field1, _value1, _field2, _value2, _field3, _value3)
 889
 890 #define EFX_POPULATE_OWORD_2(_oword,                                    \
 891             _field1, _value1, _field2, _value2)                         \
 892         EFX_POPULATE_OWORD_3(_oword, EFX_DUMMY_FIELD, 0,                \
 893             _field1, _value1, _field2, _value2)
 894
 895 #define EFX_POPULATE_OWORD_1(_oword,                                    \
 896             _field1, _value1)                                           \
 897         EFX_POPULATE_OWORD_2(_oword, EFX_DUMMY_FIELD, 0,                \
 898             _field1, _value1)
 899
 900 #define EFX_ZERO_OWORD(_oword)                                          \
 901         EFX_POPULATE_OWORD_1(_oword, EFX_DUMMY_FIELD, 0)
 902
 903 #define EFX_SET_OWORD(_oword)                                           \
 904         EFX_POPULATE_OWORD_4(_oword,                                    \
 905             EFX_DWORD_0, 0xffffffff, EFX_DWORD_1, 0xffffffff,           \
 906             EFX_DWORD_2, 0xffffffff, EFX_DWORD_3, 0xffffffff)
 907
 908 /* Populate a quadword field with various numbers of arguments */
 909 #define EFX_POPULATE_QWORD_10 EFX_POPULATE_QWORD
 910
 911 #define EFX_POPULATE_QWORD_9(_qword,                                    \
 912             _field1, _value1, _field2, _value2, _field3, _value3,       \
 913             _field4, _value4, _field5, _value5, _field6, _value6,       \
 914             _field7, _value7, _field8, _value8, _field9, _value9)       \
 915         EFX_POPULATE_QWORD_10(_qword, EFX_DUMMY_FIELD, 0,               \
 916             _field1, _value1, _field2, _value2, _field3, _value3,       \
 917             _field4, _value4, _field5, _value5, _field6, _value6,       \
 918             _field7, _value7, _field8, _value8, _field9, _value9)
 919
 920 #define EFX_POPULATE_QWORD_8(_qword,                                    \
 921             _field1, _value1, _field2, _value2, _field3, _value3,       \
 922             _field4, _value4, _field5, _value5, _field6, _value6,       \
 923             _field7, _value7, _field8, _value8)                         \
 924         EFX_POPULATE_QWORD_9(_qword, EFX_DUMMY_FIELD, 0,                \
 925             _field1, _value1, _field2, _value2, _field3, _value3,       \
 926             _field4, _value4, _field5, _value5, _field6, _value6,       \
 927             _field7, _value7, _field8, _value8)
 928
 929 #define EFX_POPULATE_QWORD_7(_qword,                                    \
 930             _field1, _value1, _field2, _value2, _field3, _value3,       \
 931             _field4, _value4, _field5, _value5, _field6, _value6,       \
 932             _field7, _value7)                                           \
 933         EFX_POPULATE_QWORD_8(_qword, EFX_DUMMY_FIELD, 0,                \
 934             _field1, _value1, _field2, _value2, _field3, _value3,       \
 935             _field4, _value4, _field5, _value5, _field6, _value6,       \
 936             _field7, _value7)
 937
 938 #define EFX_POPULATE_QWORD_6(_qword,                                    \
 939             _field1, _value1, _field2, _value2, _field3, _value3,       \
 940             _field4, _value4, _field5, _value5, _field6, _value6)       \
 941         EFX_POPULATE_QWORD_7(_qword, EFX_DUMMY_FIELD, 0,                \
 942             _field1, _value1, _field2, _value2, _field3, _value3,       \
 943             _field4, _value4, _field5, _value5, _field6, _value6)
 944
 945 #define EFX_POPULATE_QWORD_5(_qword,                                    \
 946             _field1, _value1, _field2, _value2, _field3, _value3,       \
 947             _field4, _value4, _field5, _value5)                         \
 948         EFX_POPULATE_QWORD_6(_qword, EFX_DUMMY_FIELD, 0,                \
 949             _field1, _value1, _field2, _value2, _field3, _value3,       \
 950             _field4, _value4, _field5, _value5)
 951
 952 #define EFX_POPULATE_QWORD_4(_qword,                                    \
 953             _field1, _value1, _field2, _value2, _field3, _value3,       \
 954             _field4, _value4)                                           \
 955         EFX_POPULATE_QWORD_5(_qword, EFX_DUMMY_FIELD, 0,                \
 956             _field1, _value1, _field2, _value2, _field3, _value3,       \
 957             _field4, _value4)
 958
 959 #define EFX_POPULATE_QWORD_3(_qword,                                    \
 960             _field1, _value1, _field2, _value2, _field3, _value3)       \
 961         EFX_POPULATE_QWORD_4(_qword, EFX_DUMMY_FIELD, 0,                \
 962             _field1, _value1, _field2, _value2, _field3, _value3)
 963
 964 #define EFX_POPULATE_QWORD_2(_qword,                                    \
 965             _field1, _value1, _field2, _value2)                         \
 966         EFX_POPULATE_QWORD_3(_qword, EFX_DUMMY_FIELD, 0,                \
 967             _field1, _value1, _field2, _value2)
 968
 969 #define EFX_POPULATE_QWORD_1(_qword,                                    \
 970             _field1, _value1)                                           \
 971         EFX_POPULATE_QWORD_2(_qword, EFX_DUMMY_FIELD, 0,                \
 972             _field1, _value1)
 973
 974 #define EFX_ZERO_QWORD(_qword)                                          \
 975         EFX_POPULATE_QWORD_1(_qword, EFX_DUMMY_FIELD, 0)
 976
 977 #define EFX_SET_QWORD(_qword)                                           \
 978         EFX_POPULATE_QWORD_2(_qword,                                    \
 979             EFX_DWORD_0, 0xffffffff, EFX_DWORD_1, 0xffffffff)
 980
 981 /* Populate a dword field with various numbers of arguments */
 982 #define EFX_POPULATE_DWORD_10 EFX_POPULATE_DWORD
 983
 984 #define EFX_POPULATE_DWORD_9(_dword,                                    \
 985             _field1, _value1, _field2, _value2, _field3, _value3,       \
 986             _field4, _value4, _field5, _value5, _field6, _value6,       \
 987             _field7, _value7, _field8, _value8, _field9, _value9)       \
 988         EFX_POPULATE_DWORD_10(_dword, EFX_DUMMY_FIELD, 0,               \
 989             _field1, _value1, _field2, _value2, _field3, _value3,       \
 990             _field4, _value4, _field5, _value5, _field6, _value6,       \
 991             _field7, _value7, _field8, _value8, _field9, _value9)
 992
 993 #define EFX_POPULATE_DWORD_8(_dword,                                    \
 994             _field1, _value1, _field2, _value2, _field3, _value3,       \
 995             _field4, _value4, _field5, _value5, _field6, _value6,       \
 996             _field7, _value7, _field8, _value8)                         \
 997         EFX_POPULATE_DWORD_9(_dword, EFX_DUMMY_FIELD, 0,                \
 998             _field1, _value1, _field2, _value2, _field3, _value3,       \
 999             _field4, _value4, _field5, _value5, _field6, _value6,       \
1000             _field7, _value7, _field8, _value8)
1001
1002 #define EFX_POPULATE_DWORD_7(_dword,                                    \
1003             _field1, _value1, _field2, _value2, _field3, _value3,       \
1004             _field4, _value4, _field5, _value5, _field6, _value6,       \
1005             _field7, _value7)                                           \
1006         EFX_POPULATE_DWORD_8(_dword, EFX_DUMMY_FIELD, 0,                \
1007             _field1, _value1, _field2, _value2, _field3, _value3,       \
1008             _field4, _value4, _field5, _value5, _field6, _value6,       \
1009             _field7, _value7)
1010
1011 #define EFX_POPULATE_DWORD_6(_dword,                                    \
1012             _field1, _value1, _field2, _value2, _field3, _value3,       \
1013             _field4, _value4, _field5, _value5, _field6, _value6)       \
1014         EFX_POPULATE_DWORD_7(_dword, EFX_DUMMY_FIELD, 0,                \
1015             _field1, _value1, _field2, _value2, _field3, _value3,       \
1016             _field4, _value4, _field5, _value5, _field6, _value6)
1017
1018 #define EFX_POPULATE_DWORD_5(_dword,                                    \
1019             _field1, _value1, _field2, _value2, _field3, _value3,       \
1020             _field4, _value4, _field5, _value5)                         \
1021         EFX_POPULATE_DWORD_6(_dword, EFX_DUMMY_FIELD, 0,                \
1022             _field1, _value1, _field2, _value2, _field3, _value3,       \
1023             _field4, _value4, _field5, _value5)
1024
1025 #define EFX_POPULATE_DWORD_4(_dword,                                    \
1026             _field1, _value1, _field2, _value2, _field3, _value3,       \
1027             _field4, _value4)                                           \
1028         EFX_POPULATE_DWORD_5(_dword, EFX_DUMMY_FIELD, 0,                \
1029             _field1, _value1, _field2, _value2, _field3, _value3,       \
1030             _field4, _value4)
1031
1032 #define EFX_POPULATE_DWORD_3(_dword,                                    \
1033             _field1, _value1, _field2, _value2, _field3, _value3)       \
1034         EFX_POPULATE_DWORD_4(_dword, EFX_DUMMY_FIELD, 0,                \
1035             _field1, _value1, _field2, _value2, _field3, _value3)
1036
1037 #define EFX_POPULATE_DWORD_2(_dword,                                    \
1038             _field1, _value1, _field2, _value2)                         \
1039         EFX_POPULATE_DWORD_3(_dword, EFX_DUMMY_FIELD, 0,                \
1040             _field1, _value1, _field2, _value2)
1041
1042 #define EFX_POPULATE_DWORD_1(_dword,                                    \
1043             _field1, _value1)                                           \
1044         EFX_POPULATE_DWORD_2(_dword, EFX_DUMMY_FIELD, 0,                \
1045             _field1, _value1)
1046
1047 #define EFX_ZERO_DWORD(_dword)                                          \
1048         EFX_POPULATE_DWORD_1(_dword, EFX_DUMMY_FIELD, 0)
1049
1050 #define EFX_SET_DWORD(_dword)                                           \
1051         EFX_POPULATE_DWORD_1(_dword,                                    \
1052             EFX_DWORD_0, 0xffffffff)
1053
1054 /* Populate a word field with various numbers of arguments */
1055 #define EFX_POPULATE_WORD_10 EFX_POPULATE_WORD
1056
1057 #define EFX_POPULATE_WORD_9(_word,                                      \
1058             _field1, _value1, _field2, _value2, _field3, _value3,       \
1059             _field4, _value4, _field5, _value5, _field6, _value6,       \
1060             _field7, _value7, _field8, _value8, _field9, _value9)       \
1061         EFX_POPULATE_WORD_10(_word, EFX_DUMMY_FIELD, 0,                 \
1062             _field1, _value1, _field2, _value2, _field3, _value3,       \
1063             _field4, _value4, _field5, _value5, _field6, _value6,       \
1064             _field7, _value7, _field8, _value8, _field9, _value9)
1065
1066 #define EFX_POPULATE_WORD_8(_word,                                      \
1067             _field1, _value1, _field2, _value2, _field3, _value3,       \
1068             _field4, _value4, _field5, _value5, _field6, _value6,       \
1069             _field7, _value7, _field8, _value8)                         \
1070         EFX_POPULATE_WORD_9(_word, EFX_DUMMY_FIELD, 0,                  \
1071             _field1, _value1, _field2, _value2, _field3, _value3,       \
1072             _field4, _value4, _field5, _value5, _field6, _value6,       \
1073             _field7, _value7, _field8, _value8)
1074
1075 #define EFX_POPULATE_WORD_7(_word,                                      \
1076             _field1, _value1, _field2, _value2, _field3, _value3,       \
1077             _field4, _value4, _field5, _value5, _field6, _value6,       \
1078             _field7, _value7)                                           \
1079         EFX_POPULATE_WORD_8(_word, EFX_DUMMY_FIELD, 0,                  \
1080             _field1, _value1, _field2, _value2, _field3, _value3,       \
1081             _field4, _value4, _field5, _value5, _field6, _value6,       \
1082             _field7, _value7)
1083
1084 #define EFX_POPULATE_WORD_6(_word,                                      \
1085             _field1, _value1, _field2, _value2, _field3, _value3,       \
1086             _field4, _value4, _field5, _value5, _field6, _value6)       \
1087         EFX_POPULATE_WORD_7(_word, EFX_DUMMY_FIELD, 0,                  \
1088             _field1, _value1, _field2, _value2, _field3, _value3,       \
1089             _field4, _value4, _field5, _value5, _field6, _value6)
1090
1091 #define EFX_POPULATE_WORD_5(_word,                                      \
1092             _field1, _value1, _field2, _value2, _field3, _value3,       \
1093             _field4, _value4, _field5, _value5)                         \
1094         EFX_POPULATE_WORD_6(_word, EFX_DUMMY_FIELD, 0,                  \
1095             _field1, _value1, _field2, _value2, _field3, _value3,       \
1096             _field4, _value4, _field5, _value5)
1097
1098 #define EFX_POPULATE_WORD_4(_word,                                      \
1099             _field1, _value1, _field2, _value2, _field3, _value3,       \
1100             _field4, _value4)                                           \
1101         EFX_POPULATE_WORD_5(_word, EFX_DUMMY_FIELD, 0,                  \
1102             _field1, _value1, _field2, _value2, _field3, _value3,       \
1103             _field4, _value4)
1104
1105 #define EFX_POPULATE_WORD_3(_word,                                      \
1106             _field1, _value1, _field2, _value2, _field3, _value3)       \
1107         EFX_POPULATE_WORD_4(_word, EFX_DUMMY_FIELD, 0,                  \
1108             _field1, _value1, _field2, _value2, _field3, _value3)
1109
1110 #define EFX_POPULATE_WORD_2(_word,                                      \
1111             _field1, _value1, _field2, _value2)                         \
1112         EFX_POPULATE_WORD_3(_word, EFX_DUMMY_FIELD, 0,                  \
1113             _field1, _value1, _field2, _value2)
1114
1115 #define EFX_POPULATE_WORD_1(_word,                                      \
1116             _field1, _value1)                                           \
1117         EFX_POPULATE_WORD_2(_word, EFX_DUMMY_FIELD, 0,                  \
1118             _field1, _value1)
1119
1120 #define EFX_ZERO_WORD(_word)                                            \
1121         EFX_POPULATE_WORD_1(_word, EFX_DUMMY_FIELD, 0)
1122
1123 #define EFX_SET_WORD(_word)                                             \
1124         EFX_POPULATE_WORD_1(_word,                                      \
1125             EFX_WORD_0, 0xffff)
1126
1127 /* Populate a byte field with various numbers of arguments */
1128 #define EFX_POPULATE_BYTE_10 EFX_POPULATE_BYTE
1129
1130 #define EFX_POPULATE_BYTE_9(_byte,                                      \
1131             _field1, _value1, _field2, _value2, _field3, _value3,       \
1132             _field4, _value4, _field5, _value5, _field6, _value6,       \
1133             _field7, _value7, _field8, _value8, _field9, _value9)       \
1134         EFX_POPULATE_BYTE_10(_byte, EFX_DUMMY_FIELD, 0,                 \
1135             _field1, _value1, _field2, _value2, _field3, _value3,       \
1136             _field4, _value4, _field5, _value5, _field6, _value6,       \
1137             _field7, _value7, _field8, _value8, _field9, _value9)
1138
1139 #define EFX_POPULATE_BYTE_8(_byte,                                      \
1140             _field1, _value1, _field2, _value2, _field3, _value3,       \
1141             _field4, _value4, _field5, _value5, _field6, _value6,       \
1142             _field7, _value7, _field8, _value8)                         \
1143         EFX_POPULATE_BYTE_9(_byte, EFX_DUMMY_FIELD, 0,                  \
1144             _field1, _value1, _field2, _value2, _field3, _value3,       \
1145             _field4, _value4, _field5, _value5, _field6, _value6,       \
1146             _field7, _value7, _field8, _value8)
1147
1148 #define EFX_POPULATE_BYTE_7(_byte,                                      \
1149             _field1, _value1, _field2, _value2, _field3, _value3,       \
1150             _field4, _value4, _field5, _value5, _field6, _value6,       \
1151             _field7, _value7)                                           \
1152         EFX_POPULATE_BYTE_8(_byte, EFX_DUMMY_FIELD, 0,                  \
1153             _field1, _value1, _field2, _value2, _field3, _value3,       \
1154             _field4, _value4, _field5, _value5, _field6, _value6,       \
1155             _field7, _value7)
1156
1157 #define EFX_POPULATE_BYTE_6(_byte,                                      \
1158             _field1, _value1, _field2, _value2, _field3, _value3,       \
1159             _field4, _value4, _field5, _value5, _field6, _value6)       \
1160         EFX_POPULATE_BYTE_7(_byte, EFX_DUMMY_FIELD, 0,                  \
1161             _field1, _value1, _field2, _value2, _field3, _value3,       \
1162             _field4, _value4, _field5, _value5, _field6, _value6)
1163
1164 #define EFX_POPULATE_BYTE_5(_byte,                                      \
1165             _field1, _value1, _field2, _value2, _field3, _value3,       \
1166             _field4, _value4, _field5, _value5)                         \
1167         EFX_POPULATE_BYTE_6(_byte, EFX_DUMMY_FIELD, 0,                  \
1168             _field1, _value1, _field2, _value2, _field3, _value3,       \
1169             _field4, _value4, _field5, _value5)
1170
1171 #define EFX_POPULATE_BYTE_4(_byte,                                      \
1172             _field1, _value1, _field2, _value2, _field3, _value3,       \
1173             _field4, _value4)                                           \
1174         EFX_POPULATE_BYTE_5(_byte, EFX_DUMMY_FIELD, 0,                  \
1175             _field1, _value1, _field2, _value2, _field3, _value3,       \
1176             _field4, _value4)
1177
1178 #define EFX_POPULATE_BYTE_3(_byte,                                      \
1179             _field1, _value1, _field2, _value2, _field3, _value3)       \
1180         EFX_POPULATE_BYTE_4(_byte, EFX_DUMMY_FIELD, 0,                  \
1181             _field1, _value1, _field2, _value2, _field3, _value3)
1182
1183 #define EFX_POPULATE_BYTE_2(_byte,                                      \
1184             _field1, _value1, _field2, _value2)                         \
1185         EFX_POPULATE_BYTE_3(_byte, EFX_DUMMY_FIELD, 0,                  \
1186             _field1, _value1, _field2, _value2)
1187
1188 #define EFX_POPULATE_BYTE_1(_byte,                                      \
1189             _field1, _value1)                                           \
1190         EFX_POPULATE_BYTE_2(_byte, EFX_DUMMY_FIELD, 0,                  \
1191             _field1, _value1)
1192
1193 #define EFX_ZERO_BYTE(_byte)                                            \
1194         EFX_POPULATE_BYTE_1(_byte, EFX_DUMMY_FIELD, 0)
1195
1196 #define EFX_SET_BYTE(_byte)                                             \
1197         EFX_POPULATE_BYTE_1(_byte,                                      \
1198             EFX_BYTE_0, 0xff)
1199
1200 /*
1201  * Modify a named field within an already-populated structure.  Used
1202  * for read-modify-write operations.
1203  */
1204
1205 #define EFX_INSERT_FIELD64(_min, _max, _field, _value)                  \
1206         __CPU_TO_LE_64(EFX_INSERT_FIELD_NATIVE64(_min, _max, _field, _value))
1207
1208 #define EFX_INSERT_FIELD32(_min, _max, _field, _value)                  \
1209         __CPU_TO_LE_32(EFX_INSERT_FIELD_NATIVE32(_min, _max, _field, _value))
1210
1211 #define EFX_INSERT_FIELD16(_min, _max, _field, _value)                  \
1212         __CPU_TO_LE_16(EFX_INSERT_FIELD_NATIVE16(_min, _max, _field, _value))
1213
1214 #define EFX_INSERT_FIELD8(_min, _max, _field, _value)                   \
1215         __NATIVE_8(EFX_INSERT_FIELD_NATIVE8(_min, _max, _field, _value))
1216
1217 #define EFX_INPLACE_MASK64(_min, _max, _field)                          \
1218         EFX_INSERT_FIELD64(_min, _max, _field, EFX_MASK64(_field))
1219
1220 #define EFX_INPLACE_MASK32(_min, _max, _field)                          \
1221         EFX_INSERT_FIELD32(_min, _max, _field, EFX_MASK32(_field))
1222
1223 #define EFX_INPLACE_MASK16(_min, _max, _field)                          \
1224         EFX_INSERT_FIELD16(_min, _max, _field, EFX_MASK16(_field))
1225
1226 #define EFX_INPLACE_MASK8(_min, _max, _field)                           \
1227         EFX_INSERT_FIELD8(_min, _max, _field, EFX_MASK8(_field))
1228
1229 #define EFX_SET_OWORD_FIELD64(_oword, _field, _value)                   \
1230         do {                                                            \
1231                 _NOTE(CONSTANTCONDITION)                                \
1232                 (_oword).eo_u64[0] = (((_oword).eo_u64[0] &             \
1233                     ~EFX_INPLACE_MASK64(0, 63, _field)) |               \
1234                     EFX_INSERT_FIELD64(0, 63, _field, _value));         \
1235                 _NOTE(CONSTANTCONDITION)                                \
1236                 (_oword).eo_u64[1] = (((_oword).eo_u64[1] &             \
1237                     ~EFX_INPLACE_MASK64(64, 127, _field)) |             \
1238                     EFX_INSERT_FIELD64(64, 127, _field, _value));       \
1239         _NOTE(CONSTANTCONDITION)                                        \
1240         } while (B_FALSE)
1241
1242 #define EFX_SET_OWORD_FIELD32(_oword, _field, _value)                   \
1243         do {                                                            \
1244                 _NOTE(CONSTANTCONDITION)                                \
1245                 (_oword).eo_u32[0] = (((_oword).eo_u32[0] &             \
1246                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1247                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1248                 _NOTE(CONSTANTCONDITION)                                \
1249                 (_oword).eo_u32[1] = (((_oword).eo_u32[1] &             \
1250                     ~EFX_INPLACE_MASK32(32, 63, _field)) |              \
1251                     EFX_INSERT_FIELD32(32, 63, _field, _value));        \
1252                 _NOTE(CONSTANTCONDITION)                                \
1253                 (_oword).eo_u32[2] = (((_oword).eo_u32[2] &             \
1254                     ~EFX_INPLACE_MASK32(64, 95, _field)) |              \
1255                     EFX_INSERT_FIELD32(64, 95, _field, _value));        \
1256                 _NOTE(CONSTANTCONDITION)                                \
1257                 (_oword).eo_u32[3] = (((_oword).eo_u32[3] &             \
1258                     ~EFX_INPLACE_MASK32(96, 127, _field)) |             \
1259                     EFX_INSERT_FIELD32(96, 127, _field, _value));       \
1260         _NOTE(CONSTANTCONDITION)                                        \
1261         } while (B_FALSE)
1262
1263 #define EFX_SET_QWORD_FIELD64(_qword, _field, _value)                   \
1264         do {                                                            \
1265                 _NOTE(CONSTANTCONDITION)                                \
1266                 (_qword).eq_u64[0] = (((_qword).eq_u64[0] &             \
1267                     ~EFX_INPLACE_MASK64(0, 63, _field)) |               \
1268                     EFX_INSERT_FIELD64(0, 63, _field, _value));         \
1269         _NOTE(CONSTANTCONDITION)                                        \
1270         } while (B_FALSE)
1271
1272 #define EFX_SET_QWORD_FIELD32(_qword, _field, _value)                   \
1273         do {                                                            \
1274                 _NOTE(CONSTANTCONDITION)                                \
1275                 (_qword).eq_u32[0] = (((_qword).eq_u32[0] &             \
1276                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1277                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1278                 _NOTE(CONSTANTCONDITION)                                \
1279                 (_qword).eq_u32[1] = (((_qword).eq_u32[1] &             \
1280                     ~EFX_INPLACE_MASK32(32, 63, _field)) |              \
1281                     EFX_INSERT_FIELD32(32, 63, _field, _value));        \
1282         _NOTE(CONSTANTCONDITION)                                        \
1283         } while (B_FALSE)
1284
1285 #define EFX_SET_DWORD_FIELD(_dword, _field, _value)                     \
1286         do {                                                            \
1287                 _NOTE(CONSTANTCONDITION)                                \
1288                 (_dword).ed_u32[0] = (((_dword).ed_u32[0] &             \
1289                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1290                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1291         _NOTE(CONSTANTCONDITION)                                        \
1292         } while (B_FALSE)
1293
1294 #define EFX_SET_WORD_FIELD(_word, _field, _value)                       \
1295         do {                                                            \
1296                 _NOTE(CONSTANTCONDITION)                                \
1297                 (_word).ew_u16[0] = (((_word).ew_u16[0] &               \
1298                     ~EFX_INPLACE_MASK16(0, 15, _field)) |               \
1299                     EFX_INSERT_FIELD16(0, 15, _field, _value));         \
1300         _NOTE(CONSTANTCONDITION)                                        \
1301         } while (B_FALSE)
1302
1303 #define EFX_SET_BYTE_FIELD(_byte, _field, _value)                       \
1304         do {                                                            \
1305                 _NOTE(CONSTANTCONDITION)                                \
1306                 (_byte).eb_u8[0] = (((_byte).eb_u8[0] &                 \
1307                     ~EFX_INPLACE_MASK8(0, 7, _field)) |                 \
1308                     EFX_INSERT_FIELD8(0, 7, _field, _value));           \
1309         _NOTE(CONSTANTCONDITION)                                        \
1310         } while (B_FALSE)
1311
1312 /*
1313  * Set or clear a numbered bit within an octword.
1314  */
1315
1316 #define EFX_SHIFT64(_bit, _base)                                        \
1317         (((_bit) >= (_base) && (_bit) < (_base) + 64) ?                 \
1318                 ((uint64_t)1 << ((_bit) - (_base))) :                   \
1319                 0U)
1320
1321 #define EFX_SHIFT32(_bit, _base)                                        \
1322         (((_bit) >= (_base) && (_bit) < (_base) + 32) ?                 \
1323                 ((uint32_t)1 << ((_bit) - (_base))) :                   \
1324                 0U)
1325
1326 #define EFX_SHIFT16(_bit, _base)                                        \
1327         (((_bit) >= (_base) && (_bit) < (_base) + 16) ?                 \
1328                 (uint16_t)(1 << ((_bit) - (_base))) :                   \
1329                 0U)
1330
1331 #define EFX_SHIFT8(_bit, _base)                                         \
1332         (((_bit) >= (_base) && (_bit) < (_base) + 8) ?                  \
1333                 (uint8_t)(1 << ((_bit) - (_base))) :                    \
1334                 0U)
1335
1336 #define EFX_SET_OWORD_BIT64(_oword, _bit)                               \
1337         do {                                                            \
1338                 _NOTE(CONSTANTCONDITION)                                \
1339                 (_oword).eo_u64[0] |=                                   \
1340                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)));     \
1341                 (_oword).eo_u64[1] |=                                   \
1342                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(64)));    \
1343         _NOTE(CONSTANTCONDITION)                                        \
1344         } while (B_FALSE)
1345
1346 #define EFX_SET_OWORD_BIT32(_oword, _bit)                               \
1347         do {                                                            \
1348                 _NOTE(CONSTANTCONDITION)                                \
1349                 (_oword).eo_u32[0] |=                                   \
1350                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1351                 (_oword).eo_u32[1] |=                                   \
1352                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)));    \
1353                 (_oword).eo_u32[2] |=                                   \
1354                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(64)));    \
1355                 (_oword).eo_u32[3] |=                                   \
1356                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(96)));    \
1357         _NOTE(CONSTANTCONDITION)                                        \
1358         } while (B_FALSE)
1359
1360 #define EFX_CLEAR_OWORD_BIT64(_oword, _bit)                             \
1361         do {                                                            \
1362                 _NOTE(CONSTANTCONDITION)                                \
1363                 (_oword).eo_u64[0] &=                                   \
1364                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(0)));    \
1365                 (_oword).eo_u64[1] &=                                   \
1366                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(64)));   \
1367         _NOTE(CONSTANTCONDITION)                                        \
1368         } while (B_FALSE)
1369
1370 #define EFX_CLEAR_OWORD_BIT32(_oword, _bit)                             \
1371         do {                                                            \
1372                 _NOTE(CONSTANTCONDITION)                                \
1373                 (_oword).eo_u32[0] &=                                   \
1374                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1375                 (_oword).eo_u32[1] &=                                   \
1376                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(32)));   \
1377                 (_oword).eo_u32[2] &=                                   \
1378                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(64)));   \
1379                 (_oword).eo_u32[3] &=                                   \
1380                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(96)));   \
1381         _NOTE(CONSTANTCONDITION)                                        \
1382         } while (B_FALSE)
1383
1384 #define EFX_TEST_OWORD_BIT64(_oword, _bit)                              \
1385         (((_oword).eo_u64[0] &                                          \
1386                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)))) ||  \
1387         ((_oword).eo_u64[1] &                                           \
1388                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(64)))))
1389
1390 #define EFX_TEST_OWORD_BIT32(_oword, _bit)                              \
1391         (((_oword).eo_u32[0] &                                          \
1392                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) ||  \
1393         ((_oword).eo_u32[1] &                                           \
1394                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)))) || \
1395         ((_oword).eo_u32[2] &                                           \
1396                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(64)))) || \
1397         ((_oword).eo_u32[3] &                                           \
1398                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(96)))))
1399
1400
1401 #define EFX_SET_QWORD_BIT64(_qword, _bit)                               \
1402         do {                                                            \
1403                 _NOTE(CONSTANTCONDITION)                                \
1404                 (_qword).eq_u64[0] |=                                   \
1405                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)));     \
1406         _NOTE(CONSTANTCONDITION)                                        \
1407         } while (B_FALSE)
1408
1409 #define EFX_SET_QWORD_BIT32(_qword, _bit)                               \
1410         do {                                                            \
1411                 _NOTE(CONSTANTCONDITION)                                \
1412                 (_qword).eq_u32[0] |=                                   \
1413                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1414                 (_qword).eq_u32[1] |=                                   \
1415                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)));    \
1416         _NOTE(CONSTANTCONDITION)                                        \
1417         } while (B_FALSE)
1418
1419 #define EFX_CLEAR_QWORD_BIT64(_qword, _bit)                             \
1420         do {                                                            \
1421                 _NOTE(CONSTANTCONDITION)                                \
1422                 (_qword).eq_u64[0] &=                                   \
1423                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(0)));    \
1424         _NOTE(CONSTANTCONDITION)                                        \
1425         } while (B_FALSE)
1426
1427 #define EFX_CLEAR_QWORD_BIT32(_qword, _bit)                             \
1428         do {                                                            \
1429                 _NOTE(CONSTANTCONDITION)                                \
1430                 (_qword).eq_u32[0] &=                                   \
1431                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1432                 (_qword).eq_u32[1] &=                                   \
1433                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(32)));   \
1434         _NOTE(CONSTANTCONDITION)                                        \
1435         } while (B_FALSE)
1436
1437 #define EFX_TEST_QWORD_BIT64(_qword, _bit)                              \
1438         (((_qword).eq_u64[0] &                                          \
1439                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)))) != 0)
1440
1441 #define EFX_TEST_QWORD_BIT32(_qword, _bit)                              \
1442         (((_qword).eq_u32[0] &                                          \
1443                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) ||  \
1444         ((_qword).eq_u32[1] &                                           \
1445                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)))))
1446
1447
1448 #define EFX_SET_DWORD_BIT(_dword, _bit)                                 \
1449         do {                                                            \
1450                 (_dword).ed_u32[0] |=                                   \
1451                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1452         _NOTE(CONSTANTCONDITION)                                        \
1453         } while (B_FALSE)
1454
1455 #define EFX_CLEAR_DWORD_BIT(_dword, _bit)                               \
1456         do {                                                            \
1457                 (_dword).ed_u32[0] &=                                   \
1458                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1459         _NOTE(CONSTANTCONDITION)                                        \
1460         } while (B_FALSE)
1461
1462 #define EFX_TEST_DWORD_BIT(_dword, _bit)                                \
1463         (((_dword).ed_u32[0] &                                          \
1464                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) != 0)
1465
1466
1467 #define EFX_SET_WORD_BIT(_word, _bit)                                   \
1468         do {                                                            \
1469                 (_word).ew_u16[0] |=                                    \
1470                     __CPU_TO_LE_16(EFX_SHIFT16(_bit, FIX_LINT(0)));     \
1471         _NOTE(CONSTANTCONDITION)                                        \
1472         } while (B_FALSE)
1473
1474 #define EFX_CLEAR_WORD_BIT(_word, _bit)                                 \
1475         do {                                                            \
1476                 (_word).ew_u32[0] &=                                    \
1477                     __CPU_TO_LE_16(~EFX_SHIFT16(_bit, FIX_LINT(0)));    \
1478         _NOTE(CONSTANTCONDITION)                                        \
1479         } while (B_FALSE)
1480
1481 #define EFX_TEST_WORD_BIT(_word, _bit)                                  \
1482         (((_word).ew_u16[0] &                                           \
1483                     __CPU_TO_LE_16(EFX_SHIFT16(_bit, FIX_LINT(0)))) != 0)
1484
1485
1486 #define EFX_SET_BYTE_BIT(_byte, _bit)                                   \
1487         do {                                                            \
1488                 (_byte).eb_u8[0] |=                                     \
1489                     __NATIVE_8(EFX_SHIFT8(_bit, FIX_LINT(0)));          \
1490         _NOTE(CONSTANTCONDITION)                                        \
1491         } while (B_FALSE)
1492
1493 #define EFX_CLEAR_BYTE_BIT(_byte, _bit)                                 \
1494         do {                                                            \
1495                 (_byte).eb_u8[0] &=                                     \
1496                     __NATIVE_8(~EFX_SHIFT8(_bit, FIX_LINT(0)));         \
1497         _NOTE(CONSTANTCONDITION)                                        \
1498         } while (B_FALSE)
1499
1500 #define EFX_TEST_BYTE_BIT(_byte, _bit)                                  \
1501         (((_byte).eb_u8[0] &                                            \
1502                     __NATIVE_8(EFX_SHIFT8(_bit, FIX_LINT(0)))) != 0)
1503
1504
1505 #define EFX_OR_OWORD64(_oword1, _oword2)                                \
1506         do {                                                            \
1507                 (_oword1).eo_u64[0] |= (_oword2).eo_u64[0];             \
1508                 (_oword1).eo_u64[1] |= (_oword2).eo_u64[1];             \
1509         _NOTE(CONSTANTCONDITION)                                        \
1510         } while (B_FALSE)
1511
1512 #define EFX_OR_OWORD32(_oword1, _oword2)                                \
1513         do {                                                            \
1514                 (_oword1).eo_u32[0] |= (_oword2).eo_u32[0];             \
1515                 (_oword1).eo_u32[1] |= (_oword2).eo_u32[1];             \
1516                 (_oword1).eo_u32[2] |= (_oword2).eo_u32[2];             \
1517                 (_oword1).eo_u32[3] |= (_oword2).eo_u32[3];             \
1518         _NOTE(CONSTANTCONDITION)                                        \
1519         } while (B_FALSE)
1520
1521 #define EFX_AND_OWORD64(_oword1, _oword2)                               \
1522         do {                                                            \
1523                 (_oword1).eo_u64[0] &= (_oword2).eo_u64[0];             \
1524                 (_oword1).eo_u64[1] &= (_oword2).eo_u64[1];             \
1525         _NOTE(CONSTANTCONDITION)                                        \
1526         } while (B_FALSE)
1527
1528 #define EFX_AND_OWORD32(_oword1, _oword2)                               \
1529         do {                                                            \
1530                 (_oword1).eo_u32[0] &= (_oword2).eo_u32[0];             \
1531                 (_oword1).eo_u32[1] &= (_oword2).eo_u32[1];             \
1532                 (_oword1).eo_u32[2] &= (_oword2).eo_u32[2];             \
1533                 (_oword1).eo_u32[3] &= (_oword2).eo_u32[3];             \
1534         _NOTE(CONSTANTCONDITION)                                        \
1535         } while (B_FALSE)
1536
1537 #define EFX_OR_QWORD64(_qword1, _qword2)                                \
1538         do {                                                            \
1539                 (_qword1).eq_u64[0] |= (_qword2).eq_u64[0];             \
1540         _NOTE(CONSTANTCONDITION)                                        \
1541         } while (B_FALSE)
1542
1543 #define EFX_OR_QWORD32(_qword1, _qword2)                                \
1544         do {                                                            \
1545                 (_qword1).eq_u32[0] |= (_qword2).eq_u32[0];             \
1546                 (_qword1).eq_u32[1] |= (_qword2).eq_u32[1];             \
1547         _NOTE(CONSTANTCONDITION)                                        \
1548         } while (B_FALSE)
1549
1550 #define EFX_AND_QWORD64(_qword1, _qword2)                               \
1551         do {                                                            \
1552                 (_qword1).eq_u64[0] &= (_qword2).eq_u64[0];             \
1553         _NOTE(CONSTANTCONDITION)                                        \
1554         } while (B_FALSE)
1555
1556 #define EFX_AND_QWORD32(_qword1, _qword2)                               \
1557         do {                                                            \
1558                 (_qword1).eq_u32[0] &= (_qword2).eq_u32[0];             \
1559                 (_qword1).eq_u32[1] &= (_qword2).eq_u32[1];             \
1560         _NOTE(CONSTANTCONDITION)                                        \
1561         } while (B_FALSE)
1562
1563 #define EFX_OR_DWORD(_dword1, _dword2)                                  \
1564         do {                                                            \
1565                 (_dword1).ed_u32[0] |= (_dword2).ed_u32[0];             \
1566         _NOTE(CONSTANTCONDITION)                                        \
1567         } while (B_FALSE)
1568
1569 #define EFX_AND_DWORD(_dword1, _dword2)                                 \
1570         do {                                                            \
1571                 (_dword1).ed_u32[0] &= (_dword2).ed_u32[0];             \
1572         _NOTE(CONSTANTCONDITION)                                        \
1573         } while (B_FALSE)
1574
1575 #define EFX_OR_WORD(_word1, _word2)                                     \
1576         do {                                                            \
1577                 (_word1).ew_u16[0] |= (_word2).ew_u16[0];               \
1578         _NOTE(CONSTANTCONDITION)                                        \
1579         } while (B_FALSE)
1580
1581 #define EFX_AND_WORD(_word1, _word2)                                    \
1582         do {                                                            \
1583                 (_word1).ew_u16[0] &= (_word2).ew_u16[0];               \
1584         _NOTE(CONSTANTCONDITION)                                        \
1585         } while (B_FALSE)
1586
1587 #define EFX_OR_BYTE(_byte1, _byte2)                                     \
1588         do {                                                            \
1589                 (_byte1).eb_u8[0] |= (_byte2).eb_u8[0];                 \
1590         _NOTE(CONSTANTCONDITION)                                        \
1591         } while (B_FALSE)
1592
1593 #define EFX_AND_BYTE(_byte1, _byte2)                                    \
1594         do {                                                            \
1595                 (_byte1).eb_u8[0] &= (_byte2).eb_u8[0];                 \
1596         _NOTE(CONSTANTCONDITION)                                        \
1597         } while (B_FALSE)
1598
1599 #if EFSYS_USE_UINT64
1600 #define EFX_OWORD_FIELD         EFX_OWORD_FIELD64
1601 #define EFX_QWORD_FIELD         EFX_QWORD_FIELD64
1602 #define EFX_OWORD_IS_EQUAL      EFX_OWORD_IS_EQUAL64
1603 #define EFX_QWORD_IS_EQUAL      EFX_QWORD_IS_EQUAL64
1604 #define EFX_OWORD_IS_ZERO       EFX_OWORD_IS_ZERO64
1605 #define EFX_QWORD_IS_ZERO       EFX_QWORD_IS_ZERO64
1606 #define EFX_OWORD_IS_SET        EFX_OWORD_IS_SET64
1607 #define EFX_QWORD_IS_SET        EFX_QWORD_IS_SET64
1608 #define EFX_POPULATE_OWORD      EFX_POPULATE_OWORD64
1609 #define EFX_POPULATE_QWORD      EFX_POPULATE_QWORD64
1610 #define EFX_SET_OWORD_FIELD     EFX_SET_OWORD_FIELD64
1611 #define EFX_SET_QWORD_FIELD     EFX_SET_QWORD_FIELD64
1612 #define EFX_SET_OWORD_BIT       EFX_SET_OWORD_BIT64
1613 #define EFX_CLEAR_OWORD_BIT     EFX_CLEAR_OWORD_BIT64
1614 #define EFX_TEST_OWORD_BIT      EFX_TEST_OWORD_BIT64
1615 #define EFX_SET_QWORD_BIT       EFX_SET_QWORD_BIT64
1616 #define EFX_CLEAR_QWORD_BIT     EFX_CLEAR_QWORD_BIT64
1617 #define EFX_TEST_QWORD_BIT      EFX_TEST_QWORD_BIT64
1618 #define EFX_OR_OWORD            EFX_OR_OWORD64
1619 #define EFX_AND_OWORD           EFX_AND_OWORD64
1620 #define EFX_OR_QWORD            EFX_OR_QWORD64
1621 #define EFX_AND_QWORD           EFX_AND_QWORD64
1622 #else
1623 #define EFX_OWORD_FIELD         EFX_OWORD_FIELD32
1624 #define EFX_QWORD_FIELD         EFX_QWORD_FIELD32
1625 #define EFX_OWORD_IS_EQUAL      EFX_OWORD_IS_EQUAL32
1626 #define EFX_QWORD_IS_EQUAL      EFX_QWORD_IS_EQUAL32
1627 #define EFX_OWORD_IS_ZERO       EFX_OWORD_IS_ZERO32
1628 #define EFX_QWORD_IS_ZERO       EFX_QWORD_IS_ZERO32
1629 #define EFX_OWORD_IS_SET        EFX_OWORD_IS_SET32
1630 #define EFX_QWORD_IS_SET        EFX_QWORD_IS_SET32
1631 #define EFX_POPULATE_OWORD      EFX_POPULATE_OWORD32
1632 #define EFX_POPULATE_QWORD      EFX_POPULATE_QWORD32
1633 #define EFX_SET_OWORD_FIELD     EFX_SET_OWORD_FIELD32
1634 #define EFX_SET_QWORD_FIELD     EFX_SET_QWORD_FIELD32
1635 #define EFX_SET_OWORD_BIT       EFX_SET_OWORD_BIT32
1636 #define EFX_CLEAR_OWORD_BIT     EFX_CLEAR_OWORD_BIT32
1637 #define EFX_TEST_OWORD_BIT      EFX_TEST_OWORD_BIT32
1638 #define EFX_SET_QWORD_BIT       EFX_SET_QWORD_BIT32
1639 #define EFX_CLEAR_QWORD_BIT     EFX_CLEAR_QWORD_BIT32
1640 #define EFX_TEST_QWORD_BIT      EFX_TEST_QWORD_BIT32
1641 #define EFX_OR_OWORD            EFX_OR_OWORD32
1642 #define EFX_AND_OWORD           EFX_AND_OWORD32
1643 #define EFX_OR_QWORD            EFX_OR_QWORD32
1644 #define EFX_AND_QWORD           EFX_AND_QWORD32
1645 #endif
1646
1647 #ifdef  __cplusplus
1648 }
1649 #endif
1650
1651 #endif  /* _SYS_EFX_TYPES_H */