sys/arm64/arm64/identcpu.c

   1 /*-
   2  * Copyright (c) 2014 Andrew Turner
   3  * Copyright (c) 2014 The FreeBSD Foundation
   4  * All rights reserved.
   5  *
   6  * Portions of this software were developed by Semihalf
   7  * under sponsorship of the FreeBSD Foundation.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  *
  30  */
  31
  32 #include <sys/cdefs.h>
  33 __FBSDID("$FreeBSD$");
  34
  35 #include <sys/param.h>
  36 #include <sys/kernel.h>
  37 #include <sys/malloc.h>
  38 #include <sys/pcpu.h>
  39 #include <sys/sbuf.h>
  40 #include <sys/smp.h>
  41 #include <sys/sysctl.h>
  42 #include <sys/systm.h>
  43
  44 #include <machine/atomic.h>
  45 #include <machine/cpu.h>
  46 #include <machine/cpufunc.h>
  47 #include <machine/elf.h>
  48 #include <machine/md_var.h>
  49 #include <machine/undefined.h>
  50
  51 static MALLOC_DEFINE(M_IDENTCPU, "CPU ID", "arm64 CPU identification memory");
  52
  53 struct cpu_desc;
  54
  55 static void print_cpu_midr(struct sbuf *sb, u_int cpu);
  56 static void print_cpu_features(u_int cpu, struct cpu_desc *desc,
  57     struct cpu_desc *prev_desc);
  58 static void print_cpu_caches(struct sbuf *sb, struct cpu_desc *desc);
  59 #ifdef COMPAT_FREEBSD32
  60 static u_long parse_cpu_features_hwcap32(void);
  61 #endif
  62
  63 char machine[] = "arm64";
  64
  65 #ifdef SCTL_MASK32
  66 extern int adaptive_machine_arch;
  67 #endif
  68
  69 static SYSCTL_NODE(_machdep, OID_AUTO, cache, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
  70     "Cache management tuning");
  71
  72 static int allow_dic = 1;
  73 SYSCTL_INT(_machdep_cache, OID_AUTO, allow_dic, CTLFLAG_RDTUN, &allow_dic, 0,
  74     "Allow optimizations based on the DIC cache bit");
  75
  76 static int allow_idc = 1;
  77 SYSCTL_INT(_machdep_cache, OID_AUTO, allow_idc, CTLFLAG_RDTUN, &allow_idc, 0,
  78     "Allow optimizations based on the IDC cache bit");
  79
  80 static void check_cpu_regs(u_int cpu, struct cpu_desc *desc,
  81     struct cpu_desc *prev_desc);
  82
  83 /*
  84  * The default implementation of I-cache sync assumes we have an
  85  * aliasing cache until we know otherwise.
  86  */
  87 void (*arm64_icache_sync_range)(vm_offset_t, vm_size_t) =
  88     &arm64_aliasing_icache_sync_range;
  89
  90 static int
  91 sysctl_hw_machine(SYSCTL_HANDLER_ARGS)
  92 {
  93 #ifdef SCTL_MASK32
  94         static const char machine32[] = "arm";
  95 #endif
  96         int error;
  97
  98 #ifdef SCTL_MASK32
  99         if ((req->flags & SCTL_MASK32) != 0 && adaptive_machine_arch)
 100                 error = SYSCTL_OUT(req, machine32, sizeof(machine32));
 101         else
 102 #endif
 103                 error = SYSCTL_OUT(req, machine, sizeof(machine));
 104         return (error);
 105 }
 106
 107 SYSCTL_PROC(_hw, HW_MACHINE, machine, CTLTYPE_STRING | CTLFLAG_RD |
 108          CTLFLAG_CAPRD | CTLFLAG_MPSAFE, NULL, 0, sysctl_hw_machine, "A",
 109          "Machine class");
 110
 111 static char cpu_model[64];
 112 SYSCTL_STRING(_hw, HW_MODEL, model, CTLFLAG_RD,
 113         cpu_model, sizeof(cpu_model), "Machine model");
 114
 115 #define MAX_CACHES      8       /* Maximum number of caches supported
 116                                    architecturally. */
 117 /*
 118  * Per-CPU affinity as provided in MPIDR_EL1
 119  * Indexed by CPU number in logical order selected by the system.
 120  * Relevant fields can be extracted using CPU_AFFn macros,
 121  * Aff3.Aff2.Aff1.Aff0 construct a unique CPU address in the system.
 122  *
 123  * Fields used by us:
 124  * Aff1 - Cluster number
 125  * Aff0 - CPU number in Aff1 cluster
 126  */
 127 uint64_t __cpu_affinity[MAXCPU];
 128 static u_int cpu_aff_levels;
 129
 130 struct cpu_desc {
 131         uint64_t        mpidr;
 132         uint64_t        id_aa64afr0;
 133         uint64_t        id_aa64afr1;
 134         uint64_t        id_aa64dfr0;
 135         uint64_t        id_aa64dfr1;
 136         uint64_t        id_aa64isar0;
 137         uint64_t        id_aa64isar1;
 138         uint64_t        id_aa64isar2;
 139         uint64_t        id_aa64mmfr0;
 140         uint64_t        id_aa64mmfr1;
 141         uint64_t        id_aa64mmfr2;
 142         uint64_t        id_aa64pfr0;
 143         uint64_t        id_aa64pfr1;
 144         uint64_t        id_aa64zfr0;
 145         uint64_t        ctr;
 146 #ifdef COMPAT_FREEBSD32
 147         uint64_t        id_isar5;
 148         uint64_t        mvfr0;
 149         uint64_t        mvfr1;
 150 #endif
 151         uint64_t        clidr;
 152         uint32_t        ccsidr[MAX_CACHES][2]; /* 2 possible types. */
 153         bool            have_sve;
 154 };
 155
 156 static struct cpu_desc cpu_desc0;
 157 static struct cpu_desc *cpu_desc;
 158 static struct cpu_desc kern_cpu_desc;
 159 static struct cpu_desc user_cpu_desc;
 160
 161 static struct cpu_desc *
 162 get_cpu_desc(u_int cpu)
 163 {
 164         /* The cpu_desc for CPU 0 is used before the allocator is ready. */
 165         if (cpu == 0)
 166                 return (&cpu_desc0);
 167
 168         MPASS(cpu_desc != NULL);
 169         return (&cpu_desc[cpu - 1]);
 170 }
 171
 172 struct cpu_parts {
 173         u_int           part_id;
 174         const char      *part_name;
 175 };
 176 #define CPU_PART_NONE   { 0, NULL }
 177
 178 struct cpu_implementers {
 179         u_int                   impl_id;
 180         const char              *impl_name;
 181         /*
 182          * Part number is implementation defined
 183          * so each vendor will have its own set of values and names.
 184          */
 185         const struct cpu_parts  *cpu_parts;
 186 };
 187 #define CPU_IMPLEMENTER_NONE    { 0, NULL, NULL }
 188
 189 /*
 190  * Per-implementer table of (PartNum, CPU Name) pairs.
 191  */
 192 /* ARM Ltd. */
 193 static const struct cpu_parts cpu_parts_arm[] = {
 194         { CPU_PART_AEM_V8, "AEMv8" },
 195         { CPU_PART_FOUNDATION, "Foundation-Model" },
 196         { CPU_PART_CORTEX_A34, "Cortex-A34" },
 197         { CPU_PART_CORTEX_A35, "Cortex-A35" },
 198         { CPU_PART_CORTEX_A53, "Cortex-A53" },
 199         { CPU_PART_CORTEX_A55, "Cortex-A55" },
 200         { CPU_PART_CORTEX_A57, "Cortex-A57" },
 201         { CPU_PART_CORTEX_A65, "Cortex-A65" },
 202         { CPU_PART_CORTEX_A65AE, "Cortex-A65AE" },
 203         { CPU_PART_CORTEX_A72, "Cortex-A72" },
 204         { CPU_PART_CORTEX_A73, "Cortex-A73" },
 205         { CPU_PART_CORTEX_A75, "Cortex-A75" },
 206         { CPU_PART_CORTEX_A76, "Cortex-A76" },
 207         { CPU_PART_CORTEX_A76AE, "Cortex-A76AE" },
 208         { CPU_PART_CORTEX_A77, "Cortex-A77" },
 209         { CPU_PART_CORTEX_A78, "Cortex-A78" },
 210         { CPU_PART_CORTEX_A78C, "Cortex-A78C" },
 211         { CPU_PART_CORTEX_A510, "Cortex-A510" },
 212         { CPU_PART_CORTEX_A710, "Cortex-A710" },
 213         { CPU_PART_CORTEX_A715, "Cortex-A715" },
 214         { CPU_PART_CORTEX_X1, "Cortex-X1" },
 215         { CPU_PART_CORTEX_X1C, "Cortex-X1C" },
 216         { CPU_PART_CORTEX_X2, "Cortex-X2" },
 217         { CPU_PART_CORTEX_X3, "Cortex-X3" },
 218         { CPU_PART_NEOVERSE_E1, "Neoverse-E1" },
 219         { CPU_PART_NEOVERSE_N1, "Neoverse-N1" },
 220         { CPU_PART_NEOVERSE_N2, "Neoverse-N2" },
 221         { CPU_PART_NEOVERSE_V1, "Neoverse-V1" },
 222         { CPU_PART_NEOVERSE_V2, "Neoverse-V2" },
 223         CPU_PART_NONE,
 224 };
 225
 226 /* Cavium */
 227 static const struct cpu_parts cpu_parts_cavium[] = {
 228         { CPU_PART_THUNDERX, "ThunderX" },
 229         { CPU_PART_THUNDERX2, "ThunderX2" },
 230         CPU_PART_NONE,
 231 };
 232
 233 /* APM / Ampere */
 234 static const struct cpu_parts cpu_parts_apm[] = {
 235         { CPU_PART_EMAG8180, "eMAG 8180" },
 236         CPU_PART_NONE,
 237 };
 238
 239 /* Qualcomm */
 240 static const struct cpu_parts cpu_parts_qcom[] = {
 241         { CPU_PART_KRYO400_GOLD, "Kryo 400 Gold" },
 242         { CPU_PART_KRYO400_SILVER, "Kryo 400 Silver" },
 243         CPU_PART_NONE,
 244 };
 245
 246 /* Unknown */
 247 static const struct cpu_parts cpu_parts_none[] = {
 248         CPU_PART_NONE,
 249 };
 250
 251 /*
 252  * Implementers table.
 253  */
 254 const struct cpu_implementers cpu_implementers[] = {
 255         { CPU_IMPL_AMPERE,      "Ampere",       cpu_parts_none },
 256         { CPU_IMPL_APPLE,       "Apple",        cpu_parts_none },
 257         { CPU_IMPL_APM,         "APM",          cpu_parts_apm },
 258         { CPU_IMPL_ARM,         "ARM",          cpu_parts_arm },
 259         { CPU_IMPL_BROADCOM,    "Broadcom",     cpu_parts_none },
 260         { CPU_IMPL_CAVIUM,      "Cavium",       cpu_parts_cavium },
 261         { CPU_IMPL_DEC,         "DEC",          cpu_parts_none },
 262         { CPU_IMPL_FREESCALE,   "Freescale",    cpu_parts_none },
 263         { CPU_IMPL_FUJITSU,     "Fujitsu",      cpu_parts_none },
 264         { CPU_IMPL_INFINEON,    "IFX",          cpu_parts_none },
 265         { CPU_IMPL_INTEL,       "Intel",        cpu_parts_none },
 266         { CPU_IMPL_MARVELL,     "Marvell",      cpu_parts_none },
 267         { CPU_IMPL_NVIDIA,      "NVIDIA",       cpu_parts_none },
 268         { CPU_IMPL_QUALCOMM,    "Qualcomm",     cpu_parts_qcom },
 269         CPU_IMPLEMENTER_NONE,
 270 };
 271
 272 #define MRS_TYPE_MASK           0xf
 273 #define MRS_INVALID             0
 274 #define MRS_EXACT               1
 275 #define MRS_EXACT_VAL(x)        (MRS_EXACT | ((x) << 4))
 276 #define MRS_EXACT_FIELD(x)      ((x) >> 4)
 277 #define MRS_LOWER               2
 278
 279 struct mrs_field_value {
 280         uint64_t        value;
 281         const char      *desc;
 282 };
 283
 284 #define MRS_FIELD_VALUE(_value, _desc)                                  \
 285         {                                                               \
 286                 .value = (_value),                                      \
 287                 .desc = (_desc),                                        \
 288         }
 289
 290 #define MRS_FIELD_VALUE_NONE_IMPL(_reg, _field, _none, _impl)           \
 291         MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _none, ""),         \
 292         MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _impl, #_field)
 293
 294 #define MRS_FIELD_VALUE_COUNT(_reg, _field, _desc)                      \
 295         MRS_FIELD_VALUE(0ul << _reg ## _ ## _field ## _SHIFT, "1 " _desc), \
 296         MRS_FIELD_VALUE(1ul << _reg ## _ ## _field ## _SHIFT, "2 " _desc "s"), \
 297         MRS_FIELD_VALUE(2ul << _reg ## _ ## _field ## _SHIFT, "3 " _desc "s"), \
 298         MRS_FIELD_VALUE(3ul << _reg ## _ ## _field ## _SHIFT, "4 " _desc "s"), \
 299         MRS_FIELD_VALUE(4ul << _reg ## _ ## _field ## _SHIFT, "5 " _desc "s"), \
 300         MRS_FIELD_VALUE(5ul << _reg ## _ ## _field ## _SHIFT, "6 " _desc "s"), \
 301         MRS_FIELD_VALUE(6ul << _reg ## _ ## _field ## _SHIFT, "7 " _desc "s"), \
 302         MRS_FIELD_VALUE(7ul << _reg ## _ ## _field ## _SHIFT, "8 " _desc "s"), \
 303         MRS_FIELD_VALUE(8ul << _reg ## _ ## _field ## _SHIFT, "9 " _desc "s"), \
 304         MRS_FIELD_VALUE(9ul << _reg ## _ ## _field ## _SHIFT, "10 "_desc "s"), \
 305         MRS_FIELD_VALUE(10ul<< _reg ## _ ## _field ## _SHIFT, "11 "_desc "s"), \
 306         MRS_FIELD_VALUE(11ul<< _reg ## _ ## _field ## _SHIFT, "12 "_desc "s"), \
 307         MRS_FIELD_VALUE(12ul<< _reg ## _ ## _field ## _SHIFT, "13 "_desc "s"), \
 308         MRS_FIELD_VALUE(13ul<< _reg ## _ ## _field ## _SHIFT, "14 "_desc "s"), \
 309         MRS_FIELD_VALUE(14ul<< _reg ## _ ## _field ## _SHIFT, "15 "_desc "s"), \
 310         MRS_FIELD_VALUE(15ul<< _reg ## _ ## _field ## _SHIFT, "16 "_desc "s")
 311
 312 #define MRS_FIELD_VALUE_END     { .desc = NULL }
 313
 314 struct mrs_field_hwcap {
 315         u_long          *hwcap;
 316         uint64_t        min;
 317         u_long          hwcap_val;
 318 };
 319
 320 #define MRS_HWCAP(_hwcap, _val, _min)                           \
 321 {                                                               \
 322         .hwcap = (_hwcap),                                      \
 323         .hwcap_val = (_val),                                    \
 324         .min = (_min),                                          \
 325 }
 326
 327 #define MRS_HWCAP_END           { .hwcap = NULL }
 328
 329 struct mrs_field {
 330         const char      *name;
 331         struct mrs_field_value *values;
 332         struct mrs_field_hwcap *hwcaps;
 333         uint64_t        mask;
 334         bool            sign;
 335         u_int           type;
 336         u_int           shift;
 337 };
 338
 339 #define MRS_FIELD_HWCAP(_register, _name, _sign, _type, _values, _hwcap) \
 340         {                                                               \
 341                 .name = #_name,                                         \
 342                 .sign = (_sign),                                        \
 343                 .type = (_type),                                        \
 344                 .shift = _register ## _ ## _name ## _SHIFT,             \
 345                 .mask = _register ## _ ## _name ## _MASK,               \
 346                 .values = (_values),                                    \
 347                 .hwcaps = (_hwcap),                                     \
 348         }
 349
 350 #define MRS_FIELD(_register, _name, _sign, _type, _values)              \
 351         MRS_FIELD_HWCAP(_register, _name, _sign, _type, _values, NULL)
 352
 353 #define MRS_FIELD_END   { .type = MRS_INVALID, }
 354
 355 /* ID_AA64AFR0_EL1 */
 356 static struct mrs_field id_aa64afr0_fields[] = {
 357         MRS_FIELD_END,
 358 };
 359
 360
 361 /* ID_AA64AFR1_EL1 */
 362 static struct mrs_field id_aa64afr1_fields[] = {
 363         MRS_FIELD_END,
 364 };
 365
 366
 367 /* ID_AA64DFR0_EL1 */
 368 static struct mrs_field_value id_aa64dfr0_tracefilt[] = {
 369         MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_NONE, ""),
 370         MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_8_4, "Trace v8.4"),
 371         MRS_FIELD_VALUE_END,
 372 };
 373
 374 static struct mrs_field_value id_aa64dfr0_doublelock[] = {
 375         MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_IMPL, "DoubleLock"),
 376         MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_NONE, ""),
 377         MRS_FIELD_VALUE_END,
 378 };
 379
 380 static struct mrs_field_value id_aa64dfr0_pmsver[] = {
 381         MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_NONE, ""),
 382         MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE, "SPE"),
 383         MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_8_3, "SPE v8.3"),
 384         MRS_FIELD_VALUE_END,
 385 };
 386
 387 static struct mrs_field_value id_aa64dfr0_ctx_cmps[] = {
 388         MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, CTX_CMPs, "CTX BKPT"),
 389         MRS_FIELD_VALUE_END,
 390 };
 391
 392 static struct mrs_field_value id_aa64dfr0_wrps[] = {
 393         MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, WRPs, "Watchpoint"),
 394         MRS_FIELD_VALUE_END,
 395 };
 396
 397 static struct mrs_field_value id_aa64dfr0_brps[] = {
 398         MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, BRPs, "Breakpoint"),
 399         MRS_FIELD_VALUE_END,
 400 };
 401
 402 static struct mrs_field_value id_aa64dfr0_pmuver[] = {
 403         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_NONE, ""),
 404         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3, "PMUv3"),
 405         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_1, "PMUv3 v8.1"),
 406         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_4, "PMUv3 v8.4"),
 407         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_5, "PMUv3 v8.5"),
 408         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_IMPL, "IMPL PMU"),
 409         MRS_FIELD_VALUE_END,
 410 };
 411
 412 static struct mrs_field_value id_aa64dfr0_tracever[] = {
 413         MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_NONE, ""),
 414         MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_IMPL, "Trace"),
 415         MRS_FIELD_VALUE_END,
 416 };
 417
 418 static struct mrs_field_value id_aa64dfr0_debugver[] = {
 419         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8, "Debugv8"),
 420         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_VHE, "Debugv8_VHE"),
 421         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_2, "Debugv8.2"),
 422         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_4, "Debugv8.4"),
 423         MRS_FIELD_VALUE_END,
 424 };
 425
 426 static struct mrs_field id_aa64dfr0_fields[] = {
 427         MRS_FIELD(ID_AA64DFR0, TraceFilt, false, MRS_EXACT,
 428             id_aa64dfr0_tracefilt),
 429         MRS_FIELD(ID_AA64DFR0, DoubleLock, false, MRS_EXACT,
 430             id_aa64dfr0_doublelock),
 431         MRS_FIELD(ID_AA64DFR0, PMSVer, false, MRS_EXACT, id_aa64dfr0_pmsver),
 432         MRS_FIELD(ID_AA64DFR0, CTX_CMPs, false, MRS_EXACT,
 433             id_aa64dfr0_ctx_cmps),
 434         MRS_FIELD(ID_AA64DFR0, WRPs, false, MRS_LOWER, id_aa64dfr0_wrps),
 435         MRS_FIELD(ID_AA64DFR0, BRPs, false, MRS_LOWER, id_aa64dfr0_brps),
 436         MRS_FIELD(ID_AA64DFR0, PMUVer, false, MRS_EXACT, id_aa64dfr0_pmuver),
 437         MRS_FIELD(ID_AA64DFR0, TraceVer, false, MRS_EXACT,
 438             id_aa64dfr0_tracever),
 439         MRS_FIELD(ID_AA64DFR0, DebugVer, false, MRS_EXACT_VAL(0x6),
 440             id_aa64dfr0_debugver),
 441         MRS_FIELD_END,
 442 };
 443
 444
 445 /* ID_AA64DFR1_EL1 */
 446 static struct mrs_field id_aa64dfr1_fields[] = {
 447         MRS_FIELD_END,
 448 };
 449
 450
 451 /* ID_AA64ISAR0_EL1 */
 452 static struct mrs_field_value id_aa64isar0_rndr[] = {
 453         MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_NONE, ""),
 454         MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_IMPL, "RNG"),
 455         MRS_FIELD_VALUE_END,
 456 };
 457
 458 static struct mrs_field_hwcap id_aa64isar0_rndr_caps[] = {
 459         MRS_HWCAP(&elf_hwcap2, HWCAP2_RNG, ID_AA64ISAR0_RNDR_IMPL),
 460         MRS_HWCAP_END
 461 };
 462
 463 static struct mrs_field_value id_aa64isar0_tlb[] = {
 464         MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_NONE, ""),
 465         MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOS, "TLBI-OS"),
 466         MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOSR, "TLBI-OSR"),
 467         MRS_FIELD_VALUE_END,
 468 };
 469
 470 static struct mrs_field_value id_aa64isar0_ts[] = {
 471         MRS_FIELD_VALUE(ID_AA64ISAR0_TS_NONE, ""),
 472         MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_4, "CondM-8.4"),
 473         MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_5, "CondM-8.5"),
 474         MRS_FIELD_VALUE_END,
 475 };
 476
 477 static struct mrs_field_hwcap id_aa64isar0_ts_caps[] = {
 478         MRS_HWCAP(&elf_hwcap, HWCAP_FLAGM, ID_AA64ISAR0_TS_CondM_8_4),
 479         MRS_HWCAP(&elf_hwcap2, HWCAP2_FLAGM2, ID_AA64ISAR0_TS_CondM_8_5),
 480         MRS_HWCAP_END
 481 };
 482
 483 static struct mrs_field_value id_aa64isar0_fhm[] = {
 484         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, FHM, NONE, IMPL),
 485         MRS_FIELD_VALUE_END,
 486 };
 487
 488 static struct mrs_field_hwcap id_aa64isar0_fhm_caps[] = {
 489         MRS_HWCAP(&elf_hwcap, HWCAP_ASIMDFHM, ID_AA64ISAR0_FHM_IMPL),
 490         MRS_HWCAP_END
 491 };
 492
 493 static struct mrs_field_value id_aa64isar0_dp[] = {
 494         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, DP, NONE, IMPL),
 495         MRS_FIELD_VALUE_END,
 496 };
 497
 498 static struct mrs_field_hwcap id_aa64isar0_dp_caps[] = {
 499         MRS_HWCAP(&elf_hwcap, HWCAP_ASIMDDP, ID_AA64ISAR0_DP_IMPL),
 500         MRS_HWCAP_END
 501 };
 502
 503 static struct mrs_field_value id_aa64isar0_sm4[] = {
 504         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM4, NONE, IMPL),
 505         MRS_FIELD_VALUE_END,
 506 };
 507
 508 static struct mrs_field_hwcap id_aa64isar0_sm4_caps[] = {
 509         MRS_HWCAP(&elf_hwcap, HWCAP_SM4, ID_AA64ISAR0_SM4_IMPL),
 510         MRS_HWCAP_END
 511 };
 512
 513 static struct mrs_field_value id_aa64isar0_sm3[] = {
 514         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM3, NONE, IMPL),
 515         MRS_FIELD_VALUE_END,
 516 };
 517
 518 static struct mrs_field_hwcap id_aa64isar0_sm3_caps[] = {
 519         MRS_HWCAP(&elf_hwcap, HWCAP_SM3, ID_AA64ISAR0_SM3_IMPL),
 520         MRS_HWCAP_END
 521 };
 522
 523 static struct mrs_field_value id_aa64isar0_sha3[] = {
 524         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA3, NONE, IMPL),
 525         MRS_FIELD_VALUE_END,
 526 };
 527
 528 static struct mrs_field_hwcap id_aa64isar0_sha3_caps[] = {
 529         MRS_HWCAP(&elf_hwcap, HWCAP_SHA3, ID_AA64ISAR0_SHA3_IMPL),
 530         MRS_HWCAP_END
 531 };
 532
 533 static struct mrs_field_value id_aa64isar0_rdm[] = {
 534         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, RDM, NONE, IMPL),
 535         MRS_FIELD_VALUE_END,
 536 };
 537
 538 static struct mrs_field_hwcap id_aa64isar0_rdm_caps[] = {
 539         MRS_HWCAP(&elf_hwcap, HWCAP_ASIMDRDM, ID_AA64ISAR0_RDM_IMPL),
 540         MRS_HWCAP_END
 541 };
 542
 543 static struct mrs_field_value id_aa64isar0_atomic[] = {
 544         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, Atomic, NONE, IMPL),
 545         MRS_FIELD_VALUE_END,
 546 };
 547
 548 static struct mrs_field_hwcap id_aa64isar0_atomic_caps[] = {
 549         MRS_HWCAP(&elf_hwcap, HWCAP_ATOMICS, ID_AA64ISAR0_Atomic_IMPL),
 550         MRS_HWCAP_END
 551 };
 552
 553 static struct mrs_field_value id_aa64isar0_crc32[] = {
 554         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, CRC32, NONE, BASE),
 555         MRS_FIELD_VALUE_END,
 556 };
 557
 558 static struct mrs_field_hwcap id_aa64isar0_crc32_caps[] = {
 559         MRS_HWCAP(&elf_hwcap, HWCAP_CRC32, ID_AA64ISAR0_CRC32_BASE),
 560         MRS_HWCAP_END
 561 };
 562
 563 static struct mrs_field_value id_aa64isar0_sha2[] = {
 564         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA2, NONE, BASE),
 565         MRS_FIELD_VALUE(ID_AA64ISAR0_SHA2_512, "SHA2+SHA512"),
 566         MRS_FIELD_VALUE_END,
 567 };
 568
 569 static struct mrs_field_hwcap id_aa64isar0_sha2_caps[] = {
 570         MRS_HWCAP(&elf_hwcap, HWCAP_SHA2, ID_AA64ISAR0_SHA2_BASE),
 571         MRS_HWCAP(&elf_hwcap, HWCAP_SHA512, ID_AA64ISAR0_SHA2_512),
 572         MRS_HWCAP_END
 573 };
 574
 575 static struct mrs_field_value id_aa64isar0_sha1[] = {
 576         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA1, NONE, BASE),
 577         MRS_FIELD_VALUE_END,
 578 };
 579
 580 static struct mrs_field_hwcap id_aa64isar0_sha1_caps[] = {
 581         MRS_HWCAP(&elf_hwcap, HWCAP_SHA1, ID_AA64ISAR0_SHA1_BASE),
 582         MRS_HWCAP_END
 583 };
 584
 585 static struct mrs_field_value id_aa64isar0_aes[] = {
 586         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, AES, NONE, BASE),
 587         MRS_FIELD_VALUE(ID_AA64ISAR0_AES_PMULL, "AES+PMULL"),
 588         MRS_FIELD_VALUE_END,
 589 };
 590
 591 static struct mrs_field_hwcap id_aa64isar0_aes_caps[] = {
 592         MRS_HWCAP(&elf_hwcap, HWCAP_AES, ID_AA64ISAR0_AES_BASE),
 593         MRS_HWCAP(&elf_hwcap, HWCAP_PMULL, ID_AA64ISAR0_AES_PMULL),
 594         MRS_HWCAP_END
 595 };
 596
 597 static struct mrs_field id_aa64isar0_fields[] = {
 598         MRS_FIELD_HWCAP(ID_AA64ISAR0, RNDR, false, MRS_LOWER,
 599             id_aa64isar0_rndr, id_aa64isar0_rndr_caps),
 600         MRS_FIELD(ID_AA64ISAR0, TLB, false, MRS_EXACT, id_aa64isar0_tlb),
 601         MRS_FIELD_HWCAP(ID_AA64ISAR0, TS, false, MRS_LOWER, id_aa64isar0_ts,
 602             id_aa64isar0_ts_caps),
 603         MRS_FIELD_HWCAP(ID_AA64ISAR0, FHM, false, MRS_LOWER, id_aa64isar0_fhm,
 604             id_aa64isar0_fhm_caps),
 605         MRS_FIELD_HWCAP(ID_AA64ISAR0, DP, false, MRS_LOWER, id_aa64isar0_dp,
 606             id_aa64isar0_dp_caps),
 607         MRS_FIELD_HWCAP(ID_AA64ISAR0, SM4, false, MRS_LOWER, id_aa64isar0_sm4,
 608             id_aa64isar0_sm4_caps),
 609         MRS_FIELD_HWCAP(ID_AA64ISAR0, SM3, false, MRS_LOWER, id_aa64isar0_sm3,
 610             id_aa64isar0_sm3_caps),
 611         MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA3, false, MRS_LOWER, id_aa64isar0_sha3,
 612             id_aa64isar0_sha3_caps),
 613         MRS_FIELD_HWCAP(ID_AA64ISAR0, RDM, false, MRS_LOWER, id_aa64isar0_rdm,
 614             id_aa64isar0_rdm_caps),
 615         MRS_FIELD_HWCAP(ID_AA64ISAR0, Atomic, false, MRS_LOWER,
 616             id_aa64isar0_atomic, id_aa64isar0_atomic_caps),
 617         MRS_FIELD_HWCAP(ID_AA64ISAR0, CRC32, false, MRS_LOWER,
 618             id_aa64isar0_crc32, id_aa64isar0_crc32_caps),
 619         MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA2, false, MRS_LOWER, id_aa64isar0_sha2,
 620             id_aa64isar0_sha2_caps),
 621         MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA1, false, MRS_LOWER,
 622             id_aa64isar0_sha1, id_aa64isar0_sha1_caps),
 623         MRS_FIELD_HWCAP(ID_AA64ISAR0, AES, false, MRS_LOWER, id_aa64isar0_aes,
 624             id_aa64isar0_aes_caps),
 625         MRS_FIELD_END,
 626 };
 627
 628
 629 /* ID_AA64ISAR1_EL1 */
 630 static struct mrs_field_value id_aa64isar1_i8mm[] = {
 631         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, I8MM, NONE, IMPL),
 632         MRS_FIELD_VALUE_END,
 633 };
 634
 635 static struct mrs_field_hwcap id_aa64isar1_i8mm_caps[] = {
 636         MRS_HWCAP(&elf_hwcap2, HWCAP2_I8MM, ID_AA64ISAR1_I8MM_IMPL),
 637         MRS_HWCAP_END
 638 };
 639
 640 static struct mrs_field_value id_aa64isar1_dgh[] = {
 641         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, DGH, NONE, IMPL),
 642         MRS_FIELD_VALUE_END,
 643 };
 644
 645 static struct mrs_field_hwcap id_aa64isar1_dgh_caps[] = {
 646         MRS_HWCAP(&elf_hwcap2, HWCAP2_DGH, ID_AA64ISAR1_DGH_IMPL),
 647         MRS_HWCAP_END
 648 };
 649
 650 static struct mrs_field_value id_aa64isar1_bf16[] = {
 651         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, BF16, NONE, IMPL),
 652         MRS_FIELD_VALUE_END,
 653 };
 654
 655 static struct mrs_field_hwcap id_aa64isar1_bf16_caps[] = {
 656         MRS_HWCAP(&elf_hwcap2, HWCAP2_BF16, ID_AA64ISAR1_BF16_IMPL),
 657         MRS_HWCAP_END
 658 };
 659
 660 static struct mrs_field_value id_aa64isar1_specres[] = {
 661         MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_NONE, ""),
 662         MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_IMPL, "PredInv"),
 663         MRS_FIELD_VALUE_END,
 664 };
 665
 666 static struct mrs_field_value id_aa64isar1_sb[] = {
 667         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, SB, NONE, IMPL),
 668         MRS_FIELD_VALUE_END,
 669 };
 670
 671 static struct mrs_field_hwcap id_aa64isar1_sb_caps[] = {
 672         MRS_HWCAP(&elf_hwcap, HWCAP_SB, ID_AA64ISAR1_SB_IMPL),
 673         MRS_HWCAP_END
 674 };
 675
 676 static struct mrs_field_value id_aa64isar1_frintts[] = {
 677         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FRINTTS, NONE, IMPL),
 678         MRS_FIELD_VALUE_END,
 679 };
 680
 681 static struct mrs_field_hwcap id_aa64isar1_frintts_caps[] = {
 682         MRS_HWCAP(&elf_hwcap2, HWCAP2_FRINT, ID_AA64ISAR1_FRINTTS_IMPL),
 683         MRS_HWCAP_END
 684 };
 685
 686 static struct mrs_field_value id_aa64isar1_gpi[] = {
 687         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPI, NONE, IMPL),
 688         MRS_FIELD_VALUE_END,
 689 };
 690
 691 static struct mrs_field_hwcap id_aa64isar1_gpi_caps[] = {
 692         MRS_HWCAP(&elf_hwcap, HWCAP_PACG, ID_AA64ISAR1_GPI_IMPL),
 693         MRS_HWCAP_END
 694 };
 695
 696 static struct mrs_field_value id_aa64isar1_gpa[] = {
 697         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPA, NONE, IMPL),
 698         MRS_FIELD_VALUE_END,
 699 };
 700
 701 static struct mrs_field_hwcap id_aa64isar1_gpa_caps[] = {
 702         MRS_HWCAP(&elf_hwcap, HWCAP_PACG, ID_AA64ISAR1_GPA_IMPL),
 703         MRS_HWCAP_END
 704 };
 705
 706 static struct mrs_field_value id_aa64isar1_lrcpc[] = {
 707         MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_NONE, ""),
 708         MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_3, "RCPC-8.3"),
 709         MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_4, "RCPC-8.4"),
 710         MRS_FIELD_VALUE_END,
 711 };
 712
 713 static struct mrs_field_hwcap id_aa64isar1_lrcpc_caps[] = {
 714         MRS_HWCAP(&elf_hwcap, HWCAP_LRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_3),
 715         MRS_HWCAP(&elf_hwcap, HWCAP_ILRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_4),
 716         MRS_HWCAP_END
 717 };
 718
 719 static struct mrs_field_value id_aa64isar1_fcma[] = {
 720         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FCMA, NONE, IMPL),
 721         MRS_FIELD_VALUE_END,
 722 };
 723
 724 static struct mrs_field_hwcap id_aa64isar1_fcma_caps[] = {
 725         MRS_HWCAP(&elf_hwcap, HWCAP_FCMA, ID_AA64ISAR1_FCMA_IMPL),
 726         MRS_HWCAP_END
 727 };
 728
 729 static struct mrs_field_value id_aa64isar1_jscvt[] = {
 730         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, JSCVT, NONE, IMPL),
 731         MRS_FIELD_VALUE_END,
 732 };
 733
 734 static struct mrs_field_hwcap id_aa64isar1_jscvt_caps[] = {
 735         MRS_HWCAP(&elf_hwcap, HWCAP_JSCVT, ID_AA64ISAR1_JSCVT_IMPL),
 736         MRS_HWCAP_END
 737 };
 738
 739 static struct mrs_field_value id_aa64isar1_api[] = {
 740         MRS_FIELD_VALUE(ID_AA64ISAR1_API_NONE, ""),
 741         MRS_FIELD_VALUE(ID_AA64ISAR1_API_PAC, "API PAC"),
 742         MRS_FIELD_VALUE(ID_AA64ISAR1_API_EPAC, "API EPAC"),
 743         MRS_FIELD_VALUE(ID_AA64ISAR1_API_EPAC2, "Impl PAuth+EPAC2"),
 744         MRS_FIELD_VALUE(ID_AA64ISAR1_API_FPAC, "Impl PAuth+FPAC"),
 745         MRS_FIELD_VALUE(ID_AA64ISAR1_API_FPAC_COMBINED,
 746             "Impl PAuth+FPAC+Combined"),
 747         MRS_FIELD_VALUE_END,
 748 };
 749
 750 static struct mrs_field_hwcap id_aa64isar1_api_caps[] = {
 751         MRS_HWCAP(&elf_hwcap, HWCAP_PACA, ID_AA64ISAR1_API_PAC),
 752         MRS_HWCAP_END
 753 };
 754
 755 static struct mrs_field_value id_aa64isar1_apa[] = {
 756         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_NONE, ""),
 757         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_PAC, "APA PAC"),
 758         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_EPAC, "APA EPAC"),
 759         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_EPAC2, "APA EPAC2"),
 760         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_FPAC, "APA FPAC"),
 761         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_FPAC_COMBINED,
 762             "APA FPAC+Combined"),
 763         MRS_FIELD_VALUE_END,
 764 };
 765
 766 static struct mrs_field_hwcap id_aa64isar1_apa_caps[] = {
 767         MRS_HWCAP(&elf_hwcap, HWCAP_PACA, ID_AA64ISAR1_APA_PAC),
 768         MRS_HWCAP_END
 769 };
 770
 771 static struct mrs_field_value id_aa64isar1_dpb[] = {
 772         MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_NONE, ""),
 773         MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVAP, "DCPoP"),
 774         MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVADP, "DCCVADP"),
 775         MRS_FIELD_VALUE_END,
 776 };
 777
 778 static struct mrs_field_hwcap id_aa64isar1_dpb_caps[] = {
 779         MRS_HWCAP(&elf_hwcap, HWCAP_DCPOP, ID_AA64ISAR1_DPB_DCCVAP),
 780         MRS_HWCAP(&elf_hwcap2, HWCAP2_DCPODP, ID_AA64ISAR1_DPB_DCCVADP),
 781         MRS_HWCAP_END
 782 };
 783
 784 static struct mrs_field id_aa64isar1_fields[] = {
 785         MRS_FIELD_HWCAP(ID_AA64ISAR1, I8MM, false, MRS_LOWER,
 786             id_aa64isar1_i8mm, id_aa64isar1_i8mm_caps),
 787         MRS_FIELD_HWCAP(ID_AA64ISAR1, DGH, false, MRS_LOWER, id_aa64isar1_dgh,
 788             id_aa64isar1_dgh_caps),
 789         MRS_FIELD_HWCAP(ID_AA64ISAR1, BF16, false, MRS_LOWER,
 790             id_aa64isar1_bf16, id_aa64isar1_bf16_caps),
 791         MRS_FIELD(ID_AA64ISAR1, SPECRES, false, MRS_EXACT,
 792             id_aa64isar1_specres),
 793         MRS_FIELD_HWCAP(ID_AA64ISAR1, SB, false, MRS_LOWER, id_aa64isar1_sb,
 794             id_aa64isar1_sb_caps),
 795         MRS_FIELD_HWCAP(ID_AA64ISAR1, FRINTTS, false, MRS_LOWER,
 796             id_aa64isar1_frintts, id_aa64isar1_frintts_caps),
 797         MRS_FIELD_HWCAP(ID_AA64ISAR1, GPI, false, MRS_EXACT, id_aa64isar1_gpi,
 798             id_aa64isar1_gpi_caps),
 799         MRS_FIELD_HWCAP(ID_AA64ISAR1, GPA, false, MRS_EXACT, id_aa64isar1_gpa,
 800             id_aa64isar1_gpa_caps),
 801         MRS_FIELD_HWCAP(ID_AA64ISAR1, LRCPC, false, MRS_LOWER,
 802             id_aa64isar1_lrcpc, id_aa64isar1_lrcpc_caps),
 803         MRS_FIELD_HWCAP(ID_AA64ISAR1, FCMA, false, MRS_LOWER,
 804             id_aa64isar1_fcma, id_aa64isar1_fcma_caps),
 805         MRS_FIELD_HWCAP(ID_AA64ISAR1, JSCVT, false, MRS_LOWER,
 806             id_aa64isar1_jscvt, id_aa64isar1_jscvt_caps),
 807         MRS_FIELD_HWCAP(ID_AA64ISAR1, API, false, MRS_EXACT, id_aa64isar1_api,
 808             id_aa64isar1_api_caps),
 809         MRS_FIELD_HWCAP(ID_AA64ISAR1, APA, false, MRS_EXACT, id_aa64isar1_apa,
 810             id_aa64isar1_apa_caps),
 811         MRS_FIELD_HWCAP(ID_AA64ISAR1, DPB, false, MRS_LOWER, id_aa64isar1_dpb,
 812             id_aa64isar1_dpb_caps),
 813         MRS_FIELD_END,
 814 };
 815
 816
 817 /* ID_AA64ISAR2_EL1 */
 818 static struct mrs_field_value id_aa64isar2_pac_frac[] = {
 819         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, PAC_frac, NONE, IMPL),
 820         MRS_FIELD_VALUE_END,
 821 };
 822
 823 static struct mrs_field_value id_aa64isar2_bc[] = {
 824         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, BC, NONE, IMPL),
 825         MRS_FIELD_VALUE_END,
 826 };
 827
 828 static struct mrs_field_value id_aa64isar2_mops[] = {
 829         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, MOPS, NONE, IMPL),
 830         MRS_FIELD_VALUE_END,
 831 };
 832
 833 static struct mrs_field_value id_aa64isar2_apa3[] = {
 834         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_NONE, ""),
 835         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_PAC, "APA3 PAC"),
 836         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_EPAC, "APA3 EPAC"),
 837         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_EPAC2, "APA3 EPAC2"),
 838         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_FPAC, "APA3 FPAC"),
 839         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_FPAC_COMBINED,
 840             "APA3 FPAC+Combined"),
 841         MRS_FIELD_VALUE_END,
 842 };
 843
 844 static struct mrs_field_hwcap id_aa64isar2_apa3_caps[] = {
 845         MRS_HWCAP(&elf_hwcap, HWCAP_PACA, ID_AA64ISAR2_APA3_PAC),
 846         MRS_HWCAP_END
 847 };
 848
 849 static struct mrs_field_value id_aa64isar2_gpa3[] = {
 850         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, GPA3, NONE, IMPL),
 851         MRS_FIELD_VALUE_END,
 852 };
 853
 854 static struct mrs_field_hwcap id_aa64isar2_gpa3_caps[] = {
 855         MRS_HWCAP(&elf_hwcap, HWCAP_PACG, ID_AA64ISAR2_GPA3_IMPL),
 856         MRS_HWCAP_END
 857 };
 858
 859 static struct mrs_field_value id_aa64isar2_rpres[] = {
 860         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, RPRES, NONE, IMPL),
 861         MRS_FIELD_VALUE_END,
 862 };
 863
 864 static struct mrs_field_value id_aa64isar2_wfxt[] = {
 865         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, WFxT, NONE, IMPL),
 866         MRS_FIELD_VALUE_END,
 867 };
 868
 869 static struct mrs_field id_aa64isar2_fields[] = {
 870         MRS_FIELD(ID_AA64ISAR2, PAC_frac, false, MRS_EXACT,
 871             id_aa64isar2_pac_frac),
 872         MRS_FIELD(ID_AA64ISAR2, BC, false, MRS_EXACT, id_aa64isar2_bc),
 873         MRS_FIELD(ID_AA64ISAR2, MOPS, false, MRS_EXACT, id_aa64isar2_mops),
 874         MRS_FIELD_HWCAP(ID_AA64ISAR2, APA3, false, MRS_EXACT,
 875             id_aa64isar2_apa3, id_aa64isar2_apa3_caps),
 876         MRS_FIELD_HWCAP(ID_AA64ISAR2, GPA3, false, MRS_EXACT,
 877             id_aa64isar2_gpa3, id_aa64isar2_gpa3_caps),
 878         MRS_FIELD(ID_AA64ISAR2, RPRES, false, MRS_EXACT, id_aa64isar2_rpres),
 879         MRS_FIELD(ID_AA64ISAR2, WFxT, false, MRS_EXACT, id_aa64isar2_wfxt),
 880         MRS_FIELD_END,
 881 };
 882
 883
 884 /* ID_AA64MMFR0_EL1 */
 885 static struct mrs_field_value id_aa64mmfr0_exs[] = {
 886         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, ExS, ALL, IMPL),
 887         MRS_FIELD_VALUE_END,
 888 };
 889
 890 static struct mrs_field_value id_aa64mmfr0_tgran4_2[] = {
 891         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_TGran4, ""),
 892         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_NONE, "No S2 TGran4"),
 893         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_IMPL, "S2 TGran4"),
 894         MRS_FIELD_VALUE_END,
 895 };
 896
 897 static struct mrs_field_value id_aa64mmfr0_tgran64_2[] = {
 898         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_TGran64, ""),
 899         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_NONE, "No S2 TGran64"),
 900         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_IMPL, "S2 TGran64"),
 901         MRS_FIELD_VALUE_END,
 902 };
 903
 904 static struct mrs_field_value id_aa64mmfr0_tgran16_2[] = {
 905         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_TGran16, ""),
 906         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_NONE, "No S2 TGran16"),
 907         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_IMPL, "S2 TGran16"),
 908         MRS_FIELD_VALUE_END,
 909 };
 910
 911 static struct mrs_field_value id_aa64mmfr0_tgran4[] = {
 912         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran4,NONE, IMPL),
 913         MRS_FIELD_VALUE_END,
 914 };
 915
 916 static struct mrs_field_value id_aa64mmfr0_tgran64[] = {
 917         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran64, NONE, IMPL),
 918         MRS_FIELD_VALUE_END,
 919 };
 920
 921 static struct mrs_field_value id_aa64mmfr0_tgran16[] = {
 922         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran16, NONE, IMPL),
 923         MRS_FIELD_VALUE_END,
 924 };
 925
 926 static struct mrs_field_value id_aa64mmfr0_bigendel0[] = {
 927         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEndEL0, FIXED, MIXED),
 928         MRS_FIELD_VALUE_END,
 929 };
 930
 931 static struct mrs_field_value id_aa64mmfr0_snsmem[] = {
 932         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, SNSMem, NONE, DISTINCT),
 933         MRS_FIELD_VALUE_END,
 934 };
 935
 936 static struct mrs_field_value id_aa64mmfr0_bigend[] = {
 937         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEnd, FIXED, MIXED),
 938         MRS_FIELD_VALUE_END,
 939 };
 940
 941 static struct mrs_field_value id_aa64mmfr0_asidbits[] = {
 942         MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_8, "8bit ASID"),
 943         MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_16, "16bit ASID"),
 944         MRS_FIELD_VALUE_END,
 945 };
 946
 947 static struct mrs_field_value id_aa64mmfr0_parange[] = {
 948         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4G, "4GB PA"),
 949         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_64G, "64GB PA"),
 950         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_1T, "1TB PA"),
 951         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4T, "4TB PA"),
 952         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_16T, "16TB PA"),
 953         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_256T, "256TB PA"),
 954         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4P, "4PB PA"),
 955         MRS_FIELD_VALUE_END,
 956 };
 957
 958 static struct mrs_field id_aa64mmfr0_fields[] = {
 959         MRS_FIELD(ID_AA64MMFR0, ExS, false, MRS_EXACT, id_aa64mmfr0_exs),
 960         MRS_FIELD(ID_AA64MMFR0, TGran4_2, false, MRS_EXACT,
 961             id_aa64mmfr0_tgran4_2),
 962         MRS_FIELD(ID_AA64MMFR0, TGran64_2, false, MRS_EXACT,
 963             id_aa64mmfr0_tgran64_2),
 964         MRS_FIELD(ID_AA64MMFR0, TGran16_2, false, MRS_EXACT,
 965             id_aa64mmfr0_tgran16_2),
 966         MRS_FIELD(ID_AA64MMFR0, TGran4, false, MRS_EXACT, id_aa64mmfr0_tgran4),
 967         MRS_FIELD(ID_AA64MMFR0, TGran64, false, MRS_EXACT,
 968             id_aa64mmfr0_tgran64),
 969         MRS_FIELD(ID_AA64MMFR0, TGran16, false, MRS_EXACT,
 970             id_aa64mmfr0_tgran16),
 971         MRS_FIELD(ID_AA64MMFR0, BigEndEL0, false, MRS_EXACT,
 972             id_aa64mmfr0_bigendel0),
 973         MRS_FIELD(ID_AA64MMFR0, SNSMem, false, MRS_EXACT, id_aa64mmfr0_snsmem),
 974         MRS_FIELD(ID_AA64MMFR0, BigEnd, false, MRS_EXACT, id_aa64mmfr0_bigend),
 975         MRS_FIELD(ID_AA64MMFR0, ASIDBits, false, MRS_EXACT,
 976             id_aa64mmfr0_asidbits),
 977         MRS_FIELD(ID_AA64MMFR0, PARange, false, MRS_EXACT,
 978             id_aa64mmfr0_parange),
 979         MRS_FIELD_END,
 980 };
 981
 982
 983 /* ID_AA64MMFR1_EL1 */
 984 static struct mrs_field_value id_aa64mmfr1_xnx[] = {
 985         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, XNX, NONE, IMPL),
 986         MRS_FIELD_VALUE_END,
 987 };
 988
 989 static struct mrs_field_value id_aa64mmfr1_specsei[] = {
 990         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, SpecSEI, NONE, IMPL),
 991         MRS_FIELD_VALUE_END,
 992 };
 993
 994 static struct mrs_field_value id_aa64mmfr1_pan[] = {
 995         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, PAN, NONE, IMPL),
 996         MRS_FIELD_VALUE(ID_AA64MMFR1_PAN_ATS1E1, "PAN+ATS1E1"),
 997         MRS_FIELD_VALUE_END,
 998 };
 999
1000 static struct mrs_field_value id_aa64mmfr1_lo[] = {
1001         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, LO, NONE, IMPL),
1002         MRS_FIELD_VALUE_END,
1003 };
1004
1005 static struct mrs_field_value id_aa64mmfr1_hpds[] = {
1006         MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_NONE, ""),
1007         MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_HPD, "HPD"),
1008         MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_TTPBHA, "HPD+TTPBHA"),
1009         MRS_FIELD_VALUE_END,
1010 };
1011
1012 static struct mrs_field_value id_aa64mmfr1_vh[] = {
1013         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, VH, NONE, IMPL),
1014         MRS_FIELD_VALUE_END,
1015 };
1016
1017 static struct mrs_field_value id_aa64mmfr1_vmidbits[] = {
1018         MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_8, "8bit VMID"),
1019         MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_16, "16bit VMID"),
1020         MRS_FIELD_VALUE_END,
1021 };
1022
1023 static struct mrs_field_value id_aa64mmfr1_hafdbs[] = {
1024         MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_NONE, ""),
1025         MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF, "HAF"),
1026         MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF_DBS, "HAF+DS"),
1027         MRS_FIELD_VALUE_END,
1028 };
1029
1030 static struct mrs_field id_aa64mmfr1_fields[] = {
1031         MRS_FIELD(ID_AA64MMFR1, XNX, false, MRS_EXACT, id_aa64mmfr1_xnx),
1032         MRS_FIELD(ID_AA64MMFR1, SpecSEI, false, MRS_EXACT,
1033             id_aa64mmfr1_specsei),
1034         MRS_FIELD(ID_AA64MMFR1, PAN, false, MRS_EXACT, id_aa64mmfr1_pan),
1035         MRS_FIELD(ID_AA64MMFR1, LO, false, MRS_EXACT, id_aa64mmfr1_lo),
1036         MRS_FIELD(ID_AA64MMFR1, HPDS, false, MRS_EXACT, id_aa64mmfr1_hpds),
1037         MRS_FIELD(ID_AA64MMFR1, VH, false, MRS_EXACT, id_aa64mmfr1_vh),
1038         MRS_FIELD(ID_AA64MMFR1, VMIDBits, false, MRS_EXACT,
1039             id_aa64mmfr1_vmidbits),
1040         MRS_FIELD(ID_AA64MMFR1, HAFDBS, false, MRS_EXACT, id_aa64mmfr1_hafdbs),
1041         MRS_FIELD_END,
1042 };
1043
1044
1045 /* ID_AA64MMFR2_EL1 */
1046 static struct mrs_field_value id_aa64mmfr2_e0pd[] = {
1047         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, E0PD, NONE, IMPL),
1048         MRS_FIELD_VALUE_END,
1049 };
1050
1051 static struct mrs_field_value id_aa64mmfr2_evt[] = {
1052         MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_NONE, ""),
1053         MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_2, "EVT-8.2"),
1054         MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_5, "EVT-8.5"),
1055         MRS_FIELD_VALUE_END,
1056 };
1057
1058 static struct mrs_field_value id_aa64mmfr2_bbm[] = {
1059         MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL0, ""),
1060         MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL1, "BBM level 1"),
1061         MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL2, "BBM level 2"),
1062         MRS_FIELD_VALUE_END,
1063 };
1064
1065 static struct mrs_field_value id_aa64mmfr2_ttl[] = {
1066         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, TTL, NONE, IMPL),
1067         MRS_FIELD_VALUE_END,
1068 };
1069
1070 static struct mrs_field_value id_aa64mmfr2_fwb[] = {
1071         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, FWB, NONE, IMPL),
1072         MRS_FIELD_VALUE_END,
1073 };
1074
1075 static struct mrs_field_value id_aa64mmfr2_ids[] = {
1076         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IDS, NONE, IMPL),
1077         MRS_FIELD_VALUE_END,
1078 };
1079
1080 static struct mrs_field_value id_aa64mmfr2_at[] = {
1081         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, AT, NONE, IMPL),
1082         MRS_FIELD_VALUE_END,
1083 };
1084
1085 static struct mrs_field_hwcap id_aa64mmfr2_at_caps[] = {
1086         MRS_HWCAP(&elf_hwcap, HWCAP_USCAT, ID_AA64MMFR2_AT_IMPL),
1087         MRS_HWCAP_END
1088 };
1089
1090 static struct mrs_field_value id_aa64mmfr2_st[] = {
1091         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, ST, NONE, IMPL),
1092         MRS_FIELD_VALUE_END,
1093 };
1094
1095 static struct mrs_field_value id_aa64mmfr2_nv[] = {
1096         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, NV, NONE, 8_3),
1097         MRS_FIELD_VALUE(ID_AA64MMFR2_NV_8_4, "NV v8.4"),
1098         MRS_FIELD_VALUE_END,
1099 };
1100
1101 static struct mrs_field_value id_aa64mmfr2_ccidx[] = {
1102         MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_32, "32bit CCIDX"),
1103         MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_64, "64bit CCIDX"),
1104         MRS_FIELD_VALUE_END,
1105 };
1106
1107 static struct mrs_field_value id_aa64mmfr2_varange[] = {
1108         MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_48, "48bit VA"),
1109         MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_52, "52bit VA"),
1110         MRS_FIELD_VALUE_END,
1111 };
1112
1113 static struct mrs_field_value id_aa64mmfr2_iesb[] = {
1114         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IESB, NONE, IMPL),
1115         MRS_FIELD_VALUE_END,
1116 };
1117
1118 static struct mrs_field_value id_aa64mmfr2_lsm[] = {
1119         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, LSM, NONE, IMPL),
1120         MRS_FIELD_VALUE_END,
1121 };
1122
1123 static struct mrs_field_value id_aa64mmfr2_uao[] = {
1124         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, UAO, NONE, IMPL),
1125         MRS_FIELD_VALUE_END,
1126 };
1127
1128 static struct mrs_field_value id_aa64mmfr2_cnp[] = {
1129         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, CnP, NONE, IMPL),
1130         MRS_FIELD_VALUE_END,
1131 };
1132
1133 static struct mrs_field id_aa64mmfr2_fields[] = {
1134         MRS_FIELD(ID_AA64MMFR2, E0PD, false, MRS_EXACT, id_aa64mmfr2_e0pd),
1135         MRS_FIELD(ID_AA64MMFR2, EVT, false, MRS_EXACT, id_aa64mmfr2_evt),
1136         MRS_FIELD(ID_AA64MMFR2, BBM, false, MRS_EXACT, id_aa64mmfr2_bbm),
1137         MRS_FIELD(ID_AA64MMFR2, TTL, false, MRS_EXACT, id_aa64mmfr2_ttl),
1138         MRS_FIELD(ID_AA64MMFR2, FWB, false, MRS_EXACT, id_aa64mmfr2_fwb),
1139         MRS_FIELD(ID_AA64MMFR2, IDS, false, MRS_EXACT, id_aa64mmfr2_ids),
1140         MRS_FIELD_HWCAP(ID_AA64MMFR2, AT, false, MRS_LOWER, id_aa64mmfr2_at,
1141             id_aa64mmfr2_at_caps),
1142         MRS_FIELD(ID_AA64MMFR2, ST, false, MRS_EXACT, id_aa64mmfr2_st),
1143         MRS_FIELD(ID_AA64MMFR2, NV, false, MRS_EXACT, id_aa64mmfr2_nv),
1144         MRS_FIELD(ID_AA64MMFR2, CCIDX, false, MRS_EXACT, id_aa64mmfr2_ccidx),
1145         MRS_FIELD(ID_AA64MMFR2, VARange, false, MRS_EXACT,
1146             id_aa64mmfr2_varange),
1147         MRS_FIELD(ID_AA64MMFR2, IESB, false, MRS_EXACT, id_aa64mmfr2_iesb),
1148         MRS_FIELD(ID_AA64MMFR2, LSM, false, MRS_EXACT, id_aa64mmfr2_lsm),
1149         MRS_FIELD(ID_AA64MMFR2, UAO, false, MRS_EXACT, id_aa64mmfr2_uao),
1150         MRS_FIELD(ID_AA64MMFR2, CnP, false, MRS_EXACT, id_aa64mmfr2_cnp),
1151         MRS_FIELD_END,
1152 };
1153
1154
1155 /* ID_AA64PFR0_EL1 */
1156 static struct mrs_field_value id_aa64pfr0_csv3[] = {
1157         MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_NONE, ""),
1158         MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_ISOLATED, "CSV3"),
1159         MRS_FIELD_VALUE_END,
1160 };
1161
1162 static struct mrs_field_value id_aa64pfr0_csv2[] = {
1163         MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_NONE, ""),
1164         MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_ISOLATED, "CSV2"),
1165         MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_SCXTNUM, "SCXTNUM"),
1166         MRS_FIELD_VALUE_END,
1167 };
1168
1169 static struct mrs_field_value id_aa64pfr0_dit[] = {
1170         MRS_FIELD_VALUE(ID_AA64PFR0_DIT_NONE, ""),
1171         MRS_FIELD_VALUE(ID_AA64PFR0_DIT_PSTATE, "PSTATE.DIT"),
1172         MRS_FIELD_VALUE_END,
1173 };
1174
1175 static struct mrs_field_hwcap id_aa64pfr0_dit_caps[] = {
1176         MRS_HWCAP(&elf_hwcap, HWCAP_DIT, ID_AA64PFR0_DIT_PSTATE),
1177         MRS_HWCAP_END
1178 };
1179
1180 static struct mrs_field_value id_aa64pfr0_amu[] = {
1181         MRS_FIELD_VALUE(ID_AA64PFR0_AMU_NONE, ""),
1182         MRS_FIELD_VALUE(ID_AA64PFR0_AMU_V1, "AMUv1"),
1183         MRS_FIELD_VALUE_END,
1184 };
1185
1186 static struct mrs_field_value id_aa64pfr0_mpam[] = {
1187         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, MPAM, NONE, IMPL),
1188         MRS_FIELD_VALUE_END,
1189 };
1190
1191 static struct mrs_field_value id_aa64pfr0_sel2[] = {
1192         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SEL2, NONE, IMPL),
1193         MRS_FIELD_VALUE_END,
1194 };
1195
1196 static struct mrs_field_value id_aa64pfr0_sve[] = {
1197         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SVE, NONE, IMPL),
1198         MRS_FIELD_VALUE_END,
1199 };
1200
1201 #if 0
1202 /* Enable when we add SVE support */
1203 static struct mrs_field_hwcap id_aa64pfr0_sve_caps[] = {
1204         MRS_HWCAP(&elf_hwcap, HWCAP_SVE, ID_AA64PFR0_SVE_IMPL),
1205         MRS_HWCAP_END
1206 };
1207 #endif
1208
1209 static struct mrs_field_value id_aa64pfr0_ras[] = {
1210         MRS_FIELD_VALUE(ID_AA64PFR0_RAS_NONE, ""),
1211         MRS_FIELD_VALUE(ID_AA64PFR0_RAS_IMPL, "RAS"),
1212         MRS_FIELD_VALUE(ID_AA64PFR0_RAS_8_4, "RAS v8.4"),
1213         MRS_FIELD_VALUE_END,
1214 };
1215
1216 static struct mrs_field_value id_aa64pfr0_gic[] = {
1217         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, GIC, CPUIF_NONE, CPUIF_EN),
1218         MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_NONE, ""),
1219         MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_EN, "GIC"),
1220         MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_4_1, "GIC 4.1"),
1221         MRS_FIELD_VALUE_END,
1222 };
1223
1224 static struct mrs_field_value id_aa64pfr0_advsimd[] = {
1225         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, AdvSIMD, NONE, IMPL),
1226         MRS_FIELD_VALUE(ID_AA64PFR0_AdvSIMD_HP, "AdvSIMD+HP"),
1227         MRS_FIELD_VALUE_END,
1228 };
1229
1230 static struct mrs_field_hwcap id_aa64pfr0_advsimd_caps[] = {
1231         MRS_HWCAP(&elf_hwcap, HWCAP_ASIMD, ID_AA64PFR0_AdvSIMD_IMPL),
1232         MRS_HWCAP(&elf_hwcap, HWCAP_ASIMDHP, ID_AA64PFR0_AdvSIMD_HP),
1233         MRS_HWCAP_END
1234 };
1235
1236 static struct mrs_field_value id_aa64pfr0_fp[] = {
1237         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, FP, NONE, IMPL),
1238         MRS_FIELD_VALUE(ID_AA64PFR0_FP_HP, "FP+HP"),
1239         MRS_FIELD_VALUE_END,
1240 };
1241
1242 static struct mrs_field_hwcap id_aa64pfr0_fp_caps[] = {
1243         MRS_HWCAP(&elf_hwcap, HWCAP_FP, ID_AA64PFR0_FP_IMPL),
1244         MRS_HWCAP(&elf_hwcap, HWCAP_FPHP, ID_AA64PFR0_FP_HP),
1245         MRS_HWCAP_END
1246 };
1247
1248 static struct mrs_field_value id_aa64pfr0_el3[] = {
1249         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL3, NONE, 64),
1250         MRS_FIELD_VALUE(ID_AA64PFR0_EL3_64_32, "EL3 32"),
1251         MRS_FIELD_VALUE_END,
1252 };
1253
1254 static struct mrs_field_value id_aa64pfr0_el2[] = {
1255         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL2, NONE, 64),
1256         MRS_FIELD_VALUE(ID_AA64PFR0_EL2_64_32, "EL2 32"),
1257         MRS_FIELD_VALUE_END,
1258 };
1259
1260 static struct mrs_field_value id_aa64pfr0_el1[] = {
1261         MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64, "EL1"),
1262         MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64_32, "EL1 32"),
1263         MRS_FIELD_VALUE_END,
1264 };
1265
1266 static struct mrs_field_value id_aa64pfr0_el0[] = {
1267         MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64, "EL0"),
1268         MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64_32, "EL0 32"),
1269         MRS_FIELD_VALUE_END,
1270 };
1271
1272 static struct mrs_field id_aa64pfr0_fields[] = {
1273         MRS_FIELD(ID_AA64PFR0, CSV3, false, MRS_EXACT, id_aa64pfr0_csv3),
1274         MRS_FIELD(ID_AA64PFR0, CSV2, false, MRS_EXACT, id_aa64pfr0_csv2),
1275         MRS_FIELD_HWCAP(ID_AA64PFR0, DIT, false, MRS_LOWER, id_aa64pfr0_dit,
1276             id_aa64pfr0_dit_caps),
1277         MRS_FIELD(ID_AA64PFR0, AMU, false, MRS_EXACT, id_aa64pfr0_amu),
1278         MRS_FIELD(ID_AA64PFR0, MPAM, false, MRS_EXACT, id_aa64pfr0_mpam),
1279         MRS_FIELD(ID_AA64PFR0, SEL2, false, MRS_EXACT, id_aa64pfr0_sel2),
1280         MRS_FIELD(ID_AA64PFR0, SVE, false, MRS_EXACT, id_aa64pfr0_sve),
1281         MRS_FIELD(ID_AA64PFR0, RAS, false, MRS_EXACT, id_aa64pfr0_ras),
1282         MRS_FIELD(ID_AA64PFR0, GIC, false, MRS_EXACT, id_aa64pfr0_gic),
1283         MRS_FIELD_HWCAP(ID_AA64PFR0, AdvSIMD, true, MRS_LOWER,
1284             id_aa64pfr0_advsimd, id_aa64pfr0_advsimd_caps),
1285         MRS_FIELD_HWCAP(ID_AA64PFR0, FP, true,  MRS_LOWER, id_aa64pfr0_fp,
1286             id_aa64pfr0_fp_caps),
1287         MRS_FIELD(ID_AA64PFR0, EL3, false, MRS_EXACT, id_aa64pfr0_el3),
1288         MRS_FIELD(ID_AA64PFR0, EL2, false, MRS_EXACT, id_aa64pfr0_el2),
1289         MRS_FIELD(ID_AA64PFR0, EL1, false, MRS_LOWER, id_aa64pfr0_el1),
1290         MRS_FIELD(ID_AA64PFR0, EL0, false, MRS_LOWER, id_aa64pfr0_el0),
1291         MRS_FIELD_END,
1292 };
1293
1294
1295 /* ID_AA64PFR1_EL1 */
1296 static struct mrs_field_value id_aa64pfr1_mte[] = {
1297         MRS_FIELD_VALUE(ID_AA64PFR1_MTE_NONE, ""),
1298         MRS_FIELD_VALUE(ID_AA64PFR1_MTE_IMPL_EL0, "MTE EL0"),
1299         MRS_FIELD_VALUE(ID_AA64PFR1_MTE_IMPL, "MTE"),
1300         MRS_FIELD_VALUE_END,
1301 };
1302
1303 static struct mrs_field_value id_aa64pfr1_ssbs[] = {
1304         MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_NONE, ""),
1305         MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE, "PSTATE.SSBS"),
1306         MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE_MSR, "PSTATE.SSBS MSR"),
1307         MRS_FIELD_VALUE_END,
1308 };
1309
1310 static struct mrs_field_hwcap id_aa64pfr1_ssbs_caps[] = {
1311         MRS_HWCAP(&elf_hwcap, HWCAP_SSBS, ID_AA64PFR1_SSBS_PSTATE),
1312         MRS_HWCAP_END
1313 };
1314
1315 static struct mrs_field_value id_aa64pfr1_bt[] = {
1316         MRS_FIELD_VALUE(ID_AA64PFR1_BT_NONE, ""),
1317         MRS_FIELD_VALUE(ID_AA64PFR1_BT_IMPL, "BTI"),
1318         MRS_FIELD_VALUE_END,
1319 };
1320
1321 #if 0
1322 /* Enable when we add BTI support */
1323 static struct mrs_field_hwcap id_aa64pfr1_bt_caps[] = {
1324         MRS_HWCAP(&elf_hwcap2, HWCAP2_BTI, ID_AA64PFR1_BT_IMPL),
1325         MRS_HWCAP_END
1326 };
1327 #endif
1328
1329 static struct mrs_field id_aa64pfr1_fields[] = {
1330         MRS_FIELD(ID_AA64PFR1, MTE, false, MRS_EXACT, id_aa64pfr1_mte),
1331         MRS_FIELD_HWCAP(ID_AA64PFR1, SSBS, false, MRS_LOWER, id_aa64pfr1_ssbs,
1332             id_aa64pfr1_ssbs_caps),
1333         MRS_FIELD(ID_AA64PFR1, BT, false, MRS_EXACT, id_aa64pfr1_bt),
1334         MRS_FIELD_END,
1335 };
1336
1337
1338 /* ID_AA64ZFR0_EL1 */
1339 static struct mrs_field_value id_aa64zfr0_f64mm[] = {
1340         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, F64MM, NONE, IMPL),
1341         MRS_FIELD_VALUE_END,
1342 };
1343
1344 static struct mrs_field_value id_aa64zfr0_f32mm[] = {
1345         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, F32MM, NONE, IMPL),
1346         MRS_FIELD_VALUE_END,
1347 };
1348
1349 static struct mrs_field_value id_aa64zfr0_i8mm[] = {
1350         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, I8MM, NONE, IMPL),
1351         MRS_FIELD_VALUE_END,
1352 };
1353
1354 static struct mrs_field_value id_aa64zfr0_sm4[] = {
1355         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, SM4, NONE, IMPL),
1356         MRS_FIELD_VALUE_END,
1357 };
1358
1359 static struct mrs_field_value id_aa64zfr0_sha3[] = {
1360         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, SHA3, NONE, IMPL),
1361         MRS_FIELD_VALUE_END,
1362 };
1363
1364 static struct mrs_field_value id_aa64zfr0_bf16[] = {
1365         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, BF16, NONE, BASE),
1366         MRS_FIELD_VALUE(ID_AA64ZFR0_BF16_EBF, "BF16+EBF"),
1367         MRS_FIELD_VALUE_END,
1368 };
1369
1370 static struct mrs_field_value id_aa64zfr0_bitperm[] = {
1371         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, BitPerm, NONE, IMPL),
1372         MRS_FIELD_VALUE_END,
1373 };
1374
1375 static struct mrs_field_value id_aa64zfr0_aes[] = {
1376         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, AES, NONE, BASE),
1377         MRS_FIELD_VALUE(ID_AA64ZFR0_AES_PMULL, "AES+PMULL"),
1378         MRS_FIELD_VALUE_END,
1379 };
1380
1381 static struct mrs_field_value id_aa64zfr0_svever[] = {
1382         MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE1, "SVE1"),
1383         MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE2, "SVE2"),
1384         MRS_FIELD_VALUE_END,
1385 };
1386
1387 static struct mrs_field id_aa64zfr0_fields[] = {
1388         MRS_FIELD(ID_AA64ZFR0, F64MM, false, MRS_EXACT, id_aa64zfr0_f64mm),
1389         MRS_FIELD(ID_AA64ZFR0, F32MM, false, MRS_EXACT, id_aa64zfr0_f32mm),
1390         MRS_FIELD(ID_AA64ZFR0, I8MM, false, MRS_EXACT, id_aa64zfr0_i8mm),
1391         MRS_FIELD(ID_AA64ZFR0, SM4, false, MRS_EXACT, id_aa64zfr0_sm4),
1392         MRS_FIELD(ID_AA64ZFR0, SHA3, false, MRS_EXACT, id_aa64zfr0_sha3),
1393         MRS_FIELD(ID_AA64ZFR0, BF16, false, MRS_EXACT, id_aa64zfr0_bf16),
1394         MRS_FIELD(ID_AA64ZFR0, BitPerm, false, MRS_EXACT, id_aa64zfr0_bitperm),
1395         MRS_FIELD(ID_AA64ZFR0, AES, false, MRS_EXACT, id_aa64zfr0_aes),
1396         MRS_FIELD(ID_AA64ZFR0, SVEver, false, MRS_EXACT, id_aa64zfr0_svever),
1397         MRS_FIELD_END,
1398 };
1399
1400
1401 #ifdef COMPAT_FREEBSD32
1402 /* ID_ISAR5_EL1 */
1403 static struct mrs_field_value id_isar5_vcma[] = {
1404         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, VCMA, NONE, IMPL),
1405         MRS_FIELD_VALUE_END,
1406 };
1407
1408 static struct mrs_field_value id_isar5_rdm[] = {
1409         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, RDM, NONE, IMPL),
1410         MRS_FIELD_VALUE_END,
1411 };
1412
1413 static struct mrs_field_value id_isar5_crc32[] = {
1414         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, CRC32, NONE, IMPL),
1415         MRS_FIELD_VALUE_END,
1416 };
1417
1418 static struct mrs_field_hwcap id_isar5_crc32_caps[] = {
1419         MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_CRC32, ID_ISAR5_CRC32_IMPL),
1420         MRS_HWCAP_END
1421 };
1422
1423 static struct mrs_field_value id_isar5_sha2[] = {
1424         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA2, NONE, IMPL),
1425         MRS_FIELD_VALUE_END,
1426 };
1427
1428 static struct mrs_field_hwcap id_isar5_sha2_caps[] = {
1429         MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_SHA2, ID_ISAR5_SHA2_IMPL),
1430         MRS_HWCAP_END
1431 };
1432
1433 static struct mrs_field_value id_isar5_sha1[] = {
1434         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA1, NONE, IMPL),
1435         MRS_FIELD_VALUE_END,
1436 };
1437
1438 static struct mrs_field_hwcap id_isar5_sha1_caps[] = {
1439         MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_SHA1, ID_ISAR5_SHA1_IMPL),
1440         MRS_HWCAP_END
1441 };
1442
1443 static struct mrs_field_value id_isar5_aes[] = {
1444         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, AES, NONE, BASE),
1445         MRS_FIELD_VALUE(ID_ISAR5_AES_VMULL, "AES+VMULL"),
1446         MRS_FIELD_VALUE_END,
1447 };
1448
1449 static struct mrs_field_hwcap id_isar5_aes_caps[] = {
1450         MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_AES, ID_ISAR5_AES_BASE),
1451         MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_PMULL, ID_ISAR5_AES_VMULL),
1452         MRS_HWCAP_END
1453 };
1454
1455 static struct mrs_field_value id_isar5_sevl[] = {
1456         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SEVL, NOP, IMPL),
1457         MRS_FIELD_VALUE_END,
1458 };
1459
1460 static struct mrs_field id_isar5_fields[] = {
1461         MRS_FIELD(ID_ISAR5, VCMA, false, MRS_LOWER, id_isar5_vcma),
1462         MRS_FIELD(ID_ISAR5, RDM, false, MRS_LOWER, id_isar5_rdm),
1463         MRS_FIELD_HWCAP(ID_ISAR5, CRC32, false, MRS_LOWER, id_isar5_crc32,
1464             id_isar5_crc32_caps),
1465         MRS_FIELD_HWCAP(ID_ISAR5, SHA2, false, MRS_LOWER, id_isar5_sha2,
1466             id_isar5_sha2_caps),
1467         MRS_FIELD_HWCAP(ID_ISAR5, SHA1, false, MRS_LOWER, id_isar5_sha1,
1468             id_isar5_sha1_caps),
1469         MRS_FIELD_HWCAP(ID_ISAR5, AES, false, MRS_LOWER, id_isar5_aes,
1470             id_isar5_aes_caps),
1471         MRS_FIELD(ID_ISAR5, SEVL, false, MRS_LOWER, id_isar5_sevl),
1472         MRS_FIELD_END,
1473 };
1474
1475 /* MVFR0 */
1476 static struct mrs_field_value mvfr0_fpround[] = {
1477         MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPRound, NONE, IMPL),
1478         MRS_FIELD_VALUE_END,
1479 };
1480
1481 static struct mrs_field_value mvfr0_fpsqrt[] = {
1482         MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPSqrt, NONE, IMPL),
1483         MRS_FIELD_VALUE_END,
1484 };
1485
1486 static struct mrs_field_value mvfr0_fpdivide[] = {
1487         MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPDivide, NONE, IMPL),
1488         MRS_FIELD_VALUE_END,
1489 };
1490
1491 static struct mrs_field_value mvfr0_fptrap[] = {
1492         MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPTrap, NONE, IMPL),
1493         MRS_FIELD_VALUE_END,
1494 };
1495
1496 static struct mrs_field_value mvfr0_fpdp[] = {
1497         MRS_FIELD_VALUE(MVFR0_FPDP_NONE, ""),
1498         MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v2, "DP VFPv2"),
1499         MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v3_v4, "DP VFPv3+v4"),
1500         MRS_FIELD_VALUE_END,
1501 };
1502
1503 static struct mrs_field_hwcap mvfr0_fpdp_caps[] = {
1504         MRS_HWCAP(&elf32_hwcap, HWCAP32_VFP, MVFR0_FPDP_VFP_v2),
1505         MRS_HWCAP(&elf32_hwcap, HWCAP32_VFPv3, MVFR0_FPDP_VFP_v3_v4),
1506 };
1507
1508 static struct mrs_field_value mvfr0_fpsp[] = {
1509         MRS_FIELD_VALUE(MVFR0_FPSP_NONE, ""),
1510         MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v2, "SP VFPv2"),
1511         MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v3_v4, "SP VFPv3+v4"),
1512         MRS_FIELD_VALUE_END,
1513 };
1514
1515 static struct mrs_field_value mvfr0_simdreg[] = {
1516         MRS_FIELD_VALUE(MVFR0_SIMDReg_NONE, ""),
1517         MRS_FIELD_VALUE(MVFR0_SIMDReg_FP, "FP 16x64"),
1518         MRS_FIELD_VALUE(MVFR0_SIMDReg_AdvSIMD, "AdvSIMD"),
1519         MRS_FIELD_VALUE_END,
1520 };
1521
1522 static struct mrs_field mvfr0_fields[] = {
1523         MRS_FIELD(MVFR0, FPRound, false, MRS_LOWER, mvfr0_fpround),
1524         MRS_FIELD(MVFR0, FPSqrt, false, MRS_LOWER, mvfr0_fpsqrt),
1525         MRS_FIELD(MVFR0, FPDivide, false, MRS_LOWER, mvfr0_fpdivide),
1526         MRS_FIELD(MVFR0, FPTrap, false, MRS_LOWER, mvfr0_fptrap),
1527         MRS_FIELD_HWCAP(MVFR0, FPDP, false, MRS_LOWER, mvfr0_fpdp,
1528             mvfr0_fpdp_caps),
1529         MRS_FIELD(MVFR0, FPSP, false, MRS_LOWER, mvfr0_fpsp),
1530         MRS_FIELD(MVFR0, SIMDReg, false, MRS_LOWER, mvfr0_simdreg),
1531         MRS_FIELD_END,
1532 };
1533
1534 /* MVFR1 */
1535 static struct mrs_field_value mvfr1_simdfmac[] = {
1536         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDFMAC, NONE, IMPL),
1537         MRS_FIELD_VALUE_END,
1538 };
1539
1540 static struct mrs_field_hwcap mvfr1_simdfmac_caps[] = {
1541         MRS_HWCAP(&elf32_hwcap, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL),
1542         MRS_HWCAP_END
1543 };
1544
1545 static struct mrs_field_value mvfr1_fphp[] = {
1546         MRS_FIELD_VALUE(MVFR1_FPHP_NONE, ""),
1547         MRS_FIELD_VALUE(MVFR1_FPHP_CONV_SP, "FPHP SP Conv"),
1548         MRS_FIELD_VALUE(MVFR1_FPHP_CONV_DP, "FPHP DP Conv"),
1549         MRS_FIELD_VALUE(MVFR1_FPHP_ARITH, "FPHP Arith"),
1550         MRS_FIELD_VALUE_END,
1551 };
1552
1553 static struct mrs_field_value mvfr1_simdhp[] = {
1554         MRS_FIELD_VALUE(MVFR1_SIMDHP_NONE, ""),
1555         MRS_FIELD_VALUE(MVFR1_SIMDHP_CONV_SP, "SIMDHP SP Conv"),
1556         MRS_FIELD_VALUE(MVFR1_SIMDHP_ARITH, "SIMDHP Arith"),
1557         MRS_FIELD_VALUE_END,
1558 };
1559
1560 static struct mrs_field_value mvfr1_simdsp[] = {
1561         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDSP, NONE, IMPL),
1562         MRS_FIELD_VALUE_END,
1563 };
1564
1565 static struct mrs_field_value mvfr1_simdint[] = {
1566         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDInt, NONE, IMPL),
1567         MRS_FIELD_VALUE_END,
1568 };
1569
1570 static struct mrs_field_value mvfr1_simdls[] = {
1571         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDLS, NONE, IMPL),
1572         MRS_FIELD_VALUE_END,
1573 };
1574
1575 static struct mrs_field_hwcap mvfr1_simdls_caps[] = {
1576         MRS_HWCAP(&elf32_hwcap, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL),
1577         MRS_HWCAP_END
1578 };
1579
1580 static struct mrs_field_value mvfr1_fpdnan[] = {
1581         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPDNaN, NONE, IMPL),
1582         MRS_FIELD_VALUE_END,
1583 };
1584
1585 static struct mrs_field_value mvfr1_fpftz[] = {
1586         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPFtZ, NONE, IMPL),
1587         MRS_FIELD_VALUE_END,
1588 };
1589
1590 static struct mrs_field mvfr1_fields[] = {
1591         MRS_FIELD_HWCAP(MVFR1, SIMDFMAC, false, MRS_LOWER, mvfr1_simdfmac,
1592             mvfr1_simdfmac_caps),
1593         MRS_FIELD(MVFR1, FPHP, false, MRS_LOWER, mvfr1_fphp),
1594         MRS_FIELD(MVFR1, SIMDHP, false, MRS_LOWER, mvfr1_simdhp),
1595         MRS_FIELD(MVFR1, SIMDSP, false, MRS_LOWER, mvfr1_simdsp),
1596         MRS_FIELD(MVFR1, SIMDInt, false, MRS_LOWER, mvfr1_simdint),
1597         MRS_FIELD_HWCAP(MVFR1, SIMDLS, false, MRS_LOWER, mvfr1_simdls,
1598             mvfr1_simdls_caps),
1599         MRS_FIELD(MVFR1, FPDNaN, false, MRS_LOWER, mvfr1_fpdnan),
1600         MRS_FIELD(MVFR1, FPFtZ, false, MRS_LOWER, mvfr1_fpftz),
1601         MRS_FIELD_END,
1602 };
1603 #endif /* COMPAT_FREEBSD32 */
1604
1605 struct mrs_user_reg {
1606         u_int           reg;
1607         u_int           CRm;
1608         u_int           Op2;
1609         size_t          offset;
1610         struct mrs_field *fields;
1611 };
1612
1613 #define USER_REG(name, field_name)                                      \
1614         {                                                               \
1615                 .reg = name,                                            \
1616                 .CRm = name##_CRm,                                      \
1617                 .Op2 = name##_op2,                                      \
1618                 .offset = __offsetof(struct cpu_desc, field_name),      \
1619                 .fields = field_name##_fields,                          \
1620         }
1621 static struct mrs_user_reg user_regs[] = {
1622         USER_REG(ID_AA64AFR0_EL1, id_aa64afr0),
1623         USER_REG(ID_AA64AFR1_EL1, id_aa64afr1),
1624
1625         USER_REG(ID_AA64DFR0_EL1, id_aa64dfr0),
1626         USER_REG(ID_AA64DFR1_EL1, id_aa64dfr1),
1627
1628         USER_REG(ID_AA64ISAR0_EL1, id_aa64isar0),
1629         USER_REG(ID_AA64ISAR1_EL1, id_aa64isar1),
1630         USER_REG(ID_AA64ISAR2_EL1, id_aa64isar2),
1631
1632         USER_REG(ID_AA64MMFR0_EL1, id_aa64mmfr0),
1633         USER_REG(ID_AA64MMFR1_EL1, id_aa64mmfr1),
1634         USER_REG(ID_AA64MMFR2_EL1, id_aa64mmfr2),
1635
1636         USER_REG(ID_AA64PFR0_EL1, id_aa64pfr0),
1637         USER_REG(ID_AA64PFR1_EL1, id_aa64pfr1),
1638
1639         USER_REG(ID_AA64ZFR0_EL1, id_aa64zfr0),
1640
1641 #ifdef COMPAT_FREEBSD32
1642         USER_REG(ID_ISAR5_EL1, id_isar5),
1643
1644         USER_REG(MVFR0_EL1, mvfr0),
1645         USER_REG(MVFR1_EL1, mvfr1),
1646 #endif /* COMPAT_FREEBSD32 */
1647 };
1648
1649 #define CPU_DESC_FIELD(desc, idx)                                       \
1650     *(uint64_t *)((char *)&(desc) + user_regs[(idx)].offset)
1651
1652 static int
1653 user_mrs_handler(vm_offset_t va, uint32_t insn, struct trapframe *frame,
1654     uint32_t esr)
1655 {
1656         uint64_t value;
1657         int CRm, Op2, i, reg;
1658
1659         if ((insn & MRS_MASK) != MRS_VALUE)
1660                 return (0);
1661
1662         /*
1663          * We only emulate Op0 == 3, Op1 == 0, CRn == 0, CRm == {0, 4-7}.
1664          * These are in the EL1 CPU identification space.
1665          * CRm == 0 holds MIDR_EL1, MPIDR_EL1, and REVID_EL1.
1666          * CRm == {4-7} holds the ID_AA64 registers.
1667          *
1668          * For full details see the ARMv8 ARM (ARM DDI 0487C.a)
1669          * Table D9-2 System instruction encodings for non-Debug System
1670          * register accesses.
1671          */
1672         if (mrs_Op0(insn) != 3 || mrs_Op1(insn) != 0 || mrs_CRn(insn) != 0)
1673                 return (0);
1674
1675         CRm = mrs_CRm(insn);
1676         if (CRm > 7 || (CRm < 4 && CRm != 0))
1677                 return (0);
1678
1679         Op2 = mrs_Op2(insn);
1680         value = 0;
1681
1682         for (i = 0; i < nitems(user_regs); i++) {
1683                 if (user_regs[i].CRm == CRm && user_regs[i].Op2 == Op2) {
1684                         value = CPU_DESC_FIELD(user_cpu_desc, i);
1685                         break;
1686                 }
1687         }
1688
1689         if (CRm == 0) {
1690                 switch (Op2) {
1691                 case 0:
1692                         value = READ_SPECIALREG(midr_el1);
1693                         break;
1694                 case 5:
1695                         value = READ_SPECIALREG(mpidr_el1);
1696                         break;
1697                 case 6:
1698                         value = READ_SPECIALREG(revidr_el1);
1699                         break;
1700                 default:
1701                         return (0);
1702                 }
1703         }
1704
1705         /*
1706          * We will handle this instruction, move to the next so we
1707          * don't trap here again.
1708          */
1709         frame->tf_elr += INSN_SIZE;
1710
1711         reg = MRS_REGISTER(insn);
1712         /* If reg is 31 then write to xzr, i.e. do nothing */
1713         if (reg == 31)
1714                 return (1);
1715
1716         if (reg < nitems(frame->tf_x))
1717                 frame->tf_x[reg] = value;
1718         else if (reg == 30)
1719                 frame->tf_lr = value;
1720
1721         return (1);
1722 }
1723
1724 bool
1725 extract_user_id_field(u_int reg, u_int field_shift, uint8_t *val)
1726 {
1727         uint64_t value;
1728         int i;
1729
1730         for (i = 0; i < nitems(user_regs); i++) {
1731                 if (user_regs[i].reg == reg) {
1732                         value = CPU_DESC_FIELD(user_cpu_desc, i);
1733                         *val = value >> field_shift;
1734                         return (true);
1735                 }
1736         }
1737
1738         return (false);
1739 }
1740
1741 bool
1742 get_kernel_reg(u_int reg, uint64_t *val)
1743 {
1744         int i;
1745
1746         for (i = 0; i < nitems(user_regs); i++) {
1747                 if (user_regs[i].reg == reg) {
1748                         *val = CPU_DESC_FIELD(kern_cpu_desc, i);
1749                         return (true);
1750                 }
1751         }
1752
1753         return (false);
1754 }
1755
1756 /*
1757  * Compares two field values that may be signed or unsigned.
1758  * Returns:
1759  *  < 0 when a is less than b
1760  *  = 0 when a equals b
1761  *  > 0 when a is greater than b
1762  */
1763 static int
1764 mrs_field_cmp(uint64_t a, uint64_t b, u_int shift, int width, bool sign)
1765 {
1766         uint64_t mask;
1767
1768         KASSERT(width > 0 && width < 64, ("%s: Invalid width %d", __func__,
1769             width));
1770
1771         mask = (1ul << width) - 1;
1772         /* Move the field to the lower bits */
1773         a = (a >> shift) & mask;
1774         b = (b >> shift) & mask;
1775
1776         if (sign) {
1777                 /*
1778                  * The field is signed. Toggle the upper bit so the comparison
1779                  * works on unsigned values as this makes positive numbers,
1780                  * i.e. those with a 0 bit, larger than negative numbers,
1781                  * i.e. those with a 1 bit, in an unsigned comparison.
1782                  */
1783                 a ^= 1ul << (width - 1);
1784                 b ^= 1ul << (width - 1);
1785         }
1786
1787         return (a - b);
1788 }
1789
1790 static uint64_t
1791 update_lower_register(uint64_t val, uint64_t new_val, u_int shift,
1792     int width, bool sign)
1793 {
1794         uint64_t mask;
1795
1796         KASSERT(width > 0 && width < 64, ("%s: Invalid width %d", __func__,
1797             width));
1798
1799         /*
1800          * If the new value is less than the existing value update it.
1801          */
1802         if (mrs_field_cmp(new_val, val, shift, width, sign) < 0) {
1803                 mask = (1ul << width) - 1;
1804                 val &= ~(mask << shift);
1805                 val |= new_val & (mask << shift);
1806         }
1807
1808         return (val);
1809 }
1810
1811 void
1812 update_special_regs(u_int cpu)
1813 {
1814         struct cpu_desc *desc;
1815         struct mrs_field *fields;
1816         uint64_t user_reg, kern_reg, value;
1817         int i, j;
1818
1819         if (cpu == 0) {
1820                 /* Create a user visible cpu description with safe values */
1821                 memset(&user_cpu_desc, 0, sizeof(user_cpu_desc));
1822                 /* Safe values for these registers */
1823                 user_cpu_desc.id_aa64pfr0 = ID_AA64PFR0_AdvSIMD_NONE |
1824                     ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 |
1825                     ID_AA64PFR0_EL0_64;
1826                 user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8;
1827         }
1828
1829         desc = get_cpu_desc(cpu);
1830         for (i = 0; i < nitems(user_regs); i++) {
1831                 value = CPU_DESC_FIELD(*desc, i);
1832                 if (cpu == 0) {
1833                         kern_reg = value;
1834                         user_reg = value;
1835                 } else {
1836                         kern_reg = CPU_DESC_FIELD(kern_cpu_desc, i);
1837                         user_reg = CPU_DESC_FIELD(user_cpu_desc, i);
1838                 }
1839
1840                 fields = user_regs[i].fields;
1841                 for (j = 0; fields[j].type != 0; j++) {
1842                         switch (fields[j].type & MRS_TYPE_MASK) {
1843                         case MRS_EXACT:
1844                                 user_reg &= ~(0xful << fields[j].shift);
1845                                 user_reg |=
1846                                     (uint64_t)MRS_EXACT_FIELD(fields[j].type) <<
1847                                     fields[j].shift;
1848                                 break;
1849                         case MRS_LOWER:
1850                                 user_reg = update_lower_register(user_reg,
1851                                     value, fields[j].shift, 4, fields[j].sign);
1852                                 break;
1853                         default:
1854                                 panic("Invalid field type: %d", fields[j].type);
1855                         }
1856                         kern_reg = update_lower_register(kern_reg, value,
1857                             fields[j].shift, 4, fields[j].sign);
1858                 }
1859
1860                 CPU_DESC_FIELD(kern_cpu_desc, i) = kern_reg;
1861                 CPU_DESC_FIELD(user_cpu_desc, i) = user_reg;
1862         }
1863 }
1864
1865 void
1866 cpu_desc_init(void)
1867 {
1868         if (mp_ncpus == 1)
1869                 return;
1870
1871         /*
1872          * Allocate memory for the non-boot CPUs to store their registers.
1873          * As this is indexed by CPU ID we need to allocate space for CPUs
1874          * 1 to mp_maxid. Because of this mp_maxid is already the correct
1875          * number of elements.
1876          */
1877         cpu_desc = mallocarray(mp_maxid, sizeof(*cpu_desc), M_IDENTCPU,
1878             M_ZERO | M_WAITOK);
1879 }
1880
1881 /* HWCAP */
1882 bool __read_frequently lse_supported = false;
1883
1884 bool __read_frequently icache_aliasing = false;
1885 bool __read_frequently icache_vmid = false;
1886
1887 int64_t dcache_line_size;       /* The minimum D cache line size */
1888 int64_t icache_line_size;       /* The minimum I cache line size */
1889 int64_t idcache_line_size;      /* The minimum cache line size */
1890
1891 /*
1892  * Find the values to export to userspace as AT_HWCAP and AT_HWCAP2.
1893  */
1894 static void
1895 parse_cpu_features(void)
1896 {
1897         struct mrs_field_hwcap *hwcaps;
1898         struct mrs_field *fields;
1899         uint64_t min, reg;
1900         int i, j, k;
1901
1902         for (i = 0; i < nitems(user_regs); i++) {
1903                 reg = CPU_DESC_FIELD(user_cpu_desc, i);
1904                 fields = user_regs[i].fields;
1905                 for (j = 0; fields[j].type != 0; j++) {
1906                         hwcaps = fields[j].hwcaps;
1907                         if (hwcaps == NULL)
1908                                 continue;
1909
1910                         for (k = 0; hwcaps[k].hwcap != NULL; k++) {
1911                                 min = hwcaps[k].min;
1912
1913                                 /*
1914                                  * If the field is greater than the minimum
1915                                  * value we can set the hwcap;
1916                                  */
1917                                 if (mrs_field_cmp(reg, min, fields[j].shift,
1918                                     4, fields[j].sign) >= 0) {
1919                                         *hwcaps[k].hwcap |= hwcaps[k].hwcap_val;
1920                                 }
1921                         }
1922                 }
1923         }
1924 }
1925
1926 static void
1927 identify_cpu_sysinit(void *dummy __unused)
1928 {
1929         struct cpu_desc *desc, *prev_desc;
1930         int cpu;
1931         bool dic, idc;
1932
1933         dic = (allow_dic != 0);
1934         idc = (allow_idc != 0);
1935
1936         prev_desc = NULL;
1937         CPU_FOREACH(cpu) {
1938                 desc = get_cpu_desc(cpu);
1939                 if (cpu != 0) {
1940                         check_cpu_regs(cpu, desc, prev_desc);
1941                         update_special_regs(cpu);
1942                 }
1943
1944                 if (CTR_DIC_VAL(desc->ctr) == 0)
1945                         dic = false;
1946                 if (CTR_IDC_VAL(desc->ctr) == 0)
1947                         idc = false;
1948                 prev_desc = desc;
1949         }
1950
1951         /* Find the values to export to userspace as AT_HWCAP and AT_HWCAP2 */
1952         parse_cpu_features();
1953
1954 #ifdef COMPAT_FREEBSD32
1955         /* Set the default caps and any that need to check multiple fields */
1956         elf32_hwcap |= parse_cpu_features_hwcap32();
1957 #endif
1958
1959         if (dic && idc) {
1960                 arm64_icache_sync_range = &arm64_dic_idc_icache_sync_range;
1961                 if (bootverbose)
1962                         printf("Enabling DIC & IDC ICache sync\n");
1963         } else if (idc) {
1964                 arm64_icache_sync_range = &arm64_idc_aliasing_icache_sync_range;
1965                 if (bootverbose)
1966                         printf("Enabling IDC ICache sync\n");
1967         }
1968
1969         if ((elf_hwcap & HWCAP_ATOMICS) != 0) {
1970                 lse_supported = true;
1971                 if (bootverbose)
1972                         printf("Enabling LSE atomics in the kernel\n");
1973         }
1974 #ifdef LSE_ATOMICS
1975         if (!lse_supported)
1976                 panic("CPU does not support LSE atomic instructions");
1977 #endif
1978
1979         install_undef_handler(true, user_mrs_handler);
1980 }
1981 SYSINIT(identify_cpu, SI_SUB_CPU, SI_ORDER_MIDDLE, identify_cpu_sysinit, NULL);
1982
1983 static void
1984 cpu_features_sysinit(void *dummy __unused)
1985 {
1986         struct sbuf sb;
1987         struct cpu_desc *desc, *prev_desc;
1988         u_int cpu;
1989
1990         prev_desc = NULL;
1991         CPU_FOREACH(cpu) {
1992                 desc = get_cpu_desc(cpu);
1993                 print_cpu_features(cpu, desc, prev_desc);
1994                 prev_desc = desc;
1995         }
1996
1997         /* Fill in cpu_model for the hw.model sysctl */
1998         sbuf_new(&sb, cpu_model, sizeof(cpu_model), SBUF_FIXEDLEN);
1999         print_cpu_midr(&sb, 0);
2000
2001         sbuf_finish(&sb);
2002         sbuf_delete(&sb);
2003
2004         free(cpu_desc, M_IDENTCPU);
2005 }
2006 /* Log features before APs are released and start printing to the dmesg. */
2007 SYSINIT(cpu_features, SI_SUB_SMP - 1, SI_ORDER_ANY, cpu_features_sysinit, NULL);
2008
2009 #ifdef COMPAT_FREEBSD32
2010 static u_long
2011 parse_cpu_features_hwcap32(void)
2012 {
2013         u_long hwcap = HWCAP32_DEFAULT;
2014
2015         if ((MVFR1_SIMDLS_VAL(user_cpu_desc.mvfr1) >=
2016              MVFR1_SIMDLS_IMPL) &&
2017             (MVFR1_SIMDInt_VAL(user_cpu_desc.mvfr1) >=
2018              MVFR1_SIMDInt_IMPL) &&
2019             (MVFR1_SIMDSP_VAL(user_cpu_desc.mvfr1) >=
2020              MVFR1_SIMDSP_IMPL))
2021                 hwcap |= HWCAP32_NEON;
2022
2023         return (hwcap);
2024 }
2025 #endif /* COMPAT_FREEBSD32 */
2026
2027 static void
2028 print_ctr_fields(struct sbuf *sb, uint64_t reg, void *arg)
2029 {
2030
2031         sbuf_printf(sb, "%u byte D-cacheline,", CTR_DLINE_SIZE(reg));
2032         sbuf_printf(sb, "%u byte I-cacheline,", CTR_ILINE_SIZE(reg));
2033         reg &= ~(CTR_DLINE_MASK | CTR_ILINE_MASK);
2034
2035         switch(CTR_L1IP_VAL(reg)) {
2036         case CTR_L1IP_VPIPT:
2037                 sbuf_printf(sb, "VPIPT");
2038                 break;
2039         case CTR_L1IP_AIVIVT:
2040                 sbuf_printf(sb, "AIVIVT");
2041                 break;
2042         case CTR_L1IP_VIPT:
2043                 sbuf_printf(sb, "VIPT");
2044                 break;
2045         case CTR_L1IP_PIPT:
2046                 sbuf_printf(sb, "PIPT");
2047                 break;
2048         }
2049         sbuf_printf(sb, " ICache,");
2050         reg &= ~CTR_L1IP_MASK;
2051
2052         sbuf_printf(sb, "%d byte ERG,", CTR_ERG_SIZE(reg));
2053         sbuf_printf(sb, "%d byte CWG", CTR_CWG_SIZE(reg));
2054         reg &= ~(CTR_ERG_MASK | CTR_CWG_MASK);
2055
2056         if (CTR_IDC_VAL(reg) != 0)
2057                 sbuf_printf(sb, ",IDC");
2058         if (CTR_DIC_VAL(reg) != 0)
2059                 sbuf_printf(sb, ",DIC");
2060         reg &= ~(CTR_IDC_MASK | CTR_DIC_MASK);
2061         reg &= ~CTR_RES1;
2062
2063         if (reg != 0)
2064                 sbuf_printf(sb, ",%lx", reg);
2065 }
2066
2067 static void
2068 print_register(struct sbuf *sb, const char *reg_name, uint64_t reg,
2069     void (*print_fields)(struct sbuf *, uint64_t, void *), void *arg)
2070 {
2071
2072         sbuf_printf(sb, "%29s = <", reg_name);
2073
2074         print_fields(sb, reg, arg);
2075
2076         sbuf_finish(sb);
2077         printf("%s>\n", sbuf_data(sb));
2078         sbuf_clear(sb);
2079 }
2080
2081 static void
2082 print_id_fields(struct sbuf *sb, uint64_t reg, void *arg)
2083 {
2084         struct mrs_field *fields = arg;
2085         struct mrs_field_value *fv;
2086         int field, i, j, printed;
2087
2088 #define SEP_STR ((printed++) == 0) ? "" : ","
2089         printed = 0;
2090         for (i = 0; fields[i].type != 0; i++) {
2091                 fv = fields[i].values;
2092
2093                 /* TODO: Handle with an unknown message */
2094                 if (fv == NULL)
2095                         continue;
2096
2097                 field = (reg & fields[i].mask) >> fields[i].shift;
2098                 for (j = 0; fv[j].desc != NULL; j++) {
2099                         if ((fv[j].value >> fields[i].shift) != field)
2100                                 continue;
2101
2102                         if (fv[j].desc[0] != '\0')
2103                                 sbuf_printf(sb, "%s%s", SEP_STR, fv[j].desc);
2104                         break;
2105                 }
2106                 if (fv[j].desc == NULL)
2107                         sbuf_printf(sb, "%sUnknown %s(%x)", SEP_STR,
2108                             fields[i].name, field);
2109
2110                 reg &= ~(0xful << fields[i].shift);
2111         }
2112
2113         if (reg != 0)
2114                 sbuf_printf(sb, "%s%#lx", SEP_STR, reg);
2115 #undef SEP_STR
2116 }
2117
2118 static void
2119 print_id_register(struct sbuf *sb, const char *reg_name, uint64_t reg,
2120     struct mrs_field *fields)
2121 {
2122
2123         print_register(sb, reg_name, reg, print_id_fields, fields);
2124 }
2125
2126 static void
2127 print_cpu_midr(struct sbuf *sb, u_int cpu)
2128 {
2129         const struct cpu_parts *cpu_partsp;
2130         const char *cpu_impl_name;
2131         const char *cpu_part_name;
2132         u_int midr;
2133         u_int impl_id;
2134         u_int part_id;
2135
2136         midr = pcpu_find(cpu)->pc_midr;
2137
2138         cpu_impl_name = NULL;
2139         cpu_partsp = NULL;
2140         impl_id = CPU_IMPL(midr);
2141         for (int i = 0; cpu_implementers[i].impl_name != NULL; i++) {
2142                 if (impl_id == cpu_implementers[i].impl_id) {
2143                         cpu_impl_name = cpu_implementers[i].impl_name;
2144                         cpu_partsp = cpu_implementers[i].cpu_parts;
2145                         break;
2146                 }
2147         }
2148         /* Unknown implementer, so unknown part */
2149         if (cpu_impl_name == NULL) {
2150                 sbuf_printf(sb, "Unknown Implementer (midr: %08x)", midr);
2151                 return;
2152         }
2153
2154         KASSERT(cpu_partsp != NULL, ("%s: No parts table for implementer %s",
2155             __func__, cpu_impl_name));
2156
2157         cpu_part_name = NULL;
2158         part_id = CPU_PART(midr);
2159         for (int i = 0; cpu_partsp[i].part_name != NULL; i++) {
2160                 if (part_id == cpu_partsp[i].part_id) {
2161                         cpu_part_name = cpu_partsp[i].part_name;
2162                         break;
2163                 }
2164         }
2165         /* Known Implementer, Unknown part */
2166         if (cpu_part_name == NULL) {
2167                 sbuf_printf(sb, "%s Unknown CPU r%dp%d (midr: %08x)",
2168                     cpu_impl_name, CPU_VAR(midr), CPU_REV(midr), midr);
2169                 return;
2170         }
2171
2172         sbuf_printf(sb, "%s %s r%dp%d", cpu_impl_name,
2173             cpu_part_name, CPU_VAR(midr), CPU_REV(midr));
2174 }
2175
2176 static void
2177 print_cpu_cache(struct cpu_desc *desc, struct sbuf *sb, uint64_t ccs,
2178     bool icache, bool unified)
2179 {
2180         size_t cache_size;
2181         size_t line_size;
2182
2183         /* LineSize is Log2(S) - 4. */
2184         line_size = 1 << ((ccs & CCSIDR_LineSize_MASK) + 4);
2185         /*
2186          * Calculate cache size (sets * ways * line size).  There are different
2187          * formats depending on the FEAT_CCIDX bit in ID_AA64MMFR2 feature
2188          * register.
2189          */
2190         if ((desc->id_aa64mmfr2 & ID_AA64MMFR2_CCIDX_64))
2191                 cache_size = (CCSIDR_NSETS_64(ccs) + 1) *
2192                     (CCSIDR_ASSOC_64(ccs) + 1);
2193         else
2194                 cache_size = (CCSIDR_NSETS(ccs) + 1) * (CCSIDR_ASSOC(ccs) + 1);
2195
2196         cache_size *= line_size;
2197         sbuf_printf(sb, "%zuKB (%s)", cache_size / 1024,
2198             icache ? "instruction" : unified ? "unified" : "data");
2199 }
2200
2201 static void
2202 print_cpu_caches(struct sbuf *sb, struct cpu_desc *desc)
2203 {
2204         /* Print out each cache combination */
2205         uint64_t clidr;
2206         int i = 1;
2207         clidr = desc->clidr;
2208
2209         for (i = 0; (clidr & CLIDR_CTYPE_MASK) != 0; i++, clidr >>= 3) {
2210                 int j = 0;
2211                 int ctype_m = (clidr & CLIDR_CTYPE_MASK);
2212
2213                 sbuf_printf(sb, " L%d cache: ", i + 1);
2214                 if ((clidr & CLIDR_CTYPE_IO)) {
2215                         print_cpu_cache(desc, sb, desc->ccsidr[i][j++], true,
2216                             false);
2217                         /* If there's more, add to the line. */
2218                         if ((ctype_m & ~CLIDR_CTYPE_IO) != 0)
2219                                 sbuf_printf(sb, ", ");
2220                 }
2221                 if ((ctype_m & ~CLIDR_CTYPE_IO) != 0) {
2222                         print_cpu_cache(desc, sb, desc->ccsidr[i][j], false,
2223                             (clidr & CLIDR_CTYPE_UNIFIED));
2224                 }
2225                 sbuf_printf(sb, "\n");
2226
2227         }
2228         sbuf_finish(sb);
2229         printf("%s", sbuf_data(sb));
2230 }
2231
2232 static void
2233 print_cpu_features(u_int cpu, struct cpu_desc *desc,
2234     struct cpu_desc *prev_desc)
2235 {
2236         struct sbuf *sb;
2237
2238         sb = sbuf_new_auto();
2239         sbuf_printf(sb, "CPU%3u: ", cpu);
2240         print_cpu_midr(sb, cpu);
2241
2242         sbuf_cat(sb, " affinity:");
2243         switch(cpu_aff_levels) {
2244         default:
2245         case 4:
2246                 sbuf_printf(sb, " %2d", CPU_AFF3(desc->mpidr));
2247                 /* FALLTHROUGH */
2248         case 3:
2249                 sbuf_printf(sb, " %2d", CPU_AFF2(desc->mpidr));
2250                 /* FALLTHROUGH */
2251         case 2:
2252                 sbuf_printf(sb, " %2d", CPU_AFF1(desc->mpidr));
2253                 /* FALLTHROUGH */
2254         case 1:
2255         case 0: /* On UP this will be zero */
2256                 sbuf_printf(sb, " %2d", CPU_AFF0(desc->mpidr));
2257                 break;
2258         }
2259         sbuf_finish(sb);
2260         printf("%s\n", sbuf_data(sb));
2261         sbuf_clear(sb);
2262
2263         /*
2264          * There is a hardware errata where, if one CPU is performing a TLB
2265          * invalidation while another is performing a store-exclusive the
2266          * store-exclusive may return the wrong status. A workaround seems
2267          * to be to use an IPI to invalidate on each CPU, however given the
2268          * limited number of affected units (pass 1.1 is the evaluation
2269          * hardware revision), and the lack of information from Cavium
2270          * this has not been implemented.
2271          *
2272          * At the time of writing this the only information is from:
2273          * https://lkml.org/lkml/2016/8/4/722
2274          */
2275         /*
2276          * XXX: CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1 on its own also
2277          * triggers on pass 2.0+.
2278          */
2279         if (cpu == 0 && CPU_VAR(PCPU_GET(midr)) == 0 &&
2280             CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1)
2281                 printf("WARNING: ThunderX Pass 1.1 detected.\nThis has known "
2282                     "hardware bugs that may cause the incorrect operation of "
2283                     "atomic operations.\n");
2284
2285 #define SHOULD_PRINT_REG(_reg)                                          \
2286     (prev_desc == NULL || desc->_reg != prev_desc->_reg)
2287
2288         /* Cache Type Register */
2289         if (SHOULD_PRINT_REG(ctr)) {
2290                 print_register(sb, "Cache Type",
2291                     desc->ctr, print_ctr_fields, NULL);
2292         }
2293
2294         /* AArch64 Instruction Set Attribute Register 0 */
2295         if (SHOULD_PRINT_REG(id_aa64isar0))
2296                 print_id_register(sb, "Instruction Set Attributes 0",
2297                     desc->id_aa64isar0, id_aa64isar0_fields);
2298
2299         /* AArch64 Instruction Set Attribute Register 1 */
2300         if (SHOULD_PRINT_REG(id_aa64isar1))
2301                 print_id_register(sb, "Instruction Set Attributes 1",
2302                     desc->id_aa64isar1, id_aa64isar1_fields);
2303
2304         /* AArch64 Instruction Set Attribute Register 2 */
2305         if (SHOULD_PRINT_REG(id_aa64isar2))
2306                 print_id_register(sb, "Instruction Set Attributes 2",
2307                     desc->id_aa64isar2, id_aa64isar2_fields);
2308
2309         /* AArch64 Processor Feature Register 0 */
2310         if (SHOULD_PRINT_REG(id_aa64pfr0))
2311                 print_id_register(sb, "Processor Features 0",
2312                     desc->id_aa64pfr0, id_aa64pfr0_fields);
2313
2314         /* AArch64 Processor Feature Register 1 */
2315         if (SHOULD_PRINT_REG(id_aa64pfr1))
2316                 print_id_register(sb, "Processor Features 1",
2317                     desc->id_aa64pfr1, id_aa64pfr1_fields);
2318
2319         /* AArch64 Memory Model Feature Register 0 */
2320         if (SHOULD_PRINT_REG(id_aa64mmfr0))
2321                 print_id_register(sb, "Memory Model Features 0",
2322                     desc->id_aa64mmfr0, id_aa64mmfr0_fields);
2323
2324         /* AArch64 Memory Model Feature Register 1 */
2325         if (SHOULD_PRINT_REG(id_aa64mmfr1))
2326                 print_id_register(sb, "Memory Model Features 1",
2327                     desc->id_aa64mmfr1, id_aa64mmfr1_fields);
2328
2329         /* AArch64 Memory Model Feature Register 2 */
2330         if (SHOULD_PRINT_REG(id_aa64mmfr2))
2331                 print_id_register(sb, "Memory Model Features 2",
2332                     desc->id_aa64mmfr2, id_aa64mmfr2_fields);
2333
2334         /* AArch64 Debug Feature Register 0 */
2335         if (SHOULD_PRINT_REG(id_aa64dfr0))
2336                 print_id_register(sb, "Debug Features 0",
2337                     desc->id_aa64dfr0, id_aa64dfr0_fields);
2338
2339         /* AArch64 Memory Model Feature Register 1 */
2340         if (SHOULD_PRINT_REG(id_aa64dfr1))
2341                 print_id_register(sb, "Debug Features 1",
2342                     desc->id_aa64dfr1, id_aa64dfr1_fields);
2343
2344         /* AArch64 Auxiliary Feature Register 0 */
2345         if (SHOULD_PRINT_REG(id_aa64afr0))
2346                 print_id_register(sb, "Auxiliary Features 0",
2347                     desc->id_aa64afr0, id_aa64afr0_fields);
2348
2349         /* AArch64 Auxiliary Feature Register 1 */
2350         if (SHOULD_PRINT_REG(id_aa64afr1))
2351                 print_id_register(sb, "Auxiliary Features 1",
2352                     desc->id_aa64afr1, id_aa64afr1_fields);
2353
2354         /* AArch64 SVE Feature Register 0 */
2355         if (desc->have_sve) {
2356                 if (SHOULD_PRINT_REG(id_aa64zfr0) ||
2357                     !prev_desc->have_sve) {
2358                         print_id_register(sb, "SVE Features 0",
2359                             desc->id_aa64zfr0, id_aa64zfr0_fields);
2360                 }
2361         }
2362
2363 #ifdef COMPAT_FREEBSD32
2364         /* AArch32 Instruction Set Attribute Register 5 */
2365         if (SHOULD_PRINT_REG(id_isar5))
2366                 print_id_register(sb, "AArch32 Instruction Set Attributes 5",
2367                      desc->id_isar5, id_isar5_fields);
2368
2369         /* AArch32 Media and VFP Feature Register 0 */
2370         if (SHOULD_PRINT_REG(mvfr0))
2371                 print_id_register(sb, "AArch32 Media and VFP Features 0",
2372                      desc->mvfr0, mvfr0_fields);
2373
2374         /* AArch32 Media and VFP Feature Register 1 */
2375         if (SHOULD_PRINT_REG(mvfr1))
2376                 print_id_register(sb, "AArch32 Media and VFP Features 1",
2377                      desc->mvfr1, mvfr1_fields);
2378 #endif
2379         if (bootverbose)
2380                 print_cpu_caches(sb, desc);
2381
2382         sbuf_delete(sb);
2383         sb = NULL;
2384 #undef SHOULD_PRINT_REG
2385 #undef SEP_STR
2386 }
2387
2388 void
2389 identify_cache(uint64_t ctr)
2390 {
2391
2392         /* Identify the L1 cache type */
2393         switch (CTR_L1IP_VAL(ctr)) {
2394         case CTR_L1IP_PIPT:
2395                 break;
2396         case CTR_L1IP_VPIPT:
2397                 icache_vmid = true;
2398                 break;
2399         default:
2400         case CTR_L1IP_VIPT:
2401                 icache_aliasing = true;
2402                 break;
2403         }
2404
2405         if (dcache_line_size == 0) {
2406                 KASSERT(icache_line_size == 0, ("%s: i-cacheline size set: %ld",
2407                     __func__, icache_line_size));
2408
2409                 /* Get the D cache line size */
2410                 dcache_line_size = CTR_DLINE_SIZE(ctr);
2411                 /* And the same for the I cache */
2412                 icache_line_size = CTR_ILINE_SIZE(ctr);
2413
2414                 idcache_line_size = MIN(dcache_line_size, icache_line_size);
2415         }
2416
2417         if (dcache_line_size != CTR_DLINE_SIZE(ctr)) {
2418                 printf("WARNING: D-cacheline size mismatch %ld != %d\n",
2419                     dcache_line_size, CTR_DLINE_SIZE(ctr));
2420         }
2421
2422         if (icache_line_size != CTR_ILINE_SIZE(ctr)) {
2423                 printf("WARNING: I-cacheline size mismatch %ld != %d\n",
2424                     icache_line_size, CTR_ILINE_SIZE(ctr));
2425         }
2426 }
2427
2428 void
2429 identify_cpu(u_int cpu)
2430 {
2431         struct cpu_desc *desc;
2432         uint64_t clidr;
2433
2434         desc = get_cpu_desc(cpu);
2435         /* Save affinity for current CPU */
2436         desc->mpidr = get_mpidr();
2437         CPU_AFFINITY(cpu) = desc->mpidr & CPU_AFF_MASK;
2438
2439         desc->ctr = READ_SPECIALREG(ctr_el0);
2440         desc->id_aa64dfr0 = READ_SPECIALREG(id_aa64dfr0_el1);
2441         desc->id_aa64dfr1 = READ_SPECIALREG(id_aa64dfr1_el1);
2442         desc->id_aa64isar0 = READ_SPECIALREG(id_aa64isar0_el1);
2443         desc->id_aa64isar1 = READ_SPECIALREG(id_aa64isar1_el1);
2444         desc->id_aa64isar2 = READ_SPECIALREG(id_aa64isar2_el1);
2445         desc->id_aa64mmfr0 = READ_SPECIALREG(id_aa64mmfr0_el1);
2446         desc->id_aa64mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1);
2447         desc->id_aa64mmfr2 = READ_SPECIALREG(id_aa64mmfr2_el1);
2448         desc->id_aa64pfr0 = READ_SPECIALREG(id_aa64pfr0_el1);
2449         desc->id_aa64pfr1 = READ_SPECIALREG(id_aa64pfr1_el1);
2450
2451         /*
2452          * ID_AA64ZFR0_EL1 is only valid when at least one of:
2453          *  - ID_AA64PFR0_EL1.SVE is non-zero
2454          *  - ID_AA64PFR1_EL1.SME is non-zero
2455          * In other cases it is zero, but still safe to read
2456          */
2457         desc->have_sve =
2458             (ID_AA64PFR0_SVE_VAL(desc->id_aa64pfr0) != 0);
2459         desc->id_aa64zfr0 = READ_SPECIALREG(ID_AA64ZFR0_EL1_REG);
2460
2461         desc->clidr = READ_SPECIALREG(clidr_el1);
2462
2463         clidr = desc->clidr;
2464
2465         for (int i = 0; (clidr & CLIDR_CTYPE_MASK) != 0; i++, clidr >>= 3) {
2466                 int j = 0;
2467                 if ((clidr & CLIDR_CTYPE_IO)) {
2468                         WRITE_SPECIALREG(csselr_el1,
2469                             CSSELR_Level(i) | CSSELR_InD);
2470                         desc->ccsidr[i][j++] =
2471                             READ_SPECIALREG(ccsidr_el1);
2472                 }
2473                 if ((clidr & ~CLIDR_CTYPE_IO) == 0)
2474                         continue;
2475                 WRITE_SPECIALREG(csselr_el1, CSSELR_Level(i));
2476                 desc->ccsidr[i][j] = READ_SPECIALREG(ccsidr_el1);
2477         }
2478
2479 #ifdef COMPAT_FREEBSD32
2480         /* Only read aarch32 SRs if EL0-32 is available */
2481         if (ID_AA64PFR0_EL0_VAL(desc->id_aa64pfr0) == ID_AA64PFR0_EL0_64_32) {
2482                 desc->id_isar5 = READ_SPECIALREG(id_isar5_el1);
2483                 desc->mvfr0 = READ_SPECIALREG(mvfr0_el1);
2484                 desc->mvfr1 = READ_SPECIALREG(mvfr1_el1);
2485         }
2486 #endif
2487 }
2488
2489 static void
2490 check_cpu_regs(u_int cpu, struct cpu_desc *desc, struct cpu_desc *prev_desc)
2491 {
2492         switch (cpu_aff_levels) {
2493         case 0:
2494                 if (CPU_AFF0(desc->mpidr) != CPU_AFF0(prev_desc->mpidr))
2495                         cpu_aff_levels = 1;
2496                 /* FALLTHROUGH */
2497         case 1:
2498                 if (CPU_AFF1(desc->mpidr) != CPU_AFF1(prev_desc->mpidr))
2499                         cpu_aff_levels = 2;
2500                 /* FALLTHROUGH */
2501         case 2:
2502                 if (CPU_AFF2(desc->mpidr) != CPU_AFF2(prev_desc->mpidr))
2503                         cpu_aff_levels = 3;
2504                 /* FALLTHROUGH */
2505         case 3:
2506                 if (CPU_AFF3(desc->mpidr) != CPU_AFF3(prev_desc->mpidr))
2507                         cpu_aff_levels = 4;
2508                 break;
2509         }
2510
2511         if (desc->ctr != prev_desc->ctr) {
2512                 /*
2513                  * If the cache type register is different we may
2514                  * have a different l1 cache type.
2515                  */
2516                 identify_cache(desc->ctr);
2517         }
2518 }