sys/arm64/arm64/identcpu.c

   1 /*-
   2  * Copyright (c) 2014 Andrew Turner
   3  * Copyright (c) 2014 The FreeBSD Foundation
   4  * All rights reserved.
   5  *
   6  * Portions of this software were developed by Semihalf
   7  * under sponsorship of the FreeBSD Foundation.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  *
  30  */
  31
  32 #include <sys/cdefs.h>
  33 __FBSDID("$FreeBSD$");
  34
  35 #include <sys/param.h>
  36 #include <sys/kernel.h>
  37 #include <sys/pcpu.h>
  38 #include <sys/sbuf.h>
  39 #include <sys/smp.h>
  40 #include <sys/sysctl.h>
  41 #include <sys/systm.h>
  42
  43 #include <machine/atomic.h>
  44 #include <machine/cpu.h>
  45 #include <machine/cpufunc.h>
  46 #include <machine/elf.h>
  47 #include <machine/md_var.h>
  48 #include <machine/undefined.h>
  49
  50 static void print_cpu_midr(struct sbuf *sb, u_int cpu);
  51 static void print_cpu_features(u_int cpu);
  52 static void print_cpu_caches(struct sbuf *sb, u_int);
  53 #ifdef COMPAT_FREEBSD32
  54 static u_long parse_cpu_features_hwcap32(void);
  55 #endif
  56
  57 char machine[] = "arm64";
  58
  59 #ifdef SCTL_MASK32
  60 extern int adaptive_machine_arch;
  61 #endif
  62
  63 static SYSCTL_NODE(_machdep, OID_AUTO, cache, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
  64     "Cache management tuning");
  65
  66 static int allow_dic = 1;
  67 SYSCTL_INT(_machdep_cache, OID_AUTO, allow_dic, CTLFLAG_RDTUN, &allow_dic, 0,
  68     "Allow optimizations based on the DIC cache bit");
  69
  70 static int allow_idc = 1;
  71 SYSCTL_INT(_machdep_cache, OID_AUTO, allow_idc, CTLFLAG_RDTUN, &allow_idc, 0,
  72     "Allow optimizations based on the IDC cache bit");
  73
  74 static void check_cpu_regs(u_int cpu);
  75
  76 /*
  77  * The default implementation of I-cache sync assumes we have an
  78  * aliasing cache until we know otherwise.
  79  */
  80 void (*arm64_icache_sync_range)(vm_offset_t, vm_size_t) =
  81     &arm64_aliasing_icache_sync_range;
  82
  83 static int
  84 sysctl_hw_machine(SYSCTL_HANDLER_ARGS)
  85 {
  86 #ifdef SCTL_MASK32
  87         static const char machine32[] = "arm";
  88 #endif
  89         int error;
  90
  91 #ifdef SCTL_MASK32
  92         if ((req->flags & SCTL_MASK32) != 0 && adaptive_machine_arch)
  93                 error = SYSCTL_OUT(req, machine32, sizeof(machine32));
  94         else
  95 #endif
  96                 error = SYSCTL_OUT(req, machine, sizeof(machine));
  97         return (error);
  98 }
  99
 100 SYSCTL_PROC(_hw, HW_MACHINE, machine, CTLTYPE_STRING | CTLFLAG_RD |
 101          CTLFLAG_CAPRD | CTLFLAG_MPSAFE, NULL, 0, sysctl_hw_machine, "A",
 102          "Machine class");
 103
 104 static char cpu_model[64];
 105 SYSCTL_STRING(_hw, HW_MODEL, model, CTLFLAG_RD,
 106         cpu_model, sizeof(cpu_model), "Machine model");
 107
 108 #define MAX_CACHES      8       /* Maximum number of caches supported
 109                                    architecturally. */
 110 /*
 111  * Per-CPU affinity as provided in MPIDR_EL1
 112  * Indexed by CPU number in logical order selected by the system.
 113  * Relevant fields can be extracted using CPU_AFFn macros,
 114  * Aff3.Aff2.Aff1.Aff0 construct a unique CPU address in the system.
 115  *
 116  * Fields used by us:
 117  * Aff1 - Cluster number
 118  * Aff0 - CPU number in Aff1 cluster
 119  */
 120 uint64_t __cpu_affinity[MAXCPU];
 121 static u_int cpu_aff_levels;
 122
 123 struct cpu_desc {
 124         uint64_t        mpidr;
 125         uint64_t        id_aa64afr0;
 126         uint64_t        id_aa64afr1;
 127         uint64_t        id_aa64dfr0;
 128         uint64_t        id_aa64dfr1;
 129         uint64_t        id_aa64isar0;
 130         uint64_t        id_aa64isar1;
 131         uint64_t        id_aa64isar2;
 132         uint64_t        id_aa64mmfr0;
 133         uint64_t        id_aa64mmfr1;
 134         uint64_t        id_aa64mmfr2;
 135         uint64_t        id_aa64pfr0;
 136         uint64_t        id_aa64pfr1;
 137         uint64_t        id_aa64zfr0;
 138         uint64_t        ctr;
 139 #ifdef COMPAT_FREEBSD32
 140         uint64_t        id_isar5;
 141         uint64_t        mvfr0;
 142         uint64_t        mvfr1;
 143 #endif
 144         uint64_t        clidr;
 145         uint32_t        ccsidr[MAX_CACHES][2]; /* 2 possible types. */
 146         bool            have_sve;
 147 };
 148
 149 static struct cpu_desc cpu_desc[MAXCPU];
 150 static struct cpu_desc kern_cpu_desc;
 151 static struct cpu_desc user_cpu_desc;
 152
 153 struct cpu_parts {
 154         u_int           part_id;
 155         const char      *part_name;
 156 };
 157 #define CPU_PART_NONE   { 0, NULL }
 158
 159 struct cpu_implementers {
 160         u_int                   impl_id;
 161         const char              *impl_name;
 162         /*
 163          * Part number is implementation defined
 164          * so each vendor will have its own set of values and names.
 165          */
 166         const struct cpu_parts  *cpu_parts;
 167 };
 168 #define CPU_IMPLEMENTER_NONE    { 0, NULL, NULL }
 169
 170 /*
 171  * Per-implementer table of (PartNum, CPU Name) pairs.
 172  */
 173 /* ARM Ltd. */
 174 static const struct cpu_parts cpu_parts_arm[] = {
 175         { CPU_PART_AEM_V8, "AEMv8" },
 176         { CPU_PART_FOUNDATION, "Foundation-Model" },
 177         { CPU_PART_CORTEX_A34, "Cortex-A34" },
 178         { CPU_PART_CORTEX_A35, "Cortex-A35" },
 179         { CPU_PART_CORTEX_A53, "Cortex-A53" },
 180         { CPU_PART_CORTEX_A55, "Cortex-A55" },
 181         { CPU_PART_CORTEX_A57, "Cortex-A57" },
 182         { CPU_PART_CORTEX_A65, "Cortex-A65" },
 183         { CPU_PART_CORTEX_A65AE, "Cortex-A65AE" },
 184         { CPU_PART_CORTEX_A72, "Cortex-A72" },
 185         { CPU_PART_CORTEX_A73, "Cortex-A73" },
 186         { CPU_PART_CORTEX_A75, "Cortex-A75" },
 187         { CPU_PART_CORTEX_A76, "Cortex-A76" },
 188         { CPU_PART_CORTEX_A76AE, "Cortex-A76AE" },
 189         { CPU_PART_CORTEX_A77, "Cortex-A77" },
 190         { CPU_PART_CORTEX_A78, "Cortex-A78" },
 191         { CPU_PART_CORTEX_A78C, "Cortex-A78C" },
 192         { CPU_PART_CORTEX_A510, "Cortex-A510" },
 193         { CPU_PART_CORTEX_A710, "Cortex-A710" },
 194         { CPU_PART_CORTEX_A715, "Cortex-A715" },
 195         { CPU_PART_CORTEX_X1, "Cortex-X1" },
 196         { CPU_PART_CORTEX_X1C, "Cortex-X1C" },
 197         { CPU_PART_CORTEX_X2, "Cortex-X2" },
 198         { CPU_PART_CORTEX_X3, "Cortex-X3" },
 199         { CPU_PART_NEOVERSE_E1, "Neoverse-E1" },
 200         { CPU_PART_NEOVERSE_N1, "Neoverse-N1" },
 201         { CPU_PART_NEOVERSE_N2, "Neoverse-N2" },
 202         { CPU_PART_NEOVERSE_V1, "Neoverse-V1" },
 203         { CPU_PART_NEOVERSE_V2, "Neoverse-V2" },
 204         CPU_PART_NONE,
 205 };
 206
 207 /* Cavium */
 208 static const struct cpu_parts cpu_parts_cavium[] = {
 209         { CPU_PART_THUNDERX, "ThunderX" },
 210         { CPU_PART_THUNDERX2, "ThunderX2" },
 211         CPU_PART_NONE,
 212 };
 213
 214 /* APM / Ampere */
 215 static const struct cpu_parts cpu_parts_apm[] = {
 216         { CPU_PART_EMAG8180, "eMAG 8180" },
 217         CPU_PART_NONE,
 218 };
 219
 220 /* Qualcomm */
 221 static const struct cpu_parts cpu_parts_qcom[] = {
 222         { CPU_PART_KRYO400_GOLD, "Kryo 400 Gold" },
 223         { CPU_PART_KRYO400_SILVER, "Kryo 400 Silver" },
 224         CPU_PART_NONE,
 225 };
 226
 227 /* Unknown */
 228 static const struct cpu_parts cpu_parts_none[] = {
 229         CPU_PART_NONE,
 230 };
 231
 232 /*
 233  * Implementers table.
 234  */
 235 const struct cpu_implementers cpu_implementers[] = {
 236         { CPU_IMPL_AMPERE,      "Ampere",       cpu_parts_none },
 237         { CPU_IMPL_APPLE,       "Apple",        cpu_parts_none },
 238         { CPU_IMPL_APM,         "APM",          cpu_parts_apm },
 239         { CPU_IMPL_ARM,         "ARM",          cpu_parts_arm },
 240         { CPU_IMPL_BROADCOM,    "Broadcom",     cpu_parts_none },
 241         { CPU_IMPL_CAVIUM,      "Cavium",       cpu_parts_cavium },
 242         { CPU_IMPL_DEC,         "DEC",          cpu_parts_none },
 243         { CPU_IMPL_FREESCALE,   "Freescale",    cpu_parts_none },
 244         { CPU_IMPL_FUJITSU,     "Fujitsu",      cpu_parts_none },
 245         { CPU_IMPL_INFINEON,    "IFX",          cpu_parts_none },
 246         { CPU_IMPL_INTEL,       "Intel",        cpu_parts_none },
 247         { CPU_IMPL_MARVELL,     "Marvell",      cpu_parts_none },
 248         { CPU_IMPL_NVIDIA,      "NVIDIA",       cpu_parts_none },
 249         { CPU_IMPL_QUALCOMM,    "Qualcomm",     cpu_parts_qcom },
 250         CPU_IMPLEMENTER_NONE,
 251 };
 252
 253 #define MRS_TYPE_MASK           0xf
 254 #define MRS_INVALID             0
 255 #define MRS_EXACT               1
 256 #define MRS_EXACT_VAL(x)        (MRS_EXACT | ((x) << 4))
 257 #define MRS_EXACT_FIELD(x)      ((x) >> 4)
 258 #define MRS_LOWER               2
 259
 260 struct mrs_field_value {
 261         uint64_t        value;
 262         const char      *desc;
 263 };
 264
 265 #define MRS_FIELD_VALUE(_value, _desc)                                  \
 266         {                                                               \
 267                 .value = (_value),                                      \
 268                 .desc = (_desc),                                        \
 269         }
 270
 271 #define MRS_FIELD_VALUE_NONE_IMPL(_reg, _field, _none, _impl)           \
 272         MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _none, ""),         \
 273         MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _impl, #_field)
 274
 275 #define MRS_FIELD_VALUE_COUNT(_reg, _field, _desc)                      \
 276         MRS_FIELD_VALUE(0ul << _reg ## _ ## _field ## _SHIFT, "1 " _desc), \
 277         MRS_FIELD_VALUE(1ul << _reg ## _ ## _field ## _SHIFT, "2 " _desc "s"), \
 278         MRS_FIELD_VALUE(2ul << _reg ## _ ## _field ## _SHIFT, "3 " _desc "s"), \
 279         MRS_FIELD_VALUE(3ul << _reg ## _ ## _field ## _SHIFT, "4 " _desc "s"), \
 280         MRS_FIELD_VALUE(4ul << _reg ## _ ## _field ## _SHIFT, "5 " _desc "s"), \
 281         MRS_FIELD_VALUE(5ul << _reg ## _ ## _field ## _SHIFT, "6 " _desc "s"), \
 282         MRS_FIELD_VALUE(6ul << _reg ## _ ## _field ## _SHIFT, "7 " _desc "s"), \
 283         MRS_FIELD_VALUE(7ul << _reg ## _ ## _field ## _SHIFT, "8 " _desc "s"), \
 284         MRS_FIELD_VALUE(8ul << _reg ## _ ## _field ## _SHIFT, "9 " _desc "s"), \
 285         MRS_FIELD_VALUE(9ul << _reg ## _ ## _field ## _SHIFT, "10 "_desc "s"), \
 286         MRS_FIELD_VALUE(10ul<< _reg ## _ ## _field ## _SHIFT, "11 "_desc "s"), \
 287         MRS_FIELD_VALUE(11ul<< _reg ## _ ## _field ## _SHIFT, "12 "_desc "s"), \
 288         MRS_FIELD_VALUE(12ul<< _reg ## _ ## _field ## _SHIFT, "13 "_desc "s"), \
 289         MRS_FIELD_VALUE(13ul<< _reg ## _ ## _field ## _SHIFT, "14 "_desc "s"), \
 290         MRS_FIELD_VALUE(14ul<< _reg ## _ ## _field ## _SHIFT, "15 "_desc "s"), \
 291         MRS_FIELD_VALUE(15ul<< _reg ## _ ## _field ## _SHIFT, "16 "_desc "s")
 292
 293 #define MRS_FIELD_VALUE_END     { .desc = NULL }
 294
 295 struct mrs_field_hwcap {
 296         u_long          *hwcap;
 297         uint64_t        min;
 298         u_long          hwcap_val;
 299 };
 300
 301 #define MRS_HWCAP(_hwcap, _val, _min)                           \
 302 {                                                               \
 303         .hwcap = (_hwcap),                                      \
 304         .hwcap_val = (_val),                                    \
 305         .min = (_min),                                          \
 306 }
 307
 308 #define MRS_HWCAP_END           { .hwcap = NULL }
 309
 310 struct mrs_field {
 311         const char      *name;
 312         struct mrs_field_value *values;
 313         struct mrs_field_hwcap *hwcaps;
 314         uint64_t        mask;
 315         bool            sign;
 316         u_int           type;
 317         u_int           shift;
 318 };
 319
 320 #define MRS_FIELD_HWCAP(_register, _name, _sign, _type, _values, _hwcap) \
 321         {                                                               \
 322                 .name = #_name,                                         \
 323                 .sign = (_sign),                                        \
 324                 .type = (_type),                                        \
 325                 .shift = _register ## _ ## _name ## _SHIFT,             \
 326                 .mask = _register ## _ ## _name ## _MASK,               \
 327                 .values = (_values),                                    \
 328                 .hwcaps = (_hwcap),                                     \
 329         }
 330
 331 #define MRS_FIELD(_register, _name, _sign, _type, _values)              \
 332         MRS_FIELD_HWCAP(_register, _name, _sign, _type, _values, NULL)
 333
 334 #define MRS_FIELD_END   { .type = MRS_INVALID, }
 335
 336 /* ID_AA64AFR0_EL1 */
 337 static struct mrs_field id_aa64afr0_fields[] = {
 338         MRS_FIELD_END,
 339 };
 340
 341
 342 /* ID_AA64AFR1_EL1 */
 343 static struct mrs_field id_aa64afr1_fields[] = {
 344         MRS_FIELD_END,
 345 };
 346
 347
 348 /* ID_AA64DFR0_EL1 */
 349 static struct mrs_field_value id_aa64dfr0_tracefilt[] = {
 350         MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_NONE, ""),
 351         MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_8_4, "Trace v8.4"),
 352         MRS_FIELD_VALUE_END,
 353 };
 354
 355 static struct mrs_field_value id_aa64dfr0_doublelock[] = {
 356         MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_IMPL, "DoubleLock"),
 357         MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_NONE, ""),
 358         MRS_FIELD_VALUE_END,
 359 };
 360
 361 static struct mrs_field_value id_aa64dfr0_pmsver[] = {
 362         MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_NONE, ""),
 363         MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE, "SPE"),
 364         MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_8_3, "SPE v8.3"),
 365         MRS_FIELD_VALUE_END,
 366 };
 367
 368 static struct mrs_field_value id_aa64dfr0_ctx_cmps[] = {
 369         MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, CTX_CMPs, "CTX BKPT"),
 370         MRS_FIELD_VALUE_END,
 371 };
 372
 373 static struct mrs_field_value id_aa64dfr0_wrps[] = {
 374         MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, WRPs, "Watchpoint"),
 375         MRS_FIELD_VALUE_END,
 376 };
 377
 378 static struct mrs_field_value id_aa64dfr0_brps[] = {
 379         MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, BRPs, "Breakpoint"),
 380         MRS_FIELD_VALUE_END,
 381 };
 382
 383 static struct mrs_field_value id_aa64dfr0_pmuver[] = {
 384         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_NONE, ""),
 385         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3, "PMUv3"),
 386         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_1, "PMUv3 v8.1"),
 387         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_4, "PMUv3 v8.4"),
 388         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_5, "PMUv3 v8.5"),
 389         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_IMPL, "IMPL PMU"),
 390         MRS_FIELD_VALUE_END,
 391 };
 392
 393 static struct mrs_field_value id_aa64dfr0_tracever[] = {
 394         MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_NONE, ""),
 395         MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_IMPL, "Trace"),
 396         MRS_FIELD_VALUE_END,
 397 };
 398
 399 static struct mrs_field_value id_aa64dfr0_debugver[] = {
 400         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8, "Debugv8"),
 401         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_VHE, "Debugv8_VHE"),
 402         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_2, "Debugv8.2"),
 403         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_4, "Debugv8.4"),
 404         MRS_FIELD_VALUE_END,
 405 };
 406
 407 static struct mrs_field id_aa64dfr0_fields[] = {
 408         MRS_FIELD(ID_AA64DFR0, TraceFilt, false, MRS_EXACT,
 409             id_aa64dfr0_tracefilt),
 410         MRS_FIELD(ID_AA64DFR0, DoubleLock, false, MRS_EXACT,
 411             id_aa64dfr0_doublelock),
 412         MRS_FIELD(ID_AA64DFR0, PMSVer, false, MRS_EXACT, id_aa64dfr0_pmsver),
 413         MRS_FIELD(ID_AA64DFR0, CTX_CMPs, false, MRS_EXACT,
 414             id_aa64dfr0_ctx_cmps),
 415         MRS_FIELD(ID_AA64DFR0, WRPs, false, MRS_LOWER, id_aa64dfr0_wrps),
 416         MRS_FIELD(ID_AA64DFR0, BRPs, false, MRS_LOWER, id_aa64dfr0_brps),
 417         MRS_FIELD(ID_AA64DFR0, PMUVer, false, MRS_EXACT, id_aa64dfr0_pmuver),
 418         MRS_FIELD(ID_AA64DFR0, TraceVer, false, MRS_EXACT,
 419             id_aa64dfr0_tracever),
 420         MRS_FIELD(ID_AA64DFR0, DebugVer, false, MRS_EXACT_VAL(0x6),
 421             id_aa64dfr0_debugver),
 422         MRS_FIELD_END,
 423 };
 424
 425
 426 /* ID_AA64DFR1_EL1 */
 427 static struct mrs_field id_aa64dfr1_fields[] = {
 428         MRS_FIELD_END,
 429 };
 430
 431
 432 /* ID_AA64ISAR0_EL1 */
 433 static struct mrs_field_value id_aa64isar0_rndr[] = {
 434         MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_NONE, ""),
 435         MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_IMPL, "RNG"),
 436         MRS_FIELD_VALUE_END,
 437 };
 438
 439 static struct mrs_field_hwcap id_aa64isar0_rndr_caps[] = {
 440         MRS_HWCAP(&elf_hwcap2, HWCAP2_RNG, ID_AA64ISAR0_RNDR_IMPL),
 441         MRS_HWCAP_END
 442 };
 443
 444 static struct mrs_field_value id_aa64isar0_tlb[] = {
 445         MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_NONE, ""),
 446         MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOS, "TLBI-OS"),
 447         MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOSR, "TLBI-OSR"),
 448         MRS_FIELD_VALUE_END,
 449 };
 450
 451 static struct mrs_field_value id_aa64isar0_ts[] = {
 452         MRS_FIELD_VALUE(ID_AA64ISAR0_TS_NONE, ""),
 453         MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_4, "CondM-8.4"),
 454         MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_5, "CondM-8.5"),
 455         MRS_FIELD_VALUE_END,
 456 };
 457
 458 static struct mrs_field_hwcap id_aa64isar0_ts_caps[] = {
 459         MRS_HWCAP(&elf_hwcap, HWCAP_FLAGM, ID_AA64ISAR0_TS_CondM_8_4),
 460         MRS_HWCAP(&elf_hwcap2, HWCAP2_FLAGM2, ID_AA64ISAR0_TS_CondM_8_5),
 461         MRS_HWCAP_END
 462 };
 463
 464 static struct mrs_field_value id_aa64isar0_fhm[] = {
 465         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, FHM, NONE, IMPL),
 466         MRS_FIELD_VALUE_END,
 467 };
 468
 469 static struct mrs_field_hwcap id_aa64isar0_fhm_caps[] = {
 470         MRS_HWCAP(&elf_hwcap, HWCAP_ASIMDFHM, ID_AA64ISAR0_FHM_IMPL),
 471         MRS_HWCAP_END
 472 };
 473
 474 static struct mrs_field_value id_aa64isar0_dp[] = {
 475         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, DP, NONE, IMPL),
 476         MRS_FIELD_VALUE_END,
 477 };
 478
 479 static struct mrs_field_hwcap id_aa64isar0_dp_caps[] = {
 480         MRS_HWCAP(&elf_hwcap, HWCAP_ASIMDDP, ID_AA64ISAR0_DP_IMPL),
 481         MRS_HWCAP_END
 482 };
 483
 484 static struct mrs_field_value id_aa64isar0_sm4[] = {
 485         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM4, NONE, IMPL),
 486         MRS_FIELD_VALUE_END,
 487 };
 488
 489 static struct mrs_field_hwcap id_aa64isar0_sm4_caps[] = {
 490         MRS_HWCAP(&elf_hwcap, HWCAP_SM4, ID_AA64ISAR0_SM4_IMPL),
 491         MRS_HWCAP_END
 492 };
 493
 494 static struct mrs_field_value id_aa64isar0_sm3[] = {
 495         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM3, NONE, IMPL),
 496         MRS_FIELD_VALUE_END,
 497 };
 498
 499 static struct mrs_field_hwcap id_aa64isar0_sm3_caps[] = {
 500         MRS_HWCAP(&elf_hwcap, HWCAP_SM3, ID_AA64ISAR0_SM3_IMPL),
 501         MRS_HWCAP_END
 502 };
 503
 504 static struct mrs_field_value id_aa64isar0_sha3[] = {
 505         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA3, NONE, IMPL),
 506         MRS_FIELD_VALUE_END,
 507 };
 508
 509 static struct mrs_field_hwcap id_aa64isar0_sha3_caps[] = {
 510         MRS_HWCAP(&elf_hwcap, HWCAP_SHA3, ID_AA64ISAR0_SHA3_IMPL),
 511         MRS_HWCAP_END
 512 };
 513
 514 static struct mrs_field_value id_aa64isar0_rdm[] = {
 515         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, RDM, NONE, IMPL),
 516         MRS_FIELD_VALUE_END,
 517 };
 518
 519 static struct mrs_field_hwcap id_aa64isar0_rdm_caps[] = {
 520         MRS_HWCAP(&elf_hwcap, HWCAP_ASIMDRDM, ID_AA64ISAR0_RDM_IMPL),
 521         MRS_HWCAP_END
 522 };
 523
 524 static struct mrs_field_value id_aa64isar0_atomic[] = {
 525         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, Atomic, NONE, IMPL),
 526         MRS_FIELD_VALUE_END,
 527 };
 528
 529 static struct mrs_field_hwcap id_aa64isar0_atomic_caps[] = {
 530         MRS_HWCAP(&elf_hwcap, HWCAP_ATOMICS, ID_AA64ISAR0_Atomic_IMPL),
 531         MRS_HWCAP_END
 532 };
 533
 534 static struct mrs_field_value id_aa64isar0_crc32[] = {
 535         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, CRC32, NONE, BASE),
 536         MRS_FIELD_VALUE_END,
 537 };
 538
 539 static struct mrs_field_hwcap id_aa64isar0_crc32_caps[] = {
 540         MRS_HWCAP(&elf_hwcap, HWCAP_CRC32, ID_AA64ISAR0_CRC32_BASE),
 541         MRS_HWCAP_END
 542 };
 543
 544 static struct mrs_field_value id_aa64isar0_sha2[] = {
 545         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA2, NONE, BASE),
 546         MRS_FIELD_VALUE(ID_AA64ISAR0_SHA2_512, "SHA2+SHA512"),
 547         MRS_FIELD_VALUE_END,
 548 };
 549
 550 static struct mrs_field_hwcap id_aa64isar0_sha2_caps[] = {
 551         MRS_HWCAP(&elf_hwcap, HWCAP_SHA2, ID_AA64ISAR0_SHA2_BASE),
 552         MRS_HWCAP(&elf_hwcap, HWCAP_SHA512, ID_AA64ISAR0_SHA2_512),
 553         MRS_HWCAP_END
 554 };
 555
 556 static struct mrs_field_value id_aa64isar0_sha1[] = {
 557         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA1, NONE, BASE),
 558         MRS_FIELD_VALUE_END,
 559 };
 560
 561 static struct mrs_field_hwcap id_aa64isar0_sha1_caps[] = {
 562         MRS_HWCAP(&elf_hwcap, HWCAP_SHA1, ID_AA64ISAR0_SHA1_BASE),
 563         MRS_HWCAP_END
 564 };
 565
 566 static struct mrs_field_value id_aa64isar0_aes[] = {
 567         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, AES, NONE, BASE),
 568         MRS_FIELD_VALUE(ID_AA64ISAR0_AES_PMULL, "AES+PMULL"),
 569         MRS_FIELD_VALUE_END,
 570 };
 571
 572 static struct mrs_field_hwcap id_aa64isar0_aes_caps[] = {
 573         MRS_HWCAP(&elf_hwcap, HWCAP_AES, ID_AA64ISAR0_AES_BASE),
 574         MRS_HWCAP(&elf_hwcap, HWCAP_PMULL, ID_AA64ISAR0_AES_PMULL),
 575         MRS_HWCAP_END
 576 };
 577
 578 static struct mrs_field id_aa64isar0_fields[] = {
 579         MRS_FIELD_HWCAP(ID_AA64ISAR0, RNDR, false, MRS_LOWER,
 580             id_aa64isar0_rndr, id_aa64isar0_rndr_caps),
 581         MRS_FIELD(ID_AA64ISAR0, TLB, false, MRS_EXACT, id_aa64isar0_tlb),
 582         MRS_FIELD_HWCAP(ID_AA64ISAR0, TS, false, MRS_LOWER, id_aa64isar0_ts,
 583             id_aa64isar0_ts_caps),
 584         MRS_FIELD_HWCAP(ID_AA64ISAR0, FHM, false, MRS_LOWER, id_aa64isar0_fhm,
 585             id_aa64isar0_fhm_caps),
 586         MRS_FIELD_HWCAP(ID_AA64ISAR0, DP, false, MRS_LOWER, id_aa64isar0_dp,
 587             id_aa64isar0_dp_caps),
 588         MRS_FIELD_HWCAP(ID_AA64ISAR0, SM4, false, MRS_LOWER, id_aa64isar0_sm4,
 589             id_aa64isar0_sm4_caps),
 590         MRS_FIELD_HWCAP(ID_AA64ISAR0, SM3, false, MRS_LOWER, id_aa64isar0_sm3,
 591             id_aa64isar0_sm3_caps),
 592         MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA3, false, MRS_LOWER, id_aa64isar0_sha3,
 593             id_aa64isar0_sha3_caps),
 594         MRS_FIELD_HWCAP(ID_AA64ISAR0, RDM, false, MRS_LOWER, id_aa64isar0_rdm,
 595             id_aa64isar0_rdm_caps),
 596         MRS_FIELD_HWCAP(ID_AA64ISAR0, Atomic, false, MRS_LOWER,
 597             id_aa64isar0_atomic, id_aa64isar0_atomic_caps),
 598         MRS_FIELD_HWCAP(ID_AA64ISAR0, CRC32, false, MRS_LOWER,
 599             id_aa64isar0_crc32, id_aa64isar0_crc32_caps),
 600         MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA2, false, MRS_LOWER, id_aa64isar0_sha2,
 601             id_aa64isar0_sha2_caps),
 602         MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA1, false, MRS_LOWER,
 603             id_aa64isar0_sha1, id_aa64isar0_sha1_caps),
 604         MRS_FIELD_HWCAP(ID_AA64ISAR0, AES, false, MRS_LOWER, id_aa64isar0_aes,
 605             id_aa64isar0_aes_caps),
 606         MRS_FIELD_END,
 607 };
 608
 609
 610 /* ID_AA64ISAR1_EL1 */
 611 static struct mrs_field_value id_aa64isar1_i8mm[] = {
 612         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, I8MM, NONE, IMPL),
 613         MRS_FIELD_VALUE_END,
 614 };
 615
 616 static struct mrs_field_hwcap id_aa64isar1_i8mm_caps[] = {
 617         MRS_HWCAP(&elf_hwcap2, HWCAP2_I8MM, ID_AA64ISAR1_I8MM_IMPL),
 618         MRS_HWCAP_END
 619 };
 620
 621 static struct mrs_field_value id_aa64isar1_dgh[] = {
 622         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, DGH, NONE, IMPL),
 623         MRS_FIELD_VALUE_END,
 624 };
 625
 626 static struct mrs_field_hwcap id_aa64isar1_dgh_caps[] = {
 627         MRS_HWCAP(&elf_hwcap2, HWCAP2_DGH, ID_AA64ISAR1_DGH_IMPL),
 628         MRS_HWCAP_END
 629 };
 630
 631 static struct mrs_field_value id_aa64isar1_bf16[] = {
 632         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, BF16, NONE, IMPL),
 633         MRS_FIELD_VALUE_END,
 634 };
 635
 636 static struct mrs_field_hwcap id_aa64isar1_bf16_caps[] = {
 637         MRS_HWCAP(&elf_hwcap2, HWCAP2_BF16, ID_AA64ISAR1_BF16_IMPL),
 638         MRS_HWCAP_END
 639 };
 640
 641 static struct mrs_field_value id_aa64isar1_specres[] = {
 642         MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_NONE, ""),
 643         MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_IMPL, "PredInv"),
 644         MRS_FIELD_VALUE_END,
 645 };
 646
 647 static struct mrs_field_value id_aa64isar1_sb[] = {
 648         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, SB, NONE, IMPL),
 649         MRS_FIELD_VALUE_END,
 650 };
 651
 652 static struct mrs_field_hwcap id_aa64isar1_sb_caps[] = {
 653         MRS_HWCAP(&elf_hwcap, HWCAP_SB, ID_AA64ISAR1_SB_IMPL),
 654         MRS_HWCAP_END
 655 };
 656
 657 static struct mrs_field_value id_aa64isar1_frintts[] = {
 658         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FRINTTS, NONE, IMPL),
 659         MRS_FIELD_VALUE_END,
 660 };
 661
 662 static struct mrs_field_hwcap id_aa64isar1_frintts_caps[] = {
 663         MRS_HWCAP(&elf_hwcap2, HWCAP2_FRINT, ID_AA64ISAR1_FRINTTS_IMPL),
 664         MRS_HWCAP_END
 665 };
 666
 667 static struct mrs_field_value id_aa64isar1_gpi[] = {
 668         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPI, NONE, IMPL),
 669         MRS_FIELD_VALUE_END,
 670 };
 671
 672 static struct mrs_field_hwcap id_aa64isar1_gpi_caps[] = {
 673         MRS_HWCAP(&elf_hwcap, HWCAP_PACG, ID_AA64ISAR1_GPI_IMPL),
 674         MRS_HWCAP_END
 675 };
 676
 677 static struct mrs_field_value id_aa64isar1_gpa[] = {
 678         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPA, NONE, IMPL),
 679         MRS_FIELD_VALUE_END,
 680 };
 681
 682 static struct mrs_field_hwcap id_aa64isar1_gpa_caps[] = {
 683         MRS_HWCAP(&elf_hwcap, HWCAP_PACG, ID_AA64ISAR1_GPA_IMPL),
 684         MRS_HWCAP_END
 685 };
 686
 687 static struct mrs_field_value id_aa64isar1_lrcpc[] = {
 688         MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_NONE, ""),
 689         MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_3, "RCPC-8.3"),
 690         MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_4, "RCPC-8.4"),
 691         MRS_FIELD_VALUE_END,
 692 };
 693
 694 static struct mrs_field_hwcap id_aa64isar1_lrcpc_caps[] = {
 695         MRS_HWCAP(&elf_hwcap, HWCAP_LRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_3),
 696         MRS_HWCAP(&elf_hwcap, HWCAP_ILRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_4),
 697         MRS_HWCAP_END
 698 };
 699
 700 static struct mrs_field_value id_aa64isar1_fcma[] = {
 701         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FCMA, NONE, IMPL),
 702         MRS_FIELD_VALUE_END,
 703 };
 704
 705 static struct mrs_field_hwcap id_aa64isar1_fcma_caps[] = {
 706         MRS_HWCAP(&elf_hwcap, HWCAP_FCMA, ID_AA64ISAR1_FCMA_IMPL),
 707         MRS_HWCAP_END
 708 };
 709
 710 static struct mrs_field_value id_aa64isar1_jscvt[] = {
 711         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, JSCVT, NONE, IMPL),
 712         MRS_FIELD_VALUE_END,
 713 };
 714
 715 static struct mrs_field_hwcap id_aa64isar1_jscvt_caps[] = {
 716         MRS_HWCAP(&elf_hwcap, HWCAP_JSCVT, ID_AA64ISAR1_JSCVT_IMPL),
 717         MRS_HWCAP_END
 718 };
 719
 720 static struct mrs_field_value id_aa64isar1_api[] = {
 721         MRS_FIELD_VALUE(ID_AA64ISAR1_API_NONE, ""),
 722         MRS_FIELD_VALUE(ID_AA64ISAR1_API_PAC, "API PAC"),
 723         MRS_FIELD_VALUE(ID_AA64ISAR1_API_EPAC, "API EPAC"),
 724         MRS_FIELD_VALUE(ID_AA64ISAR1_API_EPAC2, "Impl PAuth+EPAC2"),
 725         MRS_FIELD_VALUE(ID_AA64ISAR1_API_FPAC, "Impl PAuth+FPAC"),
 726         MRS_FIELD_VALUE(ID_AA64ISAR1_API_FPAC_COMBINED,
 727             "Impl PAuth+FPAC+Combined"),
 728         MRS_FIELD_VALUE_END,
 729 };
 730
 731 static struct mrs_field_hwcap id_aa64isar1_api_caps[] = {
 732         MRS_HWCAP(&elf_hwcap, HWCAP_PACA, ID_AA64ISAR1_API_PAC),
 733         MRS_HWCAP_END
 734 };
 735
 736 static struct mrs_field_value id_aa64isar1_apa[] = {
 737         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_NONE, ""),
 738         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_PAC, "APA PAC"),
 739         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_EPAC, "APA EPAC"),
 740         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_EPAC2, "APA EPAC2"),
 741         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_FPAC, "APA FPAC"),
 742         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_FPAC_COMBINED,
 743             "APA FPAC+Combined"),
 744         MRS_FIELD_VALUE_END,
 745 };
 746
 747 static struct mrs_field_hwcap id_aa64isar1_apa_caps[] = {
 748         MRS_HWCAP(&elf_hwcap, HWCAP_PACA, ID_AA64ISAR1_APA_PAC),
 749         MRS_HWCAP_END
 750 };
 751
 752 static struct mrs_field_value id_aa64isar1_dpb[] = {
 753         MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_NONE, ""),
 754         MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVAP, "DCPoP"),
 755         MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVADP, "DCCVADP"),
 756         MRS_FIELD_VALUE_END,
 757 };
 758
 759 static struct mrs_field_hwcap id_aa64isar1_dpb_caps[] = {
 760         MRS_HWCAP(&elf_hwcap, HWCAP_DCPOP, ID_AA64ISAR1_DPB_DCCVAP),
 761         MRS_HWCAP(&elf_hwcap2, HWCAP2_DCPODP, ID_AA64ISAR1_DPB_DCCVADP),
 762         MRS_HWCAP_END
 763 };
 764
 765 static struct mrs_field id_aa64isar1_fields[] = {
 766         MRS_FIELD_HWCAP(ID_AA64ISAR1, I8MM, false, MRS_LOWER,
 767             id_aa64isar1_i8mm, id_aa64isar1_i8mm_caps),
 768         MRS_FIELD_HWCAP(ID_AA64ISAR1, DGH, false, MRS_LOWER, id_aa64isar1_dgh,
 769             id_aa64isar1_dgh_caps),
 770         MRS_FIELD_HWCAP(ID_AA64ISAR1, BF16, false, MRS_LOWER,
 771             id_aa64isar1_bf16, id_aa64isar1_bf16_caps),
 772         MRS_FIELD(ID_AA64ISAR1, SPECRES, false, MRS_EXACT,
 773             id_aa64isar1_specres),
 774         MRS_FIELD_HWCAP(ID_AA64ISAR1, SB, false, MRS_LOWER, id_aa64isar1_sb,
 775             id_aa64isar1_sb_caps),
 776         MRS_FIELD_HWCAP(ID_AA64ISAR1, FRINTTS, false, MRS_LOWER,
 777             id_aa64isar1_frintts, id_aa64isar1_frintts_caps),
 778         MRS_FIELD_HWCAP(ID_AA64ISAR1, GPI, false, MRS_EXACT, id_aa64isar1_gpi,
 779             id_aa64isar1_gpi_caps),
 780         MRS_FIELD_HWCAP(ID_AA64ISAR1, GPA, false, MRS_EXACT, id_aa64isar1_gpa,
 781             id_aa64isar1_gpa_caps),
 782         MRS_FIELD_HWCAP(ID_AA64ISAR1, LRCPC, false, MRS_LOWER,
 783             id_aa64isar1_lrcpc, id_aa64isar1_lrcpc_caps),
 784         MRS_FIELD_HWCAP(ID_AA64ISAR1, FCMA, false, MRS_LOWER,
 785             id_aa64isar1_fcma, id_aa64isar1_fcma_caps),
 786         MRS_FIELD_HWCAP(ID_AA64ISAR1, JSCVT, false, MRS_LOWER,
 787             id_aa64isar1_jscvt, id_aa64isar1_jscvt_caps),
 788         MRS_FIELD_HWCAP(ID_AA64ISAR1, API, false, MRS_EXACT, id_aa64isar1_api,
 789             id_aa64isar1_api_caps),
 790         MRS_FIELD_HWCAP(ID_AA64ISAR1, APA, false, MRS_EXACT, id_aa64isar1_apa,
 791             id_aa64isar1_apa_caps),
 792         MRS_FIELD_HWCAP(ID_AA64ISAR1, DPB, false, MRS_LOWER, id_aa64isar1_dpb,
 793             id_aa64isar1_dpb_caps),
 794         MRS_FIELD_END,
 795 };
 796
 797
 798 /* ID_AA64ISAR2_EL1 */
 799 static struct mrs_field_value id_aa64isar2_pac_frac[] = {
 800         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, PAC_frac, NONE, IMPL),
 801         MRS_FIELD_VALUE_END,
 802 };
 803
 804 static struct mrs_field_value id_aa64isar2_bc[] = {
 805         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, BC, NONE, IMPL),
 806         MRS_FIELD_VALUE_END,
 807 };
 808
 809 static struct mrs_field_value id_aa64isar2_mops[] = {
 810         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, MOPS, NONE, IMPL),
 811         MRS_FIELD_VALUE_END,
 812 };
 813
 814 static struct mrs_field_value id_aa64isar2_apa3[] = {
 815         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_NONE, ""),
 816         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_PAC, "APA3 PAC"),
 817         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_EPAC, "APA3 EPAC"),
 818         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_EPAC2, "APA3 EPAC2"),
 819         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_FPAC, "APA3 FPAC"),
 820         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_FPAC_COMBINED,
 821             "APA3 FPAC+Combined"),
 822         MRS_FIELD_VALUE_END,
 823 };
 824
 825 static struct mrs_field_hwcap id_aa64isar2_apa3_caps[] = {
 826         MRS_HWCAP(&elf_hwcap, HWCAP_PACA, ID_AA64ISAR2_APA3_PAC),
 827         MRS_HWCAP_END
 828 };
 829
 830 static struct mrs_field_value id_aa64isar2_gpa3[] = {
 831         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, GPA3, NONE, IMPL),
 832         MRS_FIELD_VALUE_END,
 833 };
 834
 835 static struct mrs_field_hwcap id_aa64isar2_gpa3_caps[] = {
 836         MRS_HWCAP(&elf_hwcap, HWCAP_PACG, ID_AA64ISAR2_GPA3_IMPL),
 837         MRS_HWCAP_END
 838 };
 839
 840 static struct mrs_field_value id_aa64isar2_rpres[] = {
 841         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, RPRES, NONE, IMPL),
 842         MRS_FIELD_VALUE_END,
 843 };
 844
 845 static struct mrs_field_value id_aa64isar2_wfxt[] = {
 846         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, WFxT, NONE, IMPL),
 847         MRS_FIELD_VALUE_END,
 848 };
 849
 850 static struct mrs_field id_aa64isar2_fields[] = {
 851         MRS_FIELD(ID_AA64ISAR2, PAC_frac, false, MRS_EXACT,
 852             id_aa64isar2_pac_frac),
 853         MRS_FIELD(ID_AA64ISAR2, BC, false, MRS_EXACT, id_aa64isar2_bc),
 854         MRS_FIELD(ID_AA64ISAR2, MOPS, false, MRS_EXACT, id_aa64isar2_mops),
 855         MRS_FIELD_HWCAP(ID_AA64ISAR2, APA3, false, MRS_EXACT,
 856             id_aa64isar2_apa3, id_aa64isar2_apa3_caps),
 857         MRS_FIELD_HWCAP(ID_AA64ISAR2, GPA3, false, MRS_EXACT,
 858             id_aa64isar2_gpa3, id_aa64isar2_gpa3_caps),
 859         MRS_FIELD(ID_AA64ISAR2, RPRES, false, MRS_EXACT, id_aa64isar2_rpres),
 860         MRS_FIELD(ID_AA64ISAR2, WFxT, false, MRS_EXACT, id_aa64isar2_wfxt),
 861         MRS_FIELD_END,
 862 };
 863
 864
 865 /* ID_AA64MMFR0_EL1 */
 866 static struct mrs_field_value id_aa64mmfr0_exs[] = {
 867         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, ExS, ALL, IMPL),
 868         MRS_FIELD_VALUE_END,
 869 };
 870
 871 static struct mrs_field_value id_aa64mmfr0_tgran4_2[] = {
 872         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_TGran4, ""),
 873         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_NONE, "No S2 TGran4"),
 874         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_IMPL, "S2 TGran4"),
 875         MRS_FIELD_VALUE_END,
 876 };
 877
 878 static struct mrs_field_value id_aa64mmfr0_tgran64_2[] = {
 879         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_TGran64, ""),
 880         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_NONE, "No S2 TGran64"),
 881         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_IMPL, "S2 TGran64"),
 882         MRS_FIELD_VALUE_END,
 883 };
 884
 885 static struct mrs_field_value id_aa64mmfr0_tgran16_2[] = {
 886         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_TGran16, ""),
 887         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_NONE, "No S2 TGran16"),
 888         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_IMPL, "S2 TGran16"),
 889         MRS_FIELD_VALUE_END,
 890 };
 891
 892 static struct mrs_field_value id_aa64mmfr0_tgran4[] = {
 893         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran4,NONE, IMPL),
 894         MRS_FIELD_VALUE_END,
 895 };
 896
 897 static struct mrs_field_value id_aa64mmfr0_tgran64[] = {
 898         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran64, NONE, IMPL),
 899         MRS_FIELD_VALUE_END,
 900 };
 901
 902 static struct mrs_field_value id_aa64mmfr0_tgran16[] = {
 903         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran16, NONE, IMPL),
 904         MRS_FIELD_VALUE_END,
 905 };
 906
 907 static struct mrs_field_value id_aa64mmfr0_bigendel0[] = {
 908         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEndEL0, FIXED, MIXED),
 909         MRS_FIELD_VALUE_END,
 910 };
 911
 912 static struct mrs_field_value id_aa64mmfr0_snsmem[] = {
 913         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, SNSMem, NONE, DISTINCT),
 914         MRS_FIELD_VALUE_END,
 915 };
 916
 917 static struct mrs_field_value id_aa64mmfr0_bigend[] = {
 918         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEnd, FIXED, MIXED),
 919         MRS_FIELD_VALUE_END,
 920 };
 921
 922 static struct mrs_field_value id_aa64mmfr0_asidbits[] = {
 923         MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_8, "8bit ASID"),
 924         MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_16, "16bit ASID"),
 925         MRS_FIELD_VALUE_END,
 926 };
 927
 928 static struct mrs_field_value id_aa64mmfr0_parange[] = {
 929         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4G, "4GB PA"),
 930         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_64G, "64GB PA"),
 931         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_1T, "1TB PA"),
 932         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4T, "4TB PA"),
 933         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_16T, "16TB PA"),
 934         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_256T, "256TB PA"),
 935         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4P, "4PB PA"),
 936         MRS_FIELD_VALUE_END,
 937 };
 938
 939 static struct mrs_field id_aa64mmfr0_fields[] = {
 940         MRS_FIELD(ID_AA64MMFR0, ExS, false, MRS_EXACT, id_aa64mmfr0_exs),
 941         MRS_FIELD(ID_AA64MMFR0, TGran4_2, false, MRS_EXACT,
 942             id_aa64mmfr0_tgran4_2),
 943         MRS_FIELD(ID_AA64MMFR0, TGran64_2, false, MRS_EXACT,
 944             id_aa64mmfr0_tgran64_2),
 945         MRS_FIELD(ID_AA64MMFR0, TGran16_2, false, MRS_EXACT,
 946             id_aa64mmfr0_tgran16_2),
 947         MRS_FIELD(ID_AA64MMFR0, TGran4, false, MRS_EXACT, id_aa64mmfr0_tgran4),
 948         MRS_FIELD(ID_AA64MMFR0, TGran64, false, MRS_EXACT,
 949             id_aa64mmfr0_tgran64),
 950         MRS_FIELD(ID_AA64MMFR0, TGran16, false, MRS_EXACT,
 951             id_aa64mmfr0_tgran16),
 952         MRS_FIELD(ID_AA64MMFR0, BigEndEL0, false, MRS_EXACT,
 953             id_aa64mmfr0_bigendel0),
 954         MRS_FIELD(ID_AA64MMFR0, SNSMem, false, MRS_EXACT, id_aa64mmfr0_snsmem),
 955         MRS_FIELD(ID_AA64MMFR0, BigEnd, false, MRS_EXACT, id_aa64mmfr0_bigend),
 956         MRS_FIELD(ID_AA64MMFR0, ASIDBits, false, MRS_EXACT,
 957             id_aa64mmfr0_asidbits),
 958         MRS_FIELD(ID_AA64MMFR0, PARange, false, MRS_EXACT,
 959             id_aa64mmfr0_parange),
 960         MRS_FIELD_END,
 961 };
 962
 963
 964 /* ID_AA64MMFR1_EL1 */
 965 static struct mrs_field_value id_aa64mmfr1_xnx[] = {
 966         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, XNX, NONE, IMPL),
 967         MRS_FIELD_VALUE_END,
 968 };
 969
 970 static struct mrs_field_value id_aa64mmfr1_specsei[] = {
 971         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, SpecSEI, NONE, IMPL),
 972         MRS_FIELD_VALUE_END,
 973 };
 974
 975 static struct mrs_field_value id_aa64mmfr1_pan[] = {
 976         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, PAN, NONE, IMPL),
 977         MRS_FIELD_VALUE(ID_AA64MMFR1_PAN_ATS1E1, "PAN+ATS1E1"),
 978         MRS_FIELD_VALUE_END,
 979 };
 980
 981 static struct mrs_field_value id_aa64mmfr1_lo[] = {
 982         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, LO, NONE, IMPL),
 983         MRS_FIELD_VALUE_END,
 984 };
 985
 986 static struct mrs_field_value id_aa64mmfr1_hpds[] = {
 987         MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_NONE, ""),
 988         MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_HPD, "HPD"),
 989         MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_TTPBHA, "HPD+TTPBHA"),
 990         MRS_FIELD_VALUE_END,
 991 };
 992
 993 static struct mrs_field_value id_aa64mmfr1_vh[] = {
 994         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, VH, NONE, IMPL),
 995         MRS_FIELD_VALUE_END,
 996 };
 997
 998 static struct mrs_field_value id_aa64mmfr1_vmidbits[] = {
 999         MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_8, "8bit VMID"),
1000         MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_16, "16bit VMID"),
1001         MRS_FIELD_VALUE_END,
1002 };
1003
1004 static struct mrs_field_value id_aa64mmfr1_hafdbs[] = {
1005         MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_NONE, ""),
1006         MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF, "HAF"),
1007         MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF_DBS, "HAF+DS"),
1008         MRS_FIELD_VALUE_END,
1009 };
1010
1011 static struct mrs_field id_aa64mmfr1_fields[] = {
1012         MRS_FIELD(ID_AA64MMFR1, XNX, false, MRS_EXACT, id_aa64mmfr1_xnx),
1013         MRS_FIELD(ID_AA64MMFR1, SpecSEI, false, MRS_EXACT,
1014             id_aa64mmfr1_specsei),
1015         MRS_FIELD(ID_AA64MMFR1, PAN, false, MRS_EXACT, id_aa64mmfr1_pan),
1016         MRS_FIELD(ID_AA64MMFR1, LO, false, MRS_EXACT, id_aa64mmfr1_lo),
1017         MRS_FIELD(ID_AA64MMFR1, HPDS, false, MRS_EXACT, id_aa64mmfr1_hpds),
1018         MRS_FIELD(ID_AA64MMFR1, VH, false, MRS_EXACT, id_aa64mmfr1_vh),
1019         MRS_FIELD(ID_AA64MMFR1, VMIDBits, false, MRS_EXACT,
1020             id_aa64mmfr1_vmidbits),
1021         MRS_FIELD(ID_AA64MMFR1, HAFDBS, false, MRS_EXACT, id_aa64mmfr1_hafdbs),
1022         MRS_FIELD_END,
1023 };
1024
1025
1026 /* ID_AA64MMFR2_EL1 */
1027 static struct mrs_field_value id_aa64mmfr2_e0pd[] = {
1028         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, E0PD, NONE, IMPL),
1029         MRS_FIELD_VALUE_END,
1030 };
1031
1032 static struct mrs_field_value id_aa64mmfr2_evt[] = {
1033         MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_NONE, ""),
1034         MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_2, "EVT-8.2"),
1035         MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_5, "EVT-8.5"),
1036         MRS_FIELD_VALUE_END,
1037 };
1038
1039 static struct mrs_field_value id_aa64mmfr2_bbm[] = {
1040         MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL0, ""),
1041         MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL1, "BBM level 1"),
1042         MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL2, "BBM level 2"),
1043         MRS_FIELD_VALUE_END,
1044 };
1045
1046 static struct mrs_field_value id_aa64mmfr2_ttl[] = {
1047         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, TTL, NONE, IMPL),
1048         MRS_FIELD_VALUE_END,
1049 };
1050
1051 static struct mrs_field_value id_aa64mmfr2_fwb[] = {
1052         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, FWB, NONE, IMPL),
1053         MRS_FIELD_VALUE_END,
1054 };
1055
1056 static struct mrs_field_value id_aa64mmfr2_ids[] = {
1057         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IDS, NONE, IMPL),
1058         MRS_FIELD_VALUE_END,
1059 };
1060
1061 static struct mrs_field_value id_aa64mmfr2_at[] = {
1062         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, AT, NONE, IMPL),
1063         MRS_FIELD_VALUE_END,
1064 };
1065
1066 static struct mrs_field_hwcap id_aa64mmfr2_at_caps[] = {
1067         MRS_HWCAP(&elf_hwcap, HWCAP_USCAT, ID_AA64MMFR2_AT_IMPL),
1068         MRS_HWCAP_END
1069 };
1070
1071 static struct mrs_field_value id_aa64mmfr2_st[] = {
1072         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, ST, NONE, IMPL),
1073         MRS_FIELD_VALUE_END,
1074 };
1075
1076 static struct mrs_field_value id_aa64mmfr2_nv[] = {
1077         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, NV, NONE, 8_3),
1078         MRS_FIELD_VALUE(ID_AA64MMFR2_NV_8_4, "NV v8.4"),
1079         MRS_FIELD_VALUE_END,
1080 };
1081
1082 static struct mrs_field_value id_aa64mmfr2_ccidx[] = {
1083         MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_32, "32bit CCIDX"),
1084         MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_64, "64bit CCIDX"),
1085         MRS_FIELD_VALUE_END,
1086 };
1087
1088 static struct mrs_field_value id_aa64mmfr2_varange[] = {
1089         MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_48, "48bit VA"),
1090         MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_52, "52bit VA"),
1091         MRS_FIELD_VALUE_END,
1092 };
1093
1094 static struct mrs_field_value id_aa64mmfr2_iesb[] = {
1095         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IESB, NONE, IMPL),
1096         MRS_FIELD_VALUE_END,
1097 };
1098
1099 static struct mrs_field_value id_aa64mmfr2_lsm[] = {
1100         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, LSM, NONE, IMPL),
1101         MRS_FIELD_VALUE_END,
1102 };
1103
1104 static struct mrs_field_value id_aa64mmfr2_uao[] = {
1105         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, UAO, NONE, IMPL),
1106         MRS_FIELD_VALUE_END,
1107 };
1108
1109 static struct mrs_field_value id_aa64mmfr2_cnp[] = {
1110         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, CnP, NONE, IMPL),
1111         MRS_FIELD_VALUE_END,
1112 };
1113
1114 static struct mrs_field id_aa64mmfr2_fields[] = {
1115         MRS_FIELD(ID_AA64MMFR2, E0PD, false, MRS_EXACT, id_aa64mmfr2_e0pd),
1116         MRS_FIELD(ID_AA64MMFR2, EVT, false, MRS_EXACT, id_aa64mmfr2_evt),
1117         MRS_FIELD(ID_AA64MMFR2, BBM, false, MRS_EXACT, id_aa64mmfr2_bbm),
1118         MRS_FIELD(ID_AA64MMFR2, TTL, false, MRS_EXACT, id_aa64mmfr2_ttl),
1119         MRS_FIELD(ID_AA64MMFR2, FWB, false, MRS_EXACT, id_aa64mmfr2_fwb),
1120         MRS_FIELD(ID_AA64MMFR2, IDS, false, MRS_EXACT, id_aa64mmfr2_ids),
1121         MRS_FIELD_HWCAP(ID_AA64MMFR2, AT, false, MRS_LOWER, id_aa64mmfr2_at,
1122             id_aa64mmfr2_at_caps),
1123         MRS_FIELD(ID_AA64MMFR2, ST, false, MRS_EXACT, id_aa64mmfr2_st),
1124         MRS_FIELD(ID_AA64MMFR2, NV, false, MRS_EXACT, id_aa64mmfr2_nv),
1125         MRS_FIELD(ID_AA64MMFR2, CCIDX, false, MRS_EXACT, id_aa64mmfr2_ccidx),
1126         MRS_FIELD(ID_AA64MMFR2, VARange, false, MRS_EXACT,
1127             id_aa64mmfr2_varange),
1128         MRS_FIELD(ID_AA64MMFR2, IESB, false, MRS_EXACT, id_aa64mmfr2_iesb),
1129         MRS_FIELD(ID_AA64MMFR2, LSM, false, MRS_EXACT, id_aa64mmfr2_lsm),
1130         MRS_FIELD(ID_AA64MMFR2, UAO, false, MRS_EXACT, id_aa64mmfr2_uao),
1131         MRS_FIELD(ID_AA64MMFR2, CnP, false, MRS_EXACT, id_aa64mmfr2_cnp),
1132         MRS_FIELD_END,
1133 };
1134
1135
1136 /* ID_AA64PFR0_EL1 */
1137 static struct mrs_field_value id_aa64pfr0_csv3[] = {
1138         MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_NONE, ""),
1139         MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_ISOLATED, "CSV3"),
1140         MRS_FIELD_VALUE_END,
1141 };
1142
1143 static struct mrs_field_value id_aa64pfr0_csv2[] = {
1144         MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_NONE, ""),
1145         MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_ISOLATED, "CSV2"),
1146         MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_SCXTNUM, "SCXTNUM"),
1147         MRS_FIELD_VALUE_END,
1148 };
1149
1150 static struct mrs_field_value id_aa64pfr0_dit[] = {
1151         MRS_FIELD_VALUE(ID_AA64PFR0_DIT_NONE, ""),
1152         MRS_FIELD_VALUE(ID_AA64PFR0_DIT_PSTATE, "PSTATE.DIT"),
1153         MRS_FIELD_VALUE_END,
1154 };
1155
1156 static struct mrs_field_hwcap id_aa64pfr0_dit_caps[] = {
1157         MRS_HWCAP(&elf_hwcap, HWCAP_DIT, ID_AA64PFR0_DIT_PSTATE),
1158         MRS_HWCAP_END
1159 };
1160
1161 static struct mrs_field_value id_aa64pfr0_amu[] = {
1162         MRS_FIELD_VALUE(ID_AA64PFR0_AMU_NONE, ""),
1163         MRS_FIELD_VALUE(ID_AA64PFR0_AMU_V1, "AMUv1"),
1164         MRS_FIELD_VALUE_END,
1165 };
1166
1167 static struct mrs_field_value id_aa64pfr0_mpam[] = {
1168         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, MPAM, NONE, IMPL),
1169         MRS_FIELD_VALUE_END,
1170 };
1171
1172 static struct mrs_field_value id_aa64pfr0_sel2[] = {
1173         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SEL2, NONE, IMPL),
1174         MRS_FIELD_VALUE_END,
1175 };
1176
1177 static struct mrs_field_value id_aa64pfr0_sve[] = {
1178         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SVE, NONE, IMPL),
1179         MRS_FIELD_VALUE_END,
1180 };
1181
1182 #if 0
1183 /* Enable when we add SVE support */
1184 static struct mrs_field_hwcap id_aa64pfr0_sve_caps[] = {
1185         MRS_HWCAP(&elf_hwcap, HWCAP_SVE, ID_AA64PFR0_SVE_IMPL),
1186         MRS_HWCAP_END
1187 };
1188 #endif
1189
1190 static struct mrs_field_value id_aa64pfr0_ras[] = {
1191         MRS_FIELD_VALUE(ID_AA64PFR0_RAS_NONE, ""),
1192         MRS_FIELD_VALUE(ID_AA64PFR0_RAS_IMPL, "RAS"),
1193         MRS_FIELD_VALUE(ID_AA64PFR0_RAS_8_4, "RAS v8.4"),
1194         MRS_FIELD_VALUE_END,
1195 };
1196
1197 static struct mrs_field_value id_aa64pfr0_gic[] = {
1198         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, GIC, CPUIF_NONE, CPUIF_EN),
1199         MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_NONE, ""),
1200         MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_EN, "GIC"),
1201         MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_4_1, "GIC 4.1"),
1202         MRS_FIELD_VALUE_END,
1203 };
1204
1205 static struct mrs_field_value id_aa64pfr0_advsimd[] = {
1206         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, AdvSIMD, NONE, IMPL),
1207         MRS_FIELD_VALUE(ID_AA64PFR0_AdvSIMD_HP, "AdvSIMD+HP"),
1208         MRS_FIELD_VALUE_END,
1209 };
1210
1211 static struct mrs_field_hwcap id_aa64pfr0_advsimd_caps[] = {
1212         MRS_HWCAP(&elf_hwcap, HWCAP_ASIMD, ID_AA64PFR0_AdvSIMD_IMPL),
1213         MRS_HWCAP(&elf_hwcap, HWCAP_ASIMDHP, ID_AA64PFR0_AdvSIMD_HP),
1214         MRS_HWCAP_END
1215 };
1216
1217 static struct mrs_field_value id_aa64pfr0_fp[] = {
1218         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, FP, NONE, IMPL),
1219         MRS_FIELD_VALUE(ID_AA64PFR0_FP_HP, "FP+HP"),
1220         MRS_FIELD_VALUE_END,
1221 };
1222
1223 static struct mrs_field_hwcap id_aa64pfr0_fp_caps[] = {
1224         MRS_HWCAP(&elf_hwcap, HWCAP_FP, ID_AA64PFR0_FP_IMPL),
1225         MRS_HWCAP(&elf_hwcap, HWCAP_FPHP, ID_AA64PFR0_FP_HP),
1226         MRS_HWCAP_END
1227 };
1228
1229 static struct mrs_field_value id_aa64pfr0_el3[] = {
1230         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL3, NONE, 64),
1231         MRS_FIELD_VALUE(ID_AA64PFR0_EL3_64_32, "EL3 32"),
1232         MRS_FIELD_VALUE_END,
1233 };
1234
1235 static struct mrs_field_value id_aa64pfr0_el2[] = {
1236         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL2, NONE, 64),
1237         MRS_FIELD_VALUE(ID_AA64PFR0_EL2_64_32, "EL2 32"),
1238         MRS_FIELD_VALUE_END,
1239 };
1240
1241 static struct mrs_field_value id_aa64pfr0_el1[] = {
1242         MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64, "EL1"),
1243         MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64_32, "EL1 32"),
1244         MRS_FIELD_VALUE_END,
1245 };
1246
1247 static struct mrs_field_value id_aa64pfr0_el0[] = {
1248         MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64, "EL0"),
1249         MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64_32, "EL0 32"),
1250         MRS_FIELD_VALUE_END,
1251 };
1252
1253 static struct mrs_field id_aa64pfr0_fields[] = {
1254         MRS_FIELD(ID_AA64PFR0, CSV3, false, MRS_EXACT, id_aa64pfr0_csv3),
1255         MRS_FIELD(ID_AA64PFR0, CSV2, false, MRS_EXACT, id_aa64pfr0_csv2),
1256         MRS_FIELD_HWCAP(ID_AA64PFR0, DIT, false, MRS_LOWER, id_aa64pfr0_dit,
1257             id_aa64pfr0_dit_caps),
1258         MRS_FIELD(ID_AA64PFR0, AMU, false, MRS_EXACT, id_aa64pfr0_amu),
1259         MRS_FIELD(ID_AA64PFR0, MPAM, false, MRS_EXACT, id_aa64pfr0_mpam),
1260         MRS_FIELD(ID_AA64PFR0, SEL2, false, MRS_EXACT, id_aa64pfr0_sel2),
1261         MRS_FIELD(ID_AA64PFR0, SVE, false, MRS_EXACT, id_aa64pfr0_sve),
1262         MRS_FIELD(ID_AA64PFR0, RAS, false, MRS_EXACT, id_aa64pfr0_ras),
1263         MRS_FIELD(ID_AA64PFR0, GIC, false, MRS_EXACT, id_aa64pfr0_gic),
1264         MRS_FIELD_HWCAP(ID_AA64PFR0, AdvSIMD, true, MRS_LOWER,
1265             id_aa64pfr0_advsimd, id_aa64pfr0_advsimd_caps),
1266         MRS_FIELD_HWCAP(ID_AA64PFR0, FP, true,  MRS_LOWER, id_aa64pfr0_fp,
1267             id_aa64pfr0_fp_caps),
1268         MRS_FIELD(ID_AA64PFR0, EL3, false, MRS_EXACT, id_aa64pfr0_el3),
1269         MRS_FIELD(ID_AA64PFR0, EL2, false, MRS_EXACT, id_aa64pfr0_el2),
1270         MRS_FIELD(ID_AA64PFR0, EL1, false, MRS_LOWER, id_aa64pfr0_el1),
1271         MRS_FIELD(ID_AA64PFR0, EL0, false, MRS_LOWER, id_aa64pfr0_el0),
1272         MRS_FIELD_END,
1273 };
1274
1275
1276 /* ID_AA64PFR1_EL1 */
1277 static struct mrs_field_value id_aa64pfr1_mte[] = {
1278         MRS_FIELD_VALUE(ID_AA64PFR1_MTE_NONE, ""),
1279         MRS_FIELD_VALUE(ID_AA64PFR1_MTE_IMPL_EL0, "MTE EL0"),
1280         MRS_FIELD_VALUE(ID_AA64PFR1_MTE_IMPL, "MTE"),
1281         MRS_FIELD_VALUE_END,
1282 };
1283
1284 static struct mrs_field_value id_aa64pfr1_ssbs[] = {
1285         MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_NONE, ""),
1286         MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE, "PSTATE.SSBS"),
1287         MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE_MSR, "PSTATE.SSBS MSR"),
1288         MRS_FIELD_VALUE_END,
1289 };
1290
1291 static struct mrs_field_hwcap id_aa64pfr1_ssbs_caps[] = {
1292         MRS_HWCAP(&elf_hwcap, HWCAP_SSBS, ID_AA64PFR1_SSBS_PSTATE),
1293         MRS_HWCAP_END
1294 };
1295
1296 static struct mrs_field_value id_aa64pfr1_bt[] = {
1297         MRS_FIELD_VALUE(ID_AA64PFR1_BT_NONE, ""),
1298         MRS_FIELD_VALUE(ID_AA64PFR1_BT_IMPL, "BTI"),
1299         MRS_FIELD_VALUE_END,
1300 };
1301
1302 #if 0
1303 /* Enable when we add BTI support */
1304 static struct mrs_field_hwcap id_aa64pfr1_bt_caps[] = {
1305         MRS_HWCAP(&elf_hwcap2, HWCAP2_BTI, ID_AA64PFR1_BT_IMPL),
1306         MRS_HWCAP_END
1307 };
1308 #endif
1309
1310 static struct mrs_field id_aa64pfr1_fields[] = {
1311         MRS_FIELD(ID_AA64PFR1, MTE, false, MRS_EXACT, id_aa64pfr1_mte),
1312         MRS_FIELD_HWCAP(ID_AA64PFR1, SSBS, false, MRS_LOWER, id_aa64pfr1_ssbs,
1313             id_aa64pfr1_ssbs_caps),
1314         MRS_FIELD(ID_AA64PFR1, BT, false, MRS_EXACT, id_aa64pfr1_bt),
1315         MRS_FIELD_END,
1316 };
1317
1318
1319 /* ID_AA64ZFR0_EL1 */
1320 static struct mrs_field_value id_aa64zfr0_f64mm[] = {
1321         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, F64MM, NONE, IMPL),
1322         MRS_FIELD_VALUE_END,
1323 };
1324
1325 static struct mrs_field_value id_aa64zfr0_f32mm[] = {
1326         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, F32MM, NONE, IMPL),
1327         MRS_FIELD_VALUE_END,
1328 };
1329
1330 static struct mrs_field_value id_aa64zfr0_i8mm[] = {
1331         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, I8MM, NONE, IMPL),
1332         MRS_FIELD_VALUE_END,
1333 };
1334
1335 static struct mrs_field_value id_aa64zfr0_sm4[] = {
1336         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, SM4, NONE, IMPL),
1337         MRS_FIELD_VALUE_END,
1338 };
1339
1340 static struct mrs_field_value id_aa64zfr0_sha3[] = {
1341         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, SHA3, NONE, IMPL),
1342         MRS_FIELD_VALUE_END,
1343 };
1344
1345 static struct mrs_field_value id_aa64zfr0_bf16[] = {
1346         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, BF16, NONE, BASE),
1347         MRS_FIELD_VALUE(ID_AA64ZFR0_BF16_EBF, "BF16+EBF"),
1348         MRS_FIELD_VALUE_END,
1349 };
1350
1351 static struct mrs_field_value id_aa64zfr0_bitperm[] = {
1352         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, BitPerm, NONE, IMPL),
1353         MRS_FIELD_VALUE_END,
1354 };
1355
1356 static struct mrs_field_value id_aa64zfr0_aes[] = {
1357         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, AES, NONE, BASE),
1358         MRS_FIELD_VALUE(ID_AA64ZFR0_AES_PMULL, "AES+PMULL"),
1359         MRS_FIELD_VALUE_END,
1360 };
1361
1362 static struct mrs_field_value id_aa64zfr0_svever[] = {
1363         MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE1, "SVE1"),
1364         MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE2, "SVE2"),
1365         MRS_FIELD_VALUE_END,
1366 };
1367
1368 static struct mrs_field id_aa64zfr0_fields[] = {
1369         MRS_FIELD(ID_AA64ZFR0, F64MM, false, MRS_EXACT, id_aa64zfr0_f64mm),
1370         MRS_FIELD(ID_AA64ZFR0, F32MM, false, MRS_EXACT, id_aa64zfr0_f32mm),
1371         MRS_FIELD(ID_AA64ZFR0, I8MM, false, MRS_EXACT, id_aa64zfr0_i8mm),
1372         MRS_FIELD(ID_AA64ZFR0, SM4, false, MRS_EXACT, id_aa64zfr0_sm4),
1373         MRS_FIELD(ID_AA64ZFR0, SHA3, false, MRS_EXACT, id_aa64zfr0_sha3),
1374         MRS_FIELD(ID_AA64ZFR0, BF16, false, MRS_EXACT, id_aa64zfr0_bf16),
1375         MRS_FIELD(ID_AA64ZFR0, BitPerm, false, MRS_EXACT, id_aa64zfr0_bitperm),
1376         MRS_FIELD(ID_AA64ZFR0, AES, false, MRS_EXACT, id_aa64zfr0_aes),
1377         MRS_FIELD(ID_AA64ZFR0, SVEver, false, MRS_EXACT, id_aa64zfr0_svever),
1378         MRS_FIELD_END,
1379 };
1380
1381
1382 #ifdef COMPAT_FREEBSD32
1383 /* ID_ISAR5_EL1 */
1384 static struct mrs_field_value id_isar5_vcma[] = {
1385         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, VCMA, NONE, IMPL),
1386         MRS_FIELD_VALUE_END,
1387 };
1388
1389 static struct mrs_field_value id_isar5_rdm[] = {
1390         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, RDM, NONE, IMPL),
1391         MRS_FIELD_VALUE_END,
1392 };
1393
1394 static struct mrs_field_value id_isar5_crc32[] = {
1395         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, CRC32, NONE, IMPL),
1396         MRS_FIELD_VALUE_END,
1397 };
1398
1399 static struct mrs_field_hwcap id_isar5_crc32_caps[] = {
1400         MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_CRC32, ID_ISAR5_CRC32_IMPL),
1401         MRS_HWCAP_END
1402 };
1403
1404 static struct mrs_field_value id_isar5_sha2[] = {
1405         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA2, NONE, IMPL),
1406         MRS_FIELD_VALUE_END,
1407 };
1408
1409 static struct mrs_field_hwcap id_isar5_sha2_caps[] = {
1410         MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_SHA2, ID_ISAR5_SHA2_IMPL),
1411         MRS_HWCAP_END
1412 };
1413
1414 static struct mrs_field_value id_isar5_sha1[] = {
1415         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA1, NONE, IMPL),
1416         MRS_FIELD_VALUE_END,
1417 };
1418
1419 static struct mrs_field_hwcap id_isar5_sha1_caps[] = {
1420         MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_SHA1, ID_ISAR5_SHA1_IMPL),
1421         MRS_HWCAP_END
1422 };
1423
1424 static struct mrs_field_value id_isar5_aes[] = {
1425         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, AES, NONE, BASE),
1426         MRS_FIELD_VALUE(ID_ISAR5_AES_VMULL, "AES+VMULL"),
1427         MRS_FIELD_VALUE_END,
1428 };
1429
1430 static struct mrs_field_hwcap id_isar5_aes_caps[] = {
1431         MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_AES, ID_ISAR5_AES_BASE),
1432         MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_PMULL, ID_ISAR5_AES_VMULL),
1433         MRS_HWCAP_END
1434 };
1435
1436 static struct mrs_field_value id_isar5_sevl[] = {
1437         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SEVL, NOP, IMPL),
1438         MRS_FIELD_VALUE_END,
1439 };
1440
1441 static struct mrs_field id_isar5_fields[] = {
1442         MRS_FIELD(ID_ISAR5, VCMA, false, MRS_LOWER, id_isar5_vcma),
1443         MRS_FIELD(ID_ISAR5, RDM, false, MRS_LOWER, id_isar5_rdm),
1444         MRS_FIELD_HWCAP(ID_ISAR5, CRC32, false, MRS_LOWER, id_isar5_crc32,
1445             id_isar5_crc32_caps),
1446         MRS_FIELD_HWCAP(ID_ISAR5, SHA2, false, MRS_LOWER, id_isar5_sha2,
1447             id_isar5_sha2_caps),
1448         MRS_FIELD_HWCAP(ID_ISAR5, SHA1, false, MRS_LOWER, id_isar5_sha1,
1449             id_isar5_sha1_caps),
1450         MRS_FIELD_HWCAP(ID_ISAR5, AES, false, MRS_LOWER, id_isar5_aes,
1451             id_isar5_aes_caps),
1452         MRS_FIELD(ID_ISAR5, SEVL, false, MRS_LOWER, id_isar5_sevl),
1453         MRS_FIELD_END,
1454 };
1455
1456 /* MVFR0 */
1457 static struct mrs_field_value mvfr0_fpround[] = {
1458         MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPRound, NONE, IMPL),
1459         MRS_FIELD_VALUE_END,
1460 };
1461
1462 static struct mrs_field_value mvfr0_fpsqrt[] = {
1463         MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPSqrt, NONE, IMPL),
1464         MRS_FIELD_VALUE_END,
1465 };
1466
1467 static struct mrs_field_value mvfr0_fpdivide[] = {
1468         MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPDivide, NONE, IMPL),
1469         MRS_FIELD_VALUE_END,
1470 };
1471
1472 static struct mrs_field_value mvfr0_fptrap[] = {
1473         MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPTrap, NONE, IMPL),
1474         MRS_FIELD_VALUE_END,
1475 };
1476
1477 static struct mrs_field_value mvfr0_fpdp[] = {
1478         MRS_FIELD_VALUE(MVFR0_FPDP_NONE, ""),
1479         MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v2, "DP VFPv2"),
1480         MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v3_v4, "DP VFPv3+v4"),
1481         MRS_FIELD_VALUE_END,
1482 };
1483
1484 static struct mrs_field_hwcap mvfr0_fpdp_caps[] = {
1485         MRS_HWCAP(&elf32_hwcap, HWCAP32_VFP, MVFR0_FPDP_VFP_v2),
1486         MRS_HWCAP(&elf32_hwcap, HWCAP32_VFPv3, MVFR0_FPDP_VFP_v3_v4),
1487 };
1488
1489 static struct mrs_field_value mvfr0_fpsp[] = {
1490         MRS_FIELD_VALUE(MVFR0_FPSP_NONE, ""),
1491         MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v2, "SP VFPv2"),
1492         MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v3_v4, "SP VFPv3+v4"),
1493         MRS_FIELD_VALUE_END,
1494 };
1495
1496 static struct mrs_field_value mvfr0_simdreg[] = {
1497         MRS_FIELD_VALUE(MVFR0_SIMDReg_NONE, ""),
1498         MRS_FIELD_VALUE(MVFR0_SIMDReg_FP, "FP 16x64"),
1499         MRS_FIELD_VALUE(MVFR0_SIMDReg_AdvSIMD, "AdvSIMD"),
1500         MRS_FIELD_VALUE_END,
1501 };
1502
1503 static struct mrs_field mvfr0_fields[] = {
1504         MRS_FIELD(MVFR0, FPRound, false, MRS_LOWER, mvfr0_fpround),
1505         MRS_FIELD(MVFR0, FPSqrt, false, MRS_LOWER, mvfr0_fpsqrt),
1506         MRS_FIELD(MVFR0, FPDivide, false, MRS_LOWER, mvfr0_fpdivide),
1507         MRS_FIELD(MVFR0, FPTrap, false, MRS_LOWER, mvfr0_fptrap),
1508         MRS_FIELD_HWCAP(MVFR0, FPDP, false, MRS_LOWER, mvfr0_fpdp,
1509             mvfr0_fpdp_caps),
1510         MRS_FIELD(MVFR0, FPSP, false, MRS_LOWER, mvfr0_fpsp),
1511         MRS_FIELD(MVFR0, SIMDReg, false, MRS_LOWER, mvfr0_simdreg),
1512         MRS_FIELD_END,
1513 };
1514
1515 /* MVFR1 */
1516 static struct mrs_field_value mvfr1_simdfmac[] = {
1517         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDFMAC, NONE, IMPL),
1518         MRS_FIELD_VALUE_END,
1519 };
1520
1521 static struct mrs_field_hwcap mvfr1_simdfmac_caps[] = {
1522         MRS_HWCAP(&elf32_hwcap, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL),
1523         MRS_HWCAP_END
1524 };
1525
1526 static struct mrs_field_value mvfr1_fphp[] = {
1527         MRS_FIELD_VALUE(MVFR1_FPHP_NONE, ""),
1528         MRS_FIELD_VALUE(MVFR1_FPHP_CONV_SP, "FPHP SP Conv"),
1529         MRS_FIELD_VALUE(MVFR1_FPHP_CONV_DP, "FPHP DP Conv"),
1530         MRS_FIELD_VALUE(MVFR1_FPHP_ARITH, "FPHP Arith"),
1531         MRS_FIELD_VALUE_END,
1532 };
1533
1534 static struct mrs_field_value mvfr1_simdhp[] = {
1535         MRS_FIELD_VALUE(MVFR1_SIMDHP_NONE, ""),
1536         MRS_FIELD_VALUE(MVFR1_SIMDHP_CONV_SP, "SIMDHP SP Conv"),
1537         MRS_FIELD_VALUE(MVFR1_SIMDHP_ARITH, "SIMDHP Arith"),
1538         MRS_FIELD_VALUE_END,
1539 };
1540
1541 static struct mrs_field_value mvfr1_simdsp[] = {
1542         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDSP, NONE, IMPL),
1543         MRS_FIELD_VALUE_END,
1544 };
1545
1546 static struct mrs_field_value mvfr1_simdint[] = {
1547         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDInt, NONE, IMPL),
1548         MRS_FIELD_VALUE_END,
1549 };
1550
1551 static struct mrs_field_value mvfr1_simdls[] = {
1552         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDLS, NONE, IMPL),
1553         MRS_FIELD_VALUE_END,
1554 };
1555
1556 static struct mrs_field_hwcap mvfr1_simdls_caps[] = {
1557         MRS_HWCAP(&elf32_hwcap, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL),
1558         MRS_HWCAP_END
1559 };
1560
1561 static struct mrs_field_value mvfr1_fpdnan[] = {
1562         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPDNaN, NONE, IMPL),
1563         MRS_FIELD_VALUE_END,
1564 };
1565
1566 static struct mrs_field_value mvfr1_fpftz[] = {
1567         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPFtZ, NONE, IMPL),
1568         MRS_FIELD_VALUE_END,
1569 };
1570
1571 static struct mrs_field mvfr1_fields[] = {
1572         MRS_FIELD_HWCAP(MVFR1, SIMDFMAC, false, MRS_LOWER, mvfr1_simdfmac,
1573             mvfr1_simdfmac_caps),
1574         MRS_FIELD(MVFR1, FPHP, false, MRS_LOWER, mvfr1_fphp),
1575         MRS_FIELD(MVFR1, SIMDHP, false, MRS_LOWER, mvfr1_simdhp),
1576         MRS_FIELD(MVFR1, SIMDSP, false, MRS_LOWER, mvfr1_simdsp),
1577         MRS_FIELD(MVFR1, SIMDInt, false, MRS_LOWER, mvfr1_simdint),
1578         MRS_FIELD_HWCAP(MVFR1, SIMDLS, false, MRS_LOWER, mvfr1_simdls,
1579             mvfr1_simdls_caps),
1580         MRS_FIELD(MVFR1, FPDNaN, false, MRS_LOWER, mvfr1_fpdnan),
1581         MRS_FIELD(MVFR1, FPFtZ, false, MRS_LOWER, mvfr1_fpftz),
1582         MRS_FIELD_END,
1583 };
1584 #endif /* COMPAT_FREEBSD32 */
1585
1586 struct mrs_user_reg {
1587         u_int           reg;
1588         u_int           CRm;
1589         u_int           Op2;
1590         size_t          offset;
1591         struct mrs_field *fields;
1592 };
1593
1594 #define USER_REG(name, field_name)                                      \
1595         {                                                               \
1596                 .reg = name,                                            \
1597                 .CRm = name##_CRm,                                      \
1598                 .Op2 = name##_op2,                                      \
1599                 .offset = __offsetof(struct cpu_desc, field_name),      \
1600                 .fields = field_name##_fields,                          \
1601         }
1602 static struct mrs_user_reg user_regs[] = {
1603         USER_REG(ID_AA64DFR0_EL1, id_aa64dfr0),
1604
1605         USER_REG(ID_AA64ISAR0_EL1, id_aa64isar0),
1606         USER_REG(ID_AA64ISAR1_EL1, id_aa64isar1),
1607
1608         USER_REG(ID_AA64MMFR0_EL1, id_aa64mmfr0),
1609         USER_REG(ID_AA64MMFR1_EL1, id_aa64mmfr1),
1610         USER_REG(ID_AA64MMFR2_EL1, id_aa64mmfr2),
1611
1612         USER_REG(ID_AA64PFR0_EL1, id_aa64pfr0),
1613         USER_REG(ID_AA64PFR1_EL1, id_aa64pfr1),
1614 #ifdef COMPAT_FREEBSD32
1615         USER_REG(ID_ISAR5_EL1, id_isar5),
1616
1617         USER_REG(MVFR0_EL1, mvfr0),
1618         USER_REG(MVFR1_EL1, mvfr1),
1619 #endif /* COMPAT_FREEBSD32 */
1620 };
1621
1622 #define CPU_DESC_FIELD(desc, idx)                                       \
1623     *(uint64_t *)((char *)&(desc) + user_regs[(idx)].offset)
1624
1625 static int
1626 user_mrs_handler(vm_offset_t va, uint32_t insn, struct trapframe *frame,
1627     uint32_t esr)
1628 {
1629         uint64_t value;
1630         int CRm, Op2, i, reg;
1631
1632         if ((insn & MRS_MASK) != MRS_VALUE)
1633                 return (0);
1634
1635         /*
1636          * We only emulate Op0 == 3, Op1 == 0, CRn == 0, CRm == {0, 4-7}.
1637          * These are in the EL1 CPU identification space.
1638          * CRm == 0 holds MIDR_EL1, MPIDR_EL1, and REVID_EL1.
1639          * CRm == {4-7} holds the ID_AA64 registers.
1640          *
1641          * For full details see the ARMv8 ARM (ARM DDI 0487C.a)
1642          * Table D9-2 System instruction encodings for non-Debug System
1643          * register accesses.
1644          */
1645         if (mrs_Op0(insn) != 3 || mrs_Op1(insn) != 0 || mrs_CRn(insn) != 0)
1646                 return (0);
1647
1648         CRm = mrs_CRm(insn);
1649         if (CRm > 7 || (CRm < 4 && CRm != 0))
1650                 return (0);
1651
1652         Op2 = mrs_Op2(insn);
1653         value = 0;
1654
1655         for (i = 0; i < nitems(user_regs); i++) {
1656                 if (user_regs[i].CRm == CRm && user_regs[i].Op2 == Op2) {
1657                         value = CPU_DESC_FIELD(user_cpu_desc, i);
1658                         break;
1659                 }
1660         }
1661
1662         if (CRm == 0) {
1663                 switch (Op2) {
1664                 case 0:
1665                         value = READ_SPECIALREG(midr_el1);
1666                         break;
1667                 case 5:
1668                         value = READ_SPECIALREG(mpidr_el1);
1669                         break;
1670                 case 6:
1671                         value = READ_SPECIALREG(revidr_el1);
1672                         break;
1673                 default:
1674                         return (0);
1675                 }
1676         }
1677
1678         /*
1679          * We will handle this instruction, move to the next so we
1680          * don't trap here again.
1681          */
1682         frame->tf_elr += INSN_SIZE;
1683
1684         reg = MRS_REGISTER(insn);
1685         /* If reg is 31 then write to xzr, i.e. do nothing */
1686         if (reg == 31)
1687                 return (1);
1688
1689         if (reg < nitems(frame->tf_x))
1690                 frame->tf_x[reg] = value;
1691         else if (reg == 30)
1692                 frame->tf_lr = value;
1693
1694         return (1);
1695 }
1696
1697 bool
1698 extract_user_id_field(u_int reg, u_int field_shift, uint8_t *val)
1699 {
1700         uint64_t value;
1701         int i;
1702
1703         for (i = 0; i < nitems(user_regs); i++) {
1704                 if (user_regs[i].reg == reg) {
1705                         value = CPU_DESC_FIELD(user_cpu_desc, i);
1706                         *val = value >> field_shift;
1707                         return (true);
1708                 }
1709         }
1710
1711         return (false);
1712 }
1713
1714 bool
1715 get_kernel_reg(u_int reg, uint64_t *val)
1716 {
1717         int i;
1718
1719         for (i = 0; i < nitems(user_regs); i++) {
1720                 if (user_regs[i].reg == reg) {
1721                         *val = CPU_DESC_FIELD(kern_cpu_desc, i);
1722                         return (true);
1723                 }
1724         }
1725
1726         return (false);
1727 }
1728
1729 /*
1730  * Compares two field values that may be signed or unsigned.
1731  * Returns:
1732  *  < 0 when a is less than b
1733  *  = 0 when a equals b
1734  *  > 0 when a is greater than b
1735  */
1736 static int
1737 mrs_field_cmp(uint64_t a, uint64_t b, u_int shift, int width, bool sign)
1738 {
1739         uint64_t mask;
1740
1741         KASSERT(width > 0 && width < 64, ("%s: Invalid width %d", __func__,
1742             width));
1743
1744         mask = (1ul << width) - 1;
1745         /* Move the field to the lower bits */
1746         a = (a >> shift) & mask;
1747         b = (b >> shift) & mask;
1748
1749         if (sign) {
1750                 /*
1751                  * The field is signed. Toggle the upper bit so the comparison
1752                  * works on unsigned values as this makes positive numbers,
1753                  * i.e. those with a 0 bit, larger than negative numbers,
1754                  * i.e. those with a 1 bit, in an unsigned comparison.
1755                  */
1756                 a ^= 1ul << (width - 1);
1757                 b ^= 1ul << (width - 1);
1758         }
1759
1760         return (a - b);
1761 }
1762
1763 static uint64_t
1764 update_lower_register(uint64_t val, uint64_t new_val, u_int shift,
1765     int width, bool sign)
1766 {
1767         uint64_t mask;
1768
1769         KASSERT(width > 0 && width < 64, ("%s: Invalid width %d", __func__,
1770             width));
1771
1772         /*
1773          * If the new value is less than the existing value update it.
1774          */
1775         if (mrs_field_cmp(new_val, val, shift, width, sign) < 0) {
1776                 mask = (1ul << width) - 1;
1777                 val &= ~(mask << shift);
1778                 val |= new_val & (mask << shift);
1779         }
1780
1781         return (val);
1782 }
1783
1784 void
1785 update_special_regs(u_int cpu)
1786 {
1787         struct mrs_field *fields;
1788         uint64_t user_reg, kern_reg, value;
1789         int i, j;
1790
1791         if (cpu == 0) {
1792                 /* Create a user visible cpu description with safe values */
1793                 memset(&user_cpu_desc, 0, sizeof(user_cpu_desc));
1794                 /* Safe values for these registers */
1795                 user_cpu_desc.id_aa64pfr0 = ID_AA64PFR0_AdvSIMD_NONE |
1796                     ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 |
1797                     ID_AA64PFR0_EL0_64;
1798                 user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8;
1799         }
1800
1801         for (i = 0; i < nitems(user_regs); i++) {
1802                 value = CPU_DESC_FIELD(cpu_desc[cpu], i);
1803                 if (cpu == 0) {
1804                         kern_reg = value;
1805                         user_reg = value;
1806                 } else {
1807                         kern_reg = CPU_DESC_FIELD(kern_cpu_desc, i);
1808                         user_reg = CPU_DESC_FIELD(user_cpu_desc, i);
1809                 }
1810
1811                 fields = user_regs[i].fields;
1812                 for (j = 0; fields[j].type != 0; j++) {
1813                         switch (fields[j].type & MRS_TYPE_MASK) {
1814                         case MRS_EXACT:
1815                                 user_reg &= ~(0xful << fields[j].shift);
1816                                 user_reg |=
1817                                     (uint64_t)MRS_EXACT_FIELD(fields[j].type) <<
1818                                     fields[j].shift;
1819                                 break;
1820                         case MRS_LOWER:
1821                                 user_reg = update_lower_register(user_reg,
1822                                     value, fields[j].shift, 4, fields[j].sign);
1823                                 break;
1824                         default:
1825                                 panic("Invalid field type: %d", fields[j].type);
1826                         }
1827                         kern_reg = update_lower_register(kern_reg, value,
1828                             fields[j].shift, 4, fields[j].sign);
1829                 }
1830
1831                 CPU_DESC_FIELD(kern_cpu_desc, i) = kern_reg;
1832                 CPU_DESC_FIELD(user_cpu_desc, i) = user_reg;
1833         }
1834 }
1835
1836 /* HWCAP */
1837 bool __read_frequently lse_supported = false;
1838
1839 bool __read_frequently icache_aliasing = false;
1840 bool __read_frequently icache_vmid = false;
1841
1842 int64_t dcache_line_size;       /* The minimum D cache line size */
1843 int64_t icache_line_size;       /* The minimum I cache line size */
1844 int64_t idcache_line_size;      /* The minimum cache line size */
1845
1846 /*
1847  * Find the values to export to userspace as AT_HWCAP and AT_HWCAP2.
1848  */
1849 static void
1850 parse_cpu_features(void)
1851 {
1852         struct mrs_field_hwcap *hwcaps;
1853         struct mrs_field *fields;
1854         uint64_t min, reg;
1855         int i, j, k;
1856
1857         for (i = 0; i < nitems(user_regs); i++) {
1858                 reg = CPU_DESC_FIELD(user_cpu_desc, i);
1859                 fields = user_regs[i].fields;
1860                 for (j = 0; fields[j].type != 0; j++) {
1861                         hwcaps = fields[j].hwcaps;
1862                         if (hwcaps == NULL)
1863                                 continue;
1864
1865                         for (k = 0; hwcaps[k].hwcap != NULL; k++) {
1866                                 min = hwcaps[k].min;
1867
1868                                 /*
1869                                  * If the field is greater than the minimum
1870                                  * value we can set the hwcap;
1871                                  */
1872                                 if (mrs_field_cmp(reg, min, fields[j].shift,
1873                                     4, fields[j].sign) >= 0) {
1874                                         *hwcaps[k].hwcap |= hwcaps[k].hwcap_val;
1875                                 }
1876                         }
1877                 }
1878         }
1879 }
1880
1881 static void
1882 identify_cpu_sysinit(void *dummy __unused)
1883 {
1884         int cpu;
1885         bool dic, idc;
1886
1887         dic = (allow_dic != 0);
1888         idc = (allow_idc != 0);
1889
1890         CPU_FOREACH(cpu) {
1891                 check_cpu_regs(cpu);
1892                 if (cpu != 0)
1893                         update_special_regs(cpu);
1894
1895                 if (CTR_DIC_VAL(cpu_desc[cpu].ctr) == 0)
1896                         dic = false;
1897                 if (CTR_IDC_VAL(cpu_desc[cpu].ctr) == 0)
1898                         idc = false;
1899         }
1900
1901         /* Find the values to export to userspace as AT_HWCAP and AT_HWCAP2 */
1902         parse_cpu_features();
1903
1904 #ifdef COMPAT_FREEBSD32
1905         /* Set the default caps and any that need to check multiple fields */
1906         elf32_hwcap |= parse_cpu_features_hwcap32();
1907 #endif
1908
1909         if (dic && idc) {
1910                 arm64_icache_sync_range = &arm64_dic_idc_icache_sync_range;
1911                 if (bootverbose)
1912                         printf("Enabling DIC & IDC ICache sync\n");
1913         } else if (idc) {
1914                 arm64_icache_sync_range = &arm64_idc_aliasing_icache_sync_range;
1915                 if (bootverbose)
1916                         printf("Enabling IDC ICache sync\n");
1917         }
1918
1919         if ((elf_hwcap & HWCAP_ATOMICS) != 0) {
1920                 lse_supported = true;
1921                 if (bootverbose)
1922                         printf("Enabling LSE atomics in the kernel\n");
1923         }
1924 #ifdef LSE_ATOMICS
1925         if (!lse_supported)
1926                 panic("CPU does not support LSE atomic instructions");
1927 #endif
1928
1929         install_undef_handler(true, user_mrs_handler);
1930 }
1931 SYSINIT(identify_cpu, SI_SUB_CPU, SI_ORDER_MIDDLE, identify_cpu_sysinit, NULL);
1932
1933 static void
1934 cpu_features_sysinit(void *dummy __unused)
1935 {
1936         struct sbuf sb;
1937         u_int cpu;
1938
1939         CPU_FOREACH(cpu)
1940                 print_cpu_features(cpu);
1941
1942         /* Fill in cpu_model for the hw.model sysctl */
1943         sbuf_new(&sb, cpu_model, sizeof(cpu_model), SBUF_FIXEDLEN);
1944         print_cpu_midr(&sb, 0);
1945
1946         sbuf_finish(&sb);
1947         sbuf_delete(&sb);
1948 }
1949 /* Log features before APs are released and start printing to the dmesg. */
1950 SYSINIT(cpu_features, SI_SUB_SMP - 1, SI_ORDER_ANY, cpu_features_sysinit, NULL);
1951
1952 #ifdef COMPAT_FREEBSD32
1953 static u_long
1954 parse_cpu_features_hwcap32(void)
1955 {
1956         u_long hwcap = HWCAP32_DEFAULT;
1957
1958         if ((MVFR1_SIMDLS_VAL(user_cpu_desc.mvfr1) >=
1959              MVFR1_SIMDLS_IMPL) &&
1960             (MVFR1_SIMDInt_VAL(user_cpu_desc.mvfr1) >=
1961              MVFR1_SIMDInt_IMPL) &&
1962             (MVFR1_SIMDSP_VAL(user_cpu_desc.mvfr1) >=
1963              MVFR1_SIMDSP_IMPL))
1964                 hwcap |= HWCAP32_NEON;
1965
1966         return (hwcap);
1967 }
1968 #endif /* COMPAT_FREEBSD32 */
1969
1970 static void
1971 print_ctr_fields(struct sbuf *sb, uint64_t reg, void *arg)
1972 {
1973
1974         sbuf_printf(sb, "%u byte D-cacheline,", CTR_DLINE_SIZE(reg));
1975         sbuf_printf(sb, "%u byte I-cacheline,", CTR_ILINE_SIZE(reg));
1976         reg &= ~(CTR_DLINE_MASK | CTR_ILINE_MASK);
1977
1978         switch(CTR_L1IP_VAL(reg)) {
1979         case CTR_L1IP_VPIPT:
1980                 sbuf_printf(sb, "VPIPT");
1981                 break;
1982         case CTR_L1IP_AIVIVT:
1983                 sbuf_printf(sb, "AIVIVT");
1984                 break;
1985         case CTR_L1IP_VIPT:
1986                 sbuf_printf(sb, "VIPT");
1987                 break;
1988         case CTR_L1IP_PIPT:
1989                 sbuf_printf(sb, "PIPT");
1990                 break;
1991         }
1992         sbuf_printf(sb, " ICache,");
1993         reg &= ~CTR_L1IP_MASK;
1994
1995         sbuf_printf(sb, "%d byte ERG,", CTR_ERG_SIZE(reg));
1996         sbuf_printf(sb, "%d byte CWG", CTR_CWG_SIZE(reg));
1997         reg &= ~(CTR_ERG_MASK | CTR_CWG_MASK);
1998
1999         if (CTR_IDC_VAL(reg) != 0)
2000                 sbuf_printf(sb, ",IDC");
2001         if (CTR_DIC_VAL(reg) != 0)
2002                 sbuf_printf(sb, ",DIC");
2003         reg &= ~(CTR_IDC_MASK | CTR_DIC_MASK);
2004         reg &= ~CTR_RES1;
2005
2006         if (reg != 0)
2007                 sbuf_printf(sb, ",%lx", reg);
2008 }
2009
2010 static void
2011 print_register(struct sbuf *sb, const char *reg_name, uint64_t reg,
2012     void (*print_fields)(struct sbuf *, uint64_t, void *), void *arg)
2013 {
2014
2015         sbuf_printf(sb, "%29s = <", reg_name);
2016
2017         print_fields(sb, reg, arg);
2018
2019         sbuf_finish(sb);
2020         printf("%s>\n", sbuf_data(sb));
2021         sbuf_clear(sb);
2022 }
2023
2024 static void
2025 print_id_fields(struct sbuf *sb, uint64_t reg, void *arg)
2026 {
2027         struct mrs_field *fields = arg;
2028         struct mrs_field_value *fv;
2029         int field, i, j, printed;
2030
2031 #define SEP_STR ((printed++) == 0) ? "" : ","
2032         printed = 0;
2033         for (i = 0; fields[i].type != 0; i++) {
2034                 fv = fields[i].values;
2035
2036                 /* TODO: Handle with an unknown message */
2037                 if (fv == NULL)
2038                         continue;
2039
2040                 field = (reg & fields[i].mask) >> fields[i].shift;
2041                 for (j = 0; fv[j].desc != NULL; j++) {
2042                         if ((fv[j].value >> fields[i].shift) != field)
2043                                 continue;
2044
2045                         if (fv[j].desc[0] != '\0')
2046                                 sbuf_printf(sb, "%s%s", SEP_STR, fv[j].desc);
2047                         break;
2048                 }
2049                 if (fv[j].desc == NULL)
2050                         sbuf_printf(sb, "%sUnknown %s(%x)", SEP_STR,
2051                             fields[i].name, field);
2052
2053                 reg &= ~(0xful << fields[i].shift);
2054         }
2055
2056         if (reg != 0)
2057                 sbuf_printf(sb, "%s%#lx", SEP_STR, reg);
2058 #undef SEP_STR
2059 }
2060
2061 static void
2062 print_id_register(struct sbuf *sb, const char *reg_name, uint64_t reg,
2063     struct mrs_field *fields)
2064 {
2065
2066         print_register(sb, reg_name, reg, print_id_fields, fields);
2067 }
2068
2069 static void
2070 print_cpu_midr(struct sbuf *sb, u_int cpu)
2071 {
2072         const struct cpu_parts *cpu_partsp;
2073         const char *cpu_impl_name;
2074         const char *cpu_part_name;
2075         u_int midr;
2076         u_int impl_id;
2077         u_int part_id;
2078
2079         midr = pcpu_find(cpu)->pc_midr;
2080
2081         cpu_impl_name = NULL;
2082         cpu_partsp = NULL;
2083         impl_id = CPU_IMPL(midr);
2084         for (int i = 0; cpu_implementers[i].impl_name != NULL; i++) {
2085                 if (impl_id == cpu_implementers[i].impl_id) {
2086                         cpu_impl_name = cpu_implementers[i].impl_name;
2087                         cpu_partsp = cpu_implementers[i].cpu_parts;
2088                         break;
2089                 }
2090         }
2091         /* Unknown implementer, so unknown part */
2092         if (cpu_impl_name == NULL) {
2093                 sbuf_printf(sb, "Unknown Implementer (midr: %08x)", midr);
2094                 return;
2095         }
2096
2097         KASSERT(cpu_partsp != NULL, ("%s: No parts table for implementer %s",
2098             __func__, cpu_impl_name));
2099
2100         cpu_part_name = NULL;
2101         part_id = CPU_PART(midr);
2102         for (int i = 0; cpu_partsp[i].part_name != NULL; i++) {
2103                 if (part_id == cpu_partsp[i].part_id) {
2104                         cpu_part_name = cpu_partsp[i].part_name;
2105                         break;
2106                 }
2107         }
2108         /* Known Implementer, Unknown part */
2109         if (cpu_part_name == NULL) {
2110                 sbuf_printf(sb, "%s Unknown CPU r%dp%d (midr: %08x)",
2111                     cpu_impl_name, CPU_VAR(midr), CPU_REV(midr), midr);
2112                 return;
2113         }
2114
2115         sbuf_printf(sb, "%s %s r%dp%d", cpu_impl_name,
2116             cpu_part_name, CPU_VAR(midr), CPU_REV(midr));
2117 }
2118
2119 static void
2120 print_cpu_cache(u_int cpu, struct sbuf *sb, uint64_t ccs, bool icache,
2121     bool unified)
2122 {
2123         size_t cache_size;
2124         size_t line_size;
2125
2126         /* LineSize is Log2(S) - 4. */
2127         line_size = 1 << ((ccs & CCSIDR_LineSize_MASK) + 4);
2128         /*
2129          * Calculate cache size (sets * ways * line size).  There are different
2130          * formats depending on the FEAT_CCIDX bit in ID_AA64MMFR2 feature
2131          * register.
2132          */
2133         if ((cpu_desc[cpu].id_aa64mmfr2 & ID_AA64MMFR2_CCIDX_64))
2134                 cache_size = (CCSIDR_NSETS_64(ccs) + 1) *
2135                     (CCSIDR_ASSOC_64(ccs) + 1);
2136         else
2137                 cache_size = (CCSIDR_NSETS(ccs) + 1) * (CCSIDR_ASSOC(ccs) + 1);
2138
2139         cache_size *= line_size;
2140         sbuf_printf(sb, "%zuKB (%s)", cache_size / 1024,
2141             icache ? "instruction" : unified ? "unified" : "data");
2142 }
2143
2144 static void
2145 print_cpu_caches(struct sbuf *sb, u_int cpu)
2146 {
2147         /* Print out each cache combination */
2148         uint64_t clidr;
2149         int i = 1;
2150         clidr = cpu_desc[cpu].clidr;
2151
2152         for (i = 0; (clidr & CLIDR_CTYPE_MASK) != 0; i++, clidr >>= 3) {
2153                 int j = 0;
2154                 int ctype_m = (clidr & CLIDR_CTYPE_MASK);
2155
2156                 sbuf_printf(sb, " L%d cache: ", i + 1);
2157                 if ((clidr & CLIDR_CTYPE_IO)) {
2158                         print_cpu_cache(cpu, sb, cpu_desc[cpu].ccsidr[i][j++],
2159                             true, false);
2160                         /* If there's more, add to the line. */
2161                         if ((ctype_m & ~CLIDR_CTYPE_IO) != 0)
2162                                 sbuf_printf(sb, ", ");
2163                 }
2164                 if ((ctype_m & ~CLIDR_CTYPE_IO) != 0) {
2165                         print_cpu_cache(cpu, sb, cpu_desc[cpu].ccsidr[i][j],
2166                             false, (clidr & CLIDR_CTYPE_UNIFIED));
2167                 }
2168                 sbuf_printf(sb, "\n");
2169
2170         }
2171         sbuf_finish(sb);
2172         printf("%s", sbuf_data(sb));
2173 }
2174
2175 static void
2176 print_cpu_features(u_int cpu)
2177 {
2178         struct sbuf *sb;
2179
2180         sb = sbuf_new_auto();
2181         sbuf_printf(sb, "CPU%3u: ", cpu);
2182         print_cpu_midr(sb, cpu);
2183
2184         sbuf_cat(sb, " affinity:");
2185         switch(cpu_aff_levels) {
2186         default:
2187         case 4:
2188                 sbuf_printf(sb, " %2d", CPU_AFF3(cpu_desc[cpu].mpidr));
2189                 /* FALLTHROUGH */
2190         case 3:
2191                 sbuf_printf(sb, " %2d", CPU_AFF2(cpu_desc[cpu].mpidr));
2192                 /* FALLTHROUGH */
2193         case 2:
2194                 sbuf_printf(sb, " %2d", CPU_AFF1(cpu_desc[cpu].mpidr));
2195                 /* FALLTHROUGH */
2196         case 1:
2197         case 0: /* On UP this will be zero */
2198                 sbuf_printf(sb, " %2d", CPU_AFF0(cpu_desc[cpu].mpidr));
2199                 break;
2200         }
2201         sbuf_finish(sb);
2202         printf("%s\n", sbuf_data(sb));
2203         sbuf_clear(sb);
2204
2205         /*
2206          * There is a hardware errata where, if one CPU is performing a TLB
2207          * invalidation while another is performing a store-exclusive the
2208          * store-exclusive may return the wrong status. A workaround seems
2209          * to be to use an IPI to invalidate on each CPU, however given the
2210          * limited number of affected units (pass 1.1 is the evaluation
2211          * hardware revision), and the lack of information from Cavium
2212          * this has not been implemented.
2213          *
2214          * At the time of writing this the only information is from:
2215          * https://lkml.org/lkml/2016/8/4/722
2216          */
2217         /*
2218          * XXX: CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1 on its own also
2219          * triggers on pass 2.0+.
2220          */
2221         if (cpu == 0 && CPU_VAR(PCPU_GET(midr)) == 0 &&
2222             CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1)
2223                 printf("WARNING: ThunderX Pass 1.1 detected.\nThis has known "
2224                     "hardware bugs that may cause the incorrect operation of "
2225                     "atomic operations.\n");
2226
2227 #define SHOULD_PRINT_REG(_reg)                                          \
2228     (cpu == 0 || cpu_desc[cpu]._reg != cpu_desc[cpu - 1]._reg)
2229
2230         /* Cache Type Register */
2231         if (SHOULD_PRINT_REG(ctr)) {
2232                 print_register(sb, "Cache Type",
2233                     cpu_desc[cpu].ctr, print_ctr_fields, NULL);
2234         }
2235
2236         /* AArch64 Instruction Set Attribute Register 0 */
2237         if (SHOULD_PRINT_REG(id_aa64isar0))
2238                 print_id_register(sb, "Instruction Set Attributes 0",
2239                     cpu_desc[cpu].id_aa64isar0, id_aa64isar0_fields);
2240
2241         /* AArch64 Instruction Set Attribute Register 1 */
2242         if (SHOULD_PRINT_REG(id_aa64isar1))
2243                 print_id_register(sb, "Instruction Set Attributes 1",
2244                     cpu_desc[cpu].id_aa64isar1, id_aa64isar1_fields);
2245
2246         /* AArch64 Instruction Set Attribute Register 2 */
2247         if (SHOULD_PRINT_REG(id_aa64isar2))
2248                 print_id_register(sb, "Instruction Set Attributes 2",
2249                     cpu_desc[cpu].id_aa64isar2, id_aa64isar2_fields);
2250
2251         /* AArch64 Processor Feature Register 0 */
2252         if (SHOULD_PRINT_REG(id_aa64pfr0))
2253                 print_id_register(sb, "Processor Features 0",
2254                     cpu_desc[cpu].id_aa64pfr0, id_aa64pfr0_fields);
2255
2256         /* AArch64 Processor Feature Register 1 */
2257         if (SHOULD_PRINT_REG(id_aa64pfr1))
2258                 print_id_register(sb, "Processor Features 1",
2259                     cpu_desc[cpu].id_aa64pfr1, id_aa64pfr1_fields);
2260
2261         /* AArch64 Memory Model Feature Register 0 */
2262         if (SHOULD_PRINT_REG(id_aa64mmfr0))
2263                 print_id_register(sb, "Memory Model Features 0",
2264                     cpu_desc[cpu].id_aa64mmfr0, id_aa64mmfr0_fields);
2265
2266         /* AArch64 Memory Model Feature Register 1 */
2267         if (SHOULD_PRINT_REG(id_aa64mmfr1))
2268                 print_id_register(sb, "Memory Model Features 1",
2269                     cpu_desc[cpu].id_aa64mmfr1, id_aa64mmfr1_fields);
2270
2271         /* AArch64 Memory Model Feature Register 2 */
2272         if (SHOULD_PRINT_REG(id_aa64mmfr2))
2273                 print_id_register(sb, "Memory Model Features 2",
2274                     cpu_desc[cpu].id_aa64mmfr2, id_aa64mmfr2_fields);
2275
2276         /* AArch64 Debug Feature Register 0 */
2277         if (SHOULD_PRINT_REG(id_aa64dfr0))
2278                 print_id_register(sb, "Debug Features 0",
2279                     cpu_desc[cpu].id_aa64dfr0, id_aa64dfr0_fields);
2280
2281         /* AArch64 Memory Model Feature Register 1 */
2282         if (SHOULD_PRINT_REG(id_aa64dfr1))
2283                 print_id_register(sb, "Debug Features 1",
2284                     cpu_desc[cpu].id_aa64dfr1, id_aa64dfr1_fields);
2285
2286         /* AArch64 Auxiliary Feature Register 0 */
2287         if (SHOULD_PRINT_REG(id_aa64afr0))
2288                 print_id_register(sb, "Auxiliary Features 0",
2289                     cpu_desc[cpu].id_aa64afr0, id_aa64afr0_fields);
2290
2291         /* AArch64 Auxiliary Feature Register 1 */
2292         if (SHOULD_PRINT_REG(id_aa64afr1))
2293                 print_id_register(sb, "Auxiliary Features 1",
2294                     cpu_desc[cpu].id_aa64afr1, id_aa64afr1_fields);
2295
2296         /* AArch64 SVE Feature Register 0 */
2297         if (cpu_desc[cpu].have_sve) {
2298                 if (SHOULD_PRINT_REG(id_aa64zfr0) ||
2299                     !cpu_desc[cpu - 1].have_sve) {
2300                         print_id_register(sb, "SVE Features 0",
2301                             cpu_desc[cpu].id_aa64zfr0, id_aa64zfr0_fields);
2302                 }
2303         }
2304
2305 #ifdef COMPAT_FREEBSD32
2306         /* AArch32 Instruction Set Attribute Register 5 */
2307         if (SHOULD_PRINT_REG(id_isar5))
2308                 print_id_register(sb, "AArch32 Instruction Set Attributes 5",
2309                      cpu_desc[cpu].id_isar5, id_isar5_fields);
2310
2311         /* AArch32 Media and VFP Feature Register 0 */
2312         if (SHOULD_PRINT_REG(mvfr0))
2313                 print_id_register(sb, "AArch32 Media and VFP Features 0",
2314                      cpu_desc[cpu].mvfr0, mvfr0_fields);
2315
2316         /* AArch32 Media and VFP Feature Register 1 */
2317         if (SHOULD_PRINT_REG(mvfr1))
2318                 print_id_register(sb, "AArch32 Media and VFP Features 1",
2319                      cpu_desc[cpu].mvfr1, mvfr1_fields);
2320 #endif
2321         if (bootverbose)
2322                 print_cpu_caches(sb, cpu);
2323
2324         sbuf_delete(sb);
2325         sb = NULL;
2326 #undef SHOULD_PRINT_REG
2327 #undef SEP_STR
2328 }
2329
2330 void
2331 identify_cache(uint64_t ctr)
2332 {
2333
2334         /* Identify the L1 cache type */
2335         switch (CTR_L1IP_VAL(ctr)) {
2336         case CTR_L1IP_PIPT:
2337                 break;
2338         case CTR_L1IP_VPIPT:
2339                 icache_vmid = true;
2340                 break;
2341         default:
2342         case CTR_L1IP_VIPT:
2343                 icache_aliasing = true;
2344                 break;
2345         }
2346
2347         if (dcache_line_size == 0) {
2348                 KASSERT(icache_line_size == 0, ("%s: i-cacheline size set: %ld",
2349                     __func__, icache_line_size));
2350
2351                 /* Get the D cache line size */
2352                 dcache_line_size = CTR_DLINE_SIZE(ctr);
2353                 /* And the same for the I cache */
2354                 icache_line_size = CTR_ILINE_SIZE(ctr);
2355
2356                 idcache_line_size = MIN(dcache_line_size, icache_line_size);
2357         }
2358
2359         if (dcache_line_size != CTR_DLINE_SIZE(ctr)) {
2360                 printf("WARNING: D-cacheline size mismatch %ld != %d\n",
2361                     dcache_line_size, CTR_DLINE_SIZE(ctr));
2362         }
2363
2364         if (icache_line_size != CTR_ILINE_SIZE(ctr)) {
2365                 printf("WARNING: I-cacheline size mismatch %ld != %d\n",
2366                     icache_line_size, CTR_ILINE_SIZE(ctr));
2367         }
2368 }
2369
2370 void
2371 identify_cpu(u_int cpu)
2372 {
2373         uint64_t clidr;
2374
2375         /* Save affinity for current CPU */
2376         cpu_desc[cpu].mpidr = get_mpidr();
2377         CPU_AFFINITY(cpu) = cpu_desc[cpu].mpidr & CPU_AFF_MASK;
2378
2379         cpu_desc[cpu].ctr = READ_SPECIALREG(ctr_el0);
2380         cpu_desc[cpu].id_aa64dfr0 = READ_SPECIALREG(id_aa64dfr0_el1);
2381         cpu_desc[cpu].id_aa64dfr1 = READ_SPECIALREG(id_aa64dfr1_el1);
2382         cpu_desc[cpu].id_aa64isar0 = READ_SPECIALREG(id_aa64isar0_el1);
2383         cpu_desc[cpu].id_aa64isar1 = READ_SPECIALREG(id_aa64isar1_el1);
2384         cpu_desc[cpu].id_aa64isar2 = READ_SPECIALREG(id_aa64isar2_el1);
2385         cpu_desc[cpu].id_aa64mmfr0 = READ_SPECIALREG(id_aa64mmfr0_el1);
2386         cpu_desc[cpu].id_aa64mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1);
2387         cpu_desc[cpu].id_aa64mmfr2 = READ_SPECIALREG(id_aa64mmfr2_el1);
2388         cpu_desc[cpu].id_aa64pfr0 = READ_SPECIALREG(id_aa64pfr0_el1);
2389         cpu_desc[cpu].id_aa64pfr1 = READ_SPECIALREG(id_aa64pfr1_el1);
2390
2391         /*
2392          * ID_AA64ZFR0_EL1 is only valid when at least one of:
2393          *  - ID_AA64PFR0_EL1.SVE is non-zero
2394          *  - ID_AA64PFR1_EL1.SME is non-zero
2395          * In other cases it is zero, but still safe to read
2396          */
2397         cpu_desc[cpu].have_sve =
2398             (ID_AA64PFR0_SVE_VAL(cpu_desc[cpu].id_aa64pfr0) != 0);
2399         cpu_desc[cpu].id_aa64zfr0 = READ_SPECIALREG(ID_AA64ZFR0_EL1_REG);
2400
2401         cpu_desc[cpu].clidr = READ_SPECIALREG(clidr_el1);
2402
2403         clidr = cpu_desc[cpu].clidr;
2404
2405         for (int i = 0; (clidr & CLIDR_CTYPE_MASK) != 0; i++, clidr >>= 3) {
2406                 int j = 0;
2407                 if ((clidr & CLIDR_CTYPE_IO)) {
2408                         WRITE_SPECIALREG(csselr_el1,
2409                             CSSELR_Level(i) | CSSELR_InD);
2410                         cpu_desc[cpu].ccsidr[i][j++] =
2411                             READ_SPECIALREG(ccsidr_el1);
2412                 }
2413                 if ((clidr & ~CLIDR_CTYPE_IO) == 0)
2414                         continue;
2415                 WRITE_SPECIALREG(csselr_el1, CSSELR_Level(i));
2416                 cpu_desc[cpu].ccsidr[i][j] = READ_SPECIALREG(ccsidr_el1);
2417         }
2418
2419 #ifdef COMPAT_FREEBSD32
2420         /* Only read aarch32 SRs if EL0-32 is available */
2421         if (ID_AA64PFR0_EL0_VAL(cpu_desc[cpu].id_aa64pfr0) ==
2422             ID_AA64PFR0_EL0_64_32) {
2423                 cpu_desc[cpu].id_isar5 = READ_SPECIALREG(id_isar5_el1);
2424                 cpu_desc[cpu].mvfr0 = READ_SPECIALREG(mvfr0_el1);
2425                 cpu_desc[cpu].mvfr1 = READ_SPECIALREG(mvfr1_el1);
2426         }
2427 #endif
2428 }
2429
2430 static void
2431 check_cpu_regs(u_int cpu)
2432 {
2433
2434         switch (cpu_aff_levels) {
2435         case 0:
2436                 if (CPU_AFF0(cpu_desc[cpu].mpidr) !=
2437                     CPU_AFF0(cpu_desc[0].mpidr))
2438                         cpu_aff_levels = 1;
2439                 /* FALLTHROUGH */
2440         case 1:
2441                 if (CPU_AFF1(cpu_desc[cpu].mpidr) !=
2442                     CPU_AFF1(cpu_desc[0].mpidr))
2443                         cpu_aff_levels = 2;
2444                 /* FALLTHROUGH */
2445         case 2:
2446                 if (CPU_AFF2(cpu_desc[cpu].mpidr) !=
2447                     CPU_AFF2(cpu_desc[0].mpidr))
2448                         cpu_aff_levels = 3;
2449                 /* FALLTHROUGH */
2450         case 3:
2451                 if (CPU_AFF3(cpu_desc[cpu].mpidr) !=
2452                     CPU_AFF3(cpu_desc[0].mpidr))
2453                         cpu_aff_levels = 4;
2454                 break;
2455         }
2456
2457         if (cpu_desc[cpu].ctr != cpu_desc[0].ctr) {
2458                 /*
2459                  * If the cache type register is different we may
2460                  * have a different l1 cache type.
2461                  */
2462                 identify_cache(cpu_desc[cpu].ctr);
2463         }
2464 }