sys/arm64/arm64/identcpu.c

   1 /*-
   2  * Copyright (c) 2014 Andrew Turner
   3  * Copyright (c) 2014 The FreeBSD Foundation
   4  * All rights reserved.
   5  *
   6  * Portions of this software were developed by Semihalf
   7  * under sponsorship of the FreeBSD Foundation.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  *
  30  */
  31
  32 #include <sys/cdefs.h>
  33 __FBSDID("$FreeBSD$");
  34
  35 #include <sys/param.h>
  36 #include <sys/kernel.h>
  37 #include <sys/pcpu.h>
  38 #include <sys/sbuf.h>
  39 #include <sys/smp.h>
  40 #include <sys/sysctl.h>
  41 #include <sys/systm.h>
  42
  43 #include <machine/atomic.h>
  44 #include <machine/cpu.h>
  45 #include <machine/cpufunc.h>
  46 #include <machine/elf.h>
  47 #include <machine/md_var.h>
  48 #include <machine/undefined.h>
  49
  50 static void print_cpu_midr(struct sbuf *sb, u_int cpu);
  51 static void print_cpu_features(u_int cpu);
  52 static void print_cpu_caches(struct sbuf *sb, u_int);
  53 #ifdef COMPAT_FREEBSD32
  54 static u_long parse_cpu_features_hwcap32(void);
  55 #endif
  56
  57 char machine[] = "arm64";
  58
  59 #ifdef SCTL_MASK32
  60 extern int adaptive_machine_arch;
  61 #endif
  62
  63 static SYSCTL_NODE(_machdep, OID_AUTO, cache, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
  64     "Cache management tuning");
  65
  66 static int allow_dic = 1;
  67 SYSCTL_INT(_machdep_cache, OID_AUTO, allow_dic, CTLFLAG_RDTUN, &allow_dic, 0,
  68     "Allow optimizations based on the DIC cache bit");
  69
  70 static int allow_idc = 1;
  71 SYSCTL_INT(_machdep_cache, OID_AUTO, allow_idc, CTLFLAG_RDTUN, &allow_idc, 0,
  72     "Allow optimizations based on the IDC cache bit");
  73
  74 static void check_cpu_regs(u_int cpu);
  75
  76 /*
  77  * The default implementation of I-cache sync assumes we have an
  78  * aliasing cache until we know otherwise.
  79  */
  80 void (*arm64_icache_sync_range)(vm_offset_t, vm_size_t) =
  81     &arm64_aliasing_icache_sync_range;
  82
  83 static int
  84 sysctl_hw_machine(SYSCTL_HANDLER_ARGS)
  85 {
  86 #ifdef SCTL_MASK32
  87         static const char machine32[] = "arm";
  88 #endif
  89         int error;
  90
  91 #ifdef SCTL_MASK32
  92         if ((req->flags & SCTL_MASK32) != 0 && adaptive_machine_arch)
  93                 error = SYSCTL_OUT(req, machine32, sizeof(machine32));
  94         else
  95 #endif
  96                 error = SYSCTL_OUT(req, machine, sizeof(machine));
  97         return (error);
  98 }
  99
 100 SYSCTL_PROC(_hw, HW_MACHINE, machine, CTLTYPE_STRING | CTLFLAG_RD |
 101         CTLFLAG_MPSAFE, NULL, 0, sysctl_hw_machine, "A", "Machine class");
 102
 103 static char cpu_model[64];
 104 SYSCTL_STRING(_hw, HW_MODEL, model, CTLFLAG_RD,
 105         cpu_model, sizeof(cpu_model), "Machine model");
 106
 107 #define MAX_CACHES      8       /* Maximum number of caches supported
 108                                    architecturally. */
 109 /*
 110  * Per-CPU affinity as provided in MPIDR_EL1
 111  * Indexed by CPU number in logical order selected by the system.
 112  * Relevant fields can be extracted using CPU_AFFn macros,
 113  * Aff3.Aff2.Aff1.Aff0 construct a unique CPU address in the system.
 114  *
 115  * Fields used by us:
 116  * Aff1 - Cluster number
 117  * Aff0 - CPU number in Aff1 cluster
 118  */
 119 uint64_t __cpu_affinity[MAXCPU];
 120 static u_int cpu_aff_levels;
 121
 122 struct cpu_desc {
 123         uint64_t        mpidr;
 124         uint64_t        id_aa64afr0;
 125         uint64_t        id_aa64afr1;
 126         uint64_t        id_aa64dfr0;
 127         uint64_t        id_aa64dfr1;
 128         uint64_t        id_aa64isar0;
 129         uint64_t        id_aa64isar1;
 130         uint64_t        id_aa64isar2;
 131         uint64_t        id_aa64mmfr0;
 132         uint64_t        id_aa64mmfr1;
 133         uint64_t        id_aa64mmfr2;
 134         uint64_t        id_aa64pfr0;
 135         uint64_t        id_aa64pfr1;
 136         uint64_t        id_aa64zfr0;
 137         uint64_t        ctr;
 138 #ifdef COMPAT_FREEBSD32
 139         uint64_t        id_isar5;
 140         uint64_t        mvfr0;
 141         uint64_t        mvfr1;
 142 #endif
 143         uint64_t        clidr;
 144         uint32_t        ccsidr[MAX_CACHES][2]; /* 2 possible types. */
 145         bool            have_sve;
 146 };
 147
 148 static struct cpu_desc cpu_desc[MAXCPU];
 149 static struct cpu_desc kern_cpu_desc;
 150 static struct cpu_desc user_cpu_desc;
 151
 152 struct cpu_parts {
 153         u_int           part_id;
 154         const char      *part_name;
 155 };
 156 #define CPU_PART_NONE   { 0, NULL }
 157
 158 struct cpu_implementers {
 159         u_int                   impl_id;
 160         const char              *impl_name;
 161         /*
 162          * Part number is implementation defined
 163          * so each vendor will have its own set of values and names.
 164          */
 165         const struct cpu_parts  *cpu_parts;
 166 };
 167 #define CPU_IMPLEMENTER_NONE    { 0, NULL, NULL }
 168
 169 /*
 170  * Per-implementer table of (PartNum, CPU Name) pairs.
 171  */
 172 /* ARM Ltd. */
 173 static const struct cpu_parts cpu_parts_arm[] = {
 174         { CPU_PART_AEM_V8, "AEMv8" },
 175         { CPU_PART_FOUNDATION, "Foundation-Model" },
 176         { CPU_PART_CORTEX_A34, "Cortex-A34" },
 177         { CPU_PART_CORTEX_A35, "Cortex-A35" },
 178         { CPU_PART_CORTEX_A53, "Cortex-A53" },
 179         { CPU_PART_CORTEX_A55, "Cortex-A55" },
 180         { CPU_PART_CORTEX_A57, "Cortex-A57" },
 181         { CPU_PART_CORTEX_A65, "Cortex-A65" },
 182         { CPU_PART_CORTEX_A65AE, "Cortex-A65AE" },
 183         { CPU_PART_CORTEX_A72, "Cortex-A72" },
 184         { CPU_PART_CORTEX_A73, "Cortex-A73" },
 185         { CPU_PART_CORTEX_A75, "Cortex-A75" },
 186         { CPU_PART_CORTEX_A76, "Cortex-A76" },
 187         { CPU_PART_CORTEX_A76AE, "Cortex-A76AE" },
 188         { CPU_PART_CORTEX_A77, "Cortex-A77" },
 189         { CPU_PART_CORTEX_A78, "Cortex-A78" },
 190         { CPU_PART_CORTEX_A78C, "Cortex-A78C" },
 191         { CPU_PART_CORTEX_A510, "Cortex-A510" },
 192         { CPU_PART_CORTEX_A710, "Cortex-A710" },
 193         { CPU_PART_CORTEX_A715, "Cortex-A715" },
 194         { CPU_PART_CORTEX_X1, "Cortex-X1" },
 195         { CPU_PART_CORTEX_X1C, "Cortex-X1C" },
 196         { CPU_PART_CORTEX_X2, "Cortex-X2" },
 197         { CPU_PART_CORTEX_X3, "Cortex-X3" },
 198         { CPU_PART_NEOVERSE_E1, "Neoverse-E1" },
 199         { CPU_PART_NEOVERSE_N1, "Neoverse-N1" },
 200         { CPU_PART_NEOVERSE_N2, "Neoverse-N2" },
 201         { CPU_PART_NEOVERSE_V1, "Neoverse-V1" },
 202         CPU_PART_NONE,
 203 };
 204
 205 /* Cavium */
 206 static const struct cpu_parts cpu_parts_cavium[] = {
 207         { CPU_PART_THUNDERX, "ThunderX" },
 208         { CPU_PART_THUNDERX2, "ThunderX2" },
 209         CPU_PART_NONE,
 210 };
 211
 212 /* APM / Ampere */
 213 static const struct cpu_parts cpu_parts_apm[] = {
 214         { CPU_PART_EMAG8180, "eMAG 8180" },
 215         CPU_PART_NONE,
 216 };
 217
 218 /* Unknown */
 219 static const struct cpu_parts cpu_parts_none[] = {
 220         CPU_PART_NONE,
 221 };
 222
 223 /*
 224  * Implementers table.
 225  */
 226 const struct cpu_implementers cpu_implementers[] = {
 227         { CPU_IMPL_AMPERE,      "Ampere",       cpu_parts_none },
 228         { CPU_IMPL_APPLE,       "Apple",        cpu_parts_none },
 229         { CPU_IMPL_APM,         "APM",          cpu_parts_apm },
 230         { CPU_IMPL_ARM,         "ARM",          cpu_parts_arm },
 231         { CPU_IMPL_BROADCOM,    "Broadcom",     cpu_parts_none },
 232         { CPU_IMPL_CAVIUM,      "Cavium",       cpu_parts_cavium },
 233         { CPU_IMPL_DEC,         "DEC",          cpu_parts_none },
 234         { CPU_IMPL_FREESCALE,   "Freescale",    cpu_parts_none },
 235         { CPU_IMPL_FUJITSU,     "Fujitsu",      cpu_parts_none },
 236         { CPU_IMPL_INFINEON,    "IFX",          cpu_parts_none },
 237         { CPU_IMPL_INTEL,       "Intel",        cpu_parts_none },
 238         { CPU_IMPL_MARVELL,     "Marvell",      cpu_parts_none },
 239         { CPU_IMPL_NVIDIA,      "NVIDIA",       cpu_parts_none },
 240         { CPU_IMPL_QUALCOMM,    "Qualcomm",     cpu_parts_none },
 241         CPU_IMPLEMENTER_NONE,
 242 };
 243
 244 #define MRS_TYPE_MASK           0xf
 245 #define MRS_INVALID             0
 246 #define MRS_EXACT               1
 247 #define MRS_EXACT_VAL(x)        (MRS_EXACT | ((x) << 4))
 248 #define MRS_EXACT_FIELD(x)      ((x) >> 4)
 249 #define MRS_LOWER               2
 250
 251 struct mrs_field_value {
 252         uint64_t        value;
 253         const char      *desc;
 254 };
 255
 256 #define MRS_FIELD_VALUE(_value, _desc)                                  \
 257         {                                                               \
 258                 .value = (_value),                                      \
 259                 .desc = (_desc),                                        \
 260         }
 261
 262 #define MRS_FIELD_VALUE_NONE_IMPL(_reg, _field, _none, _impl)           \
 263         MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _none, ""),         \
 264         MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _impl, #_field)
 265
 266 #define MRS_FIELD_VALUE_COUNT(_reg, _field, _desc)                      \
 267         MRS_FIELD_VALUE(0ul << _reg ## _ ## _field ## _SHIFT, "1 " _desc), \
 268         MRS_FIELD_VALUE(1ul << _reg ## _ ## _field ## _SHIFT, "2 " _desc "s"), \
 269         MRS_FIELD_VALUE(2ul << _reg ## _ ## _field ## _SHIFT, "3 " _desc "s"), \
 270         MRS_FIELD_VALUE(3ul << _reg ## _ ## _field ## _SHIFT, "4 " _desc "s"), \
 271         MRS_FIELD_VALUE(4ul << _reg ## _ ## _field ## _SHIFT, "5 " _desc "s"), \
 272         MRS_FIELD_VALUE(5ul << _reg ## _ ## _field ## _SHIFT, "6 " _desc "s"), \
 273         MRS_FIELD_VALUE(6ul << _reg ## _ ## _field ## _SHIFT, "7 " _desc "s"), \
 274         MRS_FIELD_VALUE(7ul << _reg ## _ ## _field ## _SHIFT, "8 " _desc "s"), \
 275         MRS_FIELD_VALUE(8ul << _reg ## _ ## _field ## _SHIFT, "9 " _desc "s"), \
 276         MRS_FIELD_VALUE(9ul << _reg ## _ ## _field ## _SHIFT, "10 "_desc "s"), \
 277         MRS_FIELD_VALUE(10ul<< _reg ## _ ## _field ## _SHIFT, "11 "_desc "s"), \
 278         MRS_FIELD_VALUE(11ul<< _reg ## _ ## _field ## _SHIFT, "12 "_desc "s"), \
 279         MRS_FIELD_VALUE(12ul<< _reg ## _ ## _field ## _SHIFT, "13 "_desc "s"), \
 280         MRS_FIELD_VALUE(13ul<< _reg ## _ ## _field ## _SHIFT, "14 "_desc "s"), \
 281         MRS_FIELD_VALUE(14ul<< _reg ## _ ## _field ## _SHIFT, "15 "_desc "s"), \
 282         MRS_FIELD_VALUE(15ul<< _reg ## _ ## _field ## _SHIFT, "16 "_desc "s")
 283
 284 #define MRS_FIELD_VALUE_END     { .desc = NULL }
 285
 286 struct mrs_field_hwcap {
 287         u_long          *hwcap;
 288         uint64_t        min;
 289         u_long          hwcap_val;
 290 };
 291
 292 #define MRS_HWCAP(_hwcap, _val, _min)                           \
 293 {                                                               \
 294         .hwcap = (_hwcap),                                      \
 295         .hwcap_val = (_val),                                    \
 296         .min = (_min),                                          \
 297 }
 298
 299 #define MRS_HWCAP_END           { .hwcap = NULL }
 300
 301 struct mrs_field {
 302         const char      *name;
 303         struct mrs_field_value *values;
 304         struct mrs_field_hwcap *hwcaps;
 305         uint64_t        mask;
 306         bool            sign;
 307         u_int           type;
 308         u_int           shift;
 309 };
 310
 311 #define MRS_FIELD_HWCAP(_register, _name, _sign, _type, _values, _hwcap) \
 312         {                                                               \
 313                 .name = #_name,                                         \
 314                 .sign = (_sign),                                        \
 315                 .type = (_type),                                        \
 316                 .shift = _register ## _ ## _name ## _SHIFT,             \
 317                 .mask = _register ## _ ## _name ## _MASK,               \
 318                 .values = (_values),                                    \
 319                 .hwcaps = (_hwcap),                                     \
 320         }
 321
 322 #define MRS_FIELD(_register, _name, _sign, _type, _values)              \
 323         MRS_FIELD_HWCAP(_register, _name, _sign, _type, _values, NULL)
 324
 325 #define MRS_FIELD_END   { .type = MRS_INVALID, }
 326
 327 /* ID_AA64AFR0_EL1 */
 328 static struct mrs_field id_aa64afr0_fields[] = {
 329         MRS_FIELD_END,
 330 };
 331
 332
 333 /* ID_AA64AFR1_EL1 */
 334 static struct mrs_field id_aa64afr1_fields[] = {
 335         MRS_FIELD_END,
 336 };
 337
 338
 339 /* ID_AA64DFR0_EL1 */
 340 static struct mrs_field_value id_aa64dfr0_tracefilt[] = {
 341         MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_NONE, ""),
 342         MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_8_4, "Trace v8.4"),
 343         MRS_FIELD_VALUE_END,
 344 };
 345
 346 static struct mrs_field_value id_aa64dfr0_doublelock[] = {
 347         MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_IMPL, "DoubleLock"),
 348         MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_NONE, ""),
 349         MRS_FIELD_VALUE_END,
 350 };
 351
 352 static struct mrs_field_value id_aa64dfr0_pmsver[] = {
 353         MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_NONE, ""),
 354         MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE, "SPE"),
 355         MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_8_3, "SPE v8.3"),
 356         MRS_FIELD_VALUE_END,
 357 };
 358
 359 static struct mrs_field_value id_aa64dfr0_ctx_cmps[] = {
 360         MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, CTX_CMPs, "CTX BKPT"),
 361         MRS_FIELD_VALUE_END,
 362 };
 363
 364 static struct mrs_field_value id_aa64dfr0_wrps[] = {
 365         MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, WRPs, "Watchpoint"),
 366         MRS_FIELD_VALUE_END,
 367 };
 368
 369 static struct mrs_field_value id_aa64dfr0_brps[] = {
 370         MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, BRPs, "Breakpoint"),
 371         MRS_FIELD_VALUE_END,
 372 };
 373
 374 static struct mrs_field_value id_aa64dfr0_pmuver[] = {
 375         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_NONE, ""),
 376         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3, "PMUv3"),
 377         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_1, "PMUv3 v8.1"),
 378         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_4, "PMUv3 v8.4"),
 379         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_5, "PMUv3 v8.5"),
 380         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_IMPL, "IMPL PMU"),
 381         MRS_FIELD_VALUE_END,
 382 };
 383
 384 static struct mrs_field_value id_aa64dfr0_tracever[] = {
 385         MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_NONE, ""),
 386         MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_IMPL, "Trace"),
 387         MRS_FIELD_VALUE_END,
 388 };
 389
 390 static struct mrs_field_value id_aa64dfr0_debugver[] = {
 391         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8, "Debugv8"),
 392         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_VHE, "Debugv8_VHE"),
 393         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_2, "Debugv8.2"),
 394         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_4, "Debugv8.4"),
 395         MRS_FIELD_VALUE_END,
 396 };
 397
 398 static struct mrs_field id_aa64dfr0_fields[] = {
 399         MRS_FIELD(ID_AA64DFR0, TraceFilt, false, MRS_EXACT,
 400             id_aa64dfr0_tracefilt),
 401         MRS_FIELD(ID_AA64DFR0, DoubleLock, false, MRS_EXACT,
 402             id_aa64dfr0_doublelock),
 403         MRS_FIELD(ID_AA64DFR0, PMSVer, false, MRS_EXACT, id_aa64dfr0_pmsver),
 404         MRS_FIELD(ID_AA64DFR0, CTX_CMPs, false, MRS_EXACT,
 405             id_aa64dfr0_ctx_cmps),
 406         MRS_FIELD(ID_AA64DFR0, WRPs, false, MRS_LOWER, id_aa64dfr0_wrps),
 407         MRS_FIELD(ID_AA64DFR0, BRPs, false, MRS_LOWER, id_aa64dfr0_brps),
 408         MRS_FIELD(ID_AA64DFR0, PMUVer, false, MRS_EXACT, id_aa64dfr0_pmuver),
 409         MRS_FIELD(ID_AA64DFR0, TraceVer, false, MRS_EXACT,
 410             id_aa64dfr0_tracever),
 411         MRS_FIELD(ID_AA64DFR0, DebugVer, false, MRS_EXACT_VAL(0x6),
 412             id_aa64dfr0_debugver),
 413         MRS_FIELD_END,
 414 };
 415
 416
 417 /* ID_AA64DFR1_EL1 */
 418 static struct mrs_field id_aa64dfr1_fields[] = {
 419         MRS_FIELD_END,
 420 };
 421
 422
 423 /* ID_AA64ISAR0_EL1 */
 424 static struct mrs_field_value id_aa64isar0_rndr[] = {
 425         MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_NONE, ""),
 426         MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_IMPL, "RNG"),
 427         MRS_FIELD_VALUE_END,
 428 };
 429
 430 static struct mrs_field_hwcap id_aa64isar0_rndr_caps[] = {
 431         MRS_HWCAP(&elf_hwcap2, HWCAP2_RNG, ID_AA64ISAR0_RNDR_IMPL),
 432         MRS_HWCAP_END
 433 };
 434
 435 static struct mrs_field_value id_aa64isar0_tlb[] = {
 436         MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_NONE, ""),
 437         MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOS, "TLBI-OS"),
 438         MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOSR, "TLBI-OSR"),
 439         MRS_FIELD_VALUE_END,
 440 };
 441
 442 static struct mrs_field_value id_aa64isar0_ts[] = {
 443         MRS_FIELD_VALUE(ID_AA64ISAR0_TS_NONE, ""),
 444         MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_4, "CondM-8.4"),
 445         MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_5, "CondM-8.5"),
 446         MRS_FIELD_VALUE_END,
 447 };
 448
 449 static struct mrs_field_hwcap id_aa64isar0_ts_caps[] = {
 450         MRS_HWCAP(&elf_hwcap, HWCAP_FLAGM, ID_AA64ISAR0_TS_CondM_8_4),
 451         MRS_HWCAP(&elf_hwcap2, HWCAP2_FLAGM2, ID_AA64ISAR0_TS_CondM_8_5),
 452         MRS_HWCAP_END
 453 };
 454
 455 static struct mrs_field_value id_aa64isar0_fhm[] = {
 456         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, FHM, NONE, IMPL),
 457         MRS_FIELD_VALUE_END,
 458 };
 459
 460 static struct mrs_field_hwcap id_aa64isar0_fhm_caps[] = {
 461         MRS_HWCAP(&elf_hwcap, HWCAP_ASIMDFHM, ID_AA64ISAR0_FHM_IMPL),
 462         MRS_HWCAP_END
 463 };
 464
 465 static struct mrs_field_value id_aa64isar0_dp[] = {
 466         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, DP, NONE, IMPL),
 467         MRS_FIELD_VALUE_END,
 468 };
 469
 470 static struct mrs_field_hwcap id_aa64isar0_dp_caps[] = {
 471         MRS_HWCAP(&elf_hwcap, HWCAP_ASIMDDP, ID_AA64ISAR0_DP_IMPL),
 472         MRS_HWCAP_END
 473 };
 474
 475 static struct mrs_field_value id_aa64isar0_sm4[] = {
 476         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM4, NONE, IMPL),
 477         MRS_FIELD_VALUE_END,
 478 };
 479
 480 static struct mrs_field_hwcap id_aa64isar0_sm4_caps[] = {
 481         MRS_HWCAP(&elf_hwcap, HWCAP_SM4, ID_AA64ISAR0_SM4_IMPL),
 482         MRS_HWCAP_END
 483 };
 484
 485 static struct mrs_field_value id_aa64isar0_sm3[] = {
 486         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM3, NONE, IMPL),
 487         MRS_FIELD_VALUE_END,
 488 };
 489
 490 static struct mrs_field_hwcap id_aa64isar0_sm3_caps[] = {
 491         MRS_HWCAP(&elf_hwcap, HWCAP_SM3, ID_AA64ISAR0_SM3_IMPL),
 492         MRS_HWCAP_END
 493 };
 494
 495 static struct mrs_field_value id_aa64isar0_sha3[] = {
 496         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA3, NONE, IMPL),
 497         MRS_FIELD_VALUE_END,
 498 };
 499
 500 static struct mrs_field_hwcap id_aa64isar0_sha3_caps[] = {
 501         MRS_HWCAP(&elf_hwcap, HWCAP_SHA3, ID_AA64ISAR0_SHA3_IMPL),
 502         MRS_HWCAP_END
 503 };
 504
 505 static struct mrs_field_value id_aa64isar0_rdm[] = {
 506         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, RDM, NONE, IMPL),
 507         MRS_FIELD_VALUE_END,
 508 };
 509
 510 static struct mrs_field_hwcap id_aa64isar0_rdm_caps[] = {
 511         MRS_HWCAP(&elf_hwcap, HWCAP_ASIMDRDM, ID_AA64ISAR0_RDM_IMPL),
 512         MRS_HWCAP_END
 513 };
 514
 515 static struct mrs_field_value id_aa64isar0_atomic[] = {
 516         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, Atomic, NONE, IMPL),
 517         MRS_FIELD_VALUE_END,
 518 };
 519
 520 static struct mrs_field_hwcap id_aa64isar0_atomic_caps[] = {
 521         MRS_HWCAP(&elf_hwcap, HWCAP_ATOMICS, ID_AA64ISAR0_Atomic_IMPL),
 522         MRS_HWCAP_END
 523 };
 524
 525 static struct mrs_field_value id_aa64isar0_crc32[] = {
 526         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, CRC32, NONE, BASE),
 527         MRS_FIELD_VALUE_END,
 528 };
 529
 530 static struct mrs_field_hwcap id_aa64isar0_crc32_caps[] = {
 531         MRS_HWCAP(&elf_hwcap, HWCAP_CRC32, ID_AA64ISAR0_CRC32_BASE),
 532         MRS_HWCAP_END
 533 };
 534
 535 static struct mrs_field_value id_aa64isar0_sha2[] = {
 536         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA2, NONE, BASE),
 537         MRS_FIELD_VALUE(ID_AA64ISAR0_SHA2_512, "SHA2+SHA512"),
 538         MRS_FIELD_VALUE_END,
 539 };
 540
 541 static struct mrs_field_hwcap id_aa64isar0_sha2_caps[] = {
 542         MRS_HWCAP(&elf_hwcap, HWCAP_SHA2, ID_AA64ISAR0_SHA2_BASE),
 543         MRS_HWCAP(&elf_hwcap, HWCAP_SHA512, ID_AA64ISAR0_SHA2_512),
 544         MRS_HWCAP_END
 545 };
 546
 547 static struct mrs_field_value id_aa64isar0_sha1[] = {
 548         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA1, NONE, BASE),
 549         MRS_FIELD_VALUE_END,
 550 };
 551
 552 static struct mrs_field_hwcap id_aa64isar0_sha1_caps[] = {
 553         MRS_HWCAP(&elf_hwcap, HWCAP_SHA1, ID_AA64ISAR0_SHA1_BASE),
 554         MRS_HWCAP_END
 555 };
 556
 557 static struct mrs_field_value id_aa64isar0_aes[] = {
 558         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, AES, NONE, BASE),
 559         MRS_FIELD_VALUE(ID_AA64ISAR0_AES_PMULL, "AES+PMULL"),
 560         MRS_FIELD_VALUE_END,
 561 };
 562
 563 static struct mrs_field_hwcap id_aa64isar0_aes_caps[] = {
 564         MRS_HWCAP(&elf_hwcap, HWCAP_AES, ID_AA64ISAR0_AES_BASE),
 565         MRS_HWCAP(&elf_hwcap, HWCAP_PMULL, ID_AA64ISAR0_AES_PMULL),
 566         MRS_HWCAP_END
 567 };
 568
 569 static struct mrs_field id_aa64isar0_fields[] = {
 570         MRS_FIELD_HWCAP(ID_AA64ISAR0, RNDR, false, MRS_LOWER,
 571             id_aa64isar0_rndr, id_aa64isar0_rndr_caps),
 572         MRS_FIELD(ID_AA64ISAR0, TLB, false, MRS_EXACT, id_aa64isar0_tlb),
 573         MRS_FIELD_HWCAP(ID_AA64ISAR0, TS, false, MRS_LOWER, id_aa64isar0_ts,
 574             id_aa64isar0_ts_caps),
 575         MRS_FIELD_HWCAP(ID_AA64ISAR0, FHM, false, MRS_LOWER, id_aa64isar0_fhm,
 576             id_aa64isar0_fhm_caps),
 577         MRS_FIELD_HWCAP(ID_AA64ISAR0, DP, false, MRS_LOWER, id_aa64isar0_dp,
 578             id_aa64isar0_dp_caps),
 579         MRS_FIELD_HWCAP(ID_AA64ISAR0, SM4, false, MRS_LOWER, id_aa64isar0_sm4,
 580             id_aa64isar0_sm4_caps),
 581         MRS_FIELD_HWCAP(ID_AA64ISAR0, SM3, false, MRS_LOWER, id_aa64isar0_sm3,
 582             id_aa64isar0_sm3_caps),
 583         MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA3, false, MRS_LOWER, id_aa64isar0_sha3,
 584             id_aa64isar0_sha3_caps),
 585         MRS_FIELD_HWCAP(ID_AA64ISAR0, RDM, false, MRS_LOWER, id_aa64isar0_rdm,
 586             id_aa64isar0_rdm_caps),
 587         MRS_FIELD_HWCAP(ID_AA64ISAR0, Atomic, false, MRS_LOWER,
 588             id_aa64isar0_atomic, id_aa64isar0_atomic_caps),
 589         MRS_FIELD_HWCAP(ID_AA64ISAR0, CRC32, false, MRS_LOWER,
 590             id_aa64isar0_crc32, id_aa64isar0_crc32_caps),
 591         MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA2, false, MRS_LOWER, id_aa64isar0_sha2,
 592             id_aa64isar0_sha2_caps),
 593         MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA1, false, MRS_LOWER,
 594             id_aa64isar0_sha1, id_aa64isar0_sha1_caps),
 595         MRS_FIELD_HWCAP(ID_AA64ISAR0, AES, false, MRS_LOWER, id_aa64isar0_aes,
 596             id_aa64isar0_aes_caps),
 597         MRS_FIELD_END,
 598 };
 599
 600
 601 /* ID_AA64ISAR1_EL1 */
 602 static struct mrs_field_value id_aa64isar1_i8mm[] = {
 603         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, I8MM, NONE, IMPL),
 604         MRS_FIELD_VALUE_END,
 605 };
 606
 607 static struct mrs_field_hwcap id_aa64isar1_i8mm_caps[] = {
 608         MRS_HWCAP(&elf_hwcap2, HWCAP2_I8MM, ID_AA64ISAR1_I8MM_IMPL),
 609         MRS_HWCAP_END
 610 };
 611
 612 static struct mrs_field_value id_aa64isar1_dgh[] = {
 613         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, DGH, NONE, IMPL),
 614         MRS_FIELD_VALUE_END,
 615 };
 616
 617 static struct mrs_field_hwcap id_aa64isar1_dgh_caps[] = {
 618         MRS_HWCAP(&elf_hwcap2, HWCAP2_DGH, ID_AA64ISAR1_DGH_IMPL),
 619         MRS_HWCAP_END
 620 };
 621
 622 static struct mrs_field_value id_aa64isar1_bf16[] = {
 623         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, BF16, NONE, IMPL),
 624         MRS_FIELD_VALUE_END,
 625 };
 626
 627 static struct mrs_field_hwcap id_aa64isar1_bf16_caps[] = {
 628         MRS_HWCAP(&elf_hwcap2, HWCAP2_BF16, ID_AA64ISAR1_BF16_IMPL),
 629         MRS_HWCAP_END
 630 };
 631
 632 static struct mrs_field_value id_aa64isar1_specres[] = {
 633         MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_NONE, ""),
 634         MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_IMPL, "PredInv"),
 635         MRS_FIELD_VALUE_END,
 636 };
 637
 638 static struct mrs_field_value id_aa64isar1_sb[] = {
 639         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, SB, NONE, IMPL),
 640         MRS_FIELD_VALUE_END,
 641 };
 642
 643 static struct mrs_field_hwcap id_aa64isar1_sb_caps[] = {
 644         MRS_HWCAP(&elf_hwcap, HWCAP_SB, ID_AA64ISAR1_SB_IMPL),
 645         MRS_HWCAP_END
 646 };
 647
 648 static struct mrs_field_value id_aa64isar1_frintts[] = {
 649         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FRINTTS, NONE, IMPL),
 650         MRS_FIELD_VALUE_END,
 651 };
 652
 653 static struct mrs_field_hwcap id_aa64isar1_frintts_caps[] = {
 654         MRS_HWCAP(&elf_hwcap2, HWCAP2_FRINT, ID_AA64ISAR1_FRINTTS_IMPL),
 655         MRS_HWCAP_END
 656 };
 657
 658 static struct mrs_field_value id_aa64isar1_gpi[] = {
 659         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPI, NONE, IMPL),
 660         MRS_FIELD_VALUE_END,
 661 };
 662
 663 static struct mrs_field_hwcap id_aa64isar1_gpi_caps[] = {
 664         MRS_HWCAP(&elf_hwcap, HWCAP_PACG, ID_AA64ISAR1_GPI_IMPL),
 665         MRS_HWCAP_END
 666 };
 667
 668 static struct mrs_field_value id_aa64isar1_gpa[] = {
 669         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPA, NONE, IMPL),
 670         MRS_FIELD_VALUE_END,
 671 };
 672
 673 static struct mrs_field_hwcap id_aa64isar1_gpa_caps[] = {
 674         MRS_HWCAP(&elf_hwcap, HWCAP_PACG, ID_AA64ISAR1_GPA_IMPL),
 675         MRS_HWCAP_END
 676 };
 677
 678 static struct mrs_field_value id_aa64isar1_lrcpc[] = {
 679         MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_NONE, ""),
 680         MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_3, "RCPC-8.3"),
 681         MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_4, "RCPC-8.4"),
 682         MRS_FIELD_VALUE_END,
 683 };
 684
 685 static struct mrs_field_hwcap id_aa64isar1_lrcpc_caps[] = {
 686         MRS_HWCAP(&elf_hwcap, HWCAP_LRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_3),
 687         MRS_HWCAP(&elf_hwcap, HWCAP_ILRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_4),
 688         MRS_HWCAP_END
 689 };
 690
 691 static struct mrs_field_value id_aa64isar1_fcma[] = {
 692         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FCMA, NONE, IMPL),
 693         MRS_FIELD_VALUE_END,
 694 };
 695
 696 static struct mrs_field_hwcap id_aa64isar1_fcma_caps[] = {
 697         MRS_HWCAP(&elf_hwcap, HWCAP_FCMA, ID_AA64ISAR1_FCMA_IMPL),
 698         MRS_HWCAP_END
 699 };
 700
 701 static struct mrs_field_value id_aa64isar1_jscvt[] = {
 702         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, JSCVT, NONE, IMPL),
 703         MRS_FIELD_VALUE_END,
 704 };
 705
 706 static struct mrs_field_hwcap id_aa64isar1_jscvt_caps[] = {
 707         MRS_HWCAP(&elf_hwcap, HWCAP_JSCVT, ID_AA64ISAR1_JSCVT_IMPL),
 708         MRS_HWCAP_END
 709 };
 710
 711 static struct mrs_field_value id_aa64isar1_api[] = {
 712         MRS_FIELD_VALUE(ID_AA64ISAR1_API_NONE, ""),
 713         MRS_FIELD_VALUE(ID_AA64ISAR1_API_PAC, "API PAC"),
 714         MRS_FIELD_VALUE(ID_AA64ISAR1_API_EPAC, "API EPAC"),
 715         MRS_FIELD_VALUE(ID_AA64ISAR1_API_EPAC2, "Impl PAuth+EPAC2"),
 716         MRS_FIELD_VALUE(ID_AA64ISAR1_API_FPAC, "Impl PAuth+FPAC"),
 717         MRS_FIELD_VALUE(ID_AA64ISAR1_API_FPAC_COMBINED,
 718             "Impl PAuth+FPAC+Combined"),
 719         MRS_FIELD_VALUE_END,
 720 };
 721
 722 static struct mrs_field_hwcap id_aa64isar1_api_caps[] = {
 723         MRS_HWCAP(&elf_hwcap, HWCAP_PACA, ID_AA64ISAR1_API_PAC),
 724         MRS_HWCAP_END
 725 };
 726
 727 static struct mrs_field_value id_aa64isar1_apa[] = {
 728         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_NONE, ""),
 729         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_PAC, "APA PAC"),
 730         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_EPAC, "APA EPAC"),
 731         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_EPAC2, "APA EPAC2"),
 732         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_FPAC, "APA FPAC"),
 733         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_FPAC_COMBINED,
 734             "APA FPAC+Combined"),
 735         MRS_FIELD_VALUE_END,
 736 };
 737
 738 static struct mrs_field_hwcap id_aa64isar1_apa_caps[] = {
 739         MRS_HWCAP(&elf_hwcap, HWCAP_PACA, ID_AA64ISAR1_APA_PAC),
 740         MRS_HWCAP_END
 741 };
 742
 743 static struct mrs_field_value id_aa64isar1_dpb[] = {
 744         MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_NONE, ""),
 745         MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVAP, "DCPoP"),
 746         MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVADP, "DCCVADP"),
 747         MRS_FIELD_VALUE_END,
 748 };
 749
 750 static struct mrs_field_hwcap id_aa64isar1_dpb_caps[] = {
 751         MRS_HWCAP(&elf_hwcap, HWCAP_DCPOP, ID_AA64ISAR1_DPB_DCCVAP),
 752         MRS_HWCAP(&elf_hwcap2, HWCAP2_DCPODP, ID_AA64ISAR1_DPB_DCCVADP),
 753         MRS_HWCAP_END
 754 };
 755
 756 static struct mrs_field id_aa64isar1_fields[] = {
 757         MRS_FIELD_HWCAP(ID_AA64ISAR1, I8MM, false, MRS_LOWER,
 758             id_aa64isar1_i8mm, id_aa64isar1_i8mm_caps),
 759         MRS_FIELD_HWCAP(ID_AA64ISAR1, DGH, false, MRS_LOWER, id_aa64isar1_dgh,
 760             id_aa64isar1_dgh_caps),
 761         MRS_FIELD_HWCAP(ID_AA64ISAR1, BF16, false, MRS_LOWER,
 762             id_aa64isar1_bf16, id_aa64isar1_bf16_caps),
 763         MRS_FIELD(ID_AA64ISAR1, SPECRES, false, MRS_EXACT,
 764             id_aa64isar1_specres),
 765         MRS_FIELD_HWCAP(ID_AA64ISAR1, SB, false, MRS_LOWER, id_aa64isar1_sb,
 766             id_aa64isar1_sb_caps),
 767         MRS_FIELD_HWCAP(ID_AA64ISAR1, FRINTTS, false, MRS_LOWER,
 768             id_aa64isar1_frintts, id_aa64isar1_frintts_caps),
 769         MRS_FIELD_HWCAP(ID_AA64ISAR1, GPI, false, MRS_EXACT, id_aa64isar1_gpi,
 770             id_aa64isar1_gpi_caps),
 771         MRS_FIELD_HWCAP(ID_AA64ISAR1, GPA, false, MRS_EXACT, id_aa64isar1_gpa,
 772             id_aa64isar1_gpa_caps),
 773         MRS_FIELD_HWCAP(ID_AA64ISAR1, LRCPC, false, MRS_LOWER,
 774             id_aa64isar1_lrcpc, id_aa64isar1_lrcpc_caps),
 775         MRS_FIELD_HWCAP(ID_AA64ISAR1, FCMA, false, MRS_LOWER,
 776             id_aa64isar1_fcma, id_aa64isar1_fcma_caps),
 777         MRS_FIELD_HWCAP(ID_AA64ISAR1, JSCVT, false, MRS_LOWER,
 778             id_aa64isar1_jscvt, id_aa64isar1_jscvt_caps),
 779         MRS_FIELD_HWCAP(ID_AA64ISAR1, API, false, MRS_EXACT, id_aa64isar1_api,
 780             id_aa64isar1_api_caps),
 781         MRS_FIELD_HWCAP(ID_AA64ISAR1, APA, false, MRS_EXACT, id_aa64isar1_apa,
 782             id_aa64isar1_apa_caps),
 783         MRS_FIELD_HWCAP(ID_AA64ISAR1, DPB, false, MRS_LOWER, id_aa64isar1_dpb,
 784             id_aa64isar1_dpb_caps),
 785         MRS_FIELD_END,
 786 };
 787
 788
 789 /* ID_AA64ISAR2_EL1 */
 790 static struct mrs_field_value id_aa64isar2_pac_frac[] = {
 791         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, PAC_frac, NONE, IMPL),
 792         MRS_FIELD_VALUE_END,
 793 };
 794
 795 static struct mrs_field_value id_aa64isar2_bc[] = {
 796         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, BC, NONE, IMPL),
 797         MRS_FIELD_VALUE_END,
 798 };
 799
 800 static struct mrs_field_value id_aa64isar2_mops[] = {
 801         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, MOPS, NONE, IMPL),
 802         MRS_FIELD_VALUE_END,
 803 };
 804
 805 static struct mrs_field_value id_aa64isar2_apa3[] = {
 806         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_NONE, ""),
 807         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_PAC, "APA3 PAC"),
 808         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_EPAC, "APA3 EPAC"),
 809         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_EPAC2, "APA3 EPAC2"),
 810         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_FPAC, "APA3 FPAC"),
 811         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_FPAC_COMBINED,
 812             "APA3 FPAC+Combined"),
 813         MRS_FIELD_VALUE_END,
 814 };
 815
 816 static struct mrs_field_hwcap id_aa64isar2_apa3_caps[] = {
 817         MRS_HWCAP(&elf_hwcap, HWCAP_PACA, ID_AA64ISAR2_APA3_PAC),
 818         MRS_HWCAP_END
 819 };
 820
 821 static struct mrs_field_value id_aa64isar2_gpa3[] = {
 822         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, GPA3, NONE, IMPL),
 823         MRS_FIELD_VALUE_END,
 824 };
 825
 826 static struct mrs_field_hwcap id_aa64isar2_gpa3_caps[] = {
 827         MRS_HWCAP(&elf_hwcap, HWCAP_PACG, ID_AA64ISAR2_GPA3_IMPL),
 828         MRS_HWCAP_END
 829 };
 830
 831 static struct mrs_field_value id_aa64isar2_rpres[] = {
 832         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, RPRES, NONE, IMPL),
 833         MRS_FIELD_VALUE_END,
 834 };
 835
 836 static struct mrs_field_value id_aa64isar2_wfxt[] = {
 837         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, WFxT, NONE, IMPL),
 838         MRS_FIELD_VALUE_END,
 839 };
 840
 841 static struct mrs_field id_aa64isar2_fields[] = {
 842         MRS_FIELD(ID_AA64ISAR2, PAC_frac, false, MRS_EXACT,
 843             id_aa64isar2_pac_frac),
 844         MRS_FIELD(ID_AA64ISAR2, BC, false, MRS_EXACT, id_aa64isar2_bc),
 845         MRS_FIELD(ID_AA64ISAR2, MOPS, false, MRS_EXACT, id_aa64isar2_mops),
 846         MRS_FIELD_HWCAP(ID_AA64ISAR2, APA3, false, MRS_EXACT,
 847             id_aa64isar2_apa3, id_aa64isar2_apa3_caps),
 848         MRS_FIELD_HWCAP(ID_AA64ISAR2, GPA3, false, MRS_EXACT,
 849             id_aa64isar2_gpa3, id_aa64isar2_gpa3_caps),
 850         MRS_FIELD(ID_AA64ISAR2, RPRES, false, MRS_EXACT, id_aa64isar2_rpres),
 851         MRS_FIELD(ID_AA64ISAR2, WFxT, false, MRS_EXACT, id_aa64isar2_wfxt),
 852         MRS_FIELD_END,
 853 };
 854
 855
 856 /* ID_AA64MMFR0_EL1 */
 857 static struct mrs_field_value id_aa64mmfr0_exs[] = {
 858         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, ExS, ALL, IMPL),
 859         MRS_FIELD_VALUE_END,
 860 };
 861
 862 static struct mrs_field_value id_aa64mmfr0_tgran4_2[] = {
 863         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_TGran4, ""),
 864         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_NONE, "No S2 TGran4"),
 865         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_IMPL, "S2 TGran4"),
 866         MRS_FIELD_VALUE_END,
 867 };
 868
 869 static struct mrs_field_value id_aa64mmfr0_tgran64_2[] = {
 870         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_TGran64, ""),
 871         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_NONE, "No S2 TGran64"),
 872         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_IMPL, "S2 TGran64"),
 873         MRS_FIELD_VALUE_END,
 874 };
 875
 876 static struct mrs_field_value id_aa64mmfr0_tgran16_2[] = {
 877         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_TGran16, ""),
 878         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_NONE, "No S2 TGran16"),
 879         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_IMPL, "S2 TGran16"),
 880         MRS_FIELD_VALUE_END,
 881 };
 882
 883 static struct mrs_field_value id_aa64mmfr0_tgran4[] = {
 884         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran4,NONE, IMPL),
 885         MRS_FIELD_VALUE_END,
 886 };
 887
 888 static struct mrs_field_value id_aa64mmfr0_tgran64[] = {
 889         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran64, NONE, IMPL),
 890         MRS_FIELD_VALUE_END,
 891 };
 892
 893 static struct mrs_field_value id_aa64mmfr0_tgran16[] = {
 894         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran16, NONE, IMPL),
 895         MRS_FIELD_VALUE_END,
 896 };
 897
 898 static struct mrs_field_value id_aa64mmfr0_bigendel0[] = {
 899         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEndEL0, FIXED, MIXED),
 900         MRS_FIELD_VALUE_END,
 901 };
 902
 903 static struct mrs_field_value id_aa64mmfr0_snsmem[] = {
 904         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, SNSMem, NONE, DISTINCT),
 905         MRS_FIELD_VALUE_END,
 906 };
 907
 908 static struct mrs_field_value id_aa64mmfr0_bigend[] = {
 909         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEnd, FIXED, MIXED),
 910         MRS_FIELD_VALUE_END,
 911 };
 912
 913 static struct mrs_field_value id_aa64mmfr0_asidbits[] = {
 914         MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_8, "8bit ASID"),
 915         MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_16, "16bit ASID"),
 916         MRS_FIELD_VALUE_END,
 917 };
 918
 919 static struct mrs_field_value id_aa64mmfr0_parange[] = {
 920         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4G, "4GB PA"),
 921         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_64G, "64GB PA"),
 922         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_1T, "1TB PA"),
 923         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4T, "4TB PA"),
 924         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_16T, "16TB PA"),
 925         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_256T, "256TB PA"),
 926         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4P, "4PB PA"),
 927         MRS_FIELD_VALUE_END,
 928 };
 929
 930 static struct mrs_field id_aa64mmfr0_fields[] = {
 931         MRS_FIELD(ID_AA64MMFR0, ExS, false, MRS_EXACT, id_aa64mmfr0_exs),
 932         MRS_FIELD(ID_AA64MMFR0, TGran4_2, false, MRS_EXACT,
 933             id_aa64mmfr0_tgran4_2),
 934         MRS_FIELD(ID_AA64MMFR0, TGran64_2, false, MRS_EXACT,
 935             id_aa64mmfr0_tgran64_2),
 936         MRS_FIELD(ID_AA64MMFR0, TGran16_2, false, MRS_EXACT,
 937             id_aa64mmfr0_tgran16_2),
 938         MRS_FIELD(ID_AA64MMFR0, TGran4, false, MRS_EXACT, id_aa64mmfr0_tgran4),
 939         MRS_FIELD(ID_AA64MMFR0, TGran64, false, MRS_EXACT,
 940             id_aa64mmfr0_tgran64),
 941         MRS_FIELD(ID_AA64MMFR0, TGran16, false, MRS_EXACT,
 942             id_aa64mmfr0_tgran16),
 943         MRS_FIELD(ID_AA64MMFR0, BigEndEL0, false, MRS_EXACT,
 944             id_aa64mmfr0_bigendel0),
 945         MRS_FIELD(ID_AA64MMFR0, SNSMem, false, MRS_EXACT, id_aa64mmfr0_snsmem),
 946         MRS_FIELD(ID_AA64MMFR0, BigEnd, false, MRS_EXACT, id_aa64mmfr0_bigend),
 947         MRS_FIELD(ID_AA64MMFR0, ASIDBits, false, MRS_EXACT,
 948             id_aa64mmfr0_asidbits),
 949         MRS_FIELD(ID_AA64MMFR0, PARange, false, MRS_EXACT,
 950             id_aa64mmfr0_parange),
 951         MRS_FIELD_END,
 952 };
 953
 954
 955 /* ID_AA64MMFR1_EL1 */
 956 static struct mrs_field_value id_aa64mmfr1_xnx[] = {
 957         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, XNX, NONE, IMPL),
 958         MRS_FIELD_VALUE_END,
 959 };
 960
 961 static struct mrs_field_value id_aa64mmfr1_specsei[] = {
 962         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, SpecSEI, NONE, IMPL),
 963         MRS_FIELD_VALUE_END,
 964 };
 965
 966 static struct mrs_field_value id_aa64mmfr1_pan[] = {
 967         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, PAN, NONE, IMPL),
 968         MRS_FIELD_VALUE(ID_AA64MMFR1_PAN_ATS1E1, "PAN+ATS1E1"),
 969         MRS_FIELD_VALUE_END,
 970 };
 971
 972 static struct mrs_field_value id_aa64mmfr1_lo[] = {
 973         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, LO, NONE, IMPL),
 974         MRS_FIELD_VALUE_END,
 975 };
 976
 977 static struct mrs_field_value id_aa64mmfr1_hpds[] = {
 978         MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_NONE, ""),
 979         MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_HPD, "HPD"),
 980         MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_TTPBHA, "HPD+TTPBHA"),
 981         MRS_FIELD_VALUE_END,
 982 };
 983
 984 static struct mrs_field_value id_aa64mmfr1_vh[] = {
 985         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, VH, NONE, IMPL),
 986         MRS_FIELD_VALUE_END,
 987 };
 988
 989 static struct mrs_field_value id_aa64mmfr1_vmidbits[] = {
 990         MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_8, "8bit VMID"),
 991         MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_16, "16bit VMID"),
 992         MRS_FIELD_VALUE_END,
 993 };
 994
 995 static struct mrs_field_value id_aa64mmfr1_hafdbs[] = {
 996         MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_NONE, ""),
 997         MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF, "HAF"),
 998         MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF_DBS, "HAF+DS"),
 999         MRS_FIELD_VALUE_END,
1000 };
1001
1002 static struct mrs_field id_aa64mmfr1_fields[] = {
1003         MRS_FIELD(ID_AA64MMFR1, XNX, false, MRS_EXACT, id_aa64mmfr1_xnx),
1004         MRS_FIELD(ID_AA64MMFR1, SpecSEI, false, MRS_EXACT,
1005             id_aa64mmfr1_specsei),
1006         MRS_FIELD(ID_AA64MMFR1, PAN, false, MRS_EXACT, id_aa64mmfr1_pan),
1007         MRS_FIELD(ID_AA64MMFR1, LO, false, MRS_EXACT, id_aa64mmfr1_lo),
1008         MRS_FIELD(ID_AA64MMFR1, HPDS, false, MRS_EXACT, id_aa64mmfr1_hpds),
1009         MRS_FIELD(ID_AA64MMFR1, VH, false, MRS_EXACT, id_aa64mmfr1_vh),
1010         MRS_FIELD(ID_AA64MMFR1, VMIDBits, false, MRS_EXACT,
1011             id_aa64mmfr1_vmidbits),
1012         MRS_FIELD(ID_AA64MMFR1, HAFDBS, false, MRS_EXACT, id_aa64mmfr1_hafdbs),
1013         MRS_FIELD_END,
1014 };
1015
1016
1017 /* ID_AA64MMFR2_EL1 */
1018 static struct mrs_field_value id_aa64mmfr2_e0pd[] = {
1019         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, E0PD, NONE, IMPL),
1020         MRS_FIELD_VALUE_END,
1021 };
1022
1023 static struct mrs_field_value id_aa64mmfr2_evt[] = {
1024         MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_NONE, ""),
1025         MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_2, "EVT-8.2"),
1026         MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_5, "EVT-8.5"),
1027         MRS_FIELD_VALUE_END,
1028 };
1029
1030 static struct mrs_field_value id_aa64mmfr2_bbm[] = {
1031         MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL0, ""),
1032         MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL1, "BBM level 1"),
1033         MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL2, "BBM level 2"),
1034         MRS_FIELD_VALUE_END,
1035 };
1036
1037 static struct mrs_field_value id_aa64mmfr2_ttl[] = {
1038         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, TTL, NONE, IMPL),
1039         MRS_FIELD_VALUE_END,
1040 };
1041
1042 static struct mrs_field_value id_aa64mmfr2_fwb[] = {
1043         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, FWB, NONE, IMPL),
1044         MRS_FIELD_VALUE_END,
1045 };
1046
1047 static struct mrs_field_value id_aa64mmfr2_ids[] = {
1048         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IDS, NONE, IMPL),
1049         MRS_FIELD_VALUE_END,
1050 };
1051
1052 static struct mrs_field_value id_aa64mmfr2_at[] = {
1053         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, AT, NONE, IMPL),
1054         MRS_FIELD_VALUE_END,
1055 };
1056
1057 static struct mrs_field_hwcap id_aa64mmfr2_at_caps[] = {
1058         MRS_HWCAP(&elf_hwcap, HWCAP_USCAT, ID_AA64MMFR2_AT_IMPL),
1059         MRS_HWCAP_END
1060 };
1061
1062 static struct mrs_field_value id_aa64mmfr2_st[] = {
1063         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, ST, NONE, IMPL),
1064         MRS_FIELD_VALUE_END,
1065 };
1066
1067 static struct mrs_field_value id_aa64mmfr2_nv[] = {
1068         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, NV, NONE, 8_3),
1069         MRS_FIELD_VALUE(ID_AA64MMFR2_NV_8_4, "NV v8.4"),
1070         MRS_FIELD_VALUE_END,
1071 };
1072
1073 static struct mrs_field_value id_aa64mmfr2_ccidx[] = {
1074         MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_32, "32bit CCIDX"),
1075         MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_64, "64bit CCIDX"),
1076         MRS_FIELD_VALUE_END,
1077 };
1078
1079 static struct mrs_field_value id_aa64mmfr2_varange[] = {
1080         MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_48, "48bit VA"),
1081         MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_52, "52bit VA"),
1082         MRS_FIELD_VALUE_END,
1083 };
1084
1085 static struct mrs_field_value id_aa64mmfr2_iesb[] = {
1086         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IESB, NONE, IMPL),
1087         MRS_FIELD_VALUE_END,
1088 };
1089
1090 static struct mrs_field_value id_aa64mmfr2_lsm[] = {
1091         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, LSM, NONE, IMPL),
1092         MRS_FIELD_VALUE_END,
1093 };
1094
1095 static struct mrs_field_value id_aa64mmfr2_uao[] = {
1096         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, UAO, NONE, IMPL),
1097         MRS_FIELD_VALUE_END,
1098 };
1099
1100 static struct mrs_field_value id_aa64mmfr2_cnp[] = {
1101         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, CnP, NONE, IMPL),
1102         MRS_FIELD_VALUE_END,
1103 };
1104
1105 static struct mrs_field id_aa64mmfr2_fields[] = {
1106         MRS_FIELD(ID_AA64MMFR2, E0PD, false, MRS_EXACT, id_aa64mmfr2_e0pd),
1107         MRS_FIELD(ID_AA64MMFR2, EVT, false, MRS_EXACT, id_aa64mmfr2_evt),
1108         MRS_FIELD(ID_AA64MMFR2, BBM, false, MRS_EXACT, id_aa64mmfr2_bbm),
1109         MRS_FIELD(ID_AA64MMFR2, TTL, false, MRS_EXACT, id_aa64mmfr2_ttl),
1110         MRS_FIELD(ID_AA64MMFR2, FWB, false, MRS_EXACT, id_aa64mmfr2_fwb),
1111         MRS_FIELD(ID_AA64MMFR2, IDS, false, MRS_EXACT, id_aa64mmfr2_ids),
1112         MRS_FIELD_HWCAP(ID_AA64MMFR2, AT, false, MRS_LOWER, id_aa64mmfr2_at,
1113             id_aa64mmfr2_at_caps),
1114         MRS_FIELD(ID_AA64MMFR2, ST, false, MRS_EXACT, id_aa64mmfr2_st),
1115         MRS_FIELD(ID_AA64MMFR2, NV, false, MRS_EXACT, id_aa64mmfr2_nv),
1116         MRS_FIELD(ID_AA64MMFR2, CCIDX, false, MRS_EXACT, id_aa64mmfr2_ccidx),
1117         MRS_FIELD(ID_AA64MMFR2, VARange, false, MRS_EXACT,
1118             id_aa64mmfr2_varange),
1119         MRS_FIELD(ID_AA64MMFR2, IESB, false, MRS_EXACT, id_aa64mmfr2_iesb),
1120         MRS_FIELD(ID_AA64MMFR2, LSM, false, MRS_EXACT, id_aa64mmfr2_lsm),
1121         MRS_FIELD(ID_AA64MMFR2, UAO, false, MRS_EXACT, id_aa64mmfr2_uao),
1122         MRS_FIELD(ID_AA64MMFR2, CnP, false, MRS_EXACT, id_aa64mmfr2_cnp),
1123         MRS_FIELD_END,
1124 };
1125
1126
1127 /* ID_AA64PFR0_EL1 */
1128 static struct mrs_field_value id_aa64pfr0_csv3[] = {
1129         MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_NONE, ""),
1130         MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_ISOLATED, "CSV3"),
1131         MRS_FIELD_VALUE_END,
1132 };
1133
1134 static struct mrs_field_value id_aa64pfr0_csv2[] = {
1135         MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_NONE, ""),
1136         MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_ISOLATED, "CSV2"),
1137         MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_SCXTNUM, "SCXTNUM"),
1138         MRS_FIELD_VALUE_END,
1139 };
1140
1141 static struct mrs_field_value id_aa64pfr0_dit[] = {
1142         MRS_FIELD_VALUE(ID_AA64PFR0_DIT_NONE, ""),
1143         MRS_FIELD_VALUE(ID_AA64PFR0_DIT_PSTATE, "PSTATE.DIT"),
1144         MRS_FIELD_VALUE_END,
1145 };
1146
1147 static struct mrs_field_hwcap id_aa64pfr0_dit_caps[] = {
1148         MRS_HWCAP(&elf_hwcap, HWCAP_DIT, ID_AA64PFR0_DIT_PSTATE),
1149         MRS_HWCAP_END
1150 };
1151
1152 static struct mrs_field_value id_aa64pfr0_amu[] = {
1153         MRS_FIELD_VALUE(ID_AA64PFR0_AMU_NONE, ""),
1154         MRS_FIELD_VALUE(ID_AA64PFR0_AMU_V1, "AMUv1"),
1155         MRS_FIELD_VALUE_END,
1156 };
1157
1158 static struct mrs_field_value id_aa64pfr0_mpam[] = {
1159         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, MPAM, NONE, IMPL),
1160         MRS_FIELD_VALUE_END,
1161 };
1162
1163 static struct mrs_field_value id_aa64pfr0_sel2[] = {
1164         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SEL2, NONE, IMPL),
1165         MRS_FIELD_VALUE_END,
1166 };
1167
1168 static struct mrs_field_value id_aa64pfr0_sve[] = {
1169         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SVE, NONE, IMPL),
1170         MRS_FIELD_VALUE_END,
1171 };
1172
1173 #if 0
1174 /* Enable when we add SVE support */
1175 static struct mrs_field_hwcap id_aa64pfr0_sve_caps[] = {
1176         MRS_HWCAP(&elf_hwcap, HWCAP_SVE, ID_AA64PFR0_SVE_IMPL),
1177         MRS_HWCAP_END
1178 };
1179 #endif
1180
1181 static struct mrs_field_value id_aa64pfr0_ras[] = {
1182         MRS_FIELD_VALUE(ID_AA64PFR0_RAS_NONE, ""),
1183         MRS_FIELD_VALUE(ID_AA64PFR0_RAS_IMPL, "RAS"),
1184         MRS_FIELD_VALUE(ID_AA64PFR0_RAS_8_4, "RAS v8.4"),
1185         MRS_FIELD_VALUE_END,
1186 };
1187
1188 static struct mrs_field_value id_aa64pfr0_gic[] = {
1189         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, GIC, CPUIF_NONE, CPUIF_EN),
1190         MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_NONE, ""),
1191         MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_EN, "GIC"),
1192         MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_4_1, "GIC 4.1"),
1193         MRS_FIELD_VALUE_END,
1194 };
1195
1196 static struct mrs_field_value id_aa64pfr0_advsimd[] = {
1197         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, AdvSIMD, NONE, IMPL),
1198         MRS_FIELD_VALUE(ID_AA64PFR0_AdvSIMD_HP, "AdvSIMD+HP"),
1199         MRS_FIELD_VALUE_END,
1200 };
1201
1202 static struct mrs_field_hwcap id_aa64pfr0_advsimd_caps[] = {
1203         MRS_HWCAP(&elf_hwcap, HWCAP_ASIMD, ID_AA64PFR0_AdvSIMD_IMPL),
1204         MRS_HWCAP(&elf_hwcap, HWCAP_ASIMDHP, ID_AA64PFR0_AdvSIMD_HP),
1205         MRS_HWCAP_END
1206 };
1207
1208 static struct mrs_field_value id_aa64pfr0_fp[] = {
1209         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, FP, NONE, IMPL),
1210         MRS_FIELD_VALUE(ID_AA64PFR0_FP_HP, "FP+HP"),
1211         MRS_FIELD_VALUE_END,
1212 };
1213
1214 static struct mrs_field_hwcap id_aa64pfr0_fp_caps[] = {
1215         MRS_HWCAP(&elf_hwcap, HWCAP_FP, ID_AA64PFR0_FP_IMPL),
1216         MRS_HWCAP(&elf_hwcap, HWCAP_FPHP, ID_AA64PFR0_FP_HP),
1217         MRS_HWCAP_END
1218 };
1219
1220 static struct mrs_field_value id_aa64pfr0_el3[] = {
1221         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL3, NONE, 64),
1222         MRS_FIELD_VALUE(ID_AA64PFR0_EL3_64_32, "EL3 32"),
1223         MRS_FIELD_VALUE_END,
1224 };
1225
1226 static struct mrs_field_value id_aa64pfr0_el2[] = {
1227         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL2, NONE, 64),
1228         MRS_FIELD_VALUE(ID_AA64PFR0_EL2_64_32, "EL2 32"),
1229         MRS_FIELD_VALUE_END,
1230 };
1231
1232 static struct mrs_field_value id_aa64pfr0_el1[] = {
1233         MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64, "EL1"),
1234         MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64_32, "EL1 32"),
1235         MRS_FIELD_VALUE_END,
1236 };
1237
1238 static struct mrs_field_value id_aa64pfr0_el0[] = {
1239         MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64, "EL0"),
1240         MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64_32, "EL0 32"),
1241         MRS_FIELD_VALUE_END,
1242 };
1243
1244 static struct mrs_field id_aa64pfr0_fields[] = {
1245         MRS_FIELD(ID_AA64PFR0, CSV3, false, MRS_EXACT, id_aa64pfr0_csv3),
1246         MRS_FIELD(ID_AA64PFR0, CSV2, false, MRS_EXACT, id_aa64pfr0_csv2),
1247         MRS_FIELD_HWCAP(ID_AA64PFR0, DIT, false, MRS_LOWER, id_aa64pfr0_dit,
1248             id_aa64pfr0_dit_caps),
1249         MRS_FIELD(ID_AA64PFR0, AMU, false, MRS_EXACT, id_aa64pfr0_amu),
1250         MRS_FIELD(ID_AA64PFR0, MPAM, false, MRS_EXACT, id_aa64pfr0_mpam),
1251         MRS_FIELD(ID_AA64PFR0, SEL2, false, MRS_EXACT, id_aa64pfr0_sel2),
1252         MRS_FIELD(ID_AA64PFR0, SVE, false, MRS_EXACT, id_aa64pfr0_sve),
1253         MRS_FIELD(ID_AA64PFR0, RAS, false, MRS_EXACT, id_aa64pfr0_ras),
1254         MRS_FIELD(ID_AA64PFR0, GIC, false, MRS_EXACT, id_aa64pfr0_gic),
1255         MRS_FIELD_HWCAP(ID_AA64PFR0, AdvSIMD, true, MRS_LOWER,
1256             id_aa64pfr0_advsimd, id_aa64pfr0_advsimd_caps),
1257         MRS_FIELD_HWCAP(ID_AA64PFR0, FP, true,  MRS_LOWER, id_aa64pfr0_fp,
1258             id_aa64pfr0_fp_caps),
1259         MRS_FIELD(ID_AA64PFR0, EL3, false, MRS_EXACT, id_aa64pfr0_el3),
1260         MRS_FIELD(ID_AA64PFR0, EL2, false, MRS_EXACT, id_aa64pfr0_el2),
1261         MRS_FIELD(ID_AA64PFR0, EL1, false, MRS_LOWER, id_aa64pfr0_el1),
1262         MRS_FIELD(ID_AA64PFR0, EL0, false, MRS_LOWER, id_aa64pfr0_el0),
1263         MRS_FIELD_END,
1264 };
1265
1266
1267 /* ID_AA64PFR1_EL1 */
1268 static struct mrs_field_value id_aa64pfr1_mte[] = {
1269         MRS_FIELD_VALUE(ID_AA64PFR1_MTE_NONE, ""),
1270         MRS_FIELD_VALUE(ID_AA64PFR1_MTE_IMPL_EL0, "MTE EL0"),
1271         MRS_FIELD_VALUE(ID_AA64PFR1_MTE_IMPL, "MTE"),
1272         MRS_FIELD_VALUE_END,
1273 };
1274
1275 static struct mrs_field_value id_aa64pfr1_ssbs[] = {
1276         MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_NONE, ""),
1277         MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE, "PSTATE.SSBS"),
1278         MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE_MSR, "PSTATE.SSBS MSR"),
1279         MRS_FIELD_VALUE_END,
1280 };
1281
1282 static struct mrs_field_hwcap id_aa64pfr1_ssbs_caps[] = {
1283         MRS_HWCAP(&elf_hwcap, HWCAP_SSBS, ID_AA64PFR1_SSBS_PSTATE),
1284         MRS_HWCAP_END
1285 };
1286
1287 static struct mrs_field_value id_aa64pfr1_bt[] = {
1288         MRS_FIELD_VALUE(ID_AA64PFR1_BT_NONE, ""),
1289         MRS_FIELD_VALUE(ID_AA64PFR1_BT_IMPL, "BTI"),
1290         MRS_FIELD_VALUE_END,
1291 };
1292
1293 #if 0
1294 /* Enable when we add BTI support */
1295 static struct mrs_field_hwcap id_aa64pfr1_bt_caps[] = {
1296         MRS_HWCAP(&elf_hwcap2, HWCAP2_BTI, ID_AA64PFR1_BT_IMPL),
1297         MRS_HWCAP_END
1298 };
1299 #endif
1300
1301 static struct mrs_field id_aa64pfr1_fields[] = {
1302         MRS_FIELD(ID_AA64PFR1, MTE, false, MRS_EXACT, id_aa64pfr1_mte),
1303         MRS_FIELD_HWCAP(ID_AA64PFR1, SSBS, false, MRS_LOWER, id_aa64pfr1_ssbs,
1304             id_aa64pfr1_ssbs_caps),
1305         MRS_FIELD(ID_AA64PFR1, BT, false, MRS_EXACT, id_aa64pfr1_bt),
1306         MRS_FIELD_END,
1307 };
1308
1309
1310 /* ID_AA64ZFR0_EL1 */
1311 static struct mrs_field_value id_aa64zfr0_f64mm[] = {
1312         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, F64MM, NONE, IMPL),
1313         MRS_FIELD_VALUE_END,
1314 };
1315
1316 static struct mrs_field_value id_aa64zfr0_f32mm[] = {
1317         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, F32MM, NONE, IMPL),
1318         MRS_FIELD_VALUE_END,
1319 };
1320
1321 static struct mrs_field_value id_aa64zfr0_i8mm[] = {
1322         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, I8MM, NONE, IMPL),
1323         MRS_FIELD_VALUE_END,
1324 };
1325
1326 static struct mrs_field_value id_aa64zfr0_sm4[] = {
1327         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, SM4, NONE, IMPL),
1328         MRS_FIELD_VALUE_END,
1329 };
1330
1331 static struct mrs_field_value id_aa64zfr0_sha3[] = {
1332         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, SHA3, NONE, IMPL),
1333         MRS_FIELD_VALUE_END,
1334 };
1335
1336 static struct mrs_field_value id_aa64zfr0_bf16[] = {
1337         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, BF16, NONE, BASE),
1338         MRS_FIELD_VALUE(ID_AA64ZFR0_BF16_EBF, "BF16+EBF"),
1339         MRS_FIELD_VALUE_END,
1340 };
1341
1342 static struct mrs_field_value id_aa64zfr0_bitperm[] = {
1343         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, BitPerm, NONE, IMPL),
1344         MRS_FIELD_VALUE_END,
1345 };
1346
1347 static struct mrs_field_value id_aa64zfr0_aes[] = {
1348         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, AES, NONE, BASE),
1349         MRS_FIELD_VALUE(ID_AA64ZFR0_AES_PMULL, "AES+PMULL"),
1350         MRS_FIELD_VALUE_END,
1351 };
1352
1353 static struct mrs_field_value id_aa64zfr0_svever[] = {
1354         MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE1, "SVE1"),
1355         MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE2, "SVE2"),
1356         MRS_FIELD_VALUE_END,
1357 };
1358
1359 static struct mrs_field id_aa64zfr0_fields[] = {
1360         MRS_FIELD(ID_AA64ZFR0, F64MM, false, MRS_EXACT, id_aa64zfr0_f64mm),
1361         MRS_FIELD(ID_AA64ZFR0, F32MM, false, MRS_EXACT, id_aa64zfr0_f32mm),
1362         MRS_FIELD(ID_AA64ZFR0, I8MM, false, MRS_EXACT, id_aa64zfr0_i8mm),
1363         MRS_FIELD(ID_AA64ZFR0, SM4, false, MRS_EXACT, id_aa64zfr0_sm4),
1364         MRS_FIELD(ID_AA64ZFR0, SHA3, false, MRS_EXACT, id_aa64zfr0_sha3),
1365         MRS_FIELD(ID_AA64ZFR0, BF16, false, MRS_EXACT, id_aa64zfr0_bf16),
1366         MRS_FIELD(ID_AA64ZFR0, BitPerm, false, MRS_EXACT, id_aa64zfr0_bitperm),
1367         MRS_FIELD(ID_AA64ZFR0, AES, false, MRS_EXACT, id_aa64zfr0_aes),
1368         MRS_FIELD(ID_AA64ZFR0, SVEver, false, MRS_EXACT, id_aa64zfr0_svever),
1369         MRS_FIELD_END,
1370 };
1371
1372
1373 #ifdef COMPAT_FREEBSD32
1374 /* ID_ISAR5_EL1 */
1375 static struct mrs_field_value id_isar5_vcma[] = {
1376         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, VCMA, NONE, IMPL),
1377         MRS_FIELD_VALUE_END,
1378 };
1379
1380 static struct mrs_field_value id_isar5_rdm[] = {
1381         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, RDM, NONE, IMPL),
1382         MRS_FIELD_VALUE_END,
1383 };
1384
1385 static struct mrs_field_value id_isar5_crc32[] = {
1386         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, CRC32, NONE, IMPL),
1387         MRS_FIELD_VALUE_END,
1388 };
1389
1390 static struct mrs_field_hwcap id_isar5_crc32_caps[] = {
1391         MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_CRC32, ID_ISAR5_CRC32_IMPL),
1392         MRS_HWCAP_END
1393 };
1394
1395 static struct mrs_field_value id_isar5_sha2[] = {
1396         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA2, NONE, IMPL),
1397         MRS_FIELD_VALUE_END,
1398 };
1399
1400 static struct mrs_field_hwcap id_isar5_sha2_caps[] = {
1401         MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_SHA2, ID_ISAR5_SHA2_IMPL),
1402         MRS_HWCAP_END
1403 };
1404
1405 static struct mrs_field_value id_isar5_sha1[] = {
1406         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA1, NONE, IMPL),
1407         MRS_FIELD_VALUE_END,
1408 };
1409
1410 static struct mrs_field_hwcap id_isar5_sha1_caps[] = {
1411         MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_SHA1, ID_ISAR5_SHA1_IMPL),
1412         MRS_HWCAP_END
1413 };
1414
1415 static struct mrs_field_value id_isar5_aes[] = {
1416         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, AES, NONE, BASE),
1417         MRS_FIELD_VALUE(ID_ISAR5_AES_VMULL, "AES+VMULL"),
1418         MRS_FIELD_VALUE_END,
1419 };
1420
1421 static struct mrs_field_hwcap id_isar5_aes_caps[] = {
1422         MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_AES, ID_ISAR5_AES_BASE),
1423         MRS_HWCAP(&elf32_hwcap2, HWCAP32_2_PMULL, ID_ISAR5_AES_VMULL),
1424         MRS_HWCAP_END
1425 };
1426
1427 static struct mrs_field_value id_isar5_sevl[] = {
1428         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SEVL, NOP, IMPL),
1429         MRS_FIELD_VALUE_END,
1430 };
1431
1432 static struct mrs_field id_isar5_fields[] = {
1433         MRS_FIELD(ID_ISAR5, VCMA, false, MRS_LOWER, id_isar5_vcma),
1434         MRS_FIELD(ID_ISAR5, RDM, false, MRS_LOWER, id_isar5_rdm),
1435         MRS_FIELD_HWCAP(ID_ISAR5, CRC32, false, MRS_LOWER, id_isar5_crc32,
1436             id_isar5_crc32_caps),
1437         MRS_FIELD_HWCAP(ID_ISAR5, SHA2, false, MRS_LOWER, id_isar5_sha2,
1438             id_isar5_sha2_caps),
1439         MRS_FIELD_HWCAP(ID_ISAR5, SHA1, false, MRS_LOWER, id_isar5_sha1,
1440             id_isar5_sha1_caps),
1441         MRS_FIELD_HWCAP(ID_ISAR5, AES, false, MRS_LOWER, id_isar5_aes,
1442             id_isar5_aes_caps),
1443         MRS_FIELD(ID_ISAR5, SEVL, false, MRS_LOWER, id_isar5_sevl),
1444         MRS_FIELD_END,
1445 };
1446
1447 /* MVFR0 */
1448 static struct mrs_field_value mvfr0_fpround[] = {
1449         MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPRound, NONE, IMPL),
1450         MRS_FIELD_VALUE_END,
1451 };
1452
1453 static struct mrs_field_value mvfr0_fpsqrt[] = {
1454         MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPSqrt, NONE, IMPL),
1455         MRS_FIELD_VALUE_END,
1456 };
1457
1458 static struct mrs_field_value mvfr0_fpdivide[] = {
1459         MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPDivide, NONE, IMPL),
1460         MRS_FIELD_VALUE_END,
1461 };
1462
1463 static struct mrs_field_value mvfr0_fptrap[] = {
1464         MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPTrap, NONE, IMPL),
1465         MRS_FIELD_VALUE_END,
1466 };
1467
1468 static struct mrs_field_value mvfr0_fpdp[] = {
1469         MRS_FIELD_VALUE(MVFR0_FPDP_NONE, ""),
1470         MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v2, "DP VFPv2"),
1471         MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v3_v4, "DP VFPv3+v4"),
1472         MRS_FIELD_VALUE_END,
1473 };
1474
1475 static struct mrs_field_hwcap mvfr0_fpdp_caps[] = {
1476         MRS_HWCAP(&elf32_hwcap, HWCAP32_VFP, MVFR0_FPDP_VFP_v2),
1477         MRS_HWCAP(&elf32_hwcap, HWCAP32_VFPv3, MVFR0_FPDP_VFP_v3_v4),
1478 };
1479
1480 static struct mrs_field_value mvfr0_fpsp[] = {
1481         MRS_FIELD_VALUE(MVFR0_FPSP_NONE, ""),
1482         MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v2, "SP VFPv2"),
1483         MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v3_v4, "SP VFPv3+v4"),
1484         MRS_FIELD_VALUE_END,
1485 };
1486
1487 static struct mrs_field_value mvfr0_simdreg[] = {
1488         MRS_FIELD_VALUE(MVFR0_SIMDReg_NONE, ""),
1489         MRS_FIELD_VALUE(MVFR0_SIMDReg_FP, "FP 16x64"),
1490         MRS_FIELD_VALUE(MVFR0_SIMDReg_AdvSIMD, "AdvSIMD"),
1491         MRS_FIELD_VALUE_END,
1492 };
1493
1494 static struct mrs_field mvfr0_fields[] = {
1495         MRS_FIELD(MVFR0, FPRound, false, MRS_LOWER, mvfr0_fpround),
1496         MRS_FIELD(MVFR0, FPSqrt, false, MRS_LOWER, mvfr0_fpsqrt),
1497         MRS_FIELD(MVFR0, FPDivide, false, MRS_LOWER, mvfr0_fpdivide),
1498         MRS_FIELD(MVFR0, FPTrap, false, MRS_LOWER, mvfr0_fptrap),
1499         MRS_FIELD_HWCAP(MVFR0, FPDP, false, MRS_LOWER, mvfr0_fpdp,
1500             mvfr0_fpdp_caps),
1501         MRS_FIELD(MVFR0, FPSP, false, MRS_LOWER, mvfr0_fpsp),
1502         MRS_FIELD(MVFR0, SIMDReg, false, MRS_LOWER, mvfr0_simdreg),
1503         MRS_FIELD_END,
1504 };
1505
1506 /* MVFR1 */
1507 static struct mrs_field_value mvfr1_simdfmac[] = {
1508         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDFMAC, NONE, IMPL),
1509         MRS_FIELD_VALUE_END,
1510 };
1511
1512 static struct mrs_field_hwcap mvfr1_simdfmac_caps[] = {
1513         MRS_HWCAP(&elf32_hwcap, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL),
1514         MRS_HWCAP_END
1515 };
1516
1517 static struct mrs_field_value mvfr1_fphp[] = {
1518         MRS_FIELD_VALUE(MVFR1_FPHP_NONE, ""),
1519         MRS_FIELD_VALUE(MVFR1_FPHP_CONV_SP, "FPHP SP Conv"),
1520         MRS_FIELD_VALUE(MVFR1_FPHP_CONV_DP, "FPHP DP Conv"),
1521         MRS_FIELD_VALUE(MVFR1_FPHP_ARITH, "FPHP Arith"),
1522         MRS_FIELD_VALUE_END,
1523 };
1524
1525 static struct mrs_field_value mvfr1_simdhp[] = {
1526         MRS_FIELD_VALUE(MVFR1_SIMDHP_NONE, ""),
1527         MRS_FIELD_VALUE(MVFR1_SIMDHP_CONV_SP, "SIMDHP SP Conv"),
1528         MRS_FIELD_VALUE(MVFR1_SIMDHP_ARITH, "SIMDHP Arith"),
1529         MRS_FIELD_VALUE_END,
1530 };
1531
1532 static struct mrs_field_value mvfr1_simdsp[] = {
1533         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDSP, NONE, IMPL),
1534         MRS_FIELD_VALUE_END,
1535 };
1536
1537 static struct mrs_field_value mvfr1_simdint[] = {
1538         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDInt, NONE, IMPL),
1539         MRS_FIELD_VALUE_END,
1540 };
1541
1542 static struct mrs_field_value mvfr1_simdls[] = {
1543         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDLS, NONE, IMPL),
1544         MRS_FIELD_VALUE_END,
1545 };
1546
1547 static struct mrs_field_hwcap mvfr1_simdls_caps[] = {
1548         MRS_HWCAP(&elf32_hwcap, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL),
1549         MRS_HWCAP_END
1550 };
1551
1552 static struct mrs_field_value mvfr1_fpdnan[] = {
1553         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPDNaN, NONE, IMPL),
1554         MRS_FIELD_VALUE_END,
1555 };
1556
1557 static struct mrs_field_value mvfr1_fpftz[] = {
1558         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPFtZ, NONE, IMPL),
1559         MRS_FIELD_VALUE_END,
1560 };
1561
1562 static struct mrs_field mvfr1_fields[] = {
1563         MRS_FIELD_HWCAP(MVFR1, SIMDFMAC, false, MRS_LOWER, mvfr1_simdfmac,
1564             mvfr1_simdfmac_caps),
1565         MRS_FIELD(MVFR1, FPHP, false, MRS_LOWER, mvfr1_fphp),
1566         MRS_FIELD(MVFR1, SIMDHP, false, MRS_LOWER, mvfr1_simdhp),
1567         MRS_FIELD(MVFR1, SIMDSP, false, MRS_LOWER, mvfr1_simdsp),
1568         MRS_FIELD(MVFR1, SIMDInt, false, MRS_LOWER, mvfr1_simdint),
1569         MRS_FIELD_HWCAP(MVFR1, SIMDLS, false, MRS_LOWER, mvfr1_simdls,
1570             mvfr1_simdls_caps),
1571         MRS_FIELD(MVFR1, FPDNaN, false, MRS_LOWER, mvfr1_fpdnan),
1572         MRS_FIELD(MVFR1, FPFtZ, false, MRS_LOWER, mvfr1_fpftz),
1573         MRS_FIELD_END,
1574 };
1575 #endif /* COMPAT_FREEBSD32 */
1576
1577 struct mrs_user_reg {
1578         u_int           reg;
1579         u_int           CRm;
1580         u_int           Op2;
1581         size_t          offset;
1582         struct mrs_field *fields;
1583 };
1584
1585 #define USER_REG(name, field_name)                                      \
1586         {                                                               \
1587                 .reg = name,                                            \
1588                 .CRm = name##_CRm,                                      \
1589                 .Op2 = name##_op2,                                      \
1590                 .offset = __offsetof(struct cpu_desc, field_name),      \
1591                 .fields = field_name##_fields,                          \
1592         }
1593 static struct mrs_user_reg user_regs[] = {
1594         USER_REG(ID_AA64DFR0_EL1, id_aa64dfr0),
1595
1596         USER_REG(ID_AA64ISAR0_EL1, id_aa64isar0),
1597         USER_REG(ID_AA64ISAR1_EL1, id_aa64isar1),
1598
1599         USER_REG(ID_AA64MMFR0_EL1, id_aa64mmfr0),
1600         USER_REG(ID_AA64MMFR1_EL1, id_aa64mmfr1),
1601         USER_REG(ID_AA64MMFR2_EL1, id_aa64mmfr2),
1602
1603         USER_REG(ID_AA64PFR0_EL1, id_aa64pfr0),
1604         USER_REG(ID_AA64PFR1_EL1, id_aa64pfr1),
1605 #ifdef COMPAT_FREEBSD32
1606         USER_REG(ID_ISAR5_EL1, id_isar5),
1607
1608         USER_REG(MVFR0_EL1, mvfr0),
1609         USER_REG(MVFR1_EL1, mvfr1),
1610 #endif /* COMPAT_FREEBSD32 */
1611 };
1612
1613 #define CPU_DESC_FIELD(desc, idx)                                       \
1614     *(uint64_t *)((char *)&(desc) + user_regs[(idx)].offset)
1615
1616 static int
1617 user_mrs_handler(vm_offset_t va, uint32_t insn, struct trapframe *frame,
1618     uint32_t esr)
1619 {
1620         uint64_t value;
1621         int CRm, Op2, i, reg;
1622
1623         if ((insn & MRS_MASK) != MRS_VALUE)
1624                 return (0);
1625
1626         /*
1627          * We only emulate Op0 == 3, Op1 == 0, CRn == 0, CRm == {0, 4-7}.
1628          * These are in the EL1 CPU identification space.
1629          * CRm == 0 holds MIDR_EL1, MPIDR_EL1, and REVID_EL1.
1630          * CRm == {4-7} holds the ID_AA64 registers.
1631          *
1632          * For full details see the ARMv8 ARM (ARM DDI 0487C.a)
1633          * Table D9-2 System instruction encodings for non-Debug System
1634          * register accesses.
1635          */
1636         if (mrs_Op0(insn) != 3 || mrs_Op1(insn) != 0 || mrs_CRn(insn) != 0)
1637                 return (0);
1638
1639         CRm = mrs_CRm(insn);
1640         if (CRm > 7 || (CRm < 4 && CRm != 0))
1641                 return (0);
1642
1643         Op2 = mrs_Op2(insn);
1644         value = 0;
1645
1646         for (i = 0; i < nitems(user_regs); i++) {
1647                 if (user_regs[i].CRm == CRm && user_regs[i].Op2 == Op2) {
1648                         value = CPU_DESC_FIELD(user_cpu_desc, i);
1649                         break;
1650                 }
1651         }
1652
1653         if (CRm == 0) {
1654                 switch (Op2) {
1655                 case 0:
1656                         value = READ_SPECIALREG(midr_el1);
1657                         break;
1658                 case 5:
1659                         value = READ_SPECIALREG(mpidr_el1);
1660                         break;
1661                 case 6:
1662                         value = READ_SPECIALREG(revidr_el1);
1663                         break;
1664                 default:
1665                         return (0);
1666                 }
1667         }
1668
1669         /*
1670          * We will handle this instruction, move to the next so we
1671          * don't trap here again.
1672          */
1673         frame->tf_elr += INSN_SIZE;
1674
1675         reg = MRS_REGISTER(insn);
1676         /* If reg is 31 then write to xzr, i.e. do nothing */
1677         if (reg == 31)
1678                 return (1);
1679
1680         if (reg < nitems(frame->tf_x))
1681                 frame->tf_x[reg] = value;
1682         else if (reg == 30)
1683                 frame->tf_lr = value;
1684
1685         return (1);
1686 }
1687
1688 bool
1689 extract_user_id_field(u_int reg, u_int field_shift, uint8_t *val)
1690 {
1691         uint64_t value;
1692         int i;
1693
1694         for (i = 0; i < nitems(user_regs); i++) {
1695                 if (user_regs[i].reg == reg) {
1696                         value = CPU_DESC_FIELD(user_cpu_desc, i);
1697                         *val = value >> field_shift;
1698                         return (true);
1699                 }
1700         }
1701
1702         return (false);
1703 }
1704
1705 bool
1706 get_kernel_reg(u_int reg, uint64_t *val)
1707 {
1708         int i;
1709
1710         for (i = 0; i < nitems(user_regs); i++) {
1711                 if (user_regs[i].reg == reg) {
1712                         *val = CPU_DESC_FIELD(kern_cpu_desc, i);
1713                         return (true);
1714                 }
1715         }
1716
1717         return (false);
1718 }
1719
1720 /*
1721  * Compares two field values that may be signed or unsigned.
1722  * Returns:
1723  *  < 0 when a is less than b
1724  *  = 0 when a equals b
1725  *  > 0 when a is greater than b
1726  */
1727 static int
1728 mrs_field_cmp(uint64_t a, uint64_t b, u_int shift, int width, bool sign)
1729 {
1730         uint64_t mask;
1731
1732         KASSERT(width > 0 && width < 64, ("%s: Invalid width %d", __func__,
1733             width));
1734
1735         mask = (1ul << width) - 1;
1736         /* Move the field to the lower bits */
1737         a = (a >> shift) & mask;
1738         b = (b >> shift) & mask;
1739
1740         if (sign) {
1741                 /*
1742                  * The field is signed. Toggle the upper bit so the comparison
1743                  * works on unsigned values as this makes positive numbers,
1744                  * i.e. those with a 0 bit, larger than negative numbers,
1745                  * i.e. those with a 1 bit, in an unsigned comparison.
1746                  */
1747                 a ^= 1ul << (width - 1);
1748                 b ^= 1ul << (width - 1);
1749         }
1750
1751         return (a - b);
1752 }
1753
1754 static uint64_t
1755 update_lower_register(uint64_t val, uint64_t new_val, u_int shift,
1756     int width, bool sign)
1757 {
1758         uint64_t mask;
1759
1760         KASSERT(width > 0 && width < 64, ("%s: Invalid width %d", __func__,
1761             width));
1762
1763         /*
1764          * If the new value is less than the existing value update it.
1765          */
1766         if (mrs_field_cmp(new_val, val, shift, width, sign) < 0) {
1767                 mask = (1ul << width) - 1;
1768                 val &= ~(mask << shift);
1769                 val |= new_val & (mask << shift);
1770         }
1771
1772         return (val);
1773 }
1774
1775 void
1776 update_special_regs(u_int cpu)
1777 {
1778         struct mrs_field *fields;
1779         uint64_t user_reg, kern_reg, value;
1780         int i, j;
1781
1782         if (cpu == 0) {
1783                 /* Create a user visible cpu description with safe values */
1784                 memset(&user_cpu_desc, 0, sizeof(user_cpu_desc));
1785                 /* Safe values for these registers */
1786                 user_cpu_desc.id_aa64pfr0 = ID_AA64PFR0_AdvSIMD_NONE |
1787                     ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 |
1788                     ID_AA64PFR0_EL0_64;
1789                 user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8;
1790         }
1791
1792         for (i = 0; i < nitems(user_regs); i++) {
1793                 value = CPU_DESC_FIELD(cpu_desc[cpu], i);
1794                 if (cpu == 0) {
1795                         kern_reg = value;
1796                         user_reg = value;
1797                 } else {
1798                         kern_reg = CPU_DESC_FIELD(kern_cpu_desc, i);
1799                         user_reg = CPU_DESC_FIELD(user_cpu_desc, i);
1800                 }
1801
1802                 fields = user_regs[i].fields;
1803                 for (j = 0; fields[j].type != 0; j++) {
1804                         switch (fields[j].type & MRS_TYPE_MASK) {
1805                         case MRS_EXACT:
1806                                 user_reg &= ~(0xful << fields[j].shift);
1807                                 user_reg |=
1808                                     (uint64_t)MRS_EXACT_FIELD(fields[j].type) <<
1809                                     fields[j].shift;
1810                                 break;
1811                         case MRS_LOWER:
1812                                 user_reg = update_lower_register(user_reg,
1813                                     value, fields[j].shift, 4, fields[j].sign);
1814                                 break;
1815                         default:
1816                                 panic("Invalid field type: %d", fields[j].type);
1817                         }
1818                         kern_reg = update_lower_register(kern_reg, value,
1819                             fields[j].shift, 4, fields[j].sign);
1820                 }
1821
1822                 CPU_DESC_FIELD(kern_cpu_desc, i) = kern_reg;
1823                 CPU_DESC_FIELD(user_cpu_desc, i) = user_reg;
1824         }
1825 }
1826
1827 /* HWCAP */
1828 bool __read_frequently lse_supported = false;
1829
1830 bool __read_frequently icache_aliasing = false;
1831 bool __read_frequently icache_vmid = false;
1832
1833 int64_t dcache_line_size;       /* The minimum D cache line size */
1834 int64_t icache_line_size;       /* The minimum I cache line size */
1835 int64_t idcache_line_size;      /* The minimum cache line size */
1836
1837 /*
1838  * Find the values to export to userspace as AT_HWCAP and AT_HWCAP2.
1839  */
1840 static void
1841 parse_cpu_features(void)
1842 {
1843         struct mrs_field_hwcap *hwcaps;
1844         struct mrs_field *fields;
1845         uint64_t min, reg;
1846         int i, j, k;
1847
1848         for (i = 0; i < nitems(user_regs); i++) {
1849                 reg = CPU_DESC_FIELD(user_cpu_desc, i);
1850                 fields = user_regs[i].fields;
1851                 for (j = 0; fields[j].type != 0; j++) {
1852                         hwcaps = fields[j].hwcaps;
1853                         if (hwcaps == NULL)
1854                                 continue;
1855
1856                         for (k = 0; hwcaps[k].hwcap != NULL; k++) {
1857                                 min = hwcaps[k].min;
1858
1859                                 /*
1860                                  * If the field is greater than the minimum
1861                                  * value we can set the hwcap;
1862                                  */
1863                                 if (mrs_field_cmp(reg, min, fields[j].shift,
1864                                     4, fields[j].sign) >= 0) {
1865                                         *hwcaps[k].hwcap |= hwcaps[k].hwcap_val;
1866                                 }
1867                         }
1868                 }
1869         }
1870 }
1871
1872 static void
1873 identify_cpu_sysinit(void *dummy __unused)
1874 {
1875         int cpu;
1876         bool dic, idc;
1877
1878         dic = (allow_dic != 0);
1879         idc = (allow_idc != 0);
1880
1881         CPU_FOREACH(cpu) {
1882                 check_cpu_regs(cpu);
1883                 if (cpu != 0)
1884                         update_special_regs(cpu);
1885
1886                 if (CTR_DIC_VAL(cpu_desc[cpu].ctr) == 0)
1887                         dic = false;
1888                 if (CTR_IDC_VAL(cpu_desc[cpu].ctr) == 0)
1889                         idc = false;
1890         }
1891
1892         /* Find the values to export to userspace as AT_HWCAP and AT_HWCAP2 */
1893         parse_cpu_features();
1894
1895 #ifdef COMPAT_FREEBSD32
1896         /* Set the default caps and any that need to check multiple fields */
1897         elf32_hwcap |= parse_cpu_features_hwcap32();
1898 #endif
1899
1900         if (dic && idc) {
1901                 arm64_icache_sync_range = &arm64_dic_idc_icache_sync_range;
1902                 if (bootverbose)
1903                         printf("Enabling DIC & IDC ICache sync\n");
1904         } else if (idc) {
1905                 arm64_icache_sync_range = &arm64_idc_aliasing_icache_sync_range;
1906                 if (bootverbose)
1907                         printf("Enabling IDC ICache sync\n");
1908         }
1909
1910         if ((elf_hwcap & HWCAP_ATOMICS) != 0) {
1911                 lse_supported = true;
1912                 if (bootverbose)
1913                         printf("Enabling LSE atomics in the kernel\n");
1914         }
1915 #ifdef LSE_ATOMICS
1916         if (!lse_supported)
1917                 panic("CPU does not support LSE atomic instructions");
1918 #endif
1919
1920         install_undef_handler(true, user_mrs_handler);
1921 }
1922 SYSINIT(identify_cpu, SI_SUB_CPU, SI_ORDER_MIDDLE, identify_cpu_sysinit, NULL);
1923
1924 static void
1925 cpu_features_sysinit(void *dummy __unused)
1926 {
1927         struct sbuf sb;
1928         u_int cpu;
1929
1930         CPU_FOREACH(cpu)
1931                 print_cpu_features(cpu);
1932
1933         /* Fill in cpu_model for the hw.model sysctl */
1934         sbuf_new(&sb, cpu_model, sizeof(cpu_model), SBUF_FIXEDLEN);
1935         print_cpu_midr(&sb, 0);
1936
1937         sbuf_finish(&sb);
1938         sbuf_delete(&sb);
1939 }
1940 /* Log features before APs are released and start printing to the dmesg. */
1941 SYSINIT(cpu_features, SI_SUB_SMP - 1, SI_ORDER_ANY, cpu_features_sysinit, NULL);
1942
1943 #ifdef COMPAT_FREEBSD32
1944 static u_long
1945 parse_cpu_features_hwcap32(void)
1946 {
1947         u_long hwcap = HWCAP32_DEFAULT;
1948
1949         if ((MVFR1_SIMDLS_VAL(user_cpu_desc.mvfr1) >=
1950              MVFR1_SIMDLS_IMPL) &&
1951             (MVFR1_SIMDInt_VAL(user_cpu_desc.mvfr1) >=
1952              MVFR1_SIMDInt_IMPL) &&
1953             (MVFR1_SIMDSP_VAL(user_cpu_desc.mvfr1) >=
1954              MVFR1_SIMDSP_IMPL))
1955                 hwcap |= HWCAP32_NEON;
1956
1957         return (hwcap);
1958 }
1959 #endif /* COMPAT_FREEBSD32 */
1960
1961 static void
1962 print_ctr_fields(struct sbuf *sb, uint64_t reg, void *arg)
1963 {
1964
1965         sbuf_printf(sb, "%u byte D-cacheline,", CTR_DLINE_SIZE(reg));
1966         sbuf_printf(sb, "%u byte I-cacheline,", CTR_ILINE_SIZE(reg));
1967         reg &= ~(CTR_DLINE_MASK | CTR_ILINE_MASK);
1968
1969         switch(CTR_L1IP_VAL(reg)) {
1970         case CTR_L1IP_VPIPT:
1971                 sbuf_printf(sb, "VPIPT");
1972                 break;
1973         case CTR_L1IP_AIVIVT:
1974                 sbuf_printf(sb, "AIVIVT");
1975                 break;
1976         case CTR_L1IP_VIPT:
1977                 sbuf_printf(sb, "VIPT");
1978                 break;
1979         case CTR_L1IP_PIPT:
1980                 sbuf_printf(sb, "PIPT");
1981                 break;
1982         }
1983         sbuf_printf(sb, " ICache,");
1984         reg &= ~CTR_L1IP_MASK;
1985
1986         sbuf_printf(sb, "%d byte ERG,", CTR_ERG_SIZE(reg));
1987         sbuf_printf(sb, "%d byte CWG", CTR_CWG_SIZE(reg));
1988         reg &= ~(CTR_ERG_MASK | CTR_CWG_MASK);
1989
1990         if (CTR_IDC_VAL(reg) != 0)
1991                 sbuf_printf(sb, ",IDC");
1992         if (CTR_DIC_VAL(reg) != 0)
1993                 sbuf_printf(sb, ",DIC");
1994         reg &= ~(CTR_IDC_MASK | CTR_DIC_MASK);
1995         reg &= ~CTR_RES1;
1996
1997         if (reg != 0)
1998                 sbuf_printf(sb, ",%lx", reg);
1999 }
2000
2001 static void
2002 print_register(struct sbuf *sb, const char *reg_name, uint64_t reg,
2003     void (*print_fields)(struct sbuf *, uint64_t, void *), void *arg)
2004 {
2005
2006         sbuf_printf(sb, "%29s = <", reg_name);
2007
2008         print_fields(sb, reg, arg);
2009
2010         sbuf_finish(sb);
2011         printf("%s>\n", sbuf_data(sb));
2012         sbuf_clear(sb);
2013 }
2014
2015 static void
2016 print_id_fields(struct sbuf *sb, uint64_t reg, void *arg)
2017 {
2018         struct mrs_field *fields = arg;
2019         struct mrs_field_value *fv;
2020         int field, i, j, printed;
2021
2022 #define SEP_STR ((printed++) == 0) ? "" : ","
2023         printed = 0;
2024         for (i = 0; fields[i].type != 0; i++) {
2025                 fv = fields[i].values;
2026
2027                 /* TODO: Handle with an unknown message */
2028                 if (fv == NULL)
2029                         continue;
2030
2031                 field = (reg & fields[i].mask) >> fields[i].shift;
2032                 for (j = 0; fv[j].desc != NULL; j++) {
2033                         if ((fv[j].value >> fields[i].shift) != field)
2034                                 continue;
2035
2036                         if (fv[j].desc[0] != '\0')
2037                                 sbuf_printf(sb, "%s%s", SEP_STR, fv[j].desc);
2038                         break;
2039                 }
2040                 if (fv[j].desc == NULL)
2041                         sbuf_printf(sb, "%sUnknown %s(%x)", SEP_STR,
2042                             fields[i].name, field);
2043
2044                 reg &= ~(0xful << fields[i].shift);
2045         }
2046
2047         if (reg != 0)
2048                 sbuf_printf(sb, "%s%#lx", SEP_STR, reg);
2049 #undef SEP_STR
2050 }
2051
2052 static void
2053 print_id_register(struct sbuf *sb, const char *reg_name, uint64_t reg,
2054     struct mrs_field *fields)
2055 {
2056
2057         print_register(sb, reg_name, reg, print_id_fields, fields);
2058 }
2059
2060 static void
2061 print_cpu_midr(struct sbuf *sb, u_int cpu)
2062 {
2063         const struct cpu_parts *cpu_partsp;
2064         const char *cpu_impl_name;
2065         const char *cpu_part_name;
2066         u_int midr;
2067         u_int impl_id;
2068         u_int part_id;
2069
2070         midr = pcpu_find(cpu)->pc_midr;
2071
2072         cpu_impl_name = NULL;
2073         cpu_partsp = NULL;
2074         impl_id = CPU_IMPL(midr);
2075         for (int i = 0; cpu_implementers[i].impl_name != NULL; i++) {
2076                 if (impl_id == cpu_implementers[i].impl_id) {
2077                         cpu_impl_name = cpu_implementers[i].impl_name;
2078                         cpu_partsp = cpu_implementers[i].cpu_parts;
2079                         break;
2080                 }
2081         }
2082         /* Unknown implementer, so unknown part */
2083         if (cpu_impl_name == NULL) {
2084                 sbuf_printf(sb, "Unknown Implementer (midr: %08x)", midr);
2085                 return;
2086         }
2087
2088         KASSERT(cpu_partsp != NULL, ("%s: No parts table for implementer %s",
2089             __func__, cpu_impl_name));
2090
2091         cpu_part_name = NULL;
2092         part_id = CPU_PART(midr);
2093         for (int i = 0; cpu_partsp[i].part_name != NULL; i++) {
2094                 if (part_id == cpu_partsp[i].part_id) {
2095                         cpu_part_name = cpu_partsp[i].part_name;
2096                         break;
2097                 }
2098         }
2099         /* Known Implementer, Unknown part */
2100         if (cpu_part_name == NULL) {
2101                 sbuf_printf(sb, "%s Unknown CPU r%dp%d (midr: %08x)",
2102                     cpu_impl_name, CPU_VAR(midr), CPU_REV(midr), midr);
2103                 return;
2104         }
2105
2106         sbuf_printf(sb, "%s %s r%dp%d", cpu_impl_name,
2107             cpu_part_name, CPU_VAR(midr), CPU_REV(midr));
2108 }
2109
2110 static void
2111 print_cpu_cache(u_int cpu, struct sbuf *sb, uint64_t ccs, bool icache,
2112     bool unified)
2113 {
2114         size_t cache_size;
2115         size_t line_size;
2116
2117         /* LineSize is Log2(S) - 4. */
2118         line_size = 1 << ((ccs & CCSIDR_LineSize_MASK) + 4);
2119         /*
2120          * Calculate cache size (sets * ways * line size).  There are different
2121          * formats depending on the FEAT_CCIDX bit in ID_AA64MMFR2 feature
2122          * register.
2123          */
2124         if ((cpu_desc[cpu].id_aa64mmfr2 & ID_AA64MMFR2_CCIDX_64))
2125                 cache_size = (CCSIDR_NSETS_64(ccs) + 1) *
2126                     (CCSIDR_ASSOC_64(ccs) + 1);
2127         else
2128                 cache_size = (CCSIDR_NSETS(ccs) + 1) * (CCSIDR_ASSOC(ccs) + 1);
2129
2130         cache_size *= line_size;
2131         sbuf_printf(sb, "%zuKB (%s)", cache_size / 1024,
2132             icache ? "instruction" : unified ? "unified" : "data");
2133 }
2134
2135 static void
2136 print_cpu_caches(struct sbuf *sb, u_int cpu)
2137 {
2138         /* Print out each cache combination */
2139         uint64_t clidr;
2140         int i = 1;
2141         clidr = cpu_desc[cpu].clidr;
2142
2143         for (i = 0; (clidr & CLIDR_CTYPE_MASK) != 0; i++, clidr >>= 3) {
2144                 int j = 0;
2145                 int ctype_m = (clidr & CLIDR_CTYPE_MASK);
2146
2147                 sbuf_printf(sb, " L%d cache: ", i + 1);
2148                 if ((clidr & CLIDR_CTYPE_IO)) {
2149                         print_cpu_cache(cpu, sb, cpu_desc[cpu].ccsidr[i][j++],
2150                             true, false);
2151                         /* If there's more, add to the line. */
2152                         if ((ctype_m & ~CLIDR_CTYPE_IO) != 0)
2153                                 sbuf_printf(sb, ", ");
2154                 }
2155                 if ((ctype_m & ~CLIDR_CTYPE_IO) != 0) {
2156                         print_cpu_cache(cpu, sb, cpu_desc[cpu].ccsidr[i][j],
2157                             false, (clidr & CLIDR_CTYPE_UNIFIED));
2158                 }
2159                 sbuf_printf(sb, "\n");
2160
2161         }
2162         sbuf_finish(sb);
2163         printf("%s", sbuf_data(sb));
2164 }
2165
2166 static void
2167 print_cpu_features(u_int cpu)
2168 {
2169         struct sbuf *sb;
2170
2171         sb = sbuf_new_auto();
2172         sbuf_printf(sb, "CPU%3u: ", cpu);
2173         print_cpu_midr(sb, cpu);
2174
2175         sbuf_cat(sb, " affinity:");
2176         switch(cpu_aff_levels) {
2177         default:
2178         case 4:
2179                 sbuf_printf(sb, " %2d", CPU_AFF3(cpu_desc[cpu].mpidr));
2180                 /* FALLTHROUGH */
2181         case 3:
2182                 sbuf_printf(sb, " %2d", CPU_AFF2(cpu_desc[cpu].mpidr));
2183                 /* FALLTHROUGH */
2184         case 2:
2185                 sbuf_printf(sb, " %2d", CPU_AFF1(cpu_desc[cpu].mpidr));
2186                 /* FALLTHROUGH */
2187         case 1:
2188         case 0: /* On UP this will be zero */
2189                 sbuf_printf(sb, " %2d", CPU_AFF0(cpu_desc[cpu].mpidr));
2190                 break;
2191         }
2192         sbuf_finish(sb);
2193         printf("%s\n", sbuf_data(sb));
2194         sbuf_clear(sb);
2195
2196         /*
2197          * There is a hardware errata where, if one CPU is performing a TLB
2198          * invalidation while another is performing a store-exclusive the
2199          * store-exclusive may return the wrong status. A workaround seems
2200          * to be to use an IPI to invalidate on each CPU, however given the
2201          * limited number of affected units (pass 1.1 is the evaluation
2202          * hardware revision), and the lack of information from Cavium
2203          * this has not been implemented.
2204          *
2205          * At the time of writing this the only information is from:
2206          * https://lkml.org/lkml/2016/8/4/722
2207          */
2208         /*
2209          * XXX: CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1 on its own also
2210          * triggers on pass 2.0+.
2211          */
2212         if (cpu == 0 && CPU_VAR(PCPU_GET(midr)) == 0 &&
2213             CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1)
2214                 printf("WARNING: ThunderX Pass 1.1 detected.\nThis has known "
2215                     "hardware bugs that may cause the incorrect operation of "
2216                     "atomic operations.\n");
2217
2218 #define SHOULD_PRINT_REG(_reg)                                          \
2219     (cpu == 0 || cpu_desc[cpu]._reg != cpu_desc[cpu - 1]._reg)
2220
2221         /* Cache Type Register */
2222         if (SHOULD_PRINT_REG(ctr)) {
2223                 print_register(sb, "Cache Type",
2224                     cpu_desc[cpu].ctr, print_ctr_fields, NULL);
2225         }
2226
2227         /* AArch64 Instruction Set Attribute Register 0 */
2228         if (SHOULD_PRINT_REG(id_aa64isar0))
2229                 print_id_register(sb, "Instruction Set Attributes 0",
2230                     cpu_desc[cpu].id_aa64isar0, id_aa64isar0_fields);
2231
2232         /* AArch64 Instruction Set Attribute Register 1 */
2233         if (SHOULD_PRINT_REG(id_aa64isar1))
2234                 print_id_register(sb, "Instruction Set Attributes 1",
2235                     cpu_desc[cpu].id_aa64isar1, id_aa64isar1_fields);
2236
2237         /* AArch64 Instruction Set Attribute Register 2 */
2238         if (SHOULD_PRINT_REG(id_aa64isar2))
2239                 print_id_register(sb, "Instruction Set Attributes 2",
2240                     cpu_desc[cpu].id_aa64isar2, id_aa64isar2_fields);
2241
2242         /* AArch64 Processor Feature Register 0 */
2243         if (SHOULD_PRINT_REG(id_aa64pfr0))
2244                 print_id_register(sb, "Processor Features 0",
2245                     cpu_desc[cpu].id_aa64pfr0, id_aa64pfr0_fields);
2246
2247         /* AArch64 Processor Feature Register 1 */
2248         if (SHOULD_PRINT_REG(id_aa64pfr1))
2249                 print_id_register(sb, "Processor Features 1",
2250                     cpu_desc[cpu].id_aa64pfr1, id_aa64pfr1_fields);
2251
2252         /* AArch64 Memory Model Feature Register 0 */
2253         if (SHOULD_PRINT_REG(id_aa64mmfr0))
2254                 print_id_register(sb, "Memory Model Features 0",
2255                     cpu_desc[cpu].id_aa64mmfr0, id_aa64mmfr0_fields);
2256
2257         /* AArch64 Memory Model Feature Register 1 */
2258         if (SHOULD_PRINT_REG(id_aa64mmfr1))
2259                 print_id_register(sb, "Memory Model Features 1",
2260                     cpu_desc[cpu].id_aa64mmfr1, id_aa64mmfr1_fields);
2261
2262         /* AArch64 Memory Model Feature Register 2 */
2263         if (SHOULD_PRINT_REG(id_aa64mmfr2))
2264                 print_id_register(sb, "Memory Model Features 2",
2265                     cpu_desc[cpu].id_aa64mmfr2, id_aa64mmfr2_fields);
2266
2267         /* AArch64 Debug Feature Register 0 */
2268         if (SHOULD_PRINT_REG(id_aa64dfr0))
2269                 print_id_register(sb, "Debug Features 0",
2270                     cpu_desc[cpu].id_aa64dfr0, id_aa64dfr0_fields);
2271
2272         /* AArch64 Memory Model Feature Register 1 */
2273         if (SHOULD_PRINT_REG(id_aa64dfr1))
2274                 print_id_register(sb, "Debug Features 1",
2275                     cpu_desc[cpu].id_aa64dfr1, id_aa64dfr1_fields);
2276
2277         /* AArch64 Auxiliary Feature Register 0 */
2278         if (SHOULD_PRINT_REG(id_aa64afr0))
2279                 print_id_register(sb, "Auxiliary Features 0",
2280                     cpu_desc[cpu].id_aa64afr0, id_aa64afr0_fields);
2281
2282         /* AArch64 Auxiliary Feature Register 1 */
2283         if (SHOULD_PRINT_REG(id_aa64afr1))
2284                 print_id_register(sb, "Auxiliary Features 1",
2285                     cpu_desc[cpu].id_aa64afr1, id_aa64afr1_fields);
2286
2287         /* AArch64 SVE Feature Register 0 */
2288         if (cpu_desc[cpu].have_sve) {
2289                 if (SHOULD_PRINT_REG(id_aa64zfr0) ||
2290                     !cpu_desc[cpu - 1].have_sve) {
2291                         print_id_register(sb, "SVE Features 0",
2292                             cpu_desc[cpu].id_aa64zfr0, id_aa64zfr0_fields);
2293                 }
2294         }
2295
2296 #ifdef COMPAT_FREEBSD32
2297         /* AArch32 Instruction Set Attribute Register 5 */
2298         if (SHOULD_PRINT_REG(id_isar5))
2299                 print_id_register(sb, "AArch32 Instruction Set Attributes 5",
2300                      cpu_desc[cpu].id_isar5, id_isar5_fields);
2301
2302         /* AArch32 Media and VFP Feature Register 0 */
2303         if (SHOULD_PRINT_REG(mvfr0))
2304                 print_id_register(sb, "AArch32 Media and VFP Features 0",
2305                      cpu_desc[cpu].mvfr0, mvfr0_fields);
2306
2307         /* AArch32 Media and VFP Feature Register 1 */
2308         if (SHOULD_PRINT_REG(mvfr1))
2309                 print_id_register(sb, "AArch32 Media and VFP Features 1",
2310                      cpu_desc[cpu].mvfr1, mvfr1_fields);
2311 #endif
2312         if (bootverbose)
2313                 print_cpu_caches(sb, cpu);
2314
2315         sbuf_delete(sb);
2316         sb = NULL;
2317 #undef SHOULD_PRINT_REG
2318 #undef SEP_STR
2319 }
2320
2321 void
2322 identify_cache(uint64_t ctr)
2323 {
2324
2325         /* Identify the L1 cache type */
2326         switch (CTR_L1IP_VAL(ctr)) {
2327         case CTR_L1IP_PIPT:
2328                 break;
2329         case CTR_L1IP_VPIPT:
2330                 icache_vmid = true;
2331                 break;
2332         default:
2333         case CTR_L1IP_VIPT:
2334                 icache_aliasing = true;
2335                 break;
2336         }
2337
2338         if (dcache_line_size == 0) {
2339                 KASSERT(icache_line_size == 0, ("%s: i-cacheline size set: %ld",
2340                     __func__, icache_line_size));
2341
2342                 /* Get the D cache line size */
2343                 dcache_line_size = CTR_DLINE_SIZE(ctr);
2344                 /* And the same for the I cache */
2345                 icache_line_size = CTR_ILINE_SIZE(ctr);
2346
2347                 idcache_line_size = MIN(dcache_line_size, icache_line_size);
2348         }
2349
2350         if (dcache_line_size != CTR_DLINE_SIZE(ctr)) {
2351                 printf("WARNING: D-cacheline size mismatch %ld != %d\n",
2352                     dcache_line_size, CTR_DLINE_SIZE(ctr));
2353         }
2354
2355         if (icache_line_size != CTR_ILINE_SIZE(ctr)) {
2356                 printf("WARNING: I-cacheline size mismatch %ld != %d\n",
2357                     icache_line_size, CTR_ILINE_SIZE(ctr));
2358         }
2359 }
2360
2361 void
2362 identify_cpu(u_int cpu)
2363 {
2364         uint64_t clidr;
2365
2366         /* Save affinity for current CPU */
2367         cpu_desc[cpu].mpidr = get_mpidr();
2368         CPU_AFFINITY(cpu) = cpu_desc[cpu].mpidr & CPU_AFF_MASK;
2369
2370         cpu_desc[cpu].ctr = READ_SPECIALREG(ctr_el0);
2371         cpu_desc[cpu].id_aa64dfr0 = READ_SPECIALREG(id_aa64dfr0_el1);
2372         cpu_desc[cpu].id_aa64dfr1 = READ_SPECIALREG(id_aa64dfr1_el1);
2373         cpu_desc[cpu].id_aa64isar0 = READ_SPECIALREG(id_aa64isar0_el1);
2374         cpu_desc[cpu].id_aa64isar1 = READ_SPECIALREG(id_aa64isar1_el1);
2375         cpu_desc[cpu].id_aa64isar2 = READ_SPECIALREG(id_aa64isar2_el1);
2376         cpu_desc[cpu].id_aa64mmfr0 = READ_SPECIALREG(id_aa64mmfr0_el1);
2377         cpu_desc[cpu].id_aa64mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1);
2378         cpu_desc[cpu].id_aa64mmfr2 = READ_SPECIALREG(id_aa64mmfr2_el1);
2379         cpu_desc[cpu].id_aa64pfr0 = READ_SPECIALREG(id_aa64pfr0_el1);
2380         cpu_desc[cpu].id_aa64pfr1 = READ_SPECIALREG(id_aa64pfr1_el1);
2381
2382         /*
2383          * ID_AA64ZFR0_EL1 is only valid when at least one of:
2384          *  - ID_AA64PFR0_EL1.SVE is non-zero
2385          *  - ID_AA64PFR1_EL1.SME is non-zero
2386          * In other cases it is zero, but still safe to read
2387          */
2388         cpu_desc[cpu].have_sve =
2389             (ID_AA64PFR0_SVE_VAL(cpu_desc[cpu].id_aa64pfr0) != 0);
2390         cpu_desc[cpu].id_aa64zfr0 = READ_SPECIALREG(ID_AA64ZFR0_EL1_REG);
2391
2392         cpu_desc[cpu].clidr = READ_SPECIALREG(clidr_el1);
2393
2394         clidr = cpu_desc[cpu].clidr;
2395
2396         for (int i = 0; (clidr & CLIDR_CTYPE_MASK) != 0; i++, clidr >>= 3) {
2397                 int j = 0;
2398                 if ((clidr & CLIDR_CTYPE_IO)) {
2399                         WRITE_SPECIALREG(csselr_el1,
2400                             CSSELR_Level(i) | CSSELR_InD);
2401                         cpu_desc[cpu].ccsidr[i][j++] =
2402                             READ_SPECIALREG(ccsidr_el1);
2403                 }
2404                 if ((clidr & ~CLIDR_CTYPE_IO) == 0)
2405                         continue;
2406                 WRITE_SPECIALREG(csselr_el1, CSSELR_Level(i));
2407                 cpu_desc[cpu].ccsidr[i][j] = READ_SPECIALREG(ccsidr_el1);
2408         }
2409
2410 #ifdef COMPAT_FREEBSD32
2411         /* Only read aarch32 SRs if EL0-32 is available */
2412         if (ID_AA64PFR0_EL0_VAL(cpu_desc[cpu].id_aa64pfr0) ==
2413             ID_AA64PFR0_EL0_64_32) {
2414                 cpu_desc[cpu].id_isar5 = READ_SPECIALREG(id_isar5_el1);
2415                 cpu_desc[cpu].mvfr0 = READ_SPECIALREG(mvfr0_el1);
2416                 cpu_desc[cpu].mvfr1 = READ_SPECIALREG(mvfr1_el1);
2417         }
2418 #endif
2419 }
2420
2421 static void
2422 check_cpu_regs(u_int cpu)
2423 {
2424
2425         switch (cpu_aff_levels) {
2426         case 0:
2427                 if (CPU_AFF0(cpu_desc[cpu].mpidr) !=
2428                     CPU_AFF0(cpu_desc[0].mpidr))
2429                         cpu_aff_levels = 1;
2430                 /* FALLTHROUGH */
2431         case 1:
2432                 if (CPU_AFF1(cpu_desc[cpu].mpidr) !=
2433                     CPU_AFF1(cpu_desc[0].mpidr))
2434                         cpu_aff_levels = 2;
2435                 /* FALLTHROUGH */
2436         case 2:
2437                 if (CPU_AFF2(cpu_desc[cpu].mpidr) !=
2438                     CPU_AFF2(cpu_desc[0].mpidr))
2439                         cpu_aff_levels = 3;
2440                 /* FALLTHROUGH */
2441         case 3:
2442                 if (CPU_AFF3(cpu_desc[cpu].mpidr) !=
2443                     CPU_AFF3(cpu_desc[0].mpidr))
2444                         cpu_aff_levels = 4;
2445                 break;
2446         }
2447
2448         if (cpu_desc[cpu].ctr != cpu_desc[0].ctr) {
2449                 /*
2450                  * If the cache type register is different we may
2451                  * have a different l1 cache type.
2452                  */
2453                 identify_cache(cpu_desc[cpu].ctr);
2454         }
2455 }