sys/arm64/arm64/identcpu.c

   1 /*-
   2  * Copyright (c) 2014 Andrew Turner
   3  * Copyright (c) 2014 The FreeBSD Foundation
   4  * All rights reserved.
   5  *
   6  * Portions of this software were developed by Semihalf
   7  * under sponsorship of the FreeBSD Foundation.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  *
  30  */
  31
  32 #include <sys/param.h>
  33 #include <sys/kernel.h>
  34 #include <sys/malloc.h>
  35 #include <sys/proc.h>
  36 #include <sys/pcpu.h>
  37 #include <sys/sbuf.h>
  38 #include <sys/smp.h>
  39 #include <sys/sysctl.h>
  40 #include <sys/sysent.h>
  41 #include <sys/systm.h>
  42
  43 #include <machine/atomic.h>
  44 #include <machine/cpu.h>
  45 #include <machine/cpufunc.h>
  46 #include <machine/elf.h>
  47 #include <machine/md_var.h>
  48 #include <machine/undefined.h>
  49
  50 static MALLOC_DEFINE(M_IDENTCPU, "CPU ID", "arm64 CPU identification memory");
  51
  52 struct cpu_desc;
  53
  54 static void print_cpu_midr(struct sbuf *sb, u_int cpu);
  55 static void print_cpu_features(u_int cpu, struct cpu_desc *desc,
  56     struct cpu_desc *prev_desc);
  57 static void print_cpu_caches(struct sbuf *sb, struct cpu_desc *desc);
  58 #ifdef COMPAT_FREEBSD32
  59 static u_long parse_cpu_features_hwcap32(void);
  60 #endif
  61
  62 char machine[] = "arm64";
  63
  64 #ifdef SCTL_MASK32
  65 extern int adaptive_machine_arch;
  66 #endif
  67
  68 static SYSCTL_NODE(_machdep, OID_AUTO, cache, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
  69     "Cache management tuning");
  70
  71 static int allow_dic = 1;
  72 SYSCTL_INT(_machdep_cache, OID_AUTO, allow_dic, CTLFLAG_RDTUN, &allow_dic, 0,
  73     "Allow optimizations based on the DIC cache bit");
  74
  75 static int allow_idc = 1;
  76 SYSCTL_INT(_machdep_cache, OID_AUTO, allow_idc, CTLFLAG_RDTUN, &allow_idc, 0,
  77     "Allow optimizations based on the IDC cache bit");
  78
  79 static void check_cpu_regs(u_int cpu, struct cpu_desc *desc,
  80     struct cpu_desc *prev_desc);
  81
  82 /*
  83  * The default implementation of I-cache sync assumes we have an
  84  * aliasing cache until we know otherwise.
  85  */
  86 void (*arm64_icache_sync_range)(vm_offset_t, vm_size_t) =
  87     &arm64_aliasing_icache_sync_range;
  88
  89 static int
  90 sysctl_hw_machine(SYSCTL_HANDLER_ARGS)
  91 {
  92 #ifdef SCTL_MASK32
  93         static const char machine32[] = "arm";
  94 #endif
  95         int error;
  96
  97 #ifdef SCTL_MASK32
  98         if ((req->flags & SCTL_MASK32) != 0 && adaptive_machine_arch)
  99                 error = SYSCTL_OUT(req, machine32, sizeof(machine32));
 100         else
 101 #endif
 102                 error = SYSCTL_OUT(req, machine, sizeof(machine));
 103         return (error);
 104 }
 105
 106 SYSCTL_PROC(_hw, HW_MACHINE, machine, CTLTYPE_STRING | CTLFLAG_RD |
 107          CTLFLAG_CAPRD | CTLFLAG_MPSAFE, NULL, 0, sysctl_hw_machine, "A",
 108          "Machine class");
 109
 110 static char cpu_model[64];
 111 SYSCTL_STRING(_hw, HW_MODEL, model, CTLFLAG_RD | CTLFLAG_CAPRD,
 112         cpu_model, sizeof(cpu_model), "Machine model");
 113
 114 #define MAX_CACHES      8       /* Maximum number of caches supported
 115                                    architecturally. */
 116 /*
 117  * Per-CPU affinity as provided in MPIDR_EL1
 118  * Indexed by CPU number in logical order selected by the system.
 119  * Relevant fields can be extracted using CPU_AFFn macros,
 120  * Aff3.Aff2.Aff1.Aff0 construct a unique CPU address in the system.
 121  *
 122  * Fields used by us:
 123  * Aff1 - Cluster number
 124  * Aff0 - CPU number in Aff1 cluster
 125  */
 126 uint64_t __cpu_affinity[MAXCPU];
 127 static u_int cpu_aff_levels;
 128
 129 struct cpu_desc {
 130         uint64_t        mpidr;
 131         uint64_t        id_aa64afr0;
 132         uint64_t        id_aa64afr1;
 133         uint64_t        id_aa64dfr0;
 134         uint64_t        id_aa64dfr1;
 135         uint64_t        id_aa64isar0;
 136         uint64_t        id_aa64isar1;
 137         uint64_t        id_aa64isar2;
 138         uint64_t        id_aa64mmfr0;
 139         uint64_t        id_aa64mmfr1;
 140         uint64_t        id_aa64mmfr2;
 141 #ifdef NOTYET
 142         uint64_t        id_aa64mmfr3;
 143         uint64_t        id_aa64mmfr4;
 144 #endif
 145         uint64_t        id_aa64pfr0;
 146         uint64_t        id_aa64pfr1;
 147 #ifdef NOTYET
 148         uint64_t        id_aa64pfr2;
 149 #endif
 150         uint64_t        id_aa64zfr0;
 151         uint64_t        ctr;
 152 #ifdef COMPAT_FREEBSD32
 153         uint64_t        id_isar5;
 154         uint64_t        mvfr0;
 155         uint64_t        mvfr1;
 156 #endif
 157         uint64_t        clidr;
 158         uint32_t        ccsidr[MAX_CACHES][2]; /* 2 possible types. */
 159         bool            have_sve;
 160 };
 161
 162 static struct cpu_desc cpu_desc0;
 163 static struct cpu_desc *cpu_desc;
 164 static struct cpu_desc kern_cpu_desc;
 165 static struct cpu_desc user_cpu_desc;
 166 static struct cpu_desc l_user_cpu_desc;
 167
 168 static struct cpu_desc *
 169 get_cpu_desc(u_int cpu)
 170 {
 171         /* The cpu_desc for CPU 0 is used before the allocator is ready. */
 172         if (cpu == 0)
 173                 return (&cpu_desc0);
 174
 175         MPASS(cpu_desc != NULL);
 176         return (&cpu_desc[cpu - 1]);
 177 }
 178
 179 struct cpu_parts {
 180         u_int           part_id;
 181         const char      *part_name;
 182 };
 183 #define CPU_PART_NONE   { 0, NULL }
 184
 185 struct cpu_implementers {
 186         u_int                   impl_id;
 187         const char              *impl_name;
 188         /*
 189          * Part number is implementation defined
 190          * so each vendor will have its own set of values and names.
 191          */
 192         const struct cpu_parts  *cpu_parts;
 193 };
 194 #define CPU_IMPLEMENTER_NONE    { 0, NULL, NULL }
 195
 196 /*
 197  * Per-implementer table of (PartNum, CPU Name) pairs.
 198  */
 199 /* ARM Ltd. */
 200 static const struct cpu_parts cpu_parts_arm[] = {
 201         { CPU_PART_AEM_V8, "AEMv8" },
 202         { CPU_PART_FOUNDATION, "Foundation-Model" },
 203         { CPU_PART_CORTEX_A34, "Cortex-A34" },
 204         { CPU_PART_CORTEX_A35, "Cortex-A35" },
 205         { CPU_PART_CORTEX_A53, "Cortex-A53" },
 206         { CPU_PART_CORTEX_A55, "Cortex-A55" },
 207         { CPU_PART_CORTEX_A57, "Cortex-A57" },
 208         { CPU_PART_CORTEX_A65, "Cortex-A65" },
 209         { CPU_PART_CORTEX_A65AE, "Cortex-A65AE" },
 210         { CPU_PART_CORTEX_A72, "Cortex-A72" },
 211         { CPU_PART_CORTEX_A73, "Cortex-A73" },
 212         { CPU_PART_CORTEX_A75, "Cortex-A75" },
 213         { CPU_PART_CORTEX_A76, "Cortex-A76" },
 214         { CPU_PART_CORTEX_A76AE, "Cortex-A76AE" },
 215         { CPU_PART_CORTEX_A77, "Cortex-A77" },
 216         { CPU_PART_CORTEX_A78, "Cortex-A78" },
 217         { CPU_PART_CORTEX_A78C, "Cortex-A78C" },
 218         { CPU_PART_CORTEX_A510, "Cortex-A510" },
 219         { CPU_PART_CORTEX_A710, "Cortex-A710" },
 220         { CPU_PART_CORTEX_A715, "Cortex-A715" },
 221         { CPU_PART_CORTEX_X1, "Cortex-X1" },
 222         { CPU_PART_CORTEX_X1C, "Cortex-X1C" },
 223         { CPU_PART_CORTEX_X2, "Cortex-X2" },
 224         { CPU_PART_CORTEX_X3, "Cortex-X3" },
 225         { CPU_PART_NEOVERSE_E1, "Neoverse-E1" },
 226         { CPU_PART_NEOVERSE_N1, "Neoverse-N1" },
 227         { CPU_PART_NEOVERSE_N2, "Neoverse-N2" },
 228         { CPU_PART_NEOVERSE_V1, "Neoverse-V1" },
 229         { CPU_PART_NEOVERSE_V2, "Neoverse-V2" },
 230         CPU_PART_NONE,
 231 };
 232
 233 /* Cavium */
 234 static const struct cpu_parts cpu_parts_cavium[] = {
 235         { CPU_PART_THUNDERX, "ThunderX" },
 236         { CPU_PART_THUNDERX2, "ThunderX2" },
 237         CPU_PART_NONE,
 238 };
 239
 240 /* APM / Ampere */
 241 static const struct cpu_parts cpu_parts_apm[] = {
 242         { CPU_PART_EMAG8180, "eMAG 8180" },
 243         CPU_PART_NONE,
 244 };
 245
 246 /* Qualcomm */
 247 static const struct cpu_parts cpu_parts_qcom[] = {
 248         { CPU_PART_KRYO400_GOLD, "Kryo 400 Gold" },
 249         { CPU_PART_KRYO400_SILVER, "Kryo 400 Silver" },
 250         CPU_PART_NONE,
 251 };
 252
 253 /* Unknown */
 254 static const struct cpu_parts cpu_parts_none[] = {
 255         CPU_PART_NONE,
 256 };
 257
 258 /*
 259  * Implementers table.
 260  */
 261 const struct cpu_implementers cpu_implementers[] = {
 262         { CPU_IMPL_AMPERE,      "Ampere",       cpu_parts_none },
 263         { CPU_IMPL_APPLE,       "Apple",        cpu_parts_none },
 264         { CPU_IMPL_APM,         "APM",          cpu_parts_apm },
 265         { CPU_IMPL_ARM,         "ARM",          cpu_parts_arm },
 266         { CPU_IMPL_BROADCOM,    "Broadcom",     cpu_parts_none },
 267         { CPU_IMPL_CAVIUM,      "Cavium",       cpu_parts_cavium },
 268         { CPU_IMPL_DEC,         "DEC",          cpu_parts_none },
 269         { CPU_IMPL_FREESCALE,   "Freescale",    cpu_parts_none },
 270         { CPU_IMPL_FUJITSU,     "Fujitsu",      cpu_parts_none },
 271         { CPU_IMPL_INFINEON,    "IFX",          cpu_parts_none },
 272         { CPU_IMPL_INTEL,       "Intel",        cpu_parts_none },
 273         { CPU_IMPL_MARVELL,     "Marvell",      cpu_parts_none },
 274         { CPU_IMPL_NVIDIA,      "NVIDIA",       cpu_parts_none },
 275         { CPU_IMPL_QUALCOMM,    "Qualcomm",     cpu_parts_qcom },
 276         CPU_IMPLEMENTER_NONE,
 277 };
 278
 279 #define MRS_TYPE_MASK           0xf
 280 #define MRS_TYPE_FBSD_SHIFT     0
 281 #define MRS_TYPE_LNX_SHIFT      8
 282 #define MRS_INVALID             0
 283 #define MRS_EXACT               1
 284 #define MRS_EXACT_VAL(x)        (MRS_EXACT | ((x) << 4))
 285 #define MRS_EXACT_FIELD(x)      (((x) >> 4) & 0xf)
 286 #define MRS_LOWER               2
 287
 288 struct mrs_field_value {
 289         uint64_t        value;
 290         const char      *desc;
 291 };
 292
 293 #define MRS_FIELD_VALUE(_value, _desc)                                  \
 294         {                                                               \
 295                 .value = (_value),                                      \
 296                 .desc = (_desc),                                        \
 297         }
 298
 299 #define MRS_FIELD_VALUE_NONE_IMPL(_reg, _field, _none, _impl)           \
 300         MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _none, ""),         \
 301         MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _impl, #_field)
 302
 303 #define MRS_FIELD_VALUE_COUNT(_reg, _field, _desc)                      \
 304         MRS_FIELD_VALUE(0ul << _reg ## _ ## _field ## _SHIFT, "1 " _desc), \
 305         MRS_FIELD_VALUE(1ul << _reg ## _ ## _field ## _SHIFT, "2 " _desc "s"), \
 306         MRS_FIELD_VALUE(2ul << _reg ## _ ## _field ## _SHIFT, "3 " _desc "s"), \
 307         MRS_FIELD_VALUE(3ul << _reg ## _ ## _field ## _SHIFT, "4 " _desc "s"), \
 308         MRS_FIELD_VALUE(4ul << _reg ## _ ## _field ## _SHIFT, "5 " _desc "s"), \
 309         MRS_FIELD_VALUE(5ul << _reg ## _ ## _field ## _SHIFT, "6 " _desc "s"), \
 310         MRS_FIELD_VALUE(6ul << _reg ## _ ## _field ## _SHIFT, "7 " _desc "s"), \
 311         MRS_FIELD_VALUE(7ul << _reg ## _ ## _field ## _SHIFT, "8 " _desc "s"), \
 312         MRS_FIELD_VALUE(8ul << _reg ## _ ## _field ## _SHIFT, "9 " _desc "s"), \
 313         MRS_FIELD_VALUE(9ul << _reg ## _ ## _field ## _SHIFT, "10 "_desc "s"), \
 314         MRS_FIELD_VALUE(10ul<< _reg ## _ ## _field ## _SHIFT, "11 "_desc "s"), \
 315         MRS_FIELD_VALUE(11ul<< _reg ## _ ## _field ## _SHIFT, "12 "_desc "s"), \
 316         MRS_FIELD_VALUE(12ul<< _reg ## _ ## _field ## _SHIFT, "13 "_desc "s"), \
 317         MRS_FIELD_VALUE(13ul<< _reg ## _ ## _field ## _SHIFT, "14 "_desc "s"), \
 318         MRS_FIELD_VALUE(14ul<< _reg ## _ ## _field ## _SHIFT, "15 "_desc "s"), \
 319         MRS_FIELD_VALUE(15ul<< _reg ## _ ## _field ## _SHIFT, "16 "_desc "s")
 320
 321 #define MRS_FIELD_VALUE_END     { .desc = NULL }
 322
 323 struct mrs_field_hwcap {
 324         uint64_t        min;
 325         u_long          hwcap_val;
 326         u_int           hwcap_id;
 327 };
 328
 329 #define MRS_HWCAP(_hwcap_id, _val, _min)                        \
 330 {                                                               \
 331         .hwcap_id = (_hwcap_id),                                \
 332         .hwcap_val = (_val),                                    \
 333         .min = (_min),                                          \
 334 }
 335
 336 #define MRS_HWCAP_END           { .hwcap_id = 0 }
 337
 338 struct mrs_field {
 339         const char      *name;
 340         const struct mrs_field_value *values;
 341         const struct mrs_field_hwcap *hwcaps;
 342         uint64_t        mask;
 343         bool            sign;
 344         u_int           type;
 345         u_int           shift;
 346 };
 347
 348 #define MRS_FIELD_HWCAP_SPLIT(_register, _name, _sign, _fbsd_type,      \
 349     _lnx_type, _values, _hwcap)                                         \
 350         {                                                               \
 351                 .name = #_name,                                         \
 352                 .sign = (_sign),                                        \
 353                 .type = ((_fbsd_type) << MRS_TYPE_FBSD_SHIFT) |         \
 354                     ((_lnx_type) << MRS_TYPE_LNX_SHIFT),                \
 355                 .shift = _register ## _ ## _name ## _SHIFT,             \
 356                 .mask = _register ## _ ## _name ## _MASK,               \
 357                 .values = (_values),                                    \
 358                 .hwcaps = (_hwcap),                                     \
 359         }
 360
 361 #define MRS_FIELD_HWCAP(_register, _name, _sign, _type, _values, _hwcap) \
 362         MRS_FIELD_HWCAP_SPLIT(_register, _name, _sign, _type, _type,    \
 363             _values, _hwcap)
 364
 365 #define MRS_FIELD(_register, _name, _sign, _type, _values)              \
 366         MRS_FIELD_HWCAP(_register, _name, _sign, _type, _values, NULL)
 367
 368 #define MRS_FIELD_END   { .type = MRS_INVALID, }
 369
 370 /* ID_AA64AFR0_EL1 */
 371 static const struct mrs_field id_aa64afr0_fields[] = {
 372         MRS_FIELD_END,
 373 };
 374
 375
 376 /* ID_AA64AFR1_EL1 */
 377 static const struct mrs_field id_aa64afr1_fields[] = {
 378         MRS_FIELD_END,
 379 };
 380
 381
 382 /* ID_AA64DFR0_EL1 */
 383 static const struct mrs_field_value id_aa64dfr0_hpmn0[] = {
 384         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, HPMN0, CONSTR, DEFINED),
 385         MRS_FIELD_VALUE_END,
 386 };
 387
 388 static const struct mrs_field_value id_aa64dfr0_brbe[] = {
 389         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, BRBE, NONE, IMPL),
 390         MRS_FIELD_VALUE(ID_AA64DFR0_BRBE_EL3, "BRBE EL3"),
 391         MRS_FIELD_VALUE_END,
 392 };
 393
 394 static const struct mrs_field_value id_aa64dfr0_mtpmu[] = {
 395         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, MTPMU, NONE, IMPL),
 396         MRS_FIELD_VALUE(ID_AA64DFR0_MTPMU_NONE_MT_RES0, "MTPMU res0"),
 397         MRS_FIELD_VALUE_END,
 398 };
 399
 400 static const struct mrs_field_value id_aa64dfr0_tracebuffer[] = {
 401         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, TraceBuffer, NONE, IMPL),
 402         MRS_FIELD_VALUE_END,
 403 };
 404
 405 static const struct mrs_field_value id_aa64dfr0_tracefilt[] = {
 406         MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_NONE, ""),
 407         MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_8_4, "Trace v8.4"),
 408         MRS_FIELD_VALUE_END,
 409 };
 410
 411 static const struct mrs_field_value id_aa64dfr0_doublelock[] = {
 412         MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_IMPL, "DoubleLock"),
 413         MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_NONE, ""),
 414         MRS_FIELD_VALUE_END,
 415 };
 416
 417 static const struct mrs_field_value id_aa64dfr0_pmsver[] = {
 418         MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_NONE, ""),
 419         MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE, "SPE"),
 420         MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_1, "SPEv1p1"),
 421         MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_2, "SPEv1p2"),
 422         MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_3, "SPEv1p3"),
 423         MRS_FIELD_VALUE_END,
 424 };
 425
 426 static const struct mrs_field_value id_aa64dfr0_ctx_cmps[] = {
 427         MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, CTX_CMPs, "CTX BKPT"),
 428         MRS_FIELD_VALUE_END,
 429 };
 430
 431 static const struct mrs_field_value id_aa64dfr0_wrps[] = {
 432         MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, WRPs, "Watchpoint"),
 433         MRS_FIELD_VALUE_END,
 434 };
 435
 436 static const struct mrs_field_value id_aa64dfr0_brps[] = {
 437         MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, BRPs, "Breakpoint"),
 438         MRS_FIELD_VALUE_END,
 439 };
 440
 441 static const struct mrs_field_value id_aa64dfr0_pmuver[] = {
 442         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_NONE, ""),
 443         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3, "PMUv3"),
 444         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_1, "PMUv3p1"),
 445         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_4, "PMUv3p4"),
 446         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_5, "PMUv3p5"),
 447         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_7, "PMUv3p7"),
 448         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_8, "PMUv3p8"),
 449         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_IMPL, "IMPL PMU"),
 450         MRS_FIELD_VALUE_END,
 451 };
 452
 453 static const struct mrs_field_value id_aa64dfr0_tracever[] = {
 454         MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_NONE, ""),
 455         MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_IMPL, "Trace"),
 456         MRS_FIELD_VALUE_END,
 457 };
 458
 459 static const struct mrs_field_value id_aa64dfr0_debugver[] = {
 460         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8, "Debugv8"),
 461         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_VHE, "Debugv8_VHE"),
 462         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_2, "Debugv8p2"),
 463         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_4, "Debugv8p4"),
 464         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_8, "Debugv8p8"),
 465         MRS_FIELD_VALUE_END,
 466 };
 467
 468 static const struct mrs_field id_aa64dfr0_fields[] = {
 469         MRS_FIELD(ID_AA64DFR0, HPMN0, false, MRS_EXACT, id_aa64dfr0_hpmn0),
 470         MRS_FIELD(ID_AA64DFR0, BRBE, false, MRS_EXACT, id_aa64dfr0_brbe),
 471         MRS_FIELD(ID_AA64DFR0, MTPMU, true, MRS_EXACT, id_aa64dfr0_mtpmu),
 472         MRS_FIELD(ID_AA64DFR0, TraceBuffer, false, MRS_EXACT,
 473             id_aa64dfr0_tracebuffer),
 474         MRS_FIELD(ID_AA64DFR0, TraceFilt, false, MRS_EXACT,
 475             id_aa64dfr0_tracefilt),
 476         MRS_FIELD(ID_AA64DFR0, DoubleLock, false, MRS_EXACT,
 477             id_aa64dfr0_doublelock),
 478         MRS_FIELD(ID_AA64DFR0, PMSVer, false, MRS_EXACT, id_aa64dfr0_pmsver),
 479         MRS_FIELD(ID_AA64DFR0, CTX_CMPs, false, MRS_EXACT,
 480             id_aa64dfr0_ctx_cmps),
 481         MRS_FIELD(ID_AA64DFR0, WRPs, false, MRS_LOWER, id_aa64dfr0_wrps),
 482         MRS_FIELD(ID_AA64DFR0, BRPs, false, MRS_LOWER, id_aa64dfr0_brps),
 483         MRS_FIELD(ID_AA64DFR0, PMUVer, false, MRS_EXACT, id_aa64dfr0_pmuver),
 484         MRS_FIELD(ID_AA64DFR0, TraceVer, false, MRS_EXACT,
 485             id_aa64dfr0_tracever),
 486         MRS_FIELD(ID_AA64DFR0, DebugVer, false, MRS_EXACT_VAL(0x6),
 487             id_aa64dfr0_debugver),
 488         MRS_FIELD_END,
 489 };
 490
 491
 492 /* ID_AA64DFR1_EL1 */
 493 static const struct mrs_field id_aa64dfr1_fields[] = {
 494         MRS_FIELD_END,
 495 };
 496
 497
 498 /* ID_AA64ISAR0_EL1 */
 499 static const struct mrs_field_value id_aa64isar0_rndr[] = {
 500         MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_NONE, ""),
 501         MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_IMPL, "RNG"),
 502         MRS_FIELD_VALUE_END,
 503 };
 504
 505 static const struct mrs_field_hwcap id_aa64isar0_rndr_caps[] = {
 506         MRS_HWCAP(2, HWCAP2_RNG, ID_AA64ISAR0_RNDR_IMPL),
 507         MRS_HWCAP_END
 508 };
 509
 510 static const struct mrs_field_value id_aa64isar0_tlb[] = {
 511         MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_NONE, ""),
 512         MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOS, "TLBI-OS"),
 513         MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOSR, "TLBI-OSR"),
 514         MRS_FIELD_VALUE_END,
 515 };
 516
 517 static const struct mrs_field_value id_aa64isar0_ts[] = {
 518         MRS_FIELD_VALUE(ID_AA64ISAR0_TS_NONE, ""),
 519         MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_4, "CondM-8.4"),
 520         MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_5, "CondM-8.5"),
 521         MRS_FIELD_VALUE_END,
 522 };
 523
 524 static const struct mrs_field_hwcap id_aa64isar0_ts_caps[] = {
 525         MRS_HWCAP(1, HWCAP_FLAGM, ID_AA64ISAR0_TS_CondM_8_4),
 526         MRS_HWCAP(2, HWCAP2_FLAGM2, ID_AA64ISAR0_TS_CondM_8_5),
 527         MRS_HWCAP_END
 528 };
 529
 530 static const struct mrs_field_value id_aa64isar0_fhm[] = {
 531         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, FHM, NONE, IMPL),
 532         MRS_FIELD_VALUE_END,
 533 };
 534
 535 static const struct mrs_field_hwcap id_aa64isar0_fhm_caps[] = {
 536         MRS_HWCAP(1, HWCAP_ASIMDFHM, ID_AA64ISAR0_FHM_IMPL),
 537         MRS_HWCAP_END
 538 };
 539
 540 static const struct mrs_field_value id_aa64isar0_dp[] = {
 541         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, DP, NONE, IMPL),
 542         MRS_FIELD_VALUE_END,
 543 };
 544
 545 static const struct mrs_field_hwcap id_aa64isar0_dp_caps[] = {
 546         MRS_HWCAP(1, HWCAP_ASIMDDP, ID_AA64ISAR0_DP_IMPL),
 547         MRS_HWCAP_END
 548 };
 549
 550 static const struct mrs_field_value id_aa64isar0_sm4[] = {
 551         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM4, NONE, IMPL),
 552         MRS_FIELD_VALUE_END,
 553 };
 554
 555 static const struct mrs_field_hwcap id_aa64isar0_sm4_caps[] = {
 556         MRS_HWCAP(1, HWCAP_SM4, ID_AA64ISAR0_SM4_IMPL),
 557         MRS_HWCAP_END
 558 };
 559
 560 static const struct mrs_field_value id_aa64isar0_sm3[] = {
 561         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM3, NONE, IMPL),
 562         MRS_FIELD_VALUE_END,
 563 };
 564
 565 static const struct mrs_field_hwcap id_aa64isar0_sm3_caps[] = {
 566         MRS_HWCAP(1, HWCAP_SM3, ID_AA64ISAR0_SM3_IMPL),
 567         MRS_HWCAP_END
 568 };
 569
 570 static const struct mrs_field_value id_aa64isar0_sha3[] = {
 571         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA3, NONE, IMPL),
 572         MRS_FIELD_VALUE_END,
 573 };
 574
 575 static const struct mrs_field_hwcap id_aa64isar0_sha3_caps[] = {
 576         MRS_HWCAP(1, HWCAP_SHA3, ID_AA64ISAR0_SHA3_IMPL),
 577         MRS_HWCAP_END
 578 };
 579
 580 static const struct mrs_field_value id_aa64isar0_rdm[] = {
 581         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, RDM, NONE, IMPL),
 582         MRS_FIELD_VALUE_END,
 583 };
 584
 585 static const struct mrs_field_hwcap id_aa64isar0_rdm_caps[] = {
 586         MRS_HWCAP(1, HWCAP_ASIMDRDM, ID_AA64ISAR0_RDM_IMPL),
 587         MRS_HWCAP_END
 588 };
 589
 590 static const struct mrs_field_value id_aa64isar0_tme[] = {
 591         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, TME, NONE, IMPL),
 592         MRS_FIELD_VALUE_END,
 593 };
 594
 595 static const struct mrs_field_value id_aa64isar0_atomic[] = {
 596         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, Atomic, NONE, IMPL),
 597         MRS_FIELD_VALUE_END,
 598 };
 599
 600 static const struct mrs_field_hwcap id_aa64isar0_atomic_caps[] = {
 601         MRS_HWCAP(1, HWCAP_ATOMICS, ID_AA64ISAR0_Atomic_IMPL),
 602         MRS_HWCAP_END
 603 };
 604
 605 static const struct mrs_field_value id_aa64isar0_crc32[] = {
 606         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, CRC32, NONE, BASE),
 607         MRS_FIELD_VALUE_END,
 608 };
 609
 610 static const struct mrs_field_hwcap id_aa64isar0_crc32_caps[] = {
 611         MRS_HWCAP(1, HWCAP_CRC32, ID_AA64ISAR0_CRC32_BASE),
 612         MRS_HWCAP_END
 613 };
 614
 615 static const struct mrs_field_value id_aa64isar0_sha2[] = {
 616         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA2, NONE, BASE),
 617         MRS_FIELD_VALUE(ID_AA64ISAR0_SHA2_512, "SHA2+SHA512"),
 618         MRS_FIELD_VALUE_END,
 619 };
 620
 621 static const struct mrs_field_hwcap id_aa64isar0_sha2_caps[] = {
 622         MRS_HWCAP(1, HWCAP_SHA2, ID_AA64ISAR0_SHA2_BASE),
 623         MRS_HWCAP(1, HWCAP_SHA512, ID_AA64ISAR0_SHA2_512),
 624         MRS_HWCAP_END
 625 };
 626
 627 static const struct mrs_field_value id_aa64isar0_sha1[] = {
 628         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA1, NONE, BASE),
 629         MRS_FIELD_VALUE_END,
 630 };
 631
 632 static const struct mrs_field_hwcap id_aa64isar0_sha1_caps[] = {
 633         MRS_HWCAP(1, HWCAP_SHA1, ID_AA64ISAR0_SHA1_BASE),
 634         MRS_HWCAP_END
 635 };
 636
 637 static const struct mrs_field_value id_aa64isar0_aes[] = {
 638         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, AES, NONE, BASE),
 639         MRS_FIELD_VALUE(ID_AA64ISAR0_AES_PMULL, "AES+PMULL"),
 640         MRS_FIELD_VALUE_END,
 641 };
 642
 643 static const struct mrs_field_hwcap id_aa64isar0_aes_caps[] = {
 644         MRS_HWCAP(1, HWCAP_AES, ID_AA64ISAR0_AES_BASE),
 645         MRS_HWCAP(1, HWCAP_PMULL, ID_AA64ISAR0_AES_PMULL),
 646         MRS_HWCAP_END
 647 };
 648
 649 static const struct mrs_field id_aa64isar0_fields[] = {
 650         MRS_FIELD_HWCAP(ID_AA64ISAR0, RNDR, false, MRS_LOWER,
 651             id_aa64isar0_rndr, id_aa64isar0_rndr_caps),
 652         MRS_FIELD(ID_AA64ISAR0, TLB, false, MRS_EXACT, id_aa64isar0_tlb),
 653         MRS_FIELD_HWCAP(ID_AA64ISAR0, TS, false, MRS_LOWER, id_aa64isar0_ts,
 654             id_aa64isar0_ts_caps),
 655         MRS_FIELD_HWCAP(ID_AA64ISAR0, FHM, false, MRS_LOWER, id_aa64isar0_fhm,
 656             id_aa64isar0_fhm_caps),
 657         MRS_FIELD_HWCAP(ID_AA64ISAR0, DP, false, MRS_LOWER, id_aa64isar0_dp,
 658             id_aa64isar0_dp_caps),
 659         MRS_FIELD_HWCAP(ID_AA64ISAR0, SM4, false, MRS_LOWER, id_aa64isar0_sm4,
 660             id_aa64isar0_sm4_caps),
 661         MRS_FIELD_HWCAP(ID_AA64ISAR0, SM3, false, MRS_LOWER, id_aa64isar0_sm3,
 662             id_aa64isar0_sm3_caps),
 663         MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA3, false, MRS_LOWER, id_aa64isar0_sha3,
 664             id_aa64isar0_sha3_caps),
 665         MRS_FIELD_HWCAP(ID_AA64ISAR0, RDM, false, MRS_LOWER, id_aa64isar0_rdm,
 666             id_aa64isar0_rdm_caps),
 667         MRS_FIELD(ID_AA64ISAR0, TME, false, MRS_EXACT, id_aa64isar0_tme),
 668         MRS_FIELD_HWCAP(ID_AA64ISAR0, Atomic, false, MRS_LOWER,
 669             id_aa64isar0_atomic, id_aa64isar0_atomic_caps),
 670         MRS_FIELD_HWCAP(ID_AA64ISAR0, CRC32, false, MRS_LOWER,
 671             id_aa64isar0_crc32, id_aa64isar0_crc32_caps),
 672         MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA2, false, MRS_LOWER, id_aa64isar0_sha2,
 673             id_aa64isar0_sha2_caps),
 674         MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA1, false, MRS_LOWER,
 675             id_aa64isar0_sha1, id_aa64isar0_sha1_caps),
 676         MRS_FIELD_HWCAP(ID_AA64ISAR0, AES, false, MRS_LOWER, id_aa64isar0_aes,
 677             id_aa64isar0_aes_caps),
 678         MRS_FIELD_END,
 679 };
 680
 681
 682 /* ID_AA64ISAR1_EL1 */
 683 static const struct mrs_field_value id_aa64isar1_ls64[] = {
 684         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, LS64, NONE, IMPL),
 685         MRS_FIELD_VALUE(ID_AA64ISAR1_LS64_V, "LS64v"),
 686         MRS_FIELD_VALUE(ID_AA64ISAR1_LS64_ACCDATA, "LS64+ACCDATA"),
 687         MRS_FIELD_VALUE_END,
 688 };
 689
 690 static const struct mrs_field_value id_aa64isar1_xs[] = {
 691         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, XS, NONE, IMPL),
 692         MRS_FIELD_VALUE_END,
 693 };
 694
 695 static const struct mrs_field_value id_aa64isar1_i8mm[] = {
 696         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, I8MM, NONE, IMPL),
 697         MRS_FIELD_VALUE_END,
 698 };
 699
 700 static const struct mrs_field_hwcap id_aa64isar1_i8mm_caps[] = {
 701         MRS_HWCAP(2, HWCAP2_I8MM, ID_AA64ISAR1_I8MM_IMPL),
 702         MRS_HWCAP_END
 703 };
 704
 705 static const struct mrs_field_value id_aa64isar1_dgh[] = {
 706         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, DGH, NONE, IMPL),
 707         MRS_FIELD_VALUE_END,
 708 };
 709
 710 static const struct mrs_field_hwcap id_aa64isar1_dgh_caps[] = {
 711         MRS_HWCAP(2, HWCAP2_DGH, ID_AA64ISAR1_DGH_IMPL),
 712         MRS_HWCAP_END
 713 };
 714
 715 static const struct mrs_field_value id_aa64isar1_bf16[] = {
 716         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, BF16, NONE, IMPL),
 717         MRS_FIELD_VALUE(ID_AA64ISAR1_BF16_EBF, "EBF16"),
 718         MRS_FIELD_VALUE_END,
 719 };
 720
 721 static const struct mrs_field_hwcap id_aa64isar1_bf16_caps[] = {
 722         MRS_HWCAP(2, HWCAP2_BF16, ID_AA64ISAR1_BF16_IMPL),
 723         MRS_HWCAP_END
 724 };
 725
 726 static const struct mrs_field_value id_aa64isar1_specres[] = {
 727         MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_NONE, ""),
 728         MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_IMPL, "PredInv"),
 729         MRS_FIELD_VALUE_END,
 730 };
 731
 732 static const struct mrs_field_value id_aa64isar1_sb[] = {
 733         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, SB, NONE, IMPL),
 734         MRS_FIELD_VALUE_END,
 735 };
 736
 737 static const struct mrs_field_hwcap id_aa64isar1_sb_caps[] = {
 738         MRS_HWCAP(1, HWCAP_SB, ID_AA64ISAR1_SB_IMPL),
 739         MRS_HWCAP_END
 740 };
 741
 742 static const struct mrs_field_value id_aa64isar1_frintts[] = {
 743         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FRINTTS, NONE, IMPL),
 744         MRS_FIELD_VALUE_END,
 745 };
 746
 747 static const struct mrs_field_hwcap id_aa64isar1_frintts_caps[] = {
 748         MRS_HWCAP(2, HWCAP2_FRINT, ID_AA64ISAR1_FRINTTS_IMPL),
 749         MRS_HWCAP_END
 750 };
 751
 752 static const struct mrs_field_value id_aa64isar1_gpi[] = {
 753         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPI, NONE, IMPL),
 754         MRS_FIELD_VALUE_END,
 755 };
 756
 757 static const struct mrs_field_hwcap id_aa64isar1_gpi_caps[] = {
 758         MRS_HWCAP(1, HWCAP_PACG, ID_AA64ISAR1_GPI_IMPL),
 759         MRS_HWCAP_END
 760 };
 761
 762 static const struct mrs_field_value id_aa64isar1_gpa[] = {
 763         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPA, NONE, IMPL),
 764         MRS_FIELD_VALUE_END,
 765 };
 766
 767 static const struct mrs_field_hwcap id_aa64isar1_gpa_caps[] = {
 768         MRS_HWCAP(1, HWCAP_PACG, ID_AA64ISAR1_GPA_IMPL),
 769         MRS_HWCAP_END
 770 };
 771
 772 static const struct mrs_field_value id_aa64isar1_lrcpc[] = {
 773         MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_NONE, ""),
 774         MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_3, "RCPC-8.3"),
 775         MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_4, "RCPC-8.4"),
 776         MRS_FIELD_VALUE_END,
 777 };
 778
 779 static const struct mrs_field_hwcap id_aa64isar1_lrcpc_caps[] = {
 780         MRS_HWCAP(1, HWCAP_LRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_3),
 781         MRS_HWCAP(1, HWCAP_ILRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_4),
 782         MRS_HWCAP_END
 783 };
 784
 785 static const struct mrs_field_value id_aa64isar1_fcma[] = {
 786         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FCMA, NONE, IMPL),
 787         MRS_FIELD_VALUE_END,
 788 };
 789
 790 static const struct mrs_field_hwcap id_aa64isar1_fcma_caps[] = {
 791         MRS_HWCAP(1, HWCAP_FCMA, ID_AA64ISAR1_FCMA_IMPL),
 792         MRS_HWCAP_END
 793 };
 794
 795 static const struct mrs_field_value id_aa64isar1_jscvt[] = {
 796         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, JSCVT, NONE, IMPL),
 797         MRS_FIELD_VALUE_END,
 798 };
 799
 800 static const struct mrs_field_hwcap id_aa64isar1_jscvt_caps[] = {
 801         MRS_HWCAP(1, HWCAP_JSCVT, ID_AA64ISAR1_JSCVT_IMPL),
 802         MRS_HWCAP_END
 803 };
 804
 805 static const struct mrs_field_value id_aa64isar1_api[] = {
 806         MRS_FIELD_VALUE(ID_AA64ISAR1_API_NONE, ""),
 807         MRS_FIELD_VALUE(ID_AA64ISAR1_API_PAC, "API PAC"),
 808         MRS_FIELD_VALUE(ID_AA64ISAR1_API_EPAC, "API EPAC"),
 809         MRS_FIELD_VALUE(ID_AA64ISAR1_API_EPAC2, "Impl PAuth+EPAC2"),
 810         MRS_FIELD_VALUE(ID_AA64ISAR1_API_FPAC, "Impl PAuth+FPAC"),
 811         MRS_FIELD_VALUE(ID_AA64ISAR1_API_FPAC_COMBINED,
 812             "Impl PAuth+FPAC+Combined"),
 813         MRS_FIELD_VALUE_END,
 814 };
 815
 816 static const struct mrs_field_hwcap id_aa64isar1_api_caps[] = {
 817         MRS_HWCAP(1, HWCAP_PACA, ID_AA64ISAR1_API_PAC),
 818         MRS_HWCAP_END
 819 };
 820
 821 static const struct mrs_field_value id_aa64isar1_apa[] = {
 822         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_NONE, ""),
 823         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_PAC, "APA PAC"),
 824         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_EPAC, "APA EPAC"),
 825         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_EPAC2, "APA EPAC2"),
 826         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_FPAC, "APA FPAC"),
 827         MRS_FIELD_VALUE(ID_AA64ISAR1_APA_FPAC_COMBINED,
 828             "APA FPAC+Combined"),
 829         MRS_FIELD_VALUE_END,
 830 };
 831
 832 static const struct mrs_field_hwcap id_aa64isar1_apa_caps[] = {
 833         MRS_HWCAP(1, HWCAP_PACA, ID_AA64ISAR1_APA_PAC),
 834         MRS_HWCAP_END
 835 };
 836
 837 static const struct mrs_field_value id_aa64isar1_dpb[] = {
 838         MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_NONE, ""),
 839         MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVAP, "DCPoP"),
 840         MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVADP, "DCCVADP"),
 841         MRS_FIELD_VALUE_END,
 842 };
 843
 844 static const struct mrs_field_hwcap id_aa64isar1_dpb_caps[] = {
 845         MRS_HWCAP(1, HWCAP_DCPOP, ID_AA64ISAR1_DPB_DCCVAP),
 846         MRS_HWCAP(2, HWCAP2_DCPODP, ID_AA64ISAR1_DPB_DCCVADP),
 847         MRS_HWCAP_END
 848 };
 849
 850 static const struct mrs_field id_aa64isar1_fields[] = {
 851         MRS_FIELD(ID_AA64ISAR1, LS64, false, MRS_EXACT, id_aa64isar1_ls64),
 852         MRS_FIELD(ID_AA64ISAR1, XS, false, MRS_EXACT, id_aa64isar1_xs),
 853         MRS_FIELD_HWCAP(ID_AA64ISAR1, I8MM, false, MRS_LOWER,
 854             id_aa64isar1_i8mm, id_aa64isar1_i8mm_caps),
 855         MRS_FIELD_HWCAP(ID_AA64ISAR1, DGH, false, MRS_LOWER, id_aa64isar1_dgh,
 856             id_aa64isar1_dgh_caps),
 857         MRS_FIELD_HWCAP(ID_AA64ISAR1, BF16, false, MRS_LOWER,
 858             id_aa64isar1_bf16, id_aa64isar1_bf16_caps),
 859         MRS_FIELD(ID_AA64ISAR1, SPECRES, false, MRS_EXACT,
 860             id_aa64isar1_specres),
 861         MRS_FIELD_HWCAP(ID_AA64ISAR1, SB, false, MRS_LOWER, id_aa64isar1_sb,
 862             id_aa64isar1_sb_caps),
 863         MRS_FIELD_HWCAP(ID_AA64ISAR1, FRINTTS, false, MRS_LOWER,
 864             id_aa64isar1_frintts, id_aa64isar1_frintts_caps),
 865         MRS_FIELD_HWCAP(ID_AA64ISAR1, GPI, false, MRS_EXACT, id_aa64isar1_gpi,
 866             id_aa64isar1_gpi_caps),
 867         MRS_FIELD_HWCAP(ID_AA64ISAR1, GPA, false, MRS_EXACT, id_aa64isar1_gpa,
 868             id_aa64isar1_gpa_caps),
 869         MRS_FIELD_HWCAP(ID_AA64ISAR1, LRCPC, false, MRS_LOWER,
 870             id_aa64isar1_lrcpc, id_aa64isar1_lrcpc_caps),
 871         MRS_FIELD_HWCAP(ID_AA64ISAR1, FCMA, false, MRS_LOWER,
 872             id_aa64isar1_fcma, id_aa64isar1_fcma_caps),
 873         MRS_FIELD_HWCAP(ID_AA64ISAR1, JSCVT, false, MRS_LOWER,
 874             id_aa64isar1_jscvt, id_aa64isar1_jscvt_caps),
 875         MRS_FIELD_HWCAP(ID_AA64ISAR1, API, false, MRS_EXACT, id_aa64isar1_api,
 876             id_aa64isar1_api_caps),
 877         MRS_FIELD_HWCAP(ID_AA64ISAR1, APA, false, MRS_EXACT, id_aa64isar1_apa,
 878             id_aa64isar1_apa_caps),
 879         MRS_FIELD_HWCAP(ID_AA64ISAR1, DPB, false, MRS_LOWER, id_aa64isar1_dpb,
 880             id_aa64isar1_dpb_caps),
 881         MRS_FIELD_END,
 882 };
 883
 884
 885 /* ID_AA64ISAR2_EL1 */
 886 static const struct mrs_field_value id_aa64isar2_pac_frac[] = {
 887         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, PAC_frac, NONE, IMPL),
 888         MRS_FIELD_VALUE_END,
 889 };
 890
 891 static const struct mrs_field_value id_aa64isar2_bc[] = {
 892         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, BC, NONE, IMPL),
 893         MRS_FIELD_VALUE_END,
 894 };
 895
 896 static const struct mrs_field_value id_aa64isar2_mops[] = {
 897         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, MOPS, NONE, IMPL),
 898         MRS_FIELD_VALUE_END,
 899 };
 900
 901 static const struct mrs_field_value id_aa64isar2_apa3[] = {
 902         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_NONE, ""),
 903         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_PAC, "APA3 PAC"),
 904         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_EPAC, "APA3 EPAC"),
 905         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_EPAC2, "APA3 EPAC2"),
 906         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_FPAC, "APA3 FPAC"),
 907         MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_FPAC_COMBINED,
 908             "APA3 FPAC+Combined"),
 909         MRS_FIELD_VALUE_END,
 910 };
 911
 912 static const struct mrs_field_hwcap id_aa64isar2_apa3_caps[] = {
 913         MRS_HWCAP(1, HWCAP_PACA, ID_AA64ISAR2_APA3_PAC),
 914         MRS_HWCAP_END
 915 };
 916
 917 static const struct mrs_field_value id_aa64isar2_gpa3[] = {
 918         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, GPA3, NONE, IMPL),
 919         MRS_FIELD_VALUE_END,
 920 };
 921
 922 static const struct mrs_field_hwcap id_aa64isar2_gpa3_caps[] = {
 923         MRS_HWCAP(1, HWCAP_PACG, ID_AA64ISAR2_GPA3_IMPL),
 924         MRS_HWCAP_END
 925 };
 926
 927 static const struct mrs_field_value id_aa64isar2_rpres[] = {
 928         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, RPRES, NONE, IMPL),
 929         MRS_FIELD_VALUE_END,
 930 };
 931
 932 static const struct mrs_field_value id_aa64isar2_wfxt[] = {
 933         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, WFxT, NONE, IMPL),
 934         MRS_FIELD_VALUE_END,
 935 };
 936
 937 static const struct mrs_field id_aa64isar2_fields[] = {
 938         MRS_FIELD(ID_AA64ISAR2, PAC_frac, false, MRS_EXACT,
 939             id_aa64isar2_pac_frac),
 940         MRS_FIELD(ID_AA64ISAR2, BC, false, MRS_EXACT, id_aa64isar2_bc),
 941         MRS_FIELD(ID_AA64ISAR2, MOPS, false, MRS_EXACT, id_aa64isar2_mops),
 942         MRS_FIELD_HWCAP(ID_AA64ISAR2, APA3, false, MRS_EXACT,
 943             id_aa64isar2_apa3, id_aa64isar2_apa3_caps),
 944         MRS_FIELD_HWCAP(ID_AA64ISAR2, GPA3, false, MRS_EXACT,
 945             id_aa64isar2_gpa3, id_aa64isar2_gpa3_caps),
 946         MRS_FIELD(ID_AA64ISAR2, RPRES, false, MRS_EXACT, id_aa64isar2_rpres),
 947         MRS_FIELD(ID_AA64ISAR2, WFxT, false, MRS_EXACT, id_aa64isar2_wfxt),
 948         MRS_FIELD_END,
 949 };
 950
 951
 952 /* ID_AA64MMFR0_EL1 */
 953 static const struct mrs_field_value id_aa64mmfr0_ecv[] = {
 954         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, ECV, NONE, IMPL),
 955         MRS_FIELD_VALUE(ID_AA64MMFR0_ECV_CNTHCTL, "ECV+CNTHCTL"),
 956         MRS_FIELD_VALUE_END,
 957 };
 958
 959 static const struct mrs_field_value id_aa64mmfr0_fgt[] = {
 960         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, FGT, NONE, IMPL),
 961         MRS_FIELD_VALUE_END,
 962 };
 963
 964 static const struct mrs_field_value id_aa64mmfr0_exs[] = {
 965         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, ExS, ALL, IMPL),
 966         MRS_FIELD_VALUE_END,
 967 };
 968
 969 static const struct mrs_field_value id_aa64mmfr0_tgran4_2[] = {
 970         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_TGran4, ""),
 971         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_NONE, "No S2 TGran4"),
 972         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_IMPL, "S2 TGran4"),
 973         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_LPA2, "S2 TGran4+LPA2"),
 974         MRS_FIELD_VALUE_END,
 975 };
 976
 977 static const struct mrs_field_value id_aa64mmfr0_tgran64_2[] = {
 978         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_TGran64, ""),
 979         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_NONE, "No S2 TGran64"),
 980         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_IMPL, "S2 TGran64"),
 981         MRS_FIELD_VALUE_END,
 982 };
 983
 984 static const struct mrs_field_value id_aa64mmfr0_tgran16_2[] = {
 985         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_TGran16, ""),
 986         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_NONE, "No S2 TGran16"),
 987         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_IMPL, "S2 TGran16"),
 988         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_LPA2, "S2 TGran16+LPA2"),
 989         MRS_FIELD_VALUE_END,
 990 };
 991
 992 static const struct mrs_field_value id_aa64mmfr0_tgran4[] = {
 993         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran4, NONE, IMPL),
 994         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_LPA2, "TGran4+LPA2"),
 995         MRS_FIELD_VALUE_END,
 996 };
 997
 998 static const struct mrs_field_value id_aa64mmfr0_tgran64[] = {
 999         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran64, NONE, IMPL),
1000         MRS_FIELD_VALUE_END,
1001 };
1002
1003 static const struct mrs_field_value id_aa64mmfr0_tgran16[] = {
1004         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran16, NONE, IMPL),
1005         MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_LPA2, "TGran16+LPA2"),
1006         MRS_FIELD_VALUE_END,
1007 };
1008
1009 static const struct mrs_field_value id_aa64mmfr0_bigendel0[] = {
1010         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEndEL0, FIXED, MIXED),
1011         MRS_FIELD_VALUE_END,
1012 };
1013
1014 static const struct mrs_field_value id_aa64mmfr0_snsmem[] = {
1015         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, SNSMem, NONE, DISTINCT),
1016         MRS_FIELD_VALUE_END,
1017 };
1018
1019 static const struct mrs_field_value id_aa64mmfr0_bigend[] = {
1020         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEnd, FIXED, MIXED),
1021         MRS_FIELD_VALUE_END,
1022 };
1023
1024 static const struct mrs_field_value id_aa64mmfr0_asidbits[] = {
1025         MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_8, "8bit ASID"),
1026         MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_16, "16bit ASID"),
1027         MRS_FIELD_VALUE_END,
1028 };
1029
1030 static const struct mrs_field_value id_aa64mmfr0_parange[] = {
1031         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4G, "4GB PA"),
1032         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_64G, "64GB PA"),
1033         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_1T, "1TB PA"),
1034         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4T, "4TB PA"),
1035         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_16T, "16TB PA"),
1036         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_256T, "256TB PA"),
1037         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4P, "4PB PA"),
1038         MRS_FIELD_VALUE_END,
1039 };
1040
1041 static const struct mrs_field id_aa64mmfr0_fields[] = {
1042         MRS_FIELD(ID_AA64MMFR0, ECV, false, MRS_EXACT, id_aa64mmfr0_ecv),
1043         MRS_FIELD(ID_AA64MMFR0, FGT, false, MRS_EXACT, id_aa64mmfr0_fgt),
1044         MRS_FIELD(ID_AA64MMFR0, ExS, false, MRS_EXACT, id_aa64mmfr0_exs),
1045         MRS_FIELD(ID_AA64MMFR0, TGran4_2, false, MRS_EXACT,
1046             id_aa64mmfr0_tgran4_2),
1047         MRS_FIELD(ID_AA64MMFR0, TGran64_2, false, MRS_EXACT,
1048             id_aa64mmfr0_tgran64_2),
1049         MRS_FIELD(ID_AA64MMFR0, TGran16_2, false, MRS_EXACT,
1050             id_aa64mmfr0_tgran16_2),
1051         MRS_FIELD(ID_AA64MMFR0, TGran4, false, MRS_EXACT, id_aa64mmfr0_tgran4),
1052         MRS_FIELD(ID_AA64MMFR0, TGran64, false, MRS_EXACT,
1053             id_aa64mmfr0_tgran64),
1054         MRS_FIELD(ID_AA64MMFR0, TGran16, false, MRS_EXACT,
1055             id_aa64mmfr0_tgran16),
1056         MRS_FIELD(ID_AA64MMFR0, BigEndEL0, false, MRS_EXACT,
1057             id_aa64mmfr0_bigendel0),
1058         MRS_FIELD(ID_AA64MMFR0, SNSMem, false, MRS_EXACT, id_aa64mmfr0_snsmem),
1059         MRS_FIELD(ID_AA64MMFR0, BigEnd, false, MRS_EXACT, id_aa64mmfr0_bigend),
1060         MRS_FIELD(ID_AA64MMFR0, ASIDBits, false, MRS_EXACT,
1061             id_aa64mmfr0_asidbits),
1062         MRS_FIELD(ID_AA64MMFR0, PARange, false, MRS_EXACT,
1063             id_aa64mmfr0_parange),
1064         MRS_FIELD_END,
1065 };
1066
1067
1068 /* ID_AA64MMFR1_EL1 */
1069 static const struct mrs_field_value id_aa64mmfr1_cmovw[] = {
1070         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, CMOVW, NONE, IMPL),
1071         MRS_FIELD_VALUE_END,
1072 };
1073
1074 static const struct mrs_field_value id_aa64mmfr1_tidcp1[] = {
1075         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, TIDCP1, NONE, IMPL),
1076         MRS_FIELD_VALUE_END,
1077 };
1078
1079 static const struct mrs_field_value id_aa64mmfr1_ntlbpa[] = {
1080         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, nTLBPA, NONE, IMPL),
1081         MRS_FIELD_VALUE_END,
1082 };
1083
1084 static const struct mrs_field_value id_aa64mmfr1_afp[] = {
1085         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, AFP, NONE, IMPL),
1086         MRS_FIELD_VALUE_END,
1087 };
1088
1089 static const struct mrs_field_value id_aa64mmfr1_hcx[] = {
1090         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, HCX, NONE, IMPL),
1091         MRS_FIELD_VALUE_END,
1092 };
1093
1094 static const struct mrs_field_value id_aa64mmfr1_ets[] = {
1095         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, ETS, NONE, IMPL),
1096         MRS_FIELD_VALUE_END,
1097 };
1098
1099 static const struct mrs_field_value id_aa64mmfr1_twed[] = {
1100         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, TWED, NONE, IMPL),
1101         MRS_FIELD_VALUE_END,
1102 };
1103
1104 static const struct mrs_field_value id_aa64mmfr1_xnx[] = {
1105         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, XNX, NONE, IMPL),
1106         MRS_FIELD_VALUE_END,
1107 };
1108
1109 static const struct mrs_field_value id_aa64mmfr1_specsei[] = {
1110         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, SpecSEI, NONE, IMPL),
1111         MRS_FIELD_VALUE_END,
1112 };
1113
1114 static const struct mrs_field_value id_aa64mmfr1_pan[] = {
1115         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, PAN, NONE, IMPL),
1116         MRS_FIELD_VALUE(ID_AA64MMFR1_PAN_ATS1E1, "PAN+ATS1E1"),
1117         MRS_FIELD_VALUE(ID_AA64MMFR1_PAN_EPAN, "EPAN"),
1118         MRS_FIELD_VALUE_END,
1119 };
1120
1121 static const struct mrs_field_value id_aa64mmfr1_lo[] = {
1122         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, LO, NONE, IMPL),
1123         MRS_FIELD_VALUE_END,
1124 };
1125
1126 static const struct mrs_field_value id_aa64mmfr1_hpds[] = {
1127         MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_NONE, ""),
1128         MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_HPD, "HPD"),
1129         MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_TTPBHA, "HPD+TTPBHA"),
1130         MRS_FIELD_VALUE_END,
1131 };
1132
1133 static const struct mrs_field_value id_aa64mmfr1_vh[] = {
1134         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, VH, NONE, IMPL),
1135         MRS_FIELD_VALUE_END,
1136 };
1137
1138 static const struct mrs_field_value id_aa64mmfr1_vmidbits[] = {
1139         MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_8, "8bit VMID"),
1140         MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_16, "16bit VMID"),
1141         MRS_FIELD_VALUE_END,
1142 };
1143
1144 static const struct mrs_field_value id_aa64mmfr1_hafdbs[] = {
1145         MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_NONE, ""),
1146         MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF, "HAF"),
1147         MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF_DBS, "HAF+DS"),
1148         MRS_FIELD_VALUE_END,
1149 };
1150
1151 static const struct mrs_field id_aa64mmfr1_fields[] = {
1152         MRS_FIELD(ID_AA64MMFR1, CMOVW, false, MRS_EXACT, id_aa64mmfr1_cmovw),
1153         MRS_FIELD(ID_AA64MMFR1, TIDCP1, false, MRS_EXACT, id_aa64mmfr1_tidcp1),
1154         MRS_FIELD(ID_AA64MMFR1, nTLBPA, false, MRS_EXACT, id_aa64mmfr1_ntlbpa),
1155         MRS_FIELD(ID_AA64MMFR1, AFP, false, MRS_EXACT, id_aa64mmfr1_afp),
1156         MRS_FIELD(ID_AA64MMFR1, HCX, false, MRS_EXACT, id_aa64mmfr1_hcx),
1157         MRS_FIELD(ID_AA64MMFR1, ETS, false, MRS_EXACT, id_aa64mmfr1_ets),
1158         MRS_FIELD(ID_AA64MMFR1, TWED, false, MRS_EXACT, id_aa64mmfr1_twed),
1159         MRS_FIELD(ID_AA64MMFR1, XNX, false, MRS_EXACT, id_aa64mmfr1_xnx),
1160         MRS_FIELD(ID_AA64MMFR1, SpecSEI, false, MRS_EXACT,
1161             id_aa64mmfr1_specsei),
1162         MRS_FIELD(ID_AA64MMFR1, PAN, false, MRS_EXACT, id_aa64mmfr1_pan),
1163         MRS_FIELD(ID_AA64MMFR1, LO, false, MRS_EXACT, id_aa64mmfr1_lo),
1164         MRS_FIELD(ID_AA64MMFR1, HPDS, false, MRS_EXACT, id_aa64mmfr1_hpds),
1165         MRS_FIELD(ID_AA64MMFR1, VH, false, MRS_EXACT, id_aa64mmfr1_vh),
1166         MRS_FIELD(ID_AA64MMFR1, VMIDBits, false, MRS_EXACT,
1167             id_aa64mmfr1_vmidbits),
1168         MRS_FIELD(ID_AA64MMFR1, HAFDBS, false, MRS_EXACT, id_aa64mmfr1_hafdbs),
1169         MRS_FIELD_END,
1170 };
1171
1172
1173 /* ID_AA64MMFR2_EL1 */
1174 static const struct mrs_field_value id_aa64mmfr2_e0pd[] = {
1175         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, E0PD, NONE, IMPL),
1176         MRS_FIELD_VALUE_END,
1177 };
1178
1179 static const struct mrs_field_value id_aa64mmfr2_evt[] = {
1180         MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_NONE, ""),
1181         MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_2, "EVT-8.2"),
1182         MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_5, "EVT-8.5"),
1183         MRS_FIELD_VALUE_END,
1184 };
1185
1186 static const struct mrs_field_value id_aa64mmfr2_bbm[] = {
1187         MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL0, ""),
1188         MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL1, "BBM level 1"),
1189         MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL2, "BBM level 2"),
1190         MRS_FIELD_VALUE_END,
1191 };
1192
1193 static const struct mrs_field_value id_aa64mmfr2_ttl[] = {
1194         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, TTL, NONE, IMPL),
1195         MRS_FIELD_VALUE_END,
1196 };
1197
1198 static const struct mrs_field_value id_aa64mmfr2_fwb[] = {
1199         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, FWB, NONE, IMPL),
1200         MRS_FIELD_VALUE_END,
1201 };
1202
1203 static const struct mrs_field_value id_aa64mmfr2_ids[] = {
1204         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IDS, NONE, IMPL),
1205         MRS_FIELD_VALUE_END,
1206 };
1207
1208 static const struct mrs_field_value id_aa64mmfr2_at[] = {
1209         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, AT, NONE, IMPL),
1210         MRS_FIELD_VALUE_END,
1211 };
1212
1213 static const struct mrs_field_hwcap id_aa64mmfr2_at_caps[] = {
1214         MRS_HWCAP(1, HWCAP_USCAT, ID_AA64MMFR2_AT_IMPL),
1215         MRS_HWCAP_END
1216 };
1217
1218 static const struct mrs_field_value id_aa64mmfr2_st[] = {
1219         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, ST, NONE, IMPL),
1220         MRS_FIELD_VALUE_END,
1221 };
1222
1223 static const struct mrs_field_value id_aa64mmfr2_nv[] = {
1224         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, NV, NONE, 8_3),
1225         MRS_FIELD_VALUE(ID_AA64MMFR2_NV_8_4, "NV v8.4"),
1226         MRS_FIELD_VALUE_END,
1227 };
1228
1229 static const struct mrs_field_value id_aa64mmfr2_ccidx[] = {
1230         MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_32, "32bit CCIDX"),
1231         MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_64, "64bit CCIDX"),
1232         MRS_FIELD_VALUE_END,
1233 };
1234
1235 static const struct mrs_field_value id_aa64mmfr2_varange[] = {
1236         MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_48, "48bit VA"),
1237         MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_52, "52bit VA"),
1238         MRS_FIELD_VALUE_END,
1239 };
1240
1241 static const struct mrs_field_value id_aa64mmfr2_iesb[] = {
1242         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IESB, NONE, IMPL),
1243         MRS_FIELD_VALUE_END,
1244 };
1245
1246 static const struct mrs_field_value id_aa64mmfr2_lsm[] = {
1247         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, LSM, NONE, IMPL),
1248         MRS_FIELD_VALUE_END,
1249 };
1250
1251 static const struct mrs_field_value id_aa64mmfr2_uao[] = {
1252         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, UAO, NONE, IMPL),
1253         MRS_FIELD_VALUE_END,
1254 };
1255
1256 static const struct mrs_field_value id_aa64mmfr2_cnp[] = {
1257         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, CnP, NONE, IMPL),
1258         MRS_FIELD_VALUE_END,
1259 };
1260
1261 static const struct mrs_field id_aa64mmfr2_fields[] = {
1262         MRS_FIELD(ID_AA64MMFR2, E0PD, false, MRS_EXACT, id_aa64mmfr2_e0pd),
1263         MRS_FIELD(ID_AA64MMFR2, EVT, false, MRS_EXACT, id_aa64mmfr2_evt),
1264         MRS_FIELD(ID_AA64MMFR2, BBM, false, MRS_EXACT, id_aa64mmfr2_bbm),
1265         MRS_FIELD(ID_AA64MMFR2, TTL, false, MRS_EXACT, id_aa64mmfr2_ttl),
1266         MRS_FIELD(ID_AA64MMFR2, FWB, false, MRS_EXACT, id_aa64mmfr2_fwb),
1267         MRS_FIELD(ID_AA64MMFR2, IDS, false, MRS_EXACT, id_aa64mmfr2_ids),
1268         MRS_FIELD_HWCAP(ID_AA64MMFR2, AT, false, MRS_LOWER, id_aa64mmfr2_at,
1269             id_aa64mmfr2_at_caps),
1270         MRS_FIELD(ID_AA64MMFR2, ST, false, MRS_EXACT, id_aa64mmfr2_st),
1271         MRS_FIELD(ID_AA64MMFR2, NV, false, MRS_EXACT, id_aa64mmfr2_nv),
1272         MRS_FIELD(ID_AA64MMFR2, CCIDX, false, MRS_EXACT, id_aa64mmfr2_ccidx),
1273         MRS_FIELD(ID_AA64MMFR2, VARange, false, MRS_EXACT,
1274             id_aa64mmfr2_varange),
1275         MRS_FIELD(ID_AA64MMFR2, IESB, false, MRS_EXACT, id_aa64mmfr2_iesb),
1276         MRS_FIELD(ID_AA64MMFR2, LSM, false, MRS_EXACT, id_aa64mmfr2_lsm),
1277         MRS_FIELD(ID_AA64MMFR2, UAO, false, MRS_EXACT, id_aa64mmfr2_uao),
1278         MRS_FIELD(ID_AA64MMFR2, CnP, false, MRS_EXACT, id_aa64mmfr2_cnp),
1279         MRS_FIELD_END,
1280 };
1281
1282
1283 #ifdef NOTYET
1284 /* ID_AA64MMFR2_EL1 */
1285 static const struct mrs_field_value id_aa64mmfr3_spec_fpacc[] = {
1286         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, Spec_FPACC, NONE, IMPL),
1287         MRS_FIELD_VALUE_END,
1288 };
1289
1290 static const struct mrs_field_value id_aa64mmfr3_mec[] = {
1291         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, MEC, NONE, IMPL),
1292         MRS_FIELD_VALUE_END,
1293 };
1294
1295 static const struct mrs_field_value id_aa64mmfr3_sctlrx[] = {
1296         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, SCTLRX, NONE, IMPL),
1297         MRS_FIELD_VALUE_END,
1298 };
1299
1300 static const struct mrs_field_value id_aa64mmfr3_tcrx[] = {
1301         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, TCRX, NONE, IMPL),
1302         MRS_FIELD_VALUE_END,
1303 };
1304
1305 static const struct mrs_field id_aa64mmfr3_fields[] = {
1306         MRS_FIELD(ID_AA64MMFR3, Spec_FPACC, false, MRS_EXACT,
1307             id_aa64mmfr3_spec_fpacc),
1308         MRS_FIELD(ID_AA64MMFR3, MEC, false, MRS_EXACT, id_aa64mmfr3_mec),
1309         MRS_FIELD(ID_AA64MMFR3, SCTLRX, false, MRS_EXACT, id_aa64mmfr3_sctlrx),
1310         MRS_FIELD(ID_AA64MMFR3, TCRX, false, MRS_EXACT, id_aa64mmfr3_tcrx),
1311         MRS_FIELD_END,
1312 };
1313
1314
1315 /* ID_AA64MMFR4_EL1 */
1316 static const struct mrs_field id_aa64mmfr4_fields[] = {
1317         MRS_FIELD_END,
1318 };
1319 #endif
1320
1321
1322 /* ID_AA64PFR0_EL1 */
1323 static const struct mrs_field_value id_aa64pfr0_csv3[] = {
1324         MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_NONE, ""),
1325         MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_ISOLATED, "CSV3"),
1326         MRS_FIELD_VALUE_END,
1327 };
1328
1329 static const struct mrs_field_value id_aa64pfr0_csv2[] = {
1330         MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_NONE, ""),
1331         MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_ISOLATED, "CSV2"),
1332         MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_SCXTNUM, "CSV2_2"),
1333         MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_3, "CSV2_3"),
1334         MRS_FIELD_VALUE_END,
1335 };
1336
1337 static const struct mrs_field_value id_aa64pfr0_rme[] = {
1338         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, RME, NONE, IMPL),
1339         MRS_FIELD_VALUE_END,
1340 };
1341
1342 static const struct mrs_field_value id_aa64pfr0_dit[] = {
1343         MRS_FIELD_VALUE(ID_AA64PFR0_DIT_NONE, ""),
1344         MRS_FIELD_VALUE(ID_AA64PFR0_DIT_PSTATE, "PSTATE.DIT"),
1345         MRS_FIELD_VALUE_END,
1346 };
1347
1348 static const struct mrs_field_hwcap id_aa64pfr0_dit_caps[] = {
1349         MRS_HWCAP(1, HWCAP_DIT, ID_AA64PFR0_DIT_PSTATE),
1350         MRS_HWCAP_END
1351 };
1352
1353 static const struct mrs_field_value id_aa64pfr0_amu[] = {
1354         MRS_FIELD_VALUE(ID_AA64PFR0_AMU_NONE, ""),
1355         MRS_FIELD_VALUE(ID_AA64PFR0_AMU_V1, "AMUv1"),
1356         MRS_FIELD_VALUE(ID_AA64PFR0_AMU_V1_1, "AMUv1p1"),
1357         MRS_FIELD_VALUE_END,
1358 };
1359
1360 static const struct mrs_field_value id_aa64pfr0_mpam[] = {
1361         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, MPAM, NONE, IMPL),
1362         MRS_FIELD_VALUE_END,
1363 };
1364
1365 static const struct mrs_field_value id_aa64pfr0_sel2[] = {
1366         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SEL2, NONE, IMPL),
1367         MRS_FIELD_VALUE_END,
1368 };
1369
1370 static const struct mrs_field_value id_aa64pfr0_sve[] = {
1371         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SVE, NONE, IMPL),
1372         MRS_FIELD_VALUE_END,
1373 };
1374
1375 #if 0
1376 /* Enable when we add SVE support */
1377 static const struct mrs_field_hwcap id_aa64pfr0_sve_caps[] = {
1378         MRS_HWCAP(1, HWCAP_SVE, ID_AA64PFR0_SVE_IMPL),
1379         MRS_HWCAP_END
1380 };
1381 #endif
1382
1383 static const struct mrs_field_value id_aa64pfr0_ras[] = {
1384         MRS_FIELD_VALUE(ID_AA64PFR0_RAS_NONE, ""),
1385         MRS_FIELD_VALUE(ID_AA64PFR0_RAS_IMPL, "RAS"),
1386         MRS_FIELD_VALUE(ID_AA64PFR0_RAS_8_4, "RAS v8.4"),
1387         MRS_FIELD_VALUE_END,
1388 };
1389
1390 static const struct mrs_field_value id_aa64pfr0_gic[] = {
1391         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, GIC, CPUIF_NONE, CPUIF_EN),
1392         MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_NONE, ""),
1393         MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_EN, "GIC"),
1394         MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_4_1, "GIC 4.1"),
1395         MRS_FIELD_VALUE_END,
1396 };
1397
1398 static const struct mrs_field_value id_aa64pfr0_advsimd[] = {
1399         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, AdvSIMD, NONE, IMPL),
1400         MRS_FIELD_VALUE(ID_AA64PFR0_AdvSIMD_HP, "AdvSIMD+HP"),
1401         MRS_FIELD_VALUE_END,
1402 };
1403
1404 static const struct mrs_field_hwcap id_aa64pfr0_advsimd_caps[] = {
1405         MRS_HWCAP(1, HWCAP_ASIMD, ID_AA64PFR0_AdvSIMD_IMPL),
1406         MRS_HWCAP(1, HWCAP_ASIMDHP, ID_AA64PFR0_AdvSIMD_HP),
1407         MRS_HWCAP_END
1408 };
1409
1410 static const struct mrs_field_value id_aa64pfr0_fp[] = {
1411         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, FP, NONE, IMPL),
1412         MRS_FIELD_VALUE(ID_AA64PFR0_FP_HP, "FP+HP"),
1413         MRS_FIELD_VALUE_END,
1414 };
1415
1416 static const struct mrs_field_hwcap id_aa64pfr0_fp_caps[] = {
1417         MRS_HWCAP(1, HWCAP_FP, ID_AA64PFR0_FP_IMPL),
1418         MRS_HWCAP(1, HWCAP_FPHP, ID_AA64PFR0_FP_HP),
1419         MRS_HWCAP_END
1420 };
1421
1422 static const struct mrs_field_value id_aa64pfr0_el3[] = {
1423         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL3, NONE, 64),
1424         MRS_FIELD_VALUE(ID_AA64PFR0_EL3_64_32, "EL3 32"),
1425         MRS_FIELD_VALUE_END,
1426 };
1427
1428 static const struct mrs_field_value id_aa64pfr0_el2[] = {
1429         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL2, NONE, 64),
1430         MRS_FIELD_VALUE(ID_AA64PFR0_EL2_64_32, "EL2 32"),
1431         MRS_FIELD_VALUE_END,
1432 };
1433
1434 static const struct mrs_field_value id_aa64pfr0_el1[] = {
1435         MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64, "EL1"),
1436         MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64_32, "EL1 32"),
1437         MRS_FIELD_VALUE_END,
1438 };
1439
1440 static const struct mrs_field_value id_aa64pfr0_el0[] = {
1441         MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64, "EL0"),
1442         MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64_32, "EL0 32"),
1443         MRS_FIELD_VALUE_END,
1444 };
1445
1446 static const struct mrs_field id_aa64pfr0_fields[] = {
1447         MRS_FIELD(ID_AA64PFR0, CSV3, false, MRS_EXACT, id_aa64pfr0_csv3),
1448         MRS_FIELD(ID_AA64PFR0, CSV2, false, MRS_EXACT, id_aa64pfr0_csv2),
1449         MRS_FIELD(ID_AA64PFR0, RME, false, MRS_EXACT, id_aa64pfr0_rme),
1450         MRS_FIELD_HWCAP(ID_AA64PFR0, DIT, false, MRS_LOWER, id_aa64pfr0_dit,
1451             id_aa64pfr0_dit_caps),
1452         MRS_FIELD(ID_AA64PFR0, AMU, false, MRS_EXACT, id_aa64pfr0_amu),
1453         MRS_FIELD(ID_AA64PFR0, MPAM, false, MRS_EXACT, id_aa64pfr0_mpam),
1454         MRS_FIELD(ID_AA64PFR0, SEL2, false, MRS_EXACT, id_aa64pfr0_sel2),
1455         MRS_FIELD(ID_AA64PFR0, SVE, false, MRS_EXACT, id_aa64pfr0_sve),
1456         MRS_FIELD(ID_AA64PFR0, RAS, false, MRS_EXACT, id_aa64pfr0_ras),
1457         MRS_FIELD(ID_AA64PFR0, GIC, false, MRS_EXACT, id_aa64pfr0_gic),
1458         MRS_FIELD_HWCAP(ID_AA64PFR0, AdvSIMD, true, MRS_LOWER,
1459             id_aa64pfr0_advsimd, id_aa64pfr0_advsimd_caps),
1460         MRS_FIELD_HWCAP(ID_AA64PFR0, FP, true,  MRS_LOWER, id_aa64pfr0_fp,
1461             id_aa64pfr0_fp_caps),
1462         MRS_FIELD(ID_AA64PFR0, EL3, false, MRS_EXACT, id_aa64pfr0_el3),
1463         MRS_FIELD(ID_AA64PFR0, EL2, false, MRS_EXACT, id_aa64pfr0_el2),
1464         MRS_FIELD(ID_AA64PFR0, EL1, false, MRS_LOWER, id_aa64pfr0_el1),
1465         MRS_FIELD(ID_AA64PFR0, EL0, false, MRS_LOWER, id_aa64pfr0_el0),
1466         MRS_FIELD_END,
1467 };
1468
1469
1470 /* ID_AA64PFR1_EL1 */
1471 static const struct mrs_field_value id_aa64pfr1_nmi[] = {
1472         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, NMI, NONE, IMPL),
1473         MRS_FIELD_VALUE_END,
1474 };
1475
1476 static const struct mrs_field_value id_aa64pfr1_csv2_frac[] = {
1477         MRS_FIELD_VALUE(ID_AA64PFR1_CSV2_frac_p0, ""),
1478         MRS_FIELD_VALUE(ID_AA64PFR1_CSV2_frac_p1, "CSV2 p1"),
1479         MRS_FIELD_VALUE(ID_AA64PFR1_CSV2_frac_p2, "CSV2 p2"),
1480         MRS_FIELD_VALUE_END,
1481 };
1482
1483 static const struct mrs_field_value id_aa64pfr1_rndr_trap[] = {
1484         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, RNDR_trap, NONE, IMPL),
1485         MRS_FIELD_VALUE_END,
1486 };
1487
1488 static const struct mrs_field_value id_aa64pfr1_sme[] = {
1489         MRS_FIELD_VALUE(ID_AA64PFR1_SME_NONE, ""),
1490         MRS_FIELD_VALUE(ID_AA64PFR1_SME_SME, "SME"),
1491         MRS_FIELD_VALUE(ID_AA64PFR1_SME_SME2, "SME2"),
1492         MRS_FIELD_VALUE_END,
1493 };
1494
1495 static const struct mrs_field_value id_aa64pfr1_mpam_frac[] = {
1496         MRS_FIELD_VALUE(ID_AA64PFR1_MPAM_frac_p0, ""),
1497         MRS_FIELD_VALUE(ID_AA64PFR1_MPAM_frac_p1, "MPAM p1"),
1498         MRS_FIELD_VALUE_END,
1499 };
1500
1501 static const struct mrs_field_value id_aa64pfr1_ras_frac[] = {
1502         MRS_FIELD_VALUE(ID_AA64PFR1_RAS_frac_p0, ""),
1503         MRS_FIELD_VALUE(ID_AA64PFR1_RAS_frac_p1, "RAS p1"),
1504         MRS_FIELD_VALUE_END,
1505 };
1506
1507 static const struct mrs_field_value id_aa64pfr1_mte[] = {
1508         MRS_FIELD_VALUE(ID_AA64PFR1_MTE_NONE, ""),
1509         MRS_FIELD_VALUE(ID_AA64PFR1_MTE_MTE, "MTE"),
1510         MRS_FIELD_VALUE(ID_AA64PFR1_MTE_MTE2, "MTE2"),
1511         MRS_FIELD_VALUE(ID_AA64PFR1_MTE_MTE3, "MTE3"),
1512         MRS_FIELD_VALUE_END,
1513 };
1514
1515 static const struct mrs_field_value id_aa64pfr1_ssbs[] = {
1516         MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_NONE, ""),
1517         MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE, "PSTATE.SSBS"),
1518         MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE_MSR, "PSTATE.SSBS MSR"),
1519         MRS_FIELD_VALUE_END,
1520 };
1521
1522 static const struct mrs_field_hwcap id_aa64pfr1_ssbs_caps[] = {
1523         MRS_HWCAP(1, HWCAP_SSBS, ID_AA64PFR1_SSBS_PSTATE),
1524         MRS_HWCAP_END
1525 };
1526
1527 static const struct mrs_field_value id_aa64pfr1_bt[] = {
1528         MRS_FIELD_VALUE(ID_AA64PFR1_BT_NONE, ""),
1529         MRS_FIELD_VALUE(ID_AA64PFR1_BT_IMPL, "BTI"),
1530         MRS_FIELD_VALUE_END,
1531 };
1532
1533 #if 0
1534 /* Enable when we add BTI support */
1535 static const struct mrs_field_hwcap id_aa64pfr1_bt_caps[] = {
1536         MRS_HWCAP(2, HWCAP2_BTI, ID_AA64PFR1_BT_IMPL),
1537         MRS_HWCAP_END
1538 };
1539 #endif
1540
1541 static const struct mrs_field id_aa64pfr1_fields[] = {
1542         MRS_FIELD(ID_AA64PFR1, NMI, false, MRS_EXACT, id_aa64pfr1_nmi),
1543         MRS_FIELD(ID_AA64PFR1, CSV2_frac, false, MRS_EXACT,
1544             id_aa64pfr1_csv2_frac),
1545         MRS_FIELD(ID_AA64PFR1, RNDR_trap, false, MRS_EXACT,
1546             id_aa64pfr1_rndr_trap),
1547         MRS_FIELD(ID_AA64PFR1, SME, false, MRS_EXACT, id_aa64pfr1_sme),
1548         MRS_FIELD(ID_AA64PFR1, MPAM_frac, false, MRS_EXACT,
1549             id_aa64pfr1_mpam_frac),
1550         MRS_FIELD(ID_AA64PFR1, RAS_frac, false, MRS_EXACT,
1551             id_aa64pfr1_ras_frac),
1552         MRS_FIELD(ID_AA64PFR1, MTE, false, MRS_EXACT, id_aa64pfr1_mte),
1553         MRS_FIELD_HWCAP(ID_AA64PFR1, SSBS, false, MRS_LOWER, id_aa64pfr1_ssbs,
1554             id_aa64pfr1_ssbs_caps),
1555         MRS_FIELD(ID_AA64PFR1, BT, false, MRS_EXACT, id_aa64pfr1_bt),
1556         MRS_FIELD_END,
1557 };
1558
1559
1560 #ifdef NOTYET
1561 /* ID_AA64PFR2_EL1 */
1562 static const struct mrs_field id_aa64pfr2_fields[] = {
1563         MRS_FIELD_END,
1564 };
1565 #endif
1566
1567
1568 /* ID_AA64ZFR0_EL1 */
1569 static const struct mrs_field_value id_aa64zfr0_f64mm[] = {
1570         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, F64MM, NONE, IMPL),
1571         MRS_FIELD_VALUE_END,
1572 };
1573
1574 static const struct mrs_field_value id_aa64zfr0_f32mm[] = {
1575         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, F32MM, NONE, IMPL),
1576         MRS_FIELD_VALUE_END,
1577 };
1578
1579 static const struct mrs_field_value id_aa64zfr0_i8mm[] = {
1580         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, I8MM, NONE, IMPL),
1581         MRS_FIELD_VALUE_END,
1582 };
1583
1584 static const struct mrs_field_value id_aa64zfr0_sm4[] = {
1585         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, SM4, NONE, IMPL),
1586         MRS_FIELD_VALUE_END,
1587 };
1588
1589 static const struct mrs_field_value id_aa64zfr0_sha3[] = {
1590         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, SHA3, NONE, IMPL),
1591         MRS_FIELD_VALUE_END,
1592 };
1593
1594 static const struct mrs_field_value id_aa64zfr0_bf16[] = {
1595         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, BF16, NONE, BASE),
1596         MRS_FIELD_VALUE(ID_AA64ZFR0_BF16_EBF, "BF16+EBF"),
1597         MRS_FIELD_VALUE_END,
1598 };
1599
1600 static const struct mrs_field_value id_aa64zfr0_bitperm[] = {
1601         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, BitPerm, NONE, IMPL),
1602         MRS_FIELD_VALUE_END,
1603 };
1604
1605 static const struct mrs_field_value id_aa64zfr0_aes[] = {
1606         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, AES, NONE, BASE),
1607         MRS_FIELD_VALUE(ID_AA64ZFR0_AES_PMULL, "AES+PMULL"),
1608         MRS_FIELD_VALUE_END,
1609 };
1610
1611 static const struct mrs_field_value id_aa64zfr0_svever[] = {
1612         MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE1, "SVE1"),
1613         MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE2, "SVE2"),
1614         MRS_FIELD_VALUE_END,
1615 };
1616
1617 static const struct mrs_field id_aa64zfr0_fields[] = {
1618         MRS_FIELD(ID_AA64ZFR0, F64MM, false, MRS_EXACT, id_aa64zfr0_f64mm),
1619         MRS_FIELD(ID_AA64ZFR0, F32MM, false, MRS_EXACT, id_aa64zfr0_f32mm),
1620         MRS_FIELD(ID_AA64ZFR0, I8MM, false, MRS_EXACT, id_aa64zfr0_i8mm),
1621         MRS_FIELD(ID_AA64ZFR0, SM4, false, MRS_EXACT, id_aa64zfr0_sm4),
1622         MRS_FIELD(ID_AA64ZFR0, SHA3, false, MRS_EXACT, id_aa64zfr0_sha3),
1623         MRS_FIELD(ID_AA64ZFR0, BF16, false, MRS_EXACT, id_aa64zfr0_bf16),
1624         MRS_FIELD(ID_AA64ZFR0, BitPerm, false, MRS_EXACT, id_aa64zfr0_bitperm),
1625         MRS_FIELD(ID_AA64ZFR0, AES, false, MRS_EXACT, id_aa64zfr0_aes),
1626         MRS_FIELD(ID_AA64ZFR0, SVEver, false, MRS_EXACT, id_aa64zfr0_svever),
1627         MRS_FIELD_END,
1628 };
1629
1630
1631 #ifdef COMPAT_FREEBSD32
1632 /* ID_ISAR5_EL1 */
1633 static const struct mrs_field_value id_isar5_vcma[] = {
1634         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, VCMA, NONE, IMPL),
1635         MRS_FIELD_VALUE_END,
1636 };
1637
1638 static const struct mrs_field_value id_isar5_rdm[] = {
1639         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, RDM, NONE, IMPL),
1640         MRS_FIELD_VALUE_END,
1641 };
1642
1643 static const struct mrs_field_value id_isar5_crc32[] = {
1644         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, CRC32, NONE, IMPL),
1645         MRS_FIELD_VALUE_END,
1646 };
1647
1648 static const struct mrs_field_hwcap id_isar5_crc32_caps[] = {
1649         MRS_HWCAP(2, HWCAP32_2_CRC32, ID_ISAR5_CRC32_IMPL),
1650         MRS_HWCAP_END
1651 };
1652
1653 static const struct mrs_field_value id_isar5_sha2[] = {
1654         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA2, NONE, IMPL),
1655         MRS_FIELD_VALUE_END,
1656 };
1657
1658 static const struct mrs_field_hwcap id_isar5_sha2_caps[] = {
1659         MRS_HWCAP(2, HWCAP32_2_SHA2, ID_ISAR5_SHA2_IMPL),
1660         MRS_HWCAP_END
1661 };
1662
1663 static const struct mrs_field_value id_isar5_sha1[] = {
1664         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA1, NONE, IMPL),
1665         MRS_FIELD_VALUE_END,
1666 };
1667
1668 static const struct mrs_field_hwcap id_isar5_sha1_caps[] = {
1669         MRS_HWCAP(2, HWCAP32_2_SHA1, ID_ISAR5_SHA1_IMPL),
1670         MRS_HWCAP_END
1671 };
1672
1673 static const struct mrs_field_value id_isar5_aes[] = {
1674         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, AES, NONE, BASE),
1675         MRS_FIELD_VALUE(ID_ISAR5_AES_VMULL, "AES+VMULL"),
1676         MRS_FIELD_VALUE_END,
1677 };
1678
1679 static const struct mrs_field_hwcap id_isar5_aes_caps[] = {
1680         MRS_HWCAP(2, HWCAP32_2_AES, ID_ISAR5_AES_BASE),
1681         MRS_HWCAP(2, HWCAP32_2_PMULL, ID_ISAR5_AES_VMULL),
1682         MRS_HWCAP_END
1683 };
1684
1685 static const struct mrs_field_value id_isar5_sevl[] = {
1686         MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SEVL, NOP, IMPL),
1687         MRS_FIELD_VALUE_END,
1688 };
1689
1690 static const struct mrs_field id_isar5_fields[] = {
1691         MRS_FIELD(ID_ISAR5, VCMA, false, MRS_LOWER, id_isar5_vcma),
1692         MRS_FIELD(ID_ISAR5, RDM, false, MRS_LOWER, id_isar5_rdm),
1693         MRS_FIELD_HWCAP(ID_ISAR5, CRC32, false, MRS_LOWER, id_isar5_crc32,
1694             id_isar5_crc32_caps),
1695         MRS_FIELD_HWCAP(ID_ISAR5, SHA2, false, MRS_LOWER, id_isar5_sha2,
1696             id_isar5_sha2_caps),
1697         MRS_FIELD_HWCAP(ID_ISAR5, SHA1, false, MRS_LOWER, id_isar5_sha1,
1698             id_isar5_sha1_caps),
1699         MRS_FIELD_HWCAP(ID_ISAR5, AES, false, MRS_LOWER, id_isar5_aes,
1700             id_isar5_aes_caps),
1701         MRS_FIELD(ID_ISAR5, SEVL, false, MRS_LOWER, id_isar5_sevl),
1702         MRS_FIELD_END,
1703 };
1704
1705 /* MVFR0 */
1706 static const struct mrs_field_value mvfr0_fpround[] = {
1707         MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPRound, NONE, IMPL),
1708         MRS_FIELD_VALUE_END,
1709 };
1710
1711 static const struct mrs_field_value mvfr0_fpsqrt[] = {
1712         MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPSqrt, NONE, IMPL),
1713         MRS_FIELD_VALUE_END,
1714 };
1715
1716 static const struct mrs_field_value mvfr0_fpdivide[] = {
1717         MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPDivide, NONE, IMPL),
1718         MRS_FIELD_VALUE_END,
1719 };
1720
1721 static const struct mrs_field_value mvfr0_fptrap[] = {
1722         MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPTrap, NONE, IMPL),
1723         MRS_FIELD_VALUE_END,
1724 };
1725
1726 static const struct mrs_field_value mvfr0_fpdp[] = {
1727         MRS_FIELD_VALUE(MVFR0_FPDP_NONE, ""),
1728         MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v2, "DP VFPv2"),
1729         MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v3_v4, "DP VFPv3+v4"),
1730         MRS_FIELD_VALUE_END,
1731 };
1732
1733 static const struct mrs_field_hwcap mvfr0_fpdp_caps[] = {
1734         MRS_HWCAP(1, HWCAP32_VFP, MVFR0_FPDP_VFP_v2),
1735         MRS_HWCAP(1, HWCAP32_VFPv3, MVFR0_FPDP_VFP_v3_v4),
1736         MRS_HWCAP_END
1737 };
1738
1739 static const struct mrs_field_value mvfr0_fpsp[] = {
1740         MRS_FIELD_VALUE(MVFR0_FPSP_NONE, ""),
1741         MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v2, "SP VFPv2"),
1742         MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v3_v4, "SP VFPv3+v4"),
1743         MRS_FIELD_VALUE_END,
1744 };
1745
1746 static const struct mrs_field_value mvfr0_simdreg[] = {
1747         MRS_FIELD_VALUE(MVFR0_SIMDReg_NONE, ""),
1748         MRS_FIELD_VALUE(MVFR0_SIMDReg_FP, "FP 16x64"),
1749         MRS_FIELD_VALUE(MVFR0_SIMDReg_AdvSIMD, "AdvSIMD"),
1750         MRS_FIELD_VALUE_END,
1751 };
1752
1753 static const struct mrs_field mvfr0_fields[] = {
1754         MRS_FIELD(MVFR0, FPRound, false, MRS_LOWER, mvfr0_fpround),
1755         MRS_FIELD(MVFR0, FPSqrt, false, MRS_LOWER, mvfr0_fpsqrt),
1756         MRS_FIELD(MVFR0, FPDivide, false, MRS_LOWER, mvfr0_fpdivide),
1757         MRS_FIELD(MVFR0, FPTrap, false, MRS_LOWER, mvfr0_fptrap),
1758         MRS_FIELD_HWCAP(MVFR0, FPDP, false, MRS_LOWER, mvfr0_fpdp,
1759             mvfr0_fpdp_caps),
1760         MRS_FIELD(MVFR0, FPSP, false, MRS_LOWER, mvfr0_fpsp),
1761         MRS_FIELD(MVFR0, SIMDReg, false, MRS_LOWER, mvfr0_simdreg),
1762         MRS_FIELD_END,
1763 };
1764
1765 /* MVFR1 */
1766 static const struct mrs_field_value mvfr1_simdfmac[] = {
1767         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDFMAC, NONE, IMPL),
1768         MRS_FIELD_VALUE_END,
1769 };
1770
1771 static const struct mrs_field_hwcap mvfr1_simdfmac_caps[] = {
1772         MRS_HWCAP(1, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL),
1773         MRS_HWCAP_END
1774 };
1775
1776 static const struct mrs_field_value mvfr1_fphp[] = {
1777         MRS_FIELD_VALUE(MVFR1_FPHP_NONE, ""),
1778         MRS_FIELD_VALUE(MVFR1_FPHP_CONV_SP, "FPHP SP Conv"),
1779         MRS_FIELD_VALUE(MVFR1_FPHP_CONV_DP, "FPHP DP Conv"),
1780         MRS_FIELD_VALUE(MVFR1_FPHP_ARITH, "FPHP Arith"),
1781         MRS_FIELD_VALUE_END,
1782 };
1783
1784 static const struct mrs_field_value mvfr1_simdhp[] = {
1785         MRS_FIELD_VALUE(MVFR1_SIMDHP_NONE, ""),
1786         MRS_FIELD_VALUE(MVFR1_SIMDHP_CONV_SP, "SIMDHP SP Conv"),
1787         MRS_FIELD_VALUE(MVFR1_SIMDHP_ARITH, "SIMDHP Arith"),
1788         MRS_FIELD_VALUE_END,
1789 };
1790
1791 static const struct mrs_field_value mvfr1_simdsp[] = {
1792         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDSP, NONE, IMPL),
1793         MRS_FIELD_VALUE_END,
1794 };
1795
1796 static const struct mrs_field_value mvfr1_simdint[] = {
1797         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDInt, NONE, IMPL),
1798         MRS_FIELD_VALUE_END,
1799 };
1800
1801 static const struct mrs_field_value mvfr1_simdls[] = {
1802         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDLS, NONE, IMPL),
1803         MRS_FIELD_VALUE_END,
1804 };
1805
1806 static const struct mrs_field_hwcap mvfr1_simdls_caps[] = {
1807         MRS_HWCAP(1, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL),
1808         MRS_HWCAP_END
1809 };
1810
1811 static const struct mrs_field_value mvfr1_fpdnan[] = {
1812         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPDNaN, NONE, IMPL),
1813         MRS_FIELD_VALUE_END,
1814 };
1815
1816 static const struct mrs_field_value mvfr1_fpftz[] = {
1817         MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPFtZ, NONE, IMPL),
1818         MRS_FIELD_VALUE_END,
1819 };
1820
1821 static const struct mrs_field mvfr1_fields[] = {
1822         MRS_FIELD_HWCAP(MVFR1, SIMDFMAC, false, MRS_LOWER, mvfr1_simdfmac,
1823             mvfr1_simdfmac_caps),
1824         MRS_FIELD(MVFR1, FPHP, false, MRS_LOWER, mvfr1_fphp),
1825         MRS_FIELD(MVFR1, SIMDHP, false, MRS_LOWER, mvfr1_simdhp),
1826         MRS_FIELD(MVFR1, SIMDSP, false, MRS_LOWER, mvfr1_simdsp),
1827         MRS_FIELD(MVFR1, SIMDInt, false, MRS_LOWER, mvfr1_simdint),
1828         MRS_FIELD_HWCAP(MVFR1, SIMDLS, false, MRS_LOWER, mvfr1_simdls,
1829             mvfr1_simdls_caps),
1830         MRS_FIELD(MVFR1, FPDNaN, false, MRS_LOWER, mvfr1_fpdnan),
1831         MRS_FIELD(MVFR1, FPFtZ, false, MRS_LOWER, mvfr1_fpftz),
1832         MRS_FIELD_END,
1833 };
1834 #endif /* COMPAT_FREEBSD32 */
1835
1836 struct mrs_user_reg {
1837         u_int           reg;
1838         u_int           CRm;
1839         u_int           Op2;
1840         bool            is64bit;
1841         size_t          offset;
1842         const struct mrs_field *fields;
1843 };
1844
1845 #define USER_REG(name, field_name, _is64bit)                            \
1846         {                                                               \
1847                 .reg = name,                                            \
1848                 .CRm = name##_CRm,                                      \
1849                 .Op2 = name##_op2,                                      \
1850                 .offset = __offsetof(struct cpu_desc, field_name),      \
1851                 .fields = field_name##_fields,                          \
1852                 .is64bit = _is64bit,                                    \
1853         }
1854 static const struct mrs_user_reg user_regs[] = {
1855         USER_REG(ID_AA64AFR0_EL1, id_aa64afr0, true),
1856         USER_REG(ID_AA64AFR1_EL1, id_aa64afr1, true),
1857
1858         USER_REG(ID_AA64DFR0_EL1, id_aa64dfr0, true),
1859         USER_REG(ID_AA64DFR1_EL1, id_aa64dfr1, true),
1860
1861         USER_REG(ID_AA64ISAR0_EL1, id_aa64isar0, true),
1862         USER_REG(ID_AA64ISAR1_EL1, id_aa64isar1, true),
1863         USER_REG(ID_AA64ISAR2_EL1, id_aa64isar2, true),
1864
1865         USER_REG(ID_AA64MMFR0_EL1, id_aa64mmfr0, true),
1866         USER_REG(ID_AA64MMFR1_EL1, id_aa64mmfr1, true),
1867         USER_REG(ID_AA64MMFR2_EL1, id_aa64mmfr2, true),
1868 #ifdef NOTYET
1869         USER_REG(ID_AA64MMFR3_EL1, id_aa64mmfr3, true),
1870         USER_REG(ID_AA64MMFR4_EL1, id_aa64mmfr4, true),
1871 #endif
1872
1873         USER_REG(ID_AA64PFR0_EL1, id_aa64pfr0, true),
1874         USER_REG(ID_AA64PFR1_EL1, id_aa64pfr1, true),
1875 #ifdef NOTYET
1876         USER_REG(ID_AA64PFR2_EL1, id_aa64pfr2, true),
1877 #endif
1878
1879         USER_REG(ID_AA64ZFR0_EL1, id_aa64zfr0, true),
1880
1881 #ifdef COMPAT_FREEBSD32
1882         USER_REG(ID_ISAR5_EL1, id_isar5, false),
1883
1884         USER_REG(MVFR0_EL1, mvfr0, false),
1885         USER_REG(MVFR1_EL1, mvfr1, false),
1886 #endif /* COMPAT_FREEBSD32 */
1887 };
1888
1889 #define CPU_DESC_FIELD(desc, idx)                                       \
1890     *(uint64_t *)((char *)&(desc) + user_regs[(idx)].offset)
1891
1892 static int
1893 user_mrs_handler(vm_offset_t va, uint32_t insn, struct trapframe *frame,
1894     uint32_t esr)
1895 {
1896         uint64_t value;
1897         int CRm, Op2, i, reg;
1898
1899         if ((insn & MRS_MASK) != MRS_VALUE)
1900                 return (0);
1901
1902         /*
1903          * We only emulate Op0 == 3, Op1 == 0, CRn == 0, CRm == {0, 4-7}.
1904          * These are in the EL1 CPU identification space.
1905          * CRm == 0 holds MIDR_EL1, MPIDR_EL1, and REVID_EL1.
1906          * CRm == {4-7} holds the ID_AA64 registers.
1907          *
1908          * For full details see the ARMv8 ARM (ARM DDI 0487C.a)
1909          * Table D9-2 System instruction encodings for non-Debug System
1910          * register accesses.
1911          */
1912         if (mrs_Op0(insn) != 3 || mrs_Op1(insn) != 0 || mrs_CRn(insn) != 0)
1913                 return (0);
1914
1915         CRm = mrs_CRm(insn);
1916         if (CRm > 7 || (CRm < 4 && CRm != 0))
1917                 return (0);
1918
1919         Op2 = mrs_Op2(insn);
1920         value = 0;
1921
1922         for (i = 0; i < nitems(user_regs); i++) {
1923                 if (user_regs[i].CRm == CRm && user_regs[i].Op2 == Op2) {
1924                         if (SV_CURPROC_ABI() == SV_ABI_FREEBSD)
1925                                 value = CPU_DESC_FIELD(user_cpu_desc, i);
1926                         else
1927                                 value = CPU_DESC_FIELD(l_user_cpu_desc, i);
1928                         break;
1929                 }
1930         }
1931
1932         if (CRm == 0) {
1933                 switch (Op2) {
1934                 case 0:
1935                         value = READ_SPECIALREG(midr_el1);
1936                         break;
1937                 case 5:
1938                         value = READ_SPECIALREG(mpidr_el1);
1939                         break;
1940                 case 6:
1941                         value = READ_SPECIALREG(revidr_el1);
1942                         break;
1943                 default:
1944                         return (0);
1945                 }
1946         }
1947
1948         /*
1949          * We will handle this instruction, move to the next so we
1950          * don't trap here again.
1951          */
1952         frame->tf_elr += INSN_SIZE;
1953
1954         reg = MRS_REGISTER(insn);
1955         /* If reg is 31 then write to xzr, i.e. do nothing */
1956         if (reg == 31)
1957                 return (1);
1958
1959         if (reg < nitems(frame->tf_x))
1960                 frame->tf_x[reg] = value;
1961         else if (reg == 30)
1962                 frame->tf_lr = value;
1963
1964         return (1);
1965 }
1966
1967 /*
1968  * Compares two field values that may be signed or unsigned.
1969  * Returns:
1970  *  < 0 when a is less than b
1971  *  = 0 when a equals b
1972  *  > 0 when a is greater than b
1973  */
1974 static int
1975 mrs_field_cmp(uint64_t a, uint64_t b, u_int shift, int width, bool sign)
1976 {
1977         uint64_t mask;
1978
1979         KASSERT(width > 0 && width < 64, ("%s: Invalid width %d", __func__,
1980             width));
1981
1982         mask = (1ul << width) - 1;
1983         /* Move the field to the lower bits */
1984         a = (a >> shift) & mask;
1985         b = (b >> shift) & mask;
1986
1987         if (sign) {
1988                 /*
1989                  * The field is signed. Toggle the upper bit so the comparison
1990                  * works on unsigned values as this makes positive numbers,
1991                  * i.e. those with a 0 bit, larger than negative numbers,
1992                  * i.e. those with a 1 bit, in an unsigned comparison.
1993                  */
1994                 a ^= 1ul << (width - 1);
1995                 b ^= 1ul << (width - 1);
1996         }
1997
1998         return (a - b);
1999 }
2000
2001 static uint64_t
2002 update_lower_register(uint64_t val, uint64_t new_val, u_int shift,
2003     int width, bool sign)
2004 {
2005         uint64_t mask;
2006
2007         KASSERT(width > 0 && width < 64, ("%s: Invalid width %d", __func__,
2008             width));
2009
2010         /*
2011          * If the new value is less than the existing value update it.
2012          */
2013         if (mrs_field_cmp(new_val, val, shift, width, sign) < 0) {
2014                 mask = (1ul << width) - 1;
2015                 val &= ~(mask << shift);
2016                 val |= new_val & (mask << shift);
2017         }
2018
2019         return (val);
2020 }
2021
2022 bool
2023 extract_user_id_field(u_int reg, u_int field_shift, uint8_t *val)
2024 {
2025         uint64_t value;
2026         int i;
2027
2028         for (i = 0; i < nitems(user_regs); i++) {
2029                 if (user_regs[i].reg == reg) {
2030                         value = CPU_DESC_FIELD(user_cpu_desc, i);
2031                         *val = value >> field_shift;
2032                         return (true);
2033                 }
2034         }
2035
2036         return (false);
2037 }
2038
2039 bool
2040 get_kernel_reg(u_int reg, uint64_t *val)
2041 {
2042         int i;
2043
2044         for (i = 0; i < nitems(user_regs); i++) {
2045                 if (user_regs[i].reg == reg) {
2046                         *val = CPU_DESC_FIELD(kern_cpu_desc, i);
2047                         return (true);
2048                 }
2049         }
2050
2051         return (false);
2052 }
2053
2054 /*
2055  * Fetch the specified register's value, ensuring that individual field values
2056  * do not exceed those in the mask.
2057  */
2058 bool
2059 get_kernel_reg_masked(u_int reg, uint64_t *valp, uint64_t mask)
2060 {
2061         const struct mrs_field *fields;
2062         uint64_t val;
2063
2064         for (int i = 0; i < nitems(user_regs); i++) {
2065                 if (user_regs[i].reg == reg) {
2066                         val = CPU_DESC_FIELD(kern_cpu_desc, i);
2067                         fields = user_regs[i].fields;
2068                         for (int j = 0; fields[j].type != 0; j++) {
2069                                 mask = update_lower_register(mask, val,
2070                                     fields[j].shift, 4, fields[j].sign);
2071                         }
2072                         *valp = mask;
2073                         return (true);
2074                 }
2075         }
2076
2077         return (false);
2078 }
2079
2080 static uint64_t
2081 update_special_reg_field(uint64_t user_reg, u_int type, uint64_t value,
2082     u_int shift, bool sign)
2083 {
2084         switch (type & MRS_TYPE_MASK) {
2085         case MRS_EXACT:
2086                 user_reg &= ~(0xful << shift);
2087                 user_reg |= (uint64_t)MRS_EXACT_FIELD(type) << shift;
2088                 break;
2089         case MRS_LOWER:
2090                 user_reg = update_lower_register(user_reg, value, shift, 4,
2091                     sign);
2092                 break;
2093         default:
2094                 panic("Invalid field type: %d", type);
2095         }
2096
2097         return (user_reg);
2098 }
2099
2100 void
2101 update_special_regs(u_int cpu)
2102 {
2103         struct cpu_desc *desc;
2104         const struct mrs_field *fields;
2105         uint64_t l_user_reg, user_reg, kern_reg, value;
2106         int i, j;
2107
2108         if (cpu == 0) {
2109                 /* Create a user visible cpu description with safe values */
2110                 memset(&user_cpu_desc, 0, sizeof(user_cpu_desc));
2111                 /* Safe values for these registers */
2112                 user_cpu_desc.id_aa64pfr0 = ID_AA64PFR0_AdvSIMD_NONE |
2113                     ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 |
2114                     ID_AA64PFR0_EL0_64;
2115                 user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8;
2116                 /* Create the Linux user visible cpu description */
2117                 memcpy(&l_user_cpu_desc, &user_cpu_desc, sizeof(user_cpu_desc));
2118         }
2119
2120         desc = get_cpu_desc(cpu);
2121         for (i = 0; i < nitems(user_regs); i++) {
2122                 value = CPU_DESC_FIELD(*desc, i);
2123                 if (cpu == 0) {
2124                         kern_reg = value;
2125                         user_reg = value;
2126                         l_user_reg = value;
2127                 } else {
2128                         kern_reg = CPU_DESC_FIELD(kern_cpu_desc, i);
2129                         user_reg = CPU_DESC_FIELD(user_cpu_desc, i);
2130                         l_user_reg = CPU_DESC_FIELD(l_user_cpu_desc, i);
2131                 }
2132
2133                 fields = user_regs[i].fields;
2134                 for (j = 0; fields[j].type != 0; j++) {
2135                         /* Update the FreeBSD userspace ID register view */
2136                         user_reg = update_special_reg_field(user_reg,
2137                             fields[j].type >> MRS_TYPE_FBSD_SHIFT, value,
2138                             fields[j].shift, fields[j].sign);
2139
2140                         /* Update the Linux userspace ID register view */
2141                         l_user_reg = update_special_reg_field(l_user_reg,
2142                             fields[j].type >> MRS_TYPE_LNX_SHIFT, value,
2143                             fields[j].shift, fields[j].sign);
2144
2145                         /* Update the kernel ID register view */
2146                         kern_reg = update_lower_register(kern_reg, value,
2147                             fields[j].shift, 4, fields[j].sign);
2148                 }
2149
2150                 CPU_DESC_FIELD(kern_cpu_desc, i) = kern_reg;
2151                 CPU_DESC_FIELD(user_cpu_desc, i) = user_reg;
2152                 CPU_DESC_FIELD(l_user_cpu_desc, i) = l_user_reg;
2153         }
2154 }
2155
2156 void
2157 cpu_desc_init(void)
2158 {
2159         if (mp_ncpus == 1)
2160                 return;
2161
2162         /*
2163          * Allocate memory for the non-boot CPUs to store their registers.
2164          * As this is indexed by CPU ID we need to allocate space for CPUs
2165          * 1 to mp_maxid. Because of this mp_maxid is already the correct
2166          * number of elements.
2167          */
2168         cpu_desc = mallocarray(mp_maxid, sizeof(*cpu_desc), M_IDENTCPU,
2169             M_ZERO | M_WAITOK);
2170 }
2171
2172 /* HWCAP */
2173 bool __read_frequently lse_supported = false;
2174
2175 bool __read_frequently icache_aliasing = false;
2176 bool __read_frequently icache_vmid = false;
2177
2178 int64_t dcache_line_size;       /* The minimum D cache line size */
2179 int64_t icache_line_size;       /* The minimum I cache line size */
2180 int64_t idcache_line_size;      /* The minimum cache line size */
2181
2182 /*
2183  * Find the values to export to userspace as AT_HWCAP and AT_HWCAP2.
2184  */
2185 static void
2186 parse_cpu_features(bool is64bit, struct cpu_desc *cpu_desc, u_long *hwcap,
2187     u_long *hwcap2)
2188 {
2189         const struct mrs_field_hwcap *hwcaps;
2190         const struct mrs_field *fields;
2191         uint64_t min, reg;
2192         u_long *cur_hwcap;
2193         int i, j, k;
2194
2195         for (i = 0; i < nitems(user_regs); i++) {
2196                 if (user_regs[i].is64bit != is64bit)
2197                         continue;
2198
2199                 reg = CPU_DESC_FIELD(*cpu_desc, i);
2200                 fields = user_regs[i].fields;
2201                 for (j = 0; fields[j].type != 0; j++) {
2202                         hwcaps = fields[j].hwcaps;
2203                         if (hwcaps == NULL)
2204                                 continue;
2205
2206                         for (k = 0; hwcaps[k].hwcap_id != 0; k++) {
2207                                 KASSERT(hwcaps[k].hwcap_id == 1 ||
2208                                     hwcaps[k].hwcap_id == 2,
2209                                     ("%s: Invalid HWCAP ID %d", __func__,
2210                                     hwcaps[k].hwcap_id));
2211
2212                                 cur_hwcap = hwcaps[k].hwcap_id == 1 ?
2213                                     hwcap : hwcap2;
2214                                 min = hwcaps[k].min;
2215
2216                                 /*
2217                                  * If the field is greater than the minimum
2218                                  * value we can set the hwcap;
2219                                  */
2220                                 if (mrs_field_cmp(reg, min, fields[j].shift,
2221                                     4, fields[j].sign) >= 0) {
2222                                         *cur_hwcap |= hwcaps[k].hwcap_val;
2223                                 }
2224                         }
2225                 }
2226         }
2227 }
2228
2229 static void
2230 identify_cpu_sysinit(void *dummy __unused)
2231 {
2232         struct cpu_desc *desc, *prev_desc;
2233         int cpu;
2234         bool dic, idc;
2235
2236         dic = (allow_dic != 0);
2237         idc = (allow_idc != 0);
2238
2239         prev_desc = NULL;
2240         CPU_FOREACH(cpu) {
2241                 desc = get_cpu_desc(cpu);
2242                 if (cpu != 0) {
2243                         check_cpu_regs(cpu, desc, prev_desc);
2244                         update_special_regs(cpu);
2245                 }
2246
2247                 if (CTR_DIC_VAL(desc->ctr) == 0)
2248                         dic = false;
2249                 if (CTR_IDC_VAL(desc->ctr) == 0)
2250                         idc = false;
2251                 prev_desc = desc;
2252         }
2253
2254         /* Find the values to export to userspace as AT_HWCAP and AT_HWCAP2 */
2255         parse_cpu_features(true, &user_cpu_desc, &elf_hwcap, &elf_hwcap2);
2256         parse_cpu_features(true, &l_user_cpu_desc, &linux_elf_hwcap,
2257             &linux_elf_hwcap2);
2258 #ifdef COMPAT_FREEBSD32
2259         parse_cpu_features(false, &user_cpu_desc, &elf32_hwcap, &elf32_hwcap2);
2260 #endif
2261
2262         /* We export the CPUID registers */
2263         elf_hwcap |= HWCAP_CPUID;
2264         linux_elf_hwcap |= HWCAP_CPUID;
2265
2266 #ifdef COMPAT_FREEBSD32
2267         /* Set the default caps and any that need to check multiple fields */
2268         elf32_hwcap |= parse_cpu_features_hwcap32();
2269 #endif
2270
2271         if (dic && idc) {
2272                 arm64_icache_sync_range = &arm64_dic_idc_icache_sync_range;
2273                 if (bootverbose)
2274                         printf("Enabling DIC & IDC ICache sync\n");
2275         } else if (idc) {
2276                 arm64_icache_sync_range = &arm64_idc_aliasing_icache_sync_range;
2277                 if (bootverbose)
2278                         printf("Enabling IDC ICache sync\n");
2279         }
2280
2281         if ((elf_hwcap & HWCAP_ATOMICS) != 0) {
2282                 lse_supported = true;
2283                 if (bootverbose)
2284                         printf("Enabling LSE atomics in the kernel\n");
2285         }
2286 #ifdef LSE_ATOMICS
2287         if (!lse_supported)
2288                 panic("CPU does not support LSE atomic instructions");
2289 #endif
2290
2291         install_undef_handler(true, user_mrs_handler);
2292 }
2293 SYSINIT(identify_cpu, SI_SUB_CPU, SI_ORDER_MIDDLE, identify_cpu_sysinit, NULL);
2294
2295 static void
2296 cpu_features_sysinit(void *dummy __unused)
2297 {
2298         struct sbuf sb;
2299         struct cpu_desc *desc, *prev_desc;
2300         u_int cpu;
2301
2302         prev_desc = NULL;
2303         CPU_FOREACH(cpu) {
2304                 desc = get_cpu_desc(cpu);
2305                 print_cpu_features(cpu, desc, prev_desc);
2306                 prev_desc = desc;
2307         }
2308
2309         /* Fill in cpu_model for the hw.model sysctl */
2310         sbuf_new(&sb, cpu_model, sizeof(cpu_model), SBUF_FIXEDLEN);
2311         print_cpu_midr(&sb, 0);
2312
2313         sbuf_finish(&sb);
2314         sbuf_delete(&sb);
2315
2316         free(cpu_desc, M_IDENTCPU);
2317 }
2318 /* Log features before APs are released and start printing to the dmesg. */
2319 SYSINIT(cpu_features, SI_SUB_SMP - 1, SI_ORDER_ANY, cpu_features_sysinit, NULL);
2320
2321 static void
2322 tcr_set_e0pd1(void *arg __unused)
2323 {
2324         uint64_t tcr;
2325
2326         tcr = READ_SPECIALREG(tcr_el1);
2327         tcr |= TCR_E0PD1;
2328         WRITE_SPECIALREG(tcr_el1, tcr);
2329         isb();
2330 }
2331
2332 /* Enable support for more recent architecture features */
2333 static void
2334 cpu_feat_support(void *arg __unused)
2335 {
2336         /*
2337          * If FEAT_E0PD is supported use it to cause faults without a page
2338          * table walk if userspace tries to access kernel memory.
2339          */
2340         if (ID_AA64MMFR2_E0PD_VAL(kern_cpu_desc.id_aa64mmfr2) !=
2341             ID_AA64MMFR2_E0PD_NONE)
2342                 smp_rendezvous(NULL, tcr_set_e0pd1, NULL, NULL);
2343 }
2344 SYSINIT(cpu_feat_support, SI_SUB_SMP, SI_ORDER_ANY, cpu_feat_support, NULL);
2345
2346 #ifdef COMPAT_FREEBSD32
2347 static u_long
2348 parse_cpu_features_hwcap32(void)
2349 {
2350         u_long hwcap = HWCAP32_DEFAULT;
2351
2352         if ((MVFR1_SIMDLS_VAL(user_cpu_desc.mvfr1) >=
2353              MVFR1_SIMDLS_IMPL) &&
2354             (MVFR1_SIMDInt_VAL(user_cpu_desc.mvfr1) >=
2355              MVFR1_SIMDInt_IMPL) &&
2356             (MVFR1_SIMDSP_VAL(user_cpu_desc.mvfr1) >=
2357              MVFR1_SIMDSP_IMPL))
2358                 hwcap |= HWCAP32_NEON;
2359
2360         return (hwcap);
2361 }
2362 #endif /* COMPAT_FREEBSD32 */
2363
2364 static void
2365 print_ctr_fields(struct sbuf *sb, uint64_t reg, const void *arg __unused)
2366 {
2367
2368         sbuf_printf(sb, "%u byte D-cacheline,", CTR_DLINE_SIZE(reg));
2369         sbuf_printf(sb, "%u byte I-cacheline,", CTR_ILINE_SIZE(reg));
2370         reg &= ~(CTR_DLINE_MASK | CTR_ILINE_MASK);
2371
2372         switch(CTR_L1IP_VAL(reg)) {
2373         case CTR_L1IP_VPIPT:
2374                 sbuf_printf(sb, "VPIPT");
2375                 break;
2376         case CTR_L1IP_AIVIVT:
2377                 sbuf_printf(sb, "AIVIVT");
2378                 break;
2379         case CTR_L1IP_VIPT:
2380                 sbuf_printf(sb, "VIPT");
2381                 break;
2382         case CTR_L1IP_PIPT:
2383                 sbuf_printf(sb, "PIPT");
2384                 break;
2385         }
2386         sbuf_printf(sb, " ICache,");
2387         reg &= ~CTR_L1IP_MASK;
2388
2389         sbuf_printf(sb, "%d byte ERG,", CTR_ERG_SIZE(reg));
2390         sbuf_printf(sb, "%d byte CWG", CTR_CWG_SIZE(reg));
2391         reg &= ~(CTR_ERG_MASK | CTR_CWG_MASK);
2392
2393         if (CTR_IDC_VAL(reg) != 0)
2394                 sbuf_printf(sb, ",IDC");
2395         if (CTR_DIC_VAL(reg) != 0)
2396                 sbuf_printf(sb, ",DIC");
2397         reg &= ~(CTR_IDC_MASK | CTR_DIC_MASK);
2398         reg &= ~CTR_RES1;
2399
2400         if (reg != 0)
2401                 sbuf_printf(sb, ",%lx", reg);
2402 }
2403
2404 static void
2405 print_register(struct sbuf *sb, const char *reg_name, uint64_t reg,
2406     void (*print_fields)(struct sbuf *, uint64_t, const void *),
2407     const void *arg)
2408 {
2409
2410         sbuf_printf(sb, "%29s = <", reg_name);
2411
2412         print_fields(sb, reg, arg);
2413
2414         sbuf_finish(sb);
2415         printf("%s>\n", sbuf_data(sb));
2416         sbuf_clear(sb);
2417 }
2418
2419 static void
2420 print_id_fields(struct sbuf *sb, uint64_t reg, const void *arg)
2421 {
2422         const struct mrs_field *fields = arg;
2423         const struct mrs_field_value *fv;
2424         int field, i, j, printed;
2425
2426 #define SEP_STR ((printed++) == 0) ? "" : ","
2427         printed = 0;
2428         for (i = 0; fields[i].type != 0; i++) {
2429                 fv = fields[i].values;
2430
2431                 /* TODO: Handle with an unknown message */
2432                 if (fv == NULL)
2433                         continue;
2434
2435                 field = (reg & fields[i].mask) >> fields[i].shift;
2436                 for (j = 0; fv[j].desc != NULL; j++) {
2437                         if ((fv[j].value >> fields[i].shift) != field)
2438                                 continue;
2439
2440                         if (fv[j].desc[0] != '\0')
2441                                 sbuf_printf(sb, "%s%s", SEP_STR, fv[j].desc);
2442                         break;
2443                 }
2444                 if (fv[j].desc == NULL)
2445                         sbuf_printf(sb, "%sUnknown %s(%x)", SEP_STR,
2446                             fields[i].name, field);
2447
2448                 reg &= ~(0xful << fields[i].shift);
2449         }
2450
2451         if (reg != 0)
2452                 sbuf_printf(sb, "%s%#lx", SEP_STR, reg);
2453 #undef SEP_STR
2454 }
2455
2456 static void
2457 print_id_register(struct sbuf *sb, const char *reg_name, uint64_t reg,
2458     const struct mrs_field *fields)
2459 {
2460
2461         print_register(sb, reg_name, reg, print_id_fields, fields);
2462 }
2463
2464 static void
2465 print_cpu_midr(struct sbuf *sb, u_int cpu)
2466 {
2467         const struct cpu_parts *cpu_partsp;
2468         const char *cpu_impl_name;
2469         const char *cpu_part_name;
2470         u_int midr;
2471         u_int impl_id;
2472         u_int part_id;
2473
2474         midr = pcpu_find(cpu)->pc_midr;
2475
2476         cpu_impl_name = NULL;
2477         cpu_partsp = NULL;
2478         impl_id = CPU_IMPL(midr);
2479         for (int i = 0; cpu_implementers[i].impl_name != NULL; i++) {
2480                 if (impl_id == cpu_implementers[i].impl_id) {
2481                         cpu_impl_name = cpu_implementers[i].impl_name;
2482                         cpu_partsp = cpu_implementers[i].cpu_parts;
2483                         break;
2484                 }
2485         }
2486         /* Unknown implementer, so unknown part */
2487         if (cpu_impl_name == NULL) {
2488                 sbuf_printf(sb, "Unknown Implementer (midr: %08x)", midr);
2489                 return;
2490         }
2491
2492         KASSERT(cpu_partsp != NULL, ("%s: No parts table for implementer %s",
2493             __func__, cpu_impl_name));
2494
2495         cpu_part_name = NULL;
2496         part_id = CPU_PART(midr);
2497         for (int i = 0; cpu_partsp[i].part_name != NULL; i++) {
2498                 if (part_id == cpu_partsp[i].part_id) {
2499                         cpu_part_name = cpu_partsp[i].part_name;
2500                         break;
2501                 }
2502         }
2503         /* Known Implementer, Unknown part */
2504         if (cpu_part_name == NULL) {
2505                 sbuf_printf(sb, "%s Unknown CPU r%dp%d (midr: %08x)",
2506                     cpu_impl_name, CPU_VAR(midr), CPU_REV(midr), midr);
2507                 return;
2508         }
2509
2510         sbuf_printf(sb, "%s %s r%dp%d", cpu_impl_name,
2511             cpu_part_name, CPU_VAR(midr), CPU_REV(midr));
2512 }
2513
2514 static void
2515 print_cpu_cache(struct cpu_desc *desc, struct sbuf *sb, uint64_t ccs,
2516     bool icache, bool unified)
2517 {
2518         size_t cache_size;
2519         size_t line_size;
2520
2521         /* LineSize is Log2(S) - 4. */
2522         line_size = 1 << ((ccs & CCSIDR_LineSize_MASK) + 4);
2523         /*
2524          * Calculate cache size (sets * ways * line size).  There are different
2525          * formats depending on the FEAT_CCIDX bit in ID_AA64MMFR2 feature
2526          * register.
2527          */
2528         if ((desc->id_aa64mmfr2 & ID_AA64MMFR2_CCIDX_64))
2529                 cache_size = (CCSIDR_NSETS_64(ccs) + 1) *
2530                     (CCSIDR_ASSOC_64(ccs) + 1);
2531         else
2532                 cache_size = (CCSIDR_NSETS(ccs) + 1) * (CCSIDR_ASSOC(ccs) + 1);
2533
2534         cache_size *= line_size;
2535         sbuf_printf(sb, "%zuKB (%s)", cache_size / 1024,
2536             icache ? "instruction" : unified ? "unified" : "data");
2537 }
2538
2539 static void
2540 print_cpu_caches(struct sbuf *sb, struct cpu_desc *desc)
2541 {
2542         /* Print out each cache combination */
2543         uint64_t clidr;
2544         int i = 1;
2545         clidr = desc->clidr;
2546
2547         for (i = 0; (clidr & CLIDR_CTYPE_MASK) != 0; i++, clidr >>= 3) {
2548                 int j = 0;
2549                 int ctype_m = (clidr & CLIDR_CTYPE_MASK);
2550
2551                 sbuf_printf(sb, " L%d cache: ", i + 1);
2552                 if ((clidr & CLIDR_CTYPE_IO)) {
2553                         print_cpu_cache(desc, sb, desc->ccsidr[i][j++], true,
2554                             false);
2555                         /* If there's more, add to the line. */
2556                         if ((ctype_m & ~CLIDR_CTYPE_IO) != 0)
2557                                 sbuf_printf(sb, ", ");
2558                 }
2559                 if ((ctype_m & ~CLIDR_CTYPE_IO) != 0) {
2560                         print_cpu_cache(desc, sb, desc->ccsidr[i][j], false,
2561                             (clidr & CLIDR_CTYPE_UNIFIED));
2562                 }
2563                 sbuf_printf(sb, "\n");
2564
2565         }
2566         sbuf_finish(sb);
2567         printf("%s", sbuf_data(sb));
2568 }
2569
2570 static void
2571 print_cpu_features(u_int cpu, struct cpu_desc *desc,
2572     struct cpu_desc *prev_desc)
2573 {
2574         struct sbuf *sb;
2575
2576         sb = sbuf_new_auto();
2577         sbuf_printf(sb, "CPU%3u: ", cpu);
2578         print_cpu_midr(sb, cpu);
2579
2580         sbuf_cat(sb, " affinity:");
2581         switch(cpu_aff_levels) {
2582         default:
2583         case 4:
2584                 sbuf_printf(sb, " %2d", CPU_AFF3(desc->mpidr));
2585                 /* FALLTHROUGH */
2586         case 3:
2587                 sbuf_printf(sb, " %2d", CPU_AFF2(desc->mpidr));
2588                 /* FALLTHROUGH */
2589         case 2:
2590                 sbuf_printf(sb, " %2d", CPU_AFF1(desc->mpidr));
2591                 /* FALLTHROUGH */
2592         case 1:
2593         case 0: /* On UP this will be zero */
2594                 sbuf_printf(sb, " %2d", CPU_AFF0(desc->mpidr));
2595                 break;
2596         }
2597         sbuf_finish(sb);
2598         printf("%s\n", sbuf_data(sb));
2599         sbuf_clear(sb);
2600
2601         /*
2602          * There is a hardware errata where, if one CPU is performing a TLB
2603          * invalidation while another is performing a store-exclusive the
2604          * store-exclusive may return the wrong status. A workaround seems
2605          * to be to use an IPI to invalidate on each CPU, however given the
2606          * limited number of affected units (pass 1.1 is the evaluation
2607          * hardware revision), and the lack of information from Cavium
2608          * this has not been implemented.
2609          *
2610          * At the time of writing this the only information is from:
2611          * https://lkml.org/lkml/2016/8/4/722
2612          */
2613         /*
2614          * XXX: CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1 on its own also
2615          * triggers on pass 2.0+.
2616          */
2617         if (cpu == 0 && CPU_VAR(PCPU_GET(midr)) == 0 &&
2618             CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1)
2619                 printf("WARNING: ThunderX Pass 1.1 detected.\nThis has known "
2620                     "hardware bugs that may cause the incorrect operation of "
2621                     "atomic operations.\n");
2622
2623 #define SHOULD_PRINT_REG(_reg)                                          \
2624     (prev_desc == NULL || desc->_reg != prev_desc->_reg)
2625
2626         /* Cache Type Register */
2627         if (SHOULD_PRINT_REG(ctr)) {
2628                 print_register(sb, "Cache Type",
2629                     desc->ctr, print_ctr_fields, NULL);
2630         }
2631
2632         /* AArch64 Instruction Set Attribute Register 0 */
2633         if (SHOULD_PRINT_REG(id_aa64isar0))
2634                 print_id_register(sb, "Instruction Set Attributes 0",
2635                     desc->id_aa64isar0, id_aa64isar0_fields);
2636
2637         /* AArch64 Instruction Set Attribute Register 1 */
2638         if (SHOULD_PRINT_REG(id_aa64isar1))
2639                 print_id_register(sb, "Instruction Set Attributes 1",
2640                     desc->id_aa64isar1, id_aa64isar1_fields);
2641
2642         /* AArch64 Instruction Set Attribute Register 2 */
2643         if (SHOULD_PRINT_REG(id_aa64isar2))
2644                 print_id_register(sb, "Instruction Set Attributes 2",
2645                     desc->id_aa64isar2, id_aa64isar2_fields);
2646
2647         /* AArch64 Processor Feature Register 0 */
2648         if (SHOULD_PRINT_REG(id_aa64pfr0))
2649                 print_id_register(sb, "Processor Features 0",
2650                     desc->id_aa64pfr0, id_aa64pfr0_fields);
2651
2652         /* AArch64 Processor Feature Register 1 */
2653         if (SHOULD_PRINT_REG(id_aa64pfr1))
2654                 print_id_register(sb, "Processor Features 1",
2655                     desc->id_aa64pfr1, id_aa64pfr1_fields);
2656
2657 #ifdef NOTYET
2658         /* AArch64 Processor Feature Register 2 */
2659         if (SHOULD_PRINT_REG(id_aa64pfr2))
2660                 print_id_register(sb, "Processor Features 2",
2661                     desc->id_aa64pfr2, id_aa64pfr2_fields);
2662 #endif
2663
2664         /* AArch64 Memory Model Feature Register 0 */
2665         if (SHOULD_PRINT_REG(id_aa64mmfr0))
2666                 print_id_register(sb, "Memory Model Features 0",
2667                     desc->id_aa64mmfr0, id_aa64mmfr0_fields);
2668
2669         /* AArch64 Memory Model Feature Register 1 */
2670         if (SHOULD_PRINT_REG(id_aa64mmfr1))
2671                 print_id_register(sb, "Memory Model Features 1",
2672                     desc->id_aa64mmfr1, id_aa64mmfr1_fields);
2673
2674         /* AArch64 Memory Model Feature Register 2 */
2675         if (SHOULD_PRINT_REG(id_aa64mmfr2))
2676                 print_id_register(sb, "Memory Model Features 2",
2677                     desc->id_aa64mmfr2, id_aa64mmfr2_fields);
2678
2679 #ifdef NOTYET
2680         /* AArch64 Memory Model Feature Register 3 */
2681         if (SHOULD_PRINT_REG(id_aa64mmfr3))
2682                 print_id_register(sb, "Memory Model Features 3",
2683                     desc->id_aa64mmfr3, id_aa64mmfr3_fields);
2684
2685         /* AArch64 Memory Model Feature Register 4 */
2686         if (SHOULD_PRINT_REG(id_aa64mmfr4))
2687                 print_id_register(sb, "Memory Model Features 4",
2688                     desc->id_aa64mmfr4, id_aa64mmfr4_fields);
2689 #endif
2690
2691         /* AArch64 Debug Feature Register 0 */
2692         if (SHOULD_PRINT_REG(id_aa64dfr0))
2693                 print_id_register(sb, "Debug Features 0",
2694                     desc->id_aa64dfr0, id_aa64dfr0_fields);
2695
2696         /* AArch64 Memory Model Feature Register 1 */
2697         if (SHOULD_PRINT_REG(id_aa64dfr1))
2698                 print_id_register(sb, "Debug Features 1",
2699                     desc->id_aa64dfr1, id_aa64dfr1_fields);
2700
2701         /* AArch64 Auxiliary Feature Register 0 */
2702         if (SHOULD_PRINT_REG(id_aa64afr0))
2703                 print_id_register(sb, "Auxiliary Features 0",
2704                     desc->id_aa64afr0, id_aa64afr0_fields);
2705
2706         /* AArch64 Auxiliary Feature Register 1 */
2707         if (SHOULD_PRINT_REG(id_aa64afr1))
2708                 print_id_register(sb, "Auxiliary Features 1",
2709                     desc->id_aa64afr1, id_aa64afr1_fields);
2710
2711         /* AArch64 SVE Feature Register 0 */
2712         if (desc->have_sve) {
2713                 if (SHOULD_PRINT_REG(id_aa64zfr0) ||
2714                     !prev_desc->have_sve) {
2715                         print_id_register(sb, "SVE Features 0",
2716                             desc->id_aa64zfr0, id_aa64zfr0_fields);
2717                 }
2718         }
2719
2720 #ifdef COMPAT_FREEBSD32
2721         /* AArch32 Instruction Set Attribute Register 5 */
2722         if (SHOULD_PRINT_REG(id_isar5))
2723                 print_id_register(sb, "AArch32 Instruction Set Attributes 5",
2724                      desc->id_isar5, id_isar5_fields);
2725
2726         /* AArch32 Media and VFP Feature Register 0 */
2727         if (SHOULD_PRINT_REG(mvfr0))
2728                 print_id_register(sb, "AArch32 Media and VFP Features 0",
2729                      desc->mvfr0, mvfr0_fields);
2730
2731         /* AArch32 Media and VFP Feature Register 1 */
2732         if (SHOULD_PRINT_REG(mvfr1))
2733                 print_id_register(sb, "AArch32 Media and VFP Features 1",
2734                      desc->mvfr1, mvfr1_fields);
2735 #endif
2736         if (bootverbose)
2737                 print_cpu_caches(sb, desc);
2738
2739         sbuf_delete(sb);
2740         sb = NULL;
2741 #undef SHOULD_PRINT_REG
2742 #undef SEP_STR
2743 }
2744
2745 void
2746 identify_cache(uint64_t ctr)
2747 {
2748
2749         /* Identify the L1 cache type */
2750         switch (CTR_L1IP_VAL(ctr)) {
2751         case CTR_L1IP_PIPT:
2752                 break;
2753         case CTR_L1IP_VPIPT:
2754                 icache_vmid = true;
2755                 break;
2756         default:
2757         case CTR_L1IP_VIPT:
2758                 icache_aliasing = true;
2759                 break;
2760         }
2761
2762         if (dcache_line_size == 0) {
2763                 KASSERT(icache_line_size == 0, ("%s: i-cacheline size set: %ld",
2764                     __func__, icache_line_size));
2765
2766                 /* Get the D cache line size */
2767                 dcache_line_size = CTR_DLINE_SIZE(ctr);
2768                 /* And the same for the I cache */
2769                 icache_line_size = CTR_ILINE_SIZE(ctr);
2770
2771                 idcache_line_size = MIN(dcache_line_size, icache_line_size);
2772         }
2773
2774         if (dcache_line_size != CTR_DLINE_SIZE(ctr)) {
2775                 printf("WARNING: D-cacheline size mismatch %ld != %d\n",
2776                     dcache_line_size, CTR_DLINE_SIZE(ctr));
2777         }
2778
2779         if (icache_line_size != CTR_ILINE_SIZE(ctr)) {
2780                 printf("WARNING: I-cacheline size mismatch %ld != %d\n",
2781                     icache_line_size, CTR_ILINE_SIZE(ctr));
2782         }
2783 }
2784
2785 void
2786 identify_cpu(u_int cpu)
2787 {
2788         struct cpu_desc *desc;
2789         uint64_t clidr;
2790
2791         desc = get_cpu_desc(cpu);
2792         /* Save affinity for current CPU */
2793         desc->mpidr = get_mpidr();
2794         CPU_AFFINITY(cpu) = desc->mpidr & CPU_AFF_MASK;
2795
2796         desc->ctr = READ_SPECIALREG(ctr_el0);
2797         desc->id_aa64dfr0 = READ_SPECIALREG(id_aa64dfr0_el1);
2798         desc->id_aa64dfr1 = READ_SPECIALREG(id_aa64dfr1_el1);
2799         desc->id_aa64isar0 = READ_SPECIALREG(id_aa64isar0_el1);
2800         desc->id_aa64isar1 = READ_SPECIALREG(id_aa64isar1_el1);
2801         desc->id_aa64isar2 = READ_SPECIALREG(id_aa64isar2_el1);
2802         desc->id_aa64mmfr0 = READ_SPECIALREG(id_aa64mmfr0_el1);
2803         desc->id_aa64mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1);
2804         desc->id_aa64mmfr2 = READ_SPECIALREG(id_aa64mmfr2_el1);
2805 #ifdef NOTYET
2806         desc->id_aa64mmfr3 = READ_SPECIALREG(id_aa64mmfr3_el1);
2807         desc->id_aa64mmfr4 = READ_SPECIALREG(id_aa64mmfr4_el1);
2808 #endif
2809         desc->id_aa64pfr0 = READ_SPECIALREG(id_aa64pfr0_el1);
2810         desc->id_aa64pfr1 = READ_SPECIALREG(id_aa64pfr1_el1);
2811 #ifdef NOTYET
2812         desc->id_aa64pfr2 = READ_SPECIALREG(id_aa64pfr2_el1);
2813 #endif
2814
2815         /*
2816          * ID_AA64ZFR0_EL1 is only valid when at least one of:
2817          *  - ID_AA64PFR0_EL1.SVE is non-zero
2818          *  - ID_AA64PFR1_EL1.SME is non-zero
2819          * In other cases it is zero, but still safe to read
2820          */
2821         desc->have_sve =
2822             (ID_AA64PFR0_SVE_VAL(desc->id_aa64pfr0) != 0);
2823         desc->id_aa64zfr0 = READ_SPECIALREG(ID_AA64ZFR0_EL1_REG);
2824
2825         desc->clidr = READ_SPECIALREG(clidr_el1);
2826
2827         clidr = desc->clidr;
2828
2829         for (int i = 0; (clidr & CLIDR_CTYPE_MASK) != 0; i++, clidr >>= 3) {
2830                 int j = 0;
2831                 if ((clidr & CLIDR_CTYPE_IO)) {
2832                         WRITE_SPECIALREG(csselr_el1,
2833                             CSSELR_Level(i) | CSSELR_InD);
2834                         desc->ccsidr[i][j++] =
2835                             READ_SPECIALREG(ccsidr_el1);
2836                 }
2837                 if ((clidr & ~CLIDR_CTYPE_IO) == 0)
2838                         continue;
2839                 WRITE_SPECIALREG(csselr_el1, CSSELR_Level(i));
2840                 desc->ccsidr[i][j] = READ_SPECIALREG(ccsidr_el1);
2841         }
2842
2843 #ifdef COMPAT_FREEBSD32
2844         /* Only read aarch32 SRs if EL0-32 is available */
2845         if (ID_AA64PFR0_EL0_VAL(desc->id_aa64pfr0) == ID_AA64PFR0_EL0_64_32) {
2846                 desc->id_isar5 = READ_SPECIALREG(id_isar5_el1);
2847                 desc->mvfr0 = READ_SPECIALREG(mvfr0_el1);
2848                 desc->mvfr1 = READ_SPECIALREG(mvfr1_el1);
2849         }
2850 #endif
2851 }
2852
2853 static void
2854 check_cpu_regs(u_int cpu, struct cpu_desc *desc, struct cpu_desc *prev_desc)
2855 {
2856         switch (cpu_aff_levels) {
2857         case 0:
2858                 if (CPU_AFF0(desc->mpidr) != CPU_AFF0(prev_desc->mpidr))
2859                         cpu_aff_levels = 1;
2860                 /* FALLTHROUGH */
2861         case 1:
2862                 if (CPU_AFF1(desc->mpidr) != CPU_AFF1(prev_desc->mpidr))
2863                         cpu_aff_levels = 2;
2864                 /* FALLTHROUGH */
2865         case 2:
2866                 if (CPU_AFF2(desc->mpidr) != CPU_AFF2(prev_desc->mpidr))
2867                         cpu_aff_levels = 3;
2868                 /* FALLTHROUGH */
2869         case 3:
2870                 if (CPU_AFF3(desc->mpidr) != CPU_AFF3(prev_desc->mpidr))
2871                         cpu_aff_levels = 4;
2872                 break;
2873         }
2874
2875         if (desc->ctr != prev_desc->ctr) {
2876                 /*
2877                  * If the cache type register is different we may
2878                  * have a different l1 cache type.
2879                  */
2880                 identify_cache(desc->ctr);
2881         }
2882 }