sys/arm64/arm64/identcpu.c

   1 /*-
   2  * Copyright (c) 2014 Andrew Turner
   3  * Copyright (c) 2014 The FreeBSD Foundation
   4  * All rights reserved.
   5  *
   6  * Portions of this software were developed by Semihalf
   7  * under sponsorship of the FreeBSD Foundation.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  *
  30  */
  31
  32 #include <sys/cdefs.h>
  33 __FBSDID("$FreeBSD$");
  34
  35 #include <sys/param.h>
  36 #include <sys/kernel.h>
  37 #include <sys/pcpu.h>
  38 #include <sys/sbuf.h>
  39 #include <sys/smp.h>
  40 #include <sys/sysctl.h>
  41 #include <sys/systm.h>
  42
  43 #include <machine/atomic.h>
  44 #include <machine/cpu.h>
  45 #include <machine/cpufunc.h>
  46 #include <machine/undefined.h>
  47 #include <machine/elf.h>
  48
  49 static int ident_lock;
  50 static void print_cpu_features(u_int cpu);
  51 static u_long parse_cpu_features_hwcap(u_int cpu);
  52
  53 char machine[] = "arm64";
  54
  55 #ifdef SCTL_MASK32
  56 extern int adaptive_machine_arch;
  57 #endif
  58
  59 static int
  60 sysctl_hw_machine(SYSCTL_HANDLER_ARGS)
  61 {
  62 #ifdef SCTL_MASK32
  63         static const char machine32[] = "arm";
  64 #endif
  65         int error;
  66 #ifdef SCTL_MASK32
  67         if ((req->flags & SCTL_MASK32) != 0 && adaptive_machine_arch)
  68                 error = SYSCTL_OUT(req, machine32, sizeof(machine32));
  69         else
  70 #endif
  71                 error = SYSCTL_OUT(req, machine, sizeof(machine));
  72         return (error);
  73 }
  74
  75 SYSCTL_PROC(_hw, HW_MACHINE, machine, CTLTYPE_STRING | CTLFLAG_RD |
  76         CTLFLAG_MPSAFE, NULL, 0, sysctl_hw_machine, "A", "Machine class");
  77
  78 static char cpu_model[64];
  79 SYSCTL_STRING(_hw, HW_MODEL, model, CTLFLAG_RD,
  80         cpu_model, sizeof(cpu_model), "Machine model");
  81
  82 /*
  83  * Per-CPU affinity as provided in MPIDR_EL1
  84  * Indexed by CPU number in logical order selected by the system.
  85  * Relevant fields can be extracted using CPU_AFFn macros,
  86  * Aff3.Aff2.Aff1.Aff0 construct a unique CPU address in the system.
  87  *
  88  * Fields used by us:
  89  * Aff1 - Cluster number
  90  * Aff0 - CPU number in Aff1 cluster
  91  */
  92 uint64_t __cpu_affinity[MAXCPU];
  93 static u_int cpu_aff_levels;
  94
  95 struct cpu_desc {
  96         u_int           cpu_impl;
  97         u_int           cpu_part_num;
  98         u_int           cpu_variant;
  99         u_int           cpu_revision;
 100         const char      *cpu_impl_name;
 101         const char      *cpu_part_name;
 102
 103         uint64_t        mpidr;
 104         uint64_t        id_aa64afr0;
 105         uint64_t        id_aa64afr1;
 106         uint64_t        id_aa64dfr0;
 107         uint64_t        id_aa64dfr1;
 108         uint64_t        id_aa64isar0;
 109         uint64_t        id_aa64isar1;
 110         uint64_t        id_aa64mmfr0;
 111         uint64_t        id_aa64mmfr1;
 112         uint64_t        id_aa64mmfr2;
 113         uint64_t        id_aa64pfr0;
 114         uint64_t        id_aa64pfr1;
 115 };
 116
 117 static struct cpu_desc cpu_desc[MAXCPU];
 118 static struct cpu_desc user_cpu_desc;
 119 static u_int cpu_print_regs;
 120 #define PRINT_ID_AA64_AFR0      0x00000001
 121 #define PRINT_ID_AA64_AFR1      0x00000002
 122 #define PRINT_ID_AA64_DFR0      0x00000010
 123 #define PRINT_ID_AA64_DFR1      0x00000020
 124 #define PRINT_ID_AA64_ISAR0     0x00000100
 125 #define PRINT_ID_AA64_ISAR1     0x00000200
 126 #define PRINT_ID_AA64_MMFR0     0x00001000
 127 #define PRINT_ID_AA64_MMFR1     0x00002000
 128 #define PRINT_ID_AA64_MMFR2     0x00004000
 129 #define PRINT_ID_AA64_PFR0      0x00010000
 130 #define PRINT_ID_AA64_PFR1      0x00020000
 131
 132 struct cpu_parts {
 133         u_int           part_id;
 134         const char      *part_name;
 135 };
 136 #define CPU_PART_NONE   { 0, "Unknown Processor" }
 137
 138 struct cpu_implementers {
 139         u_int                   impl_id;
 140         const char              *impl_name;
 141         /*
 142          * Part number is implementation defined
 143          * so each vendor will have its own set of values and names.
 144          */
 145         const struct cpu_parts  *cpu_parts;
 146 };
 147 #define CPU_IMPLEMENTER_NONE    { 0, "Unknown Implementer", cpu_parts_none }
 148
 149 /*
 150  * Per-implementer table of (PartNum, CPU Name) pairs.
 151  */
 152 /* ARM Ltd. */
 153 static const struct cpu_parts cpu_parts_arm[] = {
 154         { CPU_PART_FOUNDATION, "Foundation-Model" },
 155         { CPU_PART_CORTEX_A35, "Cortex-A35" },
 156         { CPU_PART_CORTEX_A53, "Cortex-A53" },
 157         { CPU_PART_CORTEX_A55, "Cortex-A55" },
 158         { CPU_PART_CORTEX_A57, "Cortex-A57" },
 159         { CPU_PART_CORTEX_A65, "Cortex-A65" },
 160         { CPU_PART_CORTEX_A72, "Cortex-A72" },
 161         { CPU_PART_CORTEX_A73, "Cortex-A73" },
 162         { CPU_PART_CORTEX_A75, "Cortex-A75" },
 163         { CPU_PART_CORTEX_A76, "Cortex-A76" },
 164         { CPU_PART_CORTEX_A76AE, "Cortex-A76AE" },
 165         { CPU_PART_CORTEX_A77, "Cortex-A77" },
 166         { CPU_PART_NEOVERSE_N1, "Neoverse-N1" },
 167         CPU_PART_NONE,
 168 };
 169 /* Cavium */
 170 static const struct cpu_parts cpu_parts_cavium[] = {
 171         { CPU_PART_THUNDERX, "ThunderX" },
 172         { CPU_PART_THUNDERX2, "ThunderX2" },
 173         CPU_PART_NONE,
 174 };
 175
 176 /* APM / Ampere */
 177 static const struct cpu_parts cpu_parts_apm[] = {
 178         { CPU_PART_EMAG8180, "eMAG 8180" },
 179         CPU_PART_NONE,
 180 };
 181
 182 /* Unknown */
 183 static const struct cpu_parts cpu_parts_none[] = {
 184         CPU_PART_NONE,
 185 };
 186
 187 /*
 188  * Implementers table.
 189  */
 190 const struct cpu_implementers cpu_implementers[] = {
 191         { CPU_IMPL_ARM,         "ARM",          cpu_parts_arm },
 192         { CPU_IMPL_BROADCOM,    "Broadcom",     cpu_parts_none },
 193         { CPU_IMPL_CAVIUM,      "Cavium",       cpu_parts_cavium },
 194         { CPU_IMPL_DEC,         "DEC",          cpu_parts_none },
 195         { CPU_IMPL_INFINEON,    "IFX",          cpu_parts_none },
 196         { CPU_IMPL_FREESCALE,   "Freescale",    cpu_parts_none },
 197         { CPU_IMPL_NVIDIA,      "NVIDIA",       cpu_parts_none },
 198         { CPU_IMPL_APM,         "APM",          cpu_parts_apm },
 199         { CPU_IMPL_QUALCOMM,    "Qualcomm",     cpu_parts_none },
 200         { CPU_IMPL_MARVELL,     "Marvell",      cpu_parts_none },
 201         { CPU_IMPL_INTEL,       "Intel",        cpu_parts_none },
 202         CPU_IMPLEMENTER_NONE,
 203 };
 204
 205 #define MRS_TYPE_MASK           0xf
 206 #define MRS_INVALID             0
 207 #define MRS_EXACT               1
 208 #define MRS_EXACT_VAL(x)        (MRS_EXACT | ((x) << 4))
 209 #define MRS_EXACT_FIELD(x)      ((x) >> 4)
 210 #define MRS_LOWER               2
 211
 212 struct mrs_field_value {
 213         uint64_t        value;
 214         const char      *desc;
 215 };
 216
 217 #define MRS_FIELD_VALUE(_value, _desc)                                  \
 218         {                                                               \
 219                 .value = (_value),                                      \
 220                 .desc = (_desc),                                        \
 221         }
 222
 223 #define MRS_FIELD_VALUE_NONE_IMPL(_reg, _field, _none, _impl)           \
 224         MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _none, ""),         \
 225         MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _impl, #_field)
 226
 227 #define MRS_FIELD_VALUE_COUNT(_reg, _field, _desc)                      \
 228         MRS_FIELD_VALUE(0ul << _reg ## _ ## _field ## _SHIFT, "1 " _desc), \
 229         MRS_FIELD_VALUE(1ul << _reg ## _ ## _field ## _SHIFT, "2 " _desc "s"), \
 230         MRS_FIELD_VALUE(2ul << _reg ## _ ## _field ## _SHIFT, "3 " _desc "s"), \
 231         MRS_FIELD_VALUE(3ul << _reg ## _ ## _field ## _SHIFT, "4 " _desc "s"), \
 232         MRS_FIELD_VALUE(4ul << _reg ## _ ## _field ## _SHIFT, "5 " _desc "s"), \
 233         MRS_FIELD_VALUE(5ul << _reg ## _ ## _field ## _SHIFT, "6 " _desc "s"), \
 234         MRS_FIELD_VALUE(6ul << _reg ## _ ## _field ## _SHIFT, "7 " _desc "s"), \
 235         MRS_FIELD_VALUE(7ul << _reg ## _ ## _field ## _SHIFT, "8 " _desc "s"), \
 236         MRS_FIELD_VALUE(8ul << _reg ## _ ## _field ## _SHIFT, "9 " _desc "s"), \
 237         MRS_FIELD_VALUE(9ul << _reg ## _ ## _field ## _SHIFT, "10 "_desc "s"), \
 238         MRS_FIELD_VALUE(10ul<< _reg ## _ ## _field ## _SHIFT, "11 "_desc "s"), \
 239         MRS_FIELD_VALUE(11ul<< _reg ## _ ## _field ## _SHIFT, "12 "_desc "s"), \
 240         MRS_FIELD_VALUE(12ul<< _reg ## _ ## _field ## _SHIFT, "13 "_desc "s"), \
 241         MRS_FIELD_VALUE(13ul<< _reg ## _ ## _field ## _SHIFT, "14 "_desc "s"), \
 242         MRS_FIELD_VALUE(14ul<< _reg ## _ ## _field ## _SHIFT, "15 "_desc "s"), \
 243         MRS_FIELD_VALUE(15ul<< _reg ## _ ## _field ## _SHIFT, "16 "_desc "s")
 244
 245 #define MRS_FIELD_VALUE_END     { .desc = NULL }
 246
 247 struct mrs_field {
 248         const char      *name;
 249         struct mrs_field_value *values;
 250         uint64_t        mask;
 251         bool            sign;
 252         u_int           type;
 253         u_int           shift;
 254 };
 255
 256 #define MRS_FIELD(_register, _name, _sign, _type, _values)              \
 257         {                                                               \
 258                 .name = #_name,                                         \
 259                 .sign = (_sign),                                        \
 260                 .type = (_type),                                        \
 261                 .shift = _register ## _ ## _name ## _SHIFT,             \
 262                 .mask = _register ## _ ## _name ## _MASK,               \
 263                 .values = (_values),                                    \
 264         }
 265
 266 #define MRS_FIELD_END   { .type = MRS_INVALID, }
 267
 268 /* ID_AA64AFR0_EL1 */
 269 static struct mrs_field id_aa64afr0_fields[] = {
 270         MRS_FIELD_END,
 271 };
 272
 273
 274 /* ID_AA64AFR1_EL1 */
 275 static struct mrs_field id_aa64afr1_fields[] = {
 276         MRS_FIELD_END,
 277 };
 278
 279
 280 /* ID_AA64DFR0_EL1 */
 281 static struct mrs_field_value id_aa64dfr0_pmsver[] = {
 282         MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_NONE, ""),
 283         MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_V1, "SPE"),
 284         MRS_FIELD_VALUE_END,
 285 };
 286
 287 static struct mrs_field_value id_aa64dfr0_ctx_cmps[] = {
 288         MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, CTX_CMPs, "CTX BKPT"),
 289         MRS_FIELD_VALUE_END,
 290 };
 291
 292 static struct mrs_field_value id_aa64dfr0_wrps[] = {
 293         MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, WRPs, "Watchpoint"),
 294         MRS_FIELD_VALUE_END,
 295 };
 296
 297 static struct mrs_field_value id_aa64dfr0_brps[] = {
 298         MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, BRPs, "Breakpoint"),
 299         MRS_FIELD_VALUE_END,
 300 };
 301
 302 static struct mrs_field_value id_aa64dfr0_pmuver[] = {
 303         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_NONE, ""),
 304         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3, "PMUv3"),
 305         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_1, "PMUv3+16 bit evtCount"),
 306         MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_IMPL, "IMPL PMU"),
 307         MRS_FIELD_VALUE_END,
 308 };
 309
 310 static struct mrs_field_value id_aa64dfr0_tracever[] = {
 311         MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_NONE, ""),
 312         MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_IMPL, "Trace"),
 313         MRS_FIELD_VALUE_END,
 314 };
 315
 316 static struct mrs_field_value id_aa64dfr0_debugver[] = {
 317         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8, "Debugv8"),
 318         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_VHE, "Debugv8_VHE"),
 319         MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_2, "Debugv8.2"),
 320         MRS_FIELD_VALUE_END,
 321 };
 322
 323 static struct mrs_field id_aa64dfr0_fields[] = {
 324         MRS_FIELD(ID_AA64DFR0, PMSVer, false, MRS_EXACT, id_aa64dfr0_pmsver),
 325         MRS_FIELD(ID_AA64DFR0, CTX_CMPs, false, MRS_EXACT,
 326             id_aa64dfr0_ctx_cmps),
 327         MRS_FIELD(ID_AA64DFR0, WRPs, false, MRS_EXACT, id_aa64dfr0_wrps),
 328         MRS_FIELD(ID_AA64DFR0, BRPs, false, MRS_LOWER, id_aa64dfr0_brps),
 329         MRS_FIELD(ID_AA64DFR0, PMUVer, false, MRS_EXACT, id_aa64dfr0_pmuver),
 330         MRS_FIELD(ID_AA64DFR0, TraceVer, false, MRS_EXACT,
 331             id_aa64dfr0_tracever),
 332         MRS_FIELD(ID_AA64DFR0, DebugVer, false, MRS_EXACT_VAL(0x6),
 333             id_aa64dfr0_debugver),
 334         MRS_FIELD_END,
 335 };
 336
 337
 338 /* ID_AA64DFR1 */
 339 static struct mrs_field id_aa64dfr1_fields[] = {
 340         MRS_FIELD_END,
 341 };
 342
 343
 344 /* ID_AA64ISAR0_EL1 */
 345 static struct mrs_field_value id_aa64isar0_dp[] = {
 346         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, DP, NONE, IMPL),
 347         MRS_FIELD_VALUE_END,
 348 };
 349
 350 static struct mrs_field_value id_aa64isar0_sm4[] = {
 351         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM4, NONE, IMPL),
 352         MRS_FIELD_VALUE_END,
 353 };
 354
 355 static struct mrs_field_value id_aa64isar0_sm3[] = {
 356         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM3, NONE, IMPL),
 357         MRS_FIELD_VALUE_END,
 358 };
 359
 360 static struct mrs_field_value id_aa64isar0_sha3[] = {
 361         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA3, NONE, IMPL),
 362         MRS_FIELD_VALUE_END,
 363 };
 364
 365 static struct mrs_field_value id_aa64isar0_rdm[] = {
 366         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, RDM, NONE, IMPL),
 367         MRS_FIELD_VALUE_END,
 368 };
 369
 370 static struct mrs_field_value id_aa64isar0_atomic[] = {
 371         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, Atomic, NONE, IMPL),
 372         MRS_FIELD_VALUE_END,
 373 };
 374
 375 static struct mrs_field_value id_aa64isar0_crc32[] = {
 376         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, CRC32, NONE, BASE),
 377         MRS_FIELD_VALUE_END,
 378 };
 379
 380 static struct mrs_field_value id_aa64isar0_sha2[] = {
 381         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA2, NONE, BASE),
 382         MRS_FIELD_VALUE(ID_AA64ISAR0_SHA2_512, "SHA2+SHA512"),
 383         MRS_FIELD_VALUE_END,
 384 };
 385
 386 static struct mrs_field_value id_aa64isar0_sha1[] = {
 387         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA1, NONE, BASE),
 388         MRS_FIELD_VALUE_END,
 389 };
 390
 391 static struct mrs_field_value id_aa64isar0_aes[] = {
 392         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, AES, NONE, BASE),
 393         MRS_FIELD_VALUE(ID_AA64ISAR0_AES_PMULL, "AES+PMULL"),
 394         MRS_FIELD_VALUE_END,
 395 };
 396
 397 static struct mrs_field id_aa64isar0_fields[] = {
 398         MRS_FIELD(ID_AA64ISAR0, DP, false, MRS_LOWER, id_aa64isar0_dp),
 399         MRS_FIELD(ID_AA64ISAR0, SM4, false, MRS_LOWER, id_aa64isar0_sm4),
 400         MRS_FIELD(ID_AA64ISAR0, SM3, false, MRS_LOWER, id_aa64isar0_sm3),
 401         MRS_FIELD(ID_AA64ISAR0, SHA3, false, MRS_LOWER, id_aa64isar0_sha3),
 402         MRS_FIELD(ID_AA64ISAR0, RDM, false, MRS_LOWER, id_aa64isar0_rdm),
 403         MRS_FIELD(ID_AA64ISAR0, Atomic, false, MRS_LOWER, id_aa64isar0_atomic),
 404         MRS_FIELD(ID_AA64ISAR0, CRC32, false, MRS_LOWER, id_aa64isar0_crc32),
 405         MRS_FIELD(ID_AA64ISAR0, SHA2, false, MRS_LOWER, id_aa64isar0_sha2),
 406         MRS_FIELD(ID_AA64ISAR0, SHA1, false, MRS_LOWER, id_aa64isar0_sha1),
 407         MRS_FIELD(ID_AA64ISAR0, AES, false, MRS_LOWER, id_aa64isar0_aes),
 408         MRS_FIELD_END,
 409 };
 410
 411
 412 /* ID_AA64ISAR1_EL1 */
 413 static struct mrs_field_value id_aa64isar1_gpi[] = {
 414         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPI, NONE, IMPL),
 415         MRS_FIELD_VALUE_END,
 416 };
 417
 418 static struct mrs_field_value id_aa64isar1_gpa[] = {
 419         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPA, NONE, IMPL),
 420         MRS_FIELD_VALUE_END,
 421 };
 422
 423 static struct mrs_field_value id_aa64isar1_lrcpc[] = {
 424         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, LRCPC, NONE, IMPL),
 425         MRS_FIELD_VALUE_END,
 426 };
 427
 428 static struct mrs_field_value id_aa64isar1_fcma[] = {
 429         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FCMA, NONE, IMPL),
 430         MRS_FIELD_VALUE_END,
 431 };
 432
 433 static struct mrs_field_value id_aa64isar1_jscvt[] = {
 434         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, JSCVT, NONE, IMPL),
 435         MRS_FIELD_VALUE_END,
 436 };
 437
 438 static struct mrs_field_value id_aa64isar1_api[] = {
 439         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, API, NONE, IMPL),
 440         MRS_FIELD_VALUE_END,
 441 };
 442
 443 static struct mrs_field_value id_aa64isar1_apa[] = {
 444         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPA, NONE, IMPL),
 445         MRS_FIELD_VALUE_END,
 446 };
 447
 448 static struct mrs_field_value id_aa64isar1_dpb[] = {
 449         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, DPB, NONE, IMPL),
 450         MRS_FIELD_VALUE_END,
 451 };
 452
 453 static struct mrs_field id_aa64isar1_fields[] = {
 454         MRS_FIELD(ID_AA64ISAR1, GPI, false, MRS_EXACT, id_aa64isar1_gpi),
 455         MRS_FIELD(ID_AA64ISAR1, GPA, false, MRS_EXACT, id_aa64isar1_gpa),
 456         MRS_FIELD(ID_AA64ISAR1, LRCPC, false, MRS_LOWER, id_aa64isar1_lrcpc),
 457         MRS_FIELD(ID_AA64ISAR1, FCMA, false, MRS_LOWER, id_aa64isar1_fcma),
 458         MRS_FIELD(ID_AA64ISAR1, JSCVT, false, MRS_LOWER, id_aa64isar1_jscvt),
 459         MRS_FIELD(ID_AA64ISAR1, API, false, MRS_EXACT, id_aa64isar1_api),
 460         MRS_FIELD(ID_AA64ISAR1, APA, false, MRS_EXACT, id_aa64isar1_apa),
 461         MRS_FIELD(ID_AA64ISAR1, DPB, false, MRS_LOWER, id_aa64isar1_dpb),
 462         MRS_FIELD_END,
 463 };
 464
 465
 466 /* ID_AA64MMFR0_EL1 */
 467 static struct mrs_field_value id_aa64mmfr0_tgran4[] = {
 468         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran4, NONE, IMPL),
 469         MRS_FIELD_VALUE_END,
 470 };
 471
 472 static struct mrs_field_value id_aa64mmfr0_tgran64[] = {
 473         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran64, NONE, IMPL),
 474         MRS_FIELD_VALUE_END,
 475 };
 476
 477 static struct mrs_field_value id_aa64mmfr0_tgran16[] = {
 478         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran16, NONE, IMPL),
 479         MRS_FIELD_VALUE_END,
 480 };
 481
 482 static struct mrs_field_value id_aa64mmfr0_bigend_el0[] = {
 483         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEndEL0, FIXED, MIXED),
 484         MRS_FIELD_VALUE_END,
 485 };
 486
 487 static struct mrs_field_value id_aa64mmfr0_snsmem[] = {
 488         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, SNSMem, NONE, DISTINCT),
 489         MRS_FIELD_VALUE_END,
 490 };
 491
 492 static struct mrs_field_value id_aa64mmfr0_bigend[] = {
 493         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEnd, FIXED, MIXED),
 494         MRS_FIELD_VALUE_END,
 495 };
 496
 497 static struct mrs_field_value id_aa64mmfr0_asid_bits[] = {
 498         MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_8, "8bit ASID"),
 499         MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_16, "16bit ASID"),
 500         MRS_FIELD_VALUE_END,
 501 };
 502
 503 static struct mrs_field_value id_aa64mmfr0_parange[] = {
 504         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4G, "4GB PA"),
 505         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_64G, "64GB PA"),
 506         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_1T, "1TB PA"),
 507         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4T, "4TB PA"),
 508         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_16T, "16TB PA"),
 509         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_256T, "256TB PA"),
 510         MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4P, "4PB PA"),
 511         MRS_FIELD_VALUE_END,
 512 };
 513
 514 static struct mrs_field id_aa64mmfr0_fields[] = {
 515         MRS_FIELD(ID_AA64MMFR0, TGran4, false, MRS_EXACT, id_aa64mmfr0_tgran4),
 516         MRS_FIELD(ID_AA64MMFR0, TGran64, false, MRS_EXACT,
 517             id_aa64mmfr0_tgran64),
 518         MRS_FIELD(ID_AA64MMFR0, TGran16, false, MRS_EXACT,
 519             id_aa64mmfr0_tgran16),
 520         MRS_FIELD(ID_AA64MMFR0, BigEndEL0, false, MRS_EXACT,
 521             id_aa64mmfr0_bigend_el0),
 522         MRS_FIELD(ID_AA64MMFR0, SNSMem, false, MRS_EXACT, id_aa64mmfr0_snsmem),
 523         MRS_FIELD(ID_AA64MMFR0, BigEnd, false, MRS_EXACT, id_aa64mmfr0_bigend),
 524         MRS_FIELD(ID_AA64MMFR0, ASIDBits, false, MRS_EXACT,
 525             id_aa64mmfr0_asid_bits),
 526         MRS_FIELD(ID_AA64MMFR0, PARange, false, MRS_EXACT,
 527             id_aa64mmfr0_parange),
 528         MRS_FIELD_END,
 529 };
 530
 531
 532 /* ID_AA64MMFR1_EL1 */
 533 static struct mrs_field_value id_aa64mmfr1_xnx[] = {
 534         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, XNX, NONE, IMPL),
 535         MRS_FIELD_VALUE_END,
 536 };
 537
 538 static struct mrs_field_value id_aa64mmfr1_specsei[] = {
 539         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, SpecSEI, NONE, IMPL),
 540         MRS_FIELD_VALUE_END,
 541 };
 542
 543 static struct mrs_field_value id_aa64mmfr1_pan[] = {
 544         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, PAN, NONE, IMPL),
 545         MRS_FIELD_VALUE(ID_AA64MMFR1_PAN_ATS1E1, "PAN+ATS1E1"),
 546         MRS_FIELD_VALUE_END,
 547 };
 548
 549 static struct mrs_field_value id_aa64mmfr1_lo[] = {
 550         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, LO, NONE, IMPL),
 551         MRS_FIELD_VALUE_END,
 552 };
 553
 554 static struct mrs_field_value id_aa64mmfr1_hpds[] = {
 555         MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_NONE, ""),
 556         MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_HPD, "HPD"),
 557         MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_TTPBHA, "HPD+TTPBHA"),
 558         MRS_FIELD_VALUE_END,
 559 };
 560
 561 static struct mrs_field_value id_aa64mmfr1_vh[] = {
 562         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, VH, NONE, IMPL),
 563         MRS_FIELD_VALUE_END,
 564 };
 565
 566 static struct mrs_field_value id_aa64mmfr1_vmidbits[] = {
 567         MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_8, "8bit VMID"),
 568         MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_16, "16bit VMID"),
 569         MRS_FIELD_VALUE_END,
 570 };
 571
 572 static struct mrs_field_value id_aa64mmfr1_hafdbs[] = {
 573         MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_NONE, ""),
 574         MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF, "HAF"),
 575         MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF_DBS, "HAF+DS"),
 576         MRS_FIELD_VALUE_END,
 577 };
 578
 579 static struct mrs_field id_aa64mmfr1_fields[] = {
 580         MRS_FIELD(ID_AA64MMFR1, XNX, false, MRS_EXACT, id_aa64mmfr1_xnx),
 581         MRS_FIELD(ID_AA64MMFR1, SpecSEI, false, MRS_EXACT,
 582             id_aa64mmfr1_specsei),
 583         MRS_FIELD(ID_AA64MMFR1, PAN, false, MRS_EXACT, id_aa64mmfr1_pan),
 584         MRS_FIELD(ID_AA64MMFR1, LO, false, MRS_EXACT, id_aa64mmfr1_lo),
 585         MRS_FIELD(ID_AA64MMFR1, HPDS, false, MRS_EXACT, id_aa64mmfr1_hpds),
 586         MRS_FIELD(ID_AA64MMFR1, VH, false, MRS_EXACT, id_aa64mmfr1_vh),
 587         MRS_FIELD(ID_AA64MMFR1, VMIDBits, false, MRS_EXACT,
 588             id_aa64mmfr1_vmidbits),
 589         MRS_FIELD(ID_AA64MMFR1, HAFDBS, false, MRS_EXACT, id_aa64mmfr1_hafdbs),
 590         MRS_FIELD_END,
 591 };
 592
 593
 594 /* ID_AA64MMFR2_EL1 */
 595 static struct mrs_field_value id_aa64mmfr2_nv[] = {
 596         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, NV, NONE, IMPL),
 597         MRS_FIELD_VALUE_END,
 598 };
 599
 600 static struct mrs_field_value id_aa64mmfr2_ccidx[] = {
 601         MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_32, "32bit CCIDX"),
 602         MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_64, "32bit CCIDX"),
 603         MRS_FIELD_VALUE_END,
 604 };
 605
 606 static struct mrs_field_value id_aa64mmfr2_varange[] = {
 607         MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_48, "48bit VA"),
 608         MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_52, "52bit VA"),
 609         MRS_FIELD_VALUE_END,
 610 };
 611
 612 static struct mrs_field_value id_aa64mmfr2_iesb[] = {
 613         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IESB, NONE, IMPL),
 614         MRS_FIELD_VALUE_END,
 615 };
 616
 617 static struct mrs_field_value id_aa64mmfr2_lsm[] = {
 618         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, LSM, NONE, IMPL),
 619         MRS_FIELD_VALUE_END,
 620 };
 621
 622 static struct mrs_field_value id_aa64mmfr2_uao[] = {
 623         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, UAO, NONE, IMPL),
 624         MRS_FIELD_VALUE_END,
 625 };
 626
 627 static struct mrs_field_value id_aa64mmfr2_cnp[] = {
 628         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, CnP, NONE, IMPL),
 629         MRS_FIELD_VALUE_END,
 630 };
 631
 632 static struct mrs_field id_aa64mmfr2_fields[] = {
 633         MRS_FIELD(ID_AA64MMFR2, NV, false, MRS_EXACT, id_aa64mmfr2_nv),
 634         MRS_FIELD(ID_AA64MMFR2, CCIDX, false, MRS_EXACT, id_aa64mmfr2_ccidx),
 635         MRS_FIELD(ID_AA64MMFR2, VARange, false, MRS_EXACT,
 636             id_aa64mmfr2_varange),
 637         MRS_FIELD(ID_AA64MMFR2, IESB, false, MRS_EXACT, id_aa64mmfr2_iesb),
 638         MRS_FIELD(ID_AA64MMFR2, LSM, false, MRS_EXACT, id_aa64mmfr2_lsm),
 639         MRS_FIELD(ID_AA64MMFR2, UAO, false, MRS_EXACT, id_aa64mmfr2_uao),
 640         MRS_FIELD(ID_AA64MMFR2, CnP, false, MRS_EXACT, id_aa64mmfr2_cnp),
 641         MRS_FIELD_END,
 642 };
 643
 644
 645 /* ID_AA64PFR0_EL1 */
 646 static struct mrs_field_value id_aa64pfr0_csv3[] = {
 647         MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_NONE, ""),
 648         MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_ISOLATED, "CSV3"),
 649         MRS_FIELD_VALUE_END,
 650 };
 651
 652 static struct mrs_field_value id_aa64pfr0_csv2[] = {
 653         MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_NONE, ""),
 654         MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_ISOLATED, "CSV2"),
 655         MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_SCXTNUM, "SCXTNUM"),
 656         MRS_FIELD_VALUE_END,
 657 };
 658
 659 static struct mrs_field_value id_aa64pfr0_dit[] = {
 660         MRS_FIELD_VALUE(ID_AA64PFR0_DIT_NONE, ""),
 661         MRS_FIELD_VALUE(ID_AA64PFR0_DIT_PSTATE, "PSTATE.DIT"),
 662         MRS_FIELD_VALUE_END,
 663 };
 664
 665 static struct mrs_field_value id_aa64pfr0_amu[] = {
 666         MRS_FIELD_VALUE(ID_AA64PFR0_AMU_NONE, ""),
 667         MRS_FIELD_VALUE(ID_AA64PFR0_AMU_V1, "AMUv1"),
 668         MRS_FIELD_VALUE_END,
 669 };
 670
 671 static struct mrs_field_value id_aa64pfr0_mpam[] = {
 672         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, MPAM, NONE, IMPL),
 673         MRS_FIELD_VALUE_END,
 674 };
 675
 676 static struct mrs_field_value id_aa64pfr0_sel2[] = {
 677         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SEL2, NONE, IMPL),
 678         MRS_FIELD_VALUE_END,
 679 };
 680
 681 static struct mrs_field_value id_aa64pfr0_sve[] = {
 682         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SVE, NONE, IMPL),
 683         MRS_FIELD_VALUE_END,
 684 };
 685
 686 static struct mrs_field_value id_aa64pfr0_ras[] = {
 687         MRS_FIELD_VALUE(ID_AA64PFR0_RAS_NONE, ""),
 688         MRS_FIELD_VALUE(ID_AA64PFR0_RAS_V1, "RASv1"),
 689         MRS_FIELD_VALUE_END,
 690 };
 691
 692 static struct mrs_field_value id_aa64pfr0_gic[] = {
 693         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, GIC, CPUIF_NONE, CPUIF_EN),
 694         MRS_FIELD_VALUE_END,
 695 };
 696
 697 static struct mrs_field_value id_aa64pfr0_advsimd[] = {
 698         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, AdvSIMD, NONE, IMPL),
 699         MRS_FIELD_VALUE(ID_AA64PFR0_AdvSIMD_HP, "AdvSIMD+HP"),
 700         MRS_FIELD_VALUE_END,
 701 };
 702
 703 static struct mrs_field_value id_aa64pfr0_fp[] = {
 704         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, FP, NONE, IMPL),
 705         MRS_FIELD_VALUE(ID_AA64PFR0_FP_HP, "FP+HP"),
 706         MRS_FIELD_VALUE_END,
 707 };
 708
 709 static struct mrs_field_value id_aa64pfr0_el3[] = {
 710         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL3, NONE, 64),
 711         MRS_FIELD_VALUE(ID_AA64PFR0_EL3_64_32, "EL3 32"),
 712         MRS_FIELD_VALUE_END,
 713 };
 714
 715 static struct mrs_field_value id_aa64pfr0_el2[] = {
 716         MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL2, NONE, 64),
 717         MRS_FIELD_VALUE(ID_AA64PFR0_EL2_64_32, "EL2 32"),
 718         MRS_FIELD_VALUE_END,
 719 };
 720
 721 static struct mrs_field_value id_aa64pfr0_el1[] = {
 722         MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64, "EL1"),
 723         MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64_32, "EL1 32"),
 724         MRS_FIELD_VALUE_END,
 725 };
 726
 727 static struct mrs_field_value id_aa64pfr0_el0[] = {
 728         MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64, "EL0"),
 729         MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64_32, "EL0 32"),
 730         MRS_FIELD_VALUE_END,
 731 };
 732
 733 static struct mrs_field id_aa64pfr0_fields[] = {
 734         MRS_FIELD(ID_AA64PFR0, CSV3, false, MRS_EXACT, id_aa64pfr0_csv3),
 735         MRS_FIELD(ID_AA64PFR0, CSV2, false, MRS_EXACT, id_aa64pfr0_csv2),
 736         MRS_FIELD(ID_AA64PFR0, DIT, false, MRS_EXACT, id_aa64pfr0_dit),
 737         MRS_FIELD(ID_AA64PFR0, AMU, false, MRS_EXACT, id_aa64pfr0_amu),
 738         MRS_FIELD(ID_AA64PFR0, MPAM, false, MRS_EXACT, id_aa64pfr0_mpam),
 739         MRS_FIELD(ID_AA64PFR0, SEL2, false, MRS_EXACT, id_aa64pfr0_sel2),
 740         MRS_FIELD(ID_AA64PFR0, SVE, false, MRS_EXACT, id_aa64pfr0_sve),
 741         MRS_FIELD(ID_AA64PFR0, RAS, false, MRS_EXACT, id_aa64pfr0_ras),
 742         MRS_FIELD(ID_AA64PFR0, GIC, false, MRS_EXACT, id_aa64pfr0_gic),
 743         MRS_FIELD(ID_AA64PFR0, AdvSIMD, true, MRS_LOWER, id_aa64pfr0_advsimd),
 744         MRS_FIELD(ID_AA64PFR0, FP, true,  MRS_LOWER, id_aa64pfr0_fp),
 745         MRS_FIELD(ID_AA64PFR0, EL3, false, MRS_EXACT, id_aa64pfr0_el3),
 746         MRS_FIELD(ID_AA64PFR0, EL2, false, MRS_EXACT, id_aa64pfr0_el2),
 747         MRS_FIELD(ID_AA64PFR0, EL1, false, MRS_LOWER, id_aa64pfr0_el1),
 748         MRS_FIELD(ID_AA64PFR0, EL0, false, MRS_LOWER, id_aa64pfr0_el0),
 749         MRS_FIELD_END,
 750 };
 751
 752
 753 /* ID_AA64PFR1_EL1 */
 754 static struct mrs_field_value id_aa64pfr1_bt[] = {
 755         MRS_FIELD_VALUE(ID_AA64PFR1_BT_NONE, ""),
 756         MRS_FIELD_VALUE(ID_AA64PFR1_BT_IMPL, "BTI"),
 757         MRS_FIELD_VALUE_END,
 758 };
 759
 760 static struct mrs_field_value id_aa64pfr1_ssbs[] = {
 761         MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_NONE, ""),
 762         MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE, "PSTATE.SSBS"),
 763         MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE_MSR, "PSTATE.SSBS MSR"),
 764         MRS_FIELD_VALUE_END,
 765 };
 766
 767 static struct mrs_field_value id_aa64pfr1_mte[] = {
 768         MRS_FIELD_VALUE(ID_AA64PFR1_MTE_NONE, ""),
 769         MRS_FIELD_VALUE(ID_AA64PFR1_MTE_IMPL_EL0, "MTE EL0"),
 770         MRS_FIELD_VALUE(ID_AA64PFR1_MTE_IMPL, "MTE"),
 771         MRS_FIELD_VALUE_END,
 772 };
 773
 774 static struct mrs_field id_aa64pfr1_fields[] = {
 775         MRS_FIELD(ID_AA64PFR1, BT, false, MRS_EXACT, id_aa64pfr1_bt),
 776         MRS_FIELD(ID_AA64PFR1, SSBS, false, MRS_EXACT, id_aa64pfr1_ssbs),
 777         MRS_FIELD(ID_AA64PFR1, MTE, false, MRS_EXACT, id_aa64pfr1_mte),
 778         MRS_FIELD_END,
 779 };
 780
 781 struct mrs_user_reg {
 782         u_int           reg;
 783         u_int           CRm;
 784         u_int           Op2;
 785         size_t          offset;
 786         struct mrs_field *fields;
 787 };
 788
 789 static struct mrs_user_reg user_regs[] = {
 790         {       /* id_aa64isar0_el1 */
 791                 .reg = ID_AA64ISAR0_EL1,
 792                 .CRm = 6,
 793                 .Op2 = 0,
 794                 .offset = __offsetof(struct cpu_desc, id_aa64isar0),
 795                 .fields = id_aa64isar0_fields,
 796         },
 797         {       /* id_aa64isar1_el1 */
 798                 .reg = ID_AA64ISAR1_EL1,
 799                 .CRm = 6,
 800                 .Op2 = 1,
 801                 .offset = __offsetof(struct cpu_desc, id_aa64isar1),
 802                 .fields = id_aa64isar1_fields,
 803         },
 804         {       /* id_aa64pfr0_el1 */
 805                 .reg = ID_AA64PFR0_EL1,
 806                 .CRm = 4,
 807                 .Op2 = 0,
 808                 .offset = __offsetof(struct cpu_desc, id_aa64pfr0),
 809                 .fields = id_aa64pfr0_fields,
 810         },
 811         {       /* id_aa64pfr0_el1 */
 812                 .reg = ID_AA64PFR1_EL1,
 813                 .CRm = 4,
 814                 .Op2 = 1,
 815                 .offset = __offsetof(struct cpu_desc, id_aa64pfr1),
 816                 .fields = id_aa64pfr1_fields,
 817         },
 818         {       /* id_aa64dfr0_el1 */
 819                 .reg = ID_AA64DFR0_EL1,
 820                 .CRm = 5,
 821                 .Op2 = 0,
 822                 .offset = __offsetof(struct cpu_desc, id_aa64dfr0),
 823                 .fields = id_aa64dfr0_fields,
 824         },
 825 };
 826
 827 #define CPU_DESC_FIELD(desc, idx)                                       \
 828     *(uint64_t *)((char *)&(desc) + user_regs[(idx)].offset)
 829
 830 static int
 831 user_mrs_handler(vm_offset_t va, uint32_t insn, struct trapframe *frame,
 832     uint32_t esr)
 833 {
 834         uint64_t value;
 835         int CRm, Op2, i, reg;
 836
 837         if ((insn & MRS_MASK) != MRS_VALUE)
 838                 return (0);
 839
 840         /*
 841          * We only emulate Op0 == 3, Op1 == 0, CRn == 0, CRm == {0, 4-7}.
 842          * These are in the EL1 CPU identification space.
 843          * CRm == 0 holds MIDR_EL1, MPIDR_EL1, and REVID_EL1.
 844          * CRm == {4-7} holds the ID_AA64 registers.
 845          *
 846          * For full details see the ARMv8 ARM (ARM DDI 0487C.a)
 847          * Table D9-2 System instruction encodings for non-Debug System
 848          * register accesses.
 849          */
 850         if (mrs_Op0(insn) != 3 || mrs_Op1(insn) != 0 || mrs_CRn(insn) != 0)
 851                 return (0);
 852
 853         CRm = mrs_CRm(insn);
 854         if (CRm > 7 || (CRm < 4 && CRm != 0))
 855                 return (0);
 856
 857         Op2 = mrs_Op2(insn);
 858         value = 0;
 859
 860         for (i = 0; i < nitems(user_regs); i++) {
 861                 if (user_regs[i].CRm == CRm && user_regs[i].Op2 == Op2) {
 862                         value = CPU_DESC_FIELD(user_cpu_desc, i);
 863                         break;
 864                 }
 865         }
 866
 867         if (CRm == 0) {
 868                 switch (Op2) {
 869                 case 0:
 870                         value = READ_SPECIALREG(midr_el1);
 871                         break;
 872                 case 5:
 873                         value = READ_SPECIALREG(mpidr_el1);
 874                         break;
 875                 case 6:
 876                         value = READ_SPECIALREG(revidr_el1);
 877                         break;
 878                 default:
 879                         return (0);
 880                 }
 881         }
 882
 883         /*
 884          * We will handle this instruction, move to the next so we
 885          * don't trap here again.
 886          */
 887         frame->tf_elr += INSN_SIZE;
 888
 889         reg = MRS_REGISTER(insn);
 890         /* If reg is 31 then write to xzr, i.e. do nothing */
 891         if (reg == 31)
 892                 return (1);
 893
 894         if (reg < nitems(frame->tf_x))
 895                 frame->tf_x[reg] = value;
 896         else if (reg == 30)
 897                 frame->tf_lr = value;
 898
 899         return (1);
 900 }
 901
 902 bool
 903 extract_user_id_field(u_int reg, u_int field_shift, uint8_t *val)
 904 {
 905         uint64_t value;
 906         int i;
 907
 908         for (i = 0; i < nitems(user_regs); i++) {
 909                 if (user_regs[i].reg == reg) {
 910                         value = CPU_DESC_FIELD(user_cpu_desc, i);
 911                         *val = value >> field_shift;
 912                         return (true);
 913                 }
 914         }
 915
 916         return (false);
 917 }
 918
 919 static void
 920 update_user_regs(u_int cpu)
 921 {
 922         struct mrs_field *fields;
 923         uint64_t cur, value;
 924         int i, j, cur_field, new_field;
 925
 926         for (i = 0; i < nitems(user_regs); i++) {
 927                 value = CPU_DESC_FIELD(cpu_desc[cpu], i);
 928                 if (cpu == 0)
 929                         cur = value;
 930                 else
 931                         cur = CPU_DESC_FIELD(user_cpu_desc, i);
 932
 933                 fields = user_regs[i].fields;
 934                 for (j = 0; fields[j].type != 0; j++) {
 935                         switch (fields[j].type & MRS_TYPE_MASK) {
 936                         case MRS_EXACT:
 937                                 cur &= ~(0xfu << fields[j].shift);
 938                                 cur |=
 939                                     (uint64_t)MRS_EXACT_FIELD(fields[j].type) <<
 940                                     fields[j].shift;
 941                                 break;
 942                         case MRS_LOWER:
 943                                 new_field = (value >> fields[j].shift) & 0xf;
 944                                 cur_field = (cur >> fields[j].shift) & 0xf;
 945                                 if ((fields[j].sign &&
 946                                      (int)new_field < (int)cur_field) ||
 947                                     (!fields[j].sign &&
 948                                      (u_int)new_field < (u_int)cur_field)) {
 949                                         cur &= ~(0xfu << fields[j].shift);
 950                                         cur |= new_field << fields[j].shift;
 951                                 }
 952                                 break;
 953                         default:
 954                                 panic("Invalid field type: %d", fields[j].type);
 955                         }
 956                 }
 957
 958                 CPU_DESC_FIELD(user_cpu_desc, i) = cur;
 959         }
 960 }
 961
 962 /* HWCAP */
 963 extern u_long elf_hwcap;
 964 bool __read_frequently lse_supported = false;
 965
 966 static void
 967 identify_cpu_sysinit(void *dummy __unused)
 968 {
 969         int cpu;
 970         u_long hwcap;
 971
 972         /* Create a user visible cpu description with safe values */
 973         memset(&user_cpu_desc, 0, sizeof(user_cpu_desc));
 974         /* Safe values for these registers */
 975         user_cpu_desc.id_aa64pfr0 = ID_AA64PFR0_AdvSIMD_NONE |
 976             ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 | ID_AA64PFR0_EL0_64;
 977         user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8;
 978
 979         CPU_FOREACH(cpu) {
 980                 print_cpu_features(cpu);
 981                 hwcap = parse_cpu_features_hwcap(cpu);
 982                 if (elf_hwcap == 0)
 983                         elf_hwcap = hwcap;
 984                 else
 985                         elf_hwcap &= hwcap;
 986                 update_user_regs(cpu);
 987         }
 988
 989         if ((elf_hwcap & HWCAP_ATOMICS) != 0) {
 990                 lse_supported = true;
 991                 if (bootverbose)
 992                         printf("Enabling LSE atomics in the kernel\n");
 993         }
 994 #ifdef LSE_ATOMICS
 995         if (!lse_supported)
 996                 panic("CPU does not support LSE atomic instructions");
 997 #endif
 998
 999         install_undef_handler(true, user_mrs_handler);
1000 }
1001 SYSINIT(idenrity_cpu, SI_SUB_SMP, SI_ORDER_ANY, identify_cpu_sysinit, NULL);
1002
1003 static u_long
1004 parse_cpu_features_hwcap(u_int cpu)
1005 {
1006         u_long hwcap = 0;
1007
1008         if (ID_AA64ISAR0_DP_VAL(cpu_desc[cpu].id_aa64isar0) == ID_AA64ISAR0_DP_IMPL)
1009                 hwcap |= HWCAP_ASIMDDP;
1010
1011         if (ID_AA64ISAR0_SM4_VAL(cpu_desc[cpu].id_aa64isar0) == ID_AA64ISAR0_SM4_IMPL)
1012                 hwcap |= HWCAP_SM4;
1013
1014         if (ID_AA64ISAR0_SM3_VAL(cpu_desc[cpu].id_aa64isar0) == ID_AA64ISAR0_SM3_IMPL)
1015                 hwcap |= HWCAP_SM3;
1016
1017         if (ID_AA64ISAR0_RDM_VAL(cpu_desc[cpu].id_aa64isar0) == ID_AA64ISAR0_RDM_IMPL)
1018                 hwcap |= HWCAP_ASIMDRDM;
1019
1020         if (ID_AA64ISAR0_Atomic_VAL(cpu_desc[cpu].id_aa64isar0) == ID_AA64ISAR0_Atomic_IMPL)
1021                 hwcap |= HWCAP_ATOMICS;
1022
1023         if (ID_AA64ISAR0_CRC32_VAL(cpu_desc[cpu].id_aa64isar0) == ID_AA64ISAR0_CRC32_BASE)
1024                 hwcap |= HWCAP_CRC32;
1025
1026         switch (ID_AA64ISAR0_SHA2_VAL(cpu_desc[cpu].id_aa64isar0)) {
1027                 case ID_AA64ISAR0_SHA2_BASE:
1028                         hwcap |= HWCAP_SHA2;
1029                         break;
1030                 case ID_AA64ISAR0_SHA2_512:
1031                         hwcap |= HWCAP_SHA2 | HWCAP_SHA512;
1032                         break;
1033         default:
1034                 break;
1035         }
1036
1037         if (ID_AA64ISAR0_SHA1_VAL(cpu_desc[cpu].id_aa64isar0))
1038                 hwcap |= HWCAP_SHA1;
1039
1040         switch (ID_AA64ISAR0_AES_VAL(cpu_desc[cpu].id_aa64isar0)) {
1041         case ID_AA64ISAR0_AES_BASE:
1042                 hwcap |= HWCAP_AES;
1043                 break;
1044         case ID_AA64ISAR0_AES_PMULL:
1045                 hwcap |= HWCAP_PMULL | HWCAP_AES;
1046                 break;
1047         default:
1048                 break;
1049         }
1050
1051         if (ID_AA64ISAR1_LRCPC_VAL(cpu_desc[cpu].id_aa64isar1) == ID_AA64ISAR1_LRCPC_IMPL)
1052                 hwcap |= HWCAP_LRCPC;
1053
1054         if (ID_AA64ISAR1_FCMA_VAL(cpu_desc[cpu].id_aa64isar1) == ID_AA64ISAR1_FCMA_IMPL)
1055                 hwcap |= HWCAP_FCMA;
1056
1057         if (ID_AA64ISAR1_JSCVT_VAL(cpu_desc[cpu].id_aa64isar1) == ID_AA64ISAR1_JSCVT_IMPL)
1058                 hwcap |= HWCAP_JSCVT;
1059
1060         if (ID_AA64ISAR1_DPB_VAL(cpu_desc[cpu].id_aa64isar1) == ID_AA64ISAR1_DPB_IMPL)
1061                 hwcap |= HWCAP_DCPOP;
1062
1063         if (ID_AA64PFR0_SVE_VAL(cpu_desc[cpu].id_aa64pfr0) == ID_AA64PFR0_SVE_IMPL)
1064                 hwcap |= HWCAP_SVE;
1065
1066         switch (ID_AA64PFR0_AdvSIMD_VAL(cpu_desc[cpu].id_aa64pfr0)) {
1067         case ID_AA64PFR0_AdvSIMD_IMPL:
1068                 hwcap |= HWCAP_ASIMD;
1069                 break;
1070         case ID_AA64PFR0_AdvSIMD_HP:
1071                 hwcap |= HWCAP_ASIMD | HWCAP_ASIMDDP;
1072                 break;
1073         default:
1074                 break;
1075         }
1076
1077         switch (ID_AA64PFR0_FP_VAL(cpu_desc[cpu].id_aa64pfr0)) {
1078         case ID_AA64PFR0_FP_IMPL:
1079                 hwcap |= HWCAP_FP;
1080                 break;
1081         case ID_AA64PFR0_FP_HP:
1082                 hwcap |= HWCAP_FP | HWCAP_FPHP;
1083                 break;
1084         default:
1085                 break;
1086         }
1087
1088         return (hwcap);
1089 }
1090
1091 static void
1092 print_id_register(struct sbuf *sb, const char *reg_name, uint64_t reg,
1093     struct mrs_field *fields)
1094 {
1095         struct mrs_field_value *fv;
1096         int field, i, j, printed;
1097
1098         sbuf_printf(sb, "%29s = <", reg_name);
1099
1100 #define SEP_STR ((printed++) == 0) ? "" : ","
1101         printed = 0;
1102         for (i = 0; fields[i].type != 0; i++) {
1103                 fv = fields[i].values;
1104
1105                 /* TODO: Handle with an unknown message */
1106                 if (fv == NULL)
1107                         continue;
1108
1109                 field = (reg & fields[i].mask) >> fields[i].shift;
1110                 for (j = 0; fv[j].desc != NULL; j++) {
1111                         if ((fv[j].value >> fields[i].shift) != field)
1112                                 continue;
1113
1114                         if (fv[j].desc[0] != '\0')
1115                                 sbuf_printf(sb, "%s%s", SEP_STR, fv[j].desc);
1116                                 break;
1117                 }
1118                 if (fv[j].desc == NULL)
1119                         sbuf_printf(sb, "%sUnknown %s(%x)", SEP_STR,
1120                             fields[i].name, field);
1121
1122                 reg &= ~(0xful << fields[i].shift);
1123         }
1124
1125         if (reg != 0)
1126                 sbuf_printf(sb, "%s%#lx", SEP_STR, reg);
1127 #undef SEP_STR
1128
1129         sbuf_finish(sb);
1130         printf("%s>\n", sbuf_data(sb));
1131         sbuf_clear(sb);
1132 }
1133
1134 static void
1135 print_cpu_features(u_int cpu)
1136 {
1137         struct sbuf *sb;
1138
1139         sb = sbuf_new_auto();
1140         sbuf_printf(sb, "CPU%3d: %s %s r%dp%d", cpu,
1141             cpu_desc[cpu].cpu_impl_name, cpu_desc[cpu].cpu_part_name,
1142             cpu_desc[cpu].cpu_variant, cpu_desc[cpu].cpu_revision);
1143
1144         sbuf_cat(sb, " affinity:");
1145         switch(cpu_aff_levels) {
1146         default:
1147         case 4:
1148                 sbuf_printf(sb, " %2d", CPU_AFF3(cpu_desc[cpu].mpidr));
1149                 /* FALLTHROUGH */
1150         case 3:
1151                 sbuf_printf(sb, " %2d", CPU_AFF2(cpu_desc[cpu].mpidr));
1152                 /* FALLTHROUGH */
1153         case 2:
1154                 sbuf_printf(sb, " %2d", CPU_AFF1(cpu_desc[cpu].mpidr));
1155                 /* FALLTHROUGH */
1156         case 1:
1157         case 0: /* On UP this will be zero */
1158                 sbuf_printf(sb, " %2d", CPU_AFF0(cpu_desc[cpu].mpidr));
1159                 break;
1160         }
1161         sbuf_finish(sb);
1162         printf("%s\n", sbuf_data(sb));
1163         sbuf_clear(sb);
1164
1165         /*
1166          * There is a hardware errata where, if one CPU is performing a TLB
1167          * invalidation while another is performing a store-exclusive the
1168          * store-exclusive may return the wrong status. A workaround seems
1169          * to be to use an IPI to invalidate on each CPU, however given the
1170          * limited number of affected units (pass 1.1 is the evaluation
1171          * hardware revision), and the lack of information from Cavium
1172          * this has not been implemented.
1173          *
1174          * At the time of writing this the only information is from:
1175          * https://lkml.org/lkml/2016/8/4/722
1176          */
1177         /*
1178          * XXX: CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1 on its own also
1179          * triggers on pass 2.0+.
1180          */
1181         if (cpu == 0 && CPU_VAR(PCPU_GET(midr)) == 0 &&
1182             CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1)
1183                 printf("WARNING: ThunderX Pass 1.1 detected.\nThis has known "
1184                     "hardware bugs that may cause the incorrect operation of "
1185                     "atomic operations.\n");
1186
1187         /* AArch64 Instruction Set Attribute Register 0 */
1188         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_ISAR0) != 0)
1189                 print_id_register(sb, "Instruction Set Attributes 0",
1190                     cpu_desc[cpu].id_aa64isar0, id_aa64isar0_fields);
1191
1192         /* AArch64 Instruction Set Attribute Register 1 */
1193         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_ISAR1) != 0)
1194                 print_id_register(sb, "Instruction Set Attributes 1",
1195                     cpu_desc[cpu].id_aa64isar1, id_aa64isar1_fields);
1196
1197         /* AArch64 Processor Feature Register 0 */
1198         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_PFR0) != 0)
1199                 print_id_register(sb, "Processor Features 0",
1200                     cpu_desc[cpu].id_aa64pfr0, id_aa64pfr0_fields);
1201
1202         /* AArch64 Processor Feature Register 1 */
1203         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_PFR1) != 0)
1204                 print_id_register(sb, "Processor Features 1",
1205                     cpu_desc[cpu].id_aa64pfr1, id_aa64pfr1_fields);
1206
1207         /* AArch64 Memory Model Feature Register 0 */
1208         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_MMFR0) != 0)
1209                 print_id_register(sb, "Memory Model Features 0",
1210                     cpu_desc[cpu].id_aa64mmfr0, id_aa64mmfr0_fields);
1211
1212         /* AArch64 Memory Model Feature Register 1 */
1213         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_MMFR1) != 0)
1214                 print_id_register(sb, "Memory Model Features 1",
1215                     cpu_desc[cpu].id_aa64mmfr1, id_aa64mmfr1_fields);
1216
1217         /* AArch64 Memory Model Feature Register 2 */
1218         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_MMFR2) != 0)
1219                 print_id_register(sb, "Memory Model Features 2",
1220                     cpu_desc[cpu].id_aa64mmfr2, id_aa64mmfr2_fields);
1221
1222         /* AArch64 Debug Feature Register 0 */
1223         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_DFR0) != 0)
1224                 print_id_register(sb, "Debug Features 0",
1225                     cpu_desc[cpu].id_aa64dfr0, id_aa64dfr0_fields);
1226
1227         /* AArch64 Memory Model Feature Register 1 */
1228         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_DFR1) != 0)
1229                 print_id_register(sb, "Debug Features 1",
1230                     cpu_desc[cpu].id_aa64dfr1, id_aa64dfr1_fields);
1231
1232         /* AArch64 Auxiliary Feature Register 0 */
1233         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_AFR0) != 0)
1234                 print_id_register(sb, "Auxiliary Features 0",
1235                     cpu_desc[cpu].id_aa64afr0, id_aa64afr0_fields);
1236
1237         /* AArch64 Auxiliary Feature Register 1 */
1238         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_AFR1) != 0)
1239                 print_id_register(sb, "Auxiliary Features 1",
1240                     cpu_desc[cpu].id_aa64afr1, id_aa64afr1_fields);
1241
1242         sbuf_delete(sb);
1243         sb = NULL;
1244 #undef SEP_STR
1245 }
1246
1247 void
1248 identify_cpu(void)
1249 {
1250         u_int midr;
1251         u_int impl_id;
1252         u_int part_id;
1253         u_int cpu;
1254         size_t i;
1255         const struct cpu_parts *cpu_partsp = NULL;
1256
1257         cpu = PCPU_GET(cpuid);
1258         midr = get_midr();
1259
1260         /*
1261          * Store midr to pcpu to allow fast reading
1262          * from EL0, EL1 and assembly code.
1263          */
1264         PCPU_SET(midr, midr);
1265
1266         impl_id = CPU_IMPL(midr);
1267         for (i = 0; i < nitems(cpu_implementers); i++) {
1268                 if (impl_id == cpu_implementers[i].impl_id ||
1269                     cpu_implementers[i].impl_id == 0) {
1270                         cpu_desc[cpu].cpu_impl = impl_id;
1271                         cpu_desc[cpu].cpu_impl_name = cpu_implementers[i].impl_name;
1272                         cpu_partsp = cpu_implementers[i].cpu_parts;
1273                         break;
1274                 }
1275         }
1276
1277         part_id = CPU_PART(midr);
1278         for (i = 0; &cpu_partsp[i] != NULL; i++) {
1279                 if (part_id == cpu_partsp[i].part_id ||
1280                     cpu_partsp[i].part_id == 0) {
1281                         cpu_desc[cpu].cpu_part_num = part_id;
1282                         cpu_desc[cpu].cpu_part_name = cpu_partsp[i].part_name;
1283                         break;
1284                 }
1285         }
1286
1287         cpu_desc[cpu].cpu_revision = CPU_REV(midr);
1288         cpu_desc[cpu].cpu_variant = CPU_VAR(midr);
1289
1290         snprintf(cpu_model, sizeof(cpu_model), "%s %s r%dp%d",
1291             cpu_desc[cpu].cpu_impl_name, cpu_desc[cpu].cpu_part_name,
1292             cpu_desc[cpu].cpu_variant, cpu_desc[cpu].cpu_revision);
1293
1294         /* Save affinity for current CPU */
1295         cpu_desc[cpu].mpidr = get_mpidr();
1296         CPU_AFFINITY(cpu) = cpu_desc[cpu].mpidr & CPU_AFF_MASK;
1297
1298         cpu_desc[cpu].id_aa64dfr0 = READ_SPECIALREG(id_aa64dfr0_el1);
1299         cpu_desc[cpu].id_aa64dfr1 = READ_SPECIALREG(id_aa64dfr1_el1);
1300         cpu_desc[cpu].id_aa64isar0 = READ_SPECIALREG(id_aa64isar0_el1);
1301         cpu_desc[cpu].id_aa64isar1 = READ_SPECIALREG(id_aa64isar1_el1);
1302         cpu_desc[cpu].id_aa64mmfr0 = READ_SPECIALREG(id_aa64mmfr0_el1);
1303         cpu_desc[cpu].id_aa64mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1);
1304         cpu_desc[cpu].id_aa64mmfr2 = READ_SPECIALREG(id_aa64mmfr2_el1);
1305         cpu_desc[cpu].id_aa64pfr0 = READ_SPECIALREG(id_aa64pfr0_el1);
1306         cpu_desc[cpu].id_aa64pfr1 = READ_SPECIALREG(id_aa64pfr1_el1);
1307
1308         if (cpu != 0) {
1309                 /*
1310                  * This code must run on one cpu at a time, but we are
1311                  * not scheduling on the current core so implement a
1312                  * simple spinlock.
1313                  */
1314                 while (atomic_cmpset_acq_int(&ident_lock, 0, 1) == 0)
1315                         __asm __volatile("wfe" ::: "memory");
1316
1317                 switch (cpu_aff_levels) {
1318                 case 0:
1319                         if (CPU_AFF0(cpu_desc[cpu].mpidr) !=
1320                             CPU_AFF0(cpu_desc[0].mpidr))
1321                                 cpu_aff_levels = 1;
1322                         /* FALLTHROUGH */
1323                 case 1:
1324                         if (CPU_AFF1(cpu_desc[cpu].mpidr) !=
1325                             CPU_AFF1(cpu_desc[0].mpidr))
1326                                 cpu_aff_levels = 2;
1327                         /* FALLTHROUGH */
1328                 case 2:
1329                         if (CPU_AFF2(cpu_desc[cpu].mpidr) !=
1330                             CPU_AFF2(cpu_desc[0].mpidr))
1331                                 cpu_aff_levels = 3;
1332                         /* FALLTHROUGH */
1333                 case 3:
1334                         if (CPU_AFF3(cpu_desc[cpu].mpidr) !=
1335                             CPU_AFF3(cpu_desc[0].mpidr))
1336                                 cpu_aff_levels = 4;
1337                         break;
1338                 }
1339
1340                 if (cpu_desc[cpu].id_aa64afr0 != cpu_desc[0].id_aa64afr0)
1341                         cpu_print_regs |= PRINT_ID_AA64_AFR0;
1342                 if (cpu_desc[cpu].id_aa64afr1 != cpu_desc[0].id_aa64afr1)
1343                         cpu_print_regs |= PRINT_ID_AA64_AFR1;
1344
1345                 if (cpu_desc[cpu].id_aa64dfr0 != cpu_desc[0].id_aa64dfr0)
1346                         cpu_print_regs |= PRINT_ID_AA64_DFR0;
1347                 if (cpu_desc[cpu].id_aa64dfr1 != cpu_desc[0].id_aa64dfr1)
1348                         cpu_print_regs |= PRINT_ID_AA64_DFR1;
1349
1350                 if (cpu_desc[cpu].id_aa64isar0 != cpu_desc[0].id_aa64isar0)
1351                         cpu_print_regs |= PRINT_ID_AA64_ISAR0;
1352                 if (cpu_desc[cpu].id_aa64isar1 != cpu_desc[0].id_aa64isar1)
1353                         cpu_print_regs |= PRINT_ID_AA64_ISAR1;
1354
1355                 if (cpu_desc[cpu].id_aa64mmfr0 != cpu_desc[0].id_aa64mmfr0)
1356                         cpu_print_regs |= PRINT_ID_AA64_MMFR0;
1357                 if (cpu_desc[cpu].id_aa64mmfr1 != cpu_desc[0].id_aa64mmfr1)
1358                         cpu_print_regs |= PRINT_ID_AA64_MMFR1;
1359                 if (cpu_desc[cpu].id_aa64mmfr2 != cpu_desc[0].id_aa64mmfr2)
1360                         cpu_print_regs |= PRINT_ID_AA64_MMFR2;
1361
1362                 if (cpu_desc[cpu].id_aa64pfr0 != cpu_desc[0].id_aa64pfr0)
1363                         cpu_print_regs |= PRINT_ID_AA64_PFR0;
1364                 if (cpu_desc[cpu].id_aa64pfr1 != cpu_desc[0].id_aa64pfr1)
1365                         cpu_print_regs |= PRINT_ID_AA64_PFR1;
1366
1367                 /* Wake up the other CPUs */
1368                 atomic_store_rel_int(&ident_lock, 0);
1369                 __asm __volatile("sev" ::: "memory");
1370         }
1371 }