sys/arm64/arm64/identcpu.c

   1 /*-
   2  * Copyright (c) 2014 Andrew Turner
   3  * Copyright (c) 2014 The FreeBSD Foundation
   4  * All rights reserved.
   5  *
   6  * Portions of this software were developed by Semihalf
   7  * under sponsorship of the FreeBSD Foundation.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  *
  30  */
  31
  32 #include <sys/cdefs.h>
  33 __FBSDID("$FreeBSD$");
  34
  35 #include <sys/param.h>
  36 #include <sys/kernel.h>
  37 #include <sys/pcpu.h>
  38 #include <sys/sbuf.h>
  39 #include <sys/smp.h>
  40 #include <sys/sysctl.h>
  41 #include <sys/systm.h>
  42
  43 #include <machine/atomic.h>
  44 #include <machine/cpu.h>
  45 #include <machine/cpufunc.h>
  46 #include <machine/undefined.h>
  47
  48 static int ident_lock;
  49
  50 char machine[] = "arm64";
  51
  52 SYSCTL_STRING(_hw, HW_MACHINE, machine, CTLFLAG_RD, machine, 0,
  53     "Machine class");
  54
  55 static char cpu_model[64];
  56 SYSCTL_STRING(_hw, HW_MODEL, model, CTLFLAG_RD,
  57         cpu_model, sizeof(cpu_model), "Machine model");
  58
  59 /*
  60  * Per-CPU affinity as provided in MPIDR_EL1
  61  * Indexed by CPU number in logical order selected by the system.
  62  * Relevant fields can be extracted using CPU_AFFn macros,
  63  * Aff3.Aff2.Aff1.Aff0 construct a unique CPU address in the system.
  64  *
  65  * Fields used by us:
  66  * Aff1 - Cluster number
  67  * Aff0 - CPU number in Aff1 cluster
  68  */
  69 uint64_t __cpu_affinity[MAXCPU];
  70 static u_int cpu_aff_levels;
  71
  72 struct cpu_desc {
  73         u_int           cpu_impl;
  74         u_int           cpu_part_num;
  75         u_int           cpu_variant;
  76         u_int           cpu_revision;
  77         const char      *cpu_impl_name;
  78         const char      *cpu_part_name;
  79
  80         uint64_t        mpidr;
  81         uint64_t        id_aa64afr0;
  82         uint64_t        id_aa64afr1;
  83         uint64_t        id_aa64dfr0;
  84         uint64_t        id_aa64dfr1;
  85         uint64_t        id_aa64isar0;
  86         uint64_t        id_aa64isar1;
  87         uint64_t        id_aa64mmfr0;
  88         uint64_t        id_aa64mmfr1;
  89         uint64_t        id_aa64mmfr2;
  90         uint64_t        id_aa64pfr0;
  91         uint64_t        id_aa64pfr1;
  92 };
  93
  94 struct cpu_desc cpu_desc[MAXCPU];
  95 struct cpu_desc user_cpu_desc;
  96 static u_int cpu_print_regs;
  97 #define PRINT_ID_AA64_AFR0      0x00000001
  98 #define PRINT_ID_AA64_AFR1      0x00000002
  99 #define PRINT_ID_AA64_DFR0      0x00000010
 100 #define PRINT_ID_AA64_DFR1      0x00000020
 101 #define PRINT_ID_AA64_ISAR0     0x00000100
 102 #define PRINT_ID_AA64_ISAR1     0x00000200
 103 #define PRINT_ID_AA64_MMFR0     0x00001000
 104 #define PRINT_ID_AA64_MMFR1     0x00002000
 105 #define PRINT_ID_AA64_MMFR2     0x00004000
 106 #define PRINT_ID_AA64_PFR0      0x00010000
 107 #define PRINT_ID_AA64_PFR1      0x00020000
 108
 109 struct cpu_parts {
 110         u_int           part_id;
 111         const char      *part_name;
 112 };
 113 #define CPU_PART_NONE   { 0, "Unknown Processor" }
 114
 115 struct cpu_implementers {
 116         u_int                   impl_id;
 117         const char              *impl_name;
 118         /*
 119          * Part number is implementation defined
 120          * so each vendor will have its own set of values and names.
 121          */
 122         const struct cpu_parts  *cpu_parts;
 123 };
 124 #define CPU_IMPLEMENTER_NONE    { 0, "Unknown Implementer", cpu_parts_none }
 125
 126 /*
 127  * Per-implementer table of (PartNum, CPU Name) pairs.
 128  */
 129 /* ARM Ltd. */
 130 static const struct cpu_parts cpu_parts_arm[] = {
 131         { CPU_PART_FOUNDATION, "Foundation-Model" },
 132         { CPU_PART_CORTEX_A35, "Cortex-A35" },
 133         { CPU_PART_CORTEX_A53, "Cortex-A53" },
 134         { CPU_PART_CORTEX_A55, "Cortex-A55" },
 135         { CPU_PART_CORTEX_A57, "Cortex-A57" },
 136         { CPU_PART_CORTEX_A72, "Cortex-A72" },
 137         { CPU_PART_CORTEX_A73, "Cortex-A73" },
 138         { CPU_PART_CORTEX_A75, "Cortex-A75" },
 139         CPU_PART_NONE,
 140 };
 141 /* Cavium */
 142 static const struct cpu_parts cpu_parts_cavium[] = {
 143         { CPU_PART_THUNDERX, "ThunderX" },
 144         { CPU_PART_THUNDERX2, "ThunderX2" },
 145         CPU_PART_NONE,
 146 };
 147
 148 /* APM / Ampere */
 149 static const struct cpu_parts cpu_parts_apm[] = {
 150         { CPU_PART_EMAG8180, "eMAG 8180" },
 151         CPU_PART_NONE,
 152 };
 153
 154 /* Unknown */
 155 static const struct cpu_parts cpu_parts_none[] = {
 156         CPU_PART_NONE,
 157 };
 158
 159 /*
 160  * Implementers table.
 161  */
 162 const struct cpu_implementers cpu_implementers[] = {
 163         { CPU_IMPL_ARM,         "ARM",          cpu_parts_arm },
 164         { CPU_IMPL_BROADCOM,    "Broadcom",     cpu_parts_none },
 165         { CPU_IMPL_CAVIUM,      "Cavium",       cpu_parts_cavium },
 166         { CPU_IMPL_DEC,         "DEC",          cpu_parts_none },
 167         { CPU_IMPL_INFINEON,    "IFX",          cpu_parts_none },
 168         { CPU_IMPL_FREESCALE,   "Freescale",    cpu_parts_none },
 169         { CPU_IMPL_NVIDIA,      "NVIDIA",       cpu_parts_none },
 170         { CPU_IMPL_APM,         "APM",          cpu_parts_apm },
 171         { CPU_IMPL_QUALCOMM,    "Qualcomm",     cpu_parts_none },
 172         { CPU_IMPL_MARVELL,     "Marvell",      cpu_parts_none },
 173         { CPU_IMPL_INTEL,       "Intel",        cpu_parts_none },
 174         CPU_IMPLEMENTER_NONE,
 175 };
 176
 177 #define MRS_TYPE_MASK           0xf
 178 #define MRS_INVALID             0
 179 #define MRS_EXACT               1
 180 #define MRS_EXACT_VAL(x)        (MRS_EXACT | ((x) << 4))
 181 #define MRS_EXACT_FIELD(x)      ((x) >> 4)
 182 #define MRS_LOWER               2
 183
 184 struct mrs_field {
 185         bool            sign;
 186         u_int           type;
 187         u_int           shift;
 188 };
 189
 190 #define MRS_FIELD(_sign, _type, _shift)                                 \
 191         {                                                               \
 192                 .sign = (_sign),                                        \
 193                 .type = (_type),                                        \
 194                 .shift = (_shift),                                      \
 195         }
 196
 197 #define MRS_FIELD_END   { .type = MRS_INVALID, }
 198
 199 static struct mrs_field id_aa64isar0_fields[] = {
 200         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR0_DP_SHIFT),
 201         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR0_SM4_SHIFT),
 202         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR0_SM3_SHIFT),
 203         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR0_SHA3_SHIFT),
 204         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR0_RDM_SHIFT),
 205         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR0_Atomic_SHIFT),
 206         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR0_CRC32_SHIFT),
 207         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR0_SHA2_SHIFT),
 208         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR0_SHA1_SHIFT),
 209         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR0_AES_SHIFT),
 210         MRS_FIELD_END,
 211 };
 212
 213 static struct mrs_field id_aa64isar1_fields[] = {
 214         MRS_FIELD(false, MRS_EXACT, ID_AA64ISAR1_GPI_SHIFT),
 215         MRS_FIELD(false, MRS_EXACT, ID_AA64ISAR1_GPA_SHIFT),
 216         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR1_LRCPC_SHIFT),
 217         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR1_FCMA_SHIFT),
 218         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR1_JSCVT_SHIFT),
 219         MRS_FIELD(false, MRS_EXACT, ID_AA64ISAR1_API_SHIFT),
 220         MRS_FIELD(false, MRS_EXACT, ID_AA64ISAR1_APA_SHIFT),
 221         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR1_DPB_SHIFT),
 222         MRS_FIELD_END,
 223 };
 224
 225 static struct mrs_field id_aa64pfr0_fields[] = {
 226         MRS_FIELD(false, MRS_EXACT, ID_AA64PFR0_SVE_SHIFT),
 227         MRS_FIELD(false, MRS_EXACT, ID_AA64PFR0_RAS_SHIFT),
 228         MRS_FIELD(false, MRS_EXACT, ID_AA64PFR0_GIC_SHIFT),
 229         MRS_FIELD(true,  MRS_LOWER, ID_AA64PFR0_AdvSIMD_SHIFT),
 230         MRS_FIELD(true,  MRS_LOWER, ID_AA64PFR0_FP_SHIFT),
 231         MRS_FIELD(false, MRS_EXACT, ID_AA64PFR0_EL3_SHIFT),
 232         MRS_FIELD(false, MRS_EXACT, ID_AA64PFR0_EL2_SHIFT),
 233         MRS_FIELD(false, MRS_LOWER, ID_AA64PFR0_EL1_SHIFT),
 234         MRS_FIELD(false, MRS_LOWER, ID_AA64PFR0_EL0_SHIFT),
 235         MRS_FIELD_END,
 236 };
 237
 238 static struct mrs_field id_aa64dfr0_fields[] = {
 239         MRS_FIELD(false, MRS_EXACT, ID_AA64DFR0_PMSVer_SHIFT),
 240         MRS_FIELD(false, MRS_EXACT, ID_AA64DFR0_CTX_CMPs_SHIFT),
 241         MRS_FIELD(false, MRS_EXACT, ID_AA64DFR0_WRPs_SHIFT),
 242         MRS_FIELD(false, MRS_EXACT, ID_AA64DFR0_BRPs_SHIFT),
 243         MRS_FIELD(false, MRS_EXACT, ID_AA64DFR0_PMUVer_SHIFT),
 244         MRS_FIELD(false, MRS_EXACT, ID_AA64DFR0_TraceVer_SHIFT),
 245         MRS_FIELD(false, MRS_EXACT_VAL(0x6), ID_AA64DFR0_DebugVer_SHIFT),
 246         MRS_FIELD_END,
 247 };
 248
 249 struct mrs_user_reg {
 250         u_int           CRm;
 251         u_int           Op2;
 252         size_t          offset;
 253         struct mrs_field *fields;
 254 };
 255
 256 static struct mrs_user_reg user_regs[] = {
 257         {       /* id_aa64isar0_el1 */
 258                 .CRm = 6,
 259                 .Op2 = 0,
 260                 .offset = __offsetof(struct cpu_desc, id_aa64isar0),
 261                 .fields = id_aa64isar0_fields,
 262         },
 263         {       /* id_aa64isar1_el1 */
 264                 .CRm = 6,
 265                 .Op2 = 1,
 266                 .offset = __offsetof(struct cpu_desc, id_aa64isar1),
 267                 .fields = id_aa64isar1_fields,
 268         },
 269         {       /* id_aa64pfr0_el1 */
 270                 .CRm = 4,
 271                 .Op2 = 0,
 272                 .offset = __offsetof(struct cpu_desc, id_aa64pfr0),
 273                 .fields = id_aa64pfr0_fields,
 274         },
 275         {       /* id_aa64dfr0_el1 */
 276                 .CRm = 5,
 277                 .Op2 = 0,
 278                 .offset = __offsetof(struct cpu_desc, id_aa64dfr0),
 279                 .fields = id_aa64dfr0_fields,
 280         },
 281 };
 282
 283 #define CPU_DESC_FIELD(desc, idx)                                       \
 284     *(uint64_t *)((char *)&(desc) + user_regs[(idx)].offset)
 285
 286 static int
 287 user_mrs_handler(vm_offset_t va, uint32_t insn, struct trapframe *frame,
 288     uint32_t esr)
 289 {
 290         uint64_t value;
 291         int CRm, Op2, i, reg;
 292
 293         if ((insn & MRS_MASK) != MRS_VALUE)
 294                 return (0);
 295
 296         /*
 297          * We only emulate Op0 == 3, Op1 == 0, CRn == 0, CRm == {0, 4-7}.
 298          * These are in the EL1 CPU identification space.
 299          * CRm == 0 holds MIDR_EL1, MPIDR_EL1, and REVID_EL1.
 300          * CRm == {4-7} holds the ID_AA64 registers.
 301          *
 302          * For full details see the ARMv8 ARM (ARM DDI 0487C.a)
 303          * Table D9-2 System instruction encodings for non-Debug System
 304          * register accesses.
 305          */
 306         if (mrs_Op0(insn) != 3 || mrs_Op1(insn) != 0 || mrs_CRn(insn) != 0)
 307                 return (0);
 308
 309         CRm = mrs_CRm(insn);
 310         if (CRm > 7 || (CRm < 4 && CRm != 0))
 311                 return (0);
 312
 313         Op2 = mrs_Op2(insn);
 314         value = 0;
 315
 316         for (i = 0; i < nitems(user_regs); i++) {
 317                 if (user_regs[i].CRm == CRm && user_regs[i].Op2 == Op2) {
 318                         value = CPU_DESC_FIELD(user_cpu_desc, i);
 319                         break;
 320                 }
 321         }
 322
 323         if (CRm == 0) {
 324                 switch (Op2) {
 325                 case 0:
 326                         value = READ_SPECIALREG(midr_el1);
 327                         break;
 328                 case 5:
 329                         value = READ_SPECIALREG(mpidr_el1);
 330                         break;
 331                 case 6:
 332                         value = READ_SPECIALREG(revidr_el1);
 333                         break;
 334                 default:
 335                         return (0);
 336                 }
 337         }
 338
 339         /*
 340          * We will handle this instruction, move to the next so we
 341          * don't trap here again.
 342          */
 343         frame->tf_elr += INSN_SIZE;
 344
 345         reg = MRS_REGISTER(insn);
 346         /* If reg is 31 then write to xzr, i.e. do nothing */
 347         if (reg == 31)
 348                 return (1);
 349
 350         if (reg < nitems(frame->tf_x))
 351                 frame->tf_x[reg] = value;
 352         else if (reg == 30)
 353                 frame->tf_lr = value;
 354
 355         return (1);
 356 }
 357
 358 static void
 359 update_user_regs(u_int cpu)
 360 {
 361         struct mrs_field *fields;
 362         uint64_t cur, value;
 363         int i, j, cur_field, new_field;
 364
 365         for (i = 0; i < nitems(user_regs); i++) {
 366                 value = CPU_DESC_FIELD(cpu_desc[cpu], i);
 367                 if (cpu == 0)
 368                         cur = value;
 369                 else
 370                         cur = CPU_DESC_FIELD(user_cpu_desc, i);
 371
 372                 fields = user_regs[i].fields;
 373                 for (j = 0; fields[j].type != 0; j++) {
 374                         switch (fields[j].type & MRS_TYPE_MASK) {
 375                         case MRS_EXACT:
 376                                 cur &= ~(0xfu << fields[j].shift);
 377                                 cur |=
 378                                     (uint64_t)MRS_EXACT_FIELD(fields[j].type) <<
 379                                     fields[j].shift;
 380                                 break;
 381                         case MRS_LOWER:
 382                                 new_field = (value >> fields[j].shift) & 0xf;
 383                                 cur_field = (cur >> fields[j].shift) & 0xf;
 384                                 if ((fields[j].sign &&
 385                                      (int)new_field < (int)cur_field) ||
 386                                     (!fields[j].sign &&
 387                                      (u_int)new_field < (u_int)cur_field)) {
 388                                         cur &= ~(0xfu << fields[j].shift);
 389                                         cur |= new_field << fields[j].shift;
 390                                 }
 391                                 break;
 392                         default:
 393                                 panic("Invalid field type: %d", fields[j].type);
 394                         }
 395                 }
 396
 397                 CPU_DESC_FIELD(user_cpu_desc, i) = cur;
 398         }
 399 }
 400
 401 static void
 402 identify_cpu_sysinit(void *dummy __unused)
 403 {
 404         int cpu;
 405
 406         /* Create a user visible cpu description with safe values */
 407         memset(&user_cpu_desc, 0, sizeof(user_cpu_desc));
 408         /* Safe values for these registers */
 409         user_cpu_desc.id_aa64pfr0 = ID_AA64PFR0_AdvSIMD_NONE |
 410             ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 | ID_AA64PFR0_EL0_64;
 411         user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8;
 412
 413
 414         CPU_FOREACH(cpu) {
 415                 print_cpu_features(cpu);
 416                 update_user_regs(cpu);
 417         }
 418
 419         install_undef_handler(true, user_mrs_handler);
 420 }
 421 SYSINIT(idenrity_cpu, SI_SUB_SMP, SI_ORDER_ANY, identify_cpu_sysinit, NULL);
 422
 423 void
 424 print_cpu_features(u_int cpu)
 425 {
 426         struct sbuf *sb;
 427         int printed;
 428
 429         sb = sbuf_new_auto();
 430         sbuf_printf(sb, "CPU%3d: %s %s r%dp%d", cpu,
 431             cpu_desc[cpu].cpu_impl_name, cpu_desc[cpu].cpu_part_name,
 432             cpu_desc[cpu].cpu_variant, cpu_desc[cpu].cpu_revision);
 433
 434         sbuf_cat(sb, " affinity:");
 435         switch(cpu_aff_levels) {
 436         default:
 437         case 4:
 438                 sbuf_printf(sb, " %2d", CPU_AFF3(cpu_desc[cpu].mpidr));
 439                 /* FALLTHROUGH */
 440         case 3:
 441                 sbuf_printf(sb, " %2d", CPU_AFF2(cpu_desc[cpu].mpidr));
 442                 /* FALLTHROUGH */
 443         case 2:
 444                 sbuf_printf(sb, " %2d", CPU_AFF1(cpu_desc[cpu].mpidr));
 445                 /* FALLTHROUGH */
 446         case 1:
 447         case 0: /* On UP this will be zero */
 448                 sbuf_printf(sb, " %2d", CPU_AFF0(cpu_desc[cpu].mpidr));
 449                 break;
 450         }
 451         sbuf_finish(sb);
 452         printf("%s\n", sbuf_data(sb));
 453         sbuf_clear(sb);
 454
 455         /*
 456          * There is a hardware errata where, if one CPU is performing a TLB
 457          * invalidation while another is performing a store-exclusive the
 458          * store-exclusive may return the wrong status. A workaround seems
 459          * to be to use an IPI to invalidate on each CPU, however given the
 460          * limited number of affected units (pass 1.1 is the evaluation
 461          * hardware revision), and the lack of information from Cavium
 462          * this has not been implemented.
 463          *
 464          * At the time of writing this the only information is from:
 465          * https://lkml.org/lkml/2016/8/4/722
 466          */
 467         /*
 468          * XXX: CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1 on its own also
 469          * triggers on pass 2.0+.
 470          */
 471         if (cpu == 0 && CPU_VAR(PCPU_GET(midr)) == 0 &&
 472             CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1)
 473                 printf("WARNING: ThunderX Pass 1.1 detected.\nThis has known "
 474                     "hardware bugs that may cause the incorrect operation of "
 475                     "atomic operations.\n");
 476
 477         if (cpu != 0 && cpu_print_regs == 0)
 478                 return;
 479
 480 #define SEP_STR ((printed++) == 0) ? "" : ","
 481
 482         /* AArch64 Instruction Set Attribute Register 0 */
 483         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_ISAR0) != 0) {
 484                 printed = 0;
 485                 sbuf_printf(sb, " Instruction Set Attributes 0 = <");
 486
 487                 switch (ID_AA64ISAR0_DP(cpu_desc[cpu].id_aa64isar0)) {
 488                 case ID_AA64ISAR0_DP_NONE:
 489                         break;
 490                 case ID_AA64ISAR0_DP_IMPL:
 491                         sbuf_printf(sb, "%sDotProd", SEP_STR);
 492                         break;
 493                 default:
 494                         sbuf_printf(sb, "%sUnknown DP", SEP_STR);
 495                         break;
 496                 }
 497
 498                 switch (ID_AA64ISAR0_SM4(cpu_desc[cpu].id_aa64isar0)) {
 499                 case ID_AA64ISAR0_SM4_NONE:
 500                         break;
 501                 case ID_AA64ISAR0_SM4_IMPL:
 502                         sbuf_printf(sb, "%sSM4", SEP_STR);
 503                         break;
 504                 default:
 505                         sbuf_printf(sb, "%sUnknown SM4", SEP_STR);
 506                         break;
 507                 }
 508
 509                 switch (ID_AA64ISAR0_SM3(cpu_desc[cpu].id_aa64isar0)) {
 510                 case ID_AA64ISAR0_SM3_NONE:
 511                         break;
 512                 case ID_AA64ISAR0_SM3_IMPL:
 513                         sbuf_printf(sb, "%sSM3", SEP_STR);
 514                         break;
 515                 default:
 516                         sbuf_printf(sb, "%sUnknown SM3", SEP_STR);
 517                         break;
 518                 }
 519
 520                 switch (ID_AA64ISAR0_SHA3(cpu_desc[cpu].id_aa64isar0)) {
 521                 case ID_AA64ISAR0_SHA3_NONE:
 522                         break;
 523                 case ID_AA64ISAR0_SHA3_IMPL:
 524                         sbuf_printf(sb, "%sSHA3", SEP_STR);
 525                         break;
 526                 default:
 527                         sbuf_printf(sb, "%sUnknown SHA3", SEP_STR);
 528                         break;
 529                 }
 530
 531                 switch (ID_AA64ISAR0_RDM(cpu_desc[cpu].id_aa64isar0)) {
 532                 case ID_AA64ISAR0_RDM_NONE:
 533                         break;
 534                 case ID_AA64ISAR0_RDM_IMPL:
 535                         sbuf_printf(sb, "%sRDM", SEP_STR);
 536                         break;
 537                 default:
 538                         sbuf_printf(sb, "%sUnknown RDM", SEP_STR);
 539                 }
 540
 541                 switch (ID_AA64ISAR0_Atomic(cpu_desc[cpu].id_aa64isar0)) {
 542                 case ID_AA64ISAR0_Atomic_NONE:
 543                         break;
 544                 case ID_AA64ISAR0_Atomic_IMPL:
 545                         sbuf_printf(sb, "%sAtomic", SEP_STR);
 546                         break;
 547                 default:
 548                         sbuf_printf(sb, "%sUnknown Atomic", SEP_STR);
 549                 }
 550
 551                 switch (ID_AA64ISAR0_CRC32(cpu_desc[cpu].id_aa64isar0)) {
 552                 case ID_AA64ISAR0_CRC32_NONE:
 553                         break;
 554                 case ID_AA64ISAR0_CRC32_BASE:
 555                         sbuf_printf(sb, "%sCRC32", SEP_STR);
 556                         break;
 557                 default:
 558                         sbuf_printf(sb, "%sUnknown CRC32", SEP_STR);
 559                         break;
 560                 }
 561
 562                 switch (ID_AA64ISAR0_SHA2(cpu_desc[cpu].id_aa64isar0)) {
 563                 case ID_AA64ISAR0_SHA2_NONE:
 564                         break;
 565                 case ID_AA64ISAR0_SHA2_BASE:
 566                         sbuf_printf(sb, "%sSHA2", SEP_STR);
 567                         break;
 568                 case ID_AA64ISAR0_SHA2_512:
 569                         sbuf_printf(sb, "%sSHA2+SHA512", SEP_STR);
 570                         break;
 571                 default:
 572                         sbuf_printf(sb, "%sUnknown SHA2", SEP_STR);
 573                         break;
 574                 }
 575
 576                 switch (ID_AA64ISAR0_SHA1(cpu_desc[cpu].id_aa64isar0)) {
 577                 case ID_AA64ISAR0_SHA1_NONE:
 578                         break;
 579                 case ID_AA64ISAR0_SHA1_BASE:
 580                         sbuf_printf(sb, "%sSHA1", SEP_STR);
 581                         break;
 582                 default:
 583                         sbuf_printf(sb, "%sUnknown SHA1", SEP_STR);
 584                         break;
 585                 }
 586
 587                 switch (ID_AA64ISAR0_AES(cpu_desc[cpu].id_aa64isar0)) {
 588                 case ID_AA64ISAR0_AES_NONE:
 589                         break;
 590                 case ID_AA64ISAR0_AES_BASE:
 591                         sbuf_printf(sb, "%sAES", SEP_STR);
 592                         break;
 593                 case ID_AA64ISAR0_AES_PMULL:
 594                         sbuf_printf(sb, "%sAES+PMULL", SEP_STR);
 595                         break;
 596                 default:
 597                         sbuf_printf(sb, "%sUnknown AES", SEP_STR);
 598                         break;
 599                 }
 600
 601                 if ((cpu_desc[cpu].id_aa64isar0 & ~ID_AA64ISAR0_MASK) != 0)
 602                         sbuf_printf(sb, "%s%#lx", SEP_STR,
 603                             cpu_desc[cpu].id_aa64isar0 & ~ID_AA64ISAR0_MASK);
 604
 605                 sbuf_finish(sb);
 606                 printf("%s>\n", sbuf_data(sb));
 607                 sbuf_clear(sb);
 608         }
 609
 610         /* AArch64 Instruction Set Attribute Register 1 */
 611         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_ISAR1) != 0) {
 612                 printed = 0;
 613                 sbuf_printf(sb, " Instruction Set Attributes 1 = <");
 614
 615                 switch (ID_AA64ISAR1_GPI(cpu_desc[cpu].id_aa64isar1)) {
 616                 case ID_AA64ISAR1_GPI_NONE:
 617                         break;
 618                 case ID_AA64ISAR1_GPI_IMPL:
 619                         sbuf_printf(sb, "%sImpl GenericAuth", SEP_STR);
 620                         break;
 621                 default:
 622                         sbuf_printf(sb, "%sUnknown GenericAuth", SEP_STR);
 623                         break;
 624                 }
 625
 626                 switch (ID_AA64ISAR1_GPA(cpu_desc[cpu].id_aa64isar1)) {
 627                 case ID_AA64ISAR1_GPA_NONE:
 628                         break;
 629                 case ID_AA64ISAR1_GPA_IMPL:
 630                         sbuf_printf(sb, "%sPrince GenericAuth", SEP_STR);
 631                         break;
 632                 default:
 633                         sbuf_printf(sb, "%sUnknown GenericAuth", SEP_STR);
 634                         break;
 635                 }
 636
 637                 switch (ID_AA64ISAR1_LRCPC(cpu_desc[cpu].id_aa64isar1)) {
 638                 case ID_AA64ISAR1_LRCPC_NONE:
 639                         break;
 640                 case ID_AA64ISAR1_LRCPC_IMPL:
 641                         sbuf_printf(sb, "%sRCpc", SEP_STR);
 642                         break;
 643                 default:
 644                         sbuf_printf(sb, "%sUnknown RCpc", SEP_STR);
 645                         break;
 646                 }
 647
 648                 switch (ID_AA64ISAR1_FCMA(cpu_desc[cpu].id_aa64isar1)) {
 649                 case ID_AA64ISAR1_FCMA_NONE:
 650                         break;
 651                 case ID_AA64ISAR1_FCMA_IMPL:
 652                         sbuf_printf(sb, "%sFCMA", SEP_STR);
 653                         break;
 654                 default:
 655                         sbuf_printf(sb, "%sUnknown FCMA", SEP_STR);
 656                         break;
 657                 }
 658
 659                 switch (ID_AA64ISAR1_JSCVT(cpu_desc[cpu].id_aa64isar1)) {
 660                 case ID_AA64ISAR1_JSCVT_NONE:
 661                         break;
 662                 case ID_AA64ISAR1_JSCVT_IMPL:
 663                         sbuf_printf(sb, "%sJS Conv", SEP_STR);
 664                         break;
 665                 default:
 666                         sbuf_printf(sb, "%sUnknown JS Conv", SEP_STR);
 667                         break;
 668                 }
 669
 670                 switch (ID_AA64ISAR1_API(cpu_desc[cpu].id_aa64isar1)) {
 671                 case ID_AA64ISAR1_API_NONE:
 672                         break;
 673                 case ID_AA64ISAR1_API_IMPL:
 674                         sbuf_printf(sb, "%sImpl AddrAuth", SEP_STR);
 675                         break;
 676                 default:
 677                         sbuf_printf(sb, "%sUnknown Impl AddrAuth", SEP_STR);
 678                         break;
 679                 }
 680
 681                 switch (ID_AA64ISAR1_APA(cpu_desc[cpu].id_aa64isar1)) {
 682                 case ID_AA64ISAR1_APA_NONE:
 683                         break;
 684                 case ID_AA64ISAR1_APA_IMPL:
 685                         sbuf_printf(sb, "%sPrince AddrAuth", SEP_STR);
 686                         break;
 687                 default:
 688                         sbuf_printf(sb, "%sUnknown Prince AddrAuth", SEP_STR);
 689                         break;
 690                 }
 691
 692                 switch (ID_AA64ISAR1_DPB(cpu_desc[cpu].id_aa64isar1)) {
 693                 case ID_AA64ISAR1_DPB_NONE:
 694                         break;
 695                 case ID_AA64ISAR1_DPB_IMPL:
 696                         sbuf_printf(sb, "%sDC CVAP", SEP_STR);
 697                         break;
 698                 default:
 699                         sbuf_printf(sb, "%sUnknown DC CVAP", SEP_STR);
 700                         break;
 701                 }
 702
 703                 if ((cpu_desc[cpu].id_aa64isar1 & ~ID_AA64ISAR1_MASK) != 0)
 704                         sbuf_printf(sb, "%s%#lx", SEP_STR,
 705                             cpu_desc[cpu].id_aa64isar1 & ~ID_AA64ISAR1_MASK);
 706                 sbuf_finish(sb);
 707                 printf("%s>\n", sbuf_data(sb));
 708                 sbuf_clear(sb);
 709         }
 710
 711         /* AArch64 Processor Feature Register 0 */
 712         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_PFR0) != 0) {
 713                 printed = 0;
 714                 sbuf_printf(sb, "         Processor Features 0 = <");
 715
 716                 switch (ID_AA64PFR0_SVE(cpu_desc[cpu].id_aa64pfr0)) {
 717                 case ID_AA64PFR0_SVE_NONE:
 718                         break;
 719                 case ID_AA64PFR0_SVE_IMPL:
 720                         sbuf_printf(sb, "%sSVE", SEP_STR);
 721                         break;
 722                 default:
 723                         sbuf_printf(sb, "%sUnknown SVE", SEP_STR);
 724                         break;
 725                 }
 726
 727                 switch (ID_AA64PFR0_RAS(cpu_desc[cpu].id_aa64pfr0)) {
 728                 case ID_AA64PFR0_RAS_NONE:
 729                         break;
 730                 case ID_AA64PFR0_RAS_V1:
 731                         sbuf_printf(sb, "%sRASv1", SEP_STR);
 732                         break;
 733                 default:
 734                         sbuf_printf(sb, "%sUnknown RAS", SEP_STR);
 735                         break;
 736                 }
 737
 738                 switch (ID_AA64PFR0_GIC(cpu_desc[cpu].id_aa64pfr0)) {
 739                 case ID_AA64PFR0_GIC_CPUIF_NONE:
 740                         break;
 741                 case ID_AA64PFR0_GIC_CPUIF_EN:
 742                         sbuf_printf(sb, "%sGIC", SEP_STR);
 743                         break;
 744                 default:
 745                         sbuf_printf(sb, "%sUnknown GIC interface", SEP_STR);
 746                         break;
 747                 }
 748
 749                 switch (ID_AA64PFR0_AdvSIMD(cpu_desc[cpu].id_aa64pfr0)) {
 750                 case ID_AA64PFR0_AdvSIMD_NONE:
 751                         break;
 752                 case ID_AA64PFR0_AdvSIMD_IMPL:
 753                         sbuf_printf(sb, "%sAdvSIMD", SEP_STR);
 754                         break;
 755                 case ID_AA64PFR0_AdvSIMD_HP:
 756                         sbuf_printf(sb, "%sAdvSIMD+HP", SEP_STR);
 757                         break;
 758                 default:
 759                         sbuf_printf(sb, "%sUnknown AdvSIMD", SEP_STR);
 760                         break;
 761                 }
 762
 763                 switch (ID_AA64PFR0_FP(cpu_desc[cpu].id_aa64pfr0)) {
 764                 case ID_AA64PFR0_FP_NONE:
 765                         break;
 766                 case ID_AA64PFR0_FP_IMPL:
 767                         sbuf_printf(sb, "%sFloat", SEP_STR);
 768                         break;
 769                 case ID_AA64PFR0_FP_HP:
 770                         sbuf_printf(sb, "%sFloat+HP", SEP_STR);
 771                         break;
 772                 default:
 773                         sbuf_printf(sb, "%sUnknown Float", SEP_STR);
 774                         break;
 775                 }
 776
 777                 switch (ID_AA64PFR0_EL3(cpu_desc[cpu].id_aa64pfr0)) {
 778                 case ID_AA64PFR0_EL3_NONE:
 779                         sbuf_printf(sb, "%sNo EL3", SEP_STR);
 780                         break;
 781                 case ID_AA64PFR0_EL3_64:
 782                         sbuf_printf(sb, "%sEL3", SEP_STR);
 783                         break;
 784                 case ID_AA64PFR0_EL3_64_32:
 785                         sbuf_printf(sb, "%sEL3 32", SEP_STR);
 786                         break;
 787                 default:
 788                         sbuf_printf(sb, "%sUnknown EL3", SEP_STR);
 789                         break;
 790                 }
 791
 792                 switch (ID_AA64PFR0_EL2(cpu_desc[cpu].id_aa64pfr0)) {
 793                 case ID_AA64PFR0_EL2_NONE:
 794                         sbuf_printf(sb, "%sNo EL2", SEP_STR);
 795                         break;
 796                 case ID_AA64PFR0_EL2_64:
 797                         sbuf_printf(sb, "%sEL2", SEP_STR);
 798                         break;
 799                 case ID_AA64PFR0_EL2_64_32:
 800                         sbuf_printf(sb, "%sEL2 32", SEP_STR);
 801                         break;
 802                 default:
 803                         sbuf_printf(sb, "%sUnknown EL2", SEP_STR);
 804                         break;
 805                 }
 806
 807                 switch (ID_AA64PFR0_EL1(cpu_desc[cpu].id_aa64pfr0)) {
 808                 case ID_AA64PFR0_EL1_64:
 809                         sbuf_printf(sb, "%sEL1", SEP_STR);
 810                         break;
 811                 case ID_AA64PFR0_EL1_64_32:
 812                         sbuf_printf(sb, "%sEL1 32", SEP_STR);
 813                         break;
 814                 default:
 815                         sbuf_printf(sb, "%sUnknown EL1", SEP_STR);
 816                         break;
 817                 }
 818
 819                 switch (ID_AA64PFR0_EL0(cpu_desc[cpu].id_aa64pfr0)) {
 820                 case ID_AA64PFR0_EL0_64:
 821                         sbuf_printf(sb, "%sEL0", SEP_STR);
 822                         break;
 823                 case ID_AA64PFR0_EL0_64_32:
 824                         sbuf_printf(sb, "%sEL0 32", SEP_STR);
 825                         break;
 826                 default:
 827                         sbuf_printf(sb, "%sUnknown EL0", SEP_STR);
 828                         break;
 829                 }
 830
 831                 if ((cpu_desc[cpu].id_aa64pfr0 & ~ID_AA64PFR0_MASK) != 0)
 832                         sbuf_printf(sb, "%s%#lx", SEP_STR,
 833                             cpu_desc[cpu].id_aa64pfr0 & ~ID_AA64PFR0_MASK);
 834
 835                 sbuf_finish(sb);
 836                 printf("%s>\n", sbuf_data(sb));
 837                 sbuf_clear(sb);
 838         }
 839
 840         /* AArch64 Processor Feature Register 1 */
 841         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_PFR1) != 0) {
 842                 printf("         Processor Features 1 = <%#lx>\n",
 843                     cpu_desc[cpu].id_aa64pfr1);
 844         }
 845
 846         /* AArch64 Memory Model Feature Register 0 */
 847         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_MMFR0) != 0) {
 848                 printed = 0;
 849                 sbuf_printf(sb, "      Memory Model Features 0 = <");
 850                 switch (ID_AA64MMFR0_TGran4(cpu_desc[cpu].id_aa64mmfr0)) {
 851                 case ID_AA64MMFR0_TGran4_NONE:
 852                         break;
 853                 case ID_AA64MMFR0_TGran4_IMPL:
 854                         sbuf_printf(sb, "%s4k Granule", SEP_STR);
 855                         break;
 856                 default:
 857                         sbuf_printf(sb, "%sUnknown 4k Granule", SEP_STR);
 858                         break;
 859                 }
 860
 861                 switch (ID_AA64MMFR0_TGran64(cpu_desc[cpu].id_aa64mmfr0)) {
 862                 case ID_AA64MMFR0_TGran64_NONE:
 863                         break;
 864                 case ID_AA64MMFR0_TGran64_IMPL:
 865                         sbuf_printf(sb, "%s64k Granule", SEP_STR);
 866                         break;
 867                 default:
 868                         sbuf_printf(sb, "%sUnknown 64k Granule", SEP_STR);
 869                         break;
 870                 }
 871
 872                 switch (ID_AA64MMFR0_TGran16(cpu_desc[cpu].id_aa64mmfr0)) {
 873                 case ID_AA64MMFR0_TGran16_NONE:
 874                         break;
 875                 case ID_AA64MMFR0_TGran16_IMPL:
 876                         sbuf_printf(sb, "%s16k Granule", SEP_STR);
 877                         break;
 878                 default:
 879                         sbuf_printf(sb, "%sUnknown 16k Granule", SEP_STR);
 880                         break;
 881                 }
 882
 883                 switch (ID_AA64MMFR0_BigEndEL0(cpu_desc[cpu].id_aa64mmfr0)) {
 884                 case ID_AA64MMFR0_BigEndEL0_FIXED:
 885                         break;
 886                 case ID_AA64MMFR0_BigEndEL0_MIXED:
 887                         sbuf_printf(sb, "%sEL0 MixEndian", SEP_STR);
 888                         break;
 889                 default:
 890                         sbuf_printf(sb, "%sUnknown EL0 Endian switching", SEP_STR);
 891                         break;
 892                 }
 893
 894                 switch (ID_AA64MMFR0_SNSMem(cpu_desc[cpu].id_aa64mmfr0)) {
 895                 case ID_AA64MMFR0_SNSMem_NONE:
 896                         break;
 897                 case ID_AA64MMFR0_SNSMem_DISTINCT:
 898                         sbuf_printf(sb, "%sS/NS Mem", SEP_STR);
 899                         break;
 900                 default:
 901                         sbuf_printf(sb, "%sUnknown S/NS Mem", SEP_STR);
 902                         break;
 903                 }
 904
 905                 switch (ID_AA64MMFR0_BigEnd(cpu_desc[cpu].id_aa64mmfr0)) {
 906                 case ID_AA64MMFR0_BigEnd_FIXED:
 907                         break;
 908                 case ID_AA64MMFR0_BigEnd_MIXED:
 909                         sbuf_printf(sb, "%sMixedEndian", SEP_STR);
 910                         break;
 911                 default:
 912                         sbuf_printf(sb, "%sUnknown Endian switching", SEP_STR);
 913                         break;
 914                 }
 915
 916                 switch (ID_AA64MMFR0_ASIDBits(cpu_desc[cpu].id_aa64mmfr0)) {
 917                 case ID_AA64MMFR0_ASIDBits_8:
 918                         sbuf_printf(sb, "%s8bit ASID", SEP_STR);
 919                         break;
 920                 case ID_AA64MMFR0_ASIDBits_16:
 921                         sbuf_printf(sb, "%s16bit ASID", SEP_STR);
 922                         break;
 923                 default:
 924                         sbuf_printf(sb, "%sUnknown ASID", SEP_STR);
 925                         break;
 926                 }
 927
 928                 switch (ID_AA64MMFR0_PARange(cpu_desc[cpu].id_aa64mmfr0)) {
 929                 case ID_AA64MMFR0_PARange_4G:
 930                         sbuf_printf(sb, "%s4GB PA", SEP_STR);
 931                         break;
 932                 case ID_AA64MMFR0_PARange_64G:
 933                         sbuf_printf(sb, "%s64GB PA", SEP_STR);
 934                         break;
 935                 case ID_AA64MMFR0_PARange_1T:
 936                         sbuf_printf(sb, "%s1TB PA", SEP_STR);
 937                         break;
 938                 case ID_AA64MMFR0_PARange_4T:
 939                         sbuf_printf(sb, "%s4TB PA", SEP_STR);
 940                         break;
 941                 case ID_AA64MMFR0_PARange_16T:
 942                         sbuf_printf(sb, "%s16TB PA", SEP_STR);
 943                         break;
 944                 case ID_AA64MMFR0_PARange_256T:
 945                         sbuf_printf(sb, "%s256TB PA", SEP_STR);
 946                         break;
 947                 case ID_AA64MMFR0_PARange_4P:
 948                         sbuf_printf(sb, "%s4PB PA", SEP_STR);
 949                         break;
 950                 default:
 951                         sbuf_printf(sb, "%sUnknown PA Range", SEP_STR);
 952                         break;
 953                 }
 954
 955                 if ((cpu_desc[cpu].id_aa64mmfr0 & ~ID_AA64MMFR0_MASK) != 0)
 956                         sbuf_printf(sb, "%s%#lx", SEP_STR,
 957                             cpu_desc[cpu].id_aa64mmfr0 & ~ID_AA64MMFR0_MASK);
 958                 sbuf_finish(sb);
 959                 printf("%s>\n", sbuf_data(sb));
 960                 sbuf_clear(sb);
 961         }
 962
 963         /* AArch64 Memory Model Feature Register 1 */
 964         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_MMFR1) != 0) {
 965                 printed = 0;
 966                 sbuf_printf(sb, "      Memory Model Features 1 = <");
 967
 968                 switch (ID_AA64MMFR1_XNX(cpu_desc[cpu].id_aa64mmfr1)) {
 969                 case ID_AA64MMFR1_XNX_NONE:
 970                         break;
 971                 case ID_AA64MMFR1_XNX_IMPL:
 972                         sbuf_printf(sb, "%sEL2 XN", SEP_STR);
 973                         break;
 974                 default:
 975                         sbuf_printf(sb, "%sUnknown XNX", SEP_STR);
 976                         break;
 977                 }
 978
 979                 switch (ID_AA64MMFR1_SpecSEI(cpu_desc[cpu].id_aa64mmfr1)) {
 980                 case ID_AA64MMFR1_SpecSEI_NONE:
 981                         break;
 982                 case ID_AA64MMFR1_SpecSEI_IMPL:
 983                         sbuf_printf(sb, "%sSpecSEI", SEP_STR);
 984                         break;
 985                 default:
 986                         sbuf_printf(sb, "%sUnknown SpecSEI", SEP_STR);
 987                         break;
 988                 }
 989
 990                 switch (ID_AA64MMFR1_PAN(cpu_desc[cpu].id_aa64mmfr1)) {
 991                 case ID_AA64MMFR1_PAN_NONE:
 992                         break;
 993                 case ID_AA64MMFR1_PAN_IMPL:
 994                         sbuf_printf(sb, "%sPAN", SEP_STR);
 995                         break;
 996                 case ID_AA64MMFR1_PAN_ATS1E1:
 997                         sbuf_printf(sb, "%sPAN+AT", SEP_STR);
 998                         break;
 999                 default:
1000                         sbuf_printf(sb, "%sUnknown PAN", SEP_STR);
1001                         break;
1002                 }
1003
1004                 switch (ID_AA64MMFR1_LO(cpu_desc[cpu].id_aa64mmfr1)) {
1005                 case ID_AA64MMFR1_LO_NONE:
1006                         break;
1007                 case ID_AA64MMFR1_LO_IMPL:
1008                         sbuf_printf(sb, "%sLO", SEP_STR);
1009                         break;
1010                 default:
1011                         sbuf_printf(sb, "%sUnknown LO", SEP_STR);
1012                         break;
1013                 }
1014
1015                 switch (ID_AA64MMFR1_HPDS(cpu_desc[cpu].id_aa64mmfr1)) {
1016                 case ID_AA64MMFR1_HPDS_NONE:
1017                         break;
1018                 case ID_AA64MMFR1_HPDS_HPD:
1019                         sbuf_printf(sb, "%sHPDS", SEP_STR);
1020                         break;
1021                 case ID_AA64MMFR1_HPDS_TTPBHA:
1022                         sbuf_printf(sb, "%sTTPBHA", SEP_STR);
1023                         break;
1024                 default:
1025                         sbuf_printf(sb, "%sUnknown HPDS", SEP_STR);
1026                         break;
1027                 }
1028
1029                 switch (ID_AA64MMFR1_VH(cpu_desc[cpu].id_aa64mmfr1)) {
1030                 case ID_AA64MMFR1_VH_NONE:
1031                         break;
1032                 case ID_AA64MMFR1_VH_IMPL:
1033                         sbuf_printf(sb, "%sVHE", SEP_STR);
1034                         break;
1035                 default:
1036                         sbuf_printf(sb, "%sUnknown VHE", SEP_STR);
1037                         break;
1038                 }
1039
1040                 switch (ID_AA64MMFR1_VMIDBits(cpu_desc[cpu].id_aa64mmfr1)) {
1041                 case ID_AA64MMFR1_VMIDBits_8:
1042                         break;
1043                 case ID_AA64MMFR1_VMIDBits_16:
1044                         sbuf_printf(sb, "%s16 VMID bits", SEP_STR);
1045                         break;
1046                 default:
1047                         sbuf_printf(sb, "%sUnknown VMID bits", SEP_STR);
1048                         break;
1049                 }
1050
1051                 switch (ID_AA64MMFR1_HAFDBS(cpu_desc[cpu].id_aa64mmfr1)) {
1052                 case ID_AA64MMFR1_HAFDBS_NONE:
1053                         break;
1054                 case ID_AA64MMFR1_HAFDBS_AF:
1055                         sbuf_printf(sb, "%sAF", SEP_STR);
1056                         break;
1057                 case ID_AA64MMFR1_HAFDBS_AF_DBS:
1058                         sbuf_printf(sb, "%sAF+DBS", SEP_STR);
1059                         break;
1060                 default:
1061                         sbuf_printf(sb, "%sUnknown Hardware update AF/DBS", SEP_STR);
1062                         break;
1063                 }
1064
1065                 if ((cpu_desc[cpu].id_aa64mmfr1 & ~ID_AA64MMFR1_MASK) != 0)
1066                         sbuf_printf(sb, "%s%#lx", SEP_STR,
1067                             cpu_desc[cpu].id_aa64mmfr1 & ~ID_AA64MMFR1_MASK);
1068                 sbuf_finish(sb);
1069                 printf("%s>\n", sbuf_data(sb));
1070                 sbuf_clear(sb);
1071         }
1072
1073         /* AArch64 Memory Model Feature Register 2 */
1074         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_MMFR2) != 0) {
1075                 printed = 0;
1076                 sbuf_printf(sb, "      Memory Model Features 2 = <");
1077
1078                 switch (ID_AA64MMFR2_NV(cpu_desc[cpu].id_aa64mmfr2)) {
1079                 case ID_AA64MMFR2_NV_NONE:
1080                         break;
1081                 case ID_AA64MMFR2_NV_IMPL:
1082                         sbuf_printf(sb, "%sNestedVirt", SEP_STR);
1083                         break;
1084                 default:
1085                         sbuf_printf(sb, "%sUnknown NestedVirt", SEP_STR);
1086                         break;
1087                 }
1088
1089                 switch (ID_AA64MMFR2_CCIDX(cpu_desc[cpu].id_aa64mmfr2)) {
1090                 case ID_AA64MMFR2_CCIDX_32:
1091                         sbuf_printf(sb, "%s32b CCIDX", SEP_STR);
1092                         break;
1093                 case ID_AA64MMFR2_CCIDX_64:
1094                         sbuf_printf(sb, "%s64b CCIDX", SEP_STR);
1095                         break;
1096                 default:
1097                         sbuf_printf(sb, "%sUnknown CCIDX", SEP_STR);
1098                         break;
1099                 }
1100
1101                 switch (ID_AA64MMFR2_VARange(cpu_desc[cpu].id_aa64mmfr2)) {
1102                 case ID_AA64MMFR2_VARange_48:
1103                         sbuf_printf(sb, "%s48b VA", SEP_STR);
1104                         break;
1105                 case ID_AA64MMFR2_VARange_52:
1106                         sbuf_printf(sb, "%s52b VA", SEP_STR);
1107                         break;
1108                 default:
1109                         sbuf_printf(sb, "%sUnknown VA Range", SEP_STR);
1110                         break;
1111                 }
1112
1113                 switch (ID_AA64MMFR2_IESB(cpu_desc[cpu].id_aa64mmfr2)) {
1114                 case ID_AA64MMFR2_IESB_NONE:
1115                         break;
1116                 case ID_AA64MMFR2_IESB_IMPL:
1117                         sbuf_printf(sb, "%sIESB", SEP_STR);
1118                         break;
1119                 default:
1120                         sbuf_printf(sb, "%sUnknown IESB", SEP_STR);
1121                         break;
1122                 }
1123
1124                 switch (ID_AA64MMFR2_LSM(cpu_desc[cpu].id_aa64mmfr2)) {
1125                 case ID_AA64MMFR2_LSM_NONE:
1126                         break;
1127                 case ID_AA64MMFR2_LSM_IMPL:
1128                         sbuf_printf(sb, "%sLSM", SEP_STR);
1129                         break;
1130                 default:
1131                         sbuf_printf(sb, "%sUnknown LSM", SEP_STR);
1132                         break;
1133                 }
1134
1135                 switch (ID_AA64MMFR2_UAO(cpu_desc[cpu].id_aa64mmfr2)) {
1136                 case ID_AA64MMFR2_UAO_NONE:
1137                         break;
1138                 case ID_AA64MMFR2_UAO_IMPL:
1139                         sbuf_printf(sb, "%sUAO", SEP_STR);
1140                         break;
1141                 default:
1142                         sbuf_printf(sb, "%sUnknown UAO", SEP_STR);
1143                         break;
1144                 }
1145
1146                 switch (ID_AA64MMFR2_CnP(cpu_desc[cpu].id_aa64mmfr2)) {
1147                 case ID_AA64MMFR2_CnP_NONE:
1148                         break;
1149                 case ID_AA64MMFR2_CnP_IMPL:
1150                         sbuf_printf(sb, "%sCnP", SEP_STR);
1151                         break;
1152                 default:
1153                         sbuf_printf(sb, "%sUnknown CnP", SEP_STR);
1154                         break;
1155                 }
1156
1157                 if ((cpu_desc[cpu].id_aa64mmfr2 & ~ID_AA64MMFR2_MASK) != 0)
1158                         sbuf_printf(sb, "%s%#lx", SEP_STR,
1159                             cpu_desc[cpu].id_aa64mmfr2 & ~ID_AA64MMFR2_MASK);
1160                 sbuf_finish(sb);
1161                 printf("%s>\n", sbuf_data(sb));
1162                 sbuf_clear(sb);
1163         }
1164
1165         /* AArch64 Debug Feature Register 0 */
1166         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_DFR0) != 0) {
1167                 printed = 0;
1168                 sbuf_printf(sb, "             Debug Features 0 = <");
1169                 switch(ID_AA64DFR0_PMSVer(cpu_desc[cpu].id_aa64dfr0)) {
1170                 case ID_AA64DFR0_PMSVer_NONE:
1171                         break;
1172                 case ID_AA64DFR0_PMSVer_V1:
1173                         sbuf_printf(sb, "%sSPE v1", SEP_STR);
1174                         break;
1175                 default:
1176                         sbuf_printf(sb, "%sUnknown SPE", SEP_STR);
1177                         break;
1178                 }
1179
1180                 sbuf_printf(sb, "%s%lu CTX Breakpoints", SEP_STR,
1181                     ID_AA64DFR0_CTX_CMPs(cpu_desc[cpu].id_aa64dfr0));
1182
1183                 sbuf_printf(sb, "%s%lu Watchpoints", SEP_STR,
1184                     ID_AA64DFR0_WRPs(cpu_desc[cpu].id_aa64dfr0));
1185
1186                 sbuf_printf(sb, "%s%lu Breakpoints", SEP_STR,
1187                     ID_AA64DFR0_BRPs(cpu_desc[cpu].id_aa64dfr0));
1188
1189                 switch (ID_AA64DFR0_PMUVer(cpu_desc[cpu].id_aa64dfr0)) {
1190                 case ID_AA64DFR0_PMUVer_NONE:
1191                         break;
1192                 case ID_AA64DFR0_PMUVer_3:
1193                         sbuf_printf(sb, "%sPMUv3", SEP_STR);
1194                         break;
1195                 case ID_AA64DFR0_PMUVer_3_1:
1196                         sbuf_printf(sb, "%sPMUv3+16 bit evtCount", SEP_STR);
1197                         break;
1198                 case ID_AA64DFR0_PMUVer_IMPL:
1199                         sbuf_printf(sb, "%sImplementation defined PMU", SEP_STR);
1200                         break;
1201                 default:
1202                         sbuf_printf(sb, "%sUnknown PMU", SEP_STR);
1203                         break;
1204                 }
1205
1206                 switch (ID_AA64DFR0_TraceVer(cpu_desc[cpu].id_aa64dfr0)) {
1207                 case ID_AA64DFR0_TraceVer_NONE:
1208                         break;
1209                 case ID_AA64DFR0_TraceVer_IMPL:
1210                         sbuf_printf(sb, "%sTrace", SEP_STR);
1211                         break;
1212                 default:
1213                         sbuf_printf(sb, "%sUnknown Trace", SEP_STR);
1214                         break;
1215                 }
1216
1217                 switch (ID_AA64DFR0_DebugVer(cpu_desc[cpu].id_aa64dfr0)) {
1218                 case ID_AA64DFR0_DebugVer_8:
1219                         sbuf_printf(sb, "%sDebug v8", SEP_STR);
1220                         break;
1221                 case ID_AA64DFR0_DebugVer_8_VHE:
1222                         sbuf_printf(sb, "%sDebug v8+VHE", SEP_STR);
1223                         break;
1224                 case ID_AA64DFR0_DebugVer_8_2:
1225                         sbuf_printf(sb, "%sDebug v8.2", SEP_STR);
1226                         break;
1227                 default:
1228                         sbuf_printf(sb, "%sUnknown Debug", SEP_STR);
1229                         break;
1230                 }
1231
1232                 if (cpu_desc[cpu].id_aa64dfr0 & ~ID_AA64DFR0_MASK)
1233                         sbuf_printf(sb, "%s%#lx", SEP_STR,
1234                             cpu_desc[cpu].id_aa64dfr0 & ~ID_AA64DFR0_MASK);
1235                 sbuf_finish(sb);
1236                 printf("%s>\n", sbuf_data(sb));
1237                 sbuf_clear(sb);
1238         }
1239
1240         /* AArch64 Memory Model Feature Register 1 */
1241         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_DFR1) != 0) {
1242                 printf("             Debug Features 1 = <%#lx>\n",
1243                     cpu_desc[cpu].id_aa64dfr1);
1244         }
1245
1246         /* AArch64 Auxiliary Feature Register 0 */
1247         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_AFR0) != 0) {
1248                 printf("         Auxiliary Features 0 = <%#lx>\n",
1249                     cpu_desc[cpu].id_aa64afr0);
1250         }
1251
1252         /* AArch64 Auxiliary Feature Register 1 */
1253         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_AFR1) != 0) {
1254                 printf("         Auxiliary Features 1 = <%#lx>\n",
1255                     cpu_desc[cpu].id_aa64afr1);
1256         }
1257
1258         sbuf_delete(sb);
1259         sb = NULL;
1260 #undef SEP_STR
1261 }
1262
1263 void
1264 identify_cpu(void)
1265 {
1266         u_int midr;
1267         u_int impl_id;
1268         u_int part_id;
1269         u_int cpu;
1270         size_t i;
1271         const struct cpu_parts *cpu_partsp = NULL;
1272
1273         cpu = PCPU_GET(cpuid);
1274         midr = get_midr();
1275
1276         /*
1277          * Store midr to pcpu to allow fast reading
1278          * from EL0, EL1 and assembly code.
1279          */
1280         PCPU_SET(midr, midr);
1281
1282         impl_id = CPU_IMPL(midr);
1283         for (i = 0; i < nitems(cpu_implementers); i++) {
1284                 if (impl_id == cpu_implementers[i].impl_id ||
1285                     cpu_implementers[i].impl_id == 0) {
1286                         cpu_desc[cpu].cpu_impl = impl_id;
1287                         cpu_desc[cpu].cpu_impl_name = cpu_implementers[i].impl_name;
1288                         cpu_partsp = cpu_implementers[i].cpu_parts;
1289                         break;
1290                 }
1291         }
1292
1293         part_id = CPU_PART(midr);
1294         for (i = 0; &cpu_partsp[i] != NULL; i++) {
1295                 if (part_id == cpu_partsp[i].part_id ||
1296                     cpu_partsp[i].part_id == 0) {
1297                         cpu_desc[cpu].cpu_part_num = part_id;
1298                         cpu_desc[cpu].cpu_part_name = cpu_partsp[i].part_name;
1299                         break;
1300                 }
1301         }
1302
1303         cpu_desc[cpu].cpu_revision = CPU_REV(midr);
1304         cpu_desc[cpu].cpu_variant = CPU_VAR(midr);
1305
1306         snprintf(cpu_model, sizeof(cpu_model), "%s %s r%dp%d",
1307             cpu_desc[cpu].cpu_impl_name, cpu_desc[cpu].cpu_part_name,
1308             cpu_desc[cpu].cpu_variant, cpu_desc[cpu].cpu_revision);
1309
1310         /* Save affinity for current CPU */
1311         cpu_desc[cpu].mpidr = get_mpidr();
1312         CPU_AFFINITY(cpu) = cpu_desc[cpu].mpidr & CPU_AFF_MASK;
1313
1314         cpu_desc[cpu].id_aa64dfr0 = READ_SPECIALREG(ID_AA64DFR0_EL1);
1315         cpu_desc[cpu].id_aa64dfr1 = READ_SPECIALREG(ID_AA64DFR1_EL1);
1316         cpu_desc[cpu].id_aa64isar0 = READ_SPECIALREG(ID_AA64ISAR0_EL1);
1317         cpu_desc[cpu].id_aa64isar1 = READ_SPECIALREG(ID_AA64ISAR1_EL1);
1318         cpu_desc[cpu].id_aa64mmfr0 = READ_SPECIALREG(ID_AA64MMFR0_EL1);
1319         cpu_desc[cpu].id_aa64mmfr1 = READ_SPECIALREG(ID_AA64MMFR1_EL1);
1320         cpu_desc[cpu].id_aa64mmfr2 = READ_SPECIALREG(ID_AA64MMFR2_EL1);
1321         cpu_desc[cpu].id_aa64pfr0 = READ_SPECIALREG(ID_AA64PFR0_EL1);
1322         cpu_desc[cpu].id_aa64pfr1 = READ_SPECIALREG(ID_AA64PFR1_EL1);
1323
1324         if (cpu != 0) {
1325                 /*
1326                  * This code must run on one cpu at a time, but we are
1327                  * not scheduling on the current core so implement a
1328                  * simple spinlock.
1329                  */
1330                 while (atomic_cmpset_acq_int(&ident_lock, 0, 1) == 0)
1331                         __asm __volatile("wfe" ::: "memory");
1332
1333                 switch (cpu_aff_levels) {
1334                 case 0:
1335                         if (CPU_AFF0(cpu_desc[cpu].mpidr) !=
1336                             CPU_AFF0(cpu_desc[0].mpidr))
1337                                 cpu_aff_levels = 1;
1338                         /* FALLTHROUGH */
1339                 case 1:
1340                         if (CPU_AFF1(cpu_desc[cpu].mpidr) !=
1341                             CPU_AFF1(cpu_desc[0].mpidr))
1342                                 cpu_aff_levels = 2;
1343                         /* FALLTHROUGH */
1344                 case 2:
1345                         if (CPU_AFF2(cpu_desc[cpu].mpidr) !=
1346                             CPU_AFF2(cpu_desc[0].mpidr))
1347                                 cpu_aff_levels = 3;
1348                         /* FALLTHROUGH */
1349                 case 3:
1350                         if (CPU_AFF3(cpu_desc[cpu].mpidr) !=
1351                             CPU_AFF3(cpu_desc[0].mpidr))
1352                                 cpu_aff_levels = 4;
1353                         break;
1354                 }
1355
1356                 if (cpu_desc[cpu].id_aa64afr0 != cpu_desc[0].id_aa64afr0)
1357                         cpu_print_regs |= PRINT_ID_AA64_AFR0;
1358                 if (cpu_desc[cpu].id_aa64afr1 != cpu_desc[0].id_aa64afr1)
1359                         cpu_print_regs |= PRINT_ID_AA64_AFR1;
1360
1361                 if (cpu_desc[cpu].id_aa64dfr0 != cpu_desc[0].id_aa64dfr0)
1362                         cpu_print_regs |= PRINT_ID_AA64_DFR0;
1363                 if (cpu_desc[cpu].id_aa64dfr1 != cpu_desc[0].id_aa64dfr1)
1364                         cpu_print_regs |= PRINT_ID_AA64_DFR1;
1365
1366                 if (cpu_desc[cpu].id_aa64isar0 != cpu_desc[0].id_aa64isar0)
1367                         cpu_print_regs |= PRINT_ID_AA64_ISAR0;
1368                 if (cpu_desc[cpu].id_aa64isar1 != cpu_desc[0].id_aa64isar1)
1369                         cpu_print_regs |= PRINT_ID_AA64_ISAR1;
1370
1371                 if (cpu_desc[cpu].id_aa64mmfr0 != cpu_desc[0].id_aa64mmfr0)
1372                         cpu_print_regs |= PRINT_ID_AA64_MMFR0;
1373                 if (cpu_desc[cpu].id_aa64mmfr1 != cpu_desc[0].id_aa64mmfr1)
1374                         cpu_print_regs |= PRINT_ID_AA64_MMFR1;
1375                 if (cpu_desc[cpu].id_aa64mmfr2 != cpu_desc[0].id_aa64mmfr2)
1376                         cpu_print_regs |= PRINT_ID_AA64_MMFR2;
1377
1378                 if (cpu_desc[cpu].id_aa64pfr0 != cpu_desc[0].id_aa64pfr0)
1379                         cpu_print_regs |= PRINT_ID_AA64_PFR0;
1380                 if (cpu_desc[cpu].id_aa64pfr1 != cpu_desc[0].id_aa64pfr1)
1381                         cpu_print_regs |= PRINT_ID_AA64_PFR1;
1382
1383                 /* Wake up the other CPUs */
1384                 atomic_store_rel_int(&ident_lock, 0);
1385                 __asm __volatile("sev" ::: "memory");
1386         }
1387 }