sys/arm64/arm64/identcpu.c

   1 /*-
   2  * Copyright (c) 2014 Andrew Turner
   3  * Copyright (c) 2014 The FreeBSD Foundation
   4  * All rights reserved.
   5  *
   6  * Portions of this software were developed by Semihalf
   7  * under sponsorship of the FreeBSD Foundation.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  *
  30  */
  31
  32 #include <sys/cdefs.h>
  33 __FBSDID("$FreeBSD$");
  34
  35 #include <sys/param.h>
  36 #include <sys/kernel.h>
  37 #include <sys/pcpu.h>
  38 #include <sys/sbuf.h>
  39 #include <sys/smp.h>
  40 #include <sys/sysctl.h>
  41 #include <sys/systm.h>
  42
  43 #include <machine/atomic.h>
  44 #include <machine/cpu.h>
  45 #include <machine/cpufunc.h>
  46 #include <machine/undefined.h>
  47 #include <machine/elf.h>
  48
  49 static int ident_lock;
  50 static void print_cpu_features(u_int cpu);
  51 static u_long parse_cpu_features_hwcap(u_int cpu);
  52
  53 char machine[] = "arm64";
  54
  55 #ifdef SCTL_MASK32
  56 extern int adaptive_machine_arch;
  57 #endif
  58
  59 static int
  60 sysctl_hw_machine(SYSCTL_HANDLER_ARGS)
  61 {
  62 #ifdef SCTL_MASK32
  63         static const char machine32[] = "arm";
  64 #endif
  65         int error;
  66 #ifdef SCTL_MASK32
  67         if ((req->flags & SCTL_MASK32) != 0 && adaptive_machine_arch)
  68                 error = SYSCTL_OUT(req, machine32, sizeof(machine32));
  69         else
  70 #endif
  71                 error = SYSCTL_OUT(req, machine, sizeof(machine));
  72         return (error);
  73 }
  74
  75 SYSCTL_PROC(_hw, HW_MACHINE, machine, CTLTYPE_STRING | CTLFLAG_RD |
  76         CTLFLAG_MPSAFE, NULL, 0, sysctl_hw_machine, "A", "Machine class");
  77
  78 static char cpu_model[64];
  79 SYSCTL_STRING(_hw, HW_MODEL, model, CTLFLAG_RD,
  80         cpu_model, sizeof(cpu_model), "Machine model");
  81
  82 /*
  83  * Per-CPU affinity as provided in MPIDR_EL1
  84  * Indexed by CPU number in logical order selected by the system.
  85  * Relevant fields can be extracted using CPU_AFFn macros,
  86  * Aff3.Aff2.Aff1.Aff0 construct a unique CPU address in the system.
  87  *
  88  * Fields used by us:
  89  * Aff1 - Cluster number
  90  * Aff0 - CPU number in Aff1 cluster
  91  */
  92 uint64_t __cpu_affinity[MAXCPU];
  93 static u_int cpu_aff_levels;
  94
  95 struct cpu_desc {
  96         u_int           cpu_impl;
  97         u_int           cpu_part_num;
  98         u_int           cpu_variant;
  99         u_int           cpu_revision;
 100         const char      *cpu_impl_name;
 101         const char      *cpu_part_name;
 102
 103         uint64_t        mpidr;
 104         uint64_t        id_aa64afr0;
 105         uint64_t        id_aa64afr1;
 106         uint64_t        id_aa64dfr0;
 107         uint64_t        id_aa64dfr1;
 108         uint64_t        id_aa64isar0;
 109         uint64_t        id_aa64isar1;
 110         uint64_t        id_aa64mmfr0;
 111         uint64_t        id_aa64mmfr1;
 112         uint64_t        id_aa64mmfr2;
 113         uint64_t        id_aa64pfr0;
 114         uint64_t        id_aa64pfr1;
 115 };
 116
 117 struct cpu_desc cpu_desc[MAXCPU];
 118 struct cpu_desc user_cpu_desc;
 119 static u_int cpu_print_regs;
 120 #define PRINT_ID_AA64_AFR0      0x00000001
 121 #define PRINT_ID_AA64_AFR1      0x00000002
 122 #define PRINT_ID_AA64_DFR0      0x00000010
 123 #define PRINT_ID_AA64_DFR1      0x00000020
 124 #define PRINT_ID_AA64_ISAR0     0x00000100
 125 #define PRINT_ID_AA64_ISAR1     0x00000200
 126 #define PRINT_ID_AA64_MMFR0     0x00001000
 127 #define PRINT_ID_AA64_MMFR1     0x00002000
 128 #define PRINT_ID_AA64_MMFR2     0x00004000
 129 #define PRINT_ID_AA64_PFR0      0x00010000
 130 #define PRINT_ID_AA64_PFR1      0x00020000
 131
 132 struct cpu_parts {
 133         u_int           part_id;
 134         const char      *part_name;
 135 };
 136 #define CPU_PART_NONE   { 0, "Unknown Processor" }
 137
 138 struct cpu_implementers {
 139         u_int                   impl_id;
 140         const char              *impl_name;
 141         /*
 142          * Part number is implementation defined
 143          * so each vendor will have its own set of values and names.
 144          */
 145         const struct cpu_parts  *cpu_parts;
 146 };
 147 #define CPU_IMPLEMENTER_NONE    { 0, "Unknown Implementer", cpu_parts_none }
 148
 149 /*
 150  * Per-implementer table of (PartNum, CPU Name) pairs.
 151  */
 152 /* ARM Ltd. */
 153 static const struct cpu_parts cpu_parts_arm[] = {
 154         { CPU_PART_FOUNDATION, "Foundation-Model" },
 155         { CPU_PART_CORTEX_A35, "Cortex-A35" },
 156         { CPU_PART_CORTEX_A53, "Cortex-A53" },
 157         { CPU_PART_CORTEX_A55, "Cortex-A55" },
 158         { CPU_PART_CORTEX_A57, "Cortex-A57" },
 159         { CPU_PART_CORTEX_A72, "Cortex-A72" },
 160         { CPU_PART_CORTEX_A73, "Cortex-A73" },
 161         { CPU_PART_CORTEX_A75, "Cortex-A75" },
 162         CPU_PART_NONE,
 163 };
 164 /* Cavium */
 165 static const struct cpu_parts cpu_parts_cavium[] = {
 166         { CPU_PART_THUNDERX, "ThunderX" },
 167         { CPU_PART_THUNDERX2, "ThunderX2" },
 168         CPU_PART_NONE,
 169 };
 170
 171 /* Unknown */
 172 static const struct cpu_parts cpu_parts_none[] = {
 173         CPU_PART_NONE,
 174 };
 175
 176 /*
 177  * Implementers table.
 178  */
 179 const struct cpu_implementers cpu_implementers[] = {
 180         { CPU_IMPL_ARM,         "ARM",          cpu_parts_arm },
 181         { CPU_IMPL_BROADCOM,    "Broadcom",     cpu_parts_none },
 182         { CPU_IMPL_CAVIUM,      "Cavium",       cpu_parts_cavium },
 183         { CPU_IMPL_DEC,         "DEC",          cpu_parts_none },
 184         { CPU_IMPL_INFINEON,    "IFX",          cpu_parts_none },
 185         { CPU_IMPL_FREESCALE,   "Freescale",    cpu_parts_none },
 186         { CPU_IMPL_NVIDIA,      "NVIDIA",       cpu_parts_none },
 187         { CPU_IMPL_APM,         "APM",          cpu_parts_none },
 188         { CPU_IMPL_QUALCOMM,    "Qualcomm",     cpu_parts_none },
 189         { CPU_IMPL_MARVELL,     "Marvell",      cpu_parts_none },
 190         { CPU_IMPL_INTEL,       "Intel",        cpu_parts_none },
 191         CPU_IMPLEMENTER_NONE,
 192 };
 193
 194 #define MRS_TYPE_MASK           0xf
 195 #define MRS_INVALID             0
 196 #define MRS_EXACT               1
 197 #define MRS_EXACT_VAL(x)        (MRS_EXACT | ((x) << 4))
 198 #define MRS_EXACT_FIELD(x)      ((x) >> 4)
 199 #define MRS_LOWER               2
 200
 201 struct mrs_field {
 202         bool            sign;
 203         u_int           type;
 204         u_int           shift;
 205 };
 206
 207 #define MRS_FIELD(_sign, _type, _shift)                                 \
 208         {                                                               \
 209                 .sign = (_sign),                                        \
 210                 .type = (_type),                                        \
 211                 .shift = (_shift),                                      \
 212         }
 213
 214 #define MRS_FIELD_END   { .type = MRS_INVALID, }
 215
 216 static struct mrs_field id_aa64isar0_fields[] = {
 217         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR0_DP_SHIFT),
 218         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR0_SM4_SHIFT),
 219         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR0_SM3_SHIFT),
 220         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR0_SHA3_SHIFT),
 221         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR0_RDM_SHIFT),
 222         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR0_Atomic_SHIFT),
 223         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR0_CRC32_SHIFT),
 224         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR0_SHA2_SHIFT),
 225         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR0_SHA1_SHIFT),
 226         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR0_AES_SHIFT),
 227         MRS_FIELD_END,
 228 };
 229
 230 static struct mrs_field id_aa64isar1_fields[] = {
 231         MRS_FIELD(false, MRS_EXACT, ID_AA64ISAR1_GPI_SHIFT),
 232         MRS_FIELD(false, MRS_EXACT, ID_AA64ISAR1_GPA_SHIFT),
 233         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR1_LRCPC_SHIFT),
 234         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR1_FCMA_SHIFT),
 235         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR1_JSCVT_SHIFT),
 236         MRS_FIELD(false, MRS_EXACT, ID_AA64ISAR1_API_SHIFT),
 237         MRS_FIELD(false, MRS_EXACT, ID_AA64ISAR1_APA_SHIFT),
 238         MRS_FIELD(false, MRS_LOWER, ID_AA64ISAR1_DPB_SHIFT),
 239         MRS_FIELD_END,
 240 };
 241
 242 static struct mrs_field id_aa64pfr0_fields[] = {
 243         MRS_FIELD(false, MRS_EXACT, ID_AA64PFR0_SVE_SHIFT),
 244         MRS_FIELD(false, MRS_EXACT, ID_AA64PFR0_RAS_SHIFT),
 245         MRS_FIELD(false, MRS_EXACT, ID_AA64PFR0_GIC_SHIFT),
 246         MRS_FIELD(true,  MRS_LOWER, ID_AA64PFR0_AdvSIMD_SHIFT),
 247         MRS_FIELD(true,  MRS_LOWER, ID_AA64PFR0_FP_SHIFT),
 248         MRS_FIELD(false, MRS_EXACT, ID_AA64PFR0_EL3_SHIFT),
 249         MRS_FIELD(false, MRS_EXACT, ID_AA64PFR0_EL2_SHIFT),
 250         MRS_FIELD(false, MRS_LOWER, ID_AA64PFR0_EL1_SHIFT),
 251         MRS_FIELD(false, MRS_LOWER, ID_AA64PFR0_EL0_SHIFT),
 252         MRS_FIELD_END,
 253 };
 254
 255 static struct mrs_field id_aa64dfr0_fields[] = {
 256         MRS_FIELD(false, MRS_EXACT, ID_AA64DFR0_PMSVer_SHIFT),
 257         MRS_FIELD(false, MRS_EXACT, ID_AA64DFR0_CTX_CMPs_SHIFT),
 258         MRS_FIELD(false, MRS_EXACT, ID_AA64DFR0_WRPs_SHIFT),
 259         MRS_FIELD(false, MRS_EXACT, ID_AA64DFR0_BRPs_SHIFT),
 260         MRS_FIELD(false, MRS_EXACT, ID_AA64DFR0_PMUVer_SHIFT),
 261         MRS_FIELD(false, MRS_EXACT, ID_AA64DFR0_TraceVer_SHIFT),
 262         MRS_FIELD(false, MRS_EXACT_VAL(0x6), ID_AA64DFR0_DebugVer_SHIFT),
 263         MRS_FIELD_END,
 264 };
 265
 266 struct mrs_user_reg {
 267         u_int           CRm;
 268         u_int           Op2;
 269         size_t          offset;
 270         struct mrs_field *fields;
 271 };
 272
 273 static struct mrs_user_reg user_regs[] = {
 274         {       /* id_aa64isar0_el1 */
 275                 .CRm = 6,
 276                 .Op2 = 0,
 277                 .offset = __offsetof(struct cpu_desc, id_aa64isar0),
 278                 .fields = id_aa64isar0_fields,
 279         },
 280         {       /* id_aa64isar1_el1 */
 281                 .CRm = 6,
 282                 .Op2 = 1,
 283                 .offset = __offsetof(struct cpu_desc, id_aa64isar1),
 284                 .fields = id_aa64isar1_fields,
 285         },
 286         {       /* id_aa64pfr0_el1 */
 287                 .CRm = 4,
 288                 .Op2 = 0,
 289                 .offset = __offsetof(struct cpu_desc, id_aa64pfr0),
 290                 .fields = id_aa64pfr0_fields,
 291         },
 292         {       /* id_aa64dfr0_el1 */
 293                 .CRm = 5,
 294                 .Op2 = 0,
 295                 .offset = __offsetof(struct cpu_desc, id_aa64dfr0),
 296                 .fields = id_aa64dfr0_fields,
 297         },
 298 };
 299
 300 #define CPU_DESC_FIELD(desc, idx)                                       \
 301     *(uint64_t *)((char *)&(desc) + user_regs[(idx)].offset)
 302
 303 static int
 304 user_mrs_handler(vm_offset_t va, uint32_t insn, struct trapframe *frame,
 305     uint32_t esr)
 306 {
 307         uint64_t value;
 308         int CRm, Op2, i, reg;
 309
 310         if ((insn & MRS_MASK) != MRS_VALUE)
 311                 return (0);
 312
 313         /*
 314          * We only emulate Op0 == 3, Op1 == 0, CRn == 0, CRm == {0, 4-7}.
 315          * These are in the EL1 CPU identification space.
 316          * CRm == 0 holds MIDR_EL1, MPIDR_EL1, and REVID_EL1.
 317          * CRm == {4-7} holds the ID_AA64 registers.
 318          *
 319          * For full details see the ARMv8 ARM (ARM DDI 0487C.a)
 320          * Table D9-2 System instruction encodings for non-Debug System
 321          * register accesses.
 322          */
 323         if (mrs_Op0(insn) != 3 || mrs_Op1(insn) != 0 || mrs_CRn(insn) != 0)
 324                 return (0);
 325
 326         CRm = mrs_CRm(insn);
 327         if (CRm > 7 || (CRm < 4 && CRm != 0))
 328                 return (0);
 329
 330         Op2 = mrs_Op2(insn);
 331         value = 0;
 332
 333         for (i = 0; i < nitems(user_regs); i++) {
 334                 if (user_regs[i].CRm == CRm && user_regs[i].Op2 == Op2) {
 335                         value = CPU_DESC_FIELD(user_cpu_desc, i);
 336                         break;
 337                 }
 338         }
 339
 340         if (CRm == 0) {
 341                 switch (Op2) {
 342                 case 0:
 343                         value = READ_SPECIALREG(midr_el1);
 344                         break;
 345                 case 5:
 346                         value = READ_SPECIALREG(mpidr_el1);
 347                         break;
 348                 case 6:
 349                         value = READ_SPECIALREG(revidr_el1);
 350                         break;
 351                 default:
 352                         return (0);
 353                 }
 354         }
 355
 356         /*
 357          * We will handle this instruction, move to the next so we
 358          * don't trap here again.
 359          */
 360         frame->tf_elr += INSN_SIZE;
 361
 362         reg = MRS_REGISTER(insn);
 363         /* If reg is 31 then write to xzr, i.e. do nothing */
 364         if (reg == 31)
 365                 return (1);
 366
 367         if (reg < nitems(frame->tf_x))
 368                 frame->tf_x[reg] = value;
 369         else if (reg == 30)
 370                 frame->tf_lr = value;
 371
 372         return (1);
 373 }
 374
 375 static void
 376 update_user_regs(u_int cpu)
 377 {
 378         struct mrs_field *fields;
 379         uint64_t cur, value;
 380         int i, j, cur_field, new_field;
 381
 382         for (i = 0; i < nitems(user_regs); i++) {
 383                 value = CPU_DESC_FIELD(cpu_desc[cpu], i);
 384                 if (cpu == 0)
 385                         cur = value;
 386                 else
 387                         cur = CPU_DESC_FIELD(user_cpu_desc, i);
 388
 389                 fields = user_regs[i].fields;
 390                 for (j = 0; fields[j].type != 0; j++) {
 391                         switch (fields[j].type & MRS_TYPE_MASK) {
 392                         case MRS_EXACT:
 393                                 cur &= ~(0xfu << fields[j].shift);
 394                                 cur |=
 395                                     (uint64_t)MRS_EXACT_FIELD(fields[j].type) <<
 396                                     fields[j].shift;
 397                                 break;
 398                         case MRS_LOWER:
 399                                 new_field = (value >> fields[j].shift) & 0xf;
 400                                 cur_field = (cur >> fields[j].shift) & 0xf;
 401                                 if ((fields[j].sign &&
 402                                      (int)new_field < (int)cur_field) ||
 403                                     (!fields[j].sign &&
 404                                      (u_int)new_field < (u_int)cur_field)) {
 405                                         cur &= ~(0xfu << fields[j].shift);
 406                                         cur |= new_field << fields[j].shift;
 407                                 }
 408                                 break;
 409                         default:
 410                                 panic("Invalid field type: %d", fields[j].type);
 411                         }
 412                 }
 413
 414                 CPU_DESC_FIELD(user_cpu_desc, i) = cur;
 415         }
 416 }
 417
 418 /* HWCAP */
 419 extern u_long elf_hwcap;
 420
 421 static void
 422 identify_cpu_sysinit(void *dummy __unused)
 423 {
 424         int cpu;
 425         u_long hwcap;
 426
 427         /* Create a user visible cpu description with safe values */
 428         memset(&user_cpu_desc, 0, sizeof(user_cpu_desc));
 429         /* Safe values for these registers */
 430         user_cpu_desc.id_aa64pfr0 = ID_AA64PFR0_AdvSIMD_NONE |
 431             ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 | ID_AA64PFR0_EL0_64;
 432         user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8;
 433
 434
 435         CPU_FOREACH(cpu) {
 436                 print_cpu_features(cpu);
 437                 hwcap = parse_cpu_features_hwcap(cpu);
 438                 if (elf_hwcap == 0)
 439                         elf_hwcap = hwcap;
 440                 else
 441                         elf_hwcap &= hwcap;
 442                 update_user_regs(cpu);
 443         }
 444
 445         install_undef_handler(true, user_mrs_handler);
 446 }
 447 SYSINIT(idenrity_cpu, SI_SUB_SMP, SI_ORDER_ANY, identify_cpu_sysinit, NULL);
 448
 449 static u_long
 450 parse_cpu_features_hwcap(u_int cpu)
 451 {
 452         u_long hwcap = 0;
 453
 454         if (ID_AA64ISAR0_DP(cpu_desc[cpu].id_aa64isar0) == ID_AA64ISAR0_DP_IMPL)
 455                 hwcap |= HWCAP_ASIMDDP;
 456
 457         if (ID_AA64ISAR0_SM4(cpu_desc[cpu].id_aa64isar0) == ID_AA64ISAR0_SM4_IMPL)
 458                 hwcap |= HWCAP_SM4;
 459
 460         if (ID_AA64ISAR0_SM3(cpu_desc[cpu].id_aa64isar0) == ID_AA64ISAR0_SM3_IMPL)
 461                 hwcap |= HWCAP_SM3;
 462
 463         if (ID_AA64ISAR0_RDM(cpu_desc[cpu].id_aa64isar0) == ID_AA64ISAR0_RDM_IMPL)
 464                 hwcap |= HWCAP_ASIMDRDM;
 465
 466         if (ID_AA64ISAR0_Atomic(cpu_desc[cpu].id_aa64isar0) == ID_AA64ISAR0_Atomic_IMPL)
 467                 hwcap |= HWCAP_ATOMICS;
 468
 469         if (ID_AA64ISAR0_CRC32(cpu_desc[cpu].id_aa64isar0) == ID_AA64ISAR0_CRC32_BASE)
 470                 hwcap |= HWCAP_CRC32;
 471
 472         switch (ID_AA64ISAR0_SHA2(cpu_desc[cpu].id_aa64isar0)) {
 473                 case ID_AA64ISAR0_SHA2_BASE:
 474                         hwcap |= HWCAP_SHA2;
 475                         break;
 476                 case ID_AA64ISAR0_SHA2_512:
 477                         hwcap |= HWCAP_SHA2 | HWCAP_SHA512;
 478                         break;
 479         default:
 480                 break;
 481         }
 482
 483         if (ID_AA64ISAR0_SHA1(cpu_desc[cpu].id_aa64isar0))
 484                 hwcap |= HWCAP_SHA1;
 485
 486         switch (ID_AA64ISAR0_AES(cpu_desc[cpu].id_aa64isar0)) {
 487         case ID_AA64ISAR0_AES_BASE:
 488                 hwcap |= HWCAP_AES;
 489                 break;
 490         case ID_AA64ISAR0_AES_PMULL:
 491                 hwcap |= HWCAP_PMULL | HWCAP_AES;
 492                 break;
 493         default:
 494                 break;
 495         }
 496
 497         if (ID_AA64ISAR1_LRCPC(cpu_desc[cpu].id_aa64isar1) == ID_AA64ISAR1_LRCPC_IMPL)
 498                 hwcap |= HWCAP_LRCPC;
 499
 500         if (ID_AA64ISAR1_FCMA(cpu_desc[cpu].id_aa64isar1) == ID_AA64ISAR1_FCMA_IMPL)
 501                 hwcap |= HWCAP_FCMA;
 502
 503         if (ID_AA64ISAR1_JSCVT(cpu_desc[cpu].id_aa64isar1) == ID_AA64ISAR1_JSCVT_IMPL)
 504                 hwcap |= HWCAP_JSCVT;
 505
 506         if (ID_AA64ISAR1_DPB(cpu_desc[cpu].id_aa64isar1) == ID_AA64ISAR1_DPB_IMPL)
 507                 hwcap |= HWCAP_DCPOP;
 508
 509         if (ID_AA64PFR0_SVE(cpu_desc[cpu].id_aa64pfr0) == ID_AA64PFR0_SVE_IMPL)
 510                 hwcap |= HWCAP_SVE;
 511
 512         switch (ID_AA64PFR0_AdvSIMD(cpu_desc[cpu].id_aa64pfr0)) {
 513         case ID_AA64PFR0_AdvSIMD_IMPL:
 514                 hwcap |= HWCAP_ASIMD;
 515                 break;
 516         case ID_AA64PFR0_AdvSIMD_HP:
 517                 hwcap |= HWCAP_ASIMD | HWCAP_ASIMDDP;
 518                 break;
 519         default:
 520                 break;
 521         }
 522
 523         switch (ID_AA64PFR0_FP(cpu_desc[cpu].id_aa64pfr0)) {
 524         case ID_AA64PFR0_FP_IMPL:
 525                 hwcap |= HWCAP_FP;
 526                 break;
 527         case ID_AA64PFR0_FP_HP:
 528                 hwcap |= HWCAP_FP | HWCAP_FPHP;
 529                 break;
 530         default:
 531                 break;
 532         }
 533
 534         return (hwcap);
 535 }
 536
 537 static void
 538 print_cpu_features(u_int cpu)
 539 {
 540         struct sbuf *sb;
 541         int printed;
 542
 543         sb = sbuf_new_auto();
 544         sbuf_printf(sb, "CPU%3d: %s %s r%dp%d", cpu,
 545             cpu_desc[cpu].cpu_impl_name, cpu_desc[cpu].cpu_part_name,
 546             cpu_desc[cpu].cpu_variant, cpu_desc[cpu].cpu_revision);
 547
 548         sbuf_cat(sb, " affinity:");
 549         switch(cpu_aff_levels) {
 550         default:
 551         case 4:
 552                 sbuf_printf(sb, " %2d", CPU_AFF3(cpu_desc[cpu].mpidr));
 553                 /* FALLTHROUGH */
 554         case 3:
 555                 sbuf_printf(sb, " %2d", CPU_AFF2(cpu_desc[cpu].mpidr));
 556                 /* FALLTHROUGH */
 557         case 2:
 558                 sbuf_printf(sb, " %2d", CPU_AFF1(cpu_desc[cpu].mpidr));
 559                 /* FALLTHROUGH */
 560         case 1:
 561         case 0: /* On UP this will be zero */
 562                 sbuf_printf(sb, " %2d", CPU_AFF0(cpu_desc[cpu].mpidr));
 563                 break;
 564         }
 565         sbuf_finish(sb);
 566         printf("%s\n", sbuf_data(sb));
 567         sbuf_clear(sb);
 568
 569         /*
 570          * There is a hardware errata where, if one CPU is performing a TLB
 571          * invalidation while another is performing a store-exclusive the
 572          * store-exclusive may return the wrong status. A workaround seems
 573          * to be to use an IPI to invalidate on each CPU, however given the
 574          * limited number of affected units (pass 1.1 is the evaluation
 575          * hardware revision), and the lack of information from Cavium
 576          * this has not been implemented.
 577          *
 578          * At the time of writing this the only information is from:
 579          * https://lkml.org/lkml/2016/8/4/722
 580          */
 581         /*
 582          * XXX: CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1 on its own also
 583          * triggers on pass 2.0+.
 584          */
 585         if (cpu == 0 && CPU_VAR(PCPU_GET(midr)) == 0 &&
 586             CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1)
 587                 printf("WARNING: ThunderX Pass 1.1 detected.\nThis has known "
 588                     "hardware bugs that may cause the incorrect operation of "
 589                     "atomic operations.\n");
 590
 591 #define SEP_STR ((printed++) == 0) ? "" : ","
 592
 593         /* AArch64 Instruction Set Attribute Register 0 */
 594         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_ISAR0) != 0) {
 595                 printed = 0;
 596                 sbuf_printf(sb, " Instruction Set Attributes 0 = <");
 597
 598                 switch (ID_AA64ISAR0_DP(cpu_desc[cpu].id_aa64isar0)) {
 599                 case ID_AA64ISAR0_DP_NONE:
 600                         break;
 601                 case ID_AA64ISAR0_DP_IMPL:
 602                         sbuf_printf(sb, "%sDotProd", SEP_STR);
 603                         break;
 604                 default:
 605                         sbuf_printf(sb, "%sUnknown DP", SEP_STR);
 606                         break;
 607                 }
 608
 609                 switch (ID_AA64ISAR0_SM4(cpu_desc[cpu].id_aa64isar0)) {
 610                 case ID_AA64ISAR0_SM4_NONE:
 611                         break;
 612                 case ID_AA64ISAR0_SM4_IMPL:
 613                         sbuf_printf(sb, "%sSM4", SEP_STR);
 614                         break;
 615                 default:
 616                         sbuf_printf(sb, "%sUnknown SM4", SEP_STR);
 617                         break;
 618                 }
 619
 620                 switch (ID_AA64ISAR0_SM3(cpu_desc[cpu].id_aa64isar0)) {
 621                 case ID_AA64ISAR0_SM3_NONE:
 622                         break;
 623                 case ID_AA64ISAR0_SM3_IMPL:
 624                         sbuf_printf(sb, "%sSM3", SEP_STR);
 625                         break;
 626                 default:
 627                         sbuf_printf(sb, "%sUnknown SM3", SEP_STR);
 628                         break;
 629                 }
 630
 631                 switch (ID_AA64ISAR0_SHA3(cpu_desc[cpu].id_aa64isar0)) {
 632                 case ID_AA64ISAR0_SHA3_NONE:
 633                         break;
 634                 case ID_AA64ISAR0_SHA3_IMPL:
 635                         sbuf_printf(sb, "%sSHA3", SEP_STR);
 636                         break;
 637                 default:
 638                         sbuf_printf(sb, "%sUnknown SHA3", SEP_STR);
 639                         break;
 640                 }
 641
 642                 switch (ID_AA64ISAR0_RDM(cpu_desc[cpu].id_aa64isar0)) {
 643                 case ID_AA64ISAR0_RDM_NONE:
 644                         break;
 645                 case ID_AA64ISAR0_RDM_IMPL:
 646                         sbuf_printf(sb, "%sRDM", SEP_STR);
 647                         break;
 648                 default:
 649                         sbuf_printf(sb, "%sUnknown RDM", SEP_STR);
 650                 }
 651
 652                 switch (ID_AA64ISAR0_Atomic(cpu_desc[cpu].id_aa64isar0)) {
 653                 case ID_AA64ISAR0_Atomic_NONE:
 654                         break;
 655                 case ID_AA64ISAR0_Atomic_IMPL:
 656                         sbuf_printf(sb, "%sAtomic", SEP_STR);
 657                         break;
 658                 default:
 659                         sbuf_printf(sb, "%sUnknown Atomic", SEP_STR);
 660                 }
 661
 662                 switch (ID_AA64ISAR0_CRC32(cpu_desc[cpu].id_aa64isar0)) {
 663                 case ID_AA64ISAR0_CRC32_NONE:
 664                         break;
 665                 case ID_AA64ISAR0_CRC32_BASE:
 666                         sbuf_printf(sb, "%sCRC32", SEP_STR);
 667                         break;
 668                 default:
 669                         sbuf_printf(sb, "%sUnknown CRC32", SEP_STR);
 670                         break;
 671                 }
 672
 673                 switch (ID_AA64ISAR0_SHA2(cpu_desc[cpu].id_aa64isar0)) {
 674                 case ID_AA64ISAR0_SHA2_NONE:
 675                         break;
 676                 case ID_AA64ISAR0_SHA2_BASE:
 677                         sbuf_printf(sb, "%sSHA2", SEP_STR);
 678                         break;
 679                 case ID_AA64ISAR0_SHA2_512:
 680                         sbuf_printf(sb, "%sSHA2+SHA512", SEP_STR);
 681                         break;
 682                 default:
 683                         sbuf_printf(sb, "%sUnknown SHA2", SEP_STR);
 684                         break;
 685                 }
 686
 687                 switch (ID_AA64ISAR0_SHA1(cpu_desc[cpu].id_aa64isar0)) {
 688                 case ID_AA64ISAR0_SHA1_NONE:
 689                         break;
 690                 case ID_AA64ISAR0_SHA1_BASE:
 691                         sbuf_printf(sb, "%sSHA1", SEP_STR);
 692                         break;
 693                 default:
 694                         sbuf_printf(sb, "%sUnknown SHA1", SEP_STR);
 695                         break;
 696                 }
 697
 698                 switch (ID_AA64ISAR0_AES(cpu_desc[cpu].id_aa64isar0)) {
 699                 case ID_AA64ISAR0_AES_NONE:
 700                         break;
 701                 case ID_AA64ISAR0_AES_BASE:
 702                         sbuf_printf(sb, "%sAES", SEP_STR);
 703                         break;
 704                 case ID_AA64ISAR0_AES_PMULL:
 705                         sbuf_printf(sb, "%sAES+PMULL", SEP_STR);
 706                         break;
 707                 default:
 708                         sbuf_printf(sb, "%sUnknown AES", SEP_STR);
 709                         break;
 710                 }
 711
 712                 if ((cpu_desc[cpu].id_aa64isar0 & ~ID_AA64ISAR0_MASK) != 0)
 713                         sbuf_printf(sb, "%s%#lx", SEP_STR,
 714                             cpu_desc[cpu].id_aa64isar0 & ~ID_AA64ISAR0_MASK);
 715
 716                 sbuf_finish(sb);
 717                 printf("%s>\n", sbuf_data(sb));
 718                 sbuf_clear(sb);
 719         }
 720
 721         /* AArch64 Instruction Set Attribute Register 1 */
 722         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_ISAR1) != 0) {
 723                 printed = 0;
 724                 sbuf_printf(sb, " Instruction Set Attributes 1 = <");
 725
 726                 switch (ID_AA64ISAR1_GPI(cpu_desc[cpu].id_aa64isar1)) {
 727                 case ID_AA64ISAR1_GPI_NONE:
 728                         break;
 729                 case ID_AA64ISAR1_GPI_IMPL:
 730                         sbuf_printf(sb, "%sImpl GenericAuth", SEP_STR);
 731                         break;
 732                 default:
 733                         sbuf_printf(sb, "%sUnknown GenericAuth", SEP_STR);
 734                         break;
 735                 }
 736
 737                 switch (ID_AA64ISAR1_GPA(cpu_desc[cpu].id_aa64isar1)) {
 738                 case ID_AA64ISAR1_GPA_NONE:
 739                         break;
 740                 case ID_AA64ISAR1_GPA_IMPL:
 741                         sbuf_printf(sb, "%sPrince GenericAuth", SEP_STR);
 742                         break;
 743                 default:
 744                         sbuf_printf(sb, "%sUnknown GenericAuth", SEP_STR);
 745                         break;
 746                 }
 747
 748                 switch (ID_AA64ISAR1_LRCPC(cpu_desc[cpu].id_aa64isar1)) {
 749                 case ID_AA64ISAR1_LRCPC_NONE:
 750                         break;
 751                 case ID_AA64ISAR1_LRCPC_IMPL:
 752                         sbuf_printf(sb, "%sRCpc", SEP_STR);
 753                         break;
 754                 default:
 755                         sbuf_printf(sb, "%sUnknown RCpc", SEP_STR);
 756                         break;
 757                 }
 758
 759                 switch (ID_AA64ISAR1_FCMA(cpu_desc[cpu].id_aa64isar1)) {
 760                 case ID_AA64ISAR1_FCMA_NONE:
 761                         break;
 762                 case ID_AA64ISAR1_FCMA_IMPL:
 763                         sbuf_printf(sb, "%sFCMA", SEP_STR);
 764                         break;
 765                 default:
 766                         sbuf_printf(sb, "%sUnknown FCMA", SEP_STR);
 767                         break;
 768                 }
 769
 770                 switch (ID_AA64ISAR1_JSCVT(cpu_desc[cpu].id_aa64isar1)) {
 771                 case ID_AA64ISAR1_JSCVT_NONE:
 772                         break;
 773                 case ID_AA64ISAR1_JSCVT_IMPL:
 774                         sbuf_printf(sb, "%sJS Conv", SEP_STR);
 775                         break;
 776                 default:
 777                         sbuf_printf(sb, "%sUnknown JS Conv", SEP_STR);
 778                         break;
 779                 }
 780
 781                 switch (ID_AA64ISAR1_API(cpu_desc[cpu].id_aa64isar1)) {
 782                 case ID_AA64ISAR1_API_NONE:
 783                         break;
 784                 case ID_AA64ISAR1_API_IMPL:
 785                         sbuf_printf(sb, "%sImpl AddrAuth", SEP_STR);
 786                         break;
 787                 default:
 788                         sbuf_printf(sb, "%sUnknown Impl AddrAuth", SEP_STR);
 789                         break;
 790                 }
 791
 792                 switch (ID_AA64ISAR1_APA(cpu_desc[cpu].id_aa64isar1)) {
 793                 case ID_AA64ISAR1_APA_NONE:
 794                         break;
 795                 case ID_AA64ISAR1_APA_IMPL:
 796                         sbuf_printf(sb, "%sPrince AddrAuth", SEP_STR);
 797                         break;
 798                 default:
 799                         sbuf_printf(sb, "%sUnknown Prince AddrAuth", SEP_STR);
 800                         break;
 801                 }
 802
 803                 switch (ID_AA64ISAR1_DPB(cpu_desc[cpu].id_aa64isar1)) {
 804                 case ID_AA64ISAR1_DPB_NONE:
 805                         break;
 806                 case ID_AA64ISAR1_DPB_IMPL:
 807                         sbuf_printf(sb, "%sDC CVAP", SEP_STR);
 808                         break;
 809                 default:
 810                         sbuf_printf(sb, "%sUnknown DC CVAP", SEP_STR);
 811                         break;
 812                 }
 813
 814                 if ((cpu_desc[cpu].id_aa64isar1 & ~ID_AA64ISAR1_MASK) != 0)
 815                         sbuf_printf(sb, "%s%#lx", SEP_STR,
 816                             cpu_desc[cpu].id_aa64isar1 & ~ID_AA64ISAR1_MASK);
 817                 sbuf_finish(sb);
 818                 printf("%s>\n", sbuf_data(sb));
 819                 sbuf_clear(sb);
 820         }
 821
 822         /* AArch64 Processor Feature Register 0 */
 823         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_PFR0) != 0) {
 824                 printed = 0;
 825                 sbuf_printf(sb, "         Processor Features 0 = <");
 826
 827                 switch (ID_AA64PFR0_SVE(cpu_desc[cpu].id_aa64pfr0)) {
 828                 case ID_AA64PFR0_SVE_NONE:
 829                         break;
 830                 case ID_AA64PFR0_SVE_IMPL:
 831                         sbuf_printf(sb, "%sSVE", SEP_STR);
 832                         break;
 833                 default:
 834                         sbuf_printf(sb, "%sUnknown SVE", SEP_STR);
 835                         break;
 836                 }
 837
 838                 switch (ID_AA64PFR0_RAS(cpu_desc[cpu].id_aa64pfr0)) {
 839                 case ID_AA64PFR0_RAS_NONE:
 840                         break;
 841                 case ID_AA64PFR0_RAS_V1:
 842                         sbuf_printf(sb, "%sRASv1", SEP_STR);
 843                         break;
 844                 default:
 845                         sbuf_printf(sb, "%sUnknown RAS", SEP_STR);
 846                         break;
 847                 }
 848
 849                 switch (ID_AA64PFR0_GIC(cpu_desc[cpu].id_aa64pfr0)) {
 850                 case ID_AA64PFR0_GIC_CPUIF_NONE:
 851                         break;
 852                 case ID_AA64PFR0_GIC_CPUIF_EN:
 853                         sbuf_printf(sb, "%sGIC", SEP_STR);
 854                         break;
 855                 default:
 856                         sbuf_printf(sb, "%sUnknown GIC interface", SEP_STR);
 857                         break;
 858                 }
 859
 860                 switch (ID_AA64PFR0_AdvSIMD(cpu_desc[cpu].id_aa64pfr0)) {
 861                 case ID_AA64PFR0_AdvSIMD_NONE:
 862                         break;
 863                 case ID_AA64PFR0_AdvSIMD_IMPL:
 864                         sbuf_printf(sb, "%sAdvSIMD", SEP_STR);
 865                         break;
 866                 case ID_AA64PFR0_AdvSIMD_HP:
 867                         sbuf_printf(sb, "%sAdvSIMD+HP", SEP_STR);
 868                         break;
 869                 default:
 870                         sbuf_printf(sb, "%sUnknown AdvSIMD", SEP_STR);
 871                         break;
 872                 }
 873
 874                 switch (ID_AA64PFR0_FP(cpu_desc[cpu].id_aa64pfr0)) {
 875                 case ID_AA64PFR0_FP_NONE:
 876                         break;
 877                 case ID_AA64PFR0_FP_IMPL:
 878                         sbuf_printf(sb, "%sFloat", SEP_STR);
 879                         break;
 880                 case ID_AA64PFR0_FP_HP:
 881                         sbuf_printf(sb, "%sFloat+HP", SEP_STR);
 882                         break;
 883                 default:
 884                         sbuf_printf(sb, "%sUnknown Float", SEP_STR);
 885                         break;
 886                 }
 887
 888                 switch (ID_AA64PFR0_EL3(cpu_desc[cpu].id_aa64pfr0)) {
 889                 case ID_AA64PFR0_EL3_NONE:
 890                         sbuf_printf(sb, "%sNo EL3", SEP_STR);
 891                         break;
 892                 case ID_AA64PFR0_EL3_64:
 893                         sbuf_printf(sb, "%sEL3", SEP_STR);
 894                         break;
 895                 case ID_AA64PFR0_EL3_64_32:
 896                         sbuf_printf(sb, "%sEL3 32", SEP_STR);
 897                         break;
 898                 default:
 899                         sbuf_printf(sb, "%sUnknown EL3", SEP_STR);
 900                         break;
 901                 }
 902
 903                 switch (ID_AA64PFR0_EL2(cpu_desc[cpu].id_aa64pfr0)) {
 904                 case ID_AA64PFR0_EL2_NONE:
 905                         sbuf_printf(sb, "%sNo EL2", SEP_STR);
 906                         break;
 907                 case ID_AA64PFR0_EL2_64:
 908                         sbuf_printf(sb, "%sEL2", SEP_STR);
 909                         break;
 910                 case ID_AA64PFR0_EL2_64_32:
 911                         sbuf_printf(sb, "%sEL2 32", SEP_STR);
 912                         break;
 913                 default:
 914                         sbuf_printf(sb, "%sUnknown EL2", SEP_STR);
 915                         break;
 916                 }
 917
 918                 switch (ID_AA64PFR0_EL1(cpu_desc[cpu].id_aa64pfr0)) {
 919                 case ID_AA64PFR0_EL1_64:
 920                         sbuf_printf(sb, "%sEL1", SEP_STR);
 921                         break;
 922                 case ID_AA64PFR0_EL1_64_32:
 923                         sbuf_printf(sb, "%sEL1 32", SEP_STR);
 924                         break;
 925                 default:
 926                         sbuf_printf(sb, "%sUnknown EL1", SEP_STR);
 927                         break;
 928                 }
 929
 930                 switch (ID_AA64PFR0_EL0(cpu_desc[cpu].id_aa64pfr0)) {
 931                 case ID_AA64PFR0_EL0_64:
 932                         sbuf_printf(sb, "%sEL0", SEP_STR);
 933                         break;
 934                 case ID_AA64PFR0_EL0_64_32:
 935                         sbuf_printf(sb, "%sEL0 32", SEP_STR);
 936                         break;
 937                 default:
 938                         sbuf_printf(sb, "%sUnknown EL0", SEP_STR);
 939                         break;
 940                 }
 941
 942                 if ((cpu_desc[cpu].id_aa64pfr0 & ~ID_AA64PFR0_MASK) != 0)
 943                         sbuf_printf(sb, "%s%#lx", SEP_STR,
 944                             cpu_desc[cpu].id_aa64pfr0 & ~ID_AA64PFR0_MASK);
 945
 946                 sbuf_finish(sb);
 947                 printf("%s>\n", sbuf_data(sb));
 948                 sbuf_clear(sb);
 949         }
 950
 951         /* AArch64 Processor Feature Register 1 */
 952         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_PFR1) != 0) {
 953                 printf("         Processor Features 1 = <%#lx>\n",
 954                     cpu_desc[cpu].id_aa64pfr1);
 955         }
 956
 957         /* AArch64 Memory Model Feature Register 0 */
 958         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_MMFR0) != 0) {
 959                 printed = 0;
 960                 sbuf_printf(sb, "      Memory Model Features 0 = <");
 961                 switch (ID_AA64MMFR0_TGran4(cpu_desc[cpu].id_aa64mmfr0)) {
 962                 case ID_AA64MMFR0_TGran4_NONE:
 963                         break;
 964                 case ID_AA64MMFR0_TGran4_IMPL:
 965                         sbuf_printf(sb, "%s4k Granule", SEP_STR);
 966                         break;
 967                 default:
 968                         sbuf_printf(sb, "%sUnknown 4k Granule", SEP_STR);
 969                         break;
 970                 }
 971
 972                 switch (ID_AA64MMFR0_TGran64(cpu_desc[cpu].id_aa64mmfr0)) {
 973                 case ID_AA64MMFR0_TGran64_NONE:
 974                         break;
 975                 case ID_AA64MMFR0_TGran64_IMPL:
 976                         sbuf_printf(sb, "%s64k Granule", SEP_STR);
 977                         break;
 978                 default:
 979                         sbuf_printf(sb, "%sUnknown 64k Granule", SEP_STR);
 980                         break;
 981                 }
 982
 983                 switch (ID_AA64MMFR0_TGran16(cpu_desc[cpu].id_aa64mmfr0)) {
 984                 case ID_AA64MMFR0_TGran16_NONE:
 985                         break;
 986                 case ID_AA64MMFR0_TGran16_IMPL:
 987                         sbuf_printf(sb, "%s16k Granule", SEP_STR);
 988                         break;
 989                 default:
 990                         sbuf_printf(sb, "%sUnknown 16k Granule", SEP_STR);
 991                         break;
 992                 }
 993
 994                 switch (ID_AA64MMFR0_BigEndEL0(cpu_desc[cpu].id_aa64mmfr0)) {
 995                 case ID_AA64MMFR0_BigEndEL0_FIXED:
 996                         break;
 997                 case ID_AA64MMFR0_BigEndEL0_MIXED:
 998                         sbuf_printf(sb, "%sEL0 MixEndian", SEP_STR);
 999                         break;
1000                 default:
1001                         sbuf_printf(sb, "%sUnknown EL0 Endian switching", SEP_STR);
1002                         break;
1003                 }
1004
1005                 switch (ID_AA64MMFR0_SNSMem(cpu_desc[cpu].id_aa64mmfr0)) {
1006                 case ID_AA64MMFR0_SNSMem_NONE:
1007                         break;
1008                 case ID_AA64MMFR0_SNSMem_DISTINCT:
1009                         sbuf_printf(sb, "%sS/NS Mem", SEP_STR);
1010                         break;
1011                 default:
1012                         sbuf_printf(sb, "%sUnknown S/NS Mem", SEP_STR);
1013                         break;
1014                 }
1015
1016                 switch (ID_AA64MMFR0_BigEnd(cpu_desc[cpu].id_aa64mmfr0)) {
1017                 case ID_AA64MMFR0_BigEnd_FIXED:
1018                         break;
1019                 case ID_AA64MMFR0_BigEnd_MIXED:
1020                         sbuf_printf(sb, "%sMixedEndian", SEP_STR);
1021                         break;
1022                 default:
1023                         sbuf_printf(sb, "%sUnknown Endian switching", SEP_STR);
1024                         break;
1025                 }
1026
1027                 switch (ID_AA64MMFR0_ASIDBits(cpu_desc[cpu].id_aa64mmfr0)) {
1028                 case ID_AA64MMFR0_ASIDBits_8:
1029                         sbuf_printf(sb, "%s8bit ASID", SEP_STR);
1030                         break;
1031                 case ID_AA64MMFR0_ASIDBits_16:
1032                         sbuf_printf(sb, "%s16bit ASID", SEP_STR);
1033                         break;
1034                 default:
1035                         sbuf_printf(sb, "%sUnknown ASID", SEP_STR);
1036                         break;
1037                 }
1038
1039                 switch (ID_AA64MMFR0_PARange(cpu_desc[cpu].id_aa64mmfr0)) {
1040                 case ID_AA64MMFR0_PARange_4G:
1041                         sbuf_printf(sb, "%s4GB PA", SEP_STR);
1042                         break;
1043                 case ID_AA64MMFR0_PARange_64G:
1044                         sbuf_printf(sb, "%s64GB PA", SEP_STR);
1045                         break;
1046                 case ID_AA64MMFR0_PARange_1T:
1047                         sbuf_printf(sb, "%s1TB PA", SEP_STR);
1048                         break;
1049                 case ID_AA64MMFR0_PARange_4T:
1050                         sbuf_printf(sb, "%s4TB PA", SEP_STR);
1051                         break;
1052                 case ID_AA64MMFR0_PARange_16T:
1053                         sbuf_printf(sb, "%s16TB PA", SEP_STR);
1054                         break;
1055                 case ID_AA64MMFR0_PARange_256T:
1056                         sbuf_printf(sb, "%s256TB PA", SEP_STR);
1057                         break;
1058                 case ID_AA64MMFR0_PARange_4P:
1059                         sbuf_printf(sb, "%s4PB PA", SEP_STR);
1060                         break;
1061                 default:
1062                         sbuf_printf(sb, "%sUnknown PA Range", SEP_STR);
1063                         break;
1064                 }
1065
1066                 if ((cpu_desc[cpu].id_aa64mmfr0 & ~ID_AA64MMFR0_MASK) != 0)
1067                         sbuf_printf(sb, "%s%#lx", SEP_STR,
1068                             cpu_desc[cpu].id_aa64mmfr0 & ~ID_AA64MMFR0_MASK);
1069                 sbuf_finish(sb);
1070                 printf("%s>\n", sbuf_data(sb));
1071                 sbuf_clear(sb);
1072         }
1073
1074         /* AArch64 Memory Model Feature Register 1 */
1075         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_MMFR1) != 0) {
1076                 printed = 0;
1077                 sbuf_printf(sb, "      Memory Model Features 1 = <");
1078
1079                 switch (ID_AA64MMFR1_XNX(cpu_desc[cpu].id_aa64mmfr1)) {
1080                 case ID_AA64MMFR1_XNX_NONE:
1081                         break;
1082                 case ID_AA64MMFR1_XNX_IMPL:
1083                         sbuf_printf(sb, "%sEL2 XN", SEP_STR);
1084                         break;
1085                 default:
1086                         sbuf_printf(sb, "%sUnknown XNX", SEP_STR);
1087                         break;
1088                 }
1089
1090                 switch (ID_AA64MMFR1_SpecSEI(cpu_desc[cpu].id_aa64mmfr1)) {
1091                 case ID_AA64MMFR1_SpecSEI_NONE:
1092                         break;
1093                 case ID_AA64MMFR1_SpecSEI_IMPL:
1094                         sbuf_printf(sb, "%sSpecSEI", SEP_STR);
1095                         break;
1096                 default:
1097                         sbuf_printf(sb, "%sUnknown SpecSEI", SEP_STR);
1098                         break;
1099                 }
1100
1101                 switch (ID_AA64MMFR1_PAN(cpu_desc[cpu].id_aa64mmfr1)) {
1102                 case ID_AA64MMFR1_PAN_NONE:
1103                         break;
1104                 case ID_AA64MMFR1_PAN_IMPL:
1105                         sbuf_printf(sb, "%sPAN", SEP_STR);
1106                         break;
1107                 case ID_AA64MMFR1_PAN_ATS1E1:
1108                         sbuf_printf(sb, "%sPAN+AT", SEP_STR);
1109                         break;
1110                 default:
1111                         sbuf_printf(sb, "%sUnknown PAN", SEP_STR);
1112                         break;
1113                 }
1114
1115                 switch (ID_AA64MMFR1_LO(cpu_desc[cpu].id_aa64mmfr1)) {
1116                 case ID_AA64MMFR1_LO_NONE:
1117                         break;
1118                 case ID_AA64MMFR1_LO_IMPL:
1119                         sbuf_printf(sb, "%sLO", SEP_STR);
1120                         break;
1121                 default:
1122                         sbuf_printf(sb, "%sUnknown LO", SEP_STR);
1123                         break;
1124                 }
1125
1126                 switch (ID_AA64MMFR1_HPDS(cpu_desc[cpu].id_aa64mmfr1)) {
1127                 case ID_AA64MMFR1_HPDS_NONE:
1128                         break;
1129                 case ID_AA64MMFR1_HPDS_HPD:
1130                         sbuf_printf(sb, "%sHPDS", SEP_STR);
1131                         break;
1132                 case ID_AA64MMFR1_HPDS_TTPBHA:
1133                         sbuf_printf(sb, "%sTTPBHA", SEP_STR);
1134                         break;
1135                 default:
1136                         sbuf_printf(sb, "%sUnknown HPDS", SEP_STR);
1137                         break;
1138                 }
1139
1140                 switch (ID_AA64MMFR1_VH(cpu_desc[cpu].id_aa64mmfr1)) {
1141                 case ID_AA64MMFR1_VH_NONE:
1142                         break;
1143                 case ID_AA64MMFR1_VH_IMPL:
1144                         sbuf_printf(sb, "%sVHE", SEP_STR);
1145                         break;
1146                 default:
1147                         sbuf_printf(sb, "%sUnknown VHE", SEP_STR);
1148                         break;
1149                 }
1150
1151                 switch (ID_AA64MMFR1_VMIDBits(cpu_desc[cpu].id_aa64mmfr1)) {
1152                 case ID_AA64MMFR1_VMIDBits_8:
1153                         break;
1154                 case ID_AA64MMFR1_VMIDBits_16:
1155                         sbuf_printf(sb, "%s16 VMID bits", SEP_STR);
1156                         break;
1157                 default:
1158                         sbuf_printf(sb, "%sUnknown VMID bits", SEP_STR);
1159                         break;
1160                 }
1161
1162                 switch (ID_AA64MMFR1_HAFDBS(cpu_desc[cpu].id_aa64mmfr1)) {
1163                 case ID_AA64MMFR1_HAFDBS_NONE:
1164                         break;
1165                 case ID_AA64MMFR1_HAFDBS_AF:
1166                         sbuf_printf(sb, "%sAF", SEP_STR);
1167                         break;
1168                 case ID_AA64MMFR1_HAFDBS_AF_DBS:
1169                         sbuf_printf(sb, "%sAF+DBS", SEP_STR);
1170                         break;
1171                 default:
1172                         sbuf_printf(sb, "%sUnknown Hardware update AF/DBS", SEP_STR);
1173                         break;
1174                 }
1175
1176                 if ((cpu_desc[cpu].id_aa64mmfr1 & ~ID_AA64MMFR1_MASK) != 0)
1177                         sbuf_printf(sb, "%s%#lx", SEP_STR,
1178                             cpu_desc[cpu].id_aa64mmfr1 & ~ID_AA64MMFR1_MASK);
1179                 sbuf_finish(sb);
1180                 printf("%s>\n", sbuf_data(sb));
1181                 sbuf_clear(sb);
1182         }
1183
1184         /* AArch64 Memory Model Feature Register 2 */
1185         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_MMFR2) != 0) {
1186                 printed = 0;
1187                 sbuf_printf(sb, "      Memory Model Features 2 = <");
1188
1189                 switch (ID_AA64MMFR2_NV(cpu_desc[cpu].id_aa64mmfr2)) {
1190                 case ID_AA64MMFR2_NV_NONE:
1191                         break;
1192                 case ID_AA64MMFR2_NV_IMPL:
1193                         sbuf_printf(sb, "%sNestedVirt", SEP_STR);
1194                         break;
1195                 default:
1196                         sbuf_printf(sb, "%sUnknown NestedVirt", SEP_STR);
1197                         break;
1198                 }
1199
1200                 switch (ID_AA64MMFR2_CCIDX(cpu_desc[cpu].id_aa64mmfr2)) {
1201                 case ID_AA64MMFR2_CCIDX_32:
1202                         sbuf_printf(sb, "%s32b CCIDX", SEP_STR);
1203                         break;
1204                 case ID_AA64MMFR2_CCIDX_64:
1205                         sbuf_printf(sb, "%s64b CCIDX", SEP_STR);
1206                         break;
1207                 default:
1208                         sbuf_printf(sb, "%sUnknown CCIDX", SEP_STR);
1209                         break;
1210                 }
1211
1212                 switch (ID_AA64MMFR2_VARange(cpu_desc[cpu].id_aa64mmfr2)) {
1213                 case ID_AA64MMFR2_VARange_48:
1214                         sbuf_printf(sb, "%s48b VA", SEP_STR);
1215                         break;
1216                 case ID_AA64MMFR2_VARange_52:
1217                         sbuf_printf(sb, "%s52b VA", SEP_STR);
1218                         break;
1219                 default:
1220                         sbuf_printf(sb, "%sUnknown VA Range", SEP_STR);
1221                         break;
1222                 }
1223
1224                 switch (ID_AA64MMFR2_IESB(cpu_desc[cpu].id_aa64mmfr2)) {
1225                 case ID_AA64MMFR2_IESB_NONE:
1226                         break;
1227                 case ID_AA64MMFR2_IESB_IMPL:
1228                         sbuf_printf(sb, "%sIESB", SEP_STR);
1229                         break;
1230                 default:
1231                         sbuf_printf(sb, "%sUnknown IESB", SEP_STR);
1232                         break;
1233                 }
1234
1235                 switch (ID_AA64MMFR2_LSM(cpu_desc[cpu].id_aa64mmfr2)) {
1236                 case ID_AA64MMFR2_LSM_NONE:
1237                         break;
1238                 case ID_AA64MMFR2_LSM_IMPL:
1239                         sbuf_printf(sb, "%sLSM", SEP_STR);
1240                         break;
1241                 default:
1242                         sbuf_printf(sb, "%sUnknown LSM", SEP_STR);
1243                         break;
1244                 }
1245
1246                 switch (ID_AA64MMFR2_UAO(cpu_desc[cpu].id_aa64mmfr2)) {
1247                 case ID_AA64MMFR2_UAO_NONE:
1248                         break;
1249                 case ID_AA64MMFR2_UAO_IMPL:
1250                         sbuf_printf(sb, "%sUAO", SEP_STR);
1251                         break;
1252                 default:
1253                         sbuf_printf(sb, "%sUnknown UAO", SEP_STR);
1254                         break;
1255                 }
1256
1257                 switch (ID_AA64MMFR2_CnP(cpu_desc[cpu].id_aa64mmfr2)) {
1258                 case ID_AA64MMFR2_CnP_NONE:
1259                         break;
1260                 case ID_AA64MMFR2_CnP_IMPL:
1261                         sbuf_printf(sb, "%sCnP", SEP_STR);
1262                         break;
1263                 default:
1264                         sbuf_printf(sb, "%sUnknown CnP", SEP_STR);
1265                         break;
1266                 }
1267
1268                 if ((cpu_desc[cpu].id_aa64mmfr2 & ~ID_AA64MMFR2_MASK) != 0)
1269                         sbuf_printf(sb, "%s%#lx", SEP_STR,
1270                             cpu_desc[cpu].id_aa64mmfr2 & ~ID_AA64MMFR2_MASK);
1271                 sbuf_finish(sb);
1272                 printf("%s>\n", sbuf_data(sb));
1273                 sbuf_clear(sb);
1274         }
1275
1276         /* AArch64 Debug Feature Register 0 */
1277         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_DFR0) != 0) {
1278                 printed = 0;
1279                 sbuf_printf(sb, "             Debug Features 0 = <");
1280                 switch(ID_AA64DFR0_PMSVer(cpu_desc[cpu].id_aa64dfr0)) {
1281                 case ID_AA64DFR0_PMSVer_NONE:
1282                         break;
1283                 case ID_AA64DFR0_PMSVer_V1:
1284                         sbuf_printf(sb, "%sSPE v1", SEP_STR);
1285                         break;
1286                 default:
1287                         sbuf_printf(sb, "%sUnknown SPE", SEP_STR);
1288                         break;
1289                 }
1290
1291                 sbuf_printf(sb, "%s%lu CTX Breakpoints", SEP_STR,
1292                     ID_AA64DFR0_CTX_CMPs(cpu_desc[cpu].id_aa64dfr0));
1293
1294                 sbuf_printf(sb, "%s%lu Watchpoints", SEP_STR,
1295                     ID_AA64DFR0_WRPs(cpu_desc[cpu].id_aa64dfr0));
1296
1297                 sbuf_printf(sb, "%s%lu Breakpoints", SEP_STR,
1298                     ID_AA64DFR0_BRPs(cpu_desc[cpu].id_aa64dfr0));
1299
1300                 switch (ID_AA64DFR0_PMUVer(cpu_desc[cpu].id_aa64dfr0)) {
1301                 case ID_AA64DFR0_PMUVer_NONE:
1302                         break;
1303                 case ID_AA64DFR0_PMUVer_3:
1304                         sbuf_printf(sb, "%sPMUv3", SEP_STR);
1305                         break;
1306                 case ID_AA64DFR0_PMUVer_3_1:
1307                         sbuf_printf(sb, "%sPMUv3+16 bit evtCount", SEP_STR);
1308                         break;
1309                 case ID_AA64DFR0_PMUVer_IMPL:
1310                         sbuf_printf(sb, "%sImplementation defined PMU", SEP_STR);
1311                         break;
1312                 default:
1313                         sbuf_printf(sb, "%sUnknown PMU", SEP_STR);
1314                         break;
1315                 }
1316
1317                 switch (ID_AA64DFR0_TraceVer(cpu_desc[cpu].id_aa64dfr0)) {
1318                 case ID_AA64DFR0_TraceVer_NONE:
1319                         break;
1320                 case ID_AA64DFR0_TraceVer_IMPL:
1321                         sbuf_printf(sb, "%sTrace", SEP_STR);
1322                         break;
1323                 default:
1324                         sbuf_printf(sb, "%sUnknown Trace", SEP_STR);
1325                         break;
1326                 }
1327
1328                 switch (ID_AA64DFR0_DebugVer(cpu_desc[cpu].id_aa64dfr0)) {
1329                 case ID_AA64DFR0_DebugVer_8:
1330                         sbuf_printf(sb, "%sDebug v8", SEP_STR);
1331                         break;
1332                 case ID_AA64DFR0_DebugVer_8_VHE:
1333                         sbuf_printf(sb, "%sDebug v8+VHE", SEP_STR);
1334                         break;
1335                 case ID_AA64DFR0_DebugVer_8_2:
1336                         sbuf_printf(sb, "%sDebug v8.2", SEP_STR);
1337                         break;
1338                 default:
1339                         sbuf_printf(sb, "%sUnknown Debug", SEP_STR);
1340                         break;
1341                 }
1342
1343                 if (cpu_desc[cpu].id_aa64dfr0 & ~ID_AA64DFR0_MASK)
1344                         sbuf_printf(sb, "%s%#lx", SEP_STR,
1345                             cpu_desc[cpu].id_aa64dfr0 & ~ID_AA64DFR0_MASK);
1346                 sbuf_finish(sb);
1347                 printf("%s>\n", sbuf_data(sb));
1348                 sbuf_clear(sb);
1349         }
1350
1351         /* AArch64 Memory Model Feature Register 1 */
1352         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_DFR1) != 0) {
1353                 printf("             Debug Features 1 = <%#lx>\n",
1354                     cpu_desc[cpu].id_aa64dfr1);
1355         }
1356
1357         /* AArch64 Auxiliary Feature Register 0 */
1358         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_AFR0) != 0) {
1359                 printf("         Auxiliary Features 0 = <%#lx>\n",
1360                     cpu_desc[cpu].id_aa64afr0);
1361         }
1362
1363         /* AArch64 Auxiliary Feature Register 1 */
1364         if (cpu == 0 || (cpu_print_regs & PRINT_ID_AA64_AFR1) != 0) {
1365                 printf("         Auxiliary Features 1 = <%#lx>\n",
1366                     cpu_desc[cpu].id_aa64afr1);
1367         }
1368
1369         sbuf_delete(sb);
1370         sb = NULL;
1371 #undef SEP_STR
1372 }
1373
1374 void
1375 identify_cpu(void)
1376 {
1377         u_int midr;
1378         u_int impl_id;
1379         u_int part_id;
1380         u_int cpu;
1381         size_t i;
1382         const struct cpu_parts *cpu_partsp = NULL;
1383
1384         cpu = PCPU_GET(cpuid);
1385         midr = get_midr();
1386
1387         /*
1388          * Store midr to pcpu to allow fast reading
1389          * from EL0, EL1 and assembly code.
1390          */
1391         PCPU_SET(midr, midr);
1392
1393         impl_id = CPU_IMPL(midr);
1394         for (i = 0; i < nitems(cpu_implementers); i++) {
1395                 if (impl_id == cpu_implementers[i].impl_id ||
1396                     cpu_implementers[i].impl_id == 0) {
1397                         cpu_desc[cpu].cpu_impl = impl_id;
1398                         cpu_desc[cpu].cpu_impl_name = cpu_implementers[i].impl_name;
1399                         cpu_partsp = cpu_implementers[i].cpu_parts;
1400                         break;
1401                 }
1402         }
1403
1404         part_id = CPU_PART(midr);
1405         for (i = 0; &cpu_partsp[i] != NULL; i++) {
1406                 if (part_id == cpu_partsp[i].part_id ||
1407                     cpu_partsp[i].part_id == 0) {
1408                         cpu_desc[cpu].cpu_part_num = part_id;
1409                         cpu_desc[cpu].cpu_part_name = cpu_partsp[i].part_name;
1410                         break;
1411                 }
1412         }
1413
1414         cpu_desc[cpu].cpu_revision = CPU_REV(midr);
1415         cpu_desc[cpu].cpu_variant = CPU_VAR(midr);
1416
1417         snprintf(cpu_model, sizeof(cpu_model), "%s %s r%dp%d",
1418             cpu_desc[cpu].cpu_impl_name, cpu_desc[cpu].cpu_part_name,
1419             cpu_desc[cpu].cpu_variant, cpu_desc[cpu].cpu_revision);
1420
1421         /* Save affinity for current CPU */
1422         cpu_desc[cpu].mpidr = get_mpidr();
1423         CPU_AFFINITY(cpu) = cpu_desc[cpu].mpidr & CPU_AFF_MASK;
1424
1425         cpu_desc[cpu].id_aa64dfr0 = READ_SPECIALREG(ID_AA64DFR0_EL1);
1426         cpu_desc[cpu].id_aa64dfr1 = READ_SPECIALREG(ID_AA64DFR1_EL1);
1427         cpu_desc[cpu].id_aa64isar0 = READ_SPECIALREG(ID_AA64ISAR0_EL1);
1428         cpu_desc[cpu].id_aa64isar1 = READ_SPECIALREG(ID_AA64ISAR1_EL1);
1429         cpu_desc[cpu].id_aa64mmfr0 = READ_SPECIALREG(ID_AA64MMFR0_EL1);
1430         cpu_desc[cpu].id_aa64mmfr1 = READ_SPECIALREG(ID_AA64MMFR1_EL1);
1431         cpu_desc[cpu].id_aa64mmfr2 = READ_SPECIALREG(ID_AA64MMFR2_EL1);
1432         cpu_desc[cpu].id_aa64pfr0 = READ_SPECIALREG(ID_AA64PFR0_EL1);
1433         cpu_desc[cpu].id_aa64pfr1 = READ_SPECIALREG(ID_AA64PFR1_EL1);
1434
1435         if (cpu != 0) {
1436                 /*
1437                  * This code must run on one cpu at a time, but we are
1438                  * not scheduling on the current core so implement a
1439                  * simple spinlock.
1440                  */
1441                 while (atomic_cmpset_acq_int(&ident_lock, 0, 1) == 0)
1442                         __asm __volatile("wfe" ::: "memory");
1443
1444                 switch (cpu_aff_levels) {
1445                 case 0:
1446                         if (CPU_AFF0(cpu_desc[cpu].mpidr) !=
1447                             CPU_AFF0(cpu_desc[0].mpidr))
1448                                 cpu_aff_levels = 1;
1449                         /* FALLTHROUGH */
1450                 case 1:
1451                         if (CPU_AFF1(cpu_desc[cpu].mpidr) !=
1452                             CPU_AFF1(cpu_desc[0].mpidr))
1453                                 cpu_aff_levels = 2;
1454                         /* FALLTHROUGH */
1455                 case 2:
1456                         if (CPU_AFF2(cpu_desc[cpu].mpidr) !=
1457                             CPU_AFF2(cpu_desc[0].mpidr))
1458                                 cpu_aff_levels = 3;
1459                         /* FALLTHROUGH */
1460                 case 3:
1461                         if (CPU_AFF3(cpu_desc[cpu].mpidr) !=
1462                             CPU_AFF3(cpu_desc[0].mpidr))
1463                                 cpu_aff_levels = 4;
1464                         break;
1465                 }
1466
1467                 if (cpu_desc[cpu].id_aa64afr0 != cpu_desc[0].id_aa64afr0)
1468                         cpu_print_regs |= PRINT_ID_AA64_AFR0;
1469                 if (cpu_desc[cpu].id_aa64afr1 != cpu_desc[0].id_aa64afr1)
1470                         cpu_print_regs |= PRINT_ID_AA64_AFR1;
1471
1472                 if (cpu_desc[cpu].id_aa64dfr0 != cpu_desc[0].id_aa64dfr0)
1473                         cpu_print_regs |= PRINT_ID_AA64_DFR0;
1474                 if (cpu_desc[cpu].id_aa64dfr1 != cpu_desc[0].id_aa64dfr1)
1475                         cpu_print_regs |= PRINT_ID_AA64_DFR1;
1476
1477                 if (cpu_desc[cpu].id_aa64isar0 != cpu_desc[0].id_aa64isar0)
1478                         cpu_print_regs |= PRINT_ID_AA64_ISAR0;
1479                 if (cpu_desc[cpu].id_aa64isar1 != cpu_desc[0].id_aa64isar1)
1480                         cpu_print_regs |= PRINT_ID_AA64_ISAR1;
1481
1482                 if (cpu_desc[cpu].id_aa64mmfr0 != cpu_desc[0].id_aa64mmfr0)
1483                         cpu_print_regs |= PRINT_ID_AA64_MMFR0;
1484                 if (cpu_desc[cpu].id_aa64mmfr1 != cpu_desc[0].id_aa64mmfr1)
1485                         cpu_print_regs |= PRINT_ID_AA64_MMFR1;
1486                 if (cpu_desc[cpu].id_aa64mmfr2 != cpu_desc[0].id_aa64mmfr2)
1487                         cpu_print_regs |= PRINT_ID_AA64_MMFR2;
1488
1489                 if (cpu_desc[cpu].id_aa64pfr0 != cpu_desc[0].id_aa64pfr0)
1490                         cpu_print_regs |= PRINT_ID_AA64_PFR0;
1491                 if (cpu_desc[cpu].id_aa64pfr1 != cpu_desc[0].id_aa64pfr1)
1492                         cpu_print_regs |= PRINT_ID_AA64_PFR1;
1493
1494                 /* Wake up the other CPUs */
1495                 atomic_store_rel_int(&ident_lock, 0);
1496                 __asm __volatile("sev" ::: "memory");
1497         }
1498 }