sys/arm64/arm64/cpu_errata.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause
   3  *
   4  * Copyright (c) 2018 Andrew Turner
   5  * All rights reserved.
   6  *
   7  * This software was developed by SRI International and the University of
   8  * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237
   9  * ("CTSRD"), as part of the DARPA CRASH research programme.
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in the
  18  *    documentation and/or other materials provided with the distribution.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  30  * SUCH DAMAGE.
  31  */
  32
  33 #include "opt_platform.h"
  34
  35 #include <sys/param.h>
  36 #include <sys/kernel.h>
  37 #include <sys/pcpu.h>
  38 #include <sys/systm.h>
  39
  40 #include <machine/cpu.h>
  41
  42 #include <dev/psci/psci.h>
  43 #include <dev/psci/smccc.h>
  44
  45 typedef void (cpu_quirk_install)(void);
  46 struct cpu_quirks {
  47         cpu_quirk_install *quirk_install;
  48         u_int           midr_mask;
  49         u_int           midr_value;
  50 #define CPU_QUIRK_POST_DEVICE   (1 << 0)        /* After device attach */
  51                                                 /* e.g. needs SMCCC */
  52         u_int           flags;
  53 };
  54
  55 static enum {
  56         SSBD_FORCE_ON,
  57         SSBD_FORCE_OFF,
  58         SSBD_KERNEL,
  59 } ssbd_method = SSBD_KERNEL;
  60
  61 static cpu_quirk_install install_psci_bp_hardening;
  62 static cpu_quirk_install install_ssbd_workaround;
  63 static cpu_quirk_install install_thunderx_bcast_tlbi_workaround;
  64
  65 static struct cpu_quirks cpu_quirks[] = {
  66         {
  67                 .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK,
  68                 .midr_value = CPU_ID_RAW(CPU_IMPL_ARM, CPU_PART_CORTEX_A57,0,0),
  69                 .quirk_install = install_psci_bp_hardening,
  70                 .flags = CPU_QUIRK_POST_DEVICE,
  71         },
  72         {
  73                 .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK,
  74                 .midr_value = CPU_ID_RAW(CPU_IMPL_ARM, CPU_PART_CORTEX_A72,0,0),
  75                 .quirk_install = install_psci_bp_hardening,
  76                 .flags = CPU_QUIRK_POST_DEVICE,
  77         },
  78         {
  79                 .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK,
  80                 .midr_value = CPU_ID_RAW(CPU_IMPL_ARM, CPU_PART_CORTEX_A73,0,0),
  81                 .quirk_install = install_psci_bp_hardening,
  82                 .flags = CPU_QUIRK_POST_DEVICE,
  83         },
  84         {
  85                 .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK,
  86                 .midr_value = CPU_ID_RAW(CPU_IMPL_ARM, CPU_PART_CORTEX_A75,0,0),
  87                 .quirk_install = install_psci_bp_hardening,
  88                 .flags = CPU_QUIRK_POST_DEVICE,
  89         },
  90         {
  91                 .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK,
  92                 .midr_value =
  93                     CPU_ID_RAW(CPU_IMPL_CAVIUM, CPU_PART_THUNDERX2, 0,0),
  94                 .quirk_install = install_psci_bp_hardening,
  95                 .flags = CPU_QUIRK_POST_DEVICE,
  96         },
  97         {
  98                 .midr_mask = 0,
  99                 .midr_value = 0,
 100                 .quirk_install = install_ssbd_workaround,
 101                 .flags = CPU_QUIRK_POST_DEVICE,
 102         },
 103         {
 104                 .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK,
 105                 .midr_value =
 106                     CPU_ID_RAW(CPU_IMPL_CAVIUM, CPU_PART_THUNDERX, 0, 0),
 107                 .quirk_install = install_thunderx_bcast_tlbi_workaround,
 108         },
 109         {
 110                 .midr_mask = CPU_IMPL_MASK | CPU_PART_MASK,
 111                 .midr_value =
 112                     CPU_ID_RAW(CPU_IMPL_CAVIUM, CPU_PART_THUNDERX_81XX, 0, 0),
 113                 .quirk_install = install_thunderx_bcast_tlbi_workaround,
 114         },
 115 };
 116
 117 static void
 118 install_psci_bp_hardening(void)
 119 {
 120         /* SMCCC depends on PSCI. If PSCI is missing so is SMCCC */
 121         if (!psci_present)
 122                 return;
 123
 124         if (smccc_arch_features(SMCCC_ARCH_WORKAROUND_1) != SMCCC_RET_SUCCESS)
 125                 return;
 126
 127         PCPU_SET(bp_harden, smccc_arch_workaround_1);
 128 }
 129
 130 static void
 131 install_ssbd_workaround(void)
 132 {
 133         char *env;
 134
 135         if (PCPU_GET(cpuid) == 0) {
 136                 env = kern_getenv("kern.cfg.ssbd");
 137                 if (env != NULL) {
 138                         if (strcmp(env, "force-on") == 0) {
 139                                 ssbd_method = SSBD_FORCE_ON;
 140                         } else if (strcmp(env, "force-off") == 0) {
 141                                 ssbd_method = SSBD_FORCE_OFF;
 142                         }
 143                 }
 144         }
 145
 146         /* SMCCC depends on PSCI. If PSCI is missing so is SMCCC */
 147         if (!psci_present)
 148                 return;
 149
 150         /* Enable the workaround on this CPU if it's enabled in the firmware */
 151         if (smccc_arch_features(SMCCC_ARCH_WORKAROUND_2) != SMCCC_RET_SUCCESS)
 152                 return;
 153
 154         switch(ssbd_method) {
 155         case SSBD_FORCE_ON:
 156                 smccc_arch_workaround_2(1);
 157                 break;
 158         case SSBD_FORCE_OFF:
 159                 smccc_arch_workaround_2(0);
 160                 break;
 161         case SSBD_KERNEL:
 162         default:
 163                 PCPU_SET(ssbd, smccc_arch_workaround_2);
 164                 break;
 165         }
 166 }
 167
 168 /*
 169  * Workaround Cavium erratum 27456.
 170  *
 171  * Invalidate the local icache when changing address spaces.
 172  */
 173 static void
 174 install_thunderx_bcast_tlbi_workaround(void)
 175 {
 176         u_int midr;
 177
 178         midr = get_midr();
 179         if (CPU_PART(midr) == CPU_PART_THUNDERX_81XX)
 180                 PCPU_SET(bcast_tlbi_workaround, 1);
 181         else if (CPU_PART(midr) == CPU_PART_THUNDERX) {
 182                 if (CPU_VAR(midr) == 0) {
 183                         /* ThunderX 1.x */
 184                         PCPU_SET(bcast_tlbi_workaround, 1);
 185                 } else if (CPU_VAR(midr) == 1 && CPU_REV(midr) <= 1) {
 186                         /* ThunderX 2.0 - 2.1 */
 187                         PCPU_SET(bcast_tlbi_workaround, 1);
 188                 }
 189         }
 190 }
 191
 192 static void
 193 install_cpu_errata_flags(u_int mask, u_int flags)
 194 {
 195         u_int midr;
 196         size_t i;
 197
 198         midr = get_midr();
 199
 200         for (i = 0; i < nitems(cpu_quirks); i++) {
 201                 if ((midr & cpu_quirks[i].midr_mask) ==
 202                     cpu_quirks[i].midr_value &&
 203                     (cpu_quirks[i].flags & mask) == flags) {
 204                         cpu_quirks[i].quirk_install();
 205                 }
 206         }
 207 }
 208
 209 /*
 210  * Install any CPU errata we need. On CPU 0 we only install the errata that
 211  * don't depend on device drivers as this is called early in the boot process.
 212  * On other CPUs the device drivers have already attached so install all
 213  * applicable errata.
 214  */
 215 void
 216 install_cpu_errata(void)
 217 {
 218         /*
 219          * Only install early CPU errata on CPU 0, device drivers may not
 220          * have attached and some workarounds depend on them, e.g. to query
 221          * SMCCC.
 222          */
 223         if (PCPU_GET(cpuid) == 0) {
 224                 install_cpu_errata_flags(CPU_QUIRK_POST_DEVICE, 0);
 225         } else {
 226                 install_cpu_errata_flags(0, 0);
 227         }
 228 }
 229
 230 /*
 231  * Install any errata workarounds that depend on device drivers, e.g. use
 232  * SMCCC to install a workaround.
 233  */
 234 static void
 235 install_cpu_errata_late(void *dummy __unused)
 236 {
 237         MPASS(PCPU_GET(cpuid) == 0);
 238         install_cpu_errata_flags(CPU_QUIRK_POST_DEVICE, CPU_QUIRK_POST_DEVICE);
 239 }
 240 SYSINIT(install_cpu_errata_late, SI_SUB_CONFIGURE, SI_ORDER_MIDDLE,
 241     install_cpu_errata_late, NULL);