sys/arm/arm/pl310.c

   1 /*-
   2  * Copyright (c) 2012 Olivier Houchard <cognet@FreeBSD.org>
   3  * Copyright (c) 2011
   4  *      Ben Gray <ben.r.gray@gmail.com>.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  * 3. The name of the company nor the name of the author may be used to
  16  *    endorse or promote products derived from this software without specific
  17  *    prior written permission.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY BEN GRAY ``AS IS'' AND ANY EXPRESS OR
  20  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  21  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  22  * IN NO EVENT SHALL BEN GRAY BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  24  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  25  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  26  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  27  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  28  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33 #include <sys/param.h>
  34 #include <sys/systm.h>
  35 #include <sys/bus.h>
  36 #include <sys/kernel.h>
  37 #include <sys/rman.h>
  38 #include <sys/module.h>
  39 #include <sys/lock.h>
  40 #include <sys/mutex.h>
  41 #include <machine/intr.h>
  42
  43 #include <machine/bus.h>
  44 #include <machine/pl310.h>
  45
  46 #include <dev/fdt/fdt_common.h>
  47 #include <dev/ofw/openfirm.h>
  48 #include <dev/ofw/ofw_bus.h>
  49 #include <dev/ofw/ofw_bus_subr.h>
  50
  51 /*
  52  * Define this if you need to disable PL310 for debugging purpose
  53  * Spec:
  54  * http://infocenter.arm.com/help/topic/com.arm.doc.ddi0246e/DDI0246E_l2c310_r3p1_trm.pdf
  55  */
  56
  57 /*
  58  * Hardcode errata for now
  59  * http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ddi0246b/pr01s02s02.html
  60  */
  61 #define PL310_ERRATA_588369
  62 #define PL310_ERRATA_753970
  63 #define PL310_ERRATA_727915
  64
  65 #define PL310_LOCK(sc) do {             \
  66         mtx_lock_spin(&(sc)->sc_mtx);   \
  67 } while(0);
  68
  69 #define PL310_UNLOCK(sc) do {           \
  70         mtx_unlock_spin(&(sc)->sc_mtx); \
  71 } while(0);
  72
  73 static int pl310_enabled = 1;
  74 TUNABLE_INT("hw.pl310.enabled", &pl310_enabled);
  75
  76 static uint32_t g_l2cache_way_mask;
  77
  78 static const uint32_t g_l2cache_line_size = 32;
  79 static const uint32_t g_l2cache_align_mask = (32 - 1);
  80
  81 static uint32_t g_l2cache_size;
  82 static uint32_t g_way_size;
  83 static uint32_t g_ways_assoc;
  84
  85 static struct pl310_softc *pl310_softc;
  86
  87 void
  88 pl310_print_config(struct pl310_softc *sc)
  89 {
  90         uint32_t aux, prefetch;
  91         const char *dis = "disabled";
  92         const char *ena = "enabled";
  93
  94         aux = pl310_read4(sc, PL310_AUX_CTRL);
  95         prefetch = pl310_read4(sc, PL310_PREFETCH_CTRL);
  96
  97         device_printf(sc->sc_dev, "Early BRESP response: %s\n",
  98                 (aux & AUX_CTRL_EARLY_BRESP) ? ena : dis);
  99         device_printf(sc->sc_dev, "Instruction prefetch: %s\n",
 100                 (aux & AUX_CTRL_INSTR_PREFETCH) ? ena : dis);
 101         device_printf(sc->sc_dev, "Data prefetch: %s\n",
 102                 (aux & AUX_CTRL_DATA_PREFETCH) ? ena : dis);
 103         device_printf(sc->sc_dev, "Non-secure interrupt control: %s\n",
 104                 (aux & AUX_CTRL_NS_INT_CTRL) ? ena : dis);
 105         device_printf(sc->sc_dev, "Non-secure lockdown: %s\n",
 106                 (aux & AUX_CTRL_NS_LOCKDOWN) ? ena : dis);
 107         device_printf(sc->sc_dev, "Share override: %s\n",
 108                 (aux & AUX_CTRL_SHARE_OVERRIDE) ? ena : dis);
 109
 110         device_printf(sc->sc_dev, "Double linefill: %s\n",
 111                 (prefetch & PREFETCH_CTRL_DL) ? ena : dis);
 112         device_printf(sc->sc_dev, "Instruction prefetch: %s\n",
 113                 (prefetch & PREFETCH_CTRL_INSTR_PREFETCH) ? ena : dis);
 114         device_printf(sc->sc_dev, "Data prefetch: %s\n",
 115                 (prefetch & PREFETCH_CTRL_DATA_PREFETCH) ? ena : dis);
 116         device_printf(sc->sc_dev, "Double linefill on WRAP request: %s\n",
 117                 (prefetch & PREFETCH_CTRL_DL_ON_WRAP) ? ena : dis);
 118         device_printf(sc->sc_dev, "Prefetch drop: %s\n",
 119                 (prefetch & PREFETCH_CTRL_PREFETCH_DROP) ? ena : dis);
 120         device_printf(sc->sc_dev, "Incr double Linefill: %s\n",
 121                 (prefetch & PREFETCH_CTRL_INCR_DL) ? ena : dis);
 122         device_printf(sc->sc_dev, "Not same ID on exclusive sequence: %s\n",
 123                 (prefetch & PREFETCH_CTRL_NOTSAMEID) ? ena : dis);
 124         device_printf(sc->sc_dev, "Prefetch offset: %d\n",
 125                 (prefetch & PREFETCH_CTRL_OFFSET_MASK));
 126 }
 127
 128 void
 129 pl310_set_ram_latency(struct pl310_softc *sc, uint32_t which_reg,
 130    uint32_t read, uint32_t write, uint32_t setup)
 131 {
 132         uint32_t v;
 133
 134         KASSERT(which_reg == PL310_TAG_RAM_CTRL ||
 135             which_reg == PL310_DATA_RAM_CTRL,
 136             ("bad pl310 ram latency register address"));
 137
 138         v = pl310_read4(sc, which_reg);
 139         if (setup != 0) {
 140                 KASSERT(setup <= 8, ("bad pl310 setup latency: %d", setup));
 141                 v &= ~RAM_CTRL_SETUP_MASK;
 142                 v |= (setup - 1) << RAM_CTRL_SETUP_SHIFT;
 143         }
 144         if (read != 0) {
 145                 KASSERT(read <= 8, ("bad pl310 read latency: %d", read));
 146                 v &= ~RAM_CTRL_READ_MASK;
 147                 v |= (read - 1) << RAM_CTRL_READ_SHIFT;
 148         }
 149         if (write != 0) {
 150                 KASSERT(write <= 8, ("bad pl310 write latency: %d", write));
 151                 v &= ~RAM_CTRL_WRITE_MASK;
 152                 v |= (write - 1) << RAM_CTRL_WRITE_SHIFT;
 153         }
 154         pl310_write4(sc, which_reg, v);
 155 }
 156
 157 static int
 158 pl310_filter(void *arg)
 159 {
 160         struct pl310_softc *sc = arg;
 161         uint32_t intr;
 162
 163         intr = pl310_read4(sc, PL310_INTR_MASK);
 164
 165         if (!sc->sc_enabled && (intr & INTR_MASK_ECNTR)) {
 166                 /*
 167                  * This is for debug purpose, so be blunt about it
 168                  * We disable PL310 only when something fishy is going
 169                  * on and we need to make sure L2 cache is 100% disabled
 170                  */
 171                 panic("pl310: caches disabled but cache event detected\n");
 172         }
 173
 174         return (FILTER_HANDLED);
 175 }
 176
 177 static __inline void
 178 pl310_wait_background_op(uint32_t off, uint32_t mask)
 179 {
 180
 181         while (pl310_read4(pl310_softc, off) & mask)
 182                 continue;
 183 }
 184
 185
 186 /**
 187  *      pl310_cache_sync - performs a cache sync operation
 188  *
 189  *      According to the TRM:
 190  *
 191  *  "Before writing to any other register you must perform an explicit
 192  *   Cache Sync operation. This is particularly important when the cache is
 193  *   enabled and changes to how the cache allocates new lines are to be made."
 194  *
 195  *
 196  */
 197 static __inline void
 198 pl310_cache_sync(void)
 199 {
 200
 201         if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
 202                 return;
 203
 204 #ifdef PL310_ERRATA_753970
 205         if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r3p0)
 206                 /* Write uncached PL310 register */
 207                 pl310_write4(pl310_softc, 0x740, 0xffffffff);
 208         else
 209 #endif
 210                 pl310_write4(pl310_softc, PL310_CACHE_SYNC, 0xffffffff);
 211 }
 212
 213
 214 static void
 215 pl310_wbinv_all(void)
 216 {
 217
 218         if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
 219                 return;
 220
 221         PL310_LOCK(pl310_softc);
 222 #ifdef PL310_ERRATA_727915
 223         if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r2p0) {
 224                 int i, j;
 225
 226                 for (i = 0; i < g_ways_assoc; i++) {
 227                         for (j = 0; j < g_way_size / g_l2cache_line_size; j++) {
 228                                 pl310_write4(pl310_softc,
 229                                     PL310_CLEAN_INV_LINE_IDX,
 230                                     (i << 28 | j << 5));
 231                         }
 232                 }
 233                 pl310_cache_sync();
 234                 PL310_UNLOCK(pl310_softc);
 235                 return;
 236
 237         }
 238         if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r3p0)
 239                 platform_pl310_write_debug(pl310_softc, 3);
 240 #endif
 241         pl310_write4(pl310_softc, PL310_CLEAN_INV_WAY, g_l2cache_way_mask);
 242         pl310_wait_background_op(PL310_CLEAN_INV_WAY, g_l2cache_way_mask);
 243         pl310_cache_sync();
 244 #ifdef PL310_ERRATA_727915
 245         if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r3p0)
 246                 platform_pl310_write_debug(pl310_softc, 0);
 247 #endif
 248         PL310_UNLOCK(pl310_softc);
 249 }
 250
 251 static void
 252 pl310_wbinv_range(vm_paddr_t start, vm_size_t size)
 253 {
 254
 255         if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
 256                 return;
 257
 258         PL310_LOCK(pl310_softc);
 259         if (start & g_l2cache_align_mask) {
 260                 size += start & g_l2cache_align_mask;
 261                 start &= ~g_l2cache_align_mask;
 262         }
 263         if (size & g_l2cache_align_mask) {
 264                 size &= ~g_l2cache_align_mask;
 265                 size += g_l2cache_line_size;
 266         }
 267
 268
 269 #ifdef PL310_ERRATA_727915
 270         platform_pl310_write_debug(pl310_softc, 3);
 271 #endif
 272         while (size > 0) {
 273 #ifdef PL310_ERRATA_588369
 274                 if (pl310_softc->sc_rtl_revision <= CACHE_ID_RELEASE_r1p0) {
 275                         /*
 276                          * Errata 588369 says that clean + inv may keep the
 277                          * cache line if it was clean, the recommanded
 278                          * workaround is to clean then invalidate the cache
 279                          * line, with write-back and cache linefill disabled.
 280                          */
 281                         pl310_write4(pl310_softc, PL310_CLEAN_LINE_PA, start);
 282                         pl310_write4(pl310_softc, PL310_INV_LINE_PA, start);
 283                 } else
 284 #endif
 285                         pl310_write4(pl310_softc, PL310_CLEAN_INV_LINE_PA,
 286                             start);
 287                 start += g_l2cache_line_size;
 288                 size -= g_l2cache_line_size;
 289         }
 290 #ifdef PL310_ERRATA_727915
 291         platform_pl310_write_debug(pl310_softc, 0);
 292 #endif
 293
 294         pl310_cache_sync();
 295         PL310_UNLOCK(pl310_softc);
 296 }
 297
 298 static void
 299 pl310_wb_range(vm_paddr_t start, vm_size_t size)
 300 {
 301
 302         if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
 303                 return;
 304
 305         PL310_LOCK(pl310_softc);
 306         if (start & g_l2cache_align_mask) {
 307                 size += start & g_l2cache_align_mask;
 308                 start &= ~g_l2cache_align_mask;
 309         }
 310
 311         if (size & g_l2cache_align_mask) {
 312                 size &= ~g_l2cache_align_mask;
 313                 size += g_l2cache_line_size;
 314         }
 315
 316         while (size > 0) {
 317                 pl310_write4(pl310_softc, PL310_CLEAN_LINE_PA, start);
 318                 start += g_l2cache_line_size;
 319                 size -= g_l2cache_line_size;
 320         }
 321
 322         pl310_cache_sync();
 323         PL310_UNLOCK(pl310_softc);
 324 }
 325
 326 static void
 327 pl310_inv_range(vm_paddr_t start, vm_size_t size)
 328 {
 329
 330         if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
 331                 return;
 332
 333         PL310_LOCK(pl310_softc);
 334         if (start & g_l2cache_align_mask) {
 335                 size += start & g_l2cache_align_mask;
 336                 start &= ~g_l2cache_align_mask;
 337         }
 338         if (size & g_l2cache_align_mask) {
 339                 size &= ~g_l2cache_align_mask;
 340                 size += g_l2cache_line_size;
 341         }
 342         while (size > 0) {
 343                 pl310_write4(pl310_softc, PL310_INV_LINE_PA, start);
 344                 start += g_l2cache_line_size;
 345                 size -= g_l2cache_line_size;
 346         }
 347
 348         pl310_cache_sync();
 349         PL310_UNLOCK(pl310_softc);
 350 }
 351
 352 static void
 353 pl310_set_way_sizes(struct pl310_softc *sc)
 354 {
 355         uint32_t aux_value;
 356
 357         aux_value = pl310_read4(sc, PL310_AUX_CTRL);
 358         g_way_size = (aux_value & AUX_CTRL_WAY_SIZE_MASK) >>
 359             AUX_CTRL_WAY_SIZE_SHIFT;
 360         g_way_size = 1 << (g_way_size + 13);
 361         if (aux_value & (1 << AUX_CTRL_ASSOCIATIVITY_SHIFT))
 362                 g_ways_assoc = 16;
 363         else
 364                 g_ways_assoc = 8;
 365         g_l2cache_way_mask = (1 << g_ways_assoc) - 1;
 366         g_l2cache_size = g_way_size * g_ways_assoc;
 367 }
 368
 369 static int
 370 pl310_probe(device_t dev)
 371 {
 372
 373         if (!ofw_bus_status_okay(dev))
 374                 return (ENXIO);
 375
 376         if (!ofw_bus_is_compatible(dev, "arm,pl310"))
 377                 return (ENXIO);
 378         device_set_desc(dev, "PL310 L2 cache controller");
 379         return (0);
 380 }
 381
 382 static int
 383 pl310_attach(device_t dev)
 384 {
 385         struct pl310_softc *sc = device_get_softc(dev);
 386         int rid;
 387         uint32_t cache_id, debug_ctrl;
 388
 389         sc->sc_dev = dev;
 390         rid = 0;
 391         sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 392             RF_ACTIVE);
 393         if (sc->sc_mem_res == NULL)
 394                 panic("%s: Cannot map registers", device_get_name(dev));
 395
 396         /* Allocate an IRQ resource */
 397         rid = 0;
 398         sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 399                                                 RF_ACTIVE | RF_SHAREABLE);
 400         if (sc->sc_irq_res == NULL) {
 401                 panic("Cannot allocate IRQ\n");
 402         }
 403
 404         pl310_softc = sc;
 405         mtx_init(&sc->sc_mtx, "pl310lock", NULL, MTX_SPIN);
 406
 407         /* activate the interrupt */
 408         bus_setup_intr(dev, sc->sc_irq_res, INTR_TYPE_MISC | INTR_MPSAFE,
 409                                 pl310_filter, NULL, sc, &sc->sc_irq_h);
 410
 411         cache_id = pl310_read4(sc, PL310_CACHE_ID);
 412         sc->sc_rtl_revision = (cache_id >> CACHE_ID_RELEASE_SHIFT) &
 413             CACHE_ID_RELEASE_MASK;
 414         device_printf(dev, "Part number: 0x%x, release: 0x%x\n",
 415             (cache_id >> CACHE_ID_PARTNUM_SHIFT) & CACHE_ID_PARTNUM_MASK,
 416             (cache_id >> CACHE_ID_RELEASE_SHIFT) & CACHE_ID_RELEASE_MASK);
 417
 418         /*
 419          * If L2 cache is already enabled then something has violated the rules,
 420          * because caches are supposed to be off at kernel entry.  The cache
 421          * must be disabled to write the configuration registers without
 422          * triggering an access error (SLVERR), but there's no documented safe
 423          * procedure for disabling the L2 cache in the manual.  So we'll try to
 424          * invent one:
 425          *  - Use the debug register to force write-through mode and prevent
 426          *    linefills (allocation of new lines on read); now anything we do
 427          *    will not cause new data to come into the L2 cache.
 428          *  - Writeback and invalidate the current contents.
 429          *  - Disable the controller.
 430          *  - Restore the original debug settings.
 431          */
 432         if (pl310_read4(sc, PL310_CTRL) & CTRL_ENABLED) {
 433                 device_printf(dev, "Warning: L2 Cache should not already be "
 434                     "active; trying to de-activate and re-initialize...\n");
 435                 sc->sc_enabled = 1;
 436                 debug_ctrl = pl310_read4(sc, PL310_DEBUG_CTRL);
 437                 platform_pl310_write_debug(sc, debug_ctrl |
 438                     DEBUG_CTRL_DISABLE_WRITEBACK | DEBUG_CTRL_DISABLE_LINEFILL);
 439                 pl310_set_way_sizes(sc);
 440                 pl310_wbinv_all();
 441                 platform_pl310_write_ctrl(sc, CTRL_DISABLED);
 442                 platform_pl310_write_debug(sc, debug_ctrl);
 443         }
 444         sc->sc_enabled = pl310_enabled;
 445
 446         if (sc->sc_enabled) {
 447                 platform_pl310_init(sc);
 448                 pl310_set_way_sizes(sc); /* platform init might change these */
 449                 pl310_write4(pl310_softc, PL310_INV_WAY, 0xffff);
 450                 pl310_wait_background_op(PL310_INV_WAY, 0xffff);
 451                 platform_pl310_write_ctrl(sc, CTRL_ENABLED);
 452                 device_printf(dev, "L2 Cache enabled: %uKB/%dB %d ways\n",
 453                     (g_l2cache_size / 1024), g_l2cache_line_size, g_ways_assoc);
 454                 if (bootverbose)
 455                         pl310_print_config(sc);
 456         } else {
 457                 /*
 458                  * Set counters so when cache event happens we'll get interrupt
 459                  * and be warned that something is off.
 460                  */
 461
 462                 /* Cache Line Eviction for Counter 0 */
 463                 pl310_write4(sc, PL310_EVENT_COUNTER0_CONF,
 464                     EVENT_COUNTER_CONF_INCR | EVENT_COUNTER_CONF_CO);
 465                 /* Data Read Request for Counter 1 */
 466                 pl310_write4(sc, PL310_EVENT_COUNTER1_CONF,
 467                     EVENT_COUNTER_CONF_INCR | EVENT_COUNTER_CONF_DRREQ);
 468
 469                 /* Enable and clear pending interrupts */
 470                 pl310_write4(sc, PL310_INTR_CLEAR, INTR_MASK_ECNTR);
 471                 pl310_write4(sc, PL310_INTR_MASK, INTR_MASK_ALL);
 472
 473                 /* Enable counters and reset C0 and C1 */
 474                 pl310_write4(sc, PL310_EVENT_COUNTER_CTRL,
 475                     EVENT_COUNTER_CTRL_ENABLED |
 476                     EVENT_COUNTER_CTRL_C0_RESET |
 477                     EVENT_COUNTER_CTRL_C1_RESET);
 478
 479                 device_printf(dev, "L2 Cache disabled\n");
 480         }
 481
 482         /* Set the l2 functions in the set of cpufuncs */
 483         cpufuncs.cf_l2cache_wbinv_all = pl310_wbinv_all;
 484         cpufuncs.cf_l2cache_wbinv_range = pl310_wbinv_range;
 485         cpufuncs.cf_l2cache_inv_range = pl310_inv_range;
 486         cpufuncs.cf_l2cache_wb_range = pl310_wb_range;
 487
 488         return (0);
 489 }
 490
 491 static device_method_t pl310_methods[] = {
 492         DEVMETHOD(device_probe, pl310_probe),
 493         DEVMETHOD(device_attach, pl310_attach),
 494         DEVMETHOD_END
 495 };
 496
 497 static driver_t pl310_driver = {
 498         "l2cache",
 499         pl310_methods,
 500         sizeof(struct pl310_softc),
 501 };
 502 static devclass_t pl310_devclass;
 503
 504 DRIVER_MODULE(pl310, simplebus, pl310_driver, pl310_devclass, 0, 0);