2 * Copyright (c) 2012 Olivier Houchard <cognet@FreeBSD.org>
4 * Ben Gray <ben.r.gray@gmail.com>.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. The name of the company nor the name of the author may be used to
16 * endorse or promote products derived from this software without specific
17 * prior written permission.
19 * THIS SOFTWARE IS PROVIDED BY BEN GRAY ``AS IS'' AND ANY EXPRESS OR
20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22 * IN NO EVENT SHALL BEN GRAY BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
24 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
25 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
26 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
27 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
28 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 #include <sys/param.h>
34 #include <sys/systm.h>
36 #include <sys/kernel.h>
38 #include <sys/module.h>
40 #include <sys/mutex.h>
41 #include <machine/intr.h>
43 #include <machine/bus.h>
44 #include <machine/pl310.h>
46 #include <dev/ofw/openfirm.h>
47 #include <dev/ofw/ofw_bus.h>
48 #include <dev/ofw/ofw_bus_subr.h>
51 * Define this if you need to disable PL310 for debugging purpose
53 * http://infocenter.arm.com/help/topic/com.arm.doc.ddi0246e/DDI0246E_l2c310_r3p1_trm.pdf
57 * Hardcode errata for now
58 * http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ddi0246b/pr01s02s02.html
60 #define PL310_ERRATA_588369
61 #define PL310_ERRATA_753970
62 #define PL310_ERRATA_727915
64 #define PL310_LOCK(sc) do { \
65 mtx_lock_spin(&(sc)->sc_mtx); \
68 #define PL310_UNLOCK(sc) do { \
69 mtx_unlock_spin(&(sc)->sc_mtx); \
72 static int pl310_enabled = 1;
73 TUNABLE_INT("hw.pl310.enabled", &pl310_enabled);
75 static uint32_t g_l2cache_way_mask;
77 static const uint32_t g_l2cache_line_size = 32;
78 static const uint32_t g_l2cache_align_mask = (32 - 1);
80 static uint32_t g_l2cache_size;
81 static uint32_t g_way_size;
82 static uint32_t g_ways_assoc;
84 static struct pl310_softc *pl310_softc;
86 static struct ofw_compat_data compat_data[] = {
87 {"arm,pl310", true}, /* Non-standard, FreeBSD. */
88 {"arm,pl310-cache", true},
93 pl310_print_config(struct pl310_softc *sc)
95 uint32_t aux, prefetch;
96 const char *dis = "disabled";
97 const char *ena = "enabled";
99 aux = pl310_read4(sc, PL310_AUX_CTRL);
100 prefetch = pl310_read4(sc, PL310_PREFETCH_CTRL);
102 device_printf(sc->sc_dev, "Early BRESP response: %s\n",
103 (aux & AUX_CTRL_EARLY_BRESP) ? ena : dis);
104 device_printf(sc->sc_dev, "Instruction prefetch: %s\n",
105 (aux & AUX_CTRL_INSTR_PREFETCH) ? ena : dis);
106 device_printf(sc->sc_dev, "Data prefetch: %s\n",
107 (aux & AUX_CTRL_DATA_PREFETCH) ? ena : dis);
108 device_printf(sc->sc_dev, "Non-secure interrupt control: %s\n",
109 (aux & AUX_CTRL_NS_INT_CTRL) ? ena : dis);
110 device_printf(sc->sc_dev, "Non-secure lockdown: %s\n",
111 (aux & AUX_CTRL_NS_LOCKDOWN) ? ena : dis);
112 device_printf(sc->sc_dev, "Share override: %s\n",
113 (aux & AUX_CTRL_SHARE_OVERRIDE) ? ena : dis);
115 device_printf(sc->sc_dev, "Double linefill: %s\n",
116 (prefetch & PREFETCH_CTRL_DL) ? ena : dis);
117 device_printf(sc->sc_dev, "Instruction prefetch: %s\n",
118 (prefetch & PREFETCH_CTRL_INSTR_PREFETCH) ? ena : dis);
119 device_printf(sc->sc_dev, "Data prefetch: %s\n",
120 (prefetch & PREFETCH_CTRL_DATA_PREFETCH) ? ena : dis);
121 device_printf(sc->sc_dev, "Double linefill on WRAP request: %s\n",
122 (prefetch & PREFETCH_CTRL_DL_ON_WRAP) ? ena : dis);
123 device_printf(sc->sc_dev, "Prefetch drop: %s\n",
124 (prefetch & PREFETCH_CTRL_PREFETCH_DROP) ? ena : dis);
125 device_printf(sc->sc_dev, "Incr double Linefill: %s\n",
126 (prefetch & PREFETCH_CTRL_INCR_DL) ? ena : dis);
127 device_printf(sc->sc_dev, "Not same ID on exclusive sequence: %s\n",
128 (prefetch & PREFETCH_CTRL_NOTSAMEID) ? ena : dis);
129 device_printf(sc->sc_dev, "Prefetch offset: %d\n",
130 (prefetch & PREFETCH_CTRL_OFFSET_MASK));
134 pl310_set_ram_latency(struct pl310_softc *sc, uint32_t which_reg,
135 uint32_t read, uint32_t write, uint32_t setup)
139 KASSERT(which_reg == PL310_TAG_RAM_CTRL ||
140 which_reg == PL310_DATA_RAM_CTRL,
141 ("bad pl310 ram latency register address"));
143 v = pl310_read4(sc, which_reg);
145 KASSERT(setup <= 8, ("bad pl310 setup latency: %d", setup));
146 v &= ~RAM_CTRL_SETUP_MASK;
147 v |= (setup - 1) << RAM_CTRL_SETUP_SHIFT;
150 KASSERT(read <= 8, ("bad pl310 read latency: %d", read));
151 v &= ~RAM_CTRL_READ_MASK;
152 v |= (read - 1) << RAM_CTRL_READ_SHIFT;
155 KASSERT(write <= 8, ("bad pl310 write latency: %d", write));
156 v &= ~RAM_CTRL_WRITE_MASK;
157 v |= (write - 1) << RAM_CTRL_WRITE_SHIFT;
159 pl310_write4(sc, which_reg, v);
163 pl310_filter(void *arg)
165 struct pl310_softc *sc = arg;
168 intr = pl310_read4(sc, PL310_INTR_MASK);
170 if (!sc->sc_enabled && (intr & INTR_MASK_ECNTR)) {
172 * This is for debug purpose, so be blunt about it
173 * We disable PL310 only when something fishy is going
174 * on and we need to make sure L2 cache is 100% disabled
176 panic("pl310: caches disabled but cache event detected\n");
179 return (FILTER_HANDLED);
183 pl310_wait_background_op(uint32_t off, uint32_t mask)
186 while (pl310_read4(pl310_softc, off) & mask)
192 * pl310_cache_sync - performs a cache sync operation
194 * According to the TRM:
196 * "Before writing to any other register you must perform an explicit
197 * Cache Sync operation. This is particularly important when the cache is
198 * enabled and changes to how the cache allocates new lines are to be made."
203 pl310_cache_sync(void)
206 if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
209 /* Do not sync outer cache on IO coherent platform */
210 if (pl310_softc->sc_io_coherent)
213 #ifdef PL310_ERRATA_753970
214 if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r3p0)
215 /* Write uncached PL310 register */
216 pl310_write4(pl310_softc, 0x740, 0xffffffff);
219 pl310_write4(pl310_softc, PL310_CACHE_SYNC, 0xffffffff);
224 pl310_wbinv_all(void)
227 if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
230 PL310_LOCK(pl310_softc);
231 #ifdef PL310_ERRATA_727915
232 if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r2p0) {
235 for (i = 0; i < g_ways_assoc; i++) {
236 for (j = 0; j < g_way_size / g_l2cache_line_size; j++) {
237 pl310_write4(pl310_softc,
238 PL310_CLEAN_INV_LINE_IDX,
243 PL310_UNLOCK(pl310_softc);
247 if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r3p0)
248 platform_pl310_write_debug(pl310_softc, 3);
250 pl310_write4(pl310_softc, PL310_CLEAN_INV_WAY, g_l2cache_way_mask);
251 pl310_wait_background_op(PL310_CLEAN_INV_WAY, g_l2cache_way_mask);
253 #ifdef PL310_ERRATA_727915
254 if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r3p0)
255 platform_pl310_write_debug(pl310_softc, 0);
257 PL310_UNLOCK(pl310_softc);
261 pl310_wbinv_range(vm_paddr_t start, vm_size_t size)
264 if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
267 PL310_LOCK(pl310_softc);
268 if (start & g_l2cache_align_mask) {
269 size += start & g_l2cache_align_mask;
270 start &= ~g_l2cache_align_mask;
272 if (size & g_l2cache_align_mask) {
273 size &= ~g_l2cache_align_mask;
274 size += g_l2cache_line_size;
278 #ifdef PL310_ERRATA_727915
279 if (pl310_softc->sc_rtl_revision >= CACHE_ID_RELEASE_r2p0 &&
280 pl310_softc->sc_rtl_revision < CACHE_ID_RELEASE_r3p1)
281 platform_pl310_write_debug(pl310_softc, 3);
284 #ifdef PL310_ERRATA_588369
285 if (pl310_softc->sc_rtl_revision <= CACHE_ID_RELEASE_r1p0) {
287 * Errata 588369 says that clean + inv may keep the
288 * cache line if it was clean, the recommanded
289 * workaround is to clean then invalidate the cache
290 * line, with write-back and cache linefill disabled.
292 pl310_write4(pl310_softc, PL310_CLEAN_LINE_PA, start);
293 pl310_write4(pl310_softc, PL310_INV_LINE_PA, start);
296 pl310_write4(pl310_softc, PL310_CLEAN_INV_LINE_PA,
298 start += g_l2cache_line_size;
299 size -= g_l2cache_line_size;
301 #ifdef PL310_ERRATA_727915
302 if (pl310_softc->sc_rtl_revision >= CACHE_ID_RELEASE_r2p0 &&
303 pl310_softc->sc_rtl_revision < CACHE_ID_RELEASE_r3p1)
304 platform_pl310_write_debug(pl310_softc, 0);
308 PL310_UNLOCK(pl310_softc);
312 pl310_wb_range(vm_paddr_t start, vm_size_t size)
315 if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
318 PL310_LOCK(pl310_softc);
319 if (start & g_l2cache_align_mask) {
320 size += start & g_l2cache_align_mask;
321 start &= ~g_l2cache_align_mask;
324 if (size & g_l2cache_align_mask) {
325 size &= ~g_l2cache_align_mask;
326 size += g_l2cache_line_size;
330 pl310_write4(pl310_softc, PL310_CLEAN_LINE_PA, start);
331 start += g_l2cache_line_size;
332 size -= g_l2cache_line_size;
336 PL310_UNLOCK(pl310_softc);
340 pl310_inv_range(vm_paddr_t start, vm_size_t size)
343 if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
346 PL310_LOCK(pl310_softc);
347 if (start & g_l2cache_align_mask) {
348 size += start & g_l2cache_align_mask;
349 start &= ~g_l2cache_align_mask;
351 if (size & g_l2cache_align_mask) {
352 size &= ~g_l2cache_align_mask;
353 size += g_l2cache_line_size;
356 pl310_write4(pl310_softc, PL310_INV_LINE_PA, start);
357 start += g_l2cache_line_size;
358 size -= g_l2cache_line_size;
362 PL310_UNLOCK(pl310_softc);
366 pl310_drain_writebuf(void)
369 if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
372 PL310_LOCK(pl310_softc);
374 PL310_UNLOCK(pl310_softc);
378 pl310_set_way_sizes(struct pl310_softc *sc)
382 aux_value = pl310_read4(sc, PL310_AUX_CTRL);
383 g_way_size = (aux_value & AUX_CTRL_WAY_SIZE_MASK) >>
384 AUX_CTRL_WAY_SIZE_SHIFT;
385 g_way_size = 1 << (g_way_size + 13);
386 if (aux_value & (1 << AUX_CTRL_ASSOCIATIVITY_SHIFT))
390 g_l2cache_way_mask = (1 << g_ways_assoc) - 1;
391 g_l2cache_size = g_way_size * g_ways_assoc;
395 * Setup interrupt handling. This is done only if the cache controller is
396 * disabled, for debugging. We set counters so when a cache event happens we'll
397 * get interrupted and be warned that something is wrong, because no cache
398 * events should happen if we're disabled.
401 pl310_config_intr(void *arg)
403 struct pl310_softc * sc;
407 /* activate the interrupt */
408 bus_setup_intr(sc->sc_dev, sc->sc_irq_res, INTR_TYPE_MISC | INTR_MPSAFE,
409 pl310_filter, NULL, sc, &sc->sc_irq_h);
411 /* Cache Line Eviction for Counter 0 */
412 pl310_write4(sc, PL310_EVENT_COUNTER0_CONF,
413 EVENT_COUNTER_CONF_INCR | EVENT_COUNTER_CONF_CO);
414 /* Data Read Request for Counter 1 */
415 pl310_write4(sc, PL310_EVENT_COUNTER1_CONF,
416 EVENT_COUNTER_CONF_INCR | EVENT_COUNTER_CONF_DRREQ);
418 /* Enable and clear pending interrupts */
419 pl310_write4(sc, PL310_INTR_CLEAR, INTR_MASK_ECNTR);
420 pl310_write4(sc, PL310_INTR_MASK, INTR_MASK_ALL);
422 /* Enable counters and reset C0 and C1 */
423 pl310_write4(sc, PL310_EVENT_COUNTER_CTRL,
424 EVENT_COUNTER_CTRL_ENABLED |
425 EVENT_COUNTER_CTRL_C0_RESET |
426 EVENT_COUNTER_CTRL_C1_RESET);
428 config_intrhook_disestablish(sc->sc_ich);
429 free(sc->sc_ich, M_DEVBUF);
434 pl310_probe(device_t dev)
437 if (!ofw_bus_status_okay(dev))
439 if (!ofw_bus_search_compatible(dev, compat_data)->ocd_data)
441 device_set_desc(dev, "PL310 L2 cache controller");
446 pl310_attach(device_t dev)
448 struct pl310_softc *sc = device_get_softc(dev);
450 uint32_t cache_id, debug_ctrl;
455 sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
457 if (sc->sc_mem_res == NULL)
458 panic("%s: Cannot map registers", device_get_name(dev));
460 /* Allocate an IRQ resource */
462 sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
463 RF_ACTIVE | RF_SHAREABLE);
464 if (sc->sc_irq_res == NULL) {
465 device_printf(dev, "cannot allocate IRQ, not using interrupt\n");
469 mtx_init(&sc->sc_mtx, "pl310lock", NULL, MTX_SPIN);
471 cache_id = pl310_read4(sc, PL310_CACHE_ID);
472 sc->sc_rtl_revision = (cache_id >> CACHE_ID_RELEASE_SHIFT) &
473 CACHE_ID_RELEASE_MASK;
474 device_printf(dev, "Part number: 0x%x, release: 0x%x\n",
475 (cache_id >> CACHE_ID_PARTNUM_SHIFT) & CACHE_ID_PARTNUM_MASK,
476 (cache_id >> CACHE_ID_RELEASE_SHIFT) & CACHE_ID_RELEASE_MASK);
479 * Test for "arm,io-coherent" property and disable sync operation if
480 * platform is I/O coherent. Outer sync operations are not needed
481 * on coherent platform and may be harmful in certain situations.
483 node = ofw_bus_get_node(dev);
484 if (OF_hasprop(node, "arm,io-coherent"))
485 sc->sc_io_coherent = true;
488 * If L2 cache is already enabled then something has violated the rules,
489 * because caches are supposed to be off at kernel entry. The cache
490 * must be disabled to write the configuration registers without
491 * triggering an access error (SLVERR), but there's no documented safe
492 * procedure for disabling the L2 cache in the manual. So we'll try to
494 * - Use the debug register to force write-through mode and prevent
495 * linefills (allocation of new lines on read); now anything we do
496 * will not cause new data to come into the L2 cache.
497 * - Writeback and invalidate the current contents.
498 * - Disable the controller.
499 * - Restore the original debug settings.
501 if (pl310_read4(sc, PL310_CTRL) & CTRL_ENABLED) {
502 device_printf(dev, "Warning: L2 Cache should not already be "
503 "active; trying to de-activate and re-initialize...\n");
505 debug_ctrl = pl310_read4(sc, PL310_DEBUG_CTRL);
506 platform_pl310_write_debug(sc, debug_ctrl |
507 DEBUG_CTRL_DISABLE_WRITEBACK | DEBUG_CTRL_DISABLE_LINEFILL);
508 pl310_set_way_sizes(sc);
510 platform_pl310_write_ctrl(sc, CTRL_DISABLED);
511 platform_pl310_write_debug(sc, debug_ctrl);
513 sc->sc_enabled = pl310_enabled;
515 if (sc->sc_enabled) {
516 platform_pl310_init(sc);
517 pl310_set_way_sizes(sc); /* platform init might change these */
518 pl310_write4(pl310_softc, PL310_INV_WAY, 0xffff);
519 pl310_wait_background_op(PL310_INV_WAY, 0xffff);
520 platform_pl310_write_ctrl(sc, CTRL_ENABLED);
521 device_printf(dev, "L2 Cache enabled: %uKB/%dB %d ways\n",
522 (g_l2cache_size / 1024), g_l2cache_line_size, g_ways_assoc);
524 pl310_print_config(sc);
526 if (sc->sc_irq_res != NULL) {
527 sc->sc_ich = malloc(sizeof(*sc->sc_ich), M_DEVBUF, M_WAITOK);
528 sc->sc_ich->ich_func = pl310_config_intr;
529 sc->sc_ich->ich_arg = sc;
530 if (config_intrhook_establish(sc->sc_ich) != 0) {
532 "config_intrhook_establish failed\n");
533 free(sc->sc_ich, M_DEVBUF);
538 device_printf(dev, "L2 Cache disabled\n");
541 /* Set the l2 functions in the set of cpufuncs */
542 cpufuncs.cf_l2cache_wbinv_all = pl310_wbinv_all;
543 cpufuncs.cf_l2cache_wbinv_range = pl310_wbinv_range;
544 cpufuncs.cf_l2cache_inv_range = pl310_inv_range;
545 cpufuncs.cf_l2cache_wb_range = pl310_wb_range;
546 cpufuncs.cf_l2cache_drain_writebuf = pl310_drain_writebuf;
551 static device_method_t pl310_methods[] = {
552 DEVMETHOD(device_probe, pl310_probe),
553 DEVMETHOD(device_attach, pl310_attach),
557 static driver_t pl310_driver = {
560 sizeof(struct pl310_softc),
562 static devclass_t pl310_devclass;
564 EARLY_DRIVER_MODULE(pl310, simplebus, pl310_driver, pl310_devclass, 0, 0,
565 BUS_PASS_CPU + BUS_PASS_ORDER_MIDDLE);