]> CyberLeo.Net >> Repos - FreeBSD/FreeBSD.git/blob - sys/arm/arm/pl310.c
IFC @r271694
[FreeBSD/FreeBSD.git] / sys / arm / arm / pl310.c
1 /*-
2  * Copyright (c) 2012 Olivier Houchard <cognet@FreeBSD.org>
3  * Copyright (c) 2011
4  *      Ben Gray <ben.r.gray@gmail.com>.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  * 3. The name of the company nor the name of the author may be used to
16  *    endorse or promote products derived from this software without specific
17  *    prior written permission.
18  *
19  * THIS SOFTWARE IS PROVIDED BY BEN GRAY ``AS IS'' AND ANY EXPRESS OR
20  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22  * IN NO EVENT SHALL BEN GRAY BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
24  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
25  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
26  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
27  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
28  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29  */
30
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/bus.h>
36 #include <sys/kernel.h>
37 #include <sys/rman.h>
38 #include <sys/module.h>
39 #include <sys/lock.h>
40 #include <sys/mutex.h>
41 #include <machine/intr.h>
42
43 #include <machine/bus.h>
44 #include <machine/pl310.h>
45
46 #include <dev/fdt/fdt_common.h>
47 #include <dev/ofw/openfirm.h>
48 #include <dev/ofw/ofw_bus.h>
49 #include <dev/ofw/ofw_bus_subr.h>
50
51 /*
52  * Define this if you need to disable PL310 for debugging purpose
53  * Spec: 
54  * http://infocenter.arm.com/help/topic/com.arm.doc.ddi0246e/DDI0246E_l2c310_r3p1_trm.pdf
55  */
56
57 /* 
58  * Hardcode errata for now
59  * http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ddi0246b/pr01s02s02.html
60  */
61 #define PL310_ERRATA_588369
62 #define PL310_ERRATA_753970
63 #define PL310_ERRATA_727915
64
65 #define PL310_LOCK(sc) do {             \
66         mtx_lock_spin(&(sc)->sc_mtx);   \
67 } while(0);
68
69 #define PL310_UNLOCK(sc) do {           \
70         mtx_unlock_spin(&(sc)->sc_mtx); \
71 } while(0);
72
73 static int pl310_enabled = 1;
74 TUNABLE_INT("hw.pl310.enabled", &pl310_enabled);
75
76 static uint32_t g_l2cache_way_mask;
77
78 static const uint32_t g_l2cache_line_size = 32;
79 static const uint32_t g_l2cache_align_mask = (32 - 1);
80
81 static uint32_t g_l2cache_size;
82 static uint32_t g_way_size;
83 static uint32_t g_ways_assoc;
84
85 static struct pl310_softc *pl310_softc;
86
87 void
88 pl310_print_config(struct pl310_softc *sc)
89 {
90         uint32_t aux, prefetch;
91         const char *dis = "disabled";
92         const char *ena = "enabled";
93
94         aux = pl310_read4(sc, PL310_AUX_CTRL);
95         prefetch = pl310_read4(sc, PL310_PREFETCH_CTRL);
96
97         device_printf(sc->sc_dev, "Early BRESP response: %s\n",
98                 (aux & AUX_CTRL_EARLY_BRESP) ? ena : dis);
99         device_printf(sc->sc_dev, "Instruction prefetch: %s\n",
100                 (aux & AUX_CTRL_INSTR_PREFETCH) ? ena : dis);
101         device_printf(sc->sc_dev, "Data prefetch: %s\n",
102                 (aux & AUX_CTRL_DATA_PREFETCH) ? ena : dis);
103         device_printf(sc->sc_dev, "Non-secure interrupt control: %s\n",
104                 (aux & AUX_CTRL_NS_INT_CTRL) ? ena : dis);
105         device_printf(sc->sc_dev, "Non-secure lockdown: %s\n",
106                 (aux & AUX_CTRL_NS_LOCKDOWN) ? ena : dis);
107         device_printf(sc->sc_dev, "Share override: %s\n",
108                 (aux & AUX_CTRL_SHARE_OVERRIDE) ? ena : dis);
109
110         device_printf(sc->sc_dev, "Double linefill: %s\n",
111                 (prefetch & PREFETCH_CTRL_DL) ? ena : dis);
112         device_printf(sc->sc_dev, "Instruction prefetch: %s\n",
113                 (prefetch & PREFETCH_CTRL_INSTR_PREFETCH) ? ena : dis);
114         device_printf(sc->sc_dev, "Data prefetch: %s\n",
115                 (prefetch & PREFETCH_CTRL_DATA_PREFETCH) ? ena : dis);
116         device_printf(sc->sc_dev, "Double linefill on WRAP request: %s\n",
117                 (prefetch & PREFETCH_CTRL_DL_ON_WRAP) ? ena : dis);
118         device_printf(sc->sc_dev, "Prefetch drop: %s\n",
119                 (prefetch & PREFETCH_CTRL_PREFETCH_DROP) ? ena : dis);
120         device_printf(sc->sc_dev, "Incr double Linefill: %s\n",
121                 (prefetch & PREFETCH_CTRL_INCR_DL) ? ena : dis);
122         device_printf(sc->sc_dev, "Not same ID on exclusive sequence: %s\n",
123                 (prefetch & PREFETCH_CTRL_NOTSAMEID) ? ena : dis);
124         device_printf(sc->sc_dev, "Prefetch offset: %d\n",
125                 (prefetch & PREFETCH_CTRL_OFFSET_MASK));
126 }
127
128 void
129 pl310_set_ram_latency(struct pl310_softc *sc, uint32_t which_reg,
130    uint32_t read, uint32_t write, uint32_t setup)
131 {
132         uint32_t v;
133
134         KASSERT(which_reg == PL310_TAG_RAM_CTRL || 
135             which_reg == PL310_DATA_RAM_CTRL,
136             ("bad pl310 ram latency register address"));
137
138         v = pl310_read4(sc, which_reg);
139         if (setup != 0) {
140                 KASSERT(setup <= 8, ("bad pl310 setup latency: %d", setup));
141                 v &= ~RAM_CTRL_SETUP_MASK;
142                 v |= (setup - 1) << RAM_CTRL_SETUP_SHIFT;
143         }
144         if (read != 0) {
145                 KASSERT(read <= 8, ("bad pl310 read latency: %d", read));
146                 v &= ~RAM_CTRL_READ_MASK;
147                 v |= (read - 1) << RAM_CTRL_READ_SHIFT;
148         }
149         if (write != 0) {
150                 KASSERT(write <= 8, ("bad pl310 write latency: %d", write));
151                 v &= ~RAM_CTRL_WRITE_MASK;
152                 v |= (write - 1) << RAM_CTRL_WRITE_SHIFT;
153         }
154         pl310_write4(sc, which_reg, v);
155 }
156
157 static int
158 pl310_filter(void *arg)
159 {
160         struct pl310_softc *sc = arg;
161         uint32_t intr;
162
163         intr = pl310_read4(sc, PL310_INTR_MASK);
164
165         if (!sc->sc_enabled && (intr & INTR_MASK_ECNTR)) {
166                 /*
167                  * This is for debug purpose, so be blunt about it
168                  * We disable PL310 only when something fishy is going
169                  * on and we need to make sure L2 cache is 100% disabled
170                  */
171                 panic("pl310: caches disabled but cache event detected\n");
172         }
173
174         return (FILTER_HANDLED);
175 }
176
177 static __inline void
178 pl310_wait_background_op(uint32_t off, uint32_t mask)
179 {
180
181         while (pl310_read4(pl310_softc, off) & mask)
182                 continue;
183 }
184
185
186 /**
187  *      pl310_cache_sync - performs a cache sync operation
188  * 
189  *      According to the TRM:
190  *
191  *  "Before writing to any other register you must perform an explicit
192  *   Cache Sync operation. This is particularly important when the cache is
193  *   enabled and changes to how the cache allocates new lines are to be made."
194  *
195  *
196  */
197 static __inline void
198 pl310_cache_sync(void)
199 {
200
201         if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
202                 return;
203
204 #ifdef PL310_ERRATA_753970
205         if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r3p0)
206                 /* Write uncached PL310 register */
207                 pl310_write4(pl310_softc, 0x740, 0xffffffff);
208         else
209 #endif
210                 pl310_write4(pl310_softc, PL310_CACHE_SYNC, 0xffffffff);
211 }
212
213
214 static void
215 pl310_wbinv_all(void)
216 {
217
218         if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
219                 return;
220
221         PL310_LOCK(pl310_softc);
222 #ifdef PL310_ERRATA_727915
223         if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r2p0) {
224                 int i, j;
225
226                 for (i = 0; i < g_ways_assoc; i++) {
227                         for (j = 0; j < g_way_size / g_l2cache_line_size; j++) {
228                                 pl310_write4(pl310_softc, 
229                                     PL310_CLEAN_INV_LINE_IDX,
230                                     (i << 28 | j << 5));
231                         }
232                 }
233                 pl310_cache_sync();
234                 PL310_UNLOCK(pl310_softc);
235                 return;
236
237         }
238         if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r3p0)
239                 platform_pl310_write_debug(pl310_softc, 3);
240 #endif
241         pl310_write4(pl310_softc, PL310_CLEAN_INV_WAY, g_l2cache_way_mask);
242         pl310_wait_background_op(PL310_CLEAN_INV_WAY, g_l2cache_way_mask);
243         pl310_cache_sync();
244 #ifdef PL310_ERRATA_727915
245         if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r3p0)
246                 platform_pl310_write_debug(pl310_softc, 0);
247 #endif
248         PL310_UNLOCK(pl310_softc);
249 }
250
251 static void
252 pl310_wbinv_range(vm_paddr_t start, vm_size_t size)
253 {
254
255         if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
256                 return;
257
258         PL310_LOCK(pl310_softc);
259         if (start & g_l2cache_align_mask) {
260                 size += start & g_l2cache_align_mask;
261                 start &= ~g_l2cache_align_mask;
262         }
263         if (size & g_l2cache_align_mask) {
264                 size &= ~g_l2cache_align_mask;
265                 size += g_l2cache_line_size;
266         }
267
268
269 #ifdef PL310_ERRATA_727915
270         platform_pl310_write_debug(pl310_softc, 3);
271 #endif
272         while (size > 0) {
273 #ifdef PL310_ERRATA_588369
274                 if (pl310_softc->sc_rtl_revision <= CACHE_ID_RELEASE_r1p0) {
275                         /* 
276                          * Errata 588369 says that clean + inv may keep the 
277                          * cache line if it was clean, the recommanded
278                          * workaround is to clean then invalidate the cache
279                          * line, with write-back and cache linefill disabled.
280                          */
281                         pl310_write4(pl310_softc, PL310_CLEAN_LINE_PA, start);
282                         pl310_write4(pl310_softc, PL310_INV_LINE_PA, start);
283                 } else
284 #endif
285                         pl310_write4(pl310_softc, PL310_CLEAN_INV_LINE_PA,
286                             start);
287                 start += g_l2cache_line_size;
288                 size -= g_l2cache_line_size;
289         }
290 #ifdef PL310_ERRATA_727915
291         platform_pl310_write_debug(pl310_softc, 0);
292 #endif
293
294         pl310_cache_sync();
295         PL310_UNLOCK(pl310_softc);
296 }
297
298 static void
299 pl310_wb_range(vm_paddr_t start, vm_size_t size)
300 {
301
302         if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
303                 return;
304
305         PL310_LOCK(pl310_softc);
306         if (start & g_l2cache_align_mask) {
307                 size += start & g_l2cache_align_mask;
308                 start &= ~g_l2cache_align_mask;
309         }
310
311         if (size & g_l2cache_align_mask) {
312                 size &= ~g_l2cache_align_mask;
313                 size += g_l2cache_line_size;
314         }
315
316         while (size > 0) {
317                 pl310_write4(pl310_softc, PL310_CLEAN_LINE_PA, start);
318                 start += g_l2cache_line_size;
319                 size -= g_l2cache_line_size;
320         }
321
322         pl310_cache_sync();
323         PL310_UNLOCK(pl310_softc);
324 }
325
326 static void
327 pl310_inv_range(vm_paddr_t start, vm_size_t size)
328 {
329
330         if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
331                 return;
332
333         PL310_LOCK(pl310_softc);
334         if (start & g_l2cache_align_mask) {
335                 size += start & g_l2cache_align_mask;
336                 start &= ~g_l2cache_align_mask;
337         }
338         if (size & g_l2cache_align_mask) {
339                 size &= ~g_l2cache_align_mask;
340                 size += g_l2cache_line_size;
341         }
342         while (size > 0) {
343                 pl310_write4(pl310_softc, PL310_INV_LINE_PA, start);
344                 start += g_l2cache_line_size;
345                 size -= g_l2cache_line_size;
346         }
347
348         pl310_cache_sync();
349         PL310_UNLOCK(pl310_softc);
350 }
351
352 static void
353 pl310_drain_writebuf(void)
354 {
355
356         if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
357                 return;
358
359         PL310_LOCK(pl310_softc);
360         pl310_cache_sync();
361         PL310_UNLOCK(pl310_softc);
362 }
363
364 static void
365 pl310_set_way_sizes(struct pl310_softc *sc)
366 {
367         uint32_t aux_value;
368
369         aux_value = pl310_read4(sc, PL310_AUX_CTRL);
370         g_way_size = (aux_value & AUX_CTRL_WAY_SIZE_MASK) >>
371             AUX_CTRL_WAY_SIZE_SHIFT;
372         g_way_size = 1 << (g_way_size + 13);
373         if (aux_value & (1 << AUX_CTRL_ASSOCIATIVITY_SHIFT))
374                 g_ways_assoc = 16;
375         else
376                 g_ways_assoc = 8;
377         g_l2cache_way_mask = (1 << g_ways_assoc) - 1;
378         g_l2cache_size = g_way_size * g_ways_assoc;
379 }
380
381 /*
382  * Setup interrupt handling.  This is done only if the cache controller is
383  * disabled, for debugging.  We set counters so when a cache event happens we'll
384  * get interrupted and be warned that something is wrong, because no cache
385  * events should happen if we're disabled.
386  */
387 static void
388 pl310_config_intr(void *arg)
389 {
390         struct pl310_softc * sc;
391
392         sc = arg;
393
394         /* activate the interrupt */
395         bus_setup_intr(sc->sc_dev, sc->sc_irq_res, INTR_TYPE_MISC | INTR_MPSAFE,
396             pl310_filter, NULL, sc, &sc->sc_irq_h);
397
398         /* Cache Line Eviction for Counter 0 */
399         pl310_write4(sc, PL310_EVENT_COUNTER0_CONF, 
400             EVENT_COUNTER_CONF_INCR | EVENT_COUNTER_CONF_CO);
401         /* Data Read Request for Counter 1 */
402         pl310_write4(sc, PL310_EVENT_COUNTER1_CONF, 
403             EVENT_COUNTER_CONF_INCR | EVENT_COUNTER_CONF_DRREQ);
404
405         /* Enable and clear pending interrupts */
406         pl310_write4(sc, PL310_INTR_CLEAR, INTR_MASK_ECNTR);
407         pl310_write4(sc, PL310_INTR_MASK, INTR_MASK_ALL);
408
409         /* Enable counters and reset C0 and C1 */
410         pl310_write4(sc, PL310_EVENT_COUNTER_CTRL, 
411             EVENT_COUNTER_CTRL_ENABLED | 
412             EVENT_COUNTER_CTRL_C0_RESET | 
413             EVENT_COUNTER_CTRL_C1_RESET);
414
415         config_intrhook_disestablish(sc->sc_ich);
416         free(sc->sc_ich, M_DEVBUF);
417 }
418
419 static int
420 pl310_probe(device_t dev)
421 {
422         
423         if (!ofw_bus_status_okay(dev))
424                 return (ENXIO);
425
426         if (!ofw_bus_is_compatible(dev, "arm,pl310"))
427                 return (ENXIO);
428         device_set_desc(dev, "PL310 L2 cache controller");
429         return (0);
430 }
431
432 static int
433 pl310_attach(device_t dev)
434 {
435         struct pl310_softc *sc = device_get_softc(dev);
436         int rid;
437         uint32_t cache_id, debug_ctrl;
438
439         sc->sc_dev = dev;
440         rid = 0;
441         sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 
442             RF_ACTIVE);
443         if (sc->sc_mem_res == NULL)
444                 panic("%s: Cannot map registers", device_get_name(dev));
445
446         /* Allocate an IRQ resource */
447         rid = 0;
448         sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
449                                                 RF_ACTIVE | RF_SHAREABLE);
450         if (sc->sc_irq_res == NULL) {
451                 panic("Cannot allocate IRQ\n");
452         }
453
454         pl310_softc = sc;
455         mtx_init(&sc->sc_mtx, "pl310lock", NULL, MTX_SPIN);
456
457         cache_id = pl310_read4(sc, PL310_CACHE_ID);
458         sc->sc_rtl_revision = (cache_id >> CACHE_ID_RELEASE_SHIFT) &
459             CACHE_ID_RELEASE_MASK;
460         device_printf(dev, "Part number: 0x%x, release: 0x%x\n",
461             (cache_id >> CACHE_ID_PARTNUM_SHIFT) & CACHE_ID_PARTNUM_MASK,
462             (cache_id >> CACHE_ID_RELEASE_SHIFT) & CACHE_ID_RELEASE_MASK);
463
464         /*
465          * If L2 cache is already enabled then something has violated the rules,
466          * because caches are supposed to be off at kernel entry.  The cache
467          * must be disabled to write the configuration registers without
468          * triggering an access error (SLVERR), but there's no documented safe
469          * procedure for disabling the L2 cache in the manual.  So we'll try to
470          * invent one:
471          *  - Use the debug register to force write-through mode and prevent
472          *    linefills (allocation of new lines on read); now anything we do
473          *    will not cause new data to come into the L2 cache.
474          *  - Writeback and invalidate the current contents.
475          *  - Disable the controller.
476          *  - Restore the original debug settings.
477          */
478         if (pl310_read4(sc, PL310_CTRL) & CTRL_ENABLED) {
479                 device_printf(dev, "Warning: L2 Cache should not already be "
480                     "active; trying to de-activate and re-initialize...\n");
481                 sc->sc_enabled = 1;
482                 debug_ctrl = pl310_read4(sc, PL310_DEBUG_CTRL);
483                 platform_pl310_write_debug(sc, debug_ctrl |
484                     DEBUG_CTRL_DISABLE_WRITEBACK | DEBUG_CTRL_DISABLE_LINEFILL);
485                 pl310_set_way_sizes(sc);
486                 pl310_wbinv_all();
487                 platform_pl310_write_ctrl(sc, CTRL_DISABLED);
488                 platform_pl310_write_debug(sc, debug_ctrl);
489         }
490         sc->sc_enabled = pl310_enabled;
491
492         if (sc->sc_enabled) {
493                 platform_pl310_init(sc);
494                 pl310_set_way_sizes(sc); /* platform init might change these */
495                 pl310_write4(pl310_softc, PL310_INV_WAY, 0xffff);
496                 pl310_wait_background_op(PL310_INV_WAY, 0xffff);
497                 platform_pl310_write_ctrl(sc, CTRL_ENABLED);
498                 device_printf(dev, "L2 Cache enabled: %uKB/%dB %d ways\n", 
499                     (g_l2cache_size / 1024), g_l2cache_line_size, g_ways_assoc);
500                 if (bootverbose)
501                         pl310_print_config(sc);
502         } else {
503                 malloc(sizeof(*sc->sc_ich), M_DEVBUF, M_WAITOK);
504                 sc->sc_ich->ich_func = pl310_config_intr;
505                 sc->sc_ich->ich_arg = sc;
506                 if (config_intrhook_establish(sc->sc_ich) != 0) {
507                         device_printf(dev,
508                             "config_intrhook_establish failed\n");
509                         return(ENXIO);
510                 }
511                 device_printf(dev, "L2 Cache disabled\n");
512         }
513
514         /* Set the l2 functions in the set of cpufuncs */
515         cpufuncs.cf_l2cache_wbinv_all = pl310_wbinv_all;
516         cpufuncs.cf_l2cache_wbinv_range = pl310_wbinv_range;
517         cpufuncs.cf_l2cache_inv_range = pl310_inv_range;
518         cpufuncs.cf_l2cache_wb_range = pl310_wb_range;
519         cpufuncs.cf_l2cache_drain_writebuf = pl310_drain_writebuf;
520
521         return (0);
522 }
523
524 static device_method_t pl310_methods[] = {
525         DEVMETHOD(device_probe, pl310_probe),
526         DEVMETHOD(device_attach, pl310_attach),
527         DEVMETHOD_END
528 };
529
530 static driver_t pl310_driver = {
531         "l2cache",
532         pl310_methods,
533         sizeof(struct pl310_softc),
534 };
535 static devclass_t pl310_devclass;
536
537 EARLY_DRIVER_MODULE(pl310, simplebus, pl310_driver, pl310_devclass, 0, 0,
538     BUS_PASS_CPU + BUS_PASS_ORDER_MIDDLE);
539