]> CyberLeo.Net >> Repos - FreeBSD/FreeBSD.git/blob - sys/arm/arm/pl310.c
Merge llvm, clang, lld, lldb, compiler-rt and libc++ r304222, and update
[FreeBSD/FreeBSD.git] / sys / arm / arm / pl310.c
1 /*-
2  * Copyright (c) 2012 Olivier Houchard <cognet@FreeBSD.org>
3  * Copyright (c) 2011
4  *      Ben Gray <ben.r.gray@gmail.com>.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  * 3. The name of the company nor the name of the author may be used to
16  *    endorse or promote products derived from this software without specific
17  *    prior written permission.
18  *
19  * THIS SOFTWARE IS PROVIDED BY BEN GRAY ``AS IS'' AND ANY EXPRESS OR
20  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22  * IN NO EVENT SHALL BEN GRAY BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
24  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
25  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
26  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
27  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
28  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29  */
30
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/bus.h>
36 #include <sys/kernel.h>
37 #include <sys/rman.h>
38 #include <sys/module.h>
39 #include <sys/lock.h>
40 #include <sys/mutex.h>
41 #include <machine/intr.h>
42
43 #include <machine/bus.h>
44 #include <machine/pl310.h>
45
46 #include <dev/ofw/openfirm.h>
47 #include <dev/ofw/ofw_bus.h>
48 #include <dev/ofw/ofw_bus_subr.h>
49
50 /*
51  * Define this if you need to disable PL310 for debugging purpose
52  * Spec:
53  * http://infocenter.arm.com/help/topic/com.arm.doc.ddi0246e/DDI0246E_l2c310_r3p1_trm.pdf
54  */
55
56 /*
57  * Hardcode errata for now
58  * http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ddi0246b/pr01s02s02.html
59  */
60 #define PL310_ERRATA_588369
61 #define PL310_ERRATA_753970
62 #define PL310_ERRATA_727915
63
64 #define PL310_LOCK(sc) do {             \
65         mtx_lock_spin(&(sc)->sc_mtx);   \
66 } while(0);
67
68 #define PL310_UNLOCK(sc) do {           \
69         mtx_unlock_spin(&(sc)->sc_mtx); \
70 } while(0);
71
72 static int pl310_enabled = 1;
73 TUNABLE_INT("hw.pl310.enabled", &pl310_enabled);
74
75 static uint32_t g_l2cache_way_mask;
76
77 static const uint32_t g_l2cache_line_size = 32;
78 static const uint32_t g_l2cache_align_mask = (32 - 1);
79
80 static uint32_t g_l2cache_size;
81 static uint32_t g_way_size;
82 static uint32_t g_ways_assoc;
83
84 static struct pl310_softc *pl310_softc;
85
86 static struct ofw_compat_data compat_data[] = {
87         {"arm,pl310",           true}, /* Non-standard, FreeBSD. */
88         {"arm,pl310-cache",     true},
89         {NULL,                  false}
90 };
91
92 static void
93 pl310_print_config(struct pl310_softc *sc)
94 {
95         uint32_t aux, prefetch;
96         const char *dis = "disabled";
97         const char *ena = "enabled";
98
99         aux = pl310_read4(sc, PL310_AUX_CTRL);
100         prefetch = pl310_read4(sc, PL310_PREFETCH_CTRL);
101
102         device_printf(sc->sc_dev, "Early BRESP response: %s\n",
103                 (aux & AUX_CTRL_EARLY_BRESP) ? ena : dis);
104         device_printf(sc->sc_dev, "Instruction prefetch: %s\n",
105                 (aux & AUX_CTRL_INSTR_PREFETCH) ? ena : dis);
106         device_printf(sc->sc_dev, "Data prefetch: %s\n",
107                 (aux & AUX_CTRL_DATA_PREFETCH) ? ena : dis);
108         device_printf(sc->sc_dev, "Non-secure interrupt control: %s\n",
109                 (aux & AUX_CTRL_NS_INT_CTRL) ? ena : dis);
110         device_printf(sc->sc_dev, "Non-secure lockdown: %s\n",
111                 (aux & AUX_CTRL_NS_LOCKDOWN) ? ena : dis);
112         device_printf(sc->sc_dev, "Share override: %s\n",
113                 (aux & AUX_CTRL_SHARE_OVERRIDE) ? ena : dis);
114
115         device_printf(sc->sc_dev, "Double linefill: %s\n",
116                 (prefetch & PREFETCH_CTRL_DL) ? ena : dis);
117         device_printf(sc->sc_dev, "Instruction prefetch: %s\n",
118                 (prefetch & PREFETCH_CTRL_INSTR_PREFETCH) ? ena : dis);
119         device_printf(sc->sc_dev, "Data prefetch: %s\n",
120                 (prefetch & PREFETCH_CTRL_DATA_PREFETCH) ? ena : dis);
121         device_printf(sc->sc_dev, "Double linefill on WRAP request: %s\n",
122                 (prefetch & PREFETCH_CTRL_DL_ON_WRAP) ? ena : dis);
123         device_printf(sc->sc_dev, "Prefetch drop: %s\n",
124                 (prefetch & PREFETCH_CTRL_PREFETCH_DROP) ? ena : dis);
125         device_printf(sc->sc_dev, "Incr double Linefill: %s\n",
126                 (prefetch & PREFETCH_CTRL_INCR_DL) ? ena : dis);
127         device_printf(sc->sc_dev, "Not same ID on exclusive sequence: %s\n",
128                 (prefetch & PREFETCH_CTRL_NOTSAMEID) ? ena : dis);
129         device_printf(sc->sc_dev, "Prefetch offset: %d\n",
130                 (prefetch & PREFETCH_CTRL_OFFSET_MASK));
131 }
132
133 void
134 pl310_set_ram_latency(struct pl310_softc *sc, uint32_t which_reg,
135    uint32_t read, uint32_t write, uint32_t setup)
136 {
137         uint32_t v;
138
139         KASSERT(which_reg == PL310_TAG_RAM_CTRL ||
140             which_reg == PL310_DATA_RAM_CTRL,
141             ("bad pl310 ram latency register address"));
142
143         v = pl310_read4(sc, which_reg);
144         if (setup != 0) {
145                 KASSERT(setup <= 8, ("bad pl310 setup latency: %d", setup));
146                 v &= ~RAM_CTRL_SETUP_MASK;
147                 v |= (setup - 1) << RAM_CTRL_SETUP_SHIFT;
148         }
149         if (read != 0) {
150                 KASSERT(read <= 8, ("bad pl310 read latency: %d", read));
151                 v &= ~RAM_CTRL_READ_MASK;
152                 v |= (read - 1) << RAM_CTRL_READ_SHIFT;
153         }
154         if (write != 0) {
155                 KASSERT(write <= 8, ("bad pl310 write latency: %d", write));
156                 v &= ~RAM_CTRL_WRITE_MASK;
157                 v |= (write - 1) << RAM_CTRL_WRITE_SHIFT;
158         }
159         pl310_write4(sc, which_reg, v);
160 }
161
162 static int
163 pl310_filter(void *arg)
164 {
165         struct pl310_softc *sc = arg;
166         uint32_t intr;
167
168         intr = pl310_read4(sc, PL310_INTR_MASK);
169
170         if (!sc->sc_enabled && (intr & INTR_MASK_ECNTR)) {
171                 /*
172                  * This is for debug purpose, so be blunt about it
173                  * We disable PL310 only when something fishy is going
174                  * on and we need to make sure L2 cache is 100% disabled
175                  */
176                 panic("pl310: caches disabled but cache event detected\n");
177         }
178
179         return (FILTER_HANDLED);
180 }
181
182 static __inline void
183 pl310_wait_background_op(uint32_t off, uint32_t mask)
184 {
185
186         while (pl310_read4(pl310_softc, off) & mask)
187                 continue;
188 }
189
190
191 /**
192  *      pl310_cache_sync - performs a cache sync operation
193  *
194  *      According to the TRM:
195  *
196  *  "Before writing to any other register you must perform an explicit
197  *   Cache Sync operation. This is particularly important when the cache is
198  *   enabled and changes to how the cache allocates new lines are to be made."
199  *
200  *
201  */
202 static __inline void
203 pl310_cache_sync(void)
204 {
205
206         if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
207                 return;
208
209 #ifdef PL310_ERRATA_753970
210         if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r3p0)
211                 /* Write uncached PL310 register */
212                 pl310_write4(pl310_softc, 0x740, 0xffffffff);
213         else
214 #endif
215                 pl310_write4(pl310_softc, PL310_CACHE_SYNC, 0xffffffff);
216 }
217
218
219 static void
220 pl310_wbinv_all(void)
221 {
222
223         if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
224                 return;
225
226         PL310_LOCK(pl310_softc);
227 #ifdef PL310_ERRATA_727915
228         if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r2p0) {
229                 int i, j;
230
231                 for (i = 0; i < g_ways_assoc; i++) {
232                         for (j = 0; j < g_way_size / g_l2cache_line_size; j++) {
233                                 pl310_write4(pl310_softc,
234                                     PL310_CLEAN_INV_LINE_IDX,
235                                     (i << 28 | j << 5));
236                         }
237                 }
238                 pl310_cache_sync();
239                 PL310_UNLOCK(pl310_softc);
240                 return;
241
242         }
243         if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r3p0)
244                 platform_pl310_write_debug(pl310_softc, 3);
245 #endif
246         pl310_write4(pl310_softc, PL310_CLEAN_INV_WAY, g_l2cache_way_mask);
247         pl310_wait_background_op(PL310_CLEAN_INV_WAY, g_l2cache_way_mask);
248         pl310_cache_sync();
249 #ifdef PL310_ERRATA_727915
250         if (pl310_softc->sc_rtl_revision == CACHE_ID_RELEASE_r3p0)
251                 platform_pl310_write_debug(pl310_softc, 0);
252 #endif
253         PL310_UNLOCK(pl310_softc);
254 }
255
256 static void
257 pl310_wbinv_range(vm_paddr_t start, vm_size_t size)
258 {
259
260         if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
261                 return;
262
263         PL310_LOCK(pl310_softc);
264         if (start & g_l2cache_align_mask) {
265                 size += start & g_l2cache_align_mask;
266                 start &= ~g_l2cache_align_mask;
267         }
268         if (size & g_l2cache_align_mask) {
269                 size &= ~g_l2cache_align_mask;
270                 size += g_l2cache_line_size;
271         }
272
273
274 #ifdef PL310_ERRATA_727915
275         if (pl310_softc->sc_rtl_revision >= CACHE_ID_RELEASE_r2p0 &&
276             pl310_softc->sc_rtl_revision < CACHE_ID_RELEASE_r3p1)
277                 platform_pl310_write_debug(pl310_softc, 3);
278 #endif
279         while (size > 0) {
280 #ifdef PL310_ERRATA_588369
281                 if (pl310_softc->sc_rtl_revision <= CACHE_ID_RELEASE_r1p0) {
282                         /*
283                          * Errata 588369 says that clean + inv may keep the
284                          * cache line if it was clean, the recommanded
285                          * workaround is to clean then invalidate the cache
286                          * line, with write-back and cache linefill disabled.
287                          */
288                         pl310_write4(pl310_softc, PL310_CLEAN_LINE_PA, start);
289                         pl310_write4(pl310_softc, PL310_INV_LINE_PA, start);
290                 } else
291 #endif
292                         pl310_write4(pl310_softc, PL310_CLEAN_INV_LINE_PA,
293                             start);
294                 start += g_l2cache_line_size;
295                 size -= g_l2cache_line_size;
296         }
297 #ifdef PL310_ERRATA_727915
298         if (pl310_softc->sc_rtl_revision >= CACHE_ID_RELEASE_r2p0 &&
299             pl310_softc->sc_rtl_revision < CACHE_ID_RELEASE_r3p1)
300                 platform_pl310_write_debug(pl310_softc, 0);
301 #endif
302
303         pl310_cache_sync();
304         PL310_UNLOCK(pl310_softc);
305 }
306
307 static void
308 pl310_wb_range(vm_paddr_t start, vm_size_t size)
309 {
310
311         if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
312                 return;
313
314         PL310_LOCK(pl310_softc);
315         if (start & g_l2cache_align_mask) {
316                 size += start & g_l2cache_align_mask;
317                 start &= ~g_l2cache_align_mask;
318         }
319
320         if (size & g_l2cache_align_mask) {
321                 size &= ~g_l2cache_align_mask;
322                 size += g_l2cache_line_size;
323         }
324
325         while (size > 0) {
326                 pl310_write4(pl310_softc, PL310_CLEAN_LINE_PA, start);
327                 start += g_l2cache_line_size;
328                 size -= g_l2cache_line_size;
329         }
330
331         pl310_cache_sync();
332         PL310_UNLOCK(pl310_softc);
333 }
334
335 static void
336 pl310_inv_range(vm_paddr_t start, vm_size_t size)
337 {
338
339         if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
340                 return;
341
342         PL310_LOCK(pl310_softc);
343         if (start & g_l2cache_align_mask) {
344                 size += start & g_l2cache_align_mask;
345                 start &= ~g_l2cache_align_mask;
346         }
347         if (size & g_l2cache_align_mask) {
348                 size &= ~g_l2cache_align_mask;
349                 size += g_l2cache_line_size;
350         }
351         while (size > 0) {
352                 pl310_write4(pl310_softc, PL310_INV_LINE_PA, start);
353                 start += g_l2cache_line_size;
354                 size -= g_l2cache_line_size;
355         }
356
357         pl310_cache_sync();
358         PL310_UNLOCK(pl310_softc);
359 }
360
361 static void
362 pl310_drain_writebuf(void)
363 {
364
365         if ((pl310_softc == NULL) || !pl310_softc->sc_enabled)
366                 return;
367
368         PL310_LOCK(pl310_softc);
369         pl310_cache_sync();
370         PL310_UNLOCK(pl310_softc);
371 }
372
373 static void
374 pl310_set_way_sizes(struct pl310_softc *sc)
375 {
376         uint32_t aux_value;
377
378         aux_value = pl310_read4(sc, PL310_AUX_CTRL);
379         g_way_size = (aux_value & AUX_CTRL_WAY_SIZE_MASK) >>
380             AUX_CTRL_WAY_SIZE_SHIFT;
381         g_way_size = 1 << (g_way_size + 13);
382         if (aux_value & (1 << AUX_CTRL_ASSOCIATIVITY_SHIFT))
383                 g_ways_assoc = 16;
384         else
385                 g_ways_assoc = 8;
386         g_l2cache_way_mask = (1 << g_ways_assoc) - 1;
387         g_l2cache_size = g_way_size * g_ways_assoc;
388 }
389
390 /*
391  * Setup interrupt handling.  This is done only if the cache controller is
392  * disabled, for debugging.  We set counters so when a cache event happens we'll
393  * get interrupted and be warned that something is wrong, because no cache
394  * events should happen if we're disabled.
395  */
396 static void
397 pl310_config_intr(void *arg)
398 {
399         struct pl310_softc * sc;
400
401         sc = arg;
402
403         /* activate the interrupt */
404         bus_setup_intr(sc->sc_dev, sc->sc_irq_res, INTR_TYPE_MISC | INTR_MPSAFE,
405             pl310_filter, NULL, sc, &sc->sc_irq_h);
406
407         /* Cache Line Eviction for Counter 0 */
408         pl310_write4(sc, PL310_EVENT_COUNTER0_CONF,
409             EVENT_COUNTER_CONF_INCR | EVENT_COUNTER_CONF_CO);
410         /* Data Read Request for Counter 1 */
411         pl310_write4(sc, PL310_EVENT_COUNTER1_CONF,
412             EVENT_COUNTER_CONF_INCR | EVENT_COUNTER_CONF_DRREQ);
413
414         /* Enable and clear pending interrupts */
415         pl310_write4(sc, PL310_INTR_CLEAR, INTR_MASK_ECNTR);
416         pl310_write4(sc, PL310_INTR_MASK, INTR_MASK_ALL);
417
418         /* Enable counters and reset C0 and C1 */
419         pl310_write4(sc, PL310_EVENT_COUNTER_CTRL,
420             EVENT_COUNTER_CTRL_ENABLED |
421             EVENT_COUNTER_CTRL_C0_RESET |
422             EVENT_COUNTER_CTRL_C1_RESET);
423
424         config_intrhook_disestablish(sc->sc_ich);
425         free(sc->sc_ich, M_DEVBUF);
426         sc->sc_ich = NULL;
427 }
428
429 static int
430 pl310_probe(device_t dev)
431 {
432
433         if (!ofw_bus_status_okay(dev))
434                 return (ENXIO);
435         if (!ofw_bus_search_compatible(dev, compat_data)->ocd_data)
436                 return (ENXIO);
437         device_set_desc(dev, "PL310 L2 cache controller");
438         return (0);
439 }
440
441 static int
442 pl310_attach(device_t dev)
443 {
444         struct pl310_softc *sc = device_get_softc(dev);
445         int rid;
446         uint32_t cache_id, debug_ctrl;
447
448         sc->sc_dev = dev;
449         rid = 0;
450         sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
451             RF_ACTIVE);
452         if (sc->sc_mem_res == NULL)
453                 panic("%s: Cannot map registers", device_get_name(dev));
454
455         /* Allocate an IRQ resource */
456         rid = 0;
457         sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
458                                                 RF_ACTIVE | RF_SHAREABLE);
459         if (sc->sc_irq_res == NULL) {
460                 device_printf(dev, "cannot allocate IRQ, not using interrupt\n");
461         }
462
463         pl310_softc = sc;
464         mtx_init(&sc->sc_mtx, "pl310lock", NULL, MTX_SPIN);
465
466         cache_id = pl310_read4(sc, PL310_CACHE_ID);
467         sc->sc_rtl_revision = (cache_id >> CACHE_ID_RELEASE_SHIFT) &
468             CACHE_ID_RELEASE_MASK;
469         device_printf(dev, "Part number: 0x%x, release: 0x%x\n",
470             (cache_id >> CACHE_ID_PARTNUM_SHIFT) & CACHE_ID_PARTNUM_MASK,
471             (cache_id >> CACHE_ID_RELEASE_SHIFT) & CACHE_ID_RELEASE_MASK);
472
473         /*
474          * If L2 cache is already enabled then something has violated the rules,
475          * because caches are supposed to be off at kernel entry.  The cache
476          * must be disabled to write the configuration registers without
477          * triggering an access error (SLVERR), but there's no documented safe
478          * procedure for disabling the L2 cache in the manual.  So we'll try to
479          * invent one:
480          *  - Use the debug register to force write-through mode and prevent
481          *    linefills (allocation of new lines on read); now anything we do
482          *    will not cause new data to come into the L2 cache.
483          *  - Writeback and invalidate the current contents.
484          *  - Disable the controller.
485          *  - Restore the original debug settings.
486          */
487         if (pl310_read4(sc, PL310_CTRL) & CTRL_ENABLED) {
488                 device_printf(dev, "Warning: L2 Cache should not already be "
489                     "active; trying to de-activate and re-initialize...\n");
490                 sc->sc_enabled = 1;
491                 debug_ctrl = pl310_read4(sc, PL310_DEBUG_CTRL);
492                 platform_pl310_write_debug(sc, debug_ctrl |
493                     DEBUG_CTRL_DISABLE_WRITEBACK | DEBUG_CTRL_DISABLE_LINEFILL);
494                 pl310_set_way_sizes(sc);
495                 pl310_wbinv_all();
496                 platform_pl310_write_ctrl(sc, CTRL_DISABLED);
497                 platform_pl310_write_debug(sc, debug_ctrl);
498         }
499         sc->sc_enabled = pl310_enabled;
500
501         if (sc->sc_enabled) {
502                 platform_pl310_init(sc);
503                 pl310_set_way_sizes(sc); /* platform init might change these */
504                 pl310_write4(pl310_softc, PL310_INV_WAY, 0xffff);
505                 pl310_wait_background_op(PL310_INV_WAY, 0xffff);
506                 platform_pl310_write_ctrl(sc, CTRL_ENABLED);
507                 device_printf(dev, "L2 Cache enabled: %uKB/%dB %d ways\n",
508                     (g_l2cache_size / 1024), g_l2cache_line_size, g_ways_assoc);
509                 if (bootverbose)
510                         pl310_print_config(sc);
511         } else {
512                 if (sc->sc_irq_res != NULL) {
513                         sc->sc_ich = malloc(sizeof(*sc->sc_ich), M_DEVBUF, M_WAITOK);
514                         sc->sc_ich->ich_func = pl310_config_intr;
515                         sc->sc_ich->ich_arg = sc;
516                         if (config_intrhook_establish(sc->sc_ich) != 0) {
517                                 device_printf(dev,
518                                     "config_intrhook_establish failed\n");
519                                 free(sc->sc_ich, M_DEVBUF);
520                                 return(ENXIO);
521                         }
522                 }
523
524                 device_printf(dev, "L2 Cache disabled\n");
525         }
526
527         /* Set the l2 functions in the set of cpufuncs */
528         cpufuncs.cf_l2cache_wbinv_all = pl310_wbinv_all;
529         cpufuncs.cf_l2cache_wbinv_range = pl310_wbinv_range;
530         cpufuncs.cf_l2cache_inv_range = pl310_inv_range;
531         cpufuncs.cf_l2cache_wb_range = pl310_wb_range;
532         cpufuncs.cf_l2cache_drain_writebuf = pl310_drain_writebuf;
533
534         return (0);
535 }
536
537 static device_method_t pl310_methods[] = {
538         DEVMETHOD(device_probe, pl310_probe),
539         DEVMETHOD(device_attach, pl310_attach),
540         DEVMETHOD_END
541 };
542
543 static driver_t pl310_driver = {
544         "l2cache",
545         pl310_methods,
546         sizeof(struct pl310_softc),
547 };
548 static devclass_t pl310_devclass;
549
550 EARLY_DRIVER_MODULE(pl310, simplebus, pl310_driver, pl310_devclass, 0, 0,
551     BUS_PASS_CPU + BUS_PASS_ORDER_MIDDLE);
552