1 /******************************************************************************
3 © 1995-2003, 2004, 2005-2011 Freescale Semiconductor, Inc.
6 This is proprietary source code of Freescale Semiconductor Inc.,
7 and its use is subject to the NetComm Device Drivers EULA.
8 The copyright notice above does not evidence any actual or intended
9 publication of such source code.
11 ALTERNATIVELY, redistribution and use in source and binary forms, with
12 or without modification, are permitted provided that the following
14 * Redistributions of source code must retain the above copyright
15 notice, this list of conditions and the following disclaimer.
16 * Redistributions in binary form must reproduce the above copyright
17 notice, this list of conditions and the following disclaimer in the
18 documentation and/or other materials provided with the distribution.
19 * Neither the name of Freescale Semiconductor nor the
20 names of its contributors may be used to endorse or promote products
21 derived from this software without specific prior written permission.
23 THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
24 EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
25 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
26 DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
27 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
28 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
29 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
30 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
32 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 **************************************************************************/
36 /******************************************************************************
39 @Description BM low-level implementation
40 *//***************************************************************************/
41 #include <sys/cdefs.h>
42 #include <sys/types.h>
43 #include <machine/atomic.h>
48 #include "error_ext.h"
50 #include "bman_private.h"
53 /***************************/
54 /* Portal register assists */
55 /***************************/
57 /* Cache-inhibited register offsets */
58 #define REG_RCR_PI_CINH 0x0000
59 #define REG_RCR_CI_CINH 0x0004
60 #define REG_RCR_ITR 0x0008
61 #define REG_CFG 0x0100
62 #define REG_SCN(n) (0x0200 + ((n) << 2))
63 #define REG_ISR 0x0e00
64 #define REG_IER 0x0e04
65 #define REG_ISDR 0x0e08
66 #define REG_IIR 0x0e0c
68 /* Cache-enabled register offsets */
73 #define CL_RCR_PI_CENA 0x3000
74 #define CL_RCR_CI_CENA 0x3100
76 /* The h/w design requires mappings to be size-aligned so that "add"s can be
77 * reduced to "or"s. The primitives below do the same for s/w. */
79 static __inline__ void *ptr_ADD(void *a, uintptr_t b)
81 return (void *)((uintptr_t)a + b);
84 /* Bitwise-OR two pointers */
85 static __inline__ void *ptr_OR(void *a, uintptr_t b)
87 return (void *)((uintptr_t)a | b);
90 /* Cache-inhibited register access */
91 static __inline__ uint32_t __bm_in(struct bm_addr *bm, uintptr_t offset)
93 uint32_t *tmp = (uint32_t *)ptr_ADD(bm->addr_ci, offset);
94 return GET_UINT32(*tmp);
96 static __inline__ void __bm_out(struct bm_addr *bm, uintptr_t offset, uint32_t val)
98 uint32_t *tmp = (uint32_t *)ptr_ADD(bm->addr_ci, offset);
99 WRITE_UINT32(*tmp, val);
101 #define bm_in(reg) __bm_in(&portal->addr, REG_##reg)
102 #define bm_out(reg, val) __bm_out(&portal->addr, REG_##reg, val)
104 /* Convert 'n' cachelines to a pointer value for bitwise OR */
105 #define bm_cl(n) (void *)((n) << 6)
107 /* Cache-enabled (index) register access */
108 static __inline__ void __bm_cl_touch_ro(struct bm_addr *bm, uintptr_t offset)
110 dcbt_ro(ptr_ADD(bm->addr_ce, offset));
112 static __inline__ void __bm_cl_touch_rw(struct bm_addr *bm, uintptr_t offset)
114 dcbt_rw(ptr_ADD(bm->addr_ce, offset));
116 static __inline__ uint32_t __bm_cl_in(struct bm_addr *bm, uintptr_t offset)
118 uint32_t *tmp = (uint32_t *)ptr_ADD(bm->addr_ce, offset);
119 return GET_UINT32(*tmp);
121 static __inline__ void __bm_cl_out(struct bm_addr *bm, uintptr_t offset, uint32_t val)
123 uint32_t *tmp = (uint32_t *)ptr_ADD(bm->addr_ce, offset);
124 WRITE_UINT32(*tmp, val);
127 static __inline__ void __bm_cl_invalidate(struct bm_addr *bm, uintptr_t offset)
129 dcbi(ptr_ADD(bm->addr_ce, offset));
131 #define bm_cl_touch_ro(reg) __bm_cl_touch_ro(&portal->addr, CL_##reg##_CENA)
132 #define bm_cl_touch_rw(reg) __bm_cl_touch_rw(&portal->addr, CL_##reg##_CENA)
133 #define bm_cl_in(reg) __bm_cl_in(&portal->addr, CL_##reg##_CENA)
134 #define bm_cl_out(reg, val) __bm_cl_out(&portal->addr, CL_##reg##_CENA, val)
135 #define bm_cl_invalidate(reg) __bm_cl_invalidate(&portal->addr, CL_##reg##_CENA)
137 /* Cyclic helper for rings. TODO: once we are able to do fine-grain perf
138 * analysis, look at using the "extra" bit in the ring index registers to avoid
140 static __inline__ uint8_t cyc_diff(uint8_t ringsize, uint8_t first, uint8_t last)
142 /* 'first' is included, 'last' is excluded */
144 return (uint8_t)(last - first);
145 return (uint8_t)(ringsize + last - first);
148 /* --------------- */
149 /* --- RCR API --- */
151 /* It's safer to code in terms of the 'rcr' object than the 'portal' object,
152 * because the latter runs the risk of copy-n-paste errors from other code where
153 * we could manipulate some other structure within 'portal'. */
154 /* #define RCR_API_START() register struct bm_rcr *rcr = &portal->rcr */
156 /* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
157 #define RCR_CARRYCLEAR(p) \
158 (void *)((uintptr_t)(p) & (~(uintptr_t)(BM_RCR_SIZE << 6)))
160 /* Bit-wise logic to convert a ring pointer to a ring index */
161 static __inline__ uint8_t RCR_PTR2IDX(struct bm_rcr_entry *e)
163 return (uint8_t)(((uintptr_t)e >> 6) & (BM_RCR_SIZE - 1));
166 /* Increment the 'cursor' ring pointer, taking 'vbit' into account */
167 static __inline__ void RCR_INC(struct bm_rcr *rcr)
169 /* NB: this is odd-looking, but experiments show that it generates
170 * fast code with essentially no branching overheads. We increment to
171 * the next RCR pointer and handle overflow and 'vbit'. */
172 struct bm_rcr_entry *partial = rcr->cursor + 1;
173 rcr->cursor = RCR_CARRYCLEAR(partial);
174 if (partial != rcr->cursor)
175 rcr->vbit ^= BM_RCR_VERB_VBIT;
178 t_Error bm_rcr_init(struct bm_portal *portal,
179 e_BmPortalProduceMode pmode,
180 e_BmPortalRcrConsumeMode cmode)
182 register struct bm_rcr *rcr = &portal->rcr;
186 rcr->ring = ptr_ADD(portal->addr.addr_ce, CL_RCR);
187 rcr->ci = (uint8_t)(bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1));
188 pi = (uint8_t)(bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1));
189 rcr->cursor = rcr->ring + pi;
190 rcr->vbit = (uint8_t)((bm_in(RCR_PI_CINH) & BM_RCR_SIZE) ? BM_RCR_VERB_VBIT : 0);
191 rcr->available = (uint8_t)(BM_RCR_SIZE - 1 - cyc_diff(BM_RCR_SIZE, rcr->ci, pi));
192 rcr->ithresh = (uint8_t)bm_in(RCR_ITR);
199 #endif /* BM_CHECKING */
200 cfg = (bm_in(CFG) & 0xffffffe0) | (pmode & 0x3); /* BCSP_CFG::RPM */
205 void bm_rcr_finish(struct bm_portal *portal)
207 register struct bm_rcr *rcr = &portal->rcr;
208 uint8_t pi = (uint8_t)(bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1));
209 uint8_t ci = (uint8_t)(bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1));
210 ASSERT_COND(!rcr->busy);
211 if (pi != RCR_PTR2IDX(rcr->cursor))
212 REPORT_ERROR(WARNING, E_INVALID_STATE, ("losing uncommitted RCR entries"));
214 REPORT_ERROR(WARNING, E_INVALID_STATE, ("missing existing RCR completions"));
215 if (rcr->ci != RCR_PTR2IDX(rcr->cursor))
216 REPORT_ERROR(WARNING, E_INVALID_STATE, ("RCR destroyed unquiesced"));
219 struct bm_rcr_entry *bm_rcr_start(struct bm_portal *portal)
221 register struct bm_rcr *rcr = &portal->rcr;
222 ASSERT_COND(!rcr->busy);
227 #endif /* BM_CHECKING */
228 dcbz_64(rcr->cursor);
232 void bm_rcr_abort(struct bm_portal *portal)
234 register struct bm_rcr *rcr = &portal->rcr;
235 ASSERT_COND(rcr->busy);
240 #endif /* BM_CHECKING */
243 struct bm_rcr_entry *bm_rcr_pend_and_next(struct bm_portal *portal, uint8_t myverb)
245 register struct bm_rcr *rcr = &portal->rcr;
246 ASSERT_COND(rcr->busy);
247 ASSERT_COND(rcr->pmode != e_BmPortalPVB);
248 if (rcr->available == 1)
250 rcr->cursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit);
251 dcbf_64(rcr->cursor);
254 dcbz_64(rcr->cursor);
258 void bm_rcr_pci_commit(struct bm_portal *portal, uint8_t myverb)
260 register struct bm_rcr *rcr = &portal->rcr;
261 ASSERT_COND(rcr->busy);
262 ASSERT_COND(rcr->pmode == e_BmPortalPCI);
263 rcr->cursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit);
267 bm_out(RCR_PI_CINH, RCR_PTR2IDX(rcr->cursor));
270 #endif /* BM_CHECKING */
273 void bm_rcr_pce_prefetch(struct bm_portal *portal)
275 ASSERT_COND(((struct bm_rcr *)&portal->rcr)->pmode == e_BmPortalPCE);
276 bm_cl_invalidate(RCR_PI);
277 bm_cl_touch_rw(RCR_PI);
280 void bm_rcr_pce_commit(struct bm_portal *portal, uint8_t myverb)
282 register struct bm_rcr *rcr = &portal->rcr;
283 ASSERT_COND(rcr->busy);
284 ASSERT_COND(rcr->pmode == e_BmPortalPCE);
285 rcr->cursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit);
289 bm_cl_out(RCR_PI, RCR_PTR2IDX(rcr->cursor));
292 #endif /* BM_CHECKING */
295 void bm_rcr_pvb_commit(struct bm_portal *portal, uint8_t myverb)
297 register struct bm_rcr *rcr = &portal->rcr;
298 struct bm_rcr_entry *rcursor;
299 ASSERT_COND(rcr->busy);
300 ASSERT_COND(rcr->pmode == e_BmPortalPVB);
302 rcursor = rcr->cursor;
303 rcursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit);
309 #endif /* BM_CHECKING */
313 uint8_t bm_rcr_cci_update(struct bm_portal *portal)
315 register struct bm_rcr *rcr = &portal->rcr;
316 uint8_t diff, old_ci = rcr->ci;
317 ASSERT_COND(rcr->cmode == e_BmPortalRcrCCI);
318 rcr->ci = (uint8_t)(bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1));
319 diff = cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
320 rcr->available += diff;
325 void bm_rcr_cce_prefetch(struct bm_portal *portal)
327 ASSERT_COND(((struct bm_rcr *)&portal->rcr)->cmode == e_BmPortalRcrCCE);
328 bm_cl_touch_ro(RCR_CI);
332 uint8_t bm_rcr_cce_update(struct bm_portal *portal)
334 register struct bm_rcr *rcr = &portal->rcr;
335 uint8_t diff, old_ci = rcr->ci;
336 ASSERT_COND(rcr->cmode == e_BmPortalRcrCCE);
337 rcr->ci = (uint8_t)(bm_cl_in(RCR_CI) & (BM_RCR_SIZE - 1));
338 bm_cl_invalidate(RCR_CI);
339 diff = cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
340 rcr->available += diff;
345 uint8_t bm_rcr_get_ithresh(struct bm_portal *portal)
347 register struct bm_rcr *rcr = &portal->rcr;
352 void bm_rcr_set_ithresh(struct bm_portal *portal, uint8_t ithresh)
354 register struct bm_rcr *rcr = &portal->rcr;
355 rcr->ithresh = ithresh;
356 bm_out(RCR_ITR, ithresh);
360 uint8_t bm_rcr_get_avail(struct bm_portal *portal)
362 register struct bm_rcr *rcr = &portal->rcr;
363 return rcr->available;
367 uint8_t bm_rcr_get_fill(struct bm_portal *portal)
369 register struct bm_rcr *rcr = &portal->rcr;
370 return (uint8_t)(BM_RCR_SIZE - 1 - rcr->available);
374 /* ------------------------------ */
375 /* --- Management command API --- */
377 /* It's safer to code in terms of the 'mc' object than the 'portal' object,
378 * because the latter runs the risk of copy-n-paste errors from other code where
379 * we could manipulate some other structure within 'portal'. */
380 /* #define MC_API_START() register struct bm_mc *mc = &portal->mc */
383 t_Error bm_mc_init(struct bm_portal *portal)
385 register struct bm_mc *mc = &portal->mc;
386 mc->cr = ptr_ADD(portal->addr.addr_ce, CL_CR);
387 mc->rr = ptr_ADD(portal->addr.addr_ce, CL_RR0);
388 mc->rridx = (uint8_t)((mc->cr->__dont_write_directly__verb & BM_MCC_VERB_VBIT) ?
390 mc->vbit = (uint8_t)(mc->rridx ? BM_MCC_VERB_VBIT : 0);
393 #endif /* BM_CHECKING */
398 void bm_mc_finish(struct bm_portal *portal)
400 register struct bm_mc *mc = &portal->mc;
401 ASSERT_COND(mc->state == mc_idle);
403 if (mc->state != mc_idle)
404 REPORT_ERROR(WARNING, E_INVALID_STATE, ("Losing incomplete MC command"));
407 #endif /* BM_CHECKING */
411 struct bm_mc_command *bm_mc_start(struct bm_portal *portal)
413 register struct bm_mc *mc = &portal->mc;
414 ASSERT_COND(mc->state == mc_idle);
417 #endif /* BM_CHECKING */
423 void bm_mc_abort(struct bm_portal *portal)
425 register struct bm_mc *mc = &portal->mc;
426 ASSERT_COND(mc->state == mc_user);
431 #endif /* BM_CHECKING */
435 void bm_mc_commit(struct bm_portal *portal, uint8_t myverb)
437 register struct bm_mc *mc = &portal->mc;
438 ASSERT_COND(mc->state == mc_user);
440 mc->cr->__dont_write_directly__verb = (uint8_t)(myverb | mc->vbit);
442 dcbit_ro(mc->rr + mc->rridx);
445 #endif /* BM_CHECKING */
449 struct bm_mc_result *bm_mc_result(struct bm_portal *portal)
451 register struct bm_mc *mc = &portal->mc;
452 struct bm_mc_result *rr = mc->rr + mc->rridx;
453 ASSERT_COND(mc->state == mc_hw);
454 /* The inactive response register's verb byte always returns zero until
455 * its command is submitted and completed. This includes the valid-bit,
456 * in case you were wondering... */
462 mc->vbit ^= BM_MCC_VERB_VBIT;
465 #endif /* BM_CHECKING */
469 /* ------------------------------------- */
470 /* --- Portal interrupt register API --- */
472 #define SCN_REG(bpid) REG_SCN((bpid) / 32)
473 #define SCN_BIT(bpid) (0x80000000 >> (bpid & 31))
474 void bm_isr_bscn_mask(struct bm_portal *portal, uint8_t bpid, int enable)
477 ASSERT_COND(bpid < BM_MAX_NUM_OF_POOLS);
478 /* REG_SCN for bpid=0..31, REG_SCN+4 for bpid=32..63 */
479 val = __bm_in(&portal->addr, SCN_REG(bpid));
481 val |= SCN_BIT(bpid);
483 val &= ~SCN_BIT(bpid);
484 __bm_out(&portal->addr, SCN_REG(bpid), val);
488 uint32_t __bm_isr_read(struct bm_portal *portal, enum bm_isr_reg n)
490 return __bm_in(&portal->addr, REG_ISR + (n << 2));
494 void __bm_isr_write(struct bm_portal *portal, enum bm_isr_reg n, uint32_t val)
496 __bm_out(&portal->addr, REG_ISR + (n << 2), val);