2 *******************************************************************************
3 Copyright (C) 2015 Annapurna Labs Ltd.
5 This file may be licensed under the terms of the Annapurna Labs Commercial
8 Alternatively, this file can be distributed under the terms of the GNU General
9 Public License V2 as published by the Free Software Foundation and can be
10 found at http://www.gnu.org/licenses/gpl-2.0.html
12 Alternatively, redistribution and use in source and binary forms, with or
13 without modification, are permitted provided that the following conditions are
16 * Redistributions of source code must retain the above copyright notice,
17 this list of conditions and the following disclaimer.
19 * Redistributions in binary form must reproduce the above copyright
20 notice, this list of conditions and the following disclaimer in
21 the documentation and/or other materials provided with the
24 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
25 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
26 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
27 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
28 ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
29 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
30 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
31 ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
32 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
33 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *******************************************************************************/
38 * @defgroup group_udma_api API
44 * @defgroup group_udma_main UDMA Main
45 * @ingroup group_udma_api
50 * @brief C Header file for the Universal DMA HAL driver
54 #ifndef __AL_HAL_UDMA_H__
55 #define __AL_HAL_UDMA_H__
57 #include "al_hal_common.h"
58 #include "al_hal_udma_regs.h"
67 #define AL_UDMA_MIN_Q_SIZE 4
68 #define AL_UDMA_MAX_Q_SIZE (1 << 16) /* hw can do more, but we limit it */
70 /* Default Max number of descriptors supported per action */
71 #define AL_UDMA_DEFAULT_MAX_ACTN_DESCS 16
73 #define AL_UDMA_REV_ID_1 1
74 #define AL_UDMA_REV_ID_2 2
76 #define DMA_RING_ID_MASK 0x3
77 /* New registers ?? */
78 /* Statistics - TBD */
80 /** UDMA submission descriptor */
88 /* TX Meta, used by upper layer */
103 /* TX desc length and control fields */
105 #define AL_M2S_DESC_CONCAT AL_BIT(31) /* concatenate */
106 #define AL_M2S_DESC_DMB AL_BIT(30)
107 /** Data Memory Barrier */
108 #define AL_M2S_DESC_NO_SNOOP_H AL_BIT(29)
109 #define AL_M2S_DESC_INT_EN AL_BIT(28) /** enable interrupt */
110 #define AL_M2S_DESC_LAST AL_BIT(27)
111 #define AL_M2S_DESC_FIRST AL_BIT(26)
112 #define AL_M2S_DESC_RING_ID_SHIFT 24
113 #define AL_M2S_DESC_RING_ID_MASK (0x3 << AL_M2S_DESC_RING_ID_SHIFT)
114 #define AL_M2S_DESC_META_DATA AL_BIT(23)
115 #define AL_M2S_DESC_DUMMY AL_BIT(22) /* for Metdata only */
116 #define AL_M2S_DESC_LEN_ADJ_SHIFT 20
117 #define AL_M2S_DESC_LEN_ADJ_MASK (0x7 << AL_M2S_DESC_LEN_ADJ_SHIFT)
118 #define AL_M2S_DESC_LEN_SHIFT 0
119 #define AL_M2S_DESC_LEN_MASK (0xfffff << AL_M2S_DESC_LEN_SHIFT)
121 #define AL_S2M_DESC_DUAL_BUF AL_BIT(31)
122 #define AL_S2M_DESC_NO_SNOOP_H AL_BIT(29)
123 #define AL_S2M_DESC_INT_EN AL_BIT(28) /** enable interrupt */
124 #define AL_S2M_DESC_RING_ID_SHIFT 24
125 #define AL_S2M_DESC_RING_ID_MASK (0x3 << AL_S2M_DESC_RING_ID_SHIFT)
126 #define AL_S2M_DESC_LEN_SHIFT 0
127 #define AL_S2M_DESC_LEN_MASK (0xffff << AL_S2M_DESC_LEN_SHIFT)
128 #define AL_S2M_DESC_LEN2_SHIFT 16
129 #define AL_S2M_DESC_LEN2_MASK (0x3fff << AL_S2M_DESC_LEN2_SHIFT)
130 #define AL_S2M_DESC_LEN2_GRANULARITY_SHIFT 6
132 /* TX/RX descriptor Target-ID field (in the buffer address 64 bit field) */
133 #define AL_UDMA_DESC_TGTID_SHIFT 48
135 /** UDMA completion descriptor */
136 union al_udma_cdesc {
148 /* TX/RX common completion desc ctrl_meta feilds */
149 #define AL_UDMA_CDESC_ERROR AL_BIT(31)
150 #define AL_UDMA_CDESC_BUF1_USED AL_BIT(30)
151 #define AL_UDMA_CDESC_DDP AL_BIT(29)
152 #define AL_UDMA_CDESC_LAST AL_BIT(27)
153 #define AL_UDMA_CDESC_FIRST AL_BIT(26)
155 #define AL_UDMA_CDESC_BUF2_USED AL_BIT(31)
156 #define AL_UDMA_CDESC_BUF2_LEN_SHIFT 16
157 #define AL_UDMA_CDESC_BUF2_LEN_MASK AL_FIELD_MASK(29, 16)
158 /** Basic Buffer structure */
160 al_phys_addr_t addr; /**< Buffer physical address */
161 uint32_t len; /**< Buffer lenght in bytes */
164 /** Block is a set of buffers that belong to same source or destination */
166 struct al_buf *bufs; /**< The buffers of the block */
167 uint32_t num; /**< Number of buffers of the block */
170 * Target-ID to be assigned to the block descriptors
171 * Requires Target-ID in descriptor to be enabled for the specific UDMA
192 extern const char *const al_udma_states_name[];
194 /** UDMA Q specific parameters from upper layer */
195 struct al_udma_q_params {
196 uint32_t size; /**< ring size (in descriptors), submission and
197 * completion rings must have same size
199 union al_udma_desc *desc_base; /**< cpu address for submission ring
202 al_phys_addr_t desc_phy_base; /**< submission ring descriptors
203 * physical base address
206 bus_dma_tag_t desc_phy_base_tag;
207 bus_dmamap_t desc_phy_base_map;
209 uint8_t *cdesc_base; /**< completion descriptors pointer, NULL */
210 /* means no completion update */
211 al_phys_addr_t cdesc_phy_base; /**< completion descriptors ring
212 * physical base address
215 bus_dma_tag_t cdesc_phy_base_tag;
216 bus_dmamap_t cdesc_phy_base_map;
218 uint32_t cdesc_size; /**< size (in bytes) of a single dma completion
222 uint8_t adapter_rev_id; /**<PCI adapter revision ID */
225 /** UDMA parameters from upper layer */
226 struct al_udma_params {
227 struct unit_regs __iomem *udma_regs_base;
228 enum al_udma_type type; /**< Tx or Rx */
229 uint8_t num_of_queues; /**< number of queues supported by the UDMA */
230 const char *name; /**< the upper layer must keep the string area */
233 /* Fordward decleration */
236 /** SW status of a queue */
237 enum al_udma_queue_status {
238 AL_QUEUE_NOT_INITIALIZED = 0,
244 /** UDMA Queue private data structure */
245 struct __cache_aligned al_udma_q {
246 uint16_t size_mask; /**< mask used for pointers wrap around
249 union udma_q_regs __iomem *q_regs; /**< pointer to the per queue UDMA
252 union al_udma_desc *desc_base_ptr; /**< base address submission ring
255 uint16_t next_desc_idx; /**< index to the next available submission
259 uint32_t desc_ring_id; /**< current submission ring id */
261 uint8_t *cdesc_base_ptr;/**< completion descriptors pointer, NULL */
262 /* means no completion */
263 uint32_t cdesc_size; /**< size (in bytes) of the udma completion ring
266 uint16_t next_cdesc_idx; /**< index in descriptors for next completing
269 uint8_t *end_cdesc_ptr; /**< used for wrap around detection */
270 uint16_t comp_head_idx; /**< completion ring head pointer register
273 volatile union al_udma_cdesc *comp_head_ptr; /**< when working in get_packet mode
274 * we maintain pointer instead of the
278 uint32_t pkt_crnt_descs; /**< holds the number of processed descriptors
279 * of the current packet
281 uint32_t comp_ring_id; /**< current completion Ring Id */
284 al_phys_addr_t desc_phy_base; /**< submission desc. physical base */
285 al_phys_addr_t cdesc_phy_base; /**< completion desc. physical base */
287 uint32_t flags; /**< flags used for completion modes */
288 uint32_t size; /**< ring size in descriptors */
289 enum al_udma_queue_status status;
290 struct al_udma *udma; /**< pointer to parent UDMA */
291 uint32_t qid; /**< the index number of the queue */
294 * The following fields are duplicated from the UDMA parent adapter
295 * due to performance considerations.
297 uint8_t adapter_rev_id; /**<PCI adapter revision ID */
303 enum al_udma_type type; /* Tx or Rx */
304 enum al_udma_state state;
305 uint8_t num_of_queues; /* number of queues supported by the UDMA */
306 union udma_regs __iomem *udma_regs; /* pointer to the UDMA registers */
307 struct udma_gen_regs *gen_regs; /* pointer to the Gen registers*/
308 struct al_udma_q udma_q[DMA_MAX_Q]; /* Array of UDMA Qs pointers */
309 unsigned int rev_id; /* UDMA revision ID */
317 /* Initializations functions */
319 * Initialize the udma engine
321 * @param udma udma data structure
322 * @param udma_params udma parameters from upper layer
324 * @return 0 on success. -EINVAL otherwise.
326 int al_udma_init(struct al_udma *udma, struct al_udma_params *udma_params);
329 * Initialize the udma queue data structure
335 * @return 0 if no error found.
336 * -EINVAL if the qid is out of range
337 * -EIO if queue was already initialized
340 int al_udma_q_init(struct al_udma *udma, uint32_t qid,
341 struct al_udma_q_params *q_params);
346 * Prior to calling this function make sure:
347 * 1. Queue interrupts are masked
348 * 2. No additional descriptors are written to the descriptor ring of the queue
349 * 3. No completed descriptors are being fetched
351 * The queue can be initialized again using 'al_udma_q_init'
355 * @return 0 if no error found.
358 int al_udma_q_reset(struct al_udma_q *udma_q);
361 * return (by reference) a pointer to a specific queue date structure.
362 * this pointer needed for calling functions (i.e. al_udma_desc_action_add) that
363 * require this pointer as input argument.
365 * @param udma udma data structure
366 * @param qid queue index
367 * @param q_handle pointer to the location where the queue structure pointer
370 * @return 0 on success. -EINVAL otherwise.
372 int al_udma_q_handle_get(struct al_udma *udma, uint32_t qid,
373 struct al_udma_q **q_handle);
376 * Change the UDMA's state
378 * @param udma udma data structure
379 * @param state the target state
383 int al_udma_state_set(struct al_udma *udma, enum al_udma_state state);
386 * return the current UDMA hardware state
388 * @param udma udma handle
390 * @return the UDMA state as reported by the hardware.
392 enum al_udma_state al_udma_state_get(struct al_udma *udma);
399 * get number of descriptors that can be submitted to the udma.
400 * keep one free descriptor to simplify full/empty management
401 * @param udma_q queue handle
403 * @return num of free descriptors.
405 static INLINE uint32_t al_udma_available_get(struct al_udma_q *udma_q)
407 uint16_t tmp = udma_q->next_cdesc_idx - (udma_q->next_desc_idx + 1);
408 tmp &= udma_q->size_mask;
410 return (uint32_t) tmp;
414 * check if queue has pending descriptors
416 * @param udma_q queue handle
418 * @return AL_TRUE if descriptors are submitted to completion ring and still
419 * not completed (with ack). AL_FALSE otherwise.
421 static INLINE al_bool al_udma_is_empty(struct al_udma_q *udma_q)
423 if (((udma_q->next_cdesc_idx - udma_q->next_desc_idx) &
424 udma_q->size_mask) == 0)
431 * get next available descriptor
432 * @param udma_q queue handle
434 * @return pointer to the next available descriptor
436 static INLINE union al_udma_desc *al_udma_desc_get(struct al_udma_q *udma_q)
438 union al_udma_desc *desc;
439 uint16_t next_desc_idx;
443 next_desc_idx = udma_q->next_desc_idx;
444 desc = udma_q->desc_base_ptr + next_desc_idx;
448 /* if reached end of queue, wrap around */
449 udma_q->next_desc_idx = next_desc_idx & udma_q->size_mask;
455 * get ring id for the last allocated descriptor
458 * @return ring id for the last allocated descriptor
459 * this function must be called each time a new descriptor is allocated
460 * by the al_udma_desc_get(), unless ring id is ignored.
462 static INLINE uint32_t al_udma_ring_id_get(struct al_udma_q *udma_q)
468 ring_id = udma_q->desc_ring_id;
470 /* calculate the ring id of the next desc */
471 /* if next_desc points to first desc, then queue wrapped around */
472 if (unlikely(udma_q->next_desc_idx) == 0)
473 udma_q->desc_ring_id = (udma_q->desc_ring_id + 1) &
478 /* add DMA action - trigger the engine */
480 * add num descriptors to the submission queue.
482 * @param udma_q queue handle
483 * @param num number of descriptors to add to the queues ring.
487 static INLINE int al_udma_desc_action_add(struct al_udma_q *udma_q,
493 al_assert((num > 0) && (num <= udma_q->size));
495 addr = &udma_q->q_regs->rings.drtp_inc;
496 /* make sure data written to the descriptors will be visible by the */
498 al_local_data_memory_barrier();
501 * As we explicitly invoke the synchronization function
502 * (al_data_memory_barrier()), then we can use the relaxed version.
504 al_reg_write32_relaxed(addr, num);
509 #define cdesc_is_first(flags) ((flags) & AL_UDMA_CDESC_FIRST)
510 #define cdesc_is_last(flags) ((flags) & AL_UDMA_CDESC_LAST)
513 * return pointer to the cdesc + offset desciptors. wrap around when needed.
515 * @param udma_q queue handle
516 * @param cdesc pointer that set by this function
517 * @param offset offset desciptors
520 static INLINE volatile union al_udma_cdesc *al_cdesc_next(
521 struct al_udma_q *udma_q,
522 volatile union al_udma_cdesc *cdesc,
525 volatile uint8_t *tmp = (volatile uint8_t *) cdesc + offset * udma_q->cdesc_size;
530 if (unlikely((tmp > udma_q->end_cdesc_ptr)))
531 return (union al_udma_cdesc *)
532 (udma_q->cdesc_base_ptr +
533 (tmp - udma_q->end_cdesc_ptr - udma_q->cdesc_size));
535 return (volatile union al_udma_cdesc *) tmp;
539 * check if the flags of the descriptor indicates that is new one
540 * the function uses the ring id from the descriptor flags to know whether it
541 * new one by comparing it with the curring ring id of the queue
543 * @param udma_q queue handle
544 * @param flags the flags of the completion descriptor
546 * @return AL_TRUE if the completion descriptor is new one.
547 * AL_FALSE if it old one.
549 static INLINE al_bool al_udma_new_cdesc(struct al_udma_q *udma_q,
552 if (((flags & AL_M2S_DESC_RING_ID_MASK) >> AL_M2S_DESC_RING_ID_SHIFT)
553 == udma_q->comp_ring_id)
559 * get next completion descriptor
560 * this function will also increment the completion ring id when the ring wraps
563 * @param udma_q queue handle
564 * @param cdesc current completion descriptor
566 * @return pointer to the completion descriptor that follows the one pointed by
569 static INLINE volatile union al_udma_cdesc *al_cdesc_next_update(
570 struct al_udma_q *udma_q,
571 volatile union al_udma_cdesc *cdesc)
573 /* if last desc, wrap around */
574 if (unlikely(((volatile uint8_t *) cdesc == udma_q->end_cdesc_ptr))) {
575 udma_q->comp_ring_id =
576 (udma_q->comp_ring_id + 1) & DMA_RING_ID_MASK;
577 return (union al_udma_cdesc *) udma_q->cdesc_base_ptr;
579 return (volatile union al_udma_cdesc *) ((volatile uint8_t *) cdesc + udma_q->cdesc_size);
583 * get next completed packet from completion ring of the queue
585 * @param udma_q udma queue handle
586 * @param desc pointer that set by this function to the first descriptor
587 * note: desc is valid only when return value is not zero
588 * @return number of descriptors that belong to the packet. 0 means no completed
589 * full packet was found.
590 * If the descriptors found in the completion queue don't form full packet (no
591 * desc with LAST flag), then this function will do the following:
592 * (1) save the number of processed descriptors.
593 * (2) save last processed descriptor, so next time it called, it will resume
596 * note: the descriptors that belong to the completed packet will still be
597 * considered as used, that means the upper layer is safe to access those
598 * descriptors when this function returns. the al_udma_cdesc_ack() should be
599 * called to inform the udma driver that those descriptors are freed.
601 uint32_t al_udma_cdesc_packet_get(
602 struct al_udma_q *udma_q,
603 volatile union al_udma_cdesc **desc);
605 /** get completion descriptor pointer from its index */
606 #define al_udma_cdesc_idx_to_ptr(udma_q, idx) \
607 ((volatile union al_udma_cdesc *) ((udma_q)->cdesc_base_ptr + \
608 (idx) * (udma_q)->cdesc_size))
612 * return number of all completed descriptors in the completion ring
614 * @param udma_q udma queue handle
615 * @param cdesc pointer that set by this function to the first descriptor
616 * note: desc is valid only when return value is not zero
617 * note: pass NULL if not interested
618 * @return number of descriptors. 0 means no completed descriptors were found.
619 * note: the descriptors that belong to the completed packet will still be
620 * considered as used, that means the upper layer is safe to access those
621 * descriptors when this function returns. the al_udma_cdesc_ack() should be
622 * called to inform the udma driver that those descriptors are freed.
624 static INLINE uint32_t al_udma_cdesc_get_all(
625 struct al_udma_q *udma_q,
626 volatile union al_udma_cdesc **cdesc)
632 udma_q->comp_head_idx = (uint16_t)
633 (al_reg_read32(&udma_q->q_regs->rings.crhp) &
636 count = (udma_q->comp_head_idx - udma_q->next_cdesc_idx) &
640 *cdesc = al_udma_cdesc_idx_to_ptr(udma_q, udma_q->next_cdesc_idx);
642 return (uint32_t)count;
646 * acknowledge the driver that the upper layer completed processing completion
649 * @param udma_q udma queue handle
650 * @param num number of descriptors to acknowledge
654 static INLINE int al_udma_cdesc_ack(struct al_udma_q *udma_q, uint32_t num)
658 udma_q->next_cdesc_idx += num;
659 udma_q->next_cdesc_idx &= udma_q->size_mask;
670 #endif /* __AL_HAL_UDMA_H__ */
671 /** @} end of UDMA group */