1 /***********************license start***************
2 * Copyright (c) 2003-2010 Cavium Inc. (support@cavium.com). All rights
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above
14 * copyright notice, this list of conditions and the following
15 * disclaimer in the documentation and/or other materials provided
16 * with the distribution.
18 * * Neither the name of Cavium Inc. nor the names of
19 * its contributors may be used to endorse or promote products
20 * derived from this software without specific prior written
23 * This Software, including technical data, may be subject to U.S. export control
24 * laws, including the U.S. Export Administration Act and its associated
25 * regulations, and may be subject to export or import regulations in other
28 * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
29 * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR
30 * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
31 * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR
32 * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
33 * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
34 * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
35 * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
36 * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
37 * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
38 ***********************license end**************************************/
44 * cvmx-shmem supplies the cross application shared memory implementation
46 * <hr>$Revision: 41586 $<hr>
49 #include "cvmx-bootmem.h"
51 #include "cvmx-shmem.h"
55 struct cvmx_shmem_smdr *__smdr = NULL;
57 #ifdef CVMX_BUILD_FOR_LINUX_USER
58 static int __cvmx_shmem_devmemfd = 0; /* fd for /dev/mem */
61 #define __CHECK_APP_SMDR do { \
62 if (__smdr == NULL) { \
63 cvmx_dprintf("cvmx_shmem: %s is not set up, Quit line %d \n", \
64 CVMX_SHMEM_DSCPTR_NAME, __LINE__ ); \
73 * Virtual sbrk, assigning virtual address in a global virtual address space.
75 * @param alignment alignment requirement in bytes
76 * @param size size in bytes
78 static inline void *__cvmx_shmem_vsbrk_64(uint64_t alignment, uint64_t size)
80 uint64_t nbase_64 = CAST64(__smdr->break64);
83 /* Skip unaligned bytes */
84 if (nbase_64 & alignment)
85 nbase_64 += ~(nbase_64 & alignment) + 1;
87 if (nbase_64 + size < CVMX_SHMEM_VADDR64_END)
89 nbase = CASTPTR(void *, nbase_64);
90 __smdr->break64 = nbase + size;
98 * Initialize all SMDR entries, only need to be called once
100 * @param smdr pointer to the SMDR
102 static inline void __smdr_new(struct cvmx_shmem_smdr *smdr) {
108 cvmx_spinlock_init (&smdr->lock);
109 cvmx_spinlock_lock (&smdr->lock);
111 for ( i = 0; i < CVMX_SHMEM_NUM_DSCPTR; i++ )
113 smdr -> shmd[i].owner = CVMX_SHMEM_OWNER_NONE;
114 smdr -> shmd[i].is_named_block = 0;
115 smdr -> shmd[i].use_count = 0;
116 smdr -> shmd[i].name = NULL;
117 smdr -> shmd[i].vaddr = NULL;
118 smdr -> shmd[i].paddr = 0;
119 smdr -> shmd[i].size = 0;
120 smdr -> shmd[i].alignment = 0;
124 smdr->break64 = (void *)CVMX_SHMEM_VADDR64_START;
125 cvmx_spinlock_unlock (&smdr->lock);
128 /* Make sure the shmem descriptor region is created */
136 * Initialize __smdr pointer, if SMDR exits already. If not, create a new
137 * one. Once SMDR is created (as a bootmem named block), it is persistent.
139 static inline struct cvmx_shmem_smdr *__smdr_init()
141 const cvmx_bootmem_named_block_desc_t *smdr_nblk = NULL;
142 size_t smdr_size = sizeof(*__smdr);
143 char *smdr_name = CVMX_SHMEM_DSCPTR_NAME;
145 __smdr = (struct cvmx_shmem_smdr *) cvmx_bootmem_alloc_named(smdr_size, 0x10000, smdr_name);
151 /* Check if SMDR exists already */
152 smdr_nblk = cvmx_bootmem_find_named_block(smdr_name);
155 __smdr = (struct cvmx_shmem_smdr *)
156 (cvmx_phys_to_ptr(smdr_nblk->base_addr));
158 cvmx_spinlock_lock (&__smdr->lock);
159 if (smdr_nblk->size != smdr_size)
161 cvmx_dprintf("SMDR named block is created by another "
162 "application with different size %lu, "
164 (long unsigned int)smdr_nblk->size, (long unsigned int)smdr_size);
167 cvmx_spinlock_unlock (&__smdr->lock);
172 cvmx_dprintf("cvmx_shmem: Failed to allocate or find SMDR from bootmem \n");
180 * Generic Iterator function for all SMDR entries
182 * @param void(*f)(dscptr) the function to be invoked for every descriptor
185 * @return the descriptor iterator stopped at.
187 static struct cvmx_shmem_dscptr *__smdr_iterator(struct cvmx_shmem_dscptr *(*f)(struct cvmx_shmem_dscptr *dscptr, void *p), void *param )
189 struct cvmx_shmem_dscptr *d, *dscptr = NULL;
194 for (i = 0; i < CVMX_SHMEM_NUM_DSCPTR; i++)
196 d = &__smdr->shmd[i];
197 if ((dscptr = (*f)(d, param)) != NULL)
198 break; /* stop iteration */
207 * SMDR name match functor. to be used for iterator.
209 * @param dscptr descriptor passed in by the iterator
210 * @param name string to match against
212 * @return !NULL descriptor matched
215 static struct cvmx_shmem_dscptr *__cvmx_shmem_smdr_match_name(struct cvmx_shmem_dscptr *dscptr, void *name)
217 char *name_to_match = (char *) name;
218 struct cvmx_shmem_dscptr *ret = NULL;
220 if (dscptr->owner == CVMX_SHMEM_OWNER_NONE)
223 if (strcmp(dscptr->name, name_to_match) == 0)
233 * @param name string to match against
235 * @return !NULL descriptor matched
238 static struct cvmx_shmem_dscptr *__cvmx_shmem_smdr_find_by_name(char *name)
240 return __smdr_iterator( __cvmx_shmem_smdr_match_name, name);
245 * SMDR is free functor. to be used for iterator.
247 * @param dscptr descriptor passed in by the iterator
250 * @return !NULL descriptor is free
251 * NULL descriptor is not free
253 static struct cvmx_shmem_dscptr *__cvmx_shmem_smdr_is_free(struct cvmx_shmem_dscptr* dscptr, void *nouse)
255 if (dscptr->owner == CVMX_SHMEM_OWNER_NONE)
263 * Search SMDR to find the first free descriptor
265 * @return !NULL free descriptor found
268 struct cvmx_shmem_dscptr *__cvmx_shmem_smdr_find_free_dscptr(void)
270 return __smdr_iterator(__cvmx_shmem_smdr_is_free, NULL);
277 * @param dscptr descriptor to be freed
279 static void __cvmx_shmem_smdr_free(struct cvmx_shmem_dscptr *dscptr)
281 dscptr->owner = CVMX_SHMEM_OWNER_NONE;
286 * Per core shmem init function
288 * @return cvmx_shmem_smdr* pointer to __smdr
290 struct cvmx_shmem_smdr *cvmx_shmem_init()
292 return __smdr_init();
296 * Open shared memory based on named block
298 * @return dscptr descriptor of the opened named block
300 struct cvmx_shmem_dscptr *cvmx_shmem_named_block_open(char *name, uint32_t size, int oflag)
302 const cvmx_bootmem_named_block_desc_t *shmem_nblk = NULL;
303 struct cvmx_shmem_dscptr *dscptr = NULL;
304 int nblk_allocated = 0; /* Assume we don't need to allocate a new
307 const uint64_t size_4k = 4*1024, size_512mb = 512*1024*1024;
311 /* Check size, Make sure it is minimal 4K, no bigger than 512MB */
312 if (size > size_512mb) {
313 cvmx_dprintf("Shared memory size can not be bigger than 512MB \n");
319 size = __upper_power_of_two(size);
321 cvmx_spinlock_lock(&__smdr->lock);
323 shmem_nblk = cvmx_bootmem_find_named_block(name);
324 if ((shmem_nblk == NULL) && (oflag & CVMX_SHMEM_O_CREAT))
327 /* The named block does not exist, create it if caller specifies
330 p = cvmx_bootmem_alloc_named(size, size, name);
332 shmem_nblk = cvmx_bootmem_find_named_block(name);
334 cvmx_dprintf("cvmx-shmem-dbg:"
335 "creating a new block %s: blk %p, shmem_nblk %p \n",
336 name, p, shmem_nblk);
340 if (shmem_nblk == NULL)
343 /* We are now holding a valid named block */
345 dscptr = __cvmx_shmem_smdr_find_by_name(name);
350 /* name conflict between bootmem name space and SMDR name space */
351 cvmx_dprintf("cvmx-shmem: SMDR descriptor name conflict, %s \n", name);
354 /* Make sure size and alignment matches with existing descriptor */
355 if ((size != dscptr->size) || (size != dscptr -> alignment))
360 /* Create a new descriptor */
361 dscptr = __cvmx_shmem_smdr_find_free_dscptr();
366 cvmx_dprintf("cvmx-shmem: SMDR out of descriptors \n");
371 /* Maintain the reference count */
373 dscptr->use_count += 1;
375 cvmx_spinlock_unlock(&__smdr->lock);
380 cvmx_dprintf("cvmx-shmem-dbg: named block open failed \n");
384 __cvmx_shmem_smdr_free(dscptr);
385 if (shmem_nblk && nblk_allocated)
386 cvmx_bootmem_free_named(name);
387 cvmx_spinlock_unlock(&__smdr->lock);
394 cvmx_dprintf("cvmx-shmem-dbg: init SMDR descriptor %p \n", dscptr);
397 /* Assign vaddr for single address space mapping */
398 vaddr = __cvmx_shmem_vsbrk_64(size, size);
400 /* Failed to allocate virtual address, clean up */
405 cvmx_dprintf("cmvx-shmem-dbg: allocated vaddr %p \n", vaddr);
407 dscptr->vaddr = vaddr;
409 /* Store descriptor information, name, alignment,size... */
410 dscptr->owner = cvmx_get_core_num();
411 dscptr->is_named_block = 1;
412 dscptr->use_count = 1;
413 dscptr->name =shmem_nblk->name ;
414 dscptr->paddr = shmem_nblk->base_addr;
416 dscptr->alignment = size;
418 /* Store permission bits */
419 if (oflag & CVMX_SHMEM_O_WRONLY)
420 dscptr->p_wronly = 1;
421 if (oflag & CVMX_SHMEM_O_RDWR)
424 cvmx_spinlock_unlock(&__smdr->lock);
431 * For stand along SE application only.
433 * Add TLB mapping to map the shared memory
435 * @param dscptr shared memory descriptor
436 * @param pflag protection flags
438 * @return vaddr the virtual address mapped for the shared memory
440 #ifndef CVMX_BUILD_FOR_LINUX_USER
441 void *__cvmx_shmem_map_standalone(struct cvmx_shmem_dscptr *dscptr, int pflag)
445 /* Find a free tlb entry */
446 free_index = cvmx_tlb_allocate_runtime_entry();
450 cvmx_dprintf("cvmx-shmem: shmem_map failed, out TLB entries \n");
455 cvmx_dprintf("cmvx-shmem-dbg:"
456 "shmem_map TLB %d: vaddr %p paddr %lx, size %x \n",
457 free_index, dscptr->vaddr, dscptr->paddr, dscptr->size );
460 cvmx_tlb_write_runtime_entry(free_index, CAST64(dscptr->vaddr),
461 dscptr->paddr, dscptr->size,
462 TLB_DIRTY | TLB_VALID | TLB_GLOBAL);
464 return dscptr -> vaddr;
471 * For Linux user application only
473 * Add mmap the shared memory
475 * @param dscptr shared memory descriptor
476 * @param pflag protection flags
478 * @return vaddr the virtual address mapped for the shared memory
480 #ifdef CVMX_BUILD_FOR_LINUX_USER
481 static inline void *__cvmx_shmem_map_linux(struct cvmx_shmem_dscptr *dscptr, int pflag)
485 if(__cvmx_shmem_devmemfd == 0)
487 __cvmx_shmem_devmemfd = open("/dev/mem", O_RDWR);
488 if (__cvmx_shmem_devmemfd < 0)
490 cvmx_dprintf("Failed to open /dev/mem\n");
495 vaddr = mmap(dscptr->vaddr, dscptr->size, PROT_READ|PROT_WRITE,
496 MAP_SHARED, __cvmx_shmem_devmemfd, 0);
498 /* Make sure the mmap maps to the same virtual address specified in
501 if ((vaddr!=NULL) && (vaddr != dscptr->vaddr))
503 munmap(vaddr, dscptr->size);
513 * Add mapping for the shared memory
515 * @param dscptr shared memory descriptor
516 * @param pflag protection flags
518 * @return vaddr the virtual address mapped for the shared memory
520 void *cvmx_shmem_map(struct cvmx_shmem_dscptr *dscptr, int pflag)
523 #ifdef CVMX_BUILD_FOR_LINUX_USER
524 vaddr = __cvmx_shmem_map_linux(dscptr, pflag);
526 vaddr = __cvmx_shmem_map_standalone(dscptr, pflag);
535 * For Linux user application only
537 * ummap the shared memory
539 * @param dscptr shared memory descriptor
542 #ifdef CVMX_BUILD_FOR_LINUX_USER
543 static inline void __cvmx_shmem_unmap_linux(struct cvmx_shmem_dscptr* dscptr)
545 if (__cvmx_shmem_devmemfd && dscptr)
546 munmap(dscptr->vaddr, dscptr->size);
554 * For stand along SE application only.
556 * ummap the shared memory
558 * @param dscptr shared memory descriptor
561 #ifndef CVMX_BUILD_FOR_LINUX_USER
563 __cvmx_shmem_unmap_standalone(struct cvmx_shmem_dscptr *dscptr)
567 index = cvmx_tlb_lookup(CAST64(dscptr->vaddr));
570 cvmx_dprintf("cmvx-shmem-dbg:"
571 "shmem_unmap TLB %d \n", index);
573 cvmx_tlb_free_runtime_entry(index);
578 * ummap the shared memory
580 * @param dscptr shared memory descriptor
583 void cvmx_shmem_unmap(struct cvmx_shmem_dscptr *dscptr)
585 #ifdef CVMX_BUILD_FOR_LINUX_USER
586 __cvmx_shmem_unmap_linux(dscptr);
588 __cvmx_shmem_unmap_standalone(dscptr);
595 * Common implementation of closing a descriptor.
597 * @param dscptr shared memory descriptor
598 * @param remove 1: remove the descriptor and named block if this
599 * this is the last user of the descriptor
605 static inline int __cvmx_shmem_close_dscptr(struct cvmx_shmem_dscptr *dscptr, int remove)
607 cvmx_spinlock_lock(&dscptr->lock);
609 if (dscptr->use_count >0)
610 dscptr->use_count-= 1;
612 if ((dscptr->use_count == 0) && remove)
614 /* Free this descriptor */
615 __cvmx_shmem_smdr_free(dscptr);
617 /* Free named block if this is the last user, and the block
618 is created by the application */
619 if (dscptr->is_named_block)
622 cvmx_dprintf("cvmx-shmem-dbg: remove named block %s \n", dscptr->name);
624 cvmx_bootmem_phy_named_block_free(dscptr->name, 0);
627 cvmx_spinlock_unlock(&dscptr->lock);
635 * For stand along SE application only.
637 * close a descriptor.
639 * @param dscptr shared memory descriptor
640 * @param remove 1: remove the descriptor and named block if this
641 * this is the last user of the descriptor
647 #ifndef CVMX_BUILD_FOR_LINUX_USER
648 static inline int __cvmx_shmem_close_standalone(struct cvmx_shmem_dscptr *dscptr, int remove)
650 return __cvmx_shmem_close_dscptr(dscptr, remove);
657 * For Linux user application only.
659 * close a descriptor.
661 * @param dscptr shared memory descriptor
662 * @param remove 1: remove the descriptor and named block if this
663 * this is the last user of the descriptor
669 #ifdef CVMX_BUILD_FOR_LINUX_USER
670 int __cvmx_shmem_close_linux(struct cvmx_shmem_dscptr *dscptr, int remove)
673 ret = __cvmx_shmem_close_dscptr(dscptr, remove);
675 if (ret && __cvmx_shmem_devmemfd)
677 close(__cvmx_shmem_devmemfd);
678 __cvmx_shmem_devmemfd=0;
688 * close a descriptor.
690 * @param dscptr shared memory descriptor
691 * @param remove 1: remove the descriptor and named block if this
692 * this is the last user of the descriptor
698 int cvmx_shmem_close(struct cvmx_shmem_dscptr *dscptr, int remove)
701 #ifdef CVMX_BUILD_FOR_LINUX_USER
702 ret = __cvmx_shmem_close_linux(dscptr, remove);
704 ret = __cvmx_shmem_close_standalone(dscptr, remove);
712 * SMDR non-free descriptor dump functor. to be used for iterator.
714 * @param dscptr descriptor passed in by the iterator
716 * @return NULL always
718 static struct cvmx_shmem_dscptr *__cvmx_shmem_smdr_display_dscptr(struct cvmx_shmem_dscptr *dscptr, void *nouse)
720 if ((dscptr != NULL ) && (dscptr -> owner != CVMX_SHMEM_OWNER_NONE))
722 cvmx_dprintf(" %s: phy: %lx, size %d, alignment %lx, virt %p use_count %d\n",
723 dscptr->name, dscptr-> paddr,
724 dscptr->size, dscptr-> alignment,
725 dscptr->vaddr, dscptr->use_count);
733 * SMDR descriptor show
735 * list all non-free descriptors
737 void cvmx_shmem_show(void)
742 cvmx_dprintf("SMDR descriptor list: \n");
743 cvmx_spinlock_lock(&__smdr->lock);
744 __smdr_iterator(__cvmx_shmem_smdr_display_dscptr, NULL);
745 cvmx_spinlock_unlock(&__smdr->lock);
746 cvmx_dprintf("\n\n");