sys/contrib/octeon-sdk/cvmx-bootmem.c

   1 /***********************license start***************
   2  *  Copyright (c) 2003-2008 Cavium Networks (support@cavium.com). All rights
   3  *  reserved.
   4  *
   5  *
   6  *  Redistribution and use in source and binary forms, with or without
   7  *  modification, are permitted provided that the following conditions are
   8  *  met:
   9  *
  10  *      * Redistributions of source code must retain the above copyright
  11  *        notice, this list of conditions and the following disclaimer.
  12  *
  13  *      * Redistributions in binary form must reproduce the above
  14  *        copyright notice, this list of conditions and the following
  15  *        disclaimer in the documentation and/or other materials provided
  16  *        with the distribution.
  17  *
  18  *      * Neither the name of Cavium Networks nor the names of
  19  *        its contributors may be used to endorse or promote products
  20  *        derived from this software without specific prior written
  21  *        permission.
  22  *
  23  *  TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
  24  *  AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS
  25  *  OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH
  26  *  RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY
  27  *  REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT
  28  *  DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES
  29  *  OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR
  30  *  PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET
  31  *  POSSESSION OR CORRESPONDENCE TO DESCRIPTION.  THE ENTIRE RISK ARISING OUT
  32  *  OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
  33  *
  34  *
  35  *  For any questions regarding licensing please contact marketing@caviumnetworks.com
  36  *
  37  ***********************license end**************************************/
  38
  39
  40
  41
  42
  43 /**
  44  * @file
  45  * Simple allocate only memory allocator.  Used to allocate memory at application
  46  * start time.
  47  *
  48  * <hr>$Revision: 41586 $<hr>
  49  *
  50  */
  51
  52 #include "cvmx.h"
  53 #include "cvmx-spinlock.h"
  54 #include "cvmx-bootmem.h"
  55
  56
  57 //#define DEBUG
  58
  59
  60 #undef  MAX
  61 #define MAX(a, b)  (((a) > (b)) ? (a) : (b))
  62
  63 #undef  MIN
  64 #define MIN(a, b)  (((a) < (b)) ? (a) : (b))
  65
  66 #define ALIGN_ADDR_UP(addr, align)     (((addr) + (~(align))) & (align))
  67
  68 static CVMX_SHARED cvmx_bootmem_desc_t *cvmx_bootmem_desc = NULL;
  69
  70 /* See header file for descriptions of functions */
  71
  72 /* Wrapper functions are provided for reading/writing the size and next block
  73 ** values as these may not be directly addressible (in 32 bit applications, for instance.)
  74 */
  75 /* Offsets of data elements in bootmem list, must match cvmx_bootmem_block_header_t */
  76 #define NEXT_OFFSET 0
  77 #define SIZE_OFFSET 8
  78 static void cvmx_bootmem_phy_set_size(uint64_t addr, uint64_t size)
  79 {
  80     cvmx_write64_uint64((addr + SIZE_OFFSET) | (1ull << 63), size);
  81 }
  82 static void cvmx_bootmem_phy_set_next(uint64_t addr, uint64_t next)
  83 {
  84     cvmx_write64_uint64((addr + NEXT_OFFSET) | (1ull << 63), next);
  85 }
  86 static uint64_t cvmx_bootmem_phy_get_size(uint64_t addr)
  87 {
  88     return(cvmx_read64_uint64((addr + SIZE_OFFSET) | (1ull << 63)));
  89 }
  90 static uint64_t cvmx_bootmem_phy_get_next(uint64_t addr)
  91 {
  92     return(cvmx_read64_uint64((addr + NEXT_OFFSET) | (1ull << 63)));
  93 }
  94
  95
  96 /* This functions takes an address range and adjusts it as necessary to
  97 ** match the ABI that is currently being used.  This is required to ensure
  98 ** that bootmem_alloc* functions only return valid pointers for 32 bit ABIs */
  99 static int __cvmx_validate_mem_range(uint64_t *min_addr_ptr, uint64_t *max_addr_ptr)
 100 {
 101
 102 #if defined(__linux__) && defined(CVMX_ABI_N32)
 103     {
 104         extern uint64_t linux_mem32_min;
 105         extern uint64_t linux_mem32_max;
 106         /* For 32 bit Linux apps, we need to restrict the allocations to the range
 107         ** of memory configured for access from userspace.  Also, we need to add mappings
 108         ** for the data structures that we access.*/
 109
 110         /* Narrow range requests to be bounded by the 32 bit limits.  octeon_phy_mem_block_alloc()
 111         ** will reject inconsistent req_size/range requests, so we don't repeat those checks here.
 112         ** If max unspecified, set to 32 bit maximum. */
 113         *min_addr_ptr = MIN(MAX(*min_addr_ptr, linux_mem32_min), linux_mem32_max);
 114         if (!*max_addr_ptr)
 115             *max_addr_ptr = linux_mem32_max;
 116         else
 117             *max_addr_ptr = MAX(MIN(*max_addr_ptr, linux_mem32_max), linux_mem32_min);
 118     }
 119 #elif defined(CVMX_ABI_N32)
 120     {
 121         uint32_t max_phys = 0x0FFFFFFF;  /* Max physical address when 1-1 mappings not used */
 122 #if CVMX_USE_1_TO_1_TLB_MAPPINGS
 123         max_phys = 0x7FFFFFFF;
 124 #endif
 125         /* We are are running standalone simple executive, so we need to limit the range
 126         ** that we allocate from */
 127
 128         /* Narrow range requests to be bounded by the 32 bit limits.  octeon_phy_mem_block_alloc()
 129         ** will reject inconsistent req_size/range requests, so we don't repeat those checks here.
 130         ** If max unspecified, set to 32 bit maximum. */
 131         *min_addr_ptr = MIN(MAX(*min_addr_ptr, 0x0), max_phys);
 132         if (!*max_addr_ptr)
 133             *max_addr_ptr = max_phys;
 134         else
 135             *max_addr_ptr = MAX(MIN(*max_addr_ptr, max_phys), 0x0);
 136     }
 137 #endif
 138
 139     return 0;
 140 }
 141
 142
 143 void *cvmx_bootmem_alloc_range(uint64_t size, uint64_t alignment, uint64_t min_addr, uint64_t max_addr)
 144 {
 145     int64_t address;
 146
 147     __cvmx_validate_mem_range(&min_addr, &max_addr);
 148     address = cvmx_bootmem_phy_alloc(size, min_addr, max_addr, alignment, 0);
 149
 150     if (address > 0)
 151         return cvmx_phys_to_ptr(address);
 152     else
 153         return NULL;
 154 }
 155
 156 void *cvmx_bootmem_alloc_address(uint64_t size, uint64_t address, uint64_t alignment)
 157 {
 158     return cvmx_bootmem_alloc_range(size, alignment, address, address + size);
 159 }
 160
 161
 162 void *cvmx_bootmem_alloc(uint64_t size, uint64_t alignment)
 163 {
 164     return cvmx_bootmem_alloc_range(size, alignment, 0, 0);
 165 }
 166
 167 void *cvmx_bootmem_alloc_named_range(uint64_t size, uint64_t min_addr, uint64_t max_addr, uint64_t align, char *name)
 168 {
 169     int64_t addr;
 170
 171     __cvmx_validate_mem_range(&min_addr, &max_addr);
 172     addr = cvmx_bootmem_phy_named_block_alloc(size, min_addr, max_addr, align, name, 0);
 173     if (addr >= 0)
 174         return cvmx_phys_to_ptr(addr);
 175     else
 176         return NULL;
 177
 178 }
 179 void *cvmx_bootmem_alloc_named_address(uint64_t size, uint64_t address, char *name)
 180 {
 181     return(cvmx_bootmem_alloc_named_range(size, address, address + size, 0, name));
 182 }
 183 void *cvmx_bootmem_alloc_named(uint64_t size, uint64_t alignment, char *name)
 184 {
 185     return(cvmx_bootmem_alloc_named_range(size, 0, 0, alignment, name));
 186 }
 187
 188 int cvmx_bootmem_free_named(char *name)
 189 {
 190     return(cvmx_bootmem_phy_named_block_free(name, 0));
 191 }
 192
 193 cvmx_bootmem_named_block_desc_t * cvmx_bootmem_find_named_block(char *name)
 194 {
 195     return(cvmx_bootmem_phy_named_block_find(name, 0));
 196 }
 197
 198 void cvmx_bootmem_print_named(void)
 199 {
 200     cvmx_bootmem_phy_named_block_print();
 201 }
 202
 203 #if defined(__linux__) && defined(CVMX_ABI_N32)
 204 cvmx_bootmem_named_block_desc_t *linux32_named_block_array_ptr;
 205 #endif
 206
 207 int cvmx_bootmem_init(void *mem_desc_ptr)
 208 {
 209     /* Verify that the size of cvmx_spinlock_t meets our assumptions */
 210     if (sizeof(cvmx_spinlock_t) != 4)
 211     {
 212         cvmx_dprintf("ERROR: Unexpected size of cvmx_spinlock_t\n");
 213         return(-1);
 214     }
 215
 216     /* Here we set the global pointer to the bootmem descriptor block.  This pointer will
 217     ** be used directly, so we will set it up to be directly usable by the application.
 218     ** It is set up as follows for the various runtime/ABI combinations:
 219     ** Linux 64 bit: Set XKPHYS bit
 220     ** Linux 32 bit: use mmap to create mapping, use virtual address
 221     ** CVMX 64 bit:  use physical address directly
 222     ** CVMX 32 bit:  use physical address directly
 223     ** Note that the CVMX environment assumes the use of 1-1 TLB mappings so that the physical addresses
 224     ** can be used directly
 225     */
 226     if (!cvmx_bootmem_desc)
 227     {
 228 #if defined(CVMX_BUILD_FOR_LINUX_USER) && defined(CVMX_ABI_N32)
 229         void *base_ptr;
 230         /* For 32 bit, we need to use mmap to create a mapping for the bootmem descriptor */
 231         int dm_fd = open("/dev/mem", O_RDWR);
 232         if (dm_fd < 0)
 233         {
 234             cvmx_dprintf("ERROR opening /dev/mem for boot descriptor mapping\n");
 235             return(-1);
 236         }
 237
 238         base_ptr = mmap(NULL,
 239                         sizeof(cvmx_bootmem_desc_t) + sysconf(_SC_PAGESIZE),
 240                         PROT_READ | PROT_WRITE,
 241                         MAP_SHARED,
 242                         dm_fd,
 243                         ((off_t)mem_desc_ptr) & ~(sysconf(_SC_PAGESIZE) - 1));
 244
 245         if (MAP_FAILED == base_ptr)
 246         {
 247             cvmx_dprintf("Error mapping bootmem descriptor!\n");
 248             close(dm_fd);
 249             return(-1);
 250         }
 251
 252         /* Adjust pointer to point to bootmem_descriptor, rather than start of page it is in */
 253         cvmx_bootmem_desc =  (cvmx_bootmem_desc_t*)((char*)base_ptr + (((off_t)mem_desc_ptr) & (sysconf(_SC_PAGESIZE) - 1)));
 254
 255         /* Also setup mapping for named memory block desc. while we are at it.  Here we must keep another
 256         ** pointer around, as the value in the bootmem descriptor is shared with other applications. */
 257         base_ptr = mmap(NULL,
 258                         sizeof(cvmx_bootmem_named_block_desc_t) * cvmx_bootmem_desc->named_block_num_blocks + sysconf(_SC_PAGESIZE),
 259                         PROT_READ | PROT_WRITE,
 260                         MAP_SHARED,
 261                         dm_fd,
 262                         ((off_t)cvmx_bootmem_desc->named_block_array_addr) & ~(sysconf(_SC_PAGESIZE) - 1));
 263
 264         close(dm_fd);
 265
 266         if (MAP_FAILED == base_ptr)
 267         {
 268             cvmx_dprintf("Error mapping named block descriptor!\n");
 269             return(-1);
 270         }
 271
 272         /* Adjust pointer to point to named block array, rather than start of page it is in */
 273         linux32_named_block_array_ptr = (cvmx_bootmem_named_block_desc_t*)((char*)base_ptr + (((off_t)cvmx_bootmem_desc->named_block_array_addr) & (sysconf(_SC_PAGESIZE) - 1)));
 274
 275 #elif (defined(CVMX_BUILD_FOR_LINUX_KERNEL) || defined(CVMX_BUILD_FOR_LINUX_USER)) && defined(CVMX_ABI_64)
 276         /* Set XKPHYS bit */
 277         cvmx_bootmem_desc = cvmx_phys_to_ptr(CAST64(mem_desc_ptr));
 278 #else
 279         cvmx_bootmem_desc = (cvmx_bootmem_desc_t*)mem_desc_ptr;
 280 #endif
 281     }
 282
 283
 284     return(0);
 285 }
 286
 287
 288 uint64_t cvmx_bootmem_available_mem(uint64_t min_block_size)
 289 {
 290     return(cvmx_bootmem_phy_available_mem(min_block_size));
 291 }
 292
 293
 294
 295
 296
 297 /*********************************************************************
 298 ** The cvmx_bootmem_phy* functions below return 64 bit physical addresses,
 299 ** and expose more features that the cvmx_bootmem_functions above.  These are
 300 ** required for full memory space access in 32 bit applications, as well as for
 301 ** using some advance features.
 302 ** Most applications should not need to use these.
 303 **
 304 **/
 305
 306
 307 int64_t cvmx_bootmem_phy_alloc(uint64_t req_size, uint64_t address_min, uint64_t address_max, uint64_t alignment, uint32_t flags)
 308 {
 309
 310     uint64_t head_addr;
 311     uint64_t ent_addr;
 312     uint64_t prev_addr = 0;  /* points to previous list entry, NULL current entry is head of list */
 313     uint64_t new_ent_addr = 0;
 314     uint64_t desired_min_addr;
 315     uint64_t alignment_mask = ~(alignment - 1);
 316
 317 #ifdef DEBUG
 318     cvmx_dprintf("cvmx_bootmem_phy_alloc: req_size: 0x%llx, min_addr: 0x%llx, max_addr: 0x%llx, align: 0x%llx\n",
 319            (unsigned long long)req_size, (unsigned long long)address_min, (unsigned long long)address_max, (unsigned long long)alignment);
 320 #endif
 321
 322     if (cvmx_bootmem_desc->major_version > 3)
 323     {
 324         cvmx_dprintf("ERROR: Incompatible bootmem descriptor version: %d.%d at addr: %p\n",
 325                (int)cvmx_bootmem_desc->major_version, (int)cvmx_bootmem_desc->minor_version, cvmx_bootmem_desc);
 326         goto error_out;
 327     }
 328
 329     /* Do a variety of checks to validate the arguments.  The allocator code will later assume
 330     ** that these checks have been made.  We validate that the requested constraints are not
 331     ** self-contradictory before we look through the list of available memory
 332     */
 333
 334     /* 0 is not a valid req_size for this allocator */
 335     if (!req_size)
 336         goto error_out;
 337
 338     /* Round req_size up to mult of minimum alignment bytes */
 339     req_size = (req_size + (CVMX_BOOTMEM_ALIGNMENT_SIZE - 1)) & ~(CVMX_BOOTMEM_ALIGNMENT_SIZE - 1);
 340
 341     /* Convert !0 address_min and 0 address_max to special case of range that specifies an exact
 342     ** memory block to allocate.  Do this before other checks and adjustments so that this tranformation will be validated */
 343     if (address_min && !address_max)
 344         address_max = address_min + req_size;
 345     else if (!address_min && !address_max)
 346         address_max = ~0ull;   /* If no limits given, use max limits */
 347
 348
 349
 350
 351     /* Enforce minimum alignment (this also keeps the minimum free block
 352     ** req_size the same as the alignment req_size */
 353     if (alignment < CVMX_BOOTMEM_ALIGNMENT_SIZE)
 354     {
 355         alignment = CVMX_BOOTMEM_ALIGNMENT_SIZE;
 356     }
 357     alignment_mask = ~(alignment - 1);
 358
 359     /* Adjust address minimum based on requested alignment (round up to meet alignment).  Do this here so we can
 360     ** reject impossible requests up front. (NOP for address_min == 0) */
 361     if (alignment)
 362         address_min = (address_min + (alignment - 1)) & ~(alignment - 1);
 363
 364
 365     /* Reject inconsistent args.  We have adjusted these, so this may fail due to our internal changes
 366     ** even if this check would pass for the values the user supplied. */
 367     if (req_size > address_max - address_min)
 368         goto error_out;
 369
 370     /* Walk through the list entries - first fit found is returned */
 371
 372     if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
 373         cvmx_spinlock_lock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
 374     head_addr = cvmx_bootmem_desc->head_addr;
 375     ent_addr = head_addr;
 376     while (ent_addr)
 377     {
 378         uint64_t usable_base, usable_max;
 379         uint64_t ent_size = cvmx_bootmem_phy_get_size(ent_addr);
 380
 381         if (cvmx_bootmem_phy_get_next(ent_addr) && ent_addr > cvmx_bootmem_phy_get_next(ent_addr))
 382         {
 383             cvmx_dprintf("Internal bootmem_alloc() error: ent: 0x%llx, next: 0x%llx\n",
 384                    (unsigned long long)ent_addr, (unsigned long long)cvmx_bootmem_phy_get_next(ent_addr));
 385             goto error_out;
 386         }
 387
 388         /* Determine if this is an entry that can satisify the request */
 389         /* Check to make sure entry is large enough to satisfy request */
 390         usable_base = ALIGN_ADDR_UP(MAX(address_min, ent_addr), alignment_mask);
 391         usable_max = MIN(address_max, ent_addr + ent_size);
 392         /* We should be able to allocate block at address usable_base */
 393
 394         desired_min_addr = usable_base;
 395
 396         /* Determine if request can be satisfied from the current entry */
 397         if ((((ent_addr + ent_size) > usable_base && ent_addr < address_max))
 398             && req_size <= usable_max - usable_base)
 399         {
 400             /* We have found an entry that has room to satisfy the request, so allocate it from this entry */
 401
 402             /* If end CVMX_BOOTMEM_FLAG_END_ALLOC set, then allocate from the end of this block
 403             ** rather than the beginning */
 404             if (flags & CVMX_BOOTMEM_FLAG_END_ALLOC)
 405             {
 406                 desired_min_addr = usable_max - req_size;
 407                 /* Align desired address down to required alignment */
 408                 desired_min_addr &= alignment_mask;
 409             }
 410
 411             /* Match at start of entry */
 412             if (desired_min_addr == ent_addr)
 413             {
 414                 if (req_size < ent_size)
 415                 {
 416                     /* big enough to create a new block from top portion of block */
 417                     new_ent_addr = ent_addr + req_size;
 418                     cvmx_bootmem_phy_set_next(new_ent_addr, cvmx_bootmem_phy_get_next(ent_addr));
 419                     cvmx_bootmem_phy_set_size(new_ent_addr, ent_size - req_size);
 420
 421                     /* Adjust next pointer as following code uses this */
 422                     cvmx_bootmem_phy_set_next(ent_addr, new_ent_addr);
 423                 }
 424
 425                 /* adjust prev ptr or head to remove this entry from list */
 426                 if (prev_addr)
 427                 {
 428                     cvmx_bootmem_phy_set_next(prev_addr, cvmx_bootmem_phy_get_next(ent_addr));
 429                 }
 430                 else
 431                 {
 432                     /* head of list being returned, so update head ptr */
 433                     cvmx_bootmem_desc->head_addr = cvmx_bootmem_phy_get_next(ent_addr);
 434                 }
 435                 if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
 436                     cvmx_spinlock_unlock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
 437                 return(desired_min_addr);
 438             }
 439
 440
 441             /* block returned doesn't start at beginning of entry, so we know
 442             ** that we will be splitting a block off the front of this one.  Create a new block
 443             ** from the beginning, add to list, and go to top of loop again.
 444             **
 445             ** create new block from high portion of block, so that top block
 446             ** starts at desired addr
 447             **/
 448             new_ent_addr = desired_min_addr;
 449             cvmx_bootmem_phy_set_next(new_ent_addr, cvmx_bootmem_phy_get_next(ent_addr));
 450             cvmx_bootmem_phy_set_size(new_ent_addr, cvmx_bootmem_phy_get_size(ent_addr) - (desired_min_addr - ent_addr));
 451             cvmx_bootmem_phy_set_size(ent_addr, desired_min_addr - ent_addr);
 452             cvmx_bootmem_phy_set_next(ent_addr, new_ent_addr);
 453             /* Loop again to handle actual alloc from new block */
 454         }
 455
 456         prev_addr = ent_addr;
 457         ent_addr = cvmx_bootmem_phy_get_next(ent_addr);
 458     }
 459 error_out:
 460     /* We didn't find anything, so return error */
 461     if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
 462         cvmx_spinlock_unlock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
 463     return(-1);
 464 }
 465
 466
 467
 468 int __cvmx_bootmem_phy_free(uint64_t phy_addr, uint64_t size, uint32_t flags)
 469 {
 470     uint64_t cur_addr;
 471     uint64_t prev_addr = 0;  /* zero is invalid */
 472     int retval = 0;
 473
 474 #ifdef DEBUG
 475     cvmx_dprintf("__cvmx_bootmem_phy_free addr: 0x%llx, size: 0x%llx\n", (unsigned long long)phy_addr, (unsigned long long)size);
 476 #endif
 477     if (cvmx_bootmem_desc->major_version > 3)
 478     {
 479         cvmx_dprintf("ERROR: Incompatible bootmem descriptor version: %d.%d at addr: %p\n",
 480                (int)cvmx_bootmem_desc->major_version, (int)cvmx_bootmem_desc->minor_version, cvmx_bootmem_desc);
 481         return(0);
 482     }
 483
 484     /* 0 is not a valid size for this allocator */
 485     if (!size)
 486         return(0);
 487
 488
 489     if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
 490         cvmx_spinlock_lock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
 491     cur_addr = cvmx_bootmem_desc->head_addr;
 492     if (cur_addr == 0 || phy_addr < cur_addr)
 493     {
 494         /* add at front of list - special case with changing head ptr */
 495         if (cur_addr && phy_addr + size > cur_addr)
 496             goto bootmem_free_done; /* error, overlapping section */
 497         else if (phy_addr + size == cur_addr)
 498         {
 499             /* Add to front of existing first block */
 500             cvmx_bootmem_phy_set_next(phy_addr, cvmx_bootmem_phy_get_next(cur_addr));
 501             cvmx_bootmem_phy_set_size(phy_addr, cvmx_bootmem_phy_get_size(cur_addr) + size);
 502             cvmx_bootmem_desc->head_addr = phy_addr;
 503
 504         }
 505         else
 506         {
 507             /* New block before first block */
 508             cvmx_bootmem_phy_set_next(phy_addr, cur_addr);  /* OK if cur_addr is 0 */
 509             cvmx_bootmem_phy_set_size(phy_addr, size);
 510             cvmx_bootmem_desc->head_addr = phy_addr;
 511         }
 512         retval = 1;
 513         goto bootmem_free_done;
 514     }
 515
 516     /* Find place in list to add block */
 517     while (cur_addr && phy_addr > cur_addr)
 518     {
 519         prev_addr = cur_addr;
 520         cur_addr = cvmx_bootmem_phy_get_next(cur_addr);
 521     }
 522
 523     if (!cur_addr)
 524     {
 525         /* We have reached the end of the list, add on to end, checking
 526         ** to see if we need to combine with last block
 527         **/
 528         if (prev_addr +  cvmx_bootmem_phy_get_size(prev_addr) == phy_addr)
 529         {
 530             cvmx_bootmem_phy_set_size(prev_addr, cvmx_bootmem_phy_get_size(prev_addr) + size);
 531         }
 532         else
 533         {
 534             cvmx_bootmem_phy_set_next(prev_addr, phy_addr);
 535             cvmx_bootmem_phy_set_size(phy_addr, size);
 536             cvmx_bootmem_phy_set_next(phy_addr, 0);
 537         }
 538         retval = 1;
 539         goto bootmem_free_done;
 540     }
 541     else
 542     {
 543         /* insert between prev and cur nodes, checking for merge with either/both */
 544
 545         if (prev_addr +  cvmx_bootmem_phy_get_size(prev_addr) == phy_addr)
 546         {
 547             /* Merge with previous */
 548             cvmx_bootmem_phy_set_size(prev_addr, cvmx_bootmem_phy_get_size(prev_addr) + size);
 549             if (phy_addr + size == cur_addr)
 550             {
 551                 /* Also merge with current */
 552                 cvmx_bootmem_phy_set_size(prev_addr, cvmx_bootmem_phy_get_size(cur_addr) + cvmx_bootmem_phy_get_size(prev_addr));
 553                 cvmx_bootmem_phy_set_next(prev_addr, cvmx_bootmem_phy_get_next(cur_addr));
 554             }
 555             retval = 1;
 556             goto bootmem_free_done;
 557         }
 558         else if (phy_addr + size == cur_addr)
 559         {
 560             /* Merge with current */
 561             cvmx_bootmem_phy_set_size(phy_addr, cvmx_bootmem_phy_get_size(cur_addr) + size);
 562             cvmx_bootmem_phy_set_next(phy_addr, cvmx_bootmem_phy_get_next(cur_addr));
 563             cvmx_bootmem_phy_set_next(prev_addr, phy_addr);
 564             retval = 1;
 565             goto bootmem_free_done;
 566         }
 567
 568         /* It is a standalone block, add in between prev and cur */
 569         cvmx_bootmem_phy_set_size(phy_addr, size);
 570         cvmx_bootmem_phy_set_next(phy_addr, cur_addr);
 571         cvmx_bootmem_phy_set_next(prev_addr, phy_addr);
 572
 573
 574     }
 575     retval = 1;
 576
 577 bootmem_free_done:
 578     if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
 579         cvmx_spinlock_unlock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
 580     return(retval);
 581
 582 }
 583
 584
 585
 586 void cvmx_bootmem_phy_list_print(void)
 587 {
 588     uint64_t addr;
 589
 590     addr = cvmx_bootmem_desc->head_addr;
 591     cvmx_dprintf("\n\n\nPrinting bootmem block list, descriptor: %p,  head is 0x%llx\n",
 592            cvmx_bootmem_desc, (unsigned long long)addr);
 593     cvmx_dprintf("Descriptor version: %d.%d\n", (int)cvmx_bootmem_desc->major_version, (int)cvmx_bootmem_desc->minor_version);
 594     if (cvmx_bootmem_desc->major_version > 3)
 595     {
 596         cvmx_dprintf("Warning: Bootmem descriptor version is newer than expected\n");
 597     }
 598     if (!addr)
 599     {
 600         cvmx_dprintf("mem list is empty!\n");
 601     }
 602     while (addr)
 603     {
 604         cvmx_dprintf("Block address: 0x%08qx, size: 0x%08qx, next: 0x%08qx\n",
 605                (unsigned long long)addr,
 606                (unsigned long long)cvmx_bootmem_phy_get_size(addr),
 607                (unsigned long long)cvmx_bootmem_phy_get_next(addr));
 608         addr = cvmx_bootmem_phy_get_next(addr);
 609     }
 610     cvmx_dprintf("\n\n");
 611
 612 }
 613
 614
 615 uint64_t cvmx_bootmem_phy_available_mem(uint64_t min_block_size)
 616 {
 617     uint64_t addr;
 618
 619     uint64_t available_mem = 0;
 620
 621     cvmx_spinlock_lock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
 622     addr = cvmx_bootmem_desc->head_addr;
 623     while (addr)
 624     {
 625         if (cvmx_bootmem_phy_get_size(addr) >= min_block_size)
 626             available_mem += cvmx_bootmem_phy_get_size(addr);
 627         addr = cvmx_bootmem_phy_get_next(addr);
 628     }
 629     cvmx_spinlock_unlock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
 630     return(available_mem);
 631
 632 }
 633
 634
 635
 636 cvmx_bootmem_named_block_desc_t * cvmx_bootmem_phy_named_block_find(char *name, uint32_t flags)
 637 {
 638     unsigned int i;
 639     cvmx_bootmem_named_block_desc_t *named_block_array_ptr;
 640
 641
 642 #ifdef DEBUG
 643     cvmx_dprintf("cvmx_bootmem_phy_named_block_find: %s\n", name);
 644 #endif
 645     /* Lock the structure to make sure that it is not being changed while we are
 646     ** examining it.
 647     */
 648     if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
 649         cvmx_spinlock_lock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
 650
 651 #if defined(__linux__) && !defined(CONFIG_OCTEON_U_BOOT)
 652 #ifdef CVMX_ABI_N32
 653     /* Need to use mmapped named block pointer in 32 bit linux apps */
 654 extern cvmx_bootmem_named_block_desc_t *linux32_named_block_array_ptr;
 655     named_block_array_ptr = linux32_named_block_array_ptr;
 656 #else
 657     /* Use XKPHYS for 64 bit linux */
 658     named_block_array_ptr = (cvmx_bootmem_named_block_desc_t *)cvmx_phys_to_ptr(cvmx_bootmem_desc->named_block_array_addr);
 659 #endif
 660 #else
 661     /* Simple executive case. (and u-boot)
 662     ** This could be in the low 1 meg of memory that is not 1-1 mapped, so we need use XKPHYS/KSEG0 addressing for it */
 663     named_block_array_ptr = CASTPTR(cvmx_bootmem_named_block_desc_t, CVMX_ADD_SEG32(CVMX_MIPS32_SPACE_KSEG0,cvmx_bootmem_desc->named_block_array_addr));
 664 #endif
 665
 666 #ifdef DEBUG
 667     cvmx_dprintf("cvmx_bootmem_phy_named_block_find: named_block_array_ptr: %p\n", named_block_array_ptr);
 668 #endif
 669     if (cvmx_bootmem_desc->major_version == 3)
 670     {
 671         for (i = 0; i < cvmx_bootmem_desc->named_block_num_blocks; i++)
 672         {
 673             if ((name && named_block_array_ptr[i].size && !strncmp(name, named_block_array_ptr[i].name, cvmx_bootmem_desc->named_block_name_len - 1))
 674                 || (!name && !named_block_array_ptr[i].size))
 675             {
 676                 if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
 677                     cvmx_spinlock_unlock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
 678
 679                 return(&(named_block_array_ptr[i]));
 680             }
 681         }
 682     }
 683     else
 684     {
 685         cvmx_dprintf("ERROR: Incompatible bootmem descriptor version: %d.%d at addr: %p\n",
 686                (int)cvmx_bootmem_desc->major_version, (int)cvmx_bootmem_desc->minor_version, cvmx_bootmem_desc);
 687     }
 688     if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
 689         cvmx_spinlock_unlock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
 690
 691     return(NULL);
 692 }
 693
 694 int cvmx_bootmem_phy_named_block_free(char *name, uint32_t flags)
 695 {
 696     cvmx_bootmem_named_block_desc_t *named_block_ptr;
 697
 698     if (cvmx_bootmem_desc->major_version != 3)
 699     {
 700         cvmx_dprintf("ERROR: Incompatible bootmem descriptor version: %d.%d at addr: %p\n",
 701                (int)cvmx_bootmem_desc->major_version, (int)cvmx_bootmem_desc->minor_version, cvmx_bootmem_desc);
 702         return(0);
 703     }
 704 #ifdef DEBUG
 705     cvmx_dprintf("cvmx_bootmem_phy_named_block_free: %s\n", name);
 706 #endif
 707
 708     /* Take lock here, as name lookup/block free/name free need to be atomic */
 709     cvmx_spinlock_lock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
 710
 711     named_block_ptr = cvmx_bootmem_phy_named_block_find(name, CVMX_BOOTMEM_FLAG_NO_LOCKING);
 712     if (named_block_ptr)
 713     {
 714 #ifdef DEBUG
 715         cvmx_dprintf("cvmx_bootmem_phy_named_block_free: %s, base: 0x%llx, size: 0x%llx\n", name, (unsigned long long)named_block_ptr->base_addr, (unsigned long long)named_block_ptr->size);
 716 #endif
 717         __cvmx_bootmem_phy_free(named_block_ptr->base_addr, named_block_ptr->size, CVMX_BOOTMEM_FLAG_NO_LOCKING);
 718         named_block_ptr->size = 0;
 719         /* Set size to zero to indicate block not used. */
 720     }
 721
 722     cvmx_spinlock_unlock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
 723
 724     return(!!named_block_ptr);  /* 0 on failure, 1 on success */
 725 }
 726
 727
 728
 729
 730
 731 int64_t cvmx_bootmem_phy_named_block_alloc(uint64_t size, uint64_t min_addr, uint64_t max_addr, uint64_t alignment, char *name, uint32_t flags)
 732 {
 733     int64_t addr_allocated;
 734     cvmx_bootmem_named_block_desc_t *named_block_desc_ptr;
 735
 736 #ifdef DEBUG
 737     cvmx_dprintf("cvmx_bootmem_phy_named_block_alloc: size: 0x%llx, min: 0x%llx, max: 0x%llx, align: 0x%llx, name: %s\n",
 738                  (unsigned long long)size,
 739                  (unsigned long long)min_addr,
 740                  (unsigned long long)max_addr,
 741                  (unsigned long long)alignment,
 742                  name);
 743 #endif
 744     if (cvmx_bootmem_desc->major_version != 3)
 745     {
 746         cvmx_dprintf("ERROR: Incompatible bootmem descriptor version: %d.%d at addr: %p\n",
 747                (int)cvmx_bootmem_desc->major_version, (int)cvmx_bootmem_desc->minor_version, cvmx_bootmem_desc);
 748         return(-1);
 749     }
 750
 751
 752     /* Take lock here, as name lookup/block alloc/name add need to be atomic */
 753
 754     if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
 755         cvmx_spinlock_lock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
 756
 757     /* Get pointer to first available named block descriptor */
 758     named_block_desc_ptr = cvmx_bootmem_phy_named_block_find(NULL, flags | CVMX_BOOTMEM_FLAG_NO_LOCKING);
 759
 760     /* Check to see if name already in use, return error if name
 761     ** not available or no more room for blocks.
 762     */
 763     if (cvmx_bootmem_phy_named_block_find(name, flags | CVMX_BOOTMEM_FLAG_NO_LOCKING) || !named_block_desc_ptr)
 764     {
 765         if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
 766             cvmx_spinlock_unlock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
 767         return(-1);
 768     }
 769
 770
 771     /* Round size up to mult of minimum alignment bytes
 772     ** We need the actual size allocated to allow for blocks to be coallesced
 773     ** when they are freed.  The alloc routine does the same rounding up
 774     ** on all allocations. */
 775     size = (size + (CVMX_BOOTMEM_ALIGNMENT_SIZE - 1)) & ~(CVMX_BOOTMEM_ALIGNMENT_SIZE - 1);
 776
 777     addr_allocated = cvmx_bootmem_phy_alloc(size, min_addr, max_addr, alignment, flags | CVMX_BOOTMEM_FLAG_NO_LOCKING);
 778     if (addr_allocated >= 0)
 779     {
 780         named_block_desc_ptr->base_addr = addr_allocated;
 781         named_block_desc_ptr->size = size;
 782         strncpy(named_block_desc_ptr->name, name, cvmx_bootmem_desc->named_block_name_len);
 783         named_block_desc_ptr->name[cvmx_bootmem_desc->named_block_name_len - 1] = 0;
 784     }
 785
 786     if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
 787         cvmx_spinlock_unlock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
 788
 789     return(addr_allocated);
 790 }
 791
 792
 793
 794
 795 void cvmx_bootmem_phy_named_block_print(void)
 796 {
 797     unsigned int i;
 798     int printed = 0;
 799
 800 #if defined(__linux__) && !defined(CONFIG_OCTEON_U_BOOT)
 801 #ifdef CVMX_ABI_N32
 802     /* Need to use mmapped named block pointer in 32 bit linux apps */
 803 extern cvmx_bootmem_named_block_desc_t *linux32_named_block_array_ptr;
 804     cvmx_bootmem_named_block_desc_t *named_block_array_ptr = linux32_named_block_array_ptr;
 805 #else
 806     /* Use XKPHYS for 64 bit linux */
 807     cvmx_bootmem_named_block_desc_t *named_block_array_ptr = (cvmx_bootmem_named_block_desc_t *)cvmx_phys_to_ptr(cvmx_bootmem_desc->named_block_array_addr);
 808 #endif
 809 #else
 810     /* Simple executive case. (and u-boot)
 811     ** This could be in the low 1 meg of memory that is not 1-1 mapped, so we need use XKPHYS/KSEG0 addressing for it */
 812     cvmx_bootmem_named_block_desc_t *named_block_array_ptr = CASTPTR(cvmx_bootmem_named_block_desc_t, CVMX_ADD_SEG32(CVMX_MIPS32_SPACE_KSEG0,cvmx_bootmem_desc->named_block_array_addr));
 813 #endif
 814 #ifdef DEBUG
 815     cvmx_dprintf("cvmx_bootmem_phy_named_block_print, desc addr: %p\n", cvmx_bootmem_desc);
 816 #endif
 817     if (cvmx_bootmem_desc->major_version != 3)
 818     {
 819         cvmx_dprintf("ERROR: Incompatible bootmem descriptor version: %d.%d at addr: %p\n",
 820                (int)cvmx_bootmem_desc->major_version, (int)cvmx_bootmem_desc->minor_version, cvmx_bootmem_desc);
 821         return;
 822     }
 823     cvmx_dprintf("List of currently allocated named bootmem blocks:\n");
 824     for (i = 0; i < cvmx_bootmem_desc->named_block_num_blocks; i++)
 825     {
 826         if (named_block_array_ptr[i].size)
 827         {
 828             printed++;
 829             cvmx_dprintf("Name: %s, address: 0x%08qx, size: 0x%08qx, index: %d\n",
 830                    named_block_array_ptr[i].name,
 831                    (unsigned long long)named_block_array_ptr[i].base_addr,
 832                    (unsigned long long)named_block_array_ptr[i].size,
 833                    i);
 834
 835         }
 836     }
 837     if (!printed)
 838     {
 839         cvmx_dprintf("No named bootmem blocks exist.\n");
 840     }
 841
 842 }
 843
 844
 845 /* Real physical addresses of memory regions */
 846 #define OCTEON_DDR0_BASE    (0x0ULL)
 847 #define OCTEON_DDR0_SIZE    (0x010000000ULL)
 848 #define OCTEON_DDR1_BASE    (0x410000000ULL)
 849 #define OCTEON_DDR1_SIZE    (0x010000000ULL)
 850 #define OCTEON_DDR2_BASE    (0x020000000ULL)
 851 #define OCTEON_DDR2_SIZE    (0x3e0000000ULL)
 852 #define OCTEON_MAX_PHY_MEM_SIZE (16*1024*1024*1024ULL)
 853 int64_t cvmx_bootmem_phy_mem_list_init(uint64_t mem_size, uint32_t low_reserved_bytes, cvmx_bootmem_desc_t *desc_buffer)
 854 {
 855     uint64_t cur_block_addr;
 856     int64_t addr;
 857
 858 #ifdef DEBUG
 859     cvmx_dprintf("cvmx_bootmem_phy_mem_list_init (arg desc ptr: %p, cvmx_bootmem_desc: %p)\n", desc_buffer, cvmx_bootmem_desc);
 860 #endif
 861
 862     /* Descriptor buffer needs to be in 32 bit addressable space to be compatible with
 863     ** 32 bit applications */
 864     if (!desc_buffer)
 865     {
 866         cvmx_dprintf("ERROR: no memory for cvmx_bootmem descriptor provided\n");
 867         return 0;
 868     }
 869
 870     if (mem_size > OCTEON_MAX_PHY_MEM_SIZE)
 871     {
 872         mem_size = OCTEON_MAX_PHY_MEM_SIZE;
 873         cvmx_dprintf("ERROR: requested memory size too large, truncating to maximum size\n");
 874     }
 875
 876     if (cvmx_bootmem_desc)
 877         return 1;
 878
 879     /* Initialize cvmx pointer to descriptor */
 880     cvmx_bootmem_init(desc_buffer);
 881
 882     /* Set up global pointer to start of list, exclude low 64k for exception vectors, space for global descriptor */
 883     memset(cvmx_bootmem_desc, 0x0, sizeof(cvmx_bootmem_desc_t));
 884     /* Set version of bootmem descriptor */
 885     cvmx_bootmem_desc->major_version = CVMX_BOOTMEM_DESC_MAJ_VER;
 886     cvmx_bootmem_desc->minor_version = CVMX_BOOTMEM_DESC_MIN_VER;
 887
 888     cur_block_addr = cvmx_bootmem_desc->head_addr = (OCTEON_DDR0_BASE + low_reserved_bytes);
 889
 890     cvmx_bootmem_desc->head_addr = 0;
 891
 892     if (mem_size <= OCTEON_DDR0_SIZE)
 893     {
 894         __cvmx_bootmem_phy_free(cur_block_addr, mem_size - low_reserved_bytes, 0);
 895         goto frees_done;
 896     }
 897
 898     __cvmx_bootmem_phy_free(cur_block_addr, OCTEON_DDR0_SIZE - low_reserved_bytes, 0);
 899
 900     mem_size -= OCTEON_DDR0_SIZE;
 901
 902     /* Add DDR2 block next if present */
 903     if (mem_size > OCTEON_DDR1_SIZE)
 904     {
 905         __cvmx_bootmem_phy_free(OCTEON_DDR1_BASE, OCTEON_DDR1_SIZE, 0);
 906         __cvmx_bootmem_phy_free(OCTEON_DDR2_BASE, mem_size - OCTEON_DDR1_SIZE, 0);
 907     }
 908     else
 909     {
 910         __cvmx_bootmem_phy_free(OCTEON_DDR1_BASE, mem_size, 0);
 911
 912     }
 913 frees_done:
 914
 915     /* Initialize the named block structure */
 916     cvmx_bootmem_desc->named_block_name_len = CVMX_BOOTMEM_NAME_LEN;
 917     cvmx_bootmem_desc->named_block_num_blocks = CVMX_BOOTMEM_NUM_NAMED_BLOCKS;
 918     cvmx_bootmem_desc->named_block_array_addr = 0;
 919
 920     /* Allocate this near the top of the low 256 MBytes of memory */
 921     addr = cvmx_bootmem_phy_alloc(CVMX_BOOTMEM_NUM_NAMED_BLOCKS * sizeof(cvmx_bootmem_named_block_desc_t),0, 0x10000000, 0 ,CVMX_BOOTMEM_FLAG_END_ALLOC);
 922     if (addr >= 0)
 923         cvmx_bootmem_desc->named_block_array_addr = addr;
 924
 925 #ifdef DEBUG
 926     cvmx_dprintf("cvmx_bootmem_phy_mem_list_init: named_block_array_addr: 0x%llx)\n", (unsigned long long)cvmx_bootmem_desc->named_block_array_addr);
 927 #endif
 928     if (!cvmx_bootmem_desc->named_block_array_addr)
 929     {
 930         cvmx_dprintf("FATAL ERROR: unable to allocate memory for bootmem descriptor!\n");
 931         return(0);
 932     }
 933     memset((void *)(unsigned long)cvmx_bootmem_desc->named_block_array_addr, 0x0, CVMX_BOOTMEM_NUM_NAMED_BLOCKS * sizeof(cvmx_bootmem_named_block_desc_t));
 934
 935     return(1);
 936 }
 937
 938
 939 void cvmx_bootmem_lock(void)
 940 {
 941     cvmx_spinlock_lock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
 942 }
 943
 944 void cvmx_bootmem_unlock(void)
 945 {
 946     cvmx_spinlock_unlock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
 947 }
 948
 949 void *__cvmx_bootmem_internal_get_desc_ptr(void)
 950 {
 951     return(cvmx_bootmem_desc);
 952 }