2 * Copyright(c) 2002-2011 Exar Corp.
5 * Redistribution and use in source and binary forms, with or without
6 * modification are permitted provided the following conditions are met:
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * 3. Neither the name of the Exar Corporation nor the names of its
16 * contributors may be used to endorse or promote products derived from
17 * this software without specific prior written permission.
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
33 #include <dev/vxge/vxgehal/vxgehal.h>
38 * Will resize mempool up to %num_allocate value.
40 static vxge_hal_status_e
42 vxge_hal_mempool_t *mempool,
46 u32 i, j, k, item_index, is_last;
47 u32 first_time = mempool->memblocks_allocated == 0 ? 1 : 0;
48 u32 n_items = mempool->items_per_memblock;
49 u32 start_block_idx = mempool->memblocks_allocated;
50 u32 end_block_idx = mempool->memblocks_allocated + num_allocate;
51 __hal_device_t *hldev;
53 vxge_assert(mempool != NULL);
55 hldev = (__hal_device_t *) mempool->devh;
57 vxge_hal_trace_log_mm("==> %s:%s:%d",
58 __FILE__, __func__, __LINE__);
60 vxge_hal_trace_log_mm(
61 "mempool = 0x"VXGE_OS_STXFMT", num_allocate = %d, "
62 "num_allocated = 0x"VXGE_OS_STXFMT, (ptr_t) mempool,
63 num_allocate, (ptr_t) num_allocated);
67 if (end_block_idx > mempool->memblocks_max) {
68 vxge_hal_err_log_mm("%s",
69 "__hal_mempool_grow: can grow anymore");
70 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d",
71 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY);
72 return (VXGE_HAL_ERR_OUT_OF_MEMORY);
75 for (i = start_block_idx; i < end_block_idx; i++) {
78 vxge_hal_mempool_dma_t *dma_object;
80 is_last = ((end_block_idx - 1) == i);
81 dma_object = mempool->memblocks_dma_arr + i;
84 * allocate memblock's private part. Each DMA memblock
85 * has a space allocated for item's private usage upon
86 * mempool's user request. Each time mempool grows, it will
87 * allocate new memblock and its private part at once.
88 * This helps to minimize memory usage a lot.
90 mempool->memblocks_priv_arr[i] = vxge_os_malloc(
91 ((__hal_device_t *) mempool->devh)->header.pdev,
92 mempool->items_priv_size * n_items);
93 if (mempool->memblocks_priv_arr[i] == NULL) {
95 vxge_hal_err_log_mm("memblock_priv[%d]: \
96 out of virtual memory, "
97 "requested %d(%d:%d) bytes", i,
98 mempool->items_priv_size * n_items,
99 mempool->items_priv_size, n_items);
100 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d",
101 __FILE__, __func__, __LINE__,
102 VXGE_HAL_ERR_OUT_OF_MEMORY);
103 return (VXGE_HAL_ERR_OUT_OF_MEMORY);
107 vxge_os_memzero(mempool->memblocks_priv_arr[i],
108 mempool->items_priv_size * n_items);
110 /* allocate DMA-capable memblock */
111 mempool->memblocks_arr[i] =
112 __hal_blockpool_malloc(mempool->devh,
113 mempool->memblock_size,
116 &dma_object->acc_handle);
117 if (mempool->memblocks_arr[i] == NULL) {
119 ((__hal_device_t *) mempool->devh)->header.pdev,
120 mempool->memblocks_priv_arr[i],
121 mempool->items_priv_size * n_items);
122 vxge_hal_err_log_mm("memblock[%d]: \
123 out of DMA memory", i);
124 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d",
125 __FILE__, __func__, __LINE__,
126 VXGE_HAL_ERR_OUT_OF_MEMORY);
127 return (VXGE_HAL_ERR_OUT_OF_MEMORY);
131 mempool->memblocks_allocated++;
133 vxge_os_memzero(mempool->memblocks_arr[i],
134 mempool->memblock_size);
136 the_memblock = mempool->memblocks_arr[i];
138 /* fill the items hash array */
139 for (j = 0; j < n_items; j++) {
140 item_index = i * n_items + j;
142 if (first_time && (item_index >= mempool->items_initial))
145 mempool->items_arr[item_index] =
146 ((char *) the_memblock + j *mempool->item_size);
148 /* let caller to do more job on each item */
149 if (mempool->item_func_alloc != NULL) {
150 vxge_hal_status_e status;
152 if ((status = mempool->item_func_alloc(
157 mempool->items_arr[item_index],
160 mempool->userdata)) != VXGE_HAL_OK) {
162 if (mempool->item_func_free != NULL) {
164 for (k = 0; k < j; k++) {
166 item_index = i * n_items + k;
168 (void) mempool->item_func_free(
172 mempool->items_arr[item_index],
178 vxge_os_free(((__hal_device_t *)
179 mempool->devh)->header.pdev,
180 mempool->memblocks_priv_arr[i],
181 mempool->items_priv_size *
184 __hal_blockpool_free(mempool->devh,
186 mempool->memblock_size,
189 &dma_object->acc_handle);
192 mempool->memblocks_allocated--;
197 mempool->items_current = item_index + 1;
200 vxge_hal_info_log_mm(
201 "memblock%d: allocated %dk, vaddr 0x"VXGE_OS_STXFMT", "
202 "dma_addr 0x"VXGE_OS_STXFMT,
203 i, mempool->memblock_size / 1024,
204 (ptr_t) mempool->memblocks_arr[i], dma_object->addr);
206 if (first_time && mempool->items_current ==
207 mempool->items_initial) {
212 vxge_hal_trace_log_mm("<== %s:%s:%d Result: 0",
213 __FILE__, __func__, __LINE__);
215 return (VXGE_HAL_OK);
219 * vxge_hal_mempool_create
226 * This function will create memory pool object. Pool may grow but will
227 * never shrink. Pool consists of number of dynamically allocated blocks
228 * with size enough to hold %items_initial number of items. Memory is
229 * DMA-able but client must map/unmap before interoperating with the device.
230 * See also: vxge_os_dma_map(), vxge_hal_dma_unmap(), vxge_hal_status_e {}.
233 vxge_hal_mempool_create(
234 vxge_hal_device_h devh,
240 vxge_hal_mempool_item_f item_func_alloc,
241 vxge_hal_mempool_item_f item_func_free,
244 vxge_hal_status_e status;
245 u32 memblocks_to_allocate;
246 vxge_hal_mempool_t *mempool;
247 __hal_device_t *hldev;
250 vxge_assert(devh != NULL);
252 hldev = (__hal_device_t *) devh;
254 vxge_hal_trace_log_mm("==> %s:%s:%d",
255 __FILE__, __func__, __LINE__);
257 vxge_hal_trace_log_mm(
258 "devh = 0x"VXGE_OS_STXFMT", memblock_size = %d, item_size = %d, "
259 "items_priv_size = %d, items_initial = %d, items_max = %d, "
260 "item_func_alloc = 0x"VXGE_OS_STXFMT", "
261 "item_func_free = 0x"VXGE_OS_STXFMT", "
262 "userdata = 0x"VXGE_OS_STXFMT, (ptr_t) devh,
263 memblock_size, item_size, items_priv_size,
264 items_initial, items_max, (ptr_t) item_func_alloc,
265 (ptr_t) item_func_free, (ptr_t) userdata);
267 if (memblock_size < item_size) {
269 "memblock_size %d < item_size %d: misconfiguration",
270 memblock_size, item_size);
271 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d",
272 __FILE__, __func__, __LINE__, VXGE_HAL_FAIL);
276 mempool = (vxge_hal_mempool_t *) vxge_os_malloc(
277 ((__hal_device_t *) devh)->header.pdev, sizeof(vxge_hal_mempool_t));
278 if (mempool == NULL) {
279 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d",
280 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY);
283 vxge_os_memzero(mempool, sizeof(vxge_hal_mempool_t));
285 mempool->devh = devh;
286 mempool->memblock_size = memblock_size;
287 mempool->items_max = items_max;
288 mempool->items_initial = items_initial;
289 mempool->item_size = item_size;
290 mempool->items_priv_size = items_priv_size;
291 mempool->item_func_alloc = item_func_alloc;
292 mempool->item_func_free = item_func_free;
293 mempool->userdata = userdata;
295 mempool->memblocks_allocated = 0;
297 if (memblock_size != VXGE_OS_HOST_PAGE_SIZE)
298 mempool->dma_flags = VXGE_OS_DMA_CACHELINE_ALIGNED;
300 #if defined(VXGE_HAL_DMA_CONSISTENT)
301 mempool->dma_flags |= VXGE_OS_DMA_CONSISTENT;
303 mempool->dma_flags |= VXGE_OS_DMA_STREAMING;
306 mempool->items_per_memblock = memblock_size / item_size;
308 mempool->memblocks_max = (items_max + mempool->items_per_memblock - 1) /
309 mempool->items_per_memblock;
311 /* allocate array of memblocks */
312 mempool->memblocks_arr = (void **)vxge_os_malloc(
313 ((__hal_device_t *) mempool->devh)->header.pdev,
314 sizeof(void *) * mempool->memblocks_max);
315 if (mempool->memblocks_arr == NULL) {
316 vxge_hal_mempool_destroy(mempool);
317 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d",
318 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY);
321 vxge_os_memzero(mempool->memblocks_arr,
322 sizeof(void *) * mempool->memblocks_max);
324 /* allocate array of private parts of items per memblocks */
325 mempool->memblocks_priv_arr = (void **)vxge_os_malloc(
326 ((__hal_device_t *) mempool->devh)->header.pdev,
327 sizeof(void *) * mempool->memblocks_max);
328 if (mempool->memblocks_priv_arr == NULL) {
329 vxge_hal_mempool_destroy(mempool);
330 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d",
331 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY);
334 vxge_os_memzero(mempool->memblocks_priv_arr,
335 sizeof(void *) * mempool->memblocks_max);
337 /* allocate array of memblocks DMA objects */
338 mempool->memblocks_dma_arr =
339 (vxge_hal_mempool_dma_t *) vxge_os_malloc(
340 ((__hal_device_t *) mempool->devh)->header.pdev,
341 sizeof(vxge_hal_mempool_dma_t) * mempool->memblocks_max);
343 if (mempool->memblocks_dma_arr == NULL) {
344 vxge_hal_mempool_destroy(mempool);
345 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d",
346 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY);
349 vxge_os_memzero(mempool->memblocks_dma_arr,
350 sizeof(vxge_hal_mempool_dma_t) * mempool->memblocks_max);
352 /* allocate hash array of items */
353 mempool->items_arr = (void **)vxge_os_malloc(
354 ((__hal_device_t *) mempool->devh)->header.pdev,
355 sizeof(void *) * mempool->items_max);
356 if (mempool->items_arr == NULL) {
357 vxge_hal_mempool_destroy(mempool);
358 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d",
359 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY);
362 vxge_os_memzero(mempool->items_arr,
363 sizeof(void *) * mempool->items_max);
365 mempool->shadow_items_arr = (void **)vxge_os_malloc(
366 ((__hal_device_t *) mempool->devh)->header.pdev,
367 sizeof(void *) * mempool->items_max);
368 if (mempool->shadow_items_arr == NULL) {
369 vxge_hal_mempool_destroy(mempool);
370 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d",
371 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY);
374 vxge_os_memzero(mempool->shadow_items_arr,
375 sizeof(void *) * mempool->items_max);
377 /* calculate initial number of memblocks */
378 memblocks_to_allocate = (mempool->items_initial +
379 mempool->items_per_memblock - 1) /
380 mempool->items_per_memblock;
382 vxge_hal_info_log_mm("allocating %d memblocks, "
383 "%d items per memblock", memblocks_to_allocate,
384 mempool->items_per_memblock);
386 /* pre-allocate the mempool */
387 status = __hal_mempool_grow(mempool, memblocks_to_allocate, &allocated);
388 vxge_os_memcpy(mempool->shadow_items_arr, mempool->items_arr,
389 sizeof(void *) * mempool->items_max);
390 if (status != VXGE_HAL_OK) {
391 vxge_hal_mempool_destroy(mempool);
392 vxge_hal_trace_log_mm("<== %s:%s:%d Result: %d",
393 __FILE__, __func__, __LINE__, VXGE_HAL_ERR_OUT_OF_MEMORY);
397 vxge_hal_info_log_mm(
398 "total: allocated %dk of DMA-capable memory",
399 mempool->memblock_size * allocated / 1024);
401 vxge_hal_trace_log_mm("<== %s:%s:%d Result: 0",
402 __FILE__, __func__, __LINE__);
408 * vxge_hal_mempool_destroy
411 vxge_hal_mempool_destroy(
412 vxge_hal_mempool_t *mempool)
414 u32 i, j, item_index;
415 __hal_device_t *hldev;
417 vxge_assert(mempool != NULL);
419 hldev = (__hal_device_t *) mempool->devh;
421 vxge_hal_trace_log_mm("==> %s:%s:%d",
422 __FILE__, __func__, __LINE__);
424 vxge_hal_trace_log_mm("mempool = 0x"VXGE_OS_STXFMT,
427 for (i = 0; i < mempool->memblocks_allocated; i++) {
428 vxge_hal_mempool_dma_t *dma_object;
430 vxge_assert(mempool->memblocks_arr[i]);
431 vxge_assert(mempool->memblocks_dma_arr + i);
433 dma_object = mempool->memblocks_dma_arr + i;
435 for (j = 0; j < mempool->items_per_memblock; j++) {
436 item_index = i * mempool->items_per_memblock + j;
438 /* to skip last partially filled(if any) memblock */
439 if (item_index >= mempool->items_current)
442 /* let caller to do more job on each item */
443 if (mempool->item_func_free != NULL) {
445 mempool->item_func_free(mempool,
446 mempool->memblocks_arr[i],
448 mempool->shadow_items_arr[item_index],
449 item_index, /* unused */ -1,
454 vxge_os_free(hldev->header.pdev,
455 mempool->memblocks_priv_arr[i],
456 mempool->items_priv_size * mempool->items_per_memblock);
458 __hal_blockpool_free(hldev,
459 mempool->memblocks_arr[i],
460 mempool->memblock_size,
463 &dma_object->acc_handle);
466 if (mempool->items_arr) {
467 vxge_os_free(hldev->header.pdev,
468 mempool->items_arr, sizeof(void *) * mempool->items_max);
471 if (mempool->shadow_items_arr) {
472 vxge_os_free(hldev->header.pdev,
473 mempool->shadow_items_arr,
474 sizeof(void *) * mempool->items_max);
477 if (mempool->memblocks_dma_arr) {
478 vxge_os_free(hldev->header.pdev,
479 mempool->memblocks_dma_arr,
480 sizeof(vxge_hal_mempool_dma_t) *
481 mempool->memblocks_max);
484 if (mempool->memblocks_priv_arr) {
485 vxge_os_free(hldev->header.pdev,
486 mempool->memblocks_priv_arr,
487 sizeof(void *) * mempool->memblocks_max);
490 if (mempool->memblocks_arr) {
491 vxge_os_free(hldev->header.pdev,
492 mempool->memblocks_arr,
493 sizeof(void *) * mempool->memblocks_max);
496 vxge_os_free(hldev->header.pdev,
497 mempool, sizeof(vxge_hal_mempool_t));
499 vxge_hal_trace_log_mm("<== %s:%s:%d Result: 0",
500 __FILE__, __func__, __LINE__);
504 * vxge_hal_check_alignment - Check buffer alignment and calculate the
505 * "misaligned" portion.
506 * @dma_pointer: DMA address of the buffer.
507 * @size: Buffer size, in bytes.
508 * @alignment: Alignment "granularity" (see below), in bytes.
509 * @copy_size: Maximum number of bytes to "extract" from the buffer
510 * (in order to spost it as a separate scatter-gather entry). See below.
512 * Check buffer alignment and calculate "misaligned" portion, if exists.
513 * The buffer is considered aligned if its address is multiple of
514 * the specified @alignment. If this is the case,
515 * vxge_hal_check_alignment() returns zero.
516 * Otherwise, vxge_hal_check_alignment() uses the last argument,
518 * to calculate the size to "extract" from the buffer. The @copy_size
519 * may or may not be equal @alignment. The difference between these two
520 * arguments is that the @alignment is used to make the decision: aligned
521 * or not aligned. While the @copy_size is used to calculate the portion
522 * of the buffer to "extract", i.e. to post as a separate entry in the
523 * transmit descriptor. For example, the combination
524 * @alignment = 8 and @copy_size = 64 will work okay on AMD Opteron boxes.
526 * Note: @copy_size should be a multiple of @alignment. In many practical
527 * cases @copy_size and @alignment will probably be equal.
529 * See also: vxge_hal_fifo_txdl_buffer_set_aligned().
532 vxge_hal_check_alignment(
533 dma_addr_t dma_pointer,
540 misaligned_size = (int)(dma_pointer & (alignment - 1));
541 if (!misaligned_size) {
545 if (size > copy_size) {
546 misaligned_size = (int)(dma_pointer & (copy_size - 1));
547 misaligned_size = copy_size - misaligned_size;
549 misaligned_size = size;
552 return (misaligned_size);