2 * Copyright (c) 2018-2019 Cavium, Inc.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
16 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
19 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
20 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
21 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
22 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
24 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
25 * POSSIBILITY OF SUCH DAMAGE.
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
37 #include "ecore_status.h"
38 #include "ecore_ll2.h"
39 #include "ecore_ooo.h"
40 #include "ecore_iscsi.h"
41 #include "ecore_cxt.h"
43 * Static OOO functions
46 static struct ecore_ooo_archipelago *
47 ecore_ooo_seek_archipelago(struct ecore_ooo_info *p_ooo_info, u32 cid)
49 u32 idx = (cid & 0xffff) - p_ooo_info->cid_base;
50 struct ecore_ooo_archipelago *p_archipelago;
52 if (idx >= p_ooo_info->max_num_archipelagos)
55 p_archipelago = &p_ooo_info->p_archipelagos_mem[idx];
57 if (OSAL_LIST_IS_EMPTY(&p_archipelago->isles_list))
63 static struct ecore_ooo_isle *ecore_ooo_seek_isle(struct ecore_hwfn *p_hwfn,
64 struct ecore_ooo_info *p_ooo_info,
67 struct ecore_ooo_archipelago *p_archipelago = OSAL_NULL;
68 struct ecore_ooo_isle *p_isle = OSAL_NULL;
69 u8 the_num_of_isle = 1;
71 p_archipelago = ecore_ooo_seek_archipelago(p_ooo_info, cid);
73 DP_NOTICE(p_hwfn, true,
74 "Connection %d is not found in OOO list\n", cid);
78 OSAL_LIST_FOR_EACH_ENTRY(p_isle,
79 &p_archipelago->isles_list,
80 list_entry, struct ecore_ooo_isle) {
81 if (the_num_of_isle == isle)
89 void ecore_ooo_save_history_entry(struct ecore_ooo_info *p_ooo_info,
90 struct ooo_opaque *p_cqe)
92 struct ecore_ooo_history *p_history = &p_ooo_info->ooo_history;
94 if (p_history->head_idx == p_history->num_of_cqes)
95 p_history->head_idx = 0;
96 p_history->p_cqes[p_history->head_idx] = *p_cqe;
97 p_history->head_idx++;
100 //#ifdef CONFIG_ECORE_ISCSI
101 #if defined(CONFIG_ECORE_ISCSI) || defined(CONFIG_ECORE_IWARP)
102 enum _ecore_status_t ecore_ooo_alloc(struct ecore_hwfn *p_hwfn)
104 u16 max_num_archipelagos = 0, cid_base;
105 struct ecore_ooo_info *p_ooo_info;
106 u16 max_num_isles = 0;
109 switch (p_hwfn->hw_info.personality) {
110 case ECORE_PCI_ISCSI:
111 max_num_archipelagos =
112 p_hwfn->pf_params.iscsi_pf_params.num_cons;
113 cid_base =(u16)ecore_cxt_get_proto_cid_start(p_hwfn,
116 case ECORE_PCI_ETH_RDMA:
117 case ECORE_PCI_ETH_IWARP:
118 max_num_archipelagos =
119 (u16)ecore_cxt_get_proto_cid_count(p_hwfn,
122 cid_base = (u16)ecore_cxt_get_proto_cid_start(p_hwfn,
126 DP_NOTICE(p_hwfn, true,
127 "Failed to allocate ecore_ooo_info: unknown personalization\n");
131 max_num_isles = ECORE_MAX_NUM_ISLES + max_num_archipelagos;
133 if (!max_num_archipelagos) {
134 DP_NOTICE(p_hwfn, true,
135 "Failed to allocate ecore_ooo_info: unknown amount of connections\n");
139 p_ooo_info = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL,
140 sizeof(*p_ooo_info));
142 DP_NOTICE(p_hwfn, true, "Failed to allocate ecore_ooo_info\n");
145 p_ooo_info->cid_base = cid_base; /* We look only at the icid */
146 p_ooo_info->max_num_archipelagos = max_num_archipelagos;
148 OSAL_LIST_INIT(&p_ooo_info->free_buffers_list);
149 OSAL_LIST_INIT(&p_ooo_info->ready_buffers_list);
150 OSAL_LIST_INIT(&p_ooo_info->free_isles_list);
152 p_ooo_info->p_isles_mem =
153 OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL,
154 sizeof(struct ecore_ooo_isle) *
156 if (!p_ooo_info->p_isles_mem) {
157 DP_NOTICE(p_hwfn,true,
158 "Failed to allocate ecore_ooo_info (isles)\n");
162 for (i = 0; i < max_num_isles; i++) {
163 OSAL_LIST_INIT(&p_ooo_info->p_isles_mem[i].buffers_list);
164 OSAL_LIST_PUSH_TAIL(&p_ooo_info->p_isles_mem[i].list_entry,
165 &p_ooo_info->free_isles_list);
168 p_ooo_info->p_archipelagos_mem =
169 OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL,
170 sizeof(struct ecore_ooo_archipelago) *
171 max_num_archipelagos);
172 if (!p_ooo_info->p_archipelagos_mem) {
173 DP_NOTICE(p_hwfn,true,
174 "Failed to allocate ecore_ooo_info(archpelagos)\n");
175 goto no_archipelagos_mem;
178 for (i = 0; i < max_num_archipelagos; i++) {
179 OSAL_LIST_INIT(&p_ooo_info->p_archipelagos_mem[i].isles_list);
182 p_ooo_info->ooo_history.p_cqes =
183 OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL,
184 sizeof(struct ooo_opaque) *
185 ECORE_MAX_NUM_OOO_HISTORY_ENTRIES);
186 if (!p_ooo_info->ooo_history.p_cqes) {
187 DP_NOTICE(p_hwfn,true,
188 "Failed to allocate ecore_ooo_info(history)\n");
191 p_ooo_info->ooo_history.num_of_cqes =
192 ECORE_MAX_NUM_OOO_HISTORY_ENTRIES;
194 p_hwfn->p_ooo_info = p_ooo_info;
195 return ECORE_SUCCESS;
198 OSAL_FREE(p_hwfn->p_dev, p_ooo_info->p_archipelagos_mem);
200 OSAL_FREE(p_hwfn->p_dev, p_ooo_info->p_isles_mem);
202 OSAL_FREE(p_hwfn->p_dev, p_ooo_info);
207 void ecore_ooo_release_connection_isles(struct ecore_ooo_info *p_ooo_info,
210 struct ecore_ooo_archipelago *p_archipelago;
211 struct ecore_ooo_buffer *p_buffer;
212 struct ecore_ooo_isle *p_isle;
214 p_archipelago = ecore_ooo_seek_archipelago(p_ooo_info, cid);
218 while (!OSAL_LIST_IS_EMPTY(&p_archipelago->isles_list)) {
219 p_isle = OSAL_LIST_FIRST_ENTRY(
220 &p_archipelago->isles_list,
221 struct ecore_ooo_isle, list_entry);
224 #pragma warning(suppress : 6011 28182)
226 OSAL_LIST_REMOVE_ENTRY(&p_isle->list_entry,
227 &p_archipelago->isles_list);
229 while (!OSAL_LIST_IS_EMPTY(&p_isle->buffers_list)) {
231 OSAL_LIST_FIRST_ENTRY(
232 &p_isle->buffers_list ,
233 struct ecore_ooo_buffer, list_entry);
235 if (p_buffer == OSAL_NULL)
238 #pragma warning(suppress : 6011 28182)
240 OSAL_LIST_REMOVE_ENTRY(&p_buffer->list_entry,
241 &p_isle->buffers_list);
242 OSAL_LIST_PUSH_TAIL(&p_buffer->list_entry,
243 &p_ooo_info->free_buffers_list);
245 OSAL_LIST_PUSH_TAIL(&p_isle->list_entry,
246 &p_ooo_info->free_isles_list);
251 void ecore_ooo_release_all_isles(struct ecore_ooo_info *p_ooo_info)
253 struct ecore_ooo_archipelago *p_archipelago;
254 struct ecore_ooo_buffer *p_buffer;
255 struct ecore_ooo_isle *p_isle;
258 for (i = 0; i < p_ooo_info->max_num_archipelagos; i++) {
259 p_archipelago = &(p_ooo_info->p_archipelagos_mem[i]);
262 #pragma warning(suppress : 6011 28182)
264 while (!OSAL_LIST_IS_EMPTY(&p_archipelago->isles_list)) {
265 p_isle = OSAL_LIST_FIRST_ENTRY(
266 &p_archipelago->isles_list,
267 struct ecore_ooo_isle, list_entry);
270 #pragma warning(suppress : 6011 28182)
272 OSAL_LIST_REMOVE_ENTRY(&p_isle->list_entry,
273 &p_archipelago->isles_list);
275 while (!OSAL_LIST_IS_EMPTY(&p_isle->buffers_list)) {
277 OSAL_LIST_FIRST_ENTRY(
278 &p_isle->buffers_list ,
279 struct ecore_ooo_buffer, list_entry);
281 if (p_buffer == OSAL_NULL)
284 #pragma warning(suppress : 6011 28182)
286 OSAL_LIST_REMOVE_ENTRY(&p_buffer->list_entry,
287 &p_isle->buffers_list);
288 OSAL_LIST_PUSH_TAIL(&p_buffer->list_entry,
289 &p_ooo_info->free_buffers_list);
291 OSAL_LIST_PUSH_TAIL(&p_isle->list_entry,
292 &p_ooo_info->free_isles_list);
295 if (!OSAL_LIST_IS_EMPTY(&p_ooo_info->ready_buffers_list)) {
296 OSAL_LIST_SPLICE_TAIL_INIT(&p_ooo_info->ready_buffers_list,
297 &p_ooo_info->free_buffers_list);
301 //#ifdef CONFIG_ECORE_ISCSI
302 #if defined(CONFIG_ECORE_ISCSI) || defined(CONFIG_ECORE_IWARP)
303 void ecore_ooo_setup(struct ecore_hwfn *p_hwfn)
305 ecore_ooo_release_all_isles(p_hwfn->p_ooo_info);
306 OSAL_MEM_ZERO(p_hwfn->p_ooo_info->ooo_history.p_cqes,
307 p_hwfn->p_ooo_info->ooo_history.num_of_cqes *
308 sizeof(struct ooo_opaque));
309 p_hwfn->p_ooo_info->ooo_history.head_idx = 0;
312 void ecore_ooo_free(struct ecore_hwfn *p_hwfn)
314 struct ecore_ooo_info *p_ooo_info = p_hwfn->p_ooo_info;
315 struct ecore_ooo_buffer *p_buffer;
320 ecore_ooo_release_all_isles(p_ooo_info);
321 while (!OSAL_LIST_IS_EMPTY(&p_ooo_info->free_buffers_list)) {
322 p_buffer = OSAL_LIST_FIRST_ENTRY(&p_ooo_info->
324 struct ecore_ooo_buffer,
326 if (p_buffer == OSAL_NULL)
329 #pragma warning(suppress : 6011 28182)
331 OSAL_LIST_REMOVE_ENTRY(&p_buffer->list_entry,
332 &p_ooo_info->free_buffers_list);
333 OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev,
334 p_buffer->rx_buffer_virt_addr,
335 p_buffer->rx_buffer_phys_addr,
336 p_buffer->rx_buffer_size);
337 OSAL_FREE(p_hwfn->p_dev, p_buffer);
340 OSAL_FREE(p_hwfn->p_dev, p_ooo_info->p_isles_mem);
341 OSAL_FREE(p_hwfn->p_dev, p_ooo_info->p_archipelagos_mem);
342 OSAL_FREE(p_hwfn->p_dev, p_ooo_info->ooo_history.p_cqes);
343 OSAL_FREE(p_hwfn->p_dev, p_ooo_info);
344 p_hwfn->p_ooo_info = OSAL_NULL;
348 void ecore_ooo_put_free_buffer(struct ecore_ooo_info *p_ooo_info,
349 struct ecore_ooo_buffer *p_buffer)
351 OSAL_LIST_PUSH_TAIL(&p_buffer->list_entry,
352 &p_ooo_info->free_buffers_list);
355 struct ecore_ooo_buffer *
356 ecore_ooo_get_free_buffer(struct ecore_ooo_info *p_ooo_info)
358 struct ecore_ooo_buffer *p_buffer = OSAL_NULL;
360 if (!OSAL_LIST_IS_EMPTY(&p_ooo_info->free_buffers_list)) {
362 OSAL_LIST_FIRST_ENTRY(
363 &p_ooo_info->free_buffers_list,
364 struct ecore_ooo_buffer, list_entry);
366 OSAL_LIST_REMOVE_ENTRY(&p_buffer->list_entry,
367 &p_ooo_info->free_buffers_list);
373 void ecore_ooo_put_ready_buffer(struct ecore_ooo_info *p_ooo_info,
374 struct ecore_ooo_buffer *p_buffer, u8 on_tail)
377 OSAL_LIST_PUSH_TAIL(&p_buffer->list_entry,
378 &p_ooo_info->ready_buffers_list);
380 OSAL_LIST_PUSH_HEAD(&p_buffer->list_entry,
381 &p_ooo_info->ready_buffers_list);
385 struct ecore_ooo_buffer *
386 ecore_ooo_get_ready_buffer(struct ecore_ooo_info *p_ooo_info)
388 struct ecore_ooo_buffer *p_buffer = OSAL_NULL;
390 if (!OSAL_LIST_IS_EMPTY(&p_ooo_info->ready_buffers_list)) {
392 OSAL_LIST_FIRST_ENTRY(
393 &p_ooo_info->ready_buffers_list,
394 struct ecore_ooo_buffer, list_entry);
396 OSAL_LIST_REMOVE_ENTRY(&p_buffer->list_entry,
397 &p_ooo_info->ready_buffers_list);
403 void ecore_ooo_delete_isles(struct ecore_hwfn *p_hwfn,
404 struct ecore_ooo_info *p_ooo_info,
409 struct ecore_ooo_archipelago *p_archipelago = OSAL_NULL;
410 struct ecore_ooo_isle *p_isle = OSAL_NULL;
413 p_archipelago = ecore_ooo_seek_archipelago(p_ooo_info, cid);
414 for (isle_idx = 0; isle_idx < drop_size; isle_idx++) {
415 p_isle = ecore_ooo_seek_isle(p_hwfn, p_ooo_info,
418 DP_NOTICE(p_hwfn, true,
419 "Isle %d is not found(cid %d)\n",
423 if (OSAL_LIST_IS_EMPTY(&p_isle->buffers_list)) {
424 DP_NOTICE(p_hwfn, true,
425 "Isle %d is empty(cid %d)\n",
428 OSAL_LIST_SPLICE_TAIL_INIT(&p_isle->buffers_list,
429 &p_ooo_info->free_buffers_list);
432 #pragma warning(suppress : 6011)
434 OSAL_LIST_REMOVE_ENTRY(&p_isle->list_entry,
435 &p_archipelago->isles_list);
436 p_ooo_info->cur_isles_number--;
437 OSAL_LIST_PUSH_HEAD(&p_isle->list_entry,
438 &p_ooo_info->free_isles_list);
442 void ecore_ooo_add_new_isle(struct ecore_hwfn *p_hwfn,
443 struct ecore_ooo_info *p_ooo_info,
444 u32 cid, u8 ooo_isle,
445 struct ecore_ooo_buffer *p_buffer)
447 struct ecore_ooo_archipelago *p_archipelago = OSAL_NULL;
448 struct ecore_ooo_isle *p_prev_isle = OSAL_NULL;
449 struct ecore_ooo_isle *p_isle = OSAL_NULL;
452 p_prev_isle = ecore_ooo_seek_isle(p_hwfn, p_ooo_info, cid, ooo_isle - 1);
454 DP_NOTICE(p_hwfn, true,
455 "Isle %d is not found(cid %d)\n",
460 p_archipelago = ecore_ooo_seek_archipelago(p_ooo_info, cid);
461 if (!p_archipelago && (ooo_isle != 1)) {
462 DP_NOTICE(p_hwfn, true,
463 "Connection %d is not found in OOO list\n", cid);
467 if (!OSAL_LIST_IS_EMPTY(&p_ooo_info->free_isles_list)) {
469 OSAL_LIST_FIRST_ENTRY(
470 &p_ooo_info->free_isles_list,
471 struct ecore_ooo_isle, list_entry);
473 OSAL_LIST_REMOVE_ENTRY(&p_isle->list_entry,
474 &p_ooo_info->free_isles_list);
475 if (!OSAL_LIST_IS_EMPTY(&p_isle->buffers_list)) {
476 DP_NOTICE(p_hwfn, true, "Free isle is not empty\n");
477 OSAL_LIST_INIT(&p_isle->buffers_list);
480 DP_NOTICE(p_hwfn, true, "No more free isles\n");
484 if (!p_archipelago) {
485 u32 idx = (cid & 0xffff) - p_ooo_info->cid_base;
487 p_archipelago = &p_ooo_info->p_archipelagos_mem[idx];
489 OSAL_LIST_PUSH_HEAD(&p_buffer->list_entry, &p_isle->buffers_list);
490 p_ooo_info->cur_isles_number++;
491 p_ooo_info->gen_isles_number++;
492 if (p_ooo_info->cur_isles_number > p_ooo_info->max_isles_number)
493 p_ooo_info->max_isles_number = p_ooo_info->cur_isles_number;
495 OSAL_LIST_PUSH_HEAD(&p_isle->list_entry, &p_archipelago->isles_list);
497 OSAL_LIST_INSERT_ENTRY_AFTER(&p_isle->list_entry,
498 &p_prev_isle->list_entry,
499 &p_archipelago->isles_list);
503 void ecore_ooo_add_new_buffer(struct ecore_hwfn *p_hwfn,
504 struct ecore_ooo_info *p_ooo_info,
507 struct ecore_ooo_buffer *p_buffer,
510 struct ecore_ooo_isle * p_isle = OSAL_NULL;
511 p_isle = ecore_ooo_seek_isle(p_hwfn, p_ooo_info, cid, ooo_isle);
513 DP_NOTICE(p_hwfn, true,
514 "Isle %d is not found(cid %d)\n",
518 if (buffer_side == ECORE_OOO_LEFT_BUF) {
519 OSAL_LIST_PUSH_HEAD(&p_buffer->list_entry,
520 &p_isle->buffers_list);
522 OSAL_LIST_PUSH_TAIL(&p_buffer->list_entry,
523 &p_isle->buffers_list);
527 void ecore_ooo_join_isles(struct ecore_hwfn *p_hwfn,
528 struct ecore_ooo_info *p_ooo_info,
529 u32 cid, u8 left_isle)
531 struct ecore_ooo_archipelago *p_archipelago = OSAL_NULL;
532 struct ecore_ooo_isle *p_right_isle = OSAL_NULL;
533 struct ecore_ooo_isle *p_left_isle = OSAL_NULL;
535 p_right_isle = ecore_ooo_seek_isle(p_hwfn, p_ooo_info, cid,
538 DP_NOTICE(p_hwfn, true,
539 "Right isle %d is not found(cid %d)\n",
543 p_archipelago = ecore_ooo_seek_archipelago(p_ooo_info, cid);
544 OSAL_LIST_REMOVE_ENTRY(&p_right_isle->list_entry,
545 &p_archipelago->isles_list);
546 p_ooo_info->cur_isles_number--;
548 p_left_isle = ecore_ooo_seek_isle(p_hwfn, p_ooo_info, cid,
551 DP_NOTICE(p_hwfn, true,
552 "Left isle %d is not found(cid %d)\n",
556 OSAL_LIST_SPLICE_TAIL_INIT(&p_right_isle->buffers_list,
557 &p_left_isle->buffers_list);
559 OSAL_LIST_SPLICE_TAIL_INIT(&p_right_isle->buffers_list,
560 &p_ooo_info->ready_buffers_list);
562 OSAL_LIST_PUSH_TAIL(&p_right_isle->list_entry,
563 &p_ooo_info->free_isles_list);
566 void ecore_ooo_dump_rx_event(struct ecore_hwfn *p_hwfn,
567 struct ooo_opaque *iscsi_ooo,
568 struct ecore_ooo_buffer *p_buffer)
571 u32 dp_module = ECORE_MSG_OOO;
573 u8 *packet_buffer = 0;
575 if (p_hwfn->dp_level > ECORE_LEVEL_VERBOSE)
577 if (!(p_hwfn->dp_module & dp_module))
580 packet_buffer = (u8 *)p_buffer->rx_buffer_virt_addr +
581 p_buffer->placement_offset;
582 DP_VERBOSE(p_hwfn, dp_module,
583 "******************************************************\n");
584 ph_hi = DMA_HI(p_buffer->rx_buffer_phys_addr);
585 ph_lo = DMA_LO(p_buffer->rx_buffer_phys_addr);
586 DP_VERBOSE(p_hwfn, dp_module,
587 "0x%x-%x: CID 0x%x, OP 0x%x, ISLE 0x%x\n",
589 iscsi_ooo->cid, iscsi_ooo->ooo_opcode, iscsi_ooo->ooo_isle);
590 for (i = 0; i < 64; i = i + 8) {
591 DP_VERBOSE(p_hwfn, dp_module,
592 "0x%x-%x: 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
595 packet_buffer[i + 1],
596 packet_buffer[i + 2],
597 packet_buffer[i + 3],
598 packet_buffer[i + 4],
599 packet_buffer[i + 5],
600 packet_buffer[i + 6],
601 packet_buffer[i + 7]);