2 * Copyright (c) 2007 Cisco, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
44 static void *get_wqe(struct mlx4_srq *srq, int n)
46 return srq->buf.buf + (n << srq->wqe_shift);
49 void mlx4_free_srq_wqe(struct mlx4_srq *srq, int ind)
51 struct mlx4_wqe_srq_next_seg *next;
53 pthread_spin_lock(&srq->lock);
55 next = get_wqe(srq, srq->tail);
56 next->next_wqe_index = htobe16(ind);
59 pthread_spin_unlock(&srq->lock);
62 int mlx4_post_srq_recv(struct ibv_srq *ibsrq,
63 struct ibv_recv_wr *wr,
64 struct ibv_recv_wr **bad_wr)
66 struct mlx4_srq *srq = to_msrq(ibsrq);
67 struct mlx4_wqe_srq_next_seg *next;
68 struct mlx4_wqe_data_seg *scat;
73 pthread_spin_lock(&srq->lock);
75 for (nreq = 0; wr; ++nreq, wr = wr->next) {
76 if (wr->num_sge > srq->max_gs) {
82 if (srq->head == srq->tail) {
89 srq->wrid[srq->head] = wr->wr_id;
91 next = get_wqe(srq, srq->head);
92 srq->head = be16toh(next->next_wqe_index);
93 scat = (struct mlx4_wqe_data_seg *) (next + 1);
95 for (i = 0; i < wr->num_sge; ++i) {
96 scat[i].byte_count = htobe32(wr->sg_list[i].length);
97 scat[i].lkey = htobe32(wr->sg_list[i].lkey);
98 scat[i].addr = htobe64(wr->sg_list[i].addr);
101 if (i < srq->max_gs) {
102 scat[i].byte_count = 0;
103 scat[i].lkey = htobe32(MLX4_INVALID_LKEY);
109 srq->counter += nreq;
112 * Make sure that descriptors are written before
113 * we write doorbell record.
115 udma_to_device_barrier();
117 *srq->db = htobe32(srq->counter);
120 pthread_spin_unlock(&srq->lock);
125 int mlx4_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr,
126 struct mlx4_srq *srq)
128 struct mlx4_wqe_srq_next_seg *next;
129 struct mlx4_wqe_data_seg *scatter;
134 srq->wrid = malloc(srq->max * sizeof (uint64_t));
138 size = sizeof (struct mlx4_wqe_srq_next_seg) +
139 srq->max_gs * sizeof (struct mlx4_wqe_data_seg);
141 for (srq->wqe_shift = 5; 1 << srq->wqe_shift < size; ++srq->wqe_shift)
144 buf_size = srq->max << srq->wqe_shift;
146 if (mlx4_alloc_buf(&srq->buf, buf_size,
147 to_mdev(pd->context->device)->page_size)) {
152 memset(srq->buf.buf, 0, buf_size);
155 * Now initialize the SRQ buffer so that all of the WQEs are
156 * linked into the list of free WQEs.
159 for (i = 0; i < srq->max; ++i) {
160 next = get_wqe(srq, i);
161 next->next_wqe_index = htobe16((i + 1) & (srq->max - 1));
163 for (scatter = (void *) (next + 1);
164 (void *) scatter < (void *) next + (1 << srq->wqe_shift);
166 scatter->lkey = htobe32(MLX4_INVALID_LKEY);
170 srq->tail = srq->max - 1;
175 void mlx4_init_xsrq_table(struct mlx4_xsrq_table *xsrq_table, int size)
177 memset(xsrq_table, 0, sizeof *xsrq_table);
178 xsrq_table->num_xsrq = size;
179 xsrq_table->shift = ffs(size) - 1 - MLX4_XSRQ_TABLE_BITS;
180 xsrq_table->mask = (1 << xsrq_table->shift) - 1;
182 pthread_mutex_init(&xsrq_table->mutex, NULL);
185 struct mlx4_srq *mlx4_find_xsrq(struct mlx4_xsrq_table *xsrq_table, uint32_t srqn)
189 index = (srqn & (xsrq_table->num_xsrq - 1)) >> xsrq_table->shift;
190 if (xsrq_table->xsrq_table[index].refcnt)
191 return xsrq_table->xsrq_table[index].table[srqn & xsrq_table->mask];
196 int mlx4_store_xsrq(struct mlx4_xsrq_table *xsrq_table, uint32_t srqn,
197 struct mlx4_srq *srq)
201 index = (srqn & (xsrq_table->num_xsrq - 1)) >> xsrq_table->shift;
202 pthread_mutex_lock(&xsrq_table->mutex);
203 if (!xsrq_table->xsrq_table[index].refcnt) {
204 xsrq_table->xsrq_table[index].table = calloc(xsrq_table->mask + 1,
205 sizeof(struct mlx4_srq *));
206 if (!xsrq_table->xsrq_table[index].table) {
212 xsrq_table->xsrq_table[index].refcnt++;
213 xsrq_table->xsrq_table[index].table[srqn & xsrq_table->mask] = srq;
216 pthread_mutex_unlock(&xsrq_table->mutex);
220 void mlx4_clear_xsrq(struct mlx4_xsrq_table *xsrq_table, uint32_t srqn)
224 index = (srqn & (xsrq_table->num_xsrq - 1)) >> xsrq_table->shift;
225 pthread_mutex_lock(&xsrq_table->mutex);
227 if (--xsrq_table->xsrq_table[index].refcnt)
228 xsrq_table->xsrq_table[index].table[srqn & xsrq_table->mask] = NULL;
230 free(xsrq_table->xsrq_table[index].table);
232 pthread_mutex_unlock(&xsrq_table->mutex);
235 struct ibv_srq *mlx4_create_xrc_srq(struct ibv_context *context,
236 struct ibv_srq_init_attr_ex *attr_ex)
238 struct mlx4_create_xsrq cmd;
239 struct mlx4_create_srq_resp resp;
240 struct mlx4_srq *srq;
243 /* Sanity check SRQ size before proceeding */
244 if (attr_ex->attr.max_wr > 1 << 16 || attr_ex->attr.max_sge > 64)
247 srq = calloc(1, sizeof *srq);
251 if (pthread_spin_init(&srq->lock, PTHREAD_PROCESS_PRIVATE))
254 srq->max = align_queue_size(attr_ex->attr.max_wr + 1);
255 srq->max_gs = attr_ex->attr.max_sge;
259 if (mlx4_alloc_srq_buf(attr_ex->pd, &attr_ex->attr, srq))
262 srq->db = mlx4_alloc_db(to_mctx(context), MLX4_DB_TYPE_RQ);
268 cmd.buf_addr = (uintptr_t) srq->buf.buf;
269 cmd.db_addr = (uintptr_t) srq->db;
271 ret = ibv_cmd_create_srq_ex(context, &srq->verbs_srq,
272 sizeof(srq->verbs_srq),
274 &cmd.ibv_cmd, sizeof cmd,
275 &resp.ibv_resp, sizeof resp);
279 ret = mlx4_store_xsrq(&to_mctx(context)->xsrq_table,
280 srq->verbs_srq.srq_num, srq);
284 return &srq->verbs_srq.srq;
287 ibv_cmd_destroy_srq(&srq->verbs_srq.srq);
289 mlx4_free_db(to_mctx(context), MLX4_DB_TYPE_RQ, srq->db);
292 mlx4_free_buf(&srq->buf);
298 int mlx4_destroy_xrc_srq(struct ibv_srq *srq)
300 struct mlx4_context *mctx = to_mctx(srq->context);
301 struct mlx4_srq *msrq = to_msrq(srq);
305 mcq = to_mcq(msrq->verbs_srq.cq);
306 mlx4_cq_clean(mcq, 0, msrq);
307 pthread_spin_lock(&mcq->lock);
308 mlx4_clear_xsrq(&mctx->xsrq_table, msrq->verbs_srq.srq_num);
309 pthread_spin_unlock(&mcq->lock);
311 ret = ibv_cmd_destroy_srq(srq);
313 pthread_spin_lock(&mcq->lock);
314 mlx4_store_xsrq(&mctx->xsrq_table, msrq->verbs_srq.srq_num, msrq);
315 pthread_spin_unlock(&mcq->lock);
319 mlx4_free_db(mctx, MLX4_DB_TYPE_RQ, msrq->db);
320 mlx4_free_buf(&msrq->buf);