2 * Copyright (c) 2005 Cisco Systems. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 #endif /* HAVE_CONFIG_H */
38 #include <netinet/in.h>
46 static void *get_wqe(struct mthca_srq *srq, int n)
48 return srq->buf.buf + (n << srq->wqe_shift);
52 * Return a pointer to the location within a WQE that we're using as a
53 * link when the WQE is in the free list. We use the imm field at an
54 * offset of 12 bytes because in the Tavor case, posting a WQE may
55 * overwrite the next segment of the previous WQE, but a receive WQE
56 * will never touch the imm field. This avoids corrupting our free
57 * list if the previous WQE has already completed and been put on the
58 * free list when we post the next WQE.
60 static inline int *wqe_to_link(void *wqe)
62 return (int *) (wqe + 12);
65 void mthca_free_srq_wqe(struct mthca_srq *srq, int ind)
67 struct mthca_next_seg *last_free;
69 pthread_spin_lock(&srq->lock);
71 last_free = get_wqe(srq, srq->last_free);
72 *wqe_to_link(last_free) = ind;
73 last_free->nda_op = htonl((ind << srq->wqe_shift) | 1);
74 *wqe_to_link(get_wqe(srq, ind)) = -1;
77 pthread_spin_unlock(&srq->lock);
80 int mthca_tavor_post_srq_recv(struct ibv_srq *ibsrq,
81 struct ibv_recv_wr *wr,
82 struct ibv_recv_wr **bad_wr)
84 struct mthca_srq *srq = to_msrq(ibsrq);
95 pthread_spin_lock(&srq->lock);
97 first_ind = srq->first_free;
99 for (nreq = 0; wr; wr = wr->next) {
100 ind = srq->first_free;
101 wqe = get_wqe(srq, ind);
102 next_ind = *wqe_to_link(wqe);
110 prev_wqe = srq->last;
113 ((struct mthca_next_seg *) wqe)->ee_nds = 0;
114 /* flags field will always remain 0 */
116 wqe += sizeof (struct mthca_next_seg);
118 if (wr->num_sge > srq->max_gs) {
121 srq->last = prev_wqe;
125 for (i = 0; i < wr->num_sge; ++i) {
126 ((struct mthca_data_seg *) wqe)->byte_count =
127 htonl(wr->sg_list[i].length);
128 ((struct mthca_data_seg *) wqe)->lkey =
129 htonl(wr->sg_list[i].lkey);
130 ((struct mthca_data_seg *) wqe)->addr =
131 htonll(wr->sg_list[i].addr);
132 wqe += sizeof (struct mthca_data_seg);
135 if (i < srq->max_gs) {
136 ((struct mthca_data_seg *) wqe)->byte_count = 0;
137 ((struct mthca_data_seg *) wqe)->lkey = htonl(MTHCA_INVAL_LKEY);
138 ((struct mthca_data_seg *) wqe)->addr = 0;
141 ((struct mthca_next_seg *) prev_wqe)->ee_nds =
142 htonl(MTHCA_NEXT_DBD);
144 srq->wrid[ind] = wr->wr_id;
145 srq->first_free = next_ind;
147 if (++nreq == MTHCA_TAVOR_MAX_WQES_PER_RECV_DB) {
150 doorbell[0] = htonl(first_ind << srq->wqe_shift);
151 doorbell[1] = htonl(srq->srqn << 8);
154 * Make sure that descriptors are written
155 * before doorbell is rung.
159 mthca_write64(doorbell, to_mctx(ibsrq->context), MTHCA_RECV_DOORBELL);
161 first_ind = srq->first_free;
166 doorbell[0] = htonl(first_ind << srq->wqe_shift);
167 doorbell[1] = htonl((srq->srqn << 8) | nreq);
170 * Make sure that descriptors are written before
175 mthca_write64(doorbell, to_mctx(ibsrq->context), MTHCA_RECV_DOORBELL);
178 pthread_spin_unlock(&srq->lock);
182 int mthca_arbel_post_srq_recv(struct ibv_srq *ibsrq,
183 struct ibv_recv_wr *wr,
184 struct ibv_recv_wr **bad_wr)
186 struct mthca_srq *srq = to_msrq(ibsrq);
194 pthread_spin_lock(&srq->lock);
196 for (nreq = 0; wr; ++nreq, wr = wr->next) {
197 ind = srq->first_free;
198 wqe = get_wqe(srq, ind);
199 next_ind = *wqe_to_link(wqe);
207 ((struct mthca_next_seg *) wqe)->ee_nds = 0;
208 /* flags field will always remain 0 */
210 wqe += sizeof (struct mthca_next_seg);
212 if (wr->num_sge > srq->max_gs) {
218 for (i = 0; i < wr->num_sge; ++i) {
219 ((struct mthca_data_seg *) wqe)->byte_count =
220 htonl(wr->sg_list[i].length);
221 ((struct mthca_data_seg *) wqe)->lkey =
222 htonl(wr->sg_list[i].lkey);
223 ((struct mthca_data_seg *) wqe)->addr =
224 htonll(wr->sg_list[i].addr);
225 wqe += sizeof (struct mthca_data_seg);
228 if (i < srq->max_gs) {
229 ((struct mthca_data_seg *) wqe)->byte_count = 0;
230 ((struct mthca_data_seg *) wqe)->lkey = htonl(MTHCA_INVAL_LKEY);
231 ((struct mthca_data_seg *) wqe)->addr = 0;
234 srq->wrid[ind] = wr->wr_id;
235 srq->first_free = next_ind;
239 srq->counter += nreq;
242 * Make sure that descriptors are written before
243 * we write doorbell record.
246 *srq->db = htonl(srq->counter);
249 pthread_spin_unlock(&srq->lock);
253 int mthca_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr,
254 struct mthca_srq *srq)
256 struct mthca_data_seg *scatter;
261 srq->wrid = malloc(srq->max * sizeof (uint64_t));
265 size = sizeof (struct mthca_next_seg) +
266 srq->max_gs * sizeof (struct mthca_data_seg);
268 for (srq->wqe_shift = 6; 1 << srq->wqe_shift < size; ++srq->wqe_shift)
271 srq->buf_size = srq->max << srq->wqe_shift;
273 if (mthca_alloc_buf(&srq->buf,
274 align(srq->buf_size, to_mdev(pd->context->device)->page_size),
275 to_mdev(pd->context->device)->page_size)) {
280 memset(srq->buf.buf, 0, srq->buf_size);
283 * Now initialize the SRQ buffer so that all of the WQEs are
284 * linked into the list of free WQEs. In addition, set the
285 * scatter list L_Keys to the sentry value of 0x100.
288 for (i = 0; i < srq->max; ++i) {
289 struct mthca_next_seg *next;
291 next = wqe = get_wqe(srq, i);
293 if (i < srq->max - 1) {
294 *wqe_to_link(wqe) = i + 1;
295 next->nda_op = htonl(((i + 1) << srq->wqe_shift) | 1);
297 *wqe_to_link(wqe) = -1;
301 for (scatter = wqe + sizeof (struct mthca_next_seg);
302 (void *) scatter < wqe + (1 << srq->wqe_shift);
304 scatter->lkey = htonl(MTHCA_INVAL_LKEY);
308 srq->last_free = srq->max - 1;
309 srq->last = get_wqe(srq, srq->max - 1);