2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39 #include <infiniband/driver.h>
40 #include <infiniband/arch.h>
42 #ifdef HAVE_VALGRIND_MEMCHECK_H
44 # include <valgrind/memcheck.h>
46 # if !defined(VALGRIND_MAKE_MEM_DEFINED) || !defined(VALGRIND_MAKE_MEM_UNDEFINED)
47 # warning "Valgrind support requested, but VALGRIND_MAKE_MEM_(UN)DEFINED not available"
50 #endif /* HAVE_VALGRIND_MEMCHECK_H */
52 #ifndef VALGRIND_MAKE_MEM_DEFINED
53 # define VALGRIND_MAKE_MEM_DEFINED(addr,len)
56 #ifndef VALGRIND_MAKE_MEM_UNDEFINED
57 # define VALGRIND_MAKE_MEM_UNDEFINED(addr,len)
71 #define wc_wmb() asm volatile("lock; addl $0,0(%%esp) " ::: "memory")
72 #elif defined(__x86_64__)
73 #define wc_wmb() asm volatile("sfence" ::: "memory")
74 #elif defined(__ia64__)
75 #define wc_wmb() asm volatile("fwb" ::: "memory")
77 #define wc_wmb() wmb()
82 #ifndef HAVE_IBV_MORE_OPS
83 #undef HAVE_IBV_XRC_OPS
84 #undef HAVE_IBV_CREATE_QP_EXP
87 #define HIDDEN __attribute__((visibility ("hidden")))
93 ({ typeof (a) _a = (a); \
94 typeof (b) _b = (b); \
100 ({ typeof (a) _a = (a); \
101 typeof (b) _b = (b); \
102 _a < _b ? _a : _b; })
106 MLX4_CQ_ENTRY_SIZE = 0x20
110 MLX4_STAT_RATE_OFFSET = 5
114 MLX4_QP_TABLE_BITS = 8,
115 MLX4_QP_TABLE_SIZE = 1 << MLX4_QP_TABLE_BITS,
116 MLX4_QP_TABLE_MASK = MLX4_QP_TABLE_SIZE - 1
120 MLX4_XRC_SRQ_TABLE_BITS = 8,
121 MLX4_XRC_SRQ_TABLE_SIZE = 1 << MLX4_XRC_SRQ_TABLE_BITS,
122 MLX4_XRC_SRQ_TABLE_MASK = MLX4_XRC_SRQ_TABLE_SIZE - 1
126 MLX4_XRC_QPN_BIT = (1 << 23)
136 MLX4_OPCODE_NOP = 0x00,
137 MLX4_OPCODE_SEND_INVAL = 0x01,
138 MLX4_OPCODE_RDMA_WRITE = 0x08,
139 MLX4_OPCODE_RDMA_WRITE_IMM = 0x09,
140 MLX4_OPCODE_SEND = 0x0a,
141 MLX4_OPCODE_SEND_IMM = 0x0b,
142 MLX4_OPCODE_LSO = 0x0e,
143 MLX4_OPCODE_RDMA_READ = 0x10,
144 MLX4_OPCODE_ATOMIC_CS = 0x11,
145 MLX4_OPCODE_ATOMIC_FA = 0x12,
146 MLX4_OPCODE_ATOMIC_MASK_CS = 0x14,
147 MLX4_OPCODE_ATOMIC_MASK_FA = 0x15,
148 MLX4_OPCODE_BIND_MW = 0x18,
149 MLX4_OPCODE_FMR = 0x19,
150 MLX4_OPCODE_LOCAL_INVAL = 0x1b,
151 MLX4_OPCODE_CONFIG_CMD = 0x1f,
153 MLX4_RECV_OPCODE_RDMA_WRITE_IMM = 0x00,
154 MLX4_RECV_OPCODE_SEND = 0x01,
155 MLX4_RECV_OPCODE_SEND_IMM = 0x02,
156 MLX4_RECV_OPCODE_SEND_INVAL = 0x03,
158 MLX4_CQE_OPCODE_ERROR = 0x1e,
159 MLX4_CQE_OPCODE_RESIZE = 0x16,
163 MLX4_MAX_WQE_SIZE = 1008
167 struct ibv_device ibv_dev;
173 struct mlx4_context {
174 struct ibv_context ibv_ctx;
177 pthread_spinlock_t uar_lock;
182 pthread_spinlock_t bf_lock;
185 struct mlx4_qp **table;
187 } qp_table[MLX4_QP_TABLE_SIZE];
188 pthread_mutex_t qp_table_mutex;
197 struct mlx4_srq **table;
199 } xrc_srq_table[MLX4_XRC_SRQ_TABLE_SIZE];
200 pthread_mutex_t xrc_srq_table_mutex;
202 int xrc_srq_table_shift;
203 int xrc_srq_table_mask;
205 struct mlx4_db_page *db_list[MLX4_NUM_DB_TYPE];
206 pthread_mutex_t db_list_mutex;
215 struct ibv_pd ibv_pd;
220 struct ibv_cq ibv_cq;
222 struct mlx4_buf resize_buf;
223 pthread_spinlock_t lock;
232 struct ibv_srq ibv_srq;
234 pthread_spinlock_t lock;
248 pthread_spinlock_t lock;
259 struct ibv_qp ibv_qp;
264 uint32_t doorbell_qpn;
265 uint32_t sq_signal_bits;
282 uint32_t sl_tclass_flowlabel;
288 struct ibv_ah ibv_ah;
295 struct mlx4_xrc_domain {
296 struct ibv_xrc_domain ibv_xrcd;
300 static inline unsigned long align(unsigned long val, unsigned long align)
302 return (val + align - 1) & ~(align - 1);
305 #define to_mxxx(xxx, type) \
306 ((struct mlx4_##type *) \
307 ((void *) ib##xxx - offsetof(struct mlx4_##type, ibv_##xxx)))
309 static inline struct mlx4_device *to_mdev(struct ibv_device *ibdev)
311 return to_mxxx(dev, device);
314 static inline struct mlx4_context *to_mctx(struct ibv_context *ibctx)
316 return to_mxxx(ctx, context);
319 static inline struct mlx4_pd *to_mpd(struct ibv_pd *ibpd)
321 return to_mxxx(pd, pd);
324 static inline struct mlx4_cq *to_mcq(struct ibv_cq *ibcq)
326 return to_mxxx(cq, cq);
329 static inline struct mlx4_srq *to_msrq(struct ibv_srq *ibsrq)
331 return to_mxxx(srq, srq);
334 static inline struct mlx4_qp *to_mqp(struct ibv_qp *ibqp)
336 return to_mxxx(qp, qp);
339 static inline struct mlx4_ah *to_mah(struct ibv_ah *ibah)
341 return to_mxxx(ah, ah);
344 #ifdef HAVE_IBV_XRC_OPS
345 static inline struct mlx4_xrc_domain *to_mxrcd(struct ibv_xrc_domain *ibxrcd)
347 return to_mxxx(xrcd, xrc_domain);
351 int mlx4_alloc_buf(struct mlx4_buf *buf, size_t size, int page_size);
352 void mlx4_free_buf(struct mlx4_buf *buf);
354 uint32_t *mlx4_alloc_db(struct mlx4_context *context, enum mlx4_db_type type);
355 void mlx4_free_db(struct mlx4_context *context, enum mlx4_db_type type, uint32_t *db);
357 int mlx4_query_device(struct ibv_context *context,
358 struct ibv_device_attr *attr);
359 int mlx4_query_port(struct ibv_context *context, uint8_t port,
360 struct ibv_port_attr *attr);
362 struct ibv_pd *mlx4_alloc_pd(struct ibv_context *context);
363 int mlx4_free_pd(struct ibv_pd *pd);
365 struct ibv_mr *mlx4_reg_mr(struct ibv_pd *pd, void *addr,
366 size_t length, enum ibv_access_flags access);
367 int mlx4_dereg_mr(struct ibv_mr *mr);
369 struct ibv_cq *mlx4_create_cq(struct ibv_context *context, int cqe,
370 struct ibv_comp_channel *channel,
372 int mlx4_alloc_cq_buf(struct mlx4_device *dev, struct mlx4_buf *buf, int nent);
373 int mlx4_resize_cq(struct ibv_cq *cq, int cqe);
374 int mlx4_destroy_cq(struct ibv_cq *cq);
375 int mlx4_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc);
376 int mlx4_arm_cq(struct ibv_cq *cq, int solicited);
377 void mlx4_cq_event(struct ibv_cq *cq);
378 void __mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn, struct mlx4_srq *srq);
379 void mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn, struct mlx4_srq *srq);
380 int mlx4_get_outstanding_cqes(struct mlx4_cq *cq);
381 void mlx4_cq_resize_copy_cqes(struct mlx4_cq *cq, void *buf, int new_cqe);
383 struct ibv_srq *mlx4_create_srq(struct ibv_pd *pd,
384 struct ibv_srq_init_attr *attr);
385 int mlx4_modify_srq(struct ibv_srq *srq,
386 struct ibv_srq_attr *attr,
387 enum ibv_srq_attr_mask mask);
388 int mlx4_query_srq(struct ibv_srq *srq,
389 struct ibv_srq_attr *attr);
390 int mlx4_destroy_srq(struct ibv_srq *srq);
391 int mlx4_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr,
392 struct mlx4_srq *srq);
393 void mlx4_free_srq_wqe(struct mlx4_srq *srq, int ind);
394 int mlx4_post_srq_recv(struct ibv_srq *ibsrq,
395 struct ibv_recv_wr *wr,
396 struct ibv_recv_wr **bad_wr);
397 struct mlx4_srq *mlx4_find_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn);
398 int mlx4_store_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn,
399 struct mlx4_srq *srq);
400 void mlx4_clear_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn);
402 struct ibv_qp *mlx4_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr);
403 int mlx4_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
404 enum ibv_qp_attr_mask attr_mask,
405 struct ibv_qp_init_attr *init_attr);
406 int mlx4_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
407 enum ibv_qp_attr_mask attr_mask);
408 int mlx4_destroy_qp(struct ibv_qp *qp);
409 void mlx4_init_qp_indices(struct mlx4_qp *qp);
410 void mlx4_qp_init_sq_ownership(struct mlx4_qp *qp);
411 int mlx4_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
412 struct ibv_send_wr **bad_wr);
413 int mlx4_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
414 struct ibv_recv_wr **bad_wr);
415 void mlx4_calc_sq_wqe_size(struct ibv_qp_cap *cap, enum ibv_qp_type type,
417 int num_inline_segs(int data, enum ibv_qp_type type);
418 int mlx4_alloc_qp_buf(struct ibv_pd *pd, struct ibv_qp_cap *cap,
419 enum ibv_qp_type type, struct mlx4_qp *qp);
420 void mlx4_set_sq_sizes(struct mlx4_qp *qp, struct ibv_qp_cap *cap,
421 enum ibv_qp_type type);
422 struct mlx4_qp *mlx4_find_qp(struct mlx4_context *ctx, uint32_t qpn);
423 int mlx4_store_qp(struct mlx4_context *ctx, uint32_t qpn, struct mlx4_qp *qp);
424 void mlx4_clear_qp(struct mlx4_context *ctx, uint32_t qpn);
425 struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr);
426 int mlx4_destroy_ah(struct ibv_ah *ah);
427 int mlx4_alloc_av(struct mlx4_pd *pd, struct ibv_ah_attr *attr,
429 void mlx4_free_av(struct mlx4_ah *ah);
430 #ifdef HAVE_IBV_XRC_OPS
431 struct ibv_srq *mlx4_create_xrc_srq(struct ibv_pd *pd,
432 struct ibv_xrc_domain *xrc_domain,
433 struct ibv_cq *xrc_cq,
434 struct ibv_srq_init_attr *attr);
435 struct ibv_xrc_domain *mlx4_open_xrc_domain(struct ibv_context *context,
438 int mlx4_close_xrc_domain(struct ibv_xrc_domain *d);
439 int mlx4_create_xrc_rcv_qp(struct ibv_qp_init_attr *init_attr,
440 uint32_t *xrc_qp_num);
441 int mlx4_modify_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
443 struct ibv_qp_attr *attr,
445 int mlx4_query_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
447 struct ibv_qp_attr *attr,
449 struct ibv_qp_init_attr *init_attr);
450 int mlx4_reg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
451 uint32_t xrc_qp_num);
452 int mlx4_unreg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
453 uint32_t xrc_qp_num);