1 From 6f9c7b5c83a832884b3e47a6ce52145991073586 Mon Sep 17 00:00:00 2001
2 From: Eli Cohen <eli@mellanox.co.il>
3 Date: Sun, 19 Sep 2010 11:28:40 +0200
4 Subject: [PATCH] libmlx4: fix possible inline size
6 The current driver checks required inline size by making sure it does not
7 exceed 1024. This is wrong since the whole WQE is limited to 1008 bytes.
8 Moreover, a more careful claculation is required to avoid cases where the
9 application requests inline support in a certain size that when used later
10 could cause connections to stall due to bad WQEs. This patch takes into account
11 the size of the WQE, the segements used to create a WQE and the overhead
12 incured by the inline segments themselves.
14 Signed-off-by: Eli Cohen <eli@mellanox.co.il>
16 src/verbs.c | 45 ++++++++++++++++++++++++++++++++++++++++-----
17 1 files changed, 40 insertions(+), 5 deletions(-)
19 Index: libmlx4/src/verbs.c
20 ===================================================================
21 --- libmlx4.orig/src/verbs.c 2010-09-29 11:10:17.691587848 +0200
22 +++ libmlx4/src/verbs.c 2010-09-29 11:16:11.031586721 +0200
23 @@ -402,6 +402,44 @@ int mlx4_destroy_srq(struct ibv_srq *ibs
27 +static int verify_sizes(struct ibv_qp_init_attr *attr, struct mlx4_context *context)
32 + if (attr->cap.max_send_wr > context->max_qp_wr ||
33 + attr->cap.max_recv_wr > context->max_qp_wr ||
34 + attr->cap.max_send_sge > context->max_sge ||
35 + attr->cap.max_recv_sge > context->max_sge)
38 + if (attr->cap.max_inline_data) {
39 + nsegs = num_inline_segs(attr->cap.max_inline_data, attr->qp_type);
40 + size = MLX4_MAX_WQE_SIZE - nsegs * sizeof (struct mlx4_wqe_inline_seg);
41 + switch (attr->qp_type) {
43 + size -= (sizeof (struct mlx4_wqe_ctrl_seg) +
44 + sizeof (struct mlx4_wqe_datagram_seg));
50 + size -= (sizeof (struct mlx4_wqe_ctrl_seg) +
51 + sizeof (struct mlx4_wqe_raddr_seg));
58 + if (attr->cap.max_inline_data > size)
65 struct ibv_qp *mlx4_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr)
67 struct mlx4_create_qp cmd;
68 @@ -412,11 +450,7 @@ struct ibv_qp *mlx4_create_qp(struct ibv
71 /* Sanity check QP size before proceeding */
72 - if (attr->cap.max_send_wr > context->max_qp_wr ||
73 - attr->cap.max_recv_wr > context->max_qp_wr ||
74 - attr->cap.max_send_sge > context->max_sge ||
75 - attr->cap.max_recv_sge > context->max_sge ||
76 - attr->cap.max_inline_data > 1024)
77 + if (verify_sizes(attr, context))
80 qp = malloc(sizeof *qp);
81 Index: libmlx4/src/mlx4.h
82 ===================================================================
83 --- libmlx4.orig/src/mlx4.h 2010-09-29 11:10:17.691587848 +0200
84 +++ libmlx4/src/mlx4.h 2010-09-29 11:11:35.559586971 +0200
85 @@ -159,6 +159,10 @@ enum {
86 MLX4_CQE_OPCODE_RESIZE = 0x16,
90 + MLX4_MAX_WQE_SIZE = 1008
94 struct ibv_device ibv_dev;
96 @@ -410,6 +414,7 @@ int mlx4_post_recv(struct ibv_qp *ibqp,
97 struct ibv_recv_wr **bad_wr);
98 void mlx4_calc_sq_wqe_size(struct ibv_qp_cap *cap, enum ibv_qp_type type,
100 +int num_inline_segs(int data, enum ibv_qp_type type);
101 int mlx4_alloc_qp_buf(struct ibv_pd *pd, struct ibv_qp_cap *cap,
102 enum ibv_qp_type type, struct mlx4_qp *qp);
103 void mlx4_set_sq_sizes(struct mlx4_qp *qp, struct ibv_qp_cap *cap,
104 Index: libmlx4/src/qp.c
105 ===================================================================
106 --- libmlx4.orig/src/qp.c 2010-09-29 11:10:17.691587848 +0200
107 +++ libmlx4/src/qp.c 2010-09-29 11:12:14.931587492 +0200
108 @@ -505,7 +505,7 @@ out:
112 -static int num_inline_segs(int data, enum ibv_qp_type type)
113 +int num_inline_segs(int data, enum ibv_qp_type type)
116 * Inline data segments are not allowed to cross 64 byte
117 @@ -634,7 +634,8 @@ void mlx4_set_sq_sizes(struct mlx4_qp *q
119 struct mlx4_context *ctx = to_mctx(qp->ibv_qp.context);
121 - wqe_size = (1 << qp->sq.wqe_shift) - sizeof (struct mlx4_wqe_ctrl_seg);
122 + wqe_size = min((1 << qp->sq.wqe_shift), MLX4_MAX_WQE_SIZE) -
123 + sizeof (struct mlx4_wqe_ctrl_seg);
126 wqe_size -= sizeof (struct mlx4_wqe_datagram_seg);