1 [PATCHv7 1/2] libmlx4: Add RoCEE support
3 Modify libmlx4 to support RoCEE. The change involves retrieving the MAC address
4 of a port based on its GID through a new system call, ibv_cmd_get_mac(), and
5 embedding the MAC in the address vector representation of mlx4.
7 Signed-off-by: Eli Cohen <eli@mellanox.co.il>
11 src/verbs.c | 29 +++++++++++++++++++++++++++++
13 4 files changed, 36 insertions(+), 1 deletions(-)
15 Index: libmlx4/src/mlx4.h
16 ===================================================================
17 --- libmlx4.orig/src/mlx4.h 2010-08-23 08:07:47.599964446 +0300
18 +++ libmlx4/src/mlx4.h 2010-08-23 08:08:32.039462057 +0300
19 @@ -277,11 +277,15 @@ struct mlx4_av {
21 uint32_t sl_tclass_flowlabel;
34 struct mlx4_xrc_domain {
35 Index: libmlx4/src/qp.c
36 ===================================================================
37 --- libmlx4.orig/src/qp.c 2010-08-23 08:07:46.283963844 +0300
38 +++ libmlx4/src/qp.c 2010-08-23 08:08:32.039462057 +0300
39 @@ -143,6 +143,8 @@ static void set_datagram_seg(struct mlx4
40 memcpy(dseg->av, &to_mah(wr->wr.ud.ah)->av, sizeof (struct mlx4_av));
41 dseg->dqpn = htonl(wr->wr.ud.remote_qpn);
42 dseg->qkey = htonl(wr->wr.ud.remote_qkey);
43 + dseg->vlan = htons(to_mah(wr->wr.ud.ah)->vlan);
44 + memcpy(dseg->mac, to_mah(wr->wr.ud.ah)->mac, 6);
47 static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ibv_sge *sg)
48 @@ -284,6 +286,11 @@ int mlx4_post_send(struct ibv_qp *ibqp,
49 set_datagram_seg(wqe, wr);
50 wqe += sizeof (struct mlx4_wqe_datagram_seg);
51 size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
52 + if (to_mah(wr->wr.ud.ah)->tagged) {
53 + ctrl->ins_vlan = 1 << 6;
54 + ctrl->vlan_tag = htons(to_mah(wr->wr.ud.ah)->vlan);
60 @@ -396,7 +403,7 @@ out:
62 if (nreq == 1 && inl && size > 1 && size < ctx->bf_buf_size / 16) {
63 ctrl->owner_opcode |= htonl((qp->sq.head & 0xffff) << 8);
64 - *(uint32_t *) ctrl->reserved |= qp->doorbell_qpn;
65 + *(uint32_t *) (&ctrl->vlan_tag) |= qp->doorbell_qpn;
67 * Make sure that descriptor is written to memory
68 * before writing to BlueFlame page.
69 Index: libmlx4/src/verbs.c
70 ===================================================================
71 --- libmlx4.orig/src/verbs.c 2010-08-23 08:07:48.451964305 +0300
72 +++ libmlx4/src/verbs.c 2010-08-23 08:08:32.039462057 +0300
73 @@ -643,12 +643,14 @@ int mlx4_destroy_qp(struct ibv_qp *ibqp)
74 struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr)
77 + struct ibv_port_attr port_attr;
80 ah = malloc(sizeof *ah);
84 - memset(&ah->av, 0, sizeof ah->av);
85 + memset(ah, 0, sizeof *ah);
87 ah->av.port_pd = htonl(to_mpd(pd)->pdn | (attr->port_num << 24));
88 ah->av.g_slid = attr->src_path_bits;
89 @@ -668,7 +670,32 @@ struct ibv_ah *mlx4_create_ah(struct ibv
90 memcpy(ah->av.dgid, attr->grh.dgid.raw, 16);
93 + if (ibv_query_port(pd->context, attr->port_num, &port_attr))
96 + if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET) {
97 + if (ibv_resolve_eth_gid(pd, attr->port_num,
98 + (union ibv_gid *)ah->av.dgid,
99 + attr->grh.sgid_index,
100 + ah->mac, &ah->vlan,
101 + &ah->tagged, &is_mcast))
105 + ah->av.dlid = htons(0xc000);
106 + ah->av.port_pd |= htonl(1 << 31);
109 + ah->av.port_pd |= htonl(1 << 29);
110 + ah->vlan |= (attr->sl & 7) << 13;
121 int mlx4_destroy_ah(struct ibv_ah *ah)
122 Index: libmlx4/src/wqe.h
123 ===================================================================
124 --- libmlx4.orig/src/wqe.h 2010-08-23 08:07:46.287962570 +0300
125 +++ libmlx4/src/wqe.h 2010-08-23 08:07:50.231963413 +0300
126 @@ -54,7 +54,8 @@ enum {
128 struct mlx4_wqe_ctrl_seg {
129 uint32_t owner_opcode;
130 - uint8_t reserved[3];
135 * High 24 bits are SRC remote buffer; low 8 bits are flags:
136 @@ -78,7 +79,8 @@ struct mlx4_wqe_datagram_seg {
140 - uint32_t reserved[2];
145 struct mlx4_wqe_data_seg {