contrib/ofed/libibverbs/fixes/XRC_RCV_QP.patch

   1 Added support for XRC receive-only QPs.
   2 (OFED 1.3 libibverbs commit 6e99cddf835d4715ea7ca3641944e6285f27f2df)
   3
   4 V2:
   5 1. checkpatch.pl cleanups
   6 2. Fixed u64 alignment problems in kern-abi.h
   7 3. eliminated unneeded default_symvers
   8 4. Added ibv_xrc_rcv_xxx lines to libibverbs.map IBVERBS_1.1
   9
  10 Signed-off-by: Jack Morgenstein <jackm@dev.mellanox.co.il>
  11 ---
  12  include/infiniband/driver.h   |   12 ++-
  13  include/infiniband/kern-abi.h |   99 +++++++++++++++++++-
  14  include/infiniband/verbs.h    |  123 +++++++++++++++++++++++
  15  src/cmd.c                     |  215 +++++++++++++++++++++++++++++++++++++++++
  16  src/device.c                  |   52 +++++-----
  17  src/libibverbs.map            |   10 ++
  18  src/verbs.c                   |   59 +++++++++++
  19  7 files changed, 543 insertions(+), 27 deletions(-)
  20
  21 Index: libibverbs/include/infiniband/driver.h
  22 ===================================================================
  23 --- libibverbs.orig/include/infiniband/driver.h 2009-11-01 15:18:20.624171000 +0200
  24 +++ libibverbs/include/infiniband/driver.h      2009-11-01 15:18:24.572283000 +0200
  25 @@ -144,7 +144,17 @@ int ibv_cmd_open_xrc_domain(struct ibv_c
  26                             struct ibv_open_xrc_domain_resp *resp,
  27                             size_t resp_size);
  28  int ibv_cmd_close_xrc_domain(struct ibv_xrc_domain *d);
  29 -
  30 +int ibv_cmd_create_xrc_rcv_qp(struct ibv_qp_init_attr *init_attr,
  31 +                             uint32_t *xrc_rcv_qpn);
  32 +int ibv_cmd_modify_xrc_rcv_qp(struct ibv_xrc_domain *d, uint32_t xrc_rcv_qpn,
  33 +                             struct ibv_qp_attr *attr, int attr_mask);
  34 +int ibv_cmd_query_xrc_rcv_qp(struct ibv_xrc_domain *d, uint32_t xrc_rcv_qpn,
  35 +                            struct ibv_qp_attr *attr, int attr_mask,
  36 +                            struct ibv_qp_init_attr *init_attr);
  37 +int ibv_cmd_reg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
  38 +                          uint32_t xrc_qp_num);
  39 +int ibv_cmd_unreg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
  40 +                            uint32_t xrc_qp_num);
  41
  42  /*
  43   * sysfs helper functions
  44 Index: libibverbs/include/infiniband/kern-abi.h
  45 ===================================================================
  46 --- libibverbs.orig/include/infiniband/kern-abi.h       2009-11-01 15:18:20.629168000 +0200
  47 +++ libibverbs/include/infiniband/kern-abi.h    2009-11-01 15:18:24.577283000 +0200
  48 @@ -88,7 +88,12 @@ enum {
  49         IB_USER_VERBS_CMD_POST_SRQ_RECV,
  50         IB_USER_VERBS_CMD_CREATE_XRC_SRQ,
  51         IB_USER_VERBS_CMD_OPEN_XRC_DOMAIN,
  52 -       IB_USER_VERBS_CMD_CLOSE_XRC_DOMAIN
  53 +       IB_USER_VERBS_CMD_CLOSE_XRC_DOMAIN,
  54 +       IB_USER_VERBS_CMD_CREATE_XRC_RCV_QP,
  55 +       IB_USER_VERBS_CMD_MODIFY_XRC_RCV_QP,
  56 +       IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP,
  57 +       IB_USER_VERBS_CMD_REG_XRC_RCV_QP,
  58 +       IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP,
  59  };
  60
  61  /*
  62 @@ -570,6 +575,93 @@ struct ibv_destroy_qp_resp {
  63         __u32 events_reported;
  64  };
  65
  66 +struct ibv_create_xrc_rcv_qp {
  67 +       __u32 command;
  68 +       __u16 in_words;
  69 +       __u16 out_words;
  70 +       __u64 response;
  71 +       __u64 user_handle;
  72 +       __u32 xrc_domain_handle;
  73 +       __u32 max_send_wr;
  74 +       __u32 max_recv_wr;
  75 +       __u32 max_send_sge;
  76 +       __u32 max_recv_sge;
  77 +       __u32 max_inline_data;
  78 +       __u8  sq_sig_all;
  79 +       __u8  qp_type;
  80 +       __u8  reserved[6];
  81 +       __u64 driver_data[0];
  82 +};
  83 +
  84 +struct ibv_create_xrc_rcv_qp_resp {
  85 +       __u32 qpn;
  86 +       __u32 reserved;
  87 +};
  88 +
  89 +struct ibv_modify_xrc_rcv_qp {
  90 +       __u32 command;
  91 +       __u16 in_words;
  92 +       __u16 out_words;
  93 +       __u32 xrc_domain_handle;
  94 +       __u32 qp_num;
  95 +       struct ibv_qp_dest dest;
  96 +       struct ibv_qp_dest alt_dest;
  97 +       __u32 attr_mask;
  98 +       __u32 qkey;
  99 +       __u32 rq_psn;
 100 +       __u32 sq_psn;
 101 +       __u32 dest_qp_num;
 102 +       __u32 qp_access_flags;
 103 +       __u16 pkey_index;
 104 +       __u16 alt_pkey_index;
 105 +       __u8  qp_state;
 106 +       __u8  cur_qp_state;
 107 +       __u8  path_mtu;
 108 +       __u8  path_mig_state;
 109 +       __u8  en_sqd_async_notify;
 110 +       __u8  max_rd_atomic;
 111 +       __u8  max_dest_rd_atomic;
 112 +       __u8  min_rnr_timer;
 113 +       __u8  port_num;
 114 +       __u8  timeout;
 115 +       __u8  retry_cnt;
 116 +       __u8  rnr_retry;
 117 +       __u8  alt_port_num;
 118 +       __u8  alt_timeout;
 119 +       __u8  reserved[6];
 120 +       __u64 driver_data[0];
 121 +};
 122 +
 123 +struct ibv_query_xrc_rcv_qp {
 124 +       __u32 command;
 125 +       __u16 in_words;
 126 +       __u16 out_words;
 127 +       __u64 response;
 128 +       __u32 xrc_domain_handle;
 129 +       __u32 qp_num;
 130 +       __u32 attr_mask;
 131 +       __u32 reserved;
 132 +       __u64 driver_data[0];
 133 +};
 134 +
 135 +struct ibv_reg_xrc_rcv_qp {
 136 +       __u32 command;
 137 +       __u16 in_words;
 138 +       __u16 out_words;
 139 +       __u32 xrc_domain_handle;
 140 +       __u32 qp_num;
 141 +       __u64 driver_data[0];
 142 +};
 143 +
 144 +struct ibv_unreg_xrc_rcv_qp {
 145 +       __u32 command;
 146 +       __u16 in_words;
 147 +       __u16 out_words;
 148 +       __u32 xrc_domain_handle;
 149 +       __u32 qp_num;
 150 +       __u64 driver_data[0];
 151 +};
 152 +
 153  struct ibv_kern_send_wr {
 154         __u64 wr_id;
 155         __u32 num_sge;
 156 @@ -848,6 +940,11 @@ enum {
 157         IB_USER_VERBS_CMD_CREATE_XRC_SRQ_V2 = -1,
 158         IB_USER_VERBS_CMD_OPEN_XRC_DOMAIN_V2 = -1,
 159         IB_USER_VERBS_CMD_CLOSE_XRC_DOMAIN_V2 = -1,
 160 +       IB_USER_VERBS_CMD_CREATE_XRC_RCV_QP_V2 = -1,
 161 +       IB_USER_VERBS_CMD_MODIFY_XRC_RCV_QP_V2 = -1,
 162 +       IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP_V2 = -1,
 163 +       IB_USER_VERBS_CMD_REG_XRC_RCV_QP_V2 = -1,
 164 +       IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP_V2 = -1,
 165  };
 166
 167  struct ibv_destroy_cq_v1 {
 168 Index: libibverbs/include/infiniband/verbs.h
 169 ===================================================================
 170 --- libibverbs.orig/include/infiniband/verbs.h  2009-11-01 15:18:20.635171000 +0200
 171 +++ libibverbs/include/infiniband/verbs.h       2009-11-01 15:18:24.585280000 +0200
 172 @@ -205,12 +205,17 @@ enum ibv_event_type {
 173         IBV_EVENT_CLIENT_REREGISTER
 174  };
 175
 176 +enum ibv_event_flags {
 177 +       IBV_XRC_QP_EVENT_FLAG = 0x80000000,
 178 +};
 179 +
 180  struct ibv_async_event {
 181         union {
 182                 struct ibv_cq  *cq;
 183                 struct ibv_qp  *qp;
 184                 struct ibv_srq *srq;
 185                 int             port_num;
 186 +               uint32_t        xrc_qp_num;
 187         } element;
 188         enum ibv_event_type     event_type;
 189  };
 190 @@ -648,6 +653,22 @@ struct ibv_more_ops {
 191         struct ibv_xrc_domain * (*open_xrc_domain)(struct ibv_context *context,
 192                                                    int fd, int oflag);
 193         int                     (*close_xrc_domain)(struct ibv_xrc_domain *d);
 194 +       int                     (*create_xrc_rcv_qp)(struct ibv_qp_init_attr *init_attr,
 195 +                                                    uint32_t *xrc_qp_num);
 196 +       int                     (*modify_xrc_rcv_qp)(struct ibv_xrc_domain *xrc_domain,
 197 +                                                    uint32_t xrc_qp_num,
 198 +                                                    struct ibv_qp_attr *attr,
 199 +                                                    int attr_mask);
 200 +       int                     (*query_xrc_rcv_qp)(struct ibv_xrc_domain *xrc_domain,
 201 +                                                   uint32_t xrc_qp_num,
 202 +                                                   struct ibv_qp_attr *attr,
 203 +                                                   int attr_mask,
 204 +                                                   struct ibv_qp_init_attr *init_attr);
 205 +       int                     (*reg_xrc_rcv_qp)(struct ibv_xrc_domain *xrc_domain,
 206 +                                                 uint32_t xrc_qp_num);
 207 +       int                     (*unreg_xrc_rcv_qp)(struct ibv_xrc_domain *xrc_domain,
 208 +                                                   uint32_t xrc_qp_num);
 209 +
 210  };
 211
 212  struct ibv_context_ops {
 213 @@ -1174,6 +1195,108 @@ struct ibv_xrc_domain *ibv_open_xrc_doma
 214   */
 215  int ibv_close_xrc_domain(struct ibv_xrc_domain *d);
 216
 217 +/**
 218 + * ibv_create_xrc_rcv_qp - creates an XRC QP for serving as a receive-side-only QP,
 219 + *
 220 + * This QP is created in kernel space, and persists until the last process
 221 + * registered for the QP calls ibv_unreg_xrc_rcv_qp() (at which time the QP
 222 + * is destroyed).
 223 + *
 224 + * @init_attr: init attributes to use for QP. xrc domain MUST be included here.
 225 + *            All other fields are ignored.
 226 + *
 227 + * @xrc_rcv_qpn: qp_num of created QP (if success). To be passed to the
 228 + *              remote node (sender). The remote node will use xrc_rcv_qpn
 229 + *              in ibv_post_send when sending to XRC SRQ's on this host
 230 + *              in the same xrc domain.
 231 + *
 232 + * RETURNS: success (0), or a (negative) error value.
 233 + *
 234 + * NOTE: this verb also registers the calling user-process with the QP at its
 235 + *      creation time (implicit call to ibv_reg_xrc_rcv_qp), to avoid race
 236 + *      conditions. The creating process will need to call ibv_unreg_xrc_qp()
 237 + *      for the QP to release it from this process.
 238 + */
 239 +int ibv_create_xrc_rcv_qp(struct ibv_qp_init_attr *init_attr,
 240 +                         uint32_t *xrc_rcv_qpn);
 241 +
 242 +/**
 243 + * ibv_modify_xrc_rcv_qp - modifies an xrc_rcv qp.
 244 + *
 245 + * @xrc_domain: xrc domain the QP belongs to (for verification).
 246 + * @xrc_qp_num: The (24 bit) number of the XRC QP.
 247 + * @attr: modify-qp attributes. The following fields must be specified:
 248 + *             for RESET_2_INIT: qp_state, pkey_index , port, qp_access_flags
 249 + *             for INIT_2_RTR:   qp_state, path_mtu, dest_qp_num, rq_psn,
 250 + *                               max_dest_rd_atomic, min_rnr_timer, ah_attr
 251 + *             The QP need not be brought to RTS for the QP to operate as a
 252 + *             receive-only QP.
 253 + * @attr_mask:  bitmap indicating which attributes are provided in the attr
 254 + *             struct. Used for validity checking.
 255 + *             The following bits must be set:
 256 + *             for RESET_2_INIT: IBV_QP_PKEY_INDEX, IBV_QP_PORT,
 257 + *                               IBV_QP_ACCESS_FLAGS, IBV_QP_STATE
 258 + *             for INIT_2_RTR: IBV_QP_AV, IBV_QP_PATH_MTU, IBV_QP_DEST_QPN,
 259 + *                             IBV_QP_RQ_PSN, IBV_QP_MAX_DEST_RD_ATOMIC,
 260 + *                             IBV_QP_MIN_RNR_TIMER, IBV_QP_STATE
 261 + *
 262 + * RETURNS: success (0), or a (positive) error value.
 263 + *
 264 + */
 265 +int ibv_modify_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
 266 +                         uint32_t xrc_qp_num,
 267 +                         struct ibv_qp_attr *attr, int attr_mask);
 268 +
 269 +/**
 270 + * ibv_query_xrc_rcv_qp - queries an xrc_rcv qp.
 271 + *
 272 + * @xrc_domain: xrc domain the QP belongs to (for verification).
 273 + * @xrc_qp_num: The (24 bit) number of the XRC QP.
 274 + * @attr: for returning qp attributes.
 275 + * @attr_mask:  bitmap indicating which attributes to return.
 276 + * @init_attr: for returning the init attributes
 277 + *
 278 + * RETURNS: success (0), or a (positive) error value.
 279 + *
 280 + */
 281 +int ibv_query_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain, uint32_t xrc_qp_num,
 282 +                        struct ibv_qp_attr *attr, int attr_mask,
 283 +                        struct ibv_qp_init_attr *init_attr);
 284 +
 285 +/**
 286 + * ibv_reg_xrc_rcv_qp: registers a user process with an XRC QP which serves as
 287 + *         a receive-side only QP.
 288 + *
 289 + * @xrc_domain: xrc domain the QP belongs to (for verification).
 290 + * @xrc_qp_num: The (24 bit) number of the XRC QP.
 291 + *
 292 + * RETURNS: success (0),
 293 + *     or error (EINVAL), if:
 294 + *             1. There is no such QP_num allocated.
 295 + *             2. The QP is allocated, but is not an receive XRC QP
 296 + *             3. The XRC QP does not belong to the given domain.
 297 + */
 298 +int ibv_reg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain, uint32_t xrc_qp_num);
 299 +
 300 +/**
 301 + * ibv_unreg_xrc_rcv_qp: detaches a user process from an XRC QP serving as
 302 + *         a receive-side only QP. If as a result, there are no remaining
 303 + *        userspace processes registered for this XRC QP, it is destroyed.
 304 + *
 305 + * @xrc_domain: xrc domain the QP belongs to (for verification).
 306 + * @xrc_qp_num: The (24 bit) number of the XRC QP.
 307 + *
 308 + * RETURNS: success (0),
 309 + *         or error (EINVAL), if:
 310 + *             1. There is no such QP_num allocated.
 311 + *             2. The QP is allocated, but is not an XRC QP
 312 + *             3. The XRC QP does not belong to the given domain.
 313 + * NOTE: There is no reason to return a special code if the QP is destroyed.
 314 + *      The unregister simply succeeds.
 315 + */
 316 +int ibv_unreg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
 317 +                        uint32_t xrc_qp_num);
 318 +
 319  END_C_DECLS
 320
 321  #  undef __attribute_const
 322 Index: libibverbs/src/cmd.c
 323 ===================================================================
 324 --- libibverbs.orig/src/cmd.c   2009-11-01 15:18:20.643167000 +0200
 325 +++ libibverbs/src/cmd.c        2009-11-01 15:18:24.592284000 +0200
 326 @@ -828,6 +828,188 @@ int ibv_cmd_modify_qp(struct ibv_qp *qp,
 327         return 0;
 328  }
 329
 330 +int ibv_cmd_create_xrc_rcv_qp(struct ibv_qp_init_attr *init_attr,
 331 +                            uint32_t *xrc_rcv_qpn)
 332 +{
 333 +       struct ibv_create_xrc_rcv_qp cmd;
 334 +       struct ibv_create_xrc_rcv_qp_resp resp;
 335 +
 336 +       if (abi_ver < 6)
 337 +               return ENOSYS;
 338 +
 339 +       IBV_INIT_CMD_RESP(&cmd, sizeof cmd, CREATE_XRC_RCV_QP, &resp,
 340 +                         sizeof resp);
 341 +
 342 +       cmd.xrc_domain_handle = init_attr->xrc_domain->handle;
 343 +       cmd.max_send_wr     = init_attr->cap.max_send_wr;
 344 +       cmd.max_recv_wr     = init_attr->cap.max_recv_wr;
 345 +       cmd.max_send_sge    = init_attr->cap.max_send_sge;
 346 +       cmd.max_recv_sge    = init_attr->cap.max_recv_sge;
 347 +       cmd.max_inline_data = init_attr->cap.max_inline_data;
 348 +       cmd.sq_sig_all       = init_attr->sq_sig_all;
 349 +       cmd.qp_type          = init_attr->qp_type;
 350 +       cmd.reserved[0] = cmd.reserved[1] = 0;
 351 +
 352 +       if (write(init_attr->xrc_domain->context->cmd_fd, &cmd, sizeof cmd) !=
 353 +           sizeof cmd)
 354 +               return errno;
 355 +
 356 +       *xrc_rcv_qpn = resp.qpn;
 357 +
 358 +       return 0;
 359 +}
 360 +
 361 +int ibv_cmd_modify_xrc_rcv_qp(struct ibv_xrc_domain *d, uint32_t xrc_qp_num,
 362 +                             struct ibv_qp_attr *attr, int attr_mask)
 363 +{
 364 +       struct ibv_modify_xrc_rcv_qp cmd;
 365 +
 366 +       if (abi_ver < 6)
 367 +               return ENOSYS;
 368 +
 369 +       IBV_INIT_CMD(&cmd, sizeof cmd, MODIFY_XRC_RCV_QP);
 370 +
 371 +       cmd.xrc_domain_handle    = d->handle;
 372 +       cmd.qp_num               = xrc_qp_num;
 373 +       cmd.attr_mask            = attr_mask;
 374 +       cmd.qkey                 = attr->qkey;
 375 +       cmd.rq_psn               = attr->rq_psn;
 376 +       cmd.sq_psn               = attr->sq_psn;
 377 +       cmd.dest_qp_num          = attr->dest_qp_num;
 378 +       cmd.qp_access_flags      = attr->qp_access_flags;
 379 +       cmd.pkey_index           = attr->pkey_index;
 380 +       cmd.alt_pkey_index       = attr->alt_pkey_index;
 381 +       cmd.qp_state             = attr->qp_state;
 382 +       cmd.cur_qp_state         = attr->cur_qp_state;
 383 +       cmd.path_mtu             = attr->path_mtu;
 384 +       cmd.path_mig_state       = attr->path_mig_state;
 385 +       cmd.en_sqd_async_notify  = attr->en_sqd_async_notify;
 386 +       cmd.max_rd_atomic        = attr->max_rd_atomic;
 387 +       cmd.max_dest_rd_atomic   = attr->max_dest_rd_atomic;
 388 +       cmd.min_rnr_timer        = attr->min_rnr_timer;
 389 +       cmd.port_num             = attr->port_num;
 390 +       cmd.timeout              = attr->timeout;
 391 +       cmd.retry_cnt            = attr->retry_cnt;
 392 +       cmd.rnr_retry            = attr->rnr_retry;
 393 +       cmd.alt_port_num         = attr->alt_port_num;
 394 +       cmd.alt_timeout          = attr->alt_timeout;
 395 +
 396 +       memcpy(cmd.dest.dgid, attr->ah_attr.grh.dgid.raw, 16);
 397 +       cmd.dest.flow_label         = attr->ah_attr.grh.flow_label;
 398 +       cmd.dest.dlid               = attr->ah_attr.dlid;
 399 +       cmd.dest.reserved           = 0;
 400 +       cmd.dest.sgid_index         = attr->ah_attr.grh.sgid_index;
 401 +       cmd.dest.hop_limit          = attr->ah_attr.grh.hop_limit;
 402 +       cmd.dest.traffic_class      = attr->ah_attr.grh.traffic_class;
 403 +       cmd.dest.sl                 = attr->ah_attr.sl;
 404 +       cmd.dest.src_path_bits      = attr->ah_attr.src_path_bits;
 405 +       cmd.dest.static_rate        = attr->ah_attr.static_rate;
 406 +       cmd.dest.is_global          = attr->ah_attr.is_global;
 407 +       cmd.dest.port_num           = attr->ah_attr.port_num;
 408 +
 409 +       memcpy(cmd.alt_dest.dgid, attr->alt_ah_attr.grh.dgid.raw, 16);
 410 +       cmd.alt_dest.flow_label    = attr->alt_ah_attr.grh.flow_label;
 411 +       cmd.alt_dest.dlid           = attr->alt_ah_attr.dlid;
 412 +       cmd.alt_dest.reserved       = 0;
 413 +       cmd.alt_dest.sgid_index    = attr->alt_ah_attr.grh.sgid_index;
 414 +       cmd.alt_dest.hop_limit     = attr->alt_ah_attr.grh.hop_limit;
 415 +       cmd.alt_dest.traffic_class = attr->alt_ah_attr.grh.traffic_class;
 416 +       cmd.alt_dest.sl             = attr->alt_ah_attr.sl;
 417 +       cmd.alt_dest.src_path_bits = attr->alt_ah_attr.src_path_bits;
 418 +       cmd.alt_dest.static_rate   = attr->alt_ah_attr.static_rate;
 419 +       cmd.alt_dest.is_global     = attr->alt_ah_attr.is_global;
 420 +       cmd.alt_dest.port_num       = attr->alt_ah_attr.port_num;
 421 +
 422 +       cmd.reserved[0] = cmd.reserved[1] = 0;
 423 +
 424 +       if (write(d->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
 425 +               return errno;
 426 +
 427 +       return 0;
 428 +}
 429 +
 430 +int ibv_cmd_query_xrc_rcv_qp(struct ibv_xrc_domain *d, uint32_t xrc_qp_num,
 431 +                            struct ibv_qp_attr *attr, int attr_mask,
 432 +                            struct ibv_qp_init_attr *init_attr)
 433 +{
 434 +       struct ibv_query_xrc_rcv_qp cmd;
 435 +       struct ibv_query_qp_resp resp;
 436 +
 437 +       if (abi_ver < 6)
 438 +               return ENOSYS;
 439 +
 440 +       IBV_INIT_CMD_RESP(&cmd, sizeof cmd, QUERY_XRC_RCV_QP, &resp,
 441 +                         sizeof resp);
 442 +       cmd.xrc_domain_handle = d->handle;
 443 +       cmd.qp_num = xrc_qp_num;
 444 +       cmd.attr_mask = attr_mask;
 445 +
 446 +       if (write(d->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
 447 +               return errno;
 448 +
 449 +       VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
 450 +
 451 +       attr->qkey                          = resp.qkey;
 452 +       attr->rq_psn                        = resp.rq_psn;
 453 +       attr->sq_psn                        = resp.sq_psn;
 454 +       attr->dest_qp_num                   = resp.dest_qp_num;
 455 +       attr->qp_access_flags               = resp.qp_access_flags;
 456 +       attr->pkey_index                    = resp.pkey_index;
 457 +       attr->alt_pkey_index                = resp.alt_pkey_index;
 458 +       attr->qp_state                      = resp.qp_state;
 459 +       attr->cur_qp_state                  = resp.cur_qp_state;
 460 +       attr->path_mtu                      = resp.path_mtu;
 461 +       attr->path_mig_state                = resp.path_mig_state;
 462 +       attr->sq_draining                   = resp.sq_draining;
 463 +       attr->max_rd_atomic                 = resp.max_rd_atomic;
 464 +       attr->max_dest_rd_atomic            = resp.max_dest_rd_atomic;
 465 +       attr->min_rnr_timer                 = resp.min_rnr_timer;
 466 +       attr->port_num                      = resp.port_num;
 467 +       attr->timeout                       = resp.timeout;
 468 +       attr->retry_cnt                     = resp.retry_cnt;
 469 +       attr->rnr_retry                     = resp.rnr_retry;
 470 +       attr->alt_port_num                  = resp.alt_port_num;
 471 +       attr->alt_timeout                   = resp.alt_timeout;
 472 +       attr->cap.max_send_wr               = resp.max_send_wr;
 473 +       attr->cap.max_recv_wr               = resp.max_recv_wr;
 474 +       attr->cap.max_send_sge              = resp.max_send_sge;
 475 +       attr->cap.max_recv_sge              = resp.max_recv_sge;
 476 +       attr->cap.max_inline_data           = resp.max_inline_data;
 477 +
 478 +       memcpy(attr->ah_attr.grh.dgid.raw, resp.dest.dgid, 16);
 479 +       attr->ah_attr.grh.flow_label        = resp.dest.flow_label;
 480 +       attr->ah_attr.dlid                  = resp.dest.dlid;
 481 +       attr->ah_attr.grh.sgid_index        = resp.dest.sgid_index;
 482 +       attr->ah_attr.grh.hop_limit         = resp.dest.hop_limit;
 483 +       attr->ah_attr.grh.traffic_class     = resp.dest.traffic_class;
 484 +       attr->ah_attr.sl                    = resp.dest.sl;
 485 +       attr->ah_attr.src_path_bits         = resp.dest.src_path_bits;
 486 +       attr->ah_attr.static_rate           = resp.dest.static_rate;
 487 +       attr->ah_attr.is_global             = resp.dest.is_global;
 488 +       attr->ah_attr.port_num              = resp.dest.port_num;
 489 +
 490 +       memcpy(attr->alt_ah_attr.grh.dgid.raw, resp.alt_dest.dgid, 16);
 491 +       attr->alt_ah_attr.grh.flow_label    = resp.alt_dest.flow_label;
 492 +       attr->alt_ah_attr.dlid              = resp.alt_dest.dlid;
 493 +       attr->alt_ah_attr.grh.sgid_index    = resp.alt_dest.sgid_index;
 494 +       attr->alt_ah_attr.grh.hop_limit     = resp.alt_dest.hop_limit;
 495 +       attr->alt_ah_attr.grh.traffic_class = resp.alt_dest.traffic_class;
 496 +       attr->alt_ah_attr.sl                = resp.alt_dest.sl;
 497 +       attr->alt_ah_attr.src_path_bits     = resp.alt_dest.src_path_bits;
 498 +       attr->alt_ah_attr.static_rate       = resp.alt_dest.static_rate;
 499 +       attr->alt_ah_attr.is_global         = resp.alt_dest.is_global;
 500 +       attr->alt_ah_attr.port_num          = resp.alt_dest.port_num;
 501 +
 502 +       init_attr->cap.max_send_wr          = resp.max_send_wr;
 503 +       init_attr->cap.max_recv_wr          = resp.max_recv_wr;
 504 +       init_attr->cap.max_send_sge         = resp.max_send_sge;
 505 +       init_attr->cap.max_recv_sge         = resp.max_recv_sge;
 506 +       init_attr->cap.max_inline_data      = resp.max_inline_data;
 507 +       init_attr->sq_sig_all               = resp.sq_sig_all;
 508 +
 509 +       return 0;
 510 +}
 511 +
 512  static int ibv_cmd_destroy_qp_v1(struct ibv_qp *qp)
 513  {
 514         struct ibv_destroy_qp_v1 cmd;
 515 @@ -1192,3 +1374,36 @@ int ibv_cmd_close_xrc_domain(struct ibv_
 516         return 0;
 517  }
 518
 519 +int ibv_cmd_reg_xrc_rcv_qp(struct ibv_xrc_domain *d, uint32_t xrc_qp_num)
 520 +{
 521 +       struct ibv_reg_xrc_rcv_qp cmd;
 522 +
 523 +       if (abi_ver < 6)
 524 +               return ENOSYS;
 525 +
 526 +       IBV_INIT_CMD(&cmd, sizeof cmd, REG_XRC_RCV_QP);
 527 +       cmd.xrc_domain_handle = d->handle;
 528 +       cmd.qp_num = xrc_qp_num;
 529 +
 530 +       if (write(d->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
 531 +               return errno;
 532 +       return 0;
 533 +}
 534 +
 535 +int ibv_cmd_unreg_xrc_rcv_qp(struct ibv_xrc_domain *d, uint32_t xrc_qp_num)
 536 +{
 537 +       struct ibv_unreg_xrc_rcv_qp cmd;
 538 +
 539 +       if (abi_ver < 6)
 540 +               return ENOSYS;
 541 +
 542 +       IBV_INIT_CMD(&cmd, sizeof cmd, UNREG_XRC_RCV_QP);
 543 +       cmd.xrc_domain_handle = d->handle;
 544 +       cmd.qp_num = xrc_qp_num;
 545 +
 546 +       if (write(d->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
 547 +               return errno;
 548 +       return 0;
 549 +}
 550 +
 551 +
 552 Index: libibverbs/src/device.c
 553 ===================================================================
 554 --- libibverbs.orig/src/device.c        2009-11-01 15:18:17.794116000 +0200
 555 +++ libibverbs/src/device.c     2009-11-01 15:18:24.597279000 +0200
 556 @@ -191,31 +191,33 @@ int __ibv_get_async_event(struct ibv_con
 557
 558         event->event_type = ev.event_type;
 559
 560 -       switch (event->event_type) {
 561 -       case IBV_EVENT_CQ_ERR:
 562 -               event->element.cq = (void *) (uintptr_t) ev.element;
 563 -               break;
 564 -
 565 -       case IBV_EVENT_QP_FATAL:
 566 -       case IBV_EVENT_QP_REQ_ERR:
 567 -       case IBV_EVENT_QP_ACCESS_ERR:
 568 -       case IBV_EVENT_COMM_EST:
 569 -       case IBV_EVENT_SQ_DRAINED:
 570 -       case IBV_EVENT_PATH_MIG:
 571 -       case IBV_EVENT_PATH_MIG_ERR:
 572 -       case IBV_EVENT_QP_LAST_WQE_REACHED:
 573 -               event->element.qp = (void *) (uintptr_t) ev.element;
 574 -               break;
 575 -
 576 -       case IBV_EVENT_SRQ_ERR:
 577 -       case IBV_EVENT_SRQ_LIMIT_REACHED:
 578 -               event->element.srq = (void *) (uintptr_t) ev.element;
 579 -               break;
 580 -
 581 -       default:
 582 -               event->element.port_num = ev.element;
 583 -               break;
 584 -       }
 585 +       if (event->event_type & IBV_XRC_QP_EVENT_FLAG) {
 586 +               event->element.xrc_qp_num = ev.element;
 587 +       } else
 588 +               switch (event->event_type) {
 589 +               case IBV_EVENT_CQ_ERR:
 590 +                       event->element.cq = (void *) (uintptr_t) ev.element;
 591 +                       break;
 592 +
 593 +               case IBV_EVENT_QP_FATAL:
 594 +               case IBV_EVENT_QP_REQ_ERR:
 595 +               case IBV_EVENT_QP_ACCESS_ERR:
 596 +               case IBV_EVENT_COMM_EST:
 597 +               case IBV_EVENT_SQ_DRAINED:
 598 +               case IBV_EVENT_PATH_MIG:
 599 +               case IBV_EVENT_PATH_MIG_ERR:
 600 +               case IBV_EVENT_QP_LAST_WQE_REACHED:
 601 +                       event->element.qp = (void *) (uintptr_t) ev.element;
 602 +                       break;
 603 +
 604 +               case IBV_EVENT_SRQ_ERR:
 605 +               case IBV_EVENT_SRQ_LIMIT_REACHED:
 606 +                       event->element.srq = (void *) (uintptr_t) ev.element;
 607 +                       break;
 608 +               default:
 609 +                       event->element.port_num = ev.element;
 610 +                       break;
 611 +               }
 612
 613         if (context->ops.async_event)
 614                 context->ops.async_event(event);
 615 Index: libibverbs/src/libibverbs.map
 616 ===================================================================
 617 --- libibverbs.orig/src/libibverbs.map  2009-11-01 15:18:20.646169000 +0200
 618 +++ libibverbs/src/libibverbs.map       2009-11-01 15:18:24.600279000 +0200
 619 @@ -97,6 +97,16 @@ IBVERBS_1.1 {
 620                 ibv_cmd_open_xrc_domain;
 621                 ibv_close_xrc_domain;
 622                 ibv_cmd_close_xrc_domain;
 623 +               ibv_create_xrc_rcv_qp;
 624 +               ibv_cmd_create_xrc_rcv_qp;
 625 +               ibv_modify_xrc_rcv_qp;
 626 +               ibv_cmd_modify_xrc_rcv_qp;
 627 +               ibv_query_xrc_rcv_qp;
 628 +               ibv_cmd_query_xrc_rcv_qp;
 629 +               ibv_reg_xrc_rcv_qp;
 630 +               ibv_cmd_reg_xrc_rcv_qp;
 631 +               ibv_unreg_xrc_rcv_qp;
 632 +               ibv_cmd_unreg_xrc_rcv_qp;
 633
 634                 ibv_node_type_str;
 635                 ibv_port_state_str;
 636 Index: libibverbs/src/verbs.c
 637 ===================================================================
 638 --- libibverbs.orig/src/verbs.c 2009-11-01 15:18:20.650169000 +0200
 639 +++ libibverbs/src/verbs.c      2009-11-01 15:18:24.604279000 +0200
 640 @@ -597,3 +597,62 @@ int ibv_close_xrc_domain(struct ibv_xrc_
 641
 642         return d->context->more_ops->close_xrc_domain(d);
 643  }
 644 +
 645 +int ibv_create_xrc_rcv_qp(struct ibv_qp_init_attr *init_attr,
 646 +                         uint32_t *xrc_rcv_qpn)
 647 +{
 648 +       struct ibv_context *c;
 649 +       if (!init_attr || !(init_attr->xrc_domain))
 650 +               return EINVAL;
 651 +
 652 +       c = init_attr->xrc_domain->context;
 653 +       if (!c->more_ops)
 654 +               return ENOSYS;
 655 +
 656 +       return c->more_ops->create_xrc_rcv_qp(init_attr,
 657 +                                             xrc_rcv_qpn);
 658 +}
 659 +
 660 +int ibv_modify_xrc_rcv_qp(struct ibv_xrc_domain *d,
 661 +                         uint32_t xrc_rcv_qpn,
 662 +                         struct ibv_qp_attr *attr,
 663 +                         int attr_mask)
 664 +{
 665 +       if (!d || !attr)
 666 +               return EINVAL;
 667 +
 668 +       if (!d->context->more_ops)
 669 +               return ENOSYS;
 670 +
 671 +       return d->context->more_ops->modify_xrc_rcv_qp(d, xrc_rcv_qpn, attr,
 672 +                                                      attr_mask);
 673 +}
 674 +
 675 +int ibv_query_xrc_rcv_qp(struct ibv_xrc_domain *d,
 676 +                        uint32_t xrc_rcv_qpn,
 677 +                        struct ibv_qp_attr *attr,
 678 +                        int attr_mask,
 679 +                        struct ibv_qp_init_attr *init_attr)
 680 +{
 681 +       if (!d)
 682 +               return EINVAL;
 683 +
 684 +       if (!d->context->more_ops)
 685 +               return ENOSYS;
 686 +
 687 +       return d->context->more_ops->query_xrc_rcv_qp(d, xrc_rcv_qpn, attr,
 688 +                                                     attr_mask, init_attr);
 689 +}
 690 +
 691 +int ibv_reg_xrc_rcv_qp(struct ibv_xrc_domain *d,
 692 +                      uint32_t xrc_rcv_qpn)
 693 +{
 694 +       return d->context->more_ops->reg_xrc_rcv_qp(d, xrc_rcv_qpn);
 695 +}
 696 +
 697 +int ibv_unreg_xrc_rcv_qp(struct ibv_xrc_domain *d,
 698 +                        uint32_t xrc_rcv_qpn)
 699 +{
 700 +       return d->context->more_ops->unreg_xrc_rcv_qp(d, xrc_rcv_qpn);
 701 +}
 702 +