sys/ofed/drivers/infiniband/core/cma.c

   1 /*
   2  * Copyright (c) 2005 Voltaire Inc.  All rights reserved.
   3  * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
   4  * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved.
   5  * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * OpenIB.org BSD license below:
  12  *
  13  *     Redistribution and use in source and binary forms, with or
  14  *     without modification, are permitted provided that the following
  15  *     conditions are met:
  16  *
  17  *      - Redistributions of source code must retain the above
  18  *        copyright notice, this list of conditions and the following
  19  *        disclaimer.
  20  *
  21  *      - Redistributions in binary form must reproduce the above
  22  *        copyright notice, this list of conditions and the following
  23  *        disclaimer in the documentation and/or other materials
  24  *        provided with the distribution.
  25  *
  26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  33  * SOFTWARE.
  34  */
  35
  36 #include <linux/completion.h>
  37 #include <linux/in.h>
  38 #include <linux/in6.h>
  39 #include <linux/mutex.h>
  40 #include <linux/random.h>
  41 #include <linux/idr.h>
  42 #include <linux/inetdevice.h>
  43 #include <linux/slab.h>
  44 #include <linux/module.h>
  45 #include <linux/string.h>
  46 #include <net/route.h>
  47
  48 #include <net/tcp.h>
  49 #include <net/ipv6.h>
  50
  51 #include <rdma/rdma_cm.h>
  52 #include <rdma/rdma_cm_ib.h>
  53 #include <rdma/ib_cache.h>
  54 #include <rdma/ib_cm.h>
  55 #include <rdma/ib_sa.h>
  56 #include <rdma/iw_cm.h>
  57
  58 MODULE_AUTHOR("Sean Hefty");
  59 MODULE_DESCRIPTION("Generic RDMA CM Agent");
  60 MODULE_LICENSE("Dual BSD/GPL");
  61
  62 #define CMA_CM_RESPONSE_TIMEOUT 20
  63 #define CMA_MAX_CM_RETRIES 15
  64 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
  65 #define CMA_IBOE_PACKET_LIFETIME 18
  66
  67 static int cma_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
  68 module_param_named(cma_response_timeout, cma_response_timeout, int, 0644);
  69 MODULE_PARM_DESC(cma_response_timeout, "CMA_CM_RESPONSE_TIMEOUT (default=20)");
  70
  71 static int def_prec2sl = 3;
  72 module_param_named(def_prec2sl, def_prec2sl, int, 0644);
  73 MODULE_PARM_DESC(def_prec2sl, "Default value for SL priority with RoCE. Valid values 0 - 7");
  74
  75 static int debug_level = 0;
  76 #define cma_pr(level, priv, format, arg...)             \
  77         printk(level "CMA: %p: %s: " format, ((struct rdma_id_priv *) priv) , __func__, ## arg)
  78
  79 #define cma_dbg(priv, format, arg...)           \
  80         do { if (debug_level) cma_pr(KERN_DEBUG, priv, format, ## arg); } while (0)
  81
  82 #define cma_warn(priv, format, arg...)          \
  83         cma_pr(KERN_WARNING, priv, format, ## arg)
  84
  85 #define CMA_GID_FMT        "%2.2x%2.2x:%2.2x%2.2x"
  86 #define CMA_GID_RAW_ARG(gid) ((u8 *)(gid))[12],\
  87                                    ((u8 *)(gid))[13],\
  88                                    ((u8 *)(gid))[14],\
  89                                    ((u8 *)(gid))[15]
  90
  91 #define CMA_GID_ARG(gid)   CMA_GID_RAW_ARG((gid).raw)
  92 #define cma_debug_path(priv, pfx, p) \
  93         cma_dbg(priv, pfx "sgid=" CMA_GID_FMT ",dgid="  \
  94                 CMA_GID_FMT "\n", CMA_GID_ARG(p.sgid),  \
  95                 CMA_GID_ARG(p.dgid))
  96
  97 #define cma_debug_gid(priv, g) \
  98         cma_dbg(priv, "gid=" CMA_GID_FMT "\n", CMA_GID_ARG(g)
  99
 100 module_param_named(debug_level, debug_level, int, 0644);
 101 MODULE_PARM_DESC(debug_level, "debug level default=0");
 102
 103 static void cma_add_one(struct ib_device *device);
 104 static void cma_remove_one(struct ib_device *device);
 105
 106 static struct ib_client cma_client = {
 107         .name   = "cma",
 108         .add    = cma_add_one,
 109         .remove = cma_remove_one
 110 };
 111
 112 static struct ib_sa_client sa_client;
 113 static struct rdma_addr_client addr_client;
 114 static LIST_HEAD(dev_list);
 115 static LIST_HEAD(listen_any_list);
 116 static DEFINE_MUTEX(lock);
 117 static struct workqueue_struct *cma_wq;
 118 static struct workqueue_struct *cma_free_wq;
 119 static DEFINE_IDR(sdp_ps);
 120 static DEFINE_IDR(tcp_ps);
 121 static DEFINE_IDR(udp_ps);
 122 static DEFINE_IDR(ipoib_ps);
 123 static DEFINE_IDR(ib_ps);
 124
 125 struct cma_device {
 126         struct list_head        list;
 127         struct ib_device        *device;
 128         struct completion       comp;
 129         atomic_t                refcount;
 130         struct list_head        id_list;
 131 };
 132
 133 struct rdma_bind_list {
 134         struct idr              *ps;
 135         struct hlist_head       owners;
 136         unsigned short          port;
 137 };
 138
 139 enum {
 140         CMA_OPTION_AFONLY,
 141 };
 142
 143 /*
 144  * Device removal can occur at anytime, so we need extra handling to
 145  * serialize notifying the user of device removal with other callbacks.
 146  * We do this by disabling removal notification while a callback is in process,
 147  * and reporting it after the callback completes.
 148  */
 149 struct rdma_id_private {
 150         struct rdma_cm_id       id;
 151
 152         struct rdma_bind_list   *bind_list;
 153         struct socket           *sock;
 154         struct hlist_node       node;
 155         struct list_head        list; /* listen_any_list or cma_device.list */
 156         struct list_head        listen_list; /* per device listens */
 157         struct cma_device       *cma_dev;
 158         struct list_head        mc_list;
 159
 160         int                     internal_id;
 161         enum rdma_cm_state      state;
 162         spinlock_t              lock;
 163         spinlock_t              cm_lock;
 164         struct mutex            qp_mutex;
 165
 166         struct completion       comp;
 167         atomic_t                refcount;
 168         struct mutex            handler_mutex;
 169         struct work_struct      work;  /* garbage coll */
 170
 171         int                     backlog;
 172         int                     timeout_ms;
 173         struct ib_sa_query      *query;
 174         int                     query_id;
 175         union {
 176                 struct ib_cm_id *ib;
 177                 struct iw_cm_id *iw;
 178         } cm_id;
 179
 180         u32                     seq_num;
 181         u32                     qkey;
 182         u32                     qp_num;
 183         pid_t                   owner;
 184         u32                     options;
 185         u8                      srq;
 186         u8                      tos;
 187         u8                      reuseaddr;
 188         u8                      afonly;
 189         int                     qp_timeout;
 190         /* cache for mc record params */
 191         struct ib_sa_mcmember_rec rec;
 192         int is_valid_rec;
 193 };
 194
 195 struct cma_multicast {
 196         struct rdma_id_private *id_priv;
 197         union {
 198                 struct ib_sa_multicast *ib;
 199         } multicast;
 200         struct list_head        list;
 201         void                    *context;
 202         struct sockaddr_storage addr;
 203         struct kref             mcref;
 204 };
 205
 206 struct cma_work {
 207         struct work_struct      work;
 208         struct rdma_id_private  *id;
 209         enum rdma_cm_state      old_state;
 210         enum rdma_cm_state      new_state;
 211         struct rdma_cm_event    event;
 212 };
 213
 214 struct cma_ndev_work {
 215         struct work_struct      work;
 216         struct rdma_id_private  *id;
 217         struct rdma_cm_event    event;
 218 };
 219
 220 struct iboe_mcast_work {
 221         struct work_struct       work;
 222         struct rdma_id_private  *id;
 223         struct cma_multicast    *mc;
 224 };
 225
 226 union cma_ip_addr {
 227         struct in6_addr ip6;
 228         struct {
 229                 __be32 pad[3];
 230                 __be32 addr;
 231         } ip4;
 232 };
 233
 234 struct cma_hdr {
 235         u8 cma_version;
 236         u8 ip_version;  /* IP version: 7:4 */
 237         __be16 port;
 238         union cma_ip_addr src_addr;
 239         union cma_ip_addr dst_addr;
 240 };
 241
 242 struct sdp_hh {
 243         u8 bsdh[16];
 244         u8 sdp_version; /* Major version: 7:4 */
 245         u8 ip_version;  /* IP version: 7:4 */
 246         u8 sdp_specific1[10];
 247         __be16 port;
 248         __be16 sdp_specific2;
 249         union cma_ip_addr src_addr;
 250         union cma_ip_addr dst_addr;
 251 };
 252
 253 struct sdp_hah {
 254         u8 bsdh[16];
 255         u8 sdp_version;
 256 };
 257
 258 #define CMA_VERSION 0x00
 259 #define SDP_MAJ_VERSION 0x2
 260
 261 static int cma_comp(struct rdma_id_private *id_priv, enum rdma_cm_state comp)
 262 {
 263         unsigned long flags;
 264         int ret;
 265
 266         spin_lock_irqsave(&id_priv->lock, flags);
 267         ret = (id_priv->state == comp);
 268         spin_unlock_irqrestore(&id_priv->lock, flags);
 269         return ret;
 270 }
 271
 272 static int cma_comp_exch(struct rdma_id_private *id_priv,
 273                          enum rdma_cm_state comp, enum rdma_cm_state exch)
 274 {
 275         unsigned long flags;
 276         int ret;
 277
 278         spin_lock_irqsave(&id_priv->lock, flags);
 279         if ((ret = (id_priv->state == comp)))
 280                 id_priv->state = exch;
 281         spin_unlock_irqrestore(&id_priv->lock, flags);
 282         return ret;
 283 }
 284
 285 static enum rdma_cm_state cma_exch(struct rdma_id_private *id_priv,
 286                                    enum rdma_cm_state exch)
 287 {
 288         unsigned long flags;
 289         enum rdma_cm_state old;
 290
 291         spin_lock_irqsave(&id_priv->lock, flags);
 292         old = id_priv->state;
 293         id_priv->state = exch;
 294         spin_unlock_irqrestore(&id_priv->lock, flags);
 295         return old;
 296 }
 297
 298 static inline u8 cma_get_ip_ver(struct cma_hdr *hdr)
 299 {
 300         return hdr->ip_version >> 4;
 301 }
 302
 303 static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
 304 {
 305         hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
 306 }
 307
 308 static inline u8 sdp_get_majv(u8 sdp_version)
 309 {
 310         return sdp_version >> 4;
 311 }
 312
 313 static inline u8 sdp_get_ip_ver(struct sdp_hh *hh)
 314 {
 315         return hh->ip_version >> 4;
 316 }
 317
 318 static inline void sdp_set_ip_ver(struct sdp_hh *hh, u8 ip_ver)
 319 {
 320         hh->ip_version = (ip_ver << 4) | (hh->ip_version & 0xF);
 321 }
 322
 323 static void cma_attach_to_dev(struct rdma_id_private *id_priv,
 324                               struct cma_device *cma_dev)
 325 {
 326         atomic_inc(&cma_dev->refcount);
 327         id_priv->cma_dev = cma_dev;
 328         id_priv->id.device = cma_dev->device;
 329         id_priv->id.route.addr.dev_addr.transport =
 330                 rdma_node_get_transport(cma_dev->device->node_type);
 331         list_add_tail(&id_priv->list, &cma_dev->id_list);
 332 }
 333
 334 static inline void cma_deref_dev(struct cma_device *cma_dev)
 335 {
 336         if (atomic_dec_and_test(&cma_dev->refcount))
 337                 complete(&cma_dev->comp);
 338 }
 339
 340 static inline void release_mc(struct kref *kref)
 341 {
 342         struct cma_multicast *mc = container_of(kref, struct cma_multicast, mcref);
 343
 344         kfree(mc->multicast.ib);
 345         kfree(mc);
 346 }
 347
 348 static void cma_release_dev(struct rdma_id_private *id_priv)
 349 {
 350         mutex_lock(&lock);
 351         list_del(&id_priv->list);
 352         cma_deref_dev(id_priv->cma_dev);
 353         id_priv->cma_dev = NULL;
 354         mutex_unlock(&lock);
 355 }
 356
 357 static int cma_set_qkey(struct rdma_id_private *id_priv)
 358 {
 359         struct ib_sa_mcmember_rec rec;
 360         int ret = 0;
 361
 362         if (id_priv->qkey)
 363                 return 0;
 364
 365         switch (id_priv->id.ps) {
 366         case RDMA_PS_UDP:
 367                 id_priv->qkey = RDMA_UDP_QKEY;
 368                 break;
 369         case RDMA_PS_IPOIB:
 370                 ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid);
 371                 ret = ib_sa_get_mcmember_rec(id_priv->id.device,
 372                                              id_priv->id.port_num, &rec.mgid,
 373                                              &rec);
 374                 if (!ret)
 375                         id_priv->qkey = be32_to_cpu(rec.qkey);
 376                 break;
 377         default:
 378                 break;
 379         }
 380         return ret;
 381 }
 382
 383 static int find_gid_port(struct ib_device *device, union ib_gid *gid, u8 port_num)
 384 {
 385         int i;
 386         int err;
 387         struct ib_port_attr props;
 388         union ib_gid tmp;
 389
 390         err = ib_query_port(device, port_num, &props);
 391         if (err)
 392                 return 1;
 393
 394         for (i = 0; i < props.gid_tbl_len; ++i) {
 395                 err = ib_query_gid(device, port_num, i, &tmp);
 396                 if (err)
 397                         return 1;
 398                 if (!memcmp(&tmp, gid, sizeof tmp))
 399                         return 0;
 400         }
 401
 402         return -EAGAIN;
 403 }
 404
 405 static int cma_acquire_dev(struct rdma_id_private *id_priv)
 406 {
 407         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 408         struct cma_device *cma_dev;
 409         union ib_gid gid, iboe_gid;
 410         int ret = -ENODEV;
 411         u8 port;
 412         enum rdma_link_layer dev_ll = dev_addr->dev_type == ARPHRD_INFINIBAND ?
 413                 IB_LINK_LAYER_INFINIBAND : IB_LINK_LAYER_ETHERNET;
 414
 415         if (dev_ll != IB_LINK_LAYER_INFINIBAND &&
 416             id_priv->id.ps == RDMA_PS_IPOIB)
 417                 return -EINVAL;
 418
 419         mutex_lock(&lock);
 420         rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
 421                     &iboe_gid);
 422
 423         memcpy(&gid, dev_addr->src_dev_addr +
 424                rdma_addr_gid_offset(dev_addr), sizeof gid);
 425         list_for_each_entry(cma_dev, &dev_list, list) {
 426                 for (port = 1; port <= cma_dev->device->phys_port_cnt; ++port) {
 427                         if (rdma_port_get_link_layer(cma_dev->device, port) == dev_ll) {
 428                                 if (rdma_node_get_transport(cma_dev->device->node_type) == RDMA_TRANSPORT_IB &&
 429                                     rdma_port_get_link_layer(cma_dev->device, port) == IB_LINK_LAYER_ETHERNET)
 430                                         ret = find_gid_port(cma_dev->device, &iboe_gid, port);
 431                                 else
 432                                         ret = find_gid_port(cma_dev->device, &gid, port);
 433
 434                                 if (!ret) {
 435                                         id_priv->id.port_num = port;
 436                                         goto out;
 437                                 } else if (ret == 1)
 438                         break;
 439         }
 440                 }
 441         }
 442
 443 out:
 444         if (!ret)
 445                 cma_attach_to_dev(id_priv, cma_dev);
 446
 447         mutex_unlock(&lock);
 448         return ret;
 449 }
 450
 451 static void cma_deref_id(struct rdma_id_private *id_priv)
 452 {
 453         if (atomic_dec_and_test(&id_priv->refcount))
 454                 complete(&id_priv->comp);
 455 }
 456
 457 static int cma_disable_callback(struct rdma_id_private *id_priv,
 458                                 enum rdma_cm_state state)
 459 {
 460         mutex_lock(&id_priv->handler_mutex);
 461         if (id_priv->state != state) {
 462                 mutex_unlock(&id_priv->handler_mutex);
 463                 return -EINVAL;
 464         }
 465         return 0;
 466 }
 467
 468 struct rdma_cm_id *rdma_create_id(rdma_cm_event_handler event_handler,
 469                                   void *context, enum rdma_port_space ps,
 470                                   enum ib_qp_type qp_type)
 471 {
 472         struct rdma_id_private *id_priv;
 473
 474         id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
 475         if (!id_priv)
 476                 return ERR_PTR(-ENOMEM);
 477
 478         id_priv->owner = curthread->td_proc->p_pid;
 479         id_priv->state = RDMA_CM_IDLE;
 480         id_priv->id.context = context;
 481         id_priv->id.event_handler = event_handler;
 482         id_priv->id.ps = ps;
 483         id_priv->id.qp_type = qp_type;
 484         spin_lock_init(&id_priv->lock);
 485         spin_lock_init(&id_priv->cm_lock);
 486         mutex_init(&id_priv->qp_mutex);
 487         init_completion(&id_priv->comp);
 488         atomic_set(&id_priv->refcount, 1);
 489         mutex_init(&id_priv->handler_mutex);
 490         INIT_LIST_HEAD(&id_priv->listen_list);
 491         INIT_LIST_HEAD(&id_priv->mc_list);
 492         get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
 493
 494         return &id_priv->id;
 495 }
 496 EXPORT_SYMBOL(rdma_create_id);
 497
 498 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
 499 {
 500         struct ib_qp_attr qp_attr;
 501         int qp_attr_mask, ret;
 502
 503         qp_attr.qp_state = IB_QPS_INIT;
 504         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 505         if (ret)
 506                 return ret;
 507
 508         ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
 509         if (ret)
 510                 return ret;
 511
 512         qp_attr.qp_state = IB_QPS_RTR;
 513         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
 514         if (ret)
 515                 return ret;
 516
 517         qp_attr.qp_state = IB_QPS_RTS;
 518         qp_attr.sq_psn = 0;
 519         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
 520
 521         return ret;
 522 }
 523
 524 static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
 525 {
 526         struct ib_qp_attr qp_attr;
 527         int qp_attr_mask, ret;
 528
 529         qp_attr.qp_state = IB_QPS_INIT;
 530         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 531         if (ret)
 532                 return ret;
 533
 534         return ib_modify_qp(qp, &qp_attr, qp_attr_mask);
 535 }
 536
 537 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
 538                    struct ib_qp_init_attr *qp_init_attr)
 539 {
 540         struct rdma_id_private *id_priv;
 541         struct ib_qp *qp;
 542         int ret;
 543
 544         id_priv = container_of(id, struct rdma_id_private, id);
 545         if (id->device != pd->device)
 546                 return -EINVAL;
 547
 548         qp = ib_create_qp(pd, qp_init_attr);
 549         if (IS_ERR(qp))
 550                 return PTR_ERR(qp);
 551
 552         if (id->qp_type == IB_QPT_UD)
 553                 ret = cma_init_ud_qp(id_priv, qp);
 554         else
 555                 ret = cma_init_conn_qp(id_priv, qp);
 556         if (ret)
 557                 goto err;
 558
 559         id->qp = qp;
 560         id_priv->qp_num = qp->qp_num;
 561         id_priv->srq = (qp->srq != NULL);
 562         return 0;
 563 err:
 564         ib_destroy_qp(qp);
 565         return ret;
 566 }
 567 EXPORT_SYMBOL(rdma_create_qp);
 568
 569 void rdma_destroy_qp(struct rdma_cm_id *id)
 570 {
 571         struct rdma_id_private *id_priv;
 572
 573         id_priv = container_of(id, struct rdma_id_private, id);
 574         mutex_lock(&id_priv->qp_mutex);
 575         ib_destroy_qp(id_priv->id.qp);
 576         id_priv->id.qp = NULL;
 577         mutex_unlock(&id_priv->qp_mutex);
 578 }
 579 EXPORT_SYMBOL(rdma_destroy_qp);
 580
 581 static int cma_modify_qp_rtr(struct rdma_id_private *id_priv,
 582                              struct rdma_conn_param *conn_param)
 583 {
 584         struct ib_qp_attr qp_attr;
 585         int qp_attr_mask, ret;
 586         union ib_gid sgid;
 587
 588         mutex_lock(&id_priv->qp_mutex);
 589         if (!id_priv->id.qp) {
 590                 ret = 0;
 591                 goto out;
 592         }
 593
 594         /* Need to update QP attributes from default values. */
 595         qp_attr.qp_state = IB_QPS_INIT;
 596         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 597         if (ret)
 598                 goto out;
 599
 600         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 601         if (ret)
 602                 goto out;
 603
 604         qp_attr.qp_state = IB_QPS_RTR;
 605         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 606         if (ret)
 607                 goto out;
 608         ret = ib_query_gid(id_priv->id.device, id_priv->id.port_num,
 609                            qp_attr.ah_attr.grh.sgid_index, &sgid);
 610         if (ret)
 611                 goto out;
 612
 613         if (rdma_node_get_transport(id_priv->cma_dev->device->node_type)
 614             == RDMA_TRANSPORT_IB &&
 615             rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num)
 616             == IB_LINK_LAYER_ETHERNET) {
 617                 ret = rdma_addr_find_smac_by_sgid(&sgid, qp_attr.smac, NULL);
 618
 619                 if (ret)
 620                         goto out;
 621         }
 622
 623         if (conn_param)
 624                 qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
 625         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 626 out:
 627         mutex_unlock(&id_priv->qp_mutex);
 628         return ret;
 629 }
 630
 631 static int cma_modify_qp_rts(struct rdma_id_private *id_priv,
 632                              struct rdma_conn_param *conn_param)
 633 {
 634         struct ib_qp_attr qp_attr;
 635         int qp_attr_mask, ret;
 636
 637         mutex_lock(&id_priv->qp_mutex);
 638         if (!id_priv->id.qp) {
 639                 ret = 0;
 640                 goto out;
 641         }
 642
 643         qp_attr.qp_state = IB_QPS_RTS;
 644         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 645         if (ret)
 646                 goto out;
 647
 648         if (conn_param)
 649                 qp_attr.max_rd_atomic = conn_param->initiator_depth;
 650
 651         if (id_priv->qp_timeout && id_priv->id.qp->qp_type == IB_QPT_RC) {
 652                 qp_attr.timeout = id_priv->qp_timeout;
 653                 qp_attr_mask |= IB_QP_TIMEOUT;
 654         }
 655
 656         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 657 out:
 658         mutex_unlock(&id_priv->qp_mutex);
 659         return ret;
 660 }
 661
 662 static int cma_modify_qp_err(struct rdma_id_private *id_priv)
 663 {
 664         struct ib_qp_attr qp_attr;
 665         int ret;
 666
 667         mutex_lock(&id_priv->qp_mutex);
 668         if (!id_priv->id.qp) {
 669                 ret = 0;
 670                 goto out;
 671         }
 672
 673         qp_attr.qp_state = IB_QPS_ERR;
 674         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE);
 675 out:
 676         mutex_unlock(&id_priv->qp_mutex);
 677         return ret;
 678 }
 679
 680 static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
 681                                struct ib_qp_attr *qp_attr, int *qp_attr_mask)
 682 {
 683         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 684         int ret;
 685         u16 pkey;
 686
 687         if (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num) ==
 688             IB_LINK_LAYER_INFINIBAND)
 689                 pkey = ib_addr_get_pkey(dev_addr);
 690         else
 691                 pkey = 0xffff;
 692
 693         ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
 694                                   pkey, &qp_attr->pkey_index);
 695         if (ret)
 696                 return ret;
 697
 698         qp_attr->port_num = id_priv->id.port_num;
 699         *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
 700
 701         if (id_priv->id.qp_type == IB_QPT_UD) {
 702                 ret = cma_set_qkey(id_priv);
 703                 if (ret)
 704                         return ret;
 705
 706                 qp_attr->qkey = id_priv->qkey;
 707                 *qp_attr_mask |= IB_QP_QKEY;
 708         } else {
 709                 qp_attr->qp_access_flags = 0;
 710                 *qp_attr_mask |= IB_QP_ACCESS_FLAGS;
 711         }
 712         return 0;
 713 }
 714
 715 int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
 716                        int *qp_attr_mask)
 717 {
 718         struct rdma_id_private *id_priv;
 719         int ret = 0;
 720
 721         id_priv = container_of(id, struct rdma_id_private, id);
 722         switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
 723         case RDMA_TRANSPORT_IB:
 724                 if (!id_priv->cm_id.ib || (id_priv->id.qp_type == IB_QPT_UD))
 725                         ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask);
 726                 else
 727                         ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
 728                                                  qp_attr_mask);
 729                 if (qp_attr->qp_state == IB_QPS_RTR)
 730                         qp_attr->rq_psn = id_priv->seq_num;
 731                 break;
 732         case RDMA_TRANSPORT_IWARP:
 733         case RDMA_TRANSPORT_SCIF:
 734                 if (!id_priv->cm_id.iw) {
 735                         qp_attr->qp_access_flags = 0;
 736                         *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
 737                 } else
 738                         ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
 739                                                  qp_attr_mask);
 740                 break;
 741         default:
 742                 ret = -ENOSYS;
 743                 break;
 744         }
 745
 746         return ret;
 747 }
 748 EXPORT_SYMBOL(rdma_init_qp_attr);
 749
 750 static inline int cma_zero_addr(struct sockaddr *addr)
 751 {
 752         struct in6_addr *ip6;
 753
 754         if (addr->sa_family == AF_INET)
 755                 return ipv4_is_zeronet(
 756                         ((struct sockaddr_in *)addr)->sin_addr.s_addr);
 757         else {
 758                 ip6 = &((struct sockaddr_in6 *) addr)->sin6_addr;
 759                 return (ip6->s6_addr32[0] | ip6->s6_addr32[1] |
 760                         ip6->s6_addr32[2] | ip6->s6_addr32[3]) == 0;
 761         }
 762 }
 763
 764 static inline int cma_loopback_addr(struct sockaddr *addr)
 765 {
 766         if (addr->sa_family == AF_INET)
 767                 return ipv4_is_loopback(
 768                         ((struct sockaddr_in *) addr)->sin_addr.s_addr);
 769         else
 770                 return ipv6_addr_loopback(
 771                         &((struct sockaddr_in6 *) addr)->sin6_addr);
 772 }
 773
 774 static inline int cma_any_addr(struct sockaddr *addr)
 775 {
 776         return cma_zero_addr(addr) || cma_loopback_addr(addr);
 777 }
 778
 779 static int cma_addr_cmp(struct sockaddr *src, struct sockaddr *dst)
 780 {
 781         if (src->sa_family != dst->sa_family)
 782                 return -1;
 783
 784         switch (src->sa_family) {
 785         case AF_INET:
 786                 return ((struct sockaddr_in *) src)->sin_addr.s_addr !=
 787                        ((struct sockaddr_in *) dst)->sin_addr.s_addr;
 788         default:
 789                 return ipv6_addr_cmp(&((struct sockaddr_in6 *) src)->sin6_addr,
 790                                      &((struct sockaddr_in6 *) dst)->sin6_addr);
 791         }
 792 }
 793
 794 static inline __be16 cma_port(struct sockaddr *addr)
 795 {
 796         if (addr->sa_family == AF_INET)
 797                 return ((struct sockaddr_in *) addr)->sin_port;
 798         else
 799                 return ((struct sockaddr_in6 *) addr)->sin6_port;
 800 }
 801
 802 static inline int cma_any_port(struct sockaddr *addr)
 803 {
 804         return !cma_port(addr);
 805 }
 806
 807 static int cma_get_net_info(void *hdr, enum rdma_port_space ps,
 808                             u8 *ip_ver, __be16 *port,
 809                             union cma_ip_addr **src, union cma_ip_addr **dst)
 810 {
 811         switch (ps) {
 812         case RDMA_PS_SDP:
 813                 if (sdp_get_majv(((struct sdp_hh *) hdr)->sdp_version) !=
 814                     SDP_MAJ_VERSION)
 815                         return -EINVAL;
 816
 817                 *ip_ver = sdp_get_ip_ver(hdr);
 818                 *port   = ((struct sdp_hh *) hdr)->port;
 819                 *src    = &((struct sdp_hh *) hdr)->src_addr;
 820                 *dst    = &((struct sdp_hh *) hdr)->dst_addr;
 821                 break;
 822         default:
 823                 if (((struct cma_hdr *) hdr)->cma_version != CMA_VERSION)
 824                         return -EINVAL;
 825
 826                 *ip_ver = cma_get_ip_ver(hdr);
 827                 *port   = ((struct cma_hdr *) hdr)->port;
 828                 *src    = &((struct cma_hdr *) hdr)->src_addr;
 829                 *dst    = &((struct cma_hdr *) hdr)->dst_addr;
 830                 break;
 831         }
 832
 833         if (*ip_ver != 4 && *ip_ver != 6)
 834                 return -EINVAL;
 835         return 0;
 836 }
 837
 838 static void cma_save_net_info(struct rdma_addr *addr,
 839                               struct rdma_addr *listen_addr,
 840                               u8 ip_ver, __be16 port,
 841                               union cma_ip_addr *src, union cma_ip_addr *dst)
 842 {
 843         struct sockaddr_in *listen4, *ip4;
 844         struct sockaddr_in6 *listen6, *ip6;
 845
 846         switch (ip_ver) {
 847         case 4:
 848                 listen4 = (struct sockaddr_in *) &listen_addr->src_addr;
 849                 ip4 = (struct sockaddr_in *) &addr->src_addr;
 850                 ip4->sin_family = listen4->sin_family;
 851                 ip4->sin_addr.s_addr = dst->ip4.addr;
 852                 ip4->sin_port = listen4->sin_port;
 853
 854                 ip4 = (struct sockaddr_in *) &addr->dst_addr;
 855                 ip4->sin_family = listen4->sin_family;
 856                 ip4->sin_addr.s_addr = src->ip4.addr;
 857                 ip4->sin_port = port;
 858                 break;
 859         case 6:
 860                 listen6 = (struct sockaddr_in6 *) &listen_addr->src_addr;
 861                 ip6 = (struct sockaddr_in6 *) &addr->src_addr;
 862                 ip6->sin6_family = listen6->sin6_family;
 863                 ip6->sin6_addr = dst->ip6;
 864                 ip6->sin6_port = listen6->sin6_port;
 865
 866                 ip6 = (struct sockaddr_in6 *) &addr->dst_addr;
 867                 ip6->sin6_family = listen6->sin6_family;
 868                 ip6->sin6_addr = src->ip6;
 869                 ip6->sin6_port = port;
 870                 break;
 871         default:
 872                 break;
 873         }
 874 }
 875
 876 static inline int cma_user_data_offset(enum rdma_port_space ps)
 877 {
 878         switch (ps) {
 879         case RDMA_PS_SDP:
 880                 return 0;
 881         default:
 882                 return sizeof(struct cma_hdr);
 883         }
 884 }
 885
 886 static void cma_cancel_route(struct rdma_id_private *id_priv)
 887 {
 888         switch (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num)) {
 889         case IB_LINK_LAYER_INFINIBAND:
 890                 if (id_priv->query)
 891                         ib_sa_cancel_query(id_priv->query_id, id_priv->query);
 892                 break;
 893         default:
 894                 break;
 895         }
 896 }
 897
 898 static void cma_cancel_listens(struct rdma_id_private *id_priv)
 899 {
 900         struct rdma_id_private *dev_id_priv;
 901
 902         /*
 903          * Remove from listen_any_list to prevent added devices from spawning
 904          * additional listen requests.
 905          */
 906         mutex_lock(&lock);
 907         list_del(&id_priv->list);
 908
 909         while (!list_empty(&id_priv->listen_list)) {
 910                 dev_id_priv = list_entry(id_priv->listen_list.next,
 911                                          struct rdma_id_private, listen_list);
 912                 /* sync with device removal to avoid duplicate destruction */
 913                 list_del_init(&dev_id_priv->list);
 914                 list_del(&dev_id_priv->listen_list);
 915                 mutex_unlock(&lock);
 916
 917                 rdma_destroy_id(&dev_id_priv->id);
 918                 mutex_lock(&lock);
 919         }
 920         mutex_unlock(&lock);
 921 }
 922
 923 static void cma_cancel_operation(struct rdma_id_private *id_priv,
 924                                  enum rdma_cm_state state)
 925 {
 926         switch (state) {
 927         case RDMA_CM_ADDR_QUERY:
 928                 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr);
 929                 break;
 930         case RDMA_CM_ROUTE_QUERY:
 931                 cma_cancel_route(id_priv);
 932                 break;
 933         case RDMA_CM_LISTEN:
 934                 if (cma_any_addr((struct sockaddr *) &id_priv->id.route.addr.src_addr)
 935                                 && !id_priv->cma_dev)
 936                         cma_cancel_listens(id_priv);
 937                 break;
 938         default:
 939                 break;
 940         }
 941 }
 942
 943 static void cma_release_port(struct rdma_id_private *id_priv)
 944 {
 945         struct rdma_bind_list *bind_list;
 946
 947         mutex_lock(&lock);
 948         bind_list = id_priv->bind_list;
 949         if (!bind_list) {
 950                 mutex_unlock(&lock);
 951                 return;
 952         }
 953         hlist_del(&id_priv->node);
 954         id_priv->bind_list = NULL;
 955         if (hlist_empty(&bind_list->owners)) {
 956                 idr_remove(bind_list->ps, bind_list->port);
 957                 kfree(bind_list);
 958         }
 959         mutex_unlock(&lock);
 960 }
 961
 962 static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
 963 {
 964         struct cma_multicast *mc;
 965
 966         while (!list_empty(&id_priv->mc_list)) {
 967                 mc = container_of(id_priv->mc_list.next,
 968                                   struct cma_multicast, list);
 969                 list_del(&mc->list);
 970                 switch (rdma_port_get_link_layer(id_priv->cma_dev->device, id_priv->id.port_num)) {
 971                 case IB_LINK_LAYER_INFINIBAND:
 972                         ib_sa_free_multicast(mc->multicast.ib);
 973                         kfree(mc);
 974                         break;
 975                 case IB_LINK_LAYER_ETHERNET:
 976                         kref_put(&mc->mcref, release_mc);
 977                         break;
 978                 default:
 979                         break;
 980                 }
 981         }
 982 }
 983 static void __rdma_free(struct work_struct *work)
 984 {
 985         struct rdma_id_private *id_priv;
 986         id_priv = container_of(work, struct rdma_id_private, work);
 987
 988         wait_for_completion(&id_priv->comp);
 989
 990         if (id_priv->internal_id)
 991                 cma_deref_id(id_priv->id.context);
 992
 993         kfree(id_priv->id.route.path_rec);
 994         kfree(id_priv);
 995 }
 996
 997 void rdma_destroy_id(struct rdma_cm_id *id)
 998 {
 999         struct rdma_id_private *id_priv;
1000         enum rdma_cm_state state;
1001         unsigned long flags;
1002         struct ib_cm_id *ib;
1003
1004         id_priv = container_of(id, struct rdma_id_private, id);
1005         state = cma_exch(id_priv, RDMA_CM_DESTROYING);
1006         cma_cancel_operation(id_priv, state);
1007
1008         /*
1009          * Wait for any active callback to finish.  New callbacks will find
1010          * the id_priv state set to destroying and abort.
1011          */
1012         mutex_lock(&id_priv->handler_mutex);
1013         mutex_unlock(&id_priv->handler_mutex);
1014
1015         if (id_priv->cma_dev) {
1016                 switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
1017                 case RDMA_TRANSPORT_IB:
1018                         spin_lock_irqsave(&id_priv->cm_lock, flags);
1019                         if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib)) {
1020                                 ib = id_priv->cm_id.ib;
1021                                 id_priv->cm_id.ib = NULL;
1022                                 spin_unlock_irqrestore(&id_priv->cm_lock, flags);
1023                                 ib_destroy_cm_id(ib);
1024                         } else
1025                                 spin_unlock_irqrestore(&id_priv->cm_lock, flags);
1026                         break;
1027                 case RDMA_TRANSPORT_IWARP:
1028                 case RDMA_TRANSPORT_SCIF:
1029                         if (id_priv->cm_id.iw)
1030                                 iw_destroy_cm_id(id_priv->cm_id.iw);
1031                         break;
1032                 default:
1033                         break;
1034                 }
1035                 cma_leave_mc_groups(id_priv);
1036                 cma_release_dev(id_priv);
1037         }
1038
1039         cma_release_port(id_priv);
1040         cma_deref_id(id_priv);
1041         INIT_WORK(&id_priv->work, __rdma_free);
1042         queue_work(cma_free_wq, &id_priv->work);
1043 }
1044 EXPORT_SYMBOL(rdma_destroy_id);
1045
1046 static int cma_rep_recv(struct rdma_id_private *id_priv)
1047 {
1048         int ret;
1049
1050         ret = cma_modify_qp_rtr(id_priv, NULL);
1051         if (ret)
1052                 goto reject;
1053
1054         ret = cma_modify_qp_rts(id_priv, NULL);
1055         if (ret)
1056                 goto reject;
1057
1058         cma_dbg(id_priv, "sending RTU\n");
1059         ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0);
1060         if (ret)
1061                 goto reject;
1062
1063         return 0;
1064 reject:
1065         cma_modify_qp_err(id_priv);
1066         cma_dbg(id_priv, "sending REJ\n");
1067         ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,
1068                        NULL, 0, NULL, 0);
1069         return ret;
1070 }
1071
1072 static int cma_verify_rep(struct rdma_id_private *id_priv, void *data)
1073 {
1074         if (id_priv->id.ps == RDMA_PS_SDP &&
1075             sdp_get_majv(((struct sdp_hah *) data)->sdp_version) !=
1076             SDP_MAJ_VERSION)
1077                 return -EINVAL;
1078
1079         return 0;
1080 }
1081
1082 static void cma_set_rep_event_data(struct rdma_cm_event *event,
1083                                    struct ib_cm_rep_event_param *rep_data,
1084                                    void *private_data)
1085 {
1086         event->param.conn.private_data = private_data;
1087         event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
1088         event->param.conn.responder_resources = rep_data->responder_resources;
1089         event->param.conn.initiator_depth = rep_data->initiator_depth;
1090         event->param.conn.flow_control = rep_data->flow_control;
1091         event->param.conn.rnr_retry_count = rep_data->rnr_retry_count;
1092         event->param.conn.srq = rep_data->srq;
1093         event->param.conn.qp_num = rep_data->remote_qpn;
1094 }
1095
1096 static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
1097 {
1098         struct rdma_id_private *id_priv = cm_id->context;
1099         struct rdma_cm_event event;
1100         int ret = 0;
1101
1102         if ((ib_event->event != IB_CM_TIMEWAIT_EXIT &&
1103                 cma_disable_callback(id_priv, RDMA_CM_CONNECT)) ||
1104             (ib_event->event == IB_CM_TIMEWAIT_EXIT &&
1105                 cma_disable_callback(id_priv, RDMA_CM_DISCONNECT)))
1106                 return 0;
1107         memset(&event, 0, sizeof event);
1108         switch (ib_event->event) {
1109         case IB_CM_REQ_ERROR:
1110         case IB_CM_REP_ERROR:
1111                 event.event = RDMA_CM_EVENT_UNREACHABLE;
1112                 event.status = -ETIMEDOUT;
1113                 break;
1114         case IB_CM_REP_RECEIVED:
1115                 event.status = cma_verify_rep(id_priv, ib_event->private_data);
1116                 if (event.status)
1117                         event.event = RDMA_CM_EVENT_CONNECT_ERROR;
1118                 else if (id_priv->id.qp && id_priv->id.ps != RDMA_PS_SDP) {
1119                         event.status = cma_rep_recv(id_priv);
1120                         event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR :
1121                                                      RDMA_CM_EVENT_ESTABLISHED;
1122                 } else
1123                         event.event = RDMA_CM_EVENT_CONNECT_RESPONSE;
1124                 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd,
1125                                        ib_event->private_data);
1126                 break;
1127         case IB_CM_RTU_RECEIVED:
1128         case IB_CM_USER_ESTABLISHED:
1129                 event.event = RDMA_CM_EVENT_ESTABLISHED;
1130                 break;
1131         case IB_CM_DREQ_ERROR:
1132                 event.status = -ETIMEDOUT; /* fall through */
1133         case IB_CM_DREQ_RECEIVED:
1134         case IB_CM_DREP_RECEIVED:
1135                 if (!cma_comp_exch(id_priv, RDMA_CM_CONNECT,
1136                                    RDMA_CM_DISCONNECT))
1137                         goto out;
1138                 event.event = RDMA_CM_EVENT_DISCONNECTED;
1139                 break;
1140         case IB_CM_TIMEWAIT_EXIT:
1141                 event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT;
1142                 break;
1143         case IB_CM_MRA_RECEIVED:
1144                 /* ignore event */
1145                 goto out;
1146         case IB_CM_REJ_RECEIVED:
1147                 cma_modify_qp_err(id_priv);
1148                 event.status = ib_event->param.rej_rcvd.reason;
1149                 event.event = RDMA_CM_EVENT_REJECTED;
1150                 event.param.conn.private_data = ib_event->private_data;
1151                 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
1152                 break;
1153         default:
1154                 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
1155                        ib_event->event);
1156                 goto out;
1157         }
1158
1159         ret = id_priv->id.event_handler(&id_priv->id, &event);
1160         if (ret) {
1161                 /* Destroy the CM ID by returning a non-zero value. */
1162                 id_priv->cm_id.ib = NULL;
1163                 cma_exch(id_priv, RDMA_CM_DESTROYING);
1164                 mutex_unlock(&id_priv->handler_mutex);
1165                 rdma_destroy_id(&id_priv->id);
1166                 return ret;
1167         }
1168 out:
1169         mutex_unlock(&id_priv->handler_mutex);
1170         return ret;
1171 }
1172
1173 static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,
1174                                                struct ib_cm_event *ib_event)
1175 {
1176         struct rdma_id_private *id_priv;
1177         struct rdma_cm_id *id;
1178         struct rdma_route *rt;
1179         union cma_ip_addr *src, *dst;
1180         __be16 port;
1181         u8 ip_ver;
1182         int ret;
1183
1184         if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1185                              &ip_ver, &port, &src, &dst))
1186                 return NULL;
1187
1188         id = rdma_create_id(listen_id->event_handler, listen_id->context,
1189                             listen_id->ps, ib_event->param.req_rcvd.qp_type);
1190         if (IS_ERR(id))
1191                 return NULL;
1192
1193         cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1194                           ip_ver, port, src, dst);
1195
1196         rt = &id->route;
1197         rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1;
1198         rt->path_rec = kmalloc(sizeof *rt->path_rec * rt->num_paths,
1199                                GFP_KERNEL);
1200         if (!rt->path_rec)
1201                 goto err;
1202
1203         rt->path_rec[0] = *ib_event->param.req_rcvd.primary_path;
1204         if (rt->num_paths == 2)
1205                 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;
1206
1207         if (cma_any_addr((struct sockaddr *) &rt->addr.src_addr)) {
1208                 rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND;
1209                 rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid);
1210                 ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey));
1211         } else {
1212                 ret = rdma_translate_ip((struct sockaddr *) &rt->addr.src_addr,
1213                                         &rt->addr.dev_addr, NULL);
1214                 if (ret)
1215                         goto err;
1216         }
1217         rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
1218
1219         id_priv = container_of(id, struct rdma_id_private, id);
1220         id_priv->state = RDMA_CM_CONNECT;
1221         return id_priv;
1222
1223 err:
1224         rdma_destroy_id(id);
1225         return NULL;
1226 }
1227
1228 static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id,
1229                                               struct ib_cm_event *ib_event)
1230 {
1231         struct rdma_id_private *id_priv;
1232         struct rdma_cm_id *id;
1233         union cma_ip_addr *src, *dst;
1234         __be16 port;
1235         u8 ip_ver;
1236         int ret;
1237
1238         id = rdma_create_id(listen_id->event_handler, listen_id->context,
1239                             listen_id->ps, IB_QPT_UD);
1240         if (IS_ERR(id))
1241                 return NULL;
1242
1243
1244         if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1245                              &ip_ver, &port, &src, &dst))
1246                 goto err;
1247
1248         cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1249                           ip_ver, port, src, dst);
1250
1251         if (!cma_any_addr((struct sockaddr *) &id->route.addr.src_addr)) {
1252                 ret = rdma_translate_ip((struct sockaddr *) &id->route.addr.src_addr,
1253                                         &id->route.addr.dev_addr, NULL);
1254                 if (ret)
1255                         goto err;
1256         }
1257
1258         id_priv = container_of(id, struct rdma_id_private, id);
1259         id_priv->state = RDMA_CM_CONNECT;
1260         return id_priv;
1261 err:
1262         rdma_destroy_id(id);
1263         return NULL;
1264 }
1265
1266 static void cma_set_req_event_data(struct rdma_cm_event *event,
1267                                    struct ib_cm_req_event_param *req_data,
1268                                    void *private_data, int offset)
1269 {
1270         event->param.conn.private_data = private_data + offset;
1271         event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset;
1272         event->param.conn.responder_resources = req_data->responder_resources;
1273         event->param.conn.initiator_depth = req_data->initiator_depth;
1274         event->param.conn.flow_control = req_data->flow_control;
1275         event->param.conn.retry_count = req_data->retry_count;
1276         event->param.conn.rnr_retry_count = req_data->rnr_retry_count;
1277         event->param.conn.srq = req_data->srq;
1278         event->param.conn.qp_num = req_data->remote_qpn;
1279 }
1280
1281 static int cma_check_req_qp_type(struct rdma_cm_id *id, struct ib_cm_event *ib_event)
1282 {
1283         return (((ib_event->event == IB_CM_REQ_RECEIVED) &&
1284                  (ib_event->param.req_rcvd.qp_type == id->qp_type)) ||
1285                 ((ib_event->event == IB_CM_SIDR_REQ_RECEIVED) &&
1286                  (id->qp_type == IB_QPT_UD)) ||
1287                 (!id->qp_type));
1288 }
1289
1290 static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
1291 {
1292         struct rdma_id_private *listen_id, *conn_id;
1293         struct rdma_cm_event event;
1294         int offset, ret;
1295         u8 smac[ETH_ALEN];
1296         u8 alt_smac[ETH_ALEN];
1297         u8 *psmac = smac;
1298         u8 *palt_smac = alt_smac;
1299         int is_iboe = ((rdma_node_get_transport(cm_id->device->node_type) ==
1300                         RDMA_TRANSPORT_IB) &&
1301                        (rdma_port_get_link_layer(cm_id->device,
1302                         ib_event->param.req_rcvd.port) ==
1303                         IB_LINK_LAYER_ETHERNET));
1304         int is_sidr = 0;
1305
1306         listen_id = cm_id->context;
1307         if (!cma_check_req_qp_type(&listen_id->id, ib_event))
1308                 return -EINVAL;
1309
1310         if (cma_disable_callback(listen_id, RDMA_CM_LISTEN))
1311                 return -ECONNABORTED;
1312
1313         memset(&event, 0, sizeof event);
1314         offset = cma_user_data_offset(listen_id->id.ps);
1315         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1316         if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) {
1317                 is_sidr = 1;
1318                 conn_id = cma_new_udp_id(&listen_id->id, ib_event);
1319                 event.param.ud.private_data = ib_event->private_data + offset;
1320                 event.param.ud.private_data_len =
1321                                 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset;
1322         } else {
1323                 conn_id = cma_new_conn_id(&listen_id->id, ib_event);
1324                 cma_set_req_event_data(&event, &ib_event->param.req_rcvd,
1325                                        ib_event->private_data, offset);
1326         }
1327         if (!conn_id) {
1328                 ret = -ENOMEM;
1329                 goto err1;
1330         }
1331
1332         mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
1333         ret = cma_acquire_dev(conn_id);
1334         if (ret)
1335                 goto err2;
1336
1337         conn_id->cm_id.ib = cm_id;
1338         cm_id->context = conn_id;
1339         cm_id->cm_handler = cma_ib_handler;
1340
1341         /*
1342          * Protect against the user destroying conn_id from another thread
1343          * until we're done accessing it.
1344          */
1345         atomic_inc(&conn_id->refcount);
1346         ret = conn_id->id.event_handler(&conn_id->id, &event);
1347         if (ret)
1348                 goto err3;
1349
1350         if (is_iboe && !is_sidr) {
1351                 if (ib_event->param.req_rcvd.primary_path != NULL)
1352                         rdma_addr_find_smac_by_sgid(
1353                                 &ib_event->param.req_rcvd.primary_path->sgid,
1354                                 psmac, NULL);
1355                 else
1356                         psmac = NULL;
1357                 if (ib_event->param.req_rcvd.alternate_path != NULL)
1358                         rdma_addr_find_smac_by_sgid(
1359                                 &ib_event->param.req_rcvd.alternate_path->sgid,
1360                                 palt_smac, NULL);
1361                 else
1362                         palt_smac = NULL;
1363         }
1364                 /*
1365                  * Acquire mutex to prevent user executing rdma_destroy_id()
1366                  * while we're accessing the cm_id.
1367                  */
1368                 mutex_lock(&lock);
1369         if (is_iboe && !is_sidr)
1370                 ib_update_cm_av(cm_id, psmac, palt_smac);
1371         if (cma_comp(conn_id, RDMA_CM_CONNECT) && (conn_id->id.qp_type != IB_QPT_UD)) {
1372                 cma_dbg(container_of(&conn_id->id, struct rdma_id_private, id), "sending MRA\n");
1373                         ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
1374         }
1375                 mutex_unlock(&lock);
1376                 mutex_unlock(&conn_id->handler_mutex);
1377         mutex_unlock(&listen_id->handler_mutex);
1378         cma_deref_id(conn_id);
1379         return 0;
1380
1381 err3:
1382         cma_deref_id(conn_id);
1383         /* Destroy the CM ID by returning a non-zero value. */
1384         conn_id->cm_id.ib = NULL;
1385 err2:
1386         cma_exch(conn_id, RDMA_CM_DESTROYING);
1387         mutex_unlock(&conn_id->handler_mutex);
1388 err1:
1389         mutex_unlock(&listen_id->handler_mutex);
1390         if (conn_id)
1391                 rdma_destroy_id(&conn_id->id);
1392         return ret;
1393 }
1394
1395 static __be64 cma_get_service_id(enum rdma_port_space ps, struct sockaddr *addr)
1396 {
1397         return cpu_to_be64(((u64)ps << 16) + be16_to_cpu(cma_port(addr)));
1398 }
1399
1400 static void cma_set_compare_data(enum rdma_port_space ps, struct sockaddr *addr,
1401                                  struct ib_cm_compare_data *compare)
1402 {
1403         struct cma_hdr *cma_data, *cma_mask;
1404         struct sdp_hh *sdp_data, *sdp_mask;
1405         __be32 ip4_addr;
1406         struct in6_addr ip6_addr;
1407
1408         memset(compare, 0, sizeof *compare);
1409         cma_data = (void *) compare->data;
1410         cma_mask = (void *) compare->mask;
1411         sdp_data = (void *) compare->data;
1412         sdp_mask = (void *) compare->mask;
1413
1414         switch (addr->sa_family) {
1415         case AF_INET:
1416                 ip4_addr = ((struct sockaddr_in *) addr)->sin_addr.s_addr;
1417                 if (ps == RDMA_PS_SDP) {
1418                         sdp_set_ip_ver(sdp_data, 4);
1419                         sdp_set_ip_ver(sdp_mask, 0xF);
1420                         if (!cma_any_addr(addr)) {
1421                                 sdp_data->dst_addr.ip4.addr = ip4_addr;
1422                                 sdp_mask->dst_addr.ip4.addr = htonl(~0);
1423                         }
1424                 } else {
1425                         cma_set_ip_ver(cma_data, 4);
1426                         cma_set_ip_ver(cma_mask, 0xF);
1427                         if (!cma_any_addr(addr)) {
1428                                 cma_data->dst_addr.ip4.addr = ip4_addr;
1429                                 cma_mask->dst_addr.ip4.addr = htonl(~0);
1430                         }
1431                 }
1432                 break;
1433         case AF_INET6:
1434                 ip6_addr = ((struct sockaddr_in6 *) addr)->sin6_addr;
1435                 if (ps == RDMA_PS_SDP) {
1436                         sdp_set_ip_ver(sdp_data, 6);
1437                         sdp_set_ip_ver(sdp_mask, 0xF);
1438                         if (!cma_any_addr(addr)) {
1439                                 sdp_data->dst_addr.ip6 = ip6_addr;
1440                                 memset(&sdp_mask->dst_addr.ip6, 0xFF,
1441                                        sizeof(sdp_mask->dst_addr.ip6));
1442                         }
1443                 } else {
1444                         cma_set_ip_ver(cma_data, 6);
1445                         cma_set_ip_ver(cma_mask, 0xF);
1446                         if (!cma_any_addr(addr)) {
1447                                 cma_data->dst_addr.ip6 = ip6_addr;
1448                                 memset(&cma_mask->dst_addr.ip6, 0xFF,
1449                                        sizeof(cma_mask->dst_addr.ip6));
1450                         }
1451                 }
1452                 break;
1453         default:
1454                 break;
1455         }
1456 }
1457
1458 static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
1459 {
1460         struct rdma_id_private *id_priv = iw_id->context;
1461         struct rdma_cm_event event;
1462         struct sockaddr_in *sin;
1463         int ret = 0;
1464
1465         if (cma_disable_callback(id_priv, RDMA_CM_CONNECT))
1466                 return 0;
1467
1468         memset(&event, 0, sizeof event);
1469         switch (iw_event->event) {
1470         case IW_CM_EVENT_CLOSE:
1471                 event.event = RDMA_CM_EVENT_DISCONNECTED;
1472                 break;
1473         case IW_CM_EVENT_CONNECT_REPLY:
1474                 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1475                 *sin = iw_event->local_addr;
1476                 sin = (struct sockaddr_in *) &id_priv->id.route.addr.dst_addr;
1477                 *sin = iw_event->remote_addr;
1478                 switch ((int)iw_event->status) {
1479                 case 0:
1480                         event.event = RDMA_CM_EVENT_ESTABLISHED;
1481                         event.param.conn.initiator_depth = iw_event->ird;
1482                         event.param.conn.responder_resources = iw_event->ord;
1483                         break;
1484                 case -ECONNRESET:
1485                 case -ECONNREFUSED:
1486                         event.event = RDMA_CM_EVENT_REJECTED;
1487                         break;
1488                 case -ETIMEDOUT:
1489                         event.event = RDMA_CM_EVENT_UNREACHABLE;
1490                         break;
1491                 default:
1492                         event.event = RDMA_CM_EVENT_CONNECT_ERROR;
1493                         break;
1494                 }
1495                 break;
1496         case IW_CM_EVENT_ESTABLISHED:
1497                 event.event = RDMA_CM_EVENT_ESTABLISHED;
1498                 event.param.conn.initiator_depth = iw_event->ird;
1499                 event.param.conn.responder_resources = iw_event->ord;
1500                 break;
1501         default:
1502                 BUG_ON(1);
1503         }
1504
1505         event.status = iw_event->status;
1506         event.param.conn.private_data = iw_event->private_data;
1507         event.param.conn.private_data_len = iw_event->private_data_len;
1508         ret = id_priv->id.event_handler(&id_priv->id, &event);
1509         if (ret) {
1510                 /* Destroy the CM ID by returning a non-zero value. */
1511                 id_priv->cm_id.iw = NULL;
1512                 cma_exch(id_priv, RDMA_CM_DESTROYING);
1513                 mutex_unlock(&id_priv->handler_mutex);
1514                 rdma_destroy_id(&id_priv->id);
1515                 return ret;
1516         }
1517
1518         mutex_unlock(&id_priv->handler_mutex);
1519         return ret;
1520 }
1521
1522 static int iw_conn_req_handler(struct iw_cm_id *cm_id,
1523                                struct iw_cm_event *iw_event)
1524 {
1525         struct rdma_cm_id *new_cm_id;
1526         struct rdma_id_private *listen_id, *conn_id;
1527         struct sockaddr_in *sin;
1528         struct net_device *dev = NULL;
1529         struct rdma_cm_event event;
1530         int ret;
1531         struct ib_device_attr attr;
1532
1533         listen_id = cm_id->context;
1534         if (cma_disable_callback(listen_id, RDMA_CM_LISTEN))
1535                 return -ECONNABORTED;
1536
1537         /* Create a new RDMA id for the new IW CM ID */
1538         new_cm_id = rdma_create_id(listen_id->id.event_handler,
1539                                    listen_id->id.context,
1540                                    RDMA_PS_TCP, IB_QPT_RC);
1541         if (IS_ERR(new_cm_id)) {
1542                 ret = -ENOMEM;
1543                 goto out;
1544         }
1545         conn_id = container_of(new_cm_id, struct rdma_id_private, id);
1546         mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
1547         conn_id->state = RDMA_CM_CONNECT;
1548
1549         dev = ip_dev_find(&init_net, iw_event->local_addr.sin_addr.s_addr);
1550         if (!dev) {
1551                 ret = -EADDRNOTAVAIL;
1552                 mutex_unlock(&conn_id->handler_mutex);
1553                 rdma_destroy_id(new_cm_id);
1554                 goto out;
1555         }
1556         ret = rdma_copy_addr(&conn_id->id.route.addr.dev_addr, dev, NULL);
1557         if (ret) {
1558                 mutex_unlock(&conn_id->handler_mutex);
1559                 rdma_destroy_id(new_cm_id);
1560                 goto out;
1561         }
1562
1563         ret = cma_acquire_dev(conn_id);
1564         if (ret) {
1565                 mutex_unlock(&conn_id->handler_mutex);
1566                 rdma_destroy_id(new_cm_id);
1567                 goto out;
1568         }
1569
1570         conn_id->cm_id.iw = cm_id;
1571         cm_id->context = conn_id;
1572         cm_id->cm_handler = cma_iw_handler;
1573
1574         sin = (struct sockaddr_in *) &new_cm_id->route.addr.src_addr;
1575         *sin = iw_event->local_addr;
1576         sin = (struct sockaddr_in *) &new_cm_id->route.addr.dst_addr;
1577         *sin = iw_event->remote_addr;
1578
1579         ret = ib_query_device(conn_id->id.device, &attr);
1580         if (ret) {
1581                 mutex_unlock(&conn_id->handler_mutex);
1582                 rdma_destroy_id(new_cm_id);
1583                 goto out;
1584         }
1585
1586         memset(&event, 0, sizeof event);
1587         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1588         event.param.conn.private_data = iw_event->private_data;
1589         event.param.conn.private_data_len = iw_event->private_data_len;
1590         event.param.conn.initiator_depth = iw_event->ird;
1591         event.param.conn.responder_resources = iw_event->ord;
1592
1593         /*
1594          * Protect against the user destroying conn_id from another thread
1595          * until we're done accessing it.
1596          */
1597         atomic_inc(&conn_id->refcount);
1598         ret = conn_id->id.event_handler(&conn_id->id, &event);
1599         if (ret) {
1600                 /* User wants to destroy the CM ID */
1601                 conn_id->cm_id.iw = NULL;
1602                 cma_exch(conn_id, RDMA_CM_DESTROYING);
1603                 mutex_unlock(&conn_id->handler_mutex);
1604                 cma_deref_id(conn_id);
1605                 rdma_destroy_id(&conn_id->id);
1606                 goto out;
1607         }
1608
1609         mutex_unlock(&conn_id->handler_mutex);
1610         cma_deref_id(conn_id);
1611
1612 out:
1613         if (dev)
1614                 dev_put(dev);
1615         mutex_unlock(&listen_id->handler_mutex);
1616         return ret;
1617 }
1618
1619 static int cma_ib_listen(struct rdma_id_private *id_priv)
1620 {
1621         struct ib_cm_compare_data compare_data;
1622         struct sockaddr *addr;
1623         struct ib_cm_id *id;
1624         __be64 svc_id;
1625         int ret;
1626
1627         id = ib_create_cm_id(id_priv->id.device, cma_req_handler, id_priv);
1628         if (IS_ERR(id))
1629                 return PTR_ERR(id);
1630
1631         id_priv->cm_id.ib = id;
1632
1633         addr = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
1634         svc_id = cma_get_service_id(id_priv->id.ps, addr);
1635         if (cma_any_addr(addr) && !id_priv->afonly)
1636                 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, NULL);
1637         else {
1638                 cma_set_compare_data(id_priv->id.ps, addr, &compare_data);
1639                 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, &compare_data);
1640         }
1641
1642         if (ret) {
1643                 ib_destroy_cm_id(id_priv->cm_id.ib);
1644                 id_priv->cm_id.ib = NULL;
1645         }
1646
1647         return ret;
1648 }
1649
1650 static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
1651 {
1652         int ret;
1653         struct sockaddr_in *sin;
1654         struct iw_cm_id *id;
1655
1656         id = iw_create_cm_id(id_priv->id.device,
1657                                             id_priv->sock,
1658                                             iw_conn_req_handler,
1659                                             id_priv);
1660         if (IS_ERR(id))
1661                 return PTR_ERR(id);
1662
1663         id_priv->cm_id.iw = id;
1664
1665         sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1666         id_priv->cm_id.iw->local_addr = *sin;
1667
1668         ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
1669
1670         if (ret) {
1671                 iw_destroy_cm_id(id_priv->cm_id.iw);
1672                 id_priv->cm_id.iw = NULL;
1673         }
1674
1675         return ret;
1676 }
1677
1678 static int cma_listen_handler(struct rdma_cm_id *id,
1679                               struct rdma_cm_event *event)
1680 {
1681         struct rdma_id_private *id_priv = id->context;
1682
1683         id->context = id_priv->id.context;
1684         id->event_handler = id_priv->id.event_handler;
1685         return id_priv->id.event_handler(id, event);
1686 }
1687
1688 static void cma_listen_on_dev(struct rdma_id_private *id_priv,
1689                               struct cma_device *cma_dev)
1690 {
1691         struct rdma_id_private *dev_id_priv;
1692         struct rdma_cm_id *id;
1693         int ret;
1694
1695         id = rdma_create_id(cma_listen_handler, id_priv, id_priv->id.ps,
1696                             id_priv->id.qp_type);
1697         if (IS_ERR(id))
1698                 return;
1699
1700         dev_id_priv = container_of(id, struct rdma_id_private, id);
1701
1702         dev_id_priv->state = RDMA_CM_ADDR_BOUND;
1703         memcpy(&id->route.addr.src_addr, &id_priv->id.route.addr.src_addr,
1704                ip_addr_size((struct sockaddr *) &id_priv->id.route.addr.src_addr));
1705
1706         cma_attach_to_dev(dev_id_priv, cma_dev);
1707         list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
1708         atomic_inc(&id_priv->refcount);
1709         dev_id_priv->internal_id = 1;
1710         dev_id_priv->afonly = id_priv->afonly;
1711
1712         ret = rdma_listen(id, id_priv->backlog);
1713         if (ret)
1714                 cma_warn(id_priv, "cma_listen_on_dev, error %d, listening on device %s\n", ret, cma_dev->device->name);
1715 }
1716
1717 static void cma_listen_on_all(struct rdma_id_private *id_priv)
1718 {
1719         struct cma_device *cma_dev;
1720
1721         mutex_lock(&lock);
1722         list_add_tail(&id_priv->list, &listen_any_list);
1723         list_for_each_entry(cma_dev, &dev_list, list)
1724                 cma_listen_on_dev(id_priv, cma_dev);
1725         mutex_unlock(&lock);
1726 }
1727
1728 void rdma_set_service_type(struct rdma_cm_id *id, int tos)
1729 {
1730         struct rdma_id_private *id_priv;
1731
1732         id_priv = container_of(id, struct rdma_id_private, id);
1733         id_priv->tos = (u8) tos;
1734 }
1735 EXPORT_SYMBOL(rdma_set_service_type);
1736
1737 void rdma_set_timeout(struct rdma_cm_id *id, int timeout)
1738 {
1739         struct rdma_id_private *id_priv;
1740
1741         id_priv = container_of(id, struct rdma_id_private, id);
1742         id_priv->qp_timeout = (u8) timeout;
1743 }
1744 EXPORT_SYMBOL(rdma_set_timeout);
1745
1746 static void cma_query_handler(int status, struct ib_sa_path_rec *path_rec,
1747                               void *context)
1748 {
1749         struct cma_work *work = context;
1750         struct rdma_route *route;
1751
1752         route = &work->id->id.route;
1753
1754         if (!status) {
1755                 route->num_paths = 1;
1756                 *route->path_rec = *path_rec;
1757         } else {
1758                 work->old_state = RDMA_CM_ROUTE_QUERY;
1759                 work->new_state = RDMA_CM_ADDR_RESOLVED;
1760                 work->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
1761                 work->event.status = status;
1762         }
1763
1764         queue_work(cma_wq, &work->work);
1765 }
1766
1767 static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms,
1768                               struct cma_work *work)
1769 {
1770         struct rdma_addr *addr = &id_priv->id.route.addr;
1771         struct ib_sa_path_rec path_rec;
1772         ib_sa_comp_mask comp_mask;
1773         struct sockaddr_in6 *sin6;
1774
1775         memset(&path_rec, 0, sizeof path_rec);
1776         rdma_addr_get_sgid(&addr->dev_addr, &path_rec.sgid);
1777         rdma_addr_get_dgid(&addr->dev_addr, &path_rec.dgid);
1778         path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(&addr->dev_addr));
1779         path_rec.numb_path = 1;
1780         path_rec.reversible = 1;
1781         path_rec.service_id = cma_get_service_id(id_priv->id.ps,
1782                                                         (struct sockaddr *) &addr->dst_addr);
1783
1784         comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
1785                     IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH |
1786                     IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID;
1787
1788         if (addr->src_addr.ss_family == AF_INET) {
1789                 path_rec.qos_class = cpu_to_be16((u16) id_priv->tos);
1790                 comp_mask |= IB_SA_PATH_REC_QOS_CLASS;
1791         } else {
1792                 sin6 = (struct sockaddr_in6 *) &addr->src_addr;
1793                 path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20);
1794                 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
1795         }
1796
1797         id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
1798                                                id_priv->id.port_num, &path_rec,
1799                                                comp_mask, timeout_ms,
1800                                                GFP_KERNEL, cma_query_handler,
1801                                                work, &id_priv->query);
1802
1803         return (id_priv->query_id < 0) ? id_priv->query_id : 0;
1804 }
1805
1806 static void cma_work_handler(struct work_struct *_work)
1807 {
1808         struct cma_work *work = container_of(_work, struct cma_work, work);
1809         struct rdma_id_private *id_priv = work->id;
1810         int destroy = 0;
1811
1812         mutex_lock(&id_priv->handler_mutex);
1813         if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
1814                 goto out;
1815
1816         if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
1817                 cma_exch(id_priv, RDMA_CM_DESTROYING);
1818                 destroy = 1;
1819         }
1820 out:
1821         mutex_unlock(&id_priv->handler_mutex);
1822         cma_deref_id(id_priv);
1823         if (destroy)
1824                 rdma_destroy_id(&id_priv->id);
1825         kfree(work);
1826 }
1827
1828 static void cma_ndev_work_handler(struct work_struct *_work)
1829 {
1830         struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work);
1831         struct rdma_id_private *id_priv = work->id;
1832         int destroy = 0;
1833
1834         mutex_lock(&id_priv->handler_mutex);
1835         if (id_priv->state == RDMA_CM_DESTROYING ||
1836             id_priv->state == RDMA_CM_DEVICE_REMOVAL)
1837                 goto out;
1838
1839         if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
1840                 cma_exch(id_priv, RDMA_CM_DESTROYING);
1841                 destroy = 1;
1842         }
1843
1844 out:
1845         mutex_unlock(&id_priv->handler_mutex);
1846         cma_deref_id(id_priv);
1847         if (destroy)
1848                 rdma_destroy_id(&id_priv->id);
1849         kfree(work);
1850 }
1851
1852 static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms)
1853 {
1854         struct rdma_route *route = &id_priv->id.route;
1855         struct cma_work *work;
1856         int ret;
1857
1858         work = kzalloc(sizeof *work, GFP_KERNEL);
1859         if (!work)
1860                 return -ENOMEM;
1861
1862         work->id = id_priv;
1863         INIT_WORK(&work->work, cma_work_handler);
1864         work->old_state = RDMA_CM_ROUTE_QUERY;
1865         work->new_state = RDMA_CM_ROUTE_RESOLVED;
1866         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1867
1868         route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL);
1869         if (!route->path_rec) {
1870                 ret = -ENOMEM;
1871                 goto err1;
1872         }
1873
1874         ret = cma_query_ib_route(id_priv, timeout_ms, work);
1875         if (ret)
1876                 goto err2;
1877
1878         return 0;
1879 err2:
1880         kfree(route->path_rec);
1881         route->path_rec = NULL;
1882 err1:
1883         kfree(work);
1884         return ret;
1885 }
1886
1887 int rdma_set_ib_paths(struct rdma_cm_id *id,
1888                       struct ib_sa_path_rec *path_rec, int num_paths)
1889 {
1890         struct rdma_id_private *id_priv;
1891         int ret;
1892
1893         id_priv = container_of(id, struct rdma_id_private, id);
1894         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
1895                            RDMA_CM_ROUTE_RESOLVED))
1896                 return -EINVAL;
1897
1898         id->route.path_rec = kmemdup(path_rec, sizeof *path_rec * num_paths,
1899                                      GFP_KERNEL);
1900         if (!id->route.path_rec) {
1901                 ret = -ENOMEM;
1902                 goto err;
1903         }
1904
1905         id->route.num_paths = num_paths;
1906         return 0;
1907 err:
1908         cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_ADDR_RESOLVED);
1909         return ret;
1910 }
1911 EXPORT_SYMBOL(rdma_set_ib_paths);
1912
1913 static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms)
1914 {
1915         struct cma_work *work;
1916
1917         work = kzalloc(sizeof *work, GFP_KERNEL);
1918         if (!work)
1919                 return -ENOMEM;
1920
1921         work->id = id_priv;
1922         INIT_WORK(&work->work, cma_work_handler);
1923         work->old_state = RDMA_CM_ROUTE_QUERY;
1924         work->new_state = RDMA_CM_ROUTE_RESOLVED;
1925         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1926         queue_work(cma_wq, &work->work);
1927         return 0;
1928 }
1929
1930 static u8 tos_to_sl(u8 tos)
1931 {
1932         return def_prec2sl & 7;
1933 }
1934
1935 static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
1936 {
1937         struct rdma_route *route = &id_priv->id.route;
1938         struct rdma_addr *addr = &route->addr;
1939         struct cma_work *work;
1940         int ret;
1941         struct sockaddr_in *src_addr = (struct sockaddr_in *)&route->addr.src_addr;
1942         struct sockaddr_in *dst_addr = (struct sockaddr_in *)&route->addr.dst_addr;
1943         struct net_device *ndev = NULL;
1944
1945
1946         if (src_addr->sin_family != dst_addr->sin_family)
1947                 return -EINVAL;
1948
1949         work = kzalloc(sizeof *work, GFP_KERNEL);
1950         if (!work)
1951                 return -ENOMEM;
1952
1953         work->id = id_priv;
1954         INIT_WORK(&work->work, cma_work_handler);
1955
1956         route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL);
1957         if (!route->path_rec) {
1958                 ret = -ENOMEM;
1959                 goto err1;
1960         }
1961
1962         route->num_paths = 1;
1963
1964         if (addr->dev_addr.bound_dev_if)
1965                 ndev = dev_get_by_index(&init_net, addr->dev_addr.bound_dev_if);
1966         if (!ndev) {
1967                 ret = -ENODEV;
1968                 goto err2;
1969         }
1970
1971         route->path_rec->vlan_id = rdma_vlan_dev_vlan_id(ndev);
1972         memcpy(route->path_rec->dmac, addr->dev_addr.dst_dev_addr, ETH_ALEN);
1973         memcpy(route->path_rec->smac, IF_LLADDR(ndev), ndev->if_addrlen);
1974
1975
1976         rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
1977                     &route->path_rec->sgid);
1978         rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.dst_addr,
1979                     &route->path_rec->dgid);
1980
1981         route->path_rec->hop_limit = 1;
1982         route->path_rec->reversible = 1;
1983         route->path_rec->pkey = cpu_to_be16(0xffff);
1984         route->path_rec->mtu_selector = IB_SA_EQ;
1985         route->path_rec->sl = tos_to_sl(id_priv->tos);
1986
1987         route->path_rec->mtu = iboe_get_mtu(ndev->if_mtu);
1988         route->path_rec->rate_selector = IB_SA_EQ;
1989         route->path_rec->rate = iboe_get_rate(ndev);
1990         dev_put(ndev);
1991         route->path_rec->packet_life_time_selector = IB_SA_EQ;
1992         route->path_rec->packet_life_time = CMA_IBOE_PACKET_LIFETIME;
1993         if (!route->path_rec->mtu) {
1994                 ret = -EINVAL;
1995                 goto err2;
1996         }
1997
1998         work->old_state = RDMA_CM_ROUTE_QUERY;
1999         work->new_state = RDMA_CM_ROUTE_RESOLVED;
2000         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
2001         work->event.status = 0;
2002
2003         queue_work(cma_wq, &work->work);
2004
2005         return 0;
2006
2007 err2:
2008         kfree(route->path_rec);
2009         route->path_rec = NULL;
2010 err1:
2011         kfree(work);
2012         return ret;
2013 }
2014
2015 int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
2016 {
2017         struct rdma_id_private *id_priv;
2018         int ret;
2019
2020         id_priv = container_of(id, struct rdma_id_private, id);
2021         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, RDMA_CM_ROUTE_QUERY))
2022                 return -EINVAL;
2023
2024         atomic_inc(&id_priv->refcount);
2025         switch (rdma_node_get_transport(id->device->node_type)) {
2026         case RDMA_TRANSPORT_IB:
2027                 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
2028                 case IB_LINK_LAYER_INFINIBAND:
2029                         ret = cma_resolve_ib_route(id_priv, timeout_ms);
2030                         break;
2031                 case IB_LINK_LAYER_ETHERNET:
2032                         ret = cma_resolve_iboe_route(id_priv);
2033                         break;
2034                 default:
2035                         ret = -ENOSYS;
2036                 }
2037                 break;
2038         case RDMA_TRANSPORT_IWARP:
2039         case RDMA_TRANSPORT_SCIF:
2040                 ret = cma_resolve_iw_route(id_priv, timeout_ms);
2041                 break;
2042         default:
2043                 ret = -ENOSYS;
2044                 break;
2045         }
2046         if (ret)
2047                 goto err;
2048
2049         return 0;
2050 err:
2051         cma_comp_exch(id_priv, RDMA_CM_ROUTE_QUERY, RDMA_CM_ADDR_RESOLVED);
2052         cma_deref_id(id_priv);
2053         return ret;
2054 }
2055 EXPORT_SYMBOL(rdma_resolve_route);
2056
2057 int rdma_enable_apm(struct rdma_cm_id *id, enum alt_path_type alt_type)
2058 {
2059         /* APM is not supported yet */
2060         return -EINVAL;
2061 }
2062 EXPORT_SYMBOL(rdma_enable_apm);
2063
2064 static int cma_bind_loopback(struct rdma_id_private *id_priv)
2065 {
2066         struct cma_device *cma_dev;
2067         struct ib_port_attr port_attr;
2068         union ib_gid gid;
2069         u16 pkey;
2070         int ret;
2071         u8 p;
2072
2073         mutex_lock(&lock);
2074         if (list_empty(&dev_list)) {
2075                 ret = -ENODEV;
2076                 goto out;
2077         }
2078         list_for_each_entry(cma_dev, &dev_list, list)
2079                 for (p = 1; p <= cma_dev->device->phys_port_cnt; ++p)
2080                         if (!ib_query_port(cma_dev->device, p, &port_attr) &&
2081                             port_attr.state == IB_PORT_ACTIVE)
2082                                 goto port_found;
2083
2084         p = 1;
2085         cma_dev = list_entry(dev_list.next, struct cma_device, list);
2086
2087 port_found:
2088         ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid);
2089         if (ret)
2090                 goto out;
2091
2092         ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);
2093         if (ret)
2094                 goto out;
2095
2096         id_priv->id.route.addr.dev_addr.dev_type =
2097                 (rdma_port_get_link_layer(cma_dev->device, p) == IB_LINK_LAYER_INFINIBAND) ?
2098                 ARPHRD_INFINIBAND : ARPHRD_ETHER;
2099
2100         rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
2101         ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey);
2102         id_priv->id.port_num = p;
2103         cma_attach_to_dev(id_priv, cma_dev);
2104 out:
2105         mutex_unlock(&lock);
2106         return ret;
2107 }
2108
2109 static void addr_handler(int status, struct sockaddr *src_addr,
2110                          struct rdma_dev_addr *dev_addr, void *context)
2111 {
2112         struct rdma_id_private *id_priv = context;
2113         struct rdma_cm_event event;
2114
2115         memset(&event, 0, sizeof event);
2116         mutex_lock(&id_priv->handler_mutex);
2117         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY,
2118                            RDMA_CM_ADDR_RESOLVED))
2119                 goto out;
2120
2121         memcpy(&id_priv->id.route.addr.src_addr, src_addr,
2122                ip_addr_size(src_addr));
2123         if (!status && !id_priv->cma_dev)
2124                 status = cma_acquire_dev(id_priv);
2125
2126         if (status) {
2127                 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
2128                                    RDMA_CM_ADDR_BOUND))
2129                         goto out;
2130                 event.event = RDMA_CM_EVENT_ADDR_ERROR;
2131                 event.status = status;
2132         } else
2133                 event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
2134
2135         if (id_priv->id.event_handler(&id_priv->id, &event)) {
2136                 cma_exch(id_priv, RDMA_CM_DESTROYING);
2137                 mutex_unlock(&id_priv->handler_mutex);
2138                 cma_deref_id(id_priv);
2139                 rdma_destroy_id(&id_priv->id);
2140                 return;
2141         }
2142 out:
2143         mutex_unlock(&id_priv->handler_mutex);
2144         cma_deref_id(id_priv);
2145 }
2146
2147 static int cma_resolve_loopback(struct rdma_id_private *id_priv)
2148 {
2149         struct cma_work *work;
2150         struct sockaddr *src, *dst;
2151         union ib_gid gid;
2152         int ret;
2153
2154         work = kzalloc(sizeof *work, GFP_KERNEL);
2155         if (!work)
2156                 return -ENOMEM;
2157
2158         if (!id_priv->cma_dev) {
2159                 ret = cma_bind_loopback(id_priv);
2160                 if (ret)
2161                         goto err;
2162         }
2163
2164         rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
2165         rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
2166
2167         src = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
2168         if (cma_zero_addr(src)) {
2169                 dst = (struct sockaddr *) &id_priv->id.route.addr.dst_addr;
2170                 if ((src->sa_family = dst->sa_family) == AF_INET) {
2171                         ((struct sockaddr_in *)src)->sin_addr =
2172                                 ((struct sockaddr_in *)dst)->sin_addr;
2173                 } else {
2174                         ((struct sockaddr_in6 *)src)->sin6_addr =
2175                                 ((struct sockaddr_in6 *)dst)->sin6_addr;
2176                 }
2177         }
2178
2179         work->id = id_priv;
2180         INIT_WORK(&work->work, cma_work_handler);
2181         work->old_state = RDMA_CM_ADDR_QUERY;
2182         work->new_state = RDMA_CM_ADDR_RESOLVED;
2183         work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
2184         queue_work(cma_wq, &work->work);
2185         return 0;
2186 err:
2187         kfree(work);
2188         return ret;
2189 }
2190
2191 static int cma_resolve_scif(struct rdma_id_private *id_priv)
2192 {
2193         struct cma_work *work;
2194
2195         work = kzalloc(sizeof *work, GFP_KERNEL);
2196         if (!work)
2197                 return -ENOMEM;
2198
2199         /* we probably can leave it empty here */
2200
2201         work->id = id_priv;
2202         INIT_WORK(&work->work, cma_work_handler);
2203         work->old_state = RDMA_CM_ADDR_QUERY;
2204         work->new_state = RDMA_CM_ADDR_RESOLVED;
2205         work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
2206         queue_work(cma_wq, &work->work);
2207         return 0;
2208 }
2209
2210 static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
2211                          struct sockaddr *dst_addr)
2212 {
2213         if (!src_addr || !src_addr->sa_family) {
2214                 src_addr = (struct sockaddr *) &id->route.addr.src_addr;
2215                 if ((src_addr->sa_family = dst_addr->sa_family) == AF_INET6) {
2216                         ((struct sockaddr_in6 *) src_addr)->sin6_scope_id =
2217                                 ((struct sockaddr_in6 *) dst_addr)->sin6_scope_id;
2218                 }
2219         }
2220         if (!cma_any_addr(src_addr))
2221                 return rdma_bind_addr(id, src_addr);
2222         else {
2223                 struct sockaddr_in addr_in;
2224
2225                 memset(&addr_in, 0, sizeof addr_in);
2226                 addr_in.sin_family = dst_addr->sa_family;
2227                 addr_in.sin_len = sizeof addr_in;
2228                 return rdma_bind_addr(id, (struct sockaddr *) &addr_in);
2229         }
2230
2231 }
2232
2233 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
2234                       struct sockaddr *dst_addr, int timeout_ms)
2235 {
2236         struct rdma_id_private *id_priv;
2237         int ret;
2238
2239         id_priv = container_of(id, struct rdma_id_private, id);
2240         if (id_priv->state == RDMA_CM_IDLE) {
2241                 ret = cma_bind_addr(id, src_addr, dst_addr);
2242                 if (ret)
2243                         return ret;
2244         }
2245
2246         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY))
2247                 return -EINVAL;
2248
2249         atomic_inc(&id_priv->refcount);
2250         memcpy(&id->route.addr.dst_addr, dst_addr, ip_addr_size(dst_addr));
2251         if (cma_any_addr(dst_addr))
2252                 ret = cma_resolve_loopback(id_priv);
2253         else if (id_priv->id.device &&
2254                 rdma_node_get_transport(id_priv->id.device->node_type) == RDMA_TRANSPORT_SCIF)
2255                 ret = cma_resolve_scif(id_priv);
2256         else
2257                 ret = rdma_resolve_ip(&addr_client, (struct sockaddr *) &id->route.addr.src_addr,
2258                                       dst_addr, &id->route.addr.dev_addr,
2259                                       timeout_ms, addr_handler, id_priv);
2260         if (ret)
2261                 goto err;
2262
2263         return 0;
2264 err:
2265         cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
2266         cma_deref_id(id_priv);
2267         return ret;
2268 }
2269 EXPORT_SYMBOL(rdma_resolve_addr);
2270
2271 int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse)
2272 {
2273         struct rdma_id_private *id_priv;
2274         unsigned long flags;
2275         int ret;
2276
2277         id_priv = container_of(id, struct rdma_id_private, id);
2278         spin_lock_irqsave(&id_priv->lock, flags);
2279         if (id_priv->state == RDMA_CM_IDLE) {
2280                 id_priv->reuseaddr = reuse;
2281                 ret = 0;
2282         } else {
2283                 ret = -EINVAL;
2284         }
2285         spin_unlock_irqrestore(&id_priv->lock, flags);
2286         return ret;
2287 }
2288 EXPORT_SYMBOL(rdma_set_reuseaddr);
2289
2290 int rdma_set_afonly(struct rdma_cm_id *id, int afonly)
2291 {
2292         struct rdma_id_private *id_priv;
2293         unsigned long flags;
2294         int ret;
2295
2296         id_priv = container_of(id, struct rdma_id_private, id);
2297         spin_lock_irqsave(&id_priv->lock, flags);
2298         if (id_priv->state == RDMA_CM_IDLE || id_priv->state == RDMA_CM_ADDR_BOUND) {
2299                 id_priv->options |= (1 << CMA_OPTION_AFONLY);
2300                 id_priv->afonly = afonly;
2301                 ret = 0;
2302         } else {
2303                 ret = -EINVAL;
2304         }
2305         spin_unlock_irqrestore(&id_priv->lock, flags);
2306         return ret;
2307 }
2308 EXPORT_SYMBOL(rdma_set_afonly);
2309
2310 static void cma_bind_port(struct rdma_bind_list *bind_list,
2311                           struct rdma_id_private *id_priv)
2312 {
2313         struct sockaddr_in *sin;
2314
2315         sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
2316         sin->sin_port = htons(bind_list->port);
2317         id_priv->bind_list = bind_list;
2318         hlist_add_head(&id_priv->node, &bind_list->owners);
2319 }
2320
2321 static int cma_alloc_port(struct idr *ps, struct rdma_id_private *id_priv,
2322                           unsigned short snum)
2323 {
2324         struct rdma_bind_list *bind_list;
2325         int port, ret;
2326
2327         bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
2328         if (!bind_list)
2329                 return -ENOMEM;
2330
2331         do {
2332                 ret = idr_get_new_above(ps, bind_list, snum, &port);
2333         } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL));
2334
2335         if (ret)
2336                 goto err1;
2337
2338         if (port != snum) {
2339                 ret = -EADDRNOTAVAIL;
2340                 goto err2;
2341         }
2342
2343         bind_list->ps = ps;
2344         bind_list->port = (unsigned short) port;
2345         cma_bind_port(bind_list, id_priv);
2346         return 0;
2347 err2:
2348         idr_remove(ps, port);
2349 err1:
2350         kfree(bind_list);
2351         return ret;
2352 }
2353
2354 static int cma_alloc_any_port(struct idr *ps, struct rdma_id_private *id_priv)
2355 {
2356         static unsigned int last_used_port;
2357         int low, high, remaining;
2358         unsigned int rover;
2359
2360         inet_get_local_port_range(&low, &high);
2361         remaining = (high - low) + 1;
2362         rover = random() % remaining + low;
2363 retry:
2364         if (last_used_port != rover &&
2365             !idr_find(ps, (unsigned short) rover)) {
2366                 int ret = cma_alloc_port(ps, id_priv, rover);
2367                 /*
2368                  * Remember previously used port number in order to avoid
2369                  * re-using same port immediately after it is closed.
2370                  */
2371                 if (!ret)
2372                         last_used_port = rover;
2373                 if (ret != -EADDRNOTAVAIL)
2374                         return ret;
2375                 }
2376         if (--remaining) {
2377                 rover++;
2378                 if ((rover < low) || (rover > high))
2379                         rover = low;
2380                 goto retry;
2381         }
2382         return -EADDRNOTAVAIL;
2383 }
2384
2385 /*
2386  * Check that the requested port is available.  This is called when trying to
2387  * bind to a specific port, or when trying to listen on a bound port.  In
2388  * the latter case, the provided id_priv may already be on the bind_list, but
2389  * we still need to check that it's okay to start listening.
2390  */
2391 static int cma_check_port(struct rdma_bind_list *bind_list,
2392                           struct rdma_id_private *id_priv, uint8_t reuseaddr)
2393 {
2394         struct rdma_id_private *cur_id;
2395         struct sockaddr *addr, *cur_addr;
2396         struct hlist_node *node;
2397
2398         addr = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
2399         hlist_for_each_entry(cur_id, node, &bind_list->owners, node) {
2400                 if (id_priv == cur_id)
2401                         continue;
2402
2403                 if ((cur_id->state != RDMA_CM_LISTEN) && reuseaddr &&
2404                     cur_id->reuseaddr)
2405                         continue;
2406
2407                 cur_addr = (struct sockaddr *) &cur_id->id.route.addr.src_addr;
2408                 if (id_priv->afonly && cur_id->afonly &&
2409                     (addr->sa_family != cur_addr->sa_family))
2410                         continue;
2411
2412                 if (cma_any_addr(addr) || cma_any_addr(cur_addr))
2413                         return -EADDRNOTAVAIL;
2414
2415                 if (!cma_addr_cmp(addr, cur_addr))
2416                         return -EADDRINUSE;
2417         }
2418         return 0;
2419 }
2420
2421 static int cma_use_port(struct idr *ps, struct rdma_id_private *id_priv)
2422 {
2423         struct rdma_bind_list *bind_list;
2424         unsigned short snum;
2425         int ret;
2426
2427         snum = ntohs(cma_port((struct sockaddr *) &id_priv->id.route.addr.src_addr));
2428
2429         bind_list = idr_find(ps, snum);
2430         if (!bind_list) {
2431                 ret = cma_alloc_port(ps, id_priv, snum);
2432         } else {
2433                 ret = cma_check_port(bind_list, id_priv, id_priv->reuseaddr);
2434                 if (!ret)
2435                         cma_bind_port(bind_list, id_priv);
2436         }
2437         return ret;
2438 }
2439
2440 static int cma_bind_listen(struct rdma_id_private *id_priv)
2441 {
2442         struct rdma_bind_list *bind_list = id_priv->bind_list;
2443         int ret = 0;
2444
2445         mutex_lock(&lock);
2446         if (bind_list->owners.first->next)
2447                 ret = cma_check_port(bind_list, id_priv, 0);
2448         mutex_unlock(&lock);
2449         return ret;
2450 }
2451
2452 static int cma_get_port(struct rdma_id_private *id_priv)
2453 {
2454         struct idr *ps;
2455         int ret;
2456
2457         switch (id_priv->id.ps) {
2458         case RDMA_PS_SDP:
2459                 ps = &sdp_ps;
2460                 break;
2461         case RDMA_PS_TCP:
2462                 ps = &tcp_ps;
2463                 break;
2464         case RDMA_PS_UDP:
2465                 ps = &udp_ps;
2466                 break;
2467         case RDMA_PS_IPOIB:
2468                 ps = &ipoib_ps;
2469                 break;
2470         case RDMA_PS_IB:
2471                 ps = &ib_ps;
2472                 break;
2473         default:
2474                 return -EPROTONOSUPPORT;
2475         }
2476
2477         mutex_lock(&lock);
2478         if (cma_any_port((struct sockaddr *) &id_priv->id.route.addr.src_addr))
2479                 ret = cma_alloc_any_port(ps, id_priv);
2480         else
2481                 ret = cma_use_port(ps, id_priv);
2482         mutex_unlock(&lock);
2483
2484         return ret;
2485 }
2486
2487 static int cma_check_linklocal(struct rdma_dev_addr *dev_addr,
2488                                struct sockaddr *addr)
2489 {
2490 #if defined(INET6)
2491         struct sockaddr_in6 *sin6;
2492
2493         if (addr->sa_family != AF_INET6)
2494                 return 0;
2495
2496         sin6 = (struct sockaddr_in6 *) addr;
2497         if (IN6_IS_SCOPE_LINKLOCAL(&sin6->sin6_addr) &&
2498             !sin6->sin6_scope_id)
2499                         return -EINVAL;
2500
2501         dev_addr->bound_dev_if = sin6->sin6_scope_id;
2502 #endif
2503         return 0;
2504 }
2505
2506 int rdma_listen(struct rdma_cm_id *id, int backlog)
2507 {
2508         struct rdma_id_private *id_priv;
2509         int ret;
2510
2511         id_priv = container_of(id, struct rdma_id_private, id);
2512         if (id_priv->state == RDMA_CM_IDLE) {
2513                 ((struct sockaddr *) &id->route.addr.src_addr)->sa_family = AF_INET;
2514                 ret = rdma_bind_addr(id, (struct sockaddr *) &id->route.addr.src_addr);
2515                 if (ret)
2516                         return ret;
2517         }
2518
2519         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_LISTEN))
2520                 return -EINVAL;
2521
2522         if (id_priv->reuseaddr) {
2523                 ret = cma_bind_listen(id_priv);
2524                 if (ret)
2525                         goto err;
2526         }
2527
2528         id_priv->backlog = backlog;
2529         if (id->device) {
2530                 switch (rdma_node_get_transport(id->device->node_type)) {
2531                 case RDMA_TRANSPORT_IB:
2532                         ret = cma_ib_listen(id_priv);
2533                         if (ret)
2534                                 goto err;
2535                         break;
2536                 case RDMA_TRANSPORT_IWARP:
2537                 case RDMA_TRANSPORT_SCIF:
2538                         ret = cma_iw_listen(id_priv, backlog);
2539                         if (ret)
2540                                 goto err;
2541                         break;
2542                 default:
2543                         ret = -ENOSYS;
2544                         goto err;
2545                 }
2546         } else
2547                 cma_listen_on_all(id_priv);
2548
2549         return 0;
2550 err:
2551         id_priv->backlog = 0;
2552         cma_comp_exch(id_priv, RDMA_CM_LISTEN, RDMA_CM_ADDR_BOUND);
2553         return ret;
2554 }
2555 EXPORT_SYMBOL(rdma_listen);
2556
2557 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
2558 {
2559         struct rdma_id_private *id_priv;
2560         int ret;
2561 #if defined(INET6)
2562         int ipv6only;
2563         size_t var_size = sizeof(int);
2564 #endif
2565
2566         if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
2567                 return -EAFNOSUPPORT;
2568
2569         id_priv = container_of(id, struct rdma_id_private, id);
2570         if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND))
2571                 return -EINVAL;
2572
2573         ret = cma_check_linklocal(&id->route.addr.dev_addr, addr);
2574         if (ret)
2575                 goto err1;
2576
2577         memcpy(&id->route.addr.src_addr, addr, ip_addr_size(addr));
2578         if (!cma_any_addr(addr)) {
2579                 ret = rdma_translate_ip(addr, &id->route.addr.dev_addr, NULL);
2580                 if (ret)
2581                         goto err1;
2582
2583                 ret = cma_acquire_dev(id_priv);
2584                 if (ret)
2585                         goto err1;
2586         }
2587
2588         if (!(id_priv->options & (1 << CMA_OPTION_AFONLY))) {
2589                 if (addr->sa_family == AF_INET)
2590                         id_priv->afonly = 1;
2591 #if defined(INET6)
2592                 else if (addr->sa_family == AF_INET6)
2593                         id_priv->afonly = kernel_sysctlbyname(&thread0, "net.inet6.ip6.v6only",
2594                                             &ipv6only, &var_size, NULL, 0, NULL, 0);
2595 #endif
2596         }
2597         ret = cma_get_port(id_priv);
2598         if (ret)
2599                 goto err2;
2600
2601         return 0;
2602 err2:
2603         if (id_priv->cma_dev)
2604                 cma_release_dev(id_priv);
2605 err1:
2606         cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE);
2607         return ret;
2608 }
2609 EXPORT_SYMBOL(rdma_bind_addr);
2610
2611 static int cma_format_hdr(void *hdr, enum rdma_port_space ps,
2612                           struct rdma_route *route)
2613 {
2614         struct cma_hdr *cma_hdr;
2615         struct sdp_hh *sdp_hdr;
2616
2617         if (route->addr.src_addr.ss_family == AF_INET) {
2618                 struct sockaddr_in *src4, *dst4;
2619
2620                 src4 = (struct sockaddr_in *) &route->addr.src_addr;
2621                 dst4 = (struct sockaddr_in *) &route->addr.dst_addr;
2622
2623                 switch (ps) {
2624                 case RDMA_PS_SDP:
2625                         sdp_hdr = hdr;
2626                         if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
2627                                 return -EINVAL;
2628                         sdp_set_ip_ver(sdp_hdr, 4);
2629                         sdp_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2630                         sdp_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2631                         sdp_hdr->port = src4->sin_port;
2632                         break;
2633                 default:
2634                         cma_hdr = hdr;
2635                         cma_hdr->cma_version = CMA_VERSION;
2636                         cma_set_ip_ver(cma_hdr, 4);
2637                         cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2638                         cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2639                         cma_hdr->port = src4->sin_port;
2640                         break;
2641                 }
2642         } else {
2643                 struct sockaddr_in6 *src6, *dst6;
2644
2645                 src6 = (struct sockaddr_in6 *) &route->addr.src_addr;
2646                 dst6 = (struct sockaddr_in6 *) &route->addr.dst_addr;
2647
2648                 switch (ps) {
2649                 case RDMA_PS_SDP:
2650                         sdp_hdr = hdr;
2651                         if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
2652                                 return -EINVAL;
2653                         sdp_set_ip_ver(sdp_hdr, 6);
2654                         sdp_hdr->src_addr.ip6 = src6->sin6_addr;
2655                         sdp_hdr->dst_addr.ip6 = dst6->sin6_addr;
2656                         sdp_hdr->port = src6->sin6_port;
2657                         break;
2658                 default:
2659                         cma_hdr = hdr;
2660                         cma_hdr->cma_version = CMA_VERSION;
2661                         cma_set_ip_ver(cma_hdr, 6);
2662                         cma_hdr->src_addr.ip6 = src6->sin6_addr;
2663                         cma_hdr->dst_addr.ip6 = dst6->sin6_addr;
2664                         cma_hdr->port = src6->sin6_port;
2665                         break;
2666                 }
2667         }
2668         return 0;
2669 }
2670
2671 static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
2672                                 struct ib_cm_event *ib_event)
2673 {
2674         struct rdma_id_private *id_priv = cm_id->context;
2675         struct rdma_cm_event event;
2676         struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd;
2677         int ret = 0;
2678
2679         if (cma_disable_callback(id_priv, RDMA_CM_CONNECT))
2680                 return 0;
2681
2682         memset(&event, 0, sizeof event);
2683         switch (ib_event->event) {
2684         case IB_CM_SIDR_REQ_ERROR:
2685                 event.event = RDMA_CM_EVENT_UNREACHABLE;
2686                 event.status = -ETIMEDOUT;
2687                 break;
2688         case IB_CM_SIDR_REP_RECEIVED:
2689                 event.param.ud.private_data = ib_event->private_data;
2690                 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
2691                 if (rep->status != IB_SIDR_SUCCESS) {
2692                         event.event = RDMA_CM_EVENT_UNREACHABLE;
2693                         event.status = ib_event->param.sidr_rep_rcvd.status;
2694                         break;
2695                 }
2696                 ret = cma_set_qkey(id_priv);
2697                 if (ret) {
2698                         event.event = RDMA_CM_EVENT_ADDR_ERROR;
2699                         event.status = -EINVAL;
2700                         break;
2701                 }
2702                 if (id_priv->qkey != rep->qkey) {
2703                         event.event = RDMA_CM_EVENT_UNREACHABLE;
2704                         event.status = -EINVAL;
2705                         break;
2706                 }
2707                 ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num,
2708                                      id_priv->id.route.path_rec,
2709                                      &event.param.ud.ah_attr);
2710                 event.param.ud.qp_num = rep->qpn;
2711                 event.param.ud.qkey = rep->qkey;
2712                 event.event = RDMA_CM_EVENT_ESTABLISHED;
2713                 event.status = 0;
2714                 break;
2715         default:
2716                 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
2717                        ib_event->event);
2718                 goto out;
2719         }
2720
2721         ret = id_priv->id.event_handler(&id_priv->id, &event);
2722         if (ret) {
2723                 /* Destroy the CM ID by returning a non-zero value. */
2724                 id_priv->cm_id.ib = NULL;
2725                 cma_exch(id_priv, RDMA_CM_DESTROYING);
2726                 mutex_unlock(&id_priv->handler_mutex);
2727                 rdma_destroy_id(&id_priv->id);
2728                 return ret;
2729         }
2730 out:
2731         mutex_unlock(&id_priv->handler_mutex);
2732         return ret;
2733 }
2734
2735 static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
2736                               struct rdma_conn_param *conn_param)
2737 {
2738         struct ib_cm_sidr_req_param req;
2739         struct rdma_route *route;
2740         struct ib_cm_id *id;
2741         int ret;
2742
2743         req.private_data_len = sizeof(struct cma_hdr) +
2744                                conn_param->private_data_len;
2745         if (req.private_data_len < conn_param->private_data_len)
2746                 return -EINVAL;
2747
2748         req.private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2749         if (!req.private_data)
2750                 return -ENOMEM;
2751
2752         if (conn_param->private_data && conn_param->private_data_len)
2753                 memcpy((void *) req.private_data + sizeof(struct cma_hdr),
2754                        conn_param->private_data, conn_param->private_data_len);
2755
2756         route = &id_priv->id.route;
2757         ret = cma_format_hdr((void *) req.private_data, id_priv->id.ps, route);
2758         if (ret)
2759                 goto out;
2760
2761         id = ib_create_cm_id(id_priv->id.device, cma_sidr_rep_handler,
2762                              id_priv);
2763         if (IS_ERR(id)) {
2764                 ret = PTR_ERR(id);
2765                 goto out;
2766         }
2767         id_priv->cm_id.ib = id;
2768
2769         req.path = route->path_rec;
2770         req.service_id = cma_get_service_id(id_priv->id.ps,
2771                                             (struct sockaddr *) &route->addr.dst_addr);
2772         req.timeout_ms = 1 << (cma_response_timeout - 8);
2773         req.max_cm_retries = CMA_MAX_CM_RETRIES;
2774
2775         cma_dbg(id_priv, "sending SIDR\n");
2776         ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req);
2777         if (ret) {
2778                 ib_destroy_cm_id(id_priv->cm_id.ib);
2779                 id_priv->cm_id.ib = NULL;
2780         }
2781 out:
2782         kfree(req.private_data);
2783         return ret;
2784 }
2785
2786 static int cma_connect_ib(struct rdma_id_private *id_priv,
2787                           struct rdma_conn_param *conn_param)
2788 {
2789         struct ib_cm_req_param req;
2790         struct rdma_route *route;
2791         void *private_data;
2792         struct ib_cm_id *id;
2793         int offset, ret;
2794
2795         memset(&req, 0, sizeof req);
2796         offset = cma_user_data_offset(id_priv->id.ps);
2797         req.private_data_len = offset + conn_param->private_data_len;
2798         if (req.private_data_len < conn_param->private_data_len)
2799                 return -EINVAL;
2800
2801         private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2802         if (!private_data)
2803                 return -ENOMEM;
2804
2805         if (conn_param->private_data && conn_param->private_data_len)
2806                 memcpy(private_data + offset, conn_param->private_data,
2807                        conn_param->private_data_len);
2808
2809         id = ib_create_cm_id(id_priv->id.device, cma_ib_handler, id_priv);
2810         if (IS_ERR(id)) {
2811                 ret = PTR_ERR(id);
2812                 goto out;
2813         }
2814         id_priv->cm_id.ib = id;
2815
2816         route = &id_priv->id.route;
2817         ret = cma_format_hdr(private_data, id_priv->id.ps, route);
2818         if (ret)
2819                 goto out;
2820         req.private_data = private_data;
2821
2822         req.primary_path = &route->path_rec[0];
2823         if (route->num_paths == 2)
2824                 req.alternate_path = &route->path_rec[1];
2825
2826         req.service_id = cma_get_service_id(id_priv->id.ps,
2827                                             (struct sockaddr *) &route->addr.dst_addr);
2828         req.qp_num = id_priv->qp_num;
2829         req.qp_type = id_priv->id.qp_type;
2830         req.starting_psn = id_priv->seq_num;
2831         req.responder_resources = conn_param->responder_resources;
2832         req.initiator_depth = conn_param->initiator_depth;
2833         req.flow_control = conn_param->flow_control;
2834         req.retry_count = min_t(u8, 7, conn_param->retry_count);
2835         req.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
2836        req.remote_cm_response_timeout = cma_response_timeout;
2837        req.local_cm_response_timeout = cma_response_timeout;
2838         req.max_cm_retries = CMA_MAX_CM_RETRIES;
2839         req.srq = id_priv->srq ? 1 : 0;
2840
2841         cma_dbg(id_priv, "sending REQ\n");
2842         ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
2843 out:
2844         if (ret && !IS_ERR(id)) {
2845                 ib_destroy_cm_id(id);
2846                 id_priv->cm_id.ib = NULL;
2847         }
2848
2849         kfree(private_data);
2850         return ret;
2851 }
2852
2853 static int cma_connect_iw(struct rdma_id_private *id_priv,
2854                           struct rdma_conn_param *conn_param)
2855 {
2856         struct iw_cm_id *cm_id;
2857         struct sockaddr_in* sin;
2858         int ret;
2859         struct iw_cm_conn_param iw_param;
2860
2861         cm_id = iw_create_cm_id(id_priv->id.device, id_priv->sock,
2862                                         cma_iw_handler, id_priv);
2863         if (IS_ERR(cm_id))
2864                 return PTR_ERR(cm_id);
2865
2866         id_priv->cm_id.iw = cm_id;
2867
2868         sin = (struct sockaddr_in*) &id_priv->id.route.addr.src_addr;
2869         cm_id->local_addr = *sin;
2870
2871         sin = (struct sockaddr_in*) &id_priv->id.route.addr.dst_addr;
2872         cm_id->remote_addr = *sin;
2873
2874         ret = cma_modify_qp_rtr(id_priv, conn_param);
2875         if (ret)
2876                 goto out;
2877
2878         if (conn_param) {
2879         iw_param.ord = conn_param->initiator_depth;
2880         iw_param.ird = conn_param->responder_resources;
2881         iw_param.private_data = conn_param->private_data;
2882         iw_param.private_data_len = conn_param->private_data_len;
2883                 iw_param.qpn = id_priv->id.qp ? id_priv->qp_num : conn_param->qp_num;
2884         } else {
2885                 memset(&iw_param, 0, sizeof iw_param);
2886                 iw_param.qpn = id_priv->qp_num;
2887         }
2888         ret = iw_cm_connect(cm_id, &iw_param);
2889 out:
2890         if (ret) {
2891                 iw_destroy_cm_id(cm_id);
2892                 id_priv->cm_id.iw = NULL;
2893         }
2894         return ret;
2895 }
2896
2897 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2898 {
2899         struct rdma_id_private *id_priv;
2900         int ret;
2901
2902         id_priv = container_of(id, struct rdma_id_private, id);
2903         if (!cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_CONNECT))
2904                 return -EINVAL;
2905
2906         if (!id->qp) {
2907                 id_priv->qp_num = conn_param->qp_num;
2908                 id_priv->srq = conn_param->srq;
2909         }
2910
2911         switch (rdma_node_get_transport(id->device->node_type)) {
2912         case RDMA_TRANSPORT_IB:
2913                 if (id->qp_type == IB_QPT_UD)
2914                         ret = cma_resolve_ib_udp(id_priv, conn_param);
2915                 else
2916                         ret = cma_connect_ib(id_priv, conn_param);
2917                 break;
2918         case RDMA_TRANSPORT_IWARP:
2919         case RDMA_TRANSPORT_SCIF:
2920                 ret = cma_connect_iw(id_priv, conn_param);
2921                 break;
2922         default:
2923                 ret = -ENOSYS;
2924                 break;
2925         }
2926         if (ret)
2927                 goto err;
2928
2929         return 0;
2930 err:
2931         cma_comp_exch(id_priv, RDMA_CM_CONNECT, RDMA_CM_ROUTE_RESOLVED);
2932         return ret;
2933 }
2934 EXPORT_SYMBOL(rdma_connect);
2935
2936 static int cma_accept_ib(struct rdma_id_private *id_priv,
2937                          struct rdma_conn_param *conn_param)
2938 {
2939         struct ib_cm_rep_param rep;
2940         int ret;
2941
2942         ret = cma_modify_qp_rtr(id_priv, conn_param);
2943         if (ret)
2944                 goto out;
2945
2946         ret = cma_modify_qp_rts(id_priv, conn_param);
2947         if (ret)
2948                 goto out;
2949
2950         memset(&rep, 0, sizeof rep);
2951         rep.qp_num = id_priv->qp_num;
2952         rep.starting_psn = id_priv->seq_num;
2953         rep.private_data = conn_param->private_data;
2954         rep.private_data_len = conn_param->private_data_len;
2955         rep.responder_resources = conn_param->responder_resources;
2956         rep.initiator_depth = conn_param->initiator_depth;
2957         rep.failover_accepted = 0;
2958         rep.flow_control = conn_param->flow_control;
2959         rep.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
2960         rep.srq = id_priv->srq ? 1 : 0;
2961         cma_dbg(id_priv, "sending REP\n");
2962         ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep);
2963 out:
2964         return ret;
2965 }
2966
2967 static int cma_accept_iw(struct rdma_id_private *id_priv,
2968                   struct rdma_conn_param *conn_param)
2969 {
2970         struct iw_cm_conn_param iw_param;
2971         int ret;
2972
2973         if (!conn_param)
2974                 return -EINVAL;
2975
2976         ret = cma_modify_qp_rtr(id_priv, conn_param);
2977         if (ret)
2978                 return ret;
2979
2980         iw_param.ord = conn_param->initiator_depth;
2981         iw_param.ird = conn_param->responder_resources;
2982         iw_param.private_data = conn_param->private_data;
2983         iw_param.private_data_len = conn_param->private_data_len;
2984         if (id_priv->id.qp) {
2985                 iw_param.qpn = id_priv->qp_num;
2986         } else
2987                 iw_param.qpn = conn_param->qp_num;
2988
2989         return iw_cm_accept(id_priv->cm_id.iw, &iw_param);
2990 }
2991
2992 static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
2993                              enum ib_cm_sidr_status status,
2994                              const void *private_data, int private_data_len)
2995 {
2996         struct ib_cm_sidr_rep_param rep;
2997         int ret;
2998
2999         memset(&rep, 0, sizeof rep);
3000         rep.status = status;
3001         if (status == IB_SIDR_SUCCESS) {
3002                 ret = cma_set_qkey(id_priv);
3003                 if (ret)
3004                         return ret;
3005                 rep.qp_num = id_priv->qp_num;
3006                 rep.qkey = id_priv->qkey;
3007         }
3008         rep.private_data = private_data;
3009         rep.private_data_len = private_data_len;
3010
3011         cma_dbg(id_priv, "sending SIDR\n");
3012         return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
3013 }
3014
3015 int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
3016 {
3017         struct rdma_id_private *id_priv;
3018         int ret;
3019
3020         id_priv = container_of(id, struct rdma_id_private, id);
3021
3022         id_priv->owner = curthread->td_proc->p_pid;
3023         if (!cma_comp(id_priv, RDMA_CM_CONNECT))
3024                 return -EINVAL;
3025
3026         if (!id->qp && conn_param) {
3027                 id_priv->qp_num = conn_param->qp_num;
3028                 id_priv->srq = conn_param->srq;
3029         }
3030
3031         switch (rdma_node_get_transport(id->device->node_type)) {
3032         case RDMA_TRANSPORT_IB:
3033                 if (id->qp_type == IB_QPT_UD) {
3034                         if (conn_param)
3035                         ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
3036                                                 conn_param->private_data,
3037                                                 conn_param->private_data_len);
3038                         else
3039                                 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
3040                                                         NULL, 0);
3041                 } else {
3042                         if (conn_param)
3043                         ret = cma_accept_ib(id_priv, conn_param);
3044                 else
3045                         ret = cma_rep_recv(id_priv);
3046                 }
3047                 break;
3048         case RDMA_TRANSPORT_IWARP:
3049         case RDMA_TRANSPORT_SCIF:
3050                 ret = cma_accept_iw(id_priv, conn_param);
3051                 break;
3052         default:
3053                 ret = -ENOSYS;
3054                 break;
3055         }
3056
3057         if (ret)
3058                 goto reject;
3059
3060         return 0;
3061 reject:
3062         cma_modify_qp_err(id_priv);
3063         rdma_reject(id, NULL, 0);
3064         return ret;
3065 }
3066 EXPORT_SYMBOL(rdma_accept);
3067
3068 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
3069 {
3070         struct rdma_id_private *id_priv;
3071         int ret;
3072
3073         id_priv = container_of(id, struct rdma_id_private, id);
3074         if (!id_priv->cm_id.ib)
3075                 return -EINVAL;
3076
3077         switch (id->device->node_type) {
3078         case RDMA_NODE_IB_CA:
3079                 ret = ib_cm_notify(id_priv->cm_id.ib, event);
3080                 break;
3081         default:
3082                 ret = 0;
3083                 break;
3084         }
3085         return ret;
3086 }
3087 EXPORT_SYMBOL(rdma_notify);
3088
3089 int rdma_reject(struct rdma_cm_id *id, const void *private_data,
3090                 u8 private_data_len)
3091 {
3092         struct rdma_id_private *id_priv;
3093         int ret;
3094
3095         id_priv = container_of(id, struct rdma_id_private, id);
3096         if (!id_priv->cm_id.ib)
3097                 return -EINVAL;
3098
3099         switch (rdma_node_get_transport(id->device->node_type)) {
3100         case RDMA_TRANSPORT_IB:
3101                 if (id->qp_type == IB_QPT_UD)
3102                         ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT,
3103                                                 private_data, private_data_len);
3104                 else {
3105                         cma_dbg(id_priv, "sending REJ\n");
3106                         ret = ib_send_cm_rej(id_priv->cm_id.ib,
3107                                              IB_CM_REJ_CONSUMER_DEFINED, NULL,
3108                                              0, private_data, private_data_len);
3109                 }
3110                 break;
3111         case RDMA_TRANSPORT_IWARP:
3112         case RDMA_TRANSPORT_SCIF:
3113                 ret = iw_cm_reject(id_priv->cm_id.iw,
3114                                    private_data, private_data_len);
3115                 break;
3116         default:
3117                 ret = -ENOSYS;
3118                 break;
3119         }
3120         return ret;
3121 }
3122 EXPORT_SYMBOL(rdma_reject);
3123
3124 int rdma_disconnect(struct rdma_cm_id *id)
3125 {
3126         struct rdma_id_private *id_priv;
3127         int ret;
3128
3129         id_priv = container_of(id, struct rdma_id_private, id);
3130         if (!id_priv->cm_id.ib)
3131                 return -EINVAL;
3132
3133         switch (rdma_node_get_transport(id->device->node_type)) {
3134         case RDMA_TRANSPORT_IB:
3135                 ret = cma_modify_qp_err(id_priv);
3136                 if (ret)
3137                         goto out;
3138                 /* Initiate or respond to a disconnect. */
3139                 cma_dbg(id_priv, "sending DREQ\n");
3140                 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) {
3141                         cma_dbg(id_priv, "sending DREP\n");
3142                         ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0);
3143                 }
3144                 break;
3145         case RDMA_TRANSPORT_IWARP:
3146         case RDMA_TRANSPORT_SCIF:
3147                 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
3148                 break;
3149         default:
3150                 ret = -EINVAL;
3151                 break;
3152         }
3153 out:
3154         return ret;
3155 }
3156 EXPORT_SYMBOL(rdma_disconnect);
3157
3158 static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
3159 {
3160         struct rdma_id_private *id_priv;
3161         struct cma_multicast *mc = multicast->context;
3162         struct rdma_cm_event event;
3163         struct rdma_dev_addr *dev_addr;
3164         int ret;
3165         struct net_device *ndev = NULL;
3166         u16 vlan;
3167
3168         id_priv = mc->id_priv;
3169         dev_addr = &id_priv->id.route.addr.dev_addr;
3170         if (cma_disable_callback(id_priv, RDMA_CM_ADDR_BOUND) &&
3171             cma_disable_callback(id_priv, RDMA_CM_ADDR_RESOLVED))
3172                 return 0;
3173
3174         mutex_lock(&id_priv->qp_mutex);
3175         if (!status && id_priv->id.qp)
3176                 status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid,
3177                                          be16_to_cpu(multicast->rec.mlid));
3178         mutex_unlock(&id_priv->qp_mutex);
3179
3180         memset(&event, 0, sizeof event);
3181         event.status = status;
3182         event.param.ud.private_data = mc->context;
3183         ndev = dev_get_by_index(&init_net, dev_addr->bound_dev_if);
3184         if (!ndev) {
3185                 status = -ENODEV;
3186         } else {
3187                 vlan = rdma_vlan_dev_vlan_id(ndev);
3188                 dev_put(ndev);
3189         }
3190         if (!status) {
3191                 event.event = RDMA_CM_EVENT_MULTICAST_JOIN;
3192                 ib_init_ah_from_mcmember(id_priv->id.device,
3193                                          id_priv->id.port_num, &multicast->rec,
3194                                          &event.param.ud.ah_attr);
3195                 event.param.ud.ah_attr.vlan_id = vlan;
3196                 event.param.ud.qp_num = 0xFFFFFF;
3197                 event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey);
3198         } else {
3199                 event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
3200
3201                 /* mark that the cached record is no longer valid */
3202                 if (status != -ENETRESET && status != -EAGAIN) {
3203                         spin_lock(&id_priv->lock);
3204                         id_priv->is_valid_rec = 0;
3205                         spin_unlock(&id_priv->lock);
3206                 }
3207         }
3208
3209         ret = id_priv->id.event_handler(&id_priv->id, &event);
3210         if (ret) {
3211                 cma_exch(id_priv, RDMA_CM_DESTROYING);
3212                 mutex_unlock(&id_priv->handler_mutex);
3213                 rdma_destroy_id(&id_priv->id);
3214                 return 0;
3215         }
3216
3217         mutex_unlock(&id_priv->handler_mutex);
3218         return 0;
3219 }
3220
3221 static void cma_set_mgid(struct rdma_id_private *id_priv,
3222                          struct sockaddr *addr, union ib_gid *mgid)
3223 {
3224         unsigned char mc_map[MAX_ADDR_LEN];
3225         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
3226         struct sockaddr_in *sin = (struct sockaddr_in *) addr;
3227 #if defined(INET6)
3228         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr;
3229 #endif
3230
3231         if (cma_any_addr(addr)) {
3232                 memset(mgid, 0, sizeof *mgid);
3233 #if defined(INET6)
3234         } else if ((addr->sa_family == AF_INET6) &&
3235                    ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFFF0FFFF) ==
3236                                                                  0xFF10A01B)) {
3237                 /* IPv6 address is an SA assigned MGID. */
3238                 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
3239         } else if (addr->sa_family == AF_INET6) {
3240                 ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map);
3241                 if (id_priv->id.ps == RDMA_PS_UDP)
3242                         mc_map[7] = 0x01;       /* Use RDMA CM signature */
3243                 *mgid = *(union ib_gid *) (mc_map + 4);
3244 #endif
3245         } else {
3246                 ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map);
3247                 if (id_priv->id.ps == RDMA_PS_UDP)
3248                         mc_map[7] = 0x01;       /* Use RDMA CM signature */
3249                 *mgid = *(union ib_gid *) (mc_map + 4);
3250         }
3251 }
3252
3253 static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
3254                                  struct cma_multicast *mc)
3255 {
3256         struct ib_sa_mcmember_rec rec;
3257         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
3258         ib_sa_comp_mask comp_mask;
3259         int ret = 0;
3260
3261         ib_addr_get_mgid(dev_addr, &id_priv->rec.mgid);
3262
3263         /* cache ipoib bc record */
3264         spin_lock(&id_priv->lock);
3265         if (!id_priv->is_valid_rec)
3266                 ret = ib_sa_get_mcmember_rec(id_priv->id.device,
3267                                              id_priv->id.port_num,
3268                                              &id_priv->rec.mgid,
3269                                              &id_priv->rec);
3270         if (ret) {
3271                 id_priv->is_valid_rec = 0;
3272                 spin_unlock(&id_priv->lock);
3273                 return ret;
3274         } else {
3275                 rec = id_priv->rec;
3276                 id_priv->is_valid_rec = 1;
3277         }
3278         spin_unlock(&id_priv->lock);
3279
3280         cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid);
3281         if (id_priv->id.ps == RDMA_PS_UDP)
3282                 rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
3283         rdma_addr_get_sgid(dev_addr, &rec.port_gid);
3284         rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
3285         rec.join_state = 1;
3286
3287         comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
3288                     IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |
3289                     IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL |
3290                     IB_SA_MCMEMBER_REC_FLOW_LABEL |
3291                     IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
3292
3293         if (id_priv->id.ps == RDMA_PS_IPOIB)
3294                 comp_mask |= IB_SA_MCMEMBER_REC_RATE |
3295                              IB_SA_MCMEMBER_REC_RATE_SELECTOR |
3296                              IB_SA_MCMEMBER_REC_MTU_SELECTOR |
3297                              IB_SA_MCMEMBER_REC_MTU |
3298                              IB_SA_MCMEMBER_REC_HOP_LIMIT;
3299
3300         mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device,
3301                                                 id_priv->id.port_num, &rec,
3302                                                 comp_mask, GFP_KERNEL,
3303                                                 cma_ib_mc_handler, mc);
3304         return PTR_RET(mc->multicast.ib);
3305 }
3306
3307 static void iboe_mcast_work_handler(struct work_struct *work)
3308 {
3309         struct iboe_mcast_work *mw = container_of(work, struct iboe_mcast_work, work);
3310         struct cma_multicast *mc = mw->mc;
3311         struct ib_sa_multicast *m = mc->multicast.ib;
3312
3313         mc->multicast.ib->context = mc;
3314         cma_ib_mc_handler(0, m);
3315         kref_put(&mc->mcref, release_mc);
3316         kfree(mw);
3317 }
3318
3319 static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid)
3320 {
3321         struct sockaddr_in *sin = (struct sockaddr_in *)addr;
3322         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;
3323
3324         if (cma_any_addr(addr)) {
3325                 memset(mgid, 0, sizeof *mgid);
3326         } else if (addr->sa_family == AF_INET6) {
3327                 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
3328         } else {
3329                 mgid->raw[0] = 0xff;
3330                 mgid->raw[1] = 0x0e;
3331                 mgid->raw[2] = 0;
3332                 mgid->raw[3] = 0;
3333                 mgid->raw[4] = 0;
3334                 mgid->raw[5] = 0;
3335                 mgid->raw[6] = 0;
3336                 mgid->raw[7] = 0;
3337                 mgid->raw[8] = 0;
3338                 mgid->raw[9] = 0;
3339                 mgid->raw[10] = 0xff;
3340                 mgid->raw[11] = 0xff;
3341                 *(__be32 *)(&mgid->raw[12]) = sin->sin_addr.s_addr;
3342         }
3343 }
3344
3345 static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,
3346                                    struct cma_multicast *mc)
3347 {
3348         struct iboe_mcast_work *work;
3349         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
3350         int err;
3351         struct sockaddr *addr = (struct sockaddr *)&mc->addr;
3352         struct net_device *ndev = NULL;
3353
3354         if (cma_zero_addr((struct sockaddr *)&mc->addr))
3355                 return -EINVAL;
3356
3357         work = kzalloc(sizeof *work, GFP_KERNEL);
3358         if (!work)
3359                 return -ENOMEM;
3360
3361         mc->multicast.ib = kzalloc(sizeof(struct ib_sa_multicast), GFP_KERNEL);
3362         if (!mc->multicast.ib) {
3363                 err = -ENOMEM;
3364                 goto out1;
3365         }
3366
3367         cma_iboe_set_mgid(addr, &mc->multicast.ib->rec.mgid);
3368
3369         mc->multicast.ib->rec.pkey = cpu_to_be16(0xffff);
3370         if (id_priv->id.ps == RDMA_PS_UDP)
3371                 mc->multicast.ib->rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
3372
3373         if (dev_addr->bound_dev_if)
3374                 ndev = dev_get_by_index(&init_net, dev_addr->bound_dev_if);
3375         if (!ndev) {
3376                 err = -ENODEV;
3377                 goto out2;
3378         }
3379         mc->multicast.ib->rec.rate = iboe_get_rate(ndev);
3380         mc->multicast.ib->rec.hop_limit = 1;
3381         mc->multicast.ib->rec.mtu = iboe_get_mtu(ndev->if_mtu);
3382         dev_put(ndev);
3383         if (!mc->multicast.ib->rec.mtu) {
3384                 err = -EINVAL;
3385                 goto out2;
3386         }
3387         rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
3388                     &mc->multicast.ib->rec.port_gid);
3389         work->id = id_priv;
3390         work->mc = mc;
3391         INIT_WORK(&work->work, iboe_mcast_work_handler);
3392         kref_get(&mc->mcref);
3393         queue_work(cma_wq, &work->work);
3394
3395         return 0;
3396
3397 out2:
3398         kfree(mc->multicast.ib);
3399 out1:
3400         kfree(work);
3401         return err;
3402 }
3403
3404 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
3405                         void *context)
3406 {
3407         struct rdma_id_private *id_priv;
3408         struct cma_multicast *mc;
3409         int ret;
3410
3411         id_priv = container_of(id, struct rdma_id_private, id);
3412         if (!cma_comp(id_priv, RDMA_CM_ADDR_BOUND) &&
3413             !cma_comp(id_priv, RDMA_CM_ADDR_RESOLVED))
3414                 return -EINVAL;
3415
3416         mc = kmalloc(sizeof *mc, GFP_KERNEL);
3417         if (!mc)
3418                 return -ENOMEM;
3419
3420         memcpy(&mc->addr, addr, ip_addr_size(addr));
3421         mc->context = context;
3422         mc->id_priv = id_priv;
3423
3424         spin_lock(&id_priv->lock);
3425         list_add(&mc->list, &id_priv->mc_list);
3426         spin_unlock(&id_priv->lock);
3427
3428         switch (rdma_node_get_transport(id->device->node_type)) {
3429         case RDMA_TRANSPORT_IB:
3430                 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
3431                 case IB_LINK_LAYER_INFINIBAND:
3432                         ret = cma_join_ib_multicast(id_priv, mc);
3433                         break;
3434                 case IB_LINK_LAYER_ETHERNET:
3435                         kref_init(&mc->mcref);
3436                         ret = cma_iboe_join_multicast(id_priv, mc);
3437                         break;
3438                 default:
3439                         ret = -EINVAL;
3440                 }
3441                 break;
3442         default:
3443                 ret = -ENOSYS;
3444                 break;
3445         }
3446
3447         if (ret) {
3448                 spin_lock_irq(&id_priv->lock);
3449                 list_del(&mc->list);
3450                 spin_unlock_irq(&id_priv->lock);
3451                 kfree(mc);
3452         }
3453         return ret;
3454 }
3455 EXPORT_SYMBOL(rdma_join_multicast);
3456
3457 void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
3458 {
3459         struct rdma_id_private *id_priv;
3460         struct cma_multicast *mc;
3461
3462         id_priv = container_of(id, struct rdma_id_private, id);
3463         spin_lock_irq(&id_priv->lock);
3464         list_for_each_entry(mc, &id_priv->mc_list, list) {
3465                 if (!memcmp(&mc->addr, addr, ip_addr_size(addr))) {
3466                         list_del(&mc->list);
3467                         spin_unlock_irq(&id_priv->lock);
3468
3469                         if (id->qp)
3470                                 ib_detach_mcast(id->qp,
3471                                                 &mc->multicast.ib->rec.mgid,
3472                                                 be16_to_cpu(mc->multicast.ib->rec.mlid));
3473                         if (rdma_node_get_transport(id_priv->cma_dev->device->node_type) == RDMA_TRANSPORT_IB) {
3474                                 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
3475                                 case IB_LINK_LAYER_INFINIBAND:
3476                                         ib_sa_free_multicast(mc->multicast.ib);
3477                                         kfree(mc);
3478                                         break;
3479                                 case IB_LINK_LAYER_ETHERNET:
3480                                         kref_put(&mc->mcref, release_mc);
3481                                         break;
3482                                 default:
3483                                         break;
3484                                 }
3485                         }
3486                         return;
3487                 }
3488         }
3489         spin_unlock_irq(&id_priv->lock);
3490 }
3491 EXPORT_SYMBOL(rdma_leave_multicast);
3492
3493 static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv)
3494 {
3495         struct rdma_dev_addr *dev_addr;
3496         struct cma_ndev_work *work;
3497
3498         dev_addr = &id_priv->id.route.addr.dev_addr;
3499
3500         if ((dev_addr->bound_dev_if == ndev->if_index) &&
3501             memcmp(dev_addr->src_dev_addr, IF_LLADDR(ndev), ndev->if_addrlen)) {
3502                 printk(KERN_INFO "RDMA CM addr change for ndev %s used by id %p\n",
3503                        ndev->if_xname, &id_priv->id);
3504                 work = kzalloc(sizeof *work, GFP_KERNEL);
3505                 if (!work)
3506                         return -ENOMEM;
3507
3508                 INIT_WORK(&work->work, cma_ndev_work_handler);
3509                 work->id = id_priv;
3510                 work->event.event = RDMA_CM_EVENT_ADDR_CHANGE;
3511                 atomic_inc(&id_priv->refcount);
3512                 queue_work(cma_wq, &work->work);
3513         }
3514
3515         return 0;
3516 }
3517
3518 static int cma_netdev_callback(struct notifier_block *self, unsigned long event,
3519                                void *ctx)
3520 {
3521         struct net_device *ndev = (struct net_device *)ctx;
3522         struct cma_device *cma_dev;
3523         struct rdma_id_private *id_priv;
3524         int ret = NOTIFY_DONE;
3525
3526 /* BONDING related, commented out until the bonding is resolved */
3527 #if 0
3528         if (dev_net(ndev) != &init_net)
3529                 return NOTIFY_DONE;
3530
3531         if (event != NETDEV_BONDING_FAILOVER)
3532                 return NOTIFY_DONE;
3533
3534         if (!(ndev->flags & IFF_MASTER) || !(ndev->priv_flags & IFF_BONDING))
3535                 return NOTIFY_DONE;
3536 #endif
3537         if (event != NETDEV_DOWN && event != NETDEV_UNREGISTER)
3538                 return NOTIFY_DONE;
3539
3540         mutex_lock(&lock);
3541         list_for_each_entry(cma_dev, &dev_list, list)
3542                 list_for_each_entry(id_priv, &cma_dev->id_list, list) {
3543                         ret = cma_netdev_change(ndev, id_priv);
3544                         if (ret)
3545                                 goto out;
3546                 }
3547
3548 out:
3549         mutex_unlock(&lock);
3550         return ret;
3551 }
3552
3553 static struct notifier_block cma_nb = {
3554         .notifier_call = cma_netdev_callback
3555 };
3556
3557 static void cma_add_one(struct ib_device *device)
3558 {
3559         struct cma_device *cma_dev;
3560         struct rdma_id_private *id_priv;
3561
3562         cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL);
3563         if (!cma_dev)
3564                 return;
3565
3566         cma_dev->device = device;
3567
3568         init_completion(&cma_dev->comp);
3569         atomic_set(&cma_dev->refcount, 1);
3570         INIT_LIST_HEAD(&cma_dev->id_list);
3571         ib_set_client_data(device, &cma_client, cma_dev);
3572
3573         mutex_lock(&lock);
3574         list_add_tail(&cma_dev->list, &dev_list);
3575         list_for_each_entry(id_priv, &listen_any_list, list)
3576                 cma_listen_on_dev(id_priv, cma_dev);
3577         mutex_unlock(&lock);
3578 }
3579
3580 static int cma_remove_id_dev(struct rdma_id_private *id_priv)
3581 {
3582         struct rdma_cm_event event;
3583         enum rdma_cm_state state;
3584         int ret = 0;
3585
3586         /* Record that we want to remove the device */
3587         state = cma_exch(id_priv, RDMA_CM_DEVICE_REMOVAL);
3588         if (state == RDMA_CM_DESTROYING)
3589                 return 0;
3590
3591         cma_cancel_operation(id_priv, state);
3592         mutex_lock(&id_priv->handler_mutex);
3593
3594         /* Check for destruction from another callback. */
3595         if (!cma_comp(id_priv, RDMA_CM_DEVICE_REMOVAL))
3596                 goto out;
3597
3598         memset(&event, 0, sizeof event);
3599         event.event = RDMA_CM_EVENT_DEVICE_REMOVAL;
3600         ret = id_priv->id.event_handler(&id_priv->id, &event);
3601 out:
3602         mutex_unlock(&id_priv->handler_mutex);
3603         return ret;
3604 }
3605
3606 static void cma_process_remove(struct cma_device *cma_dev)
3607 {
3608         struct rdma_id_private *id_priv;
3609         int ret;
3610
3611         mutex_lock(&lock);
3612         while (!list_empty(&cma_dev->id_list)) {
3613                 id_priv = list_entry(cma_dev->id_list.next,
3614                                      struct rdma_id_private, list);
3615
3616                 list_del(&id_priv->listen_list);
3617                 list_del_init(&id_priv->list);
3618                 atomic_inc(&id_priv->refcount);
3619                 mutex_unlock(&lock);
3620
3621                 ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv);
3622                 cma_deref_id(id_priv);
3623                 if (ret)
3624                         rdma_destroy_id(&id_priv->id);
3625
3626                 mutex_lock(&lock);
3627         }
3628         mutex_unlock(&lock);
3629
3630         cma_deref_dev(cma_dev);
3631         wait_for_completion(&cma_dev->comp);
3632 }
3633
3634 static void cma_remove_one(struct ib_device *device)
3635 {
3636         struct cma_device *cma_dev;
3637
3638         cma_dev = ib_get_client_data(device, &cma_client);
3639         if (!cma_dev)
3640                 return;
3641
3642         mutex_lock(&lock);
3643         list_del(&cma_dev->list);
3644         mutex_unlock(&lock);
3645
3646         cma_process_remove(cma_dev);
3647         kfree(cma_dev);
3648 }
3649
3650 static int __init cma_init(void)
3651 {
3652         int ret = -ENOMEM;
3653
3654         cma_wq = create_singlethread_workqueue("rdma_cm");
3655         if (!cma_wq)
3656                 return -ENOMEM;
3657
3658         cma_free_wq = create_singlethread_workqueue("rdma_cm_fr");
3659         if (!cma_free_wq)
3660                 goto err1;
3661
3662         ib_sa_register_client(&sa_client);
3663         rdma_addr_register_client(&addr_client);
3664         register_netdevice_notifier(&cma_nb);
3665
3666         ret = ib_register_client(&cma_client);
3667         if (ret)
3668                 goto err;
3669
3670         return 0;
3671
3672 err:
3673         unregister_netdevice_notifier(&cma_nb);
3674         rdma_addr_unregister_client(&addr_client);
3675         ib_sa_unregister_client(&sa_client);
3676
3677         destroy_workqueue(cma_free_wq);
3678 err1:
3679         destroy_workqueue(cma_wq);
3680         return ret;
3681 }
3682
3683 static void __exit cma_cleanup(void)
3684 {
3685         ib_unregister_client(&cma_client);
3686         unregister_netdevice_notifier(&cma_nb);
3687         rdma_addr_unregister_client(&addr_client);
3688         ib_sa_unregister_client(&sa_client);
3689         flush_workqueue(cma_free_wq);
3690         destroy_workqueue(cma_free_wq);
3691         destroy_workqueue(cma_wq);
3692         idr_destroy(&sdp_ps);
3693         idr_destroy(&tcp_ps);
3694         idr_destroy(&udp_ps);
3695         idr_destroy(&ipoib_ps);
3696         idr_destroy(&ib_ps);
3697 }
3698
3699 module_init(cma_init);
3700 module_exit(cma_cleanup);