sys/ofed/drivers/infiniband/core/multicast.c

   1 /*
   2  * Copyright (c) 2006 Intel Corporation.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <linux/completion.h>
  34 #include <linux/dma-mapping.h>
  35 #include <linux/err.h>
  36 #include <linux/interrupt.h>
  37 #include <linux/module.h>
  38 #include <linux/slab.h>
  39 #include <linux/bitops.h>
  40 #include <linux/random.h>
  41 #include <linux/moduleparam.h>
  42 #include <linux/rbtree.h>
  43
  44 #include <rdma/ib_cache.h>
  45 #include "sa.h"
  46
  47 static int mcast_leave_retries = 3;
  48
  49 /*static const struct kernel_param_ops retry_ops = {
  50         .set = param_set_int,
  51         .get = param_get_int,
  52 };
  53
  54 module_param_cb(mcast_leave_retries, &retry_ops, &mcast_leave_retries, 0644);
  55 MODULE_PARM_DESC(mcast_leave_retries, "Number of retries for multicast leave "
  56                                       "requests before giving up (default: 3)");
  57 */
  58 static void mcast_add_one(struct ib_device *device);
  59 static void mcast_remove_one(struct ib_device *device);
  60
  61 static struct ib_client mcast_client = {
  62         .name   = "ib_multicast",
  63         .add    = mcast_add_one,
  64         .remove = mcast_remove_one
  65 };
  66
  67 static struct ib_sa_client      sa_client;
  68 static struct workqueue_struct  *mcast_wq;
  69 static union ib_gid mgid0;
  70
  71 struct mcast_device;
  72
  73 struct mcast_port {
  74         struct mcast_device     *dev;
  75         spinlock_t              lock;
  76         struct rb_root          table;
  77         atomic_t                refcount;
  78         struct completion       comp;
  79         u8                      port_num;
  80 };
  81
  82 struct mcast_device {
  83         struct ib_device        *device;
  84         struct ib_event_handler event_handler;
  85         int                     start_port;
  86         int                     end_port;
  87         struct mcast_port       port[0];
  88 };
  89
  90 enum mcast_state {
  91         MCAST_JOINING,
  92         MCAST_MEMBER,
  93         MCAST_ERROR,
  94 };
  95
  96 enum mcast_group_state {
  97         MCAST_IDLE,
  98         MCAST_BUSY,
  99         MCAST_GROUP_ERROR,
 100         MCAST_PKEY_EVENT
 101 };
 102
 103 enum {
 104         MCAST_INVALID_PKEY_INDEX = 0xFFFF
 105 };
 106
 107 struct mcast_member;
 108
 109 struct mcast_group {
 110         struct ib_sa_mcmember_rec rec;
 111         struct rb_node          node;
 112         struct mcast_port       *port;
 113         spinlock_t              lock;
 114         struct work_struct      work;
 115         struct list_head        pending_list;
 116         struct list_head        active_list;
 117         struct mcast_member     *last_join;
 118         int                     members[3];
 119         atomic_t                refcount;
 120         enum mcast_group_state  state;
 121         struct ib_sa_query      *query;
 122         int                     query_id;
 123         u16                     pkey_index;
 124         u8                      leave_state;
 125         int                     retries;
 126 };
 127
 128 struct mcast_member {
 129         struct ib_sa_multicast  multicast;
 130         struct ib_sa_client     *client;
 131         struct mcast_group      *group;
 132         struct list_head        list;
 133         enum mcast_state        state;
 134         atomic_t                refcount;
 135         struct completion       comp;
 136 };
 137
 138 static void join_handler(int status, struct ib_sa_mcmember_rec *rec,
 139                          void *context);
 140 static void leave_handler(int status, struct ib_sa_mcmember_rec *rec,
 141                           void *context);
 142
 143 static struct mcast_group *mcast_find(struct mcast_port *port,
 144                                       union ib_gid *mgid)
 145 {
 146         struct rb_node *node = port->table.rb_node;
 147         struct mcast_group *group;
 148         int ret;
 149
 150         while (node) {
 151                 group = rb_entry(node, struct mcast_group, node);
 152                 ret = memcmp(mgid->raw, group->rec.mgid.raw, sizeof *mgid);
 153                 if (!ret)
 154                         return group;
 155
 156                 if (ret < 0)
 157                         node = node->rb_left;
 158                 else
 159                         node = node->rb_right;
 160         }
 161         return NULL;
 162 }
 163
 164 static struct mcast_group *mcast_insert(struct mcast_port *port,
 165                                         struct mcast_group *group,
 166                                         int allow_duplicates)
 167 {
 168         struct rb_node **link = &port->table.rb_node;
 169         struct rb_node *parent = NULL;
 170         struct mcast_group *cur_group;
 171         int ret;
 172
 173         while (*link) {
 174                 parent = *link;
 175                 cur_group = rb_entry(parent, struct mcast_group, node);
 176
 177                 ret = memcmp(group->rec.mgid.raw, cur_group->rec.mgid.raw,
 178                              sizeof group->rec.mgid);
 179                 if (ret < 0)
 180                         link = &(*link)->rb_left;
 181                 else if (ret > 0)
 182                         link = &(*link)->rb_right;
 183                 else if (allow_duplicates)
 184                         link = &(*link)->rb_left;
 185                 else
 186                         return cur_group;
 187         }
 188         rb_link_node(&group->node, parent, link);
 189         rb_insert_color(&group->node, &port->table);
 190         return NULL;
 191 }
 192
 193 static void deref_port(struct mcast_port *port)
 194 {
 195         if (atomic_dec_and_test(&port->refcount))
 196                 complete(&port->comp);
 197 }
 198
 199 static void release_group(struct mcast_group *group)
 200 {
 201         struct mcast_port *port = group->port;
 202         unsigned long flags;
 203
 204         spin_lock_irqsave(&port->lock, flags);
 205         if (atomic_dec_and_test(&group->refcount)) {
 206                 rb_erase(&group->node, &port->table);
 207                 spin_unlock_irqrestore(&port->lock, flags);
 208                 kfree(group);
 209                 deref_port(port);
 210         } else
 211                 spin_unlock_irqrestore(&port->lock, flags);
 212 }
 213
 214 static void deref_member(struct mcast_member *member)
 215 {
 216         if (atomic_dec_and_test(&member->refcount))
 217                 complete(&member->comp);
 218 }
 219
 220 static void queue_join(struct mcast_member *member)
 221 {
 222         struct mcast_group *group = member->group;
 223         unsigned long flags;
 224
 225         spin_lock_irqsave(&group->lock, flags);
 226         list_add_tail(&member->list, &group->pending_list);
 227         if (group->state == MCAST_IDLE) {
 228                 group->state = MCAST_BUSY;
 229                 atomic_inc(&group->refcount);
 230                 queue_work(mcast_wq, &group->work);
 231         }
 232         spin_unlock_irqrestore(&group->lock, flags);
 233 }
 234
 235 /*
 236  * A multicast group has three types of members: full member, non member, and
 237  * send only member.  We need to keep track of the number of members of each
 238  * type based on their join state.  Adjust the number of members the belong to
 239  * the specified join states.
 240  */
 241 static void adjust_membership(struct mcast_group *group, u8 join_state, int inc)
 242 {
 243         int i;
 244
 245         for (i = 0; i < 3; i++, join_state >>= 1)
 246                 if (join_state & 0x1)
 247                         group->members[i] += inc;
 248 }
 249
 250 /*
 251  * If a multicast group has zero members left for a particular join state, but
 252  * the group is still a member with the SA, we need to leave that join state.
 253  * Determine which join states we still belong to, but that do not have any
 254  * active members.
 255  */
 256 static u8 get_leave_state(struct mcast_group *group)
 257 {
 258         u8 leave_state = 0;
 259         int i;
 260
 261         for (i = 0; i < 3; i++)
 262                 if (!group->members[i])
 263                         leave_state |= (0x1 << i);
 264
 265         return leave_state & group->rec.join_state;
 266 }
 267
 268 static int check_selector(ib_sa_comp_mask comp_mask,
 269                           ib_sa_comp_mask selector_mask,
 270                           ib_sa_comp_mask value_mask,
 271                           u8 selector, u8 src_value, u8 dst_value)
 272 {
 273         int err;
 274
 275         if (!(comp_mask & selector_mask) || !(comp_mask & value_mask))
 276                 return 0;
 277
 278         switch (selector) {
 279         case IB_SA_GT:
 280                 err = (src_value <= dst_value);
 281                 break;
 282         case IB_SA_LT:
 283                 err = (src_value >= dst_value);
 284                 break;
 285         case IB_SA_EQ:
 286                 err = (src_value != dst_value);
 287                 break;
 288         default:
 289                 err = 0;
 290                 break;
 291         }
 292
 293         return err;
 294 }
 295
 296 static int cmp_rec(struct ib_sa_mcmember_rec *src,
 297                    struct ib_sa_mcmember_rec *dst, ib_sa_comp_mask comp_mask)
 298 {
 299         /* MGID must already match */
 300
 301         if (comp_mask & IB_SA_MCMEMBER_REC_PORT_GID &&
 302             memcmp(&src->port_gid, &dst->port_gid, sizeof src->port_gid))
 303                 return -EINVAL;
 304         if (comp_mask & IB_SA_MCMEMBER_REC_QKEY && src->qkey != dst->qkey)
 305                 return -EINVAL;
 306         if (comp_mask & IB_SA_MCMEMBER_REC_MLID && src->mlid != dst->mlid)
 307                 return -EINVAL;
 308         if (check_selector(comp_mask, IB_SA_MCMEMBER_REC_MTU_SELECTOR,
 309                                  IB_SA_MCMEMBER_REC_MTU, dst->mtu_selector,
 310                                  src->mtu, dst->mtu))
 311                 return -EINVAL;
 312         if (comp_mask & IB_SA_MCMEMBER_REC_TRAFFIC_CLASS &&
 313             src->traffic_class != dst->traffic_class)
 314                 return -EINVAL;
 315         if (comp_mask & IB_SA_MCMEMBER_REC_PKEY && src->pkey != dst->pkey)
 316                 return -EINVAL;
 317         if (check_selector(comp_mask, IB_SA_MCMEMBER_REC_RATE_SELECTOR,
 318                                  IB_SA_MCMEMBER_REC_RATE, dst->rate_selector,
 319                                  src->rate, dst->rate))
 320                 return -EINVAL;
 321         if (check_selector(comp_mask,
 322                                  IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME_SELECTOR,
 323                                  IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME,
 324                                  dst->packet_life_time_selector,
 325                                  src->packet_life_time, dst->packet_life_time))
 326                 return -EINVAL;
 327         if (comp_mask & IB_SA_MCMEMBER_REC_SL && src->sl != dst->sl)
 328                 return -EINVAL;
 329         if (comp_mask & IB_SA_MCMEMBER_REC_FLOW_LABEL &&
 330             src->flow_label != dst->flow_label)
 331                 return -EINVAL;
 332         if (comp_mask & IB_SA_MCMEMBER_REC_HOP_LIMIT &&
 333             src->hop_limit != dst->hop_limit)
 334                 return -EINVAL;
 335         if (comp_mask & IB_SA_MCMEMBER_REC_SCOPE && src->scope != dst->scope)
 336                 return -EINVAL;
 337
 338         /* join_state checked separately, proxy_join ignored */
 339
 340         return 0;
 341 }
 342
 343 static int send_join(struct mcast_group *group, struct mcast_member *member)
 344 {
 345         struct mcast_port *port = group->port;
 346         int ret;
 347
 348         group->last_join = member;
 349         ret = ib_sa_mcmember_rec_query(&sa_client, port->dev->device,
 350                                        port->port_num, IB_MGMT_METHOD_SET,
 351                                        &member->multicast.rec,
 352                                        member->multicast.comp_mask,
 353                                        3000, GFP_KERNEL, join_handler, group,
 354                                        &group->query);
 355         if (ret >= 0) {
 356                 group->query_id = ret;
 357                 ret = 0;
 358         }
 359         return ret;
 360 }
 361
 362 static int send_leave(struct mcast_group *group, u8 leave_state)
 363 {
 364         struct mcast_port *port = group->port;
 365         struct ib_sa_mcmember_rec rec;
 366         int ret;
 367
 368         rec = group->rec;
 369         rec.join_state = leave_state;
 370         group->leave_state = leave_state;
 371
 372         ret = ib_sa_mcmember_rec_query(&sa_client, port->dev->device,
 373                                        port->port_num, IB_SA_METHOD_DELETE, &rec,
 374                                        IB_SA_MCMEMBER_REC_MGID     |
 375                                        IB_SA_MCMEMBER_REC_PORT_GID |
 376                                        IB_SA_MCMEMBER_REC_JOIN_STATE,
 377                                        3000, GFP_KERNEL, leave_handler,
 378                                        group, &group->query);
 379         if (ret >= 0) {
 380                 group->query_id = ret;
 381                 ret = 0;
 382         }
 383         return ret;
 384 }
 385
 386 static void join_group(struct mcast_group *group, struct mcast_member *member,
 387                        u8 join_state)
 388 {
 389         member->state = MCAST_MEMBER;
 390         adjust_membership(group, join_state, 1);
 391         group->rec.join_state |= join_state;
 392         member->multicast.rec = group->rec;
 393         member->multicast.rec.join_state = join_state;
 394         list_move(&member->list, &group->active_list);
 395 }
 396
 397 static int fail_join(struct mcast_group *group, struct mcast_member *member,
 398                      int status)
 399 {
 400         spin_lock_irq(&group->lock);
 401         list_del_init(&member->list);
 402         spin_unlock_irq(&group->lock);
 403         return member->multicast.callback(status, &member->multicast);
 404 }
 405
 406 static void process_group_error(struct mcast_group *group)
 407 {
 408         struct mcast_member *member;
 409         int ret = 0;
 410         u16 pkey_index;
 411
 412         if (group->state == MCAST_PKEY_EVENT)
 413                 ret = ib_find_pkey(group->port->dev->device,
 414                                    group->port->port_num,
 415                                    be16_to_cpu(group->rec.pkey), &pkey_index);
 416
 417         spin_lock_irq(&group->lock);
 418         if (group->state == MCAST_PKEY_EVENT && !ret &&
 419             group->pkey_index == pkey_index)
 420                 goto out;
 421
 422         while (!list_empty(&group->active_list)) {
 423                 member = list_entry(group->active_list.next,
 424                                     struct mcast_member, list);
 425                 atomic_inc(&member->refcount);
 426                 list_del_init(&member->list);
 427                 adjust_membership(group, member->multicast.rec.join_state, -1);
 428                 member->state = MCAST_ERROR;
 429                 spin_unlock_irq(&group->lock);
 430
 431                 ret = member->multicast.callback(-ENETRESET,
 432                                                  &member->multicast);
 433                 deref_member(member);
 434                 if (ret)
 435                         ib_sa_free_multicast(&member->multicast);
 436                 spin_lock_irq(&group->lock);
 437         }
 438
 439         group->rec.join_state = 0;
 440 out:
 441         group->state = MCAST_BUSY;
 442         spin_unlock_irq(&group->lock);
 443 }
 444
 445 static void mcast_work_handler(struct work_struct *work)
 446 {
 447         struct mcast_group *group;
 448         struct mcast_member *member;
 449         struct ib_sa_multicast *multicast;
 450         int status, ret;
 451         u8 join_state;
 452
 453         group = container_of(work, typeof(*group), work);
 454 retest:
 455         spin_lock_irq(&group->lock);
 456         while (!list_empty(&group->pending_list) ||
 457                (group->state != MCAST_BUSY)) {
 458
 459                 if (group->state != MCAST_BUSY) {
 460                         spin_unlock_irq(&group->lock);
 461                         process_group_error(group);
 462                         goto retest;
 463                 }
 464
 465                 member = list_entry(group->pending_list.next,
 466                                     struct mcast_member, list);
 467                 multicast = &member->multicast;
 468                 join_state = multicast->rec.join_state;
 469                 atomic_inc(&member->refcount);
 470
 471                 if (join_state == (group->rec.join_state & join_state)) {
 472                         status = cmp_rec(&group->rec, &multicast->rec,
 473                                          multicast->comp_mask);
 474                         if (!status)
 475                                 join_group(group, member, join_state);
 476                         else
 477                                 list_del_init(&member->list);
 478                         spin_unlock_irq(&group->lock);
 479                         ret = multicast->callback(status, multicast);
 480                 } else {
 481                         spin_unlock_irq(&group->lock);
 482                         status = send_join(group, member);
 483                         if (!status) {
 484                                 deref_member(member);
 485                                 return;
 486                         }
 487                         ret = fail_join(group, member, status);
 488                 }
 489
 490                 deref_member(member);
 491                 if (ret)
 492                         ib_sa_free_multicast(&member->multicast);
 493                 spin_lock_irq(&group->lock);
 494         }
 495
 496         join_state = get_leave_state(group);
 497         if (join_state) {
 498                 group->rec.join_state &= ~join_state;
 499                 spin_unlock_irq(&group->lock);
 500                 if (send_leave(group, join_state))
 501                         goto retest;
 502         } else {
 503                 group->state = MCAST_IDLE;
 504                 spin_unlock_irq(&group->lock);
 505                 release_group(group);
 506         }
 507 }
 508
 509 /*
 510  * Fail a join request if it is still active - at the head of the pending queue.
 511  */
 512 static void process_join_error(struct mcast_group *group, int status)
 513 {
 514         struct mcast_member *member;
 515         int ret;
 516
 517         spin_lock_irq(&group->lock);
 518         member = list_entry(group->pending_list.next,
 519                             struct mcast_member, list);
 520         if (group->last_join == member) {
 521                 atomic_inc(&member->refcount);
 522                 list_del_init(&member->list);
 523                 spin_unlock_irq(&group->lock);
 524                 ret = member->multicast.callback(status, &member->multicast);
 525                 deref_member(member);
 526                 if (ret)
 527                         ib_sa_free_multicast(&member->multicast);
 528         } else
 529                 spin_unlock_irq(&group->lock);
 530 }
 531
 532 static void join_handler(int status, struct ib_sa_mcmember_rec *rec,
 533                          void *context)
 534 {
 535         struct mcast_group *group = context;
 536         u16 pkey_index = MCAST_INVALID_PKEY_INDEX;
 537
 538         if (status)
 539                 process_join_error(group, status);
 540         else {
 541                 ib_find_pkey(group->port->dev->device, group->port->port_num,
 542                              be16_to_cpu(rec->pkey), &pkey_index);
 543
 544                 spin_lock_irq(&group->port->lock);
 545                 group->rec = *rec;
 546                 if (group->state == MCAST_BUSY &&
 547                     group->pkey_index == MCAST_INVALID_PKEY_INDEX)
 548                         group->pkey_index = pkey_index;
 549                 if (!memcmp(&mgid0, &group->rec.mgid, sizeof mgid0)) {
 550                         rb_erase(&group->node, &group->port->table);
 551                         mcast_insert(group->port, group, 1);
 552                 }
 553                 spin_unlock_irq(&group->port->lock);
 554         }
 555         mcast_work_handler(&group->work);
 556 }
 557
 558 static void leave_handler(int status, struct ib_sa_mcmember_rec *rec,
 559                           void *context)
 560 {
 561         struct mcast_group *group = context;
 562
 563         if (status && group->retries > 0 &&
 564             !send_leave(group, group->leave_state))
 565                 group->retries--;
 566         else {
 567                 if (status && group->retries <= 0)
 568                         printk(KERN_WARNING "reached max retry count. "
 569                                "status=%d. Giving up\n", status);
 570                 mcast_work_handler(&group->work);
 571         }
 572 }
 573
 574 static struct mcast_group *acquire_group(struct mcast_port *port,
 575                                          union ib_gid *mgid, gfp_t gfp_mask)
 576 {
 577         struct mcast_group *group, *cur_group;
 578         unsigned long flags;
 579         int is_mgid0;
 580
 581         is_mgid0 = !memcmp(&mgid0, mgid, sizeof mgid0);
 582         if (!is_mgid0) {
 583                 spin_lock_irqsave(&port->lock, flags);
 584                 group = mcast_find(port, mgid);
 585                 if (group)
 586                         goto found;
 587                 spin_unlock_irqrestore(&port->lock, flags);
 588         }
 589
 590         group = kzalloc(sizeof *group, gfp_mask);
 591         if (!group)
 592                 return NULL;
 593
 594         group->retries = mcast_leave_retries;
 595         group->port = port;
 596         group->rec.mgid = *mgid;
 597         group->pkey_index = MCAST_INVALID_PKEY_INDEX;
 598         INIT_LIST_HEAD(&group->pending_list);
 599         INIT_LIST_HEAD(&group->active_list);
 600         INIT_WORK(&group->work, mcast_work_handler);
 601         spin_lock_init(&group->lock);
 602
 603         spin_lock_irqsave(&port->lock, flags);
 604         cur_group = mcast_insert(port, group, is_mgid0);
 605         if (cur_group) {
 606                 kfree(group);
 607                 group = cur_group;
 608         } else
 609                 atomic_inc(&port->refcount);
 610 found:
 611         atomic_inc(&group->refcount);
 612         spin_unlock_irqrestore(&port->lock, flags);
 613         return group;
 614 }
 615
 616 /*
 617  * We serialize all join requests to a single group to make our lives much
 618  * easier.  Otherwise, two users could try to join the same group
 619  * simultaneously, with different configurations, one could leave while the
 620  * join is in progress, etc., which makes locking around error recovery
 621  * difficult.
 622  */
 623 struct ib_sa_multicast *
 624 ib_sa_join_multicast(struct ib_sa_client *client,
 625                      struct ib_device *device, u8 port_num,
 626                      struct ib_sa_mcmember_rec *rec,
 627                      ib_sa_comp_mask comp_mask, gfp_t gfp_mask,
 628                      int (*callback)(int status,
 629                                      struct ib_sa_multicast *multicast),
 630                      void *context)
 631 {
 632         struct mcast_device *dev;
 633         struct mcast_member *member;
 634         struct ib_sa_multicast *multicast;
 635         int ret;
 636
 637         dev = ib_get_client_data(device, &mcast_client);
 638         if (!dev)
 639                 return ERR_PTR(-ENODEV);
 640
 641         member = kmalloc(sizeof *member, gfp_mask);
 642         if (!member)
 643                 return ERR_PTR(-ENOMEM);
 644
 645         ib_sa_client_get(client);
 646         member->client = client;
 647         member->multicast.rec = *rec;
 648         member->multicast.comp_mask = comp_mask;
 649         member->multicast.callback = callback;
 650         member->multicast.context = context;
 651         init_completion(&member->comp);
 652         atomic_set(&member->refcount, 1);
 653         member->state = MCAST_JOINING;
 654
 655         member->group = acquire_group(&dev->port[port_num - dev->start_port],
 656                                       &rec->mgid, gfp_mask);
 657         if (!member->group) {
 658                 ret = -ENOMEM;
 659                 goto err;
 660         }
 661
 662         /*
 663          * The user will get the multicast structure in their callback.  They
 664          * could then free the multicast structure before we can return from
 665          * this routine.  So we save the pointer to return before queuing
 666          * any callback.
 667          */
 668         multicast = &member->multicast;
 669         queue_join(member);
 670         return multicast;
 671
 672 err:
 673         ib_sa_client_put(client);
 674         kfree(member);
 675         return ERR_PTR(ret);
 676 }
 677 EXPORT_SYMBOL(ib_sa_join_multicast);
 678
 679 void ib_sa_free_multicast(struct ib_sa_multicast *multicast)
 680 {
 681         struct mcast_member *member;
 682         struct mcast_group *group;
 683
 684         member = container_of(multicast, struct mcast_member, multicast);
 685         group = member->group;
 686
 687         spin_lock_irq(&group->lock);
 688         if (member->state == MCAST_MEMBER)
 689                 adjust_membership(group, multicast->rec.join_state, -1);
 690
 691         list_del_init(&member->list);
 692
 693         if (group->state == MCAST_IDLE) {
 694                 group->state = MCAST_BUSY;
 695                 spin_unlock_irq(&group->lock);
 696                 /* Continue to hold reference on group until callback */
 697                 queue_work(mcast_wq, &group->work);
 698         } else {
 699                 spin_unlock_irq(&group->lock);
 700                 release_group(group);
 701         }
 702
 703         deref_member(member);
 704         wait_for_completion(&member->comp);
 705         ib_sa_client_put(member->client);
 706         kfree(member);
 707 }
 708 EXPORT_SYMBOL(ib_sa_free_multicast);
 709
 710 int ib_sa_get_mcmember_rec(struct ib_device *device, u8 port_num,
 711                            union ib_gid *mgid, struct ib_sa_mcmember_rec *rec)
 712 {
 713         struct mcast_device *dev;
 714         struct mcast_port *port;
 715         struct mcast_group *group;
 716         unsigned long flags;
 717         int ret = 0;
 718
 719         dev = ib_get_client_data(device, &mcast_client);
 720         if (!dev)
 721                 return -ENODEV;
 722
 723         port = &dev->port[port_num - dev->start_port];
 724         spin_lock_irqsave(&port->lock, flags);
 725         group = mcast_find(port, mgid);
 726         if (group)
 727                 *rec = group->rec;
 728         else
 729                 ret = -EADDRNOTAVAIL;
 730         spin_unlock_irqrestore(&port->lock, flags);
 731
 732         return ret;
 733 }
 734 EXPORT_SYMBOL(ib_sa_get_mcmember_rec);
 735
 736 int ib_init_ah_from_mcmember(struct ib_device *device, u8 port_num,
 737                              struct ib_sa_mcmember_rec *rec,
 738                              struct ib_ah_attr *ah_attr)
 739 {
 740         int ret;
 741         u16 gid_index;
 742         u8 p;
 743
 744         ret = ib_find_cached_gid(device, &rec->port_gid, &p, &gid_index);
 745         if (ret)
 746                 return ret;
 747
 748         memset(ah_attr, 0, sizeof *ah_attr);
 749         ah_attr->dlid = be16_to_cpu(rec->mlid);
 750         ah_attr->sl = rec->sl;
 751         ah_attr->port_num = port_num;
 752         ah_attr->static_rate = rec->rate;
 753
 754         ah_attr->ah_flags = IB_AH_GRH;
 755         ah_attr->grh.dgid = rec->mgid;
 756
 757         ah_attr->grh.sgid_index = (u8) gid_index;
 758         ah_attr->grh.flow_label = be32_to_cpu(rec->flow_label);
 759         ah_attr->grh.hop_limit = rec->hop_limit;
 760         ah_attr->grh.traffic_class = rec->traffic_class;
 761
 762         return 0;
 763 }
 764 EXPORT_SYMBOL(ib_init_ah_from_mcmember);
 765
 766 static void mcast_groups_event(struct mcast_port *port,
 767                                enum mcast_group_state state)
 768 {
 769         struct mcast_group *group;
 770         struct rb_node *node;
 771         unsigned long flags;
 772
 773         spin_lock_irqsave(&port->lock, flags);
 774         for (node = rb_first(&port->table); node; node = rb_next(node)) {
 775                 group = rb_entry(node, struct mcast_group, node);
 776                 spin_lock(&group->lock);
 777                 if (group->state == MCAST_IDLE) {
 778                         atomic_inc(&group->refcount);
 779                         queue_work(mcast_wq, &group->work);
 780                 }
 781                 if (group->state != MCAST_GROUP_ERROR)
 782                         group->state = state;
 783                 spin_unlock(&group->lock);
 784         }
 785         spin_unlock_irqrestore(&port->lock, flags);
 786 }
 787
 788 static void mcast_event_handler(struct ib_event_handler *handler,
 789                                 struct ib_event *event)
 790 {
 791         struct mcast_device *dev;
 792         int index;
 793
 794         dev = container_of(handler, struct mcast_device, event_handler);
 795         if (rdma_port_get_link_layer(dev->device, event->element.port_num) !=
 796             IB_LINK_LAYER_INFINIBAND)
 797                 return;
 798
 799         index = event->element.port_num - dev->start_port;
 800
 801         switch (event->event) {
 802         case IB_EVENT_PORT_ERR:
 803         case IB_EVENT_LID_CHANGE:
 804         case IB_EVENT_CLIENT_REREGISTER:
 805                 mcast_groups_event(&dev->port[index], MCAST_GROUP_ERROR);
 806                 break;
 807         case IB_EVENT_PKEY_CHANGE:
 808                 mcast_groups_event(&dev->port[index], MCAST_PKEY_EVENT);
 809                 break;
 810         default:
 811                 break;
 812         }
 813 }
 814
 815 static void mcast_add_one(struct ib_device *device)
 816 {
 817         struct mcast_device *dev;
 818         struct mcast_port *port;
 819         int i;
 820         int count = 0;
 821
 822         if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB)
 823                 return;
 824
 825         dev = kmalloc(sizeof *dev + device->phys_port_cnt * sizeof *port,
 826                       GFP_KERNEL);
 827         if (!dev)
 828                 return;
 829
 830         if (device->node_type == RDMA_NODE_IB_SWITCH)
 831                 dev->start_port = dev->end_port = 0;
 832         else {
 833                 dev->start_port = 1;
 834                 dev->end_port = device->phys_port_cnt;
 835         }
 836
 837         for (i = 0; i <= dev->end_port - dev->start_port; i++) {
 838                 if (rdma_port_get_link_layer(device, dev->start_port + i) !=
 839                     IB_LINK_LAYER_INFINIBAND)
 840                         continue;
 841                 port = &dev->port[i];
 842                 port->dev = dev;
 843                 port->port_num = dev->start_port + i;
 844                 spin_lock_init(&port->lock);
 845                 port->table = RB_ROOT;
 846                 init_completion(&port->comp);
 847                 atomic_set(&port->refcount, 1);
 848                 ++count;
 849         }
 850
 851         if (!count) {
 852                 kfree(dev);
 853                 return;
 854         }
 855
 856         dev->device = device;
 857         ib_set_client_data(device, &mcast_client, dev);
 858
 859         INIT_IB_EVENT_HANDLER(&dev->event_handler, device, mcast_event_handler);
 860         ib_register_event_handler(&dev->event_handler);
 861 }
 862
 863 static void mcast_remove_one(struct ib_device *device)
 864 {
 865         struct mcast_device *dev;
 866         struct mcast_port *port;
 867         int i;
 868
 869         dev = ib_get_client_data(device, &mcast_client);
 870         if (!dev)
 871                 return;
 872
 873         ib_unregister_event_handler(&dev->event_handler);
 874         flush_workqueue(mcast_wq);
 875
 876         for (i = 0; i <= dev->end_port - dev->start_port; i++) {
 877                 if (rdma_port_get_link_layer(device, dev->start_port + i) ==
 878                     IB_LINK_LAYER_INFINIBAND) {
 879                         port = &dev->port[i];
 880                         deref_port(port);
 881                         wait_for_completion(&port->comp);
 882                 }
 883         }
 884
 885         kfree(dev);
 886 }
 887
 888 int mcast_init(void)
 889 {
 890         int ret;
 891
 892         mcast_wq = create_singlethread_workqueue("ib_mcast");
 893         if (!mcast_wq)
 894                 return -ENOMEM;
 895
 896         ib_sa_register_client(&sa_client);
 897
 898         ret = ib_register_client(&mcast_client);
 899         if (ret)
 900                 goto err;
 901         return 0;
 902
 903 err:
 904         ib_sa_unregister_client(&sa_client);
 905         destroy_workqueue(mcast_wq);
 906         return ret;
 907 }
 908
 909 void mcast_cleanup(void)
 910 {
 911         ib_unregister_client(&mcast_client);
 912         ib_sa_unregister_client(&sa_client);
 913         destroy_workqueue(mcast_wq);
 914 }