contrib/ofed/opensm/opensm/osm_vl15intf.c

   1 /*
   2  * Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
   3  * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved.
   4  * Copyright (c) 2002-2010 Mellanox Technologies LTD. All rights reserved.
   5  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * OpenIB.org BSD license below:
  12  *
  13  *     Redistribution and use in source and binary forms, with or
  14  *     without modification, are permitted provided that the following
  15  *     conditions are met:
  16  *
  17  *      - Redistributions of source code must retain the above
  18  *        copyright notice, this list of conditions and the following
  19  *        disclaimer.
  20  *
  21  *      - Redistributions in binary form must reproduce the above
  22  *        copyright notice, this list of conditions and the following
  23  *        disclaimer in the documentation and/or other materials
  24  *        provided with the distribution.
  25  *
  26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  33  * SOFTWARE.
  34  *
  35  */
  36
  37 /*
  38  * Abstract:
  39  *    Implementation of osm_vl15_t.
  40  * This object represents the VL15 Interface object.
  41  * This object is part of the opensm family of objects.
  42  */
  43
  44 #if HAVE_CONFIG_H
  45 #  include <config.h>
  46 #endif                          /* HAVE_CONFIG_H */
  47
  48 #include <string.h>
  49 #include <iba/ib_types.h>
  50 #include <complib/cl_thread.h>
  51 #include <opensm/osm_file_ids.h>
  52 #define FILE_ID OSM_FILE_VL15INTF_C
  53 #include <vendor/osm_vendor_api.h>
  54 #include <opensm/osm_vl15intf.h>
  55 #include <opensm/osm_madw.h>
  56 #include <opensm/osm_log.h>
  57 #include <opensm/osm_helper.h>
  58
  59 static void vl15_send_mad(osm_vl15_t * p_vl, osm_madw_t * p_madw)
  60 {
  61         ib_api_status_t status;
  62         boolean_t resp_expected = p_madw->resp_expected;
  63         ib_smp_t * p_smp;
  64         ib_net16_t attr_id;
  65         uint8_t method;
  66
  67         p_smp = osm_madw_get_smp_ptr(p_madw);
  68         method = p_smp->method;
  69         attr_id = p_smp->attr_id;
  70
  71         /*
  72            Non-response-expected mads are not throttled on the wire
  73            since we can have no confirmation that they arrived
  74            at their destination.
  75          */
  76         if (resp_expected)
  77                 /*
  78                    Note that other threads may not see the response MAD
  79                    arrive before send() even returns.
  80                    In that case, the wire count would temporarily go negative.
  81                    To avoid this confusion, preincrement the counts on the
  82                    assumption that send() will succeed.
  83                  */
  84                 cl_atomic_inc(&p_vl->p_stats->qp0_mads_outstanding_on_wire);
  85         else
  86                 cl_atomic_inc(&p_vl->p_stats->qp0_unicasts_sent);
  87
  88         cl_atomic_inc(&p_vl->p_stats->qp0_mads_sent);
  89
  90         status = osm_vendor_send(osm_madw_get_bind_handle(p_madw),
  91                                  p_madw, p_madw->resp_expected);
  92
  93         if (status == IB_SUCCESS) {
  94                 OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
  95                         "%u QP0 MADs on wire, %u outstanding, "
  96                         "%u unicasts sent, %u total sent\n",
  97                         p_vl->p_stats->qp0_mads_outstanding_on_wire,
  98                         p_vl->p_stats->qp0_mads_outstanding,
  99                         p_vl->p_stats->qp0_unicasts_sent,
 100                         p_vl->p_stats->qp0_mads_sent);
 101                 return;
 102         }
 103
 104         OSM_LOG(p_vl->p_log, OSM_LOG_ERROR, "ERR 3E03: "
 105                 "MAD send failed (%s)\n", ib_get_err_str(status));
 106
 107         /*
 108            The MAD was never successfully sent, so
 109            fix up the pre-incremented count values.
 110          */
 111
 112         /* Decrement qp0_mads_sent that were incremented in the code above.
 113            qp0_mads_outstanding will be decremented by send error callback
 114            (called by osm_vendor_send() */
 115         cl_atomic_dec(&p_vl->p_stats->qp0_mads_sent);
 116         if (!resp_expected) {
 117                 cl_atomic_dec(&p_vl->p_stats->qp0_unicasts_sent);
 118                 return;
 119         }
 120
 121         /* need to cause heavy-sweep if resp_expected MAD sending failed */
 122         OSM_LOG(p_vl->p_log, OSM_LOG_ERROR, "ERR 3E04: "
 123                 "%s method failed for attribute 0x%X (%s)\n",
 124                 method == IB_MAD_METHOD_SET ? "SET" : "GET",
 125                 cl_ntoh16(attr_id), ib_get_sm_attr_str(attr_id));
 126
 127         p_vl->p_subn->subnet_initialization_error = TRUE;
 128
 129 }
 130
 131 static void vl15_poller(IN void *p_ptr)
 132 {
 133         ib_api_status_t status;
 134         osm_madw_t *p_madw;
 135         osm_vl15_t *p_vl = p_ptr;
 136         cl_qlist_t *p_fifo;
 137         int32_t max_smps = p_vl->max_wire_smps;
 138         int32_t max_smps2 = p_vl->max_wire_smps2;
 139
 140         OSM_LOG_ENTER(p_vl->p_log);
 141
 142         if (p_vl->thread_state == OSM_THREAD_STATE_NONE)
 143                 p_vl->thread_state = OSM_THREAD_STATE_RUN;
 144
 145         while (p_vl->thread_state == OSM_THREAD_STATE_RUN) {
 146                 /*
 147                    Start servicing the FIFOs by pulling off MAD wrappers
 148                    and passing them to the transport interface.
 149                    There are lots of corner cases here so tread carefully.
 150
 151                    The unicast FIFO has priority, since somebody is waiting
 152                    for a timely response.
 153                  */
 154                 cl_spinlock_acquire(&p_vl->lock);
 155
 156                 if (cl_qlist_count(&p_vl->ufifo) != 0)
 157                         p_fifo = &p_vl->ufifo;
 158                 else
 159                         p_fifo = &p_vl->rfifo;
 160
 161                 p_madw = (osm_madw_t *) cl_qlist_remove_head(p_fifo);
 162
 163                 cl_spinlock_release(&p_vl->lock);
 164
 165                 if (p_madw != (osm_madw_t *) cl_qlist_end(p_fifo)) {
 166                         OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
 167                                 "Servicing p_madw = %p\n", p_madw);
 168                         if (OSM_LOG_IS_ACTIVE_V2(p_vl->p_log, OSM_LOG_FRAMES))
 169                                 osm_dump_dr_smp_v2(p_vl->p_log,
 170                                                    osm_madw_get_smp_ptr(p_madw),
 171                                                    FILE_ID, OSM_LOG_FRAMES);
 172
 173                         vl15_send_mad(p_vl, p_madw);
 174                 } else
 175                         /*
 176                            The VL15 FIFO is empty, so we have nothing left to do.
 177                          */
 178                         status = cl_event_wait_on(&p_vl->signal,
 179                                                   EVENT_NO_TIMEOUT, TRUE);
 180
 181                 while (p_vl->p_stats->qp0_mads_outstanding_on_wire >= max_smps &&
 182                        p_vl->thread_state == OSM_THREAD_STATE_RUN) {
 183                         status = cl_event_wait_on(&p_vl->signal,
 184                                                   p_vl->max_smps_timeout,
 185                                                   TRUE);
 186                         if (status == CL_TIMEOUT) {
 187                                 if (max_smps < max_smps2)
 188                                         max_smps++;
 189                                 break;
 190                         } else if (status != CL_SUCCESS) {
 191                                 OSM_LOG(p_vl->p_log, OSM_LOG_ERROR, "ERR 3E02: "
 192                                         "Event wait failed (%s)\n",
 193                                         CL_STATUS_MSG(status));
 194                                 break;
 195                         }
 196                         max_smps = p_vl->max_wire_smps;
 197                 }
 198         }
 199
 200         /*
 201            since we abort immediately when the state != OSM_THREAD_STATE_RUN
 202            we might have some mads on the queues. After the thread exits
 203            the vl15 destroy routine should put these mads back...
 204          */
 205
 206         OSM_LOG_EXIT(p_vl->p_log);
 207 }
 208
 209 void osm_vl15_construct(IN osm_vl15_t * p_vl)
 210 {
 211         memset(p_vl, 0, sizeof(*p_vl));
 212         p_vl->state = OSM_VL15_STATE_INIT;
 213         p_vl->thread_state = OSM_THREAD_STATE_NONE;
 214         cl_event_construct(&p_vl->signal);
 215         cl_spinlock_construct(&p_vl->lock);
 216         cl_qlist_init(&p_vl->rfifo);
 217         cl_qlist_init(&p_vl->ufifo);
 218         cl_thread_construct(&p_vl->poller);
 219 }
 220
 221 void osm_vl15_destroy(IN osm_vl15_t * p_vl, IN struct osm_mad_pool *p_pool)
 222 {
 223         osm_madw_t *p_madw;
 224
 225         OSM_LOG_ENTER(p_vl->p_log);
 226
 227         /*
 228            Signal our threads that we're leaving.
 229          */
 230         p_vl->thread_state = OSM_THREAD_STATE_EXIT;
 231
 232         /*
 233            Don't trigger unless event has been initialized.
 234            Destroy the thread before we tear down the other objects.
 235          */
 236         if (p_vl->state != OSM_VL15_STATE_INIT)
 237                 cl_event_signal(&p_vl->signal);
 238
 239         cl_thread_destroy(&p_vl->poller);
 240
 241         /*
 242            Return the outstanding messages to the pool
 243          */
 244
 245         cl_spinlock_acquire(&p_vl->lock);
 246
 247         while (!cl_is_qlist_empty(&p_vl->rfifo)) {
 248                 p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->rfifo);
 249                 osm_mad_pool_put(p_pool, p_madw);
 250         }
 251         while (!cl_is_qlist_empty(&p_vl->ufifo)) {
 252                 p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->ufifo);
 253                 osm_mad_pool_put(p_pool, p_madw);
 254         }
 255
 256         cl_spinlock_release(&p_vl->lock);
 257
 258         cl_event_destroy(&p_vl->signal);
 259         p_vl->state = OSM_VL15_STATE_INIT;
 260         cl_spinlock_destroy(&p_vl->lock);
 261
 262         OSM_LOG_EXIT(p_vl->p_log);
 263 }
 264
 265 ib_api_status_t osm_vl15_init(IN osm_vl15_t * p_vl, IN osm_vendor_t * p_vend,
 266                               IN osm_log_t * p_log, IN osm_stats_t * p_stats,
 267                               IN osm_subn_t * p_subn,
 268                               IN int32_t max_wire_smps,
 269                               IN int32_t max_wire_smps2,
 270                               IN uint32_t max_smps_timeout)
 271 {
 272         ib_api_status_t status = IB_SUCCESS;
 273
 274         OSM_LOG_ENTER(p_log);
 275
 276         p_vl->p_vend = p_vend;
 277         p_vl->p_log = p_log;
 278         p_vl->p_stats = p_stats;
 279         p_vl->p_subn = p_subn;
 280         p_vl->max_wire_smps = max_wire_smps;
 281         p_vl->max_wire_smps2 = max_wire_smps2;
 282         p_vl->max_smps_timeout = max_wire_smps < max_wire_smps2 ?
 283                                  max_smps_timeout : EVENT_NO_TIMEOUT;
 284
 285         status = cl_event_init(&p_vl->signal, FALSE);
 286         if (status != IB_SUCCESS)
 287                 goto Exit;
 288
 289         p_vl->state = OSM_VL15_STATE_READY;
 290
 291         status = cl_spinlock_init(&p_vl->lock);
 292         if (status != IB_SUCCESS)
 293                 goto Exit;
 294
 295         /*
 296            Initialize the thread after all other dependent objects
 297            have been initialized.
 298          */
 299         status = cl_thread_init(&p_vl->poller, vl15_poller, p_vl,
 300                                 "opensm poller");
 301 Exit:
 302         OSM_LOG_EXIT(p_log);
 303         return status;
 304 }
 305
 306 void osm_vl15_poll(IN osm_vl15_t * p_vl)
 307 {
 308         OSM_LOG_ENTER(p_vl->p_log);
 309
 310         CL_ASSERT(p_vl->state == OSM_VL15_STATE_READY);
 311
 312         /*
 313            If we have room for more VL15 MADs on the wire,
 314            then signal the poller thread.
 315
 316            This is not an airtight check, since the poller thread
 317            could be just about to send another MAD as we signal
 318            the event here.  To cover this rare case, the poller
 319            thread checks for a spurious wake-up.
 320          */
 321         if (p_vl->p_stats->qp0_mads_outstanding_on_wire <
 322             (int32_t) p_vl->max_wire_smps) {
 323                 OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
 324                         "Signalling poller thread\n");
 325                 cl_event_signal(&p_vl->signal);
 326         }
 327
 328         OSM_LOG_EXIT(p_vl->p_log);
 329 }
 330
 331 void osm_vl15_post(IN osm_vl15_t * p_vl, IN osm_madw_t * p_madw)
 332 {
 333         OSM_LOG_ENTER(p_vl->p_log);
 334
 335         CL_ASSERT(p_vl->state == OSM_VL15_STATE_READY);
 336
 337         OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG, "Posting p_madw = %p\n", p_madw);
 338
 339         /*
 340            Determine in which fifo to place the pending madw.
 341          */
 342         cl_spinlock_acquire(&p_vl->lock);
 343         if (p_madw->resp_expected == TRUE) {
 344                 cl_qlist_insert_tail(&p_vl->rfifo, &p_madw->list_item);
 345                 osm_stats_inc_qp0_outstanding(p_vl->p_stats);
 346         } else
 347                 cl_qlist_insert_tail(&p_vl->ufifo, &p_madw->list_item);
 348         cl_spinlock_release(&p_vl->lock);
 349
 350         OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
 351                 "%u QP0 MADs on wire, %u QP0 MADs outstanding\n",
 352                 p_vl->p_stats->qp0_mads_outstanding_on_wire,
 353                 p_vl->p_stats->qp0_mads_outstanding);
 354
 355         osm_vl15_poll(p_vl);
 356
 357         OSM_LOG_EXIT(p_vl->p_log);
 358 }
 359
 360 void osm_vl15_shutdown(IN osm_vl15_t * p_vl, IN osm_mad_pool_t * p_mad_pool)
 361 {
 362         osm_madw_t *p_madw;
 363
 364         OSM_LOG_ENTER(p_vl->p_log);
 365
 366         /* we only should get here after the VL15 interface was initialized */
 367         CL_ASSERT(p_vl->state == OSM_VL15_STATE_READY);
 368
 369         /* grab a lock on the object */
 370         cl_spinlock_acquire(&p_vl->lock);
 371
 372         /* go over all outstanding MADs and retire their transactions */
 373
 374         /* first we handle the list of response MADs */
 375         p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->ufifo);
 376         while (p_madw != (osm_madw_t *) cl_qlist_end(&p_vl->ufifo)) {
 377                 OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
 378                         "Releasing Response p_madw = %p\n", p_madw);
 379
 380                 osm_mad_pool_put(p_mad_pool, p_madw);
 381
 382                 p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->ufifo);
 383         }
 384
 385         /* Request MADs we send out */
 386         p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->rfifo);
 387         while (p_madw != (osm_madw_t *) cl_qlist_end(&p_vl->rfifo)) {
 388                 OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
 389                         "Releasing Request p_madw = %p\n", p_madw);
 390
 391                 osm_mad_pool_put(p_mad_pool, p_madw);
 392                 osm_stats_dec_qp0_outstanding(p_vl->p_stats);
 393
 394                 p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->rfifo);
 395         }
 396
 397         /* free the lock */
 398         cl_spinlock_release(&p_vl->lock);
 399
 400         OSM_LOG_EXIT(p_vl->p_log);
 401 }