contrib/ofed/management/opensm/opensm/osm_vl15intf.c

   1 /*
   2  * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
   3  * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
   4  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenIB.org BSD license below:
  11  *
  12  *     Redistribution and use in source and binary forms, with or
  13  *     without modification, are permitted provided that the following
  14  *     conditions are met:
  15  *
  16  *      - Redistributions of source code must retain the above
  17  *        copyright notice, this list of conditions and the following
  18  *        disclaimer.
  19  *
  20  *      - Redistributions in binary form must reproduce the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer in the documentation and/or other materials
  23  *        provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  *
  34  */
  35
  36 /*
  37  * Abstract:
  38  *    Implementation of osm_vl15_t.
  39  * This object represents the VL15 Interface object.
  40  * This object is part of the opensm family of objects.
  41  */
  42
  43 #if HAVE_CONFIG_H
  44 #  include <config.h>
  45 #endif                          /* HAVE_CONFIG_H */
  46
  47 #include <string.h>
  48 #include <iba/ib_types.h>
  49 #include <complib/cl_thread.h>
  50 #include <vendor/osm_vendor_api.h>
  51 #include <opensm/osm_vl15intf.h>
  52 #include <opensm/osm_madw.h>
  53 #include <opensm/osm_log.h>
  54 #include <opensm/osm_helper.h>
  55
  56 /**********************************************************************
  57  **********************************************************************/
  58
  59 static void vl15_send_mad(osm_vl15_t * p_vl, osm_madw_t * p_madw)
  60 {
  61         ib_api_status_t status;
  62
  63         /*
  64            Non-response-expected mads are not throttled on the wire
  65            since we can have no confirmation that they arrived
  66            at their destination.
  67          */
  68         if (p_madw->resp_expected == TRUE)
  69                 /*
  70                    Note that other threads may not see the response MAD
  71                    arrive before send() even returns.
  72                    In that case, the wire count would temporarily go negative.
  73                    To avoid this confusion, preincrement the counts on the
  74                    assumption that send() will succeed.
  75                  */
  76                 cl_atomic_inc(&p_vl->p_stats->qp0_mads_outstanding_on_wire);
  77         else
  78                 cl_atomic_inc(&p_vl->p_stats->qp0_unicasts_sent);
  79
  80         cl_atomic_inc(&p_vl->p_stats->qp0_mads_sent);
  81
  82         status = osm_vendor_send(osm_madw_get_bind_handle(p_madw),
  83                                  p_madw, p_madw->resp_expected);
  84
  85         if (status == IB_SUCCESS) {
  86                 OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
  87                         "%u QP0 MADs on wire, %u outstanding, "
  88                         "%u unicasts sent, %u total sent\n",
  89                         p_vl->p_stats->qp0_mads_outstanding_on_wire,
  90                         p_vl->p_stats->qp0_mads_outstanding,
  91                         p_vl->p_stats->qp0_unicasts_sent,
  92                         p_vl->p_stats->qp0_mads_sent);
  93                 return;
  94         }
  95
  96         OSM_LOG(p_vl->p_log, OSM_LOG_ERROR, "ERR 3E03: "
  97                 "MAD send failed (%s)\n", ib_get_err_str(status));
  98
  99         /*
 100            The MAD was never successfully sent, so
 101            fix up the pre-incremented count values.
 102          */
 103
 104         /* Decrement qp0_mads_sent that were incremented in the code above.
 105            qp0_mads_outstanding will be decremented by send error callback
 106            (called by osm_vendor_send() */
 107         cl_atomic_dec(&p_vl->p_stats->qp0_mads_sent);
 108         if (!p_madw->resp_expected)
 109                 cl_atomic_dec(&p_vl->p_stats->qp0_unicasts_sent);
 110 }
 111
 112 static void __osm_vl15_poller(IN void *p_ptr)
 113 {
 114         ib_api_status_t status;
 115         osm_madw_t *p_madw;
 116         osm_vl15_t *const p_vl = (osm_vl15_t *) p_ptr;
 117         cl_qlist_t *p_fifo;
 118
 119         OSM_LOG_ENTER(p_vl->p_log);
 120
 121         if (p_vl->thread_state == OSM_THREAD_STATE_NONE)
 122                 p_vl->thread_state = OSM_THREAD_STATE_RUN;
 123
 124         while (p_vl->thread_state == OSM_THREAD_STATE_RUN) {
 125                 /*
 126                    Start servicing the FIFOs by pulling off MAD wrappers
 127                    and passing them to the transport interface.
 128                    There are lots of corner cases here so tread carefully.
 129
 130                    The unicast FIFO has priority, since somebody is waiting
 131                    for a timely response.
 132                  */
 133                 cl_spinlock_acquire(&p_vl->lock);
 134
 135                 if (cl_qlist_count(&p_vl->ufifo) != 0)
 136                         p_fifo = &p_vl->ufifo;
 137                 else
 138                         p_fifo = &p_vl->rfifo;
 139
 140                 p_madw = (osm_madw_t *) cl_qlist_remove_head(p_fifo);
 141
 142                 cl_spinlock_release(&p_vl->lock);
 143
 144                 if (p_madw != (osm_madw_t *) cl_qlist_end(p_fifo)) {
 145                         OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
 146                                 "Servicing p_madw = %p\n", p_madw);
 147                         if (osm_log_is_active(p_vl->p_log, OSM_LOG_FRAMES))
 148                                 osm_dump_dr_smp(p_vl->p_log,
 149                                                 osm_madw_get_smp_ptr(p_madw),
 150                                                 OSM_LOG_FRAMES);
 151
 152                         vl15_send_mad(p_vl, p_madw);
 153                 } else
 154                         /*
 155                            The VL15 FIFO is empty, so we have nothing left to do.
 156                          */
 157                         status = cl_event_wait_on(&p_vl->signal,
 158                                                   EVENT_NO_TIMEOUT, TRUE);
 159
 160                 while ((p_vl->p_stats->qp0_mads_outstanding_on_wire >=
 161                         (int32_t) p_vl->max_wire_smps) &&
 162                        (p_vl->thread_state == OSM_THREAD_STATE_RUN)) {
 163                         status = cl_event_wait_on(&p_vl->signal,
 164                                                   EVENT_NO_TIMEOUT, TRUE);
 165                         if (status != CL_SUCCESS) {
 166                                 OSM_LOG(p_vl->p_log, OSM_LOG_ERROR, "ERR 3E02: "
 167                                         "Event wait failed (%s)\n",
 168                                         CL_STATUS_MSG(status));
 169                                 break;
 170                         }
 171                 }
 172         }
 173
 174         /*
 175            since we abort immediately when the state != OSM_THREAD_STATE_RUN
 176            we might have some mads on the queues. After the thread exits
 177            the vl15 destroy routine should put these mads back...
 178          */
 179
 180         OSM_LOG_EXIT(p_vl->p_log);
 181 }
 182
 183 /**********************************************************************
 184  **********************************************************************/
 185 void osm_vl15_construct(IN osm_vl15_t * const p_vl)
 186 {
 187         memset(p_vl, 0, sizeof(*p_vl));
 188         p_vl->state = OSM_VL15_STATE_INIT;
 189         p_vl->thread_state = OSM_THREAD_STATE_NONE;
 190         cl_event_construct(&p_vl->signal);
 191         cl_spinlock_construct(&p_vl->lock);
 192         cl_qlist_init(&p_vl->rfifo);
 193         cl_qlist_init(&p_vl->ufifo);
 194         cl_thread_construct(&p_vl->poller);
 195 }
 196
 197 /**********************************************************************
 198  **********************************************************************/
 199 void
 200 osm_vl15_destroy(IN osm_vl15_t * const p_vl, IN struct osm_mad_pool *p_pool)
 201 {
 202         osm_madw_t *p_madw;
 203
 204         OSM_LOG_ENTER(p_vl->p_log);
 205
 206         /*
 207            Signal our threads that we're leaving.
 208          */
 209         p_vl->thread_state = OSM_THREAD_STATE_EXIT;
 210
 211         /*
 212            Don't trigger unless event has been initialized.
 213            Destroy the thread before we tear down the other objects.
 214          */
 215         if (p_vl->state != OSM_VL15_STATE_INIT)
 216                 cl_event_signal(&p_vl->signal);
 217
 218         cl_thread_destroy(&p_vl->poller);
 219
 220         /*
 221            Return the outstanding messages to the pool
 222          */
 223
 224         cl_spinlock_acquire(&p_vl->lock);
 225
 226         while (!cl_is_qlist_empty(&p_vl->rfifo)) {
 227                 p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->rfifo);
 228                 osm_mad_pool_put(p_pool, p_madw);
 229         }
 230         while (!cl_is_qlist_empty(&p_vl->ufifo)) {
 231                 p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->ufifo);
 232                 osm_mad_pool_put(p_pool, p_madw);
 233         }
 234
 235         cl_spinlock_release(&p_vl->lock);
 236
 237         cl_event_destroy(&p_vl->signal);
 238         p_vl->state = OSM_VL15_STATE_INIT;
 239         cl_spinlock_destroy(&p_vl->lock);
 240
 241         OSM_LOG_EXIT(p_vl->p_log);
 242 }
 243
 244 /**********************************************************************
 245  **********************************************************************/
 246 ib_api_status_t
 247 osm_vl15_init(IN osm_vl15_t * const p_vl,
 248               IN osm_vendor_t * const p_vend,
 249               IN osm_log_t * const p_log,
 250               IN osm_stats_t * const p_stats, IN const int32_t max_wire_smps)
 251 {
 252         ib_api_status_t status = IB_SUCCESS;
 253
 254         OSM_LOG_ENTER(p_log);
 255
 256         p_vl->p_vend = p_vend;
 257         p_vl->p_log = p_log;
 258         p_vl->p_stats = p_stats;
 259         p_vl->max_wire_smps = max_wire_smps;
 260
 261         status = cl_event_init(&p_vl->signal, FALSE);
 262         if (status != IB_SUCCESS)
 263                 goto Exit;
 264
 265         p_vl->state = OSM_VL15_STATE_READY;
 266
 267         status = cl_spinlock_init(&p_vl->lock);
 268         if (status != IB_SUCCESS)
 269                 goto Exit;
 270
 271         /*
 272            Initialize the thread after all other dependent objects
 273            have been initialized.
 274          */
 275         status = cl_thread_init(&p_vl->poller, __osm_vl15_poller, p_vl,
 276                                 "opensm poller");
 277         if (status != IB_SUCCESS)
 278                 goto Exit;
 279
 280 Exit:
 281         OSM_LOG_EXIT(p_log);
 282         return (status);
 283 }
 284
 285 /**********************************************************************
 286  **********************************************************************/
 287 void osm_vl15_poll(IN osm_vl15_t * const p_vl)
 288 {
 289         OSM_LOG_ENTER(p_vl->p_log);
 290
 291         CL_ASSERT(p_vl->state == OSM_VL15_STATE_READY);
 292
 293         /*
 294            If we have room for more VL15 MADs on the wire,
 295            then signal the poller thread.
 296
 297            This is not an airtight check, since the poller thread
 298            could be just about to send another MAD as we signal
 299            the event here.  To cover this rare case, the poller
 300            thread checks for a spurious wake-up.
 301          */
 302         if (p_vl->p_stats->qp0_mads_outstanding_on_wire <
 303             (int32_t) p_vl->max_wire_smps) {
 304                 OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
 305                         "Signalling poller thread\n");
 306                 cl_event_signal(&p_vl->signal);
 307         }
 308
 309         OSM_LOG_EXIT(p_vl->p_log);
 310 }
 311
 312 /**********************************************************************
 313  **********************************************************************/
 314 void osm_vl15_post(IN osm_vl15_t * const p_vl, IN osm_madw_t * const p_madw)
 315 {
 316         OSM_LOG_ENTER(p_vl->p_log);
 317
 318         CL_ASSERT(p_vl->state == OSM_VL15_STATE_READY);
 319
 320         OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG, "Posting p_madw = 0x%p\n", p_madw);
 321
 322         /*
 323            Determine in which fifo to place the pending madw.
 324          */
 325         cl_spinlock_acquire(&p_vl->lock);
 326         if (p_madw->resp_expected == TRUE) {
 327                 cl_qlist_insert_tail(&p_vl->rfifo, &p_madw->list_item);
 328                 osm_stats_inc_qp0_outstanding(p_vl->p_stats);
 329         } else
 330                 cl_qlist_insert_tail(&p_vl->ufifo, &p_madw->list_item);
 331         cl_spinlock_release(&p_vl->lock);
 332
 333         OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
 334                 "%u QP0 MADs on wire, %u QP0 MADs outstanding\n",
 335                 p_vl->p_stats->qp0_mads_outstanding_on_wire,
 336                 p_vl->p_stats->qp0_mads_outstanding);
 337
 338         osm_vl15_poll(p_vl);
 339
 340         OSM_LOG_EXIT(p_vl->p_log);
 341 }
 342
 343 void
 344 osm_vl15_shutdown(IN osm_vl15_t * const p_vl,
 345                   IN osm_mad_pool_t * const p_mad_pool)
 346 {
 347         osm_madw_t *p_madw;
 348
 349         OSM_LOG_ENTER(p_vl->p_log);
 350
 351         /* we only should get here after the VL15 interface was initialized */
 352         CL_ASSERT(p_vl->state == OSM_VL15_STATE_READY);
 353
 354         /* grap a lock on the object */
 355         cl_spinlock_acquire(&p_vl->lock);
 356
 357         /* go over all outstanding MADs and retire their transactions */
 358
 359         /* first we handle the list of response MADs */
 360         p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->ufifo);
 361         while (p_madw != (osm_madw_t *) cl_qlist_end(&p_vl->ufifo)) {
 362                 OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
 363                         "Releasing Response p_madw = %p\n", p_madw);
 364
 365                 osm_mad_pool_put(p_mad_pool, p_madw);
 366
 367                 p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->ufifo);
 368         }
 369
 370         /* Request MADs we send out */
 371         p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->rfifo);
 372         while (p_madw != (osm_madw_t *) cl_qlist_end(&p_vl->rfifo)) {
 373                 OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
 374                         "Releasing Request p_madw = %p\n", p_madw);
 375
 376                 osm_mad_pool_put(p_mad_pool, p_madw);
 377                 osm_stats_dec_qp0_outstanding(p_vl->p_stats);
 378
 379                 p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->rfifo);
 380         }
 381
 382         /* free the lock */
 383         cl_spinlock_release(&p_vl->lock);
 384
 385         OSM_LOG_EXIT(p_vl->p_log);
 386 }