2 * Copyright (c) 2004-2006 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
38 #endif /* HAVE_CONFIG_H */
42 #include <opensm/osm_helper.h>
43 #include <opensm/osm_log.h>
44 #include <vendor/osm_vendor.h>
45 #include <vendor/osm_vendor_api.h>
46 #include <opensm/osm_mad_pool.h>
48 #if defined(OSM_VENDOR_INTF_MTL) | defined(OSM_VENDOR_INTF_TS)
50 #include <vendor/osm_vendor_mtl_transaction_mgr.h>
51 #ifdef OSM_VENDOR_INTF_MTL
52 #include <vendor/osm_mtl_bind.h>
55 /* this is the callback function of the timer */
56 void __osm_transaction_mgr_callback(IN void *context)
58 osm_transaction_mgr_t *trans_mgr_p;
59 osm_vendor_t *p_vend = (osm_vendor_t *) context;
60 cl_list_item_t *p_list_item;
61 cl_list_item_t *p_list_next_item;
62 osm_madw_req_t *osm_madw_req_p;
63 uint64_t current_time; /* [usec] */
64 uint32_t new_timeout; /* [msec] */
65 cl_status_t cl_status;
67 #ifdef OSM_VENDOR_INTF_MTL
68 osm_mtl_bind_info_t *p_bind;
70 osm_ts_bind_info_t *p_bind;
72 cl_list_t tmp_madw_p_list; /* this list will include all the madw_p that should be removed. */
73 cl_list_t retry_madw_p_list; /* this list will include all the madw_p that were retried and need to be removed. */
76 OSM_LOG_ENTER(p_vend->p_log);
78 trans_mgr_p = (osm_transaction_mgr_t *) p_vend->p_transaction_mgr;
80 /* initialize the tmp_madw_p_list */
81 cl_list_construct(&tmp_madw_p_list);
82 cl_status = cl_list_init(&tmp_madw_p_list, 50);
83 if (cl_status != CL_SUCCESS) {
84 osm_log(p_vend->p_log, OSM_LOG_ERROR,
85 "__osm_transaction_mgr_callback : ERROR 1000: "
86 "Failed to create tmp_madw_p_list\n");
89 cl_list_construct(&retry_madw_p_list);
90 cl_status = cl_list_init(&retry_madw_p_list, 50);
91 if (cl_status != CL_SUCCESS) {
92 osm_log(p_vend->p_log, OSM_LOG_ERROR,
93 "__osm_transaction_mgr_callback : ERROR 1000: "
94 "Failed to create retry_madw_p_list\n");
97 current_time = cl_get_time_stamp();
98 cl_spinlock_acquire(&(trans_mgr_p->transaction_mgr_lock));
99 p_list_item = cl_qlist_head(trans_mgr_p->madw_reqs_list_p);
100 if (p_list_item == cl_qlist_end(trans_mgr_p->madw_reqs_list_p)) {
101 /* the list is empty - nothing to do */
102 cl_spinlock_release(&trans_mgr_p->transaction_mgr_lock);
103 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
104 "__osm_transaction_mgr_callback : Nothing to do\n");
108 /* non empty list: */
110 /* get the osm_madw_req_p */
111 osm_madw_req_p = PARENT_STRUCT(p_list_item, osm_madw_req_t, list_item);
113 while (osm_madw_req_p->waking_time <= current_time) {
114 /* this object was supposed to have gotten a response */
115 /* we need to decide if we need to retry or done with it. */
116 if (osm_madw_req_p->retry_cnt > 0) {
117 /* add to the list of the retrys : */
118 cl_list_insert_tail(&retry_madw_p_list, osm_madw_req_p);
120 /* update wakeup time and retry count */
121 osm_madw_req_p->waking_time =
122 p_vend->timeout * 1000 + cl_get_time_stamp();
123 osm_madw_req_p->retry_cnt--;
125 /* make sure we will get some timer call if not earlier */
126 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
127 "__osm_transaction_mgr_callback : Timer restart:%u\n",
131 cl_timer_start(&trans_mgr_p->madw_list_timer,
134 /* go to the next object and check if it also needs to be removed - didn't receive response */
135 /* we need to do it before we move current item to the end of the list */
136 p_list_next_item = cl_qlist_next(p_list_item);
138 /* remove from the head */
139 cl_qlist_remove_item(trans_mgr_p->madw_reqs_list_p,
140 &(osm_madw_req_p->list_item));
142 /* insert the object to the qlist and the qmap */
143 cl_qlist_insert_tail(trans_mgr_p->madw_reqs_list_p,
144 &(osm_madw_req_p->list_item));
147 /* go to the next object and check if it also needs to be removed - didn't receive response */
148 p_list_next_item = cl_qlist_next(p_list_item);
150 /* remove from the head */
151 cl_qlist_remove_item(trans_mgr_p->madw_reqs_list_p,
152 &(osm_madw_req_p->list_item));
154 /* add it to the tmp_madw_p_list to be removed */
155 cl_list_insert_tail(&tmp_madw_p_list,
156 osm_madw_req_p->p_madw);
157 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
158 "__osm_transaction_mgr_callback : Found failed transaction madw: %p\n",
159 osm_madw_req_p->p_madw);
163 p_list_item = p_list_next_item;
164 if (p_list_item == cl_qlist_end(trans_mgr_p->madw_reqs_list_p)) {
165 /* the list is empty - nothing to do */
169 /* get the osm_madw_req_p */
171 PARENT_STRUCT(p_list_item, osm_madw_req_t, list_item);
174 /* look at the current p_list_item. If it is not the end item - then we need to */
175 /* re-start the timer */
176 if (p_list_item != cl_qlist_end(trans_mgr_p->madw_reqs_list_p)) {
177 /* get the osm_madw_req_p */
179 PARENT_STRUCT(p_list_item, osm_madw_req_t, list_item);
181 /* we have the object that still didn't get response - re-start the timer */
182 /* start the timer to the timeout (in miliseconds) */
184 (osm_madw_req_p->waking_time - cl_get_time_stamp()) / 1000 +
187 cl_timer_start(&trans_mgr_p->madw_list_timer, new_timeout);
188 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
189 "__osm_transaction_mgr_callback : Timer restart:%u\n",
192 if (cl_status != CL_SUCCESS) {
193 osm_log(p_vend->p_log, OSM_LOG_ERROR,
194 "__osm_transaction_mgr_callback : ERROR 1000: "
195 "Failed to start timer\n");
198 /* if not empty - retry on retry list: */
199 if (!cl_is_list_empty(&retry_madw_p_list)) {
201 /* remove all elements that were retried: */
204 *) (cl_list_remove_head(&retry_madw_p_list));
205 while (osm_madw_req_p != NULL) {
208 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
209 "__osm_transaction_mgr_callback : "
210 "Retry %d of madw %p\n",
211 OSM_DEFAULT_RETRY_COUNT -
212 osm_madw_req_p->retry_cnt,
213 osm_madw_req_p->p_madw);
215 /* actually send it */
216 #ifdef OSM_VENDOR_INTF_MTL
217 osm_mtl_send_mad((osm_mtl_bind_info_t *)
218 osm_madw_req_p->p_bind,
219 osm_madw_req_p->p_madw);
222 osm_ts_send_mad(osm_ts_bind_info_t * p_bind,
223 osm_madw_t * const p_madw);
224 osm_ts_send_mad((osm_ts_bind_info_t *) osm_madw_req_p->
225 p_bind, osm_madw_req_p->p_madw);
230 *) (cl_list_remove_head(&retry_madw_p_list));
234 /* if the tmp_madw_p_list has elements - need to call the send_err_callback */
235 madw_p = (osm_madw_t *) (cl_list_remove_head(&tmp_madw_p_list));
236 while (madw_p != NULL) {
237 /* need to remove it from pool */
239 /* obtain the madw_p stored as the wrid in the send call */
240 p_mad = osm_madw_get_mad_ptr(madw_p);
241 p_bind = madw_p->h_bind;
243 Return any wrappers to the pool that may have been
244 pre-emptively allocated to handle a receive.
246 if (madw_p->vend_wrap.p_resp_madw) {
247 #ifdef OSM_VENDOR_INTF_MTL
248 osm_mad_pool_put(p_bind->p_osm_pool,
249 madw_p->vend_wrap.p_resp_madw);
251 osm_mad_pool_put(p_bind->p_osm_pool,
252 madw_p->vend_wrap.p_resp_madw);
254 madw_p->vend_wrap.p_resp_madw = NULL;
258 (*(osm_vend_mad_send_err_callback_t)
259 (p_bind->send_err_callback)) (p_bind->client_context, madw_p);
260 madw_p = (osm_madw_t *) (cl_list_remove_head(&tmp_madw_p_list));
264 OSM_LOG_EXIT(p_vend->p_log);
269 * Construct and Initialize
272 void osm_transaction_mgr_init(IN osm_vendor_t * const p_vend)
274 cl_status_t cl_status;
275 osm_transaction_mgr_t *trans_mgr_p;
276 OSM_LOG_ENTER(p_vend->p_log);
278 CL_ASSERT(p_vend->p_transaction_mgr == NULL);
280 (osm_transaction_mgr_t *) p_vend->p_transaction_mgr =
281 (osm_transaction_mgr_t *) malloc(sizeof(osm_transaction_mgr_t));
283 trans_mgr_p = (osm_transaction_mgr_t *) p_vend->p_transaction_mgr;
285 /* construct lock object */
286 cl_spinlock_construct(&(trans_mgr_p->transaction_mgr_lock));
287 CL_ASSERT(cl_spinlock_init(&(trans_mgr_p->transaction_mgr_lock)) ==
290 /* initialize the qlist */
291 trans_mgr_p->madw_reqs_list_p =
292 (cl_qlist_t *) malloc(sizeof(cl_qlist_t));
293 cl_qlist_init(trans_mgr_p->madw_reqs_list_p);
295 /* initialize the qmap */
296 trans_mgr_p->madw_by_tid_map_p =
297 (cl_qmap_t *) malloc(sizeof(cl_qmap_t));
298 cl_qmap_init(trans_mgr_p->madw_by_tid_map_p);
300 /* create the timer used by the madw_req_list */
301 cl_timer_construct(&(trans_mgr_p->madw_list_timer));
303 /* init the timer with timeout. */
304 cl_status = cl_timer_init(&trans_mgr_p->madw_list_timer,
305 __osm_transaction_mgr_callback, p_vend);
307 if (cl_status != CL_SUCCESS) {
308 osm_log(p_vend->p_log, OSM_LOG_ERROR,
309 "osm_transaction_mgr_init : ERROR 1000: "
310 "Failed to initialize madw_reqs_list timer\n");
312 OSM_LOG_EXIT(p_vend->p_log);
315 void osm_transaction_mgr_destroy(IN osm_vendor_t * const p_vend)
317 osm_transaction_mgr_t *trans_mgr_p;
318 cl_list_item_t *p_list_item;
319 cl_map_item_t *p_map_item;
320 osm_madw_req_t *osm_madw_req_p;
322 OSM_LOG_ENTER(p_vend->p_log);
324 trans_mgr_p = (osm_transaction_mgr_t *) p_vend->p_transaction_mgr;
326 if (p_vend->p_transaction_mgr != NULL) {
327 /* we need to get a lock */
328 cl_spinlock_acquire(&trans_mgr_p->transaction_mgr_lock);
330 /* go over all the items in the list and remove them */
332 cl_qlist_remove_head(trans_mgr_p->madw_reqs_list_p);
333 while (p_list_item !=
334 cl_qlist_end(trans_mgr_p->madw_reqs_list_p)) {
335 osm_madw_req_p = (osm_madw_req_t *) p_list_item;
337 if (osm_madw_req_p->p_madw->p_mad)
338 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
339 "osm_transaction_mgr_destroy: "
340 "Found outstanding MADW:%p TID:<0x%"
341 PRIx64 ">.\n", osm_madw_req_p->p_madw,
342 osm_madw_req_p->p_madw->p_mad->
345 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
346 "osm_transaction_mgr_destroy: "
347 "Found outstanding MADW:%p TID:UNDEFINED.\n",
348 osm_madw_req_p->p_madw);
350 /* each item - remove it from the map */
351 p_map_item = &(osm_madw_req_p->map_item);
352 cl_qmap_remove_item(trans_mgr_p->madw_by_tid_map_p,
355 free(osm_madw_req_p);
357 cl_qlist_remove_head(trans_mgr_p->madw_reqs_list_p);
359 /* free the qlist and qmap */
360 free(trans_mgr_p->madw_reqs_list_p);
361 free(trans_mgr_p->madw_by_tid_map_p);
362 /* reliease and destroy the lock */
363 cl_spinlock_release(&trans_mgr_p->transaction_mgr_lock);
364 cl_spinlock_destroy(&(trans_mgr_p->transaction_mgr_lock));
365 /* destroy the timer */
366 cl_timer_trim(&trans_mgr_p->madw_list_timer, 1);
367 cl_timer_destroy(&trans_mgr_p->madw_list_timer);
368 /* free the transaction_manager object */
373 OSM_LOG_EXIT(p_vend->p_log);
377 osm_transaction_mgr_insert_madw(IN osm_bind_handle_t * const p_bind,
378 IN osm_madw_t * p_madw)
380 #ifdef OSM_VENDOR_INTF_MTL
381 osm_vendor_t *const p_vend = ((osm_mtl_bind_info_t *) p_bind)->p_vend;
383 osm_vendor_t *const p_vend = ((osm_ts_bind_info_t *) p_bind)->p_vend;
385 osm_transaction_mgr_t *trans_mgr_p;
386 osm_madw_req_t *osm_madw_req_p;
388 uint64_t waking_time;
389 cl_status_t cl_status;
391 const ib_mad_t *mad_p = p_madw->p_mad;
393 OSM_LOG_ENTER(p_vend->p_log);
397 trans_mgr_p = (osm_transaction_mgr_t *) p_vend->p_transaction_mgr;
399 timeout = (uint64_t) (p_vend->timeout) * 1000; /* change the miliseconds value of timeout to microseconds. */
400 waking_time = timeout + cl_get_time_stamp();
402 osm_madw_req_p = (osm_madw_req_t *) malloc(sizeof(osm_madw_req_t));
404 osm_madw_req_p->p_madw = p_madw;
405 osm_madw_req_p->waking_time = waking_time;
406 osm_madw_req_p->retry_cnt = OSM_DEFAULT_RETRY_COUNT;
407 osm_madw_req_p->p_bind = p_bind;
409 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
410 "osm_transaction_mgr_insert_madw: "
411 "Inserting MADW:%p with waking_time: <0x%" PRIx64 "> TID:<0x%"
412 PRIx64 ">.\n", p_madw, waking_time, p_madw->p_mad->trans_id);
414 /* Get the lock on the manager */
415 cl_spinlock_acquire(&(trans_mgr_p->transaction_mgr_lock));
416 /* If the list is empty - need to start the timer with timer of timeout (in miliseconds) */
417 if (cl_is_qlist_empty(trans_mgr_p->madw_reqs_list_p)) {
418 /* stop the timer if it is running */
419 cl_timer_stop(&trans_mgr_p->madw_list_timer);
421 /* start the timer to the timeout (in miliseconds) */
422 cl_status = cl_timer_start(&trans_mgr_p->madw_list_timer,
424 if (cl_status != CL_SUCCESS) {
425 osm_log(p_vend->p_log, OSM_LOG_ERROR,
426 "osm_transaction_mgr_insert_madw : ERROR 1000: "
427 "Failed to start timer\n");
431 /* insert the object to the qlist and the qmap */
432 cl_qlist_insert_tail(trans_mgr_p->madw_reqs_list_p,
433 &(osm_madw_req_p->list_item));
435 key = (uint64_t) mad_p->trans_id;
436 cl_qmap_insert(trans_mgr_p->madw_by_tid_map_p, key,
437 &(osm_madw_req_p->map_item));
438 cl_spinlock_release(&trans_mgr_p->transaction_mgr_lock);
440 OSM_LOG_EXIT(p_vend->p_log);
446 osm_transaction_mgr_erase_madw(IN osm_vendor_t * const p_vend,
449 osm_transaction_mgr_t *trans_mgr_p;
450 osm_madw_req_t *osm_madw_req_p;
452 cl_map_item_t *p_map_item;
453 OSM_LOG_ENTER(p_vend->p_log);
455 trans_mgr_p = (osm_transaction_mgr_t *) p_vend->p_transaction_mgr;
457 key = (uint64_t) p_mad->trans_id;
458 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
459 "osm_transaction_mgr_erase_madw: "
460 "Removing TID:<0x%" PRIx64 ">.\n", p_mad->trans_id);
462 cl_spinlock_acquire(&trans_mgr_p->transaction_mgr_lock);
463 p_map_item = cl_qmap_get(trans_mgr_p->madw_by_tid_map_p, key);
464 if (p_map_item != cl_qmap_end(trans_mgr_p->madw_by_tid_map_p)) {
465 /* we found such an item. */
466 /* get the osm_madw_req_p */
468 PARENT_STRUCT(p_map_item, osm_madw_req_t, map_item);
470 /* remove the item from the qlist */
471 cl_qlist_remove_item(trans_mgr_p->madw_reqs_list_p,
472 &(osm_madw_req_p->list_item));
473 /* remove the item from the qmap */
474 cl_qmap_remove_item(trans_mgr_p->madw_by_tid_map_p,
475 &(osm_madw_req_p->map_item));
477 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
478 "osm_transaction_mgr_erase_madw: "
479 "Removed TID:<0x%" PRIx64 ">.\n", p_mad->trans_id);
482 free(osm_madw_req_p);
484 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
485 "osm_transaction_mgr_erase_madw: "
486 "osm_transaction_mgr_erase_madw:<0x%" PRIx64
487 "> NOT FOUND.\n", p_mad->trans_id);
489 cl_spinlock_release(&trans_mgr_p->transaction_mgr_lock);
490 OSM_LOG_EXIT(p_vend->p_log);
496 osm_transaction_mgr_get_madw_for_tid(IN osm_vendor_t * const p_vend,
497 IN ib_mad_t * const p_mad,
498 OUT osm_madw_t ** req_madw_p)
500 osm_transaction_mgr_t *trans_mgr_p;
501 osm_madw_req_t *osm_madw_req_p;
502 cl_map_item_t *p_map_item;
504 OSM_LOG_ENTER(p_vend->p_log);
506 trans_mgr_p = (osm_transaction_mgr_t *) p_vend->p_transaction_mgr;
510 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
511 "osm_transaction_mgr_get_madw_for_tid: "
512 "Looking for TID:<0x%" PRIx64 ">.\n", p_mad->trans_id);
514 key = (uint64_t) p_mad->trans_id;
515 cl_spinlock_acquire(&(trans_mgr_p->transaction_mgr_lock));
516 p_map_item = cl_qmap_get(trans_mgr_p->madw_by_tid_map_p, key);
517 if (p_map_item != cl_qmap_end(trans_mgr_p->madw_by_tid_map_p)) {
518 /* we found such an item. */
519 /* get the osm_madw_req_p */
521 PARENT_STRUCT(p_map_item, osm_madw_req_t, map_item);
523 /* Since the Transaction was looked up and provided for */
524 /* processing we retire it */
525 cl_qlist_remove_item(trans_mgr_p->madw_reqs_list_p,
526 &(osm_madw_req_p->list_item));
527 /* remove the item from the qmap */
528 cl_qmap_remove_item(trans_mgr_p->madw_by_tid_map_p,
529 &(osm_madw_req_p->map_item));
531 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
532 "osm_transaction_mgr_get_madw_for_tid: "
533 "Removed TID:<0x%" PRIx64 ">.\n", p_mad->trans_id);
535 *req_madw_p = osm_madw_req_p->p_madw;
538 cl_spinlock_release(&(trans_mgr_p->transaction_mgr_lock));
539 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
540 "osm_transaction_mgr_get_madw_for_tid: "
541 "Got MADW:%p.\n", *req_madw_p);
542 OSM_LOG_EXIT(p_vend->p_log);