2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2002-2006 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
38 * Implementation of osm_trap_rcv_t.
39 * This object represents the Trap Receiver object.
40 * This object is part of the opensm family of objects.
45 #endif /* HAVE_CONFIG_H */
48 #include <iba/ib_types.h>
49 #include <complib/cl_qmap.h>
50 #include <complib/cl_debug.h>
51 #include <opensm/osm_madw.h>
52 #include <opensm/osm_log.h>
53 #include <opensm/osm_node.h>
54 #include <opensm/osm_helper.h>
55 #include <opensm/osm_subnet.h>
56 #include <opensm/osm_inform.h>
57 #include <opensm/osm_opensm.h>
59 extern void osm_req_get_node_desc(IN osm_sm_t * sm, osm_physp_t *p_physp);
61 /**********************************************************************
65 * Assuming traps can be caused by bad hardware we should provide
66 * a mechanism for filtering their propagation into the actual logic
67 * of OpenSM such that it is not overloaded by them.
69 * We will provide a trap filtering mechanism with "Aging" capability.
70 * This mechanism will track incoming traps, clasify them by their
71 * source and content and provide back their age.
73 * A timer running in the background will toggle a timer counter
74 * that should be referenced by the aging algorithm.
75 * To provide an efficient handling of aging. We also track all traps
76 * in a sorted list by their aging.
78 * The generic Aging Tracker mechanism is implemented in the
79 * cl_aging_tracker object.
81 **********************************************************************/
83 typedef struct osm_trap_agingracker_context {
86 } osm_trap_aging_tracker_context_t;
88 /**********************************************************************
89 **********************************************************************/
90 static osm_physp_t *get_physp_by_lid_and_num(IN osm_sm_t * sm,
91 IN uint16_t lid, IN uint8_t num)
93 cl_ptr_vector_t *p_vec = &(sm->p_subn->port_lid_tbl);
96 if (lid > cl_ptr_vector_get_size(p_vec))
99 p_port = (osm_port_t *) cl_ptr_vector_get(p_vec, lid);
103 if (osm_node_get_num_physp(p_port->p_node) < num)
106 return osm_node_get_physp_ptr(p_port->p_node, num);
109 /**********************************************************************
110 **********************************************************************/
112 osm_trap_rcv_aging_tracker_callback(IN uint64_t key,
113 IN uint32_t num_regs, IN void *context)
115 osm_sm_t *sm = context;
118 osm_physp_t *p_physp;
120 OSM_LOG_ENTER(sm->p_log);
123 /* We got an exit flag - do nothing */
126 lid = cl_ntoh16((uint16_t) ((key & 0x0000FFFF00000000ULL) >> 32));
127 port_num = (uint8_t) ((key & 0x00FF000000000000ULL) >> 48);
129 p_physp = get_physp_by_lid_and_num(sm, lid, port_num);
131 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
132 "Cannot find port num:%u with lid:%u\n",
134 /* make sure the physp is still valid */
135 /* If the health port was false - set it to true */
136 else if (!osm_physp_is_healthy(p_physp)) {
137 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
138 "Clearing health bit of port num:%u with lid:%u\n",
141 /* Clear its health bit */
142 osm_physp_set_health(p_physp, TRUE);
145 OSM_LOG_EXIT(sm->p_log);
147 /* We want to remove the event from the tracker - so
148 need to return zero. */
152 /**********************************************************************
153 * CRC calculation for notice identification
154 **********************************************************************/
156 #define CRC32_POLYNOMIAL 0xEDB88320L
158 /* calculate the crc for a given buffer */
159 static uint32_t __osm_trap_calc_crc32(void *buffer, uint32_t count)
161 uint32_t temp1, temp2;
163 unsigned char *p = (unsigned char *)buffer;
164 /* pre - calculated table for faster crc calculation */
165 static uint32_t crc_table[256];
166 static boolean_t first = TRUE;
169 /* if we need to initialize the lookup table */
171 /* calc the CRC table */
172 for (i = 0; i <= 255; i++) {
174 for (j = 8; j > 0; j--)
176 crc = (crc >> 1) ^ CRC32_POLYNOMIAL;
185 /* do the calculation */
186 while (count-- != 0) {
187 temp1 = (crc >> 8) & 0x00FFFFFFL;
188 temp2 = crc_table[((int)crc ^ *p++) & 0xFF];
194 /********************************************************************
195 ********************************************************************/
197 /* The key is created in the following manner:
203 __osm_trap_get_key(IN uint16_t lid,
205 IN ib_mad_notice_attr_t * p_ntci, OUT uint64_t * trap_key)
211 crc = __osm_trap_calc_crc32(p_ntci, sizeof(ib_mad_notice_attr_t));
212 *trap_key = ((uint64_t) port_num << 48) | ((uint64_t) lid << 32) | crc;
215 /**********************************************************************
216 **********************************************************************/
217 static int __print_num_received(IN uint32_t num_received)
221 /* Series is 10, 20, 50, 100, 200, 500, ... */
229 if (i == 1 || i == 2 || i == 5)
235 static int disable_port(osm_sm_t *sm, osm_physp_t *p)
237 uint8_t payload[IB_SMP_DATA_SIZE];
238 osm_madw_context_t context;
239 ib_port_info_t *pi = (ib_port_info_t *)payload;
242 /* select the nearest port to master opensm */
243 if (p->p_remote_physp &&
244 p->dr_path.hop_count > p->p_remote_physp->dr_path.hop_count)
245 p = p->p_remote_physp;
247 /* If trap 131, might want to disable peer port if available */
248 /* but peer port has been observed not to respond to SM requests */
250 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3810: "
251 "Disabling physical port 0x%016" PRIx64 " num:%u\n",
252 cl_ntoh64(osm_physp_get_port_guid(p)), p->port_num);
254 memcpy(payload, &p->port_info, sizeof(ib_port_info_t));
256 /* Set port to disabled/down */
257 ib_port_info_set_port_state(pi, IB_LINK_DOWN);
258 ib_port_info_set_port_phys_state(IB_PORT_PHYS_STATE_DISABLED, pi);
260 /* Issue set of PortInfo */
261 context.pi_context.node_guid = osm_node_get_node_guid(p->p_node);
262 context.pi_context.port_guid = osm_physp_get_port_guid(p);
263 context.pi_context.set_method = TRUE;
264 context.pi_context.light_sweep = FALSE;
265 context.pi_context.active_transition = FALSE;
267 ret = osm_req_set(sm, osm_physp_get_dr_path_ptr(p),
268 payload, sizeof(payload), IB_MAD_ATTR_PORT_INFO,
269 cl_hton32(osm_physp_get_port_num(p)),
270 CL_DISP_MSGID_NONE, &context);
272 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3811: "
273 "Request to set PortInfo failed\n");
278 /**********************************************************************
279 **********************************************************************/
281 __osm_trap_rcv_process_request(IN osm_sm_t * sm,
282 IN const osm_madw_t * const p_madw)
284 uint8_t payload[sizeof(ib_mad_notice_attr_t)];
286 ib_mad_notice_attr_t *p_ntci = (ib_mad_notice_attr_t *) payload;
287 ib_api_status_t status;
288 osm_madw_t tmp_madw; /* we need a copy to last after repress */
290 uint32_t num_received;
291 osm_physp_t *p_physp;
292 cl_ptr_vector_t *p_tbl;
294 ib_net16_t source_lid = 0;
295 boolean_t is_gsi = TRUE;
296 uint8_t port_num = 0;
297 boolean_t physp_change_trap = FALSE;
298 uint64_t event_wheel_timeout = OSM_DEFAULT_TRAP_SUPRESSION_TIMEOUT;
299 boolean_t run_heavy_sweep = FALSE;
301 OSM_LOG_ENTER(sm->p_log);
307 We got an exit flag - do nothing
308 Otherwise we start a sweep on the trap 144 caused by
309 cleaning up SM Cap bit...
313 /* update the is_gsi flag according to the mgmt_class field */
314 if (p_madw->p_mad->mgmt_class == IB_MCLASS_SUBN_LID ||
315 p_madw->p_mad->mgmt_class == IB_MCLASS_SUBN_DIR)
318 /* No real need to grab the lock for this function. */
319 memset(payload, 0, sizeof(payload));
320 memset(&tmp_madw, 0, sizeof(tmp_madw));
322 p_smp = osm_madw_get_smp_ptr(p_madw);
324 if (p_smp->method != IB_MAD_METHOD_TRAP) {
325 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3801: "
326 "Unsupported method 0x%X\n", p_smp->method);
331 * The NOTICE Attribute is part of the SMP CLASS attributes
332 * As such the actual attribute data resides inside the SMP
336 memcpy(payload, &(p_smp->data), IB_SMP_DATA_SIZE);
337 memcpy(&tmp_madw, p_madw, sizeof(tmp_madw));
339 if (is_gsi == FALSE) {
340 /* We are in smi flow */
342 * When we received a TRAP with dlid = 0 - it means it
343 * came from our own node. So we need to fix it.
346 if (p_madw->mad_addr.addr_type.smi.source_lid == 0) {
347 /* Check if the sm_base_lid is 0. If yes - this means
348 that the local lid wasn't configured yet. Don't send
349 a response to the trap. */
350 if (sm->p_subn->sm_base_lid == 0) {
351 OSM_LOG(sm->p_log, OSM_LOG_DEBUG,
352 "Received SLID=0 Trap with local LID=0. Ignoring MAD\n");
355 OSM_LOG(sm->p_log, OSM_LOG_DEBUG,
356 "Received SLID=0 Trap. Using local LID:%u instead\n",
357 cl_ntoh16(sm->p_subn->sm_base_lid));
358 tmp_madw.mad_addr.addr_type.smi.source_lid =
359 sm->p_subn->sm_base_lid;
362 source_lid = tmp_madw.mad_addr.addr_type.smi.source_lid;
364 /* Print some info about the incoming Trap */
365 if (ib_notice_is_generic(p_ntci)) {
366 if ((p_ntci->g_or_v.generic.trap_num == CL_HTON16(129))
367 || (p_ntci->g_or_v.generic.trap_num ==
369 || (p_ntci->g_or_v.generic.trap_num ==
371 OSM_LOG(sm->p_log, OSM_LOG_ERROR,
372 "Received Generic Notice type:%u "
373 "num:%u (%s) Producer:%u (%s) "
374 "from LID:%u Port %d TID:0x%016"
375 PRIx64 "\n", ib_notice_get_type(p_ntci),
376 cl_ntoh16(p_ntci->g_or_v.generic.
378 ib_get_trap_str(p_ntci->g_or_v.generic.
380 cl_ntoh32(ib_notice_get_prod_type
382 ib_get_producer_type_str
383 (ib_notice_get_prod_type(p_ntci)),
384 cl_hton16(source_lid),
385 p_ntci->data_details.ntc_129_131.
386 port_num, cl_ntoh64(p_smp->trans_id));
388 OSM_LOG(sm->p_log, OSM_LOG_ERROR,
389 "Received Generic Notice type:%u "
390 "num:%u (%s) Producer:%u (%s) "
391 "from LID:%u TID:0x%016" PRIx64
392 "\n", ib_notice_get_type(p_ntci),
393 cl_ntoh16(p_ntci->g_or_v.generic.
395 ib_get_trap_str(p_ntci->g_or_v.generic.
397 cl_ntoh32(ib_notice_get_prod_type
399 ib_get_producer_type_str
400 (ib_notice_get_prod_type(p_ntci)),
401 cl_hton16(source_lid),
402 cl_ntoh64(p_smp->trans_id));
404 OSM_LOG(sm->p_log, OSM_LOG_ERROR,
405 "Received Vendor Notice type:%u vend:0x%06X "
406 "dev:%u from LID:%u TID:0x%016" PRIx64 "\n",
407 ib_notice_get_type(p_ntci),
408 cl_ntoh32(ib_notice_get_vend_id(p_ntci)),
409 cl_ntoh16(p_ntci->g_or_v.vend.dev_id),
410 cl_ntoh16(source_lid),
411 cl_ntoh64(p_smp->trans_id));
414 osm_dump_notice(sm->p_log, p_ntci, OSM_LOG_VERBOSE);
416 p_physp = osm_get_physp_by_mad_addr(sm->p_log,
417 sm->p_subn, &tmp_madw.mad_addr);
419 p_smp->m_key = p_physp->port_info.m_key;
421 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3809: "
422 "Failed to find source physical port for trap\n");
424 status = osm_resp_send(sm, &tmp_madw, 0, payload);
425 if (status != IB_SUCCESS) {
426 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3802: "
427 "Error sending response (%s)\n",
428 ib_get_err_str(status));
433 * We would like to filter out recurring Traps so we track them by
434 * their source lid and content. If the same trap was already
435 * received within the aging time window more than 10 times,
436 * we simply ignore it. This is done only if we are in smi mode
439 if (is_gsi == FALSE) {
440 if (ib_notice_is_generic(p_ntci) &&
441 ((p_ntci->g_or_v.generic.trap_num == CL_HTON16(129)) ||
442 (p_ntci->g_or_v.generic.trap_num == CL_HTON16(130)) ||
443 (p_ntci->g_or_v.generic.trap_num == CL_HTON16(131)))) {
444 /* If this is a trap 129, 130, or 131 - then this is a
445 * trap signaling a change on a physical port.
446 * Mark the physp_change_trap flag as TRUE.
448 physp_change_trap = TRUE;
449 /* The source_lid should be based on the source_lid from the trap */
450 source_lid = p_ntci->data_details.ntc_129_131.lid;
453 /* If physp_change_trap is TRUE - the key will include the port number.
454 If not - the port_number in the key will be zero. */
455 if (physp_change_trap == TRUE) {
456 port_num = p_ntci->data_details.ntc_129_131.port_num;
457 __osm_trap_get_key(source_lid, port_num, p_ntci,
460 __osm_trap_get_key(source_lid, 0, p_ntci, &trap_key);
462 /* try to find it in the aging tracker */
464 cl_event_wheel_num_regs(&sm->trap_aging_tracker,
467 /* Now we know how many times it provided this trap */
468 if (num_received > 10) {
469 if (__print_num_received(num_received))
470 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3804: "
471 "Received trap %u times consecutively\n",
474 * If the trap provides info about a bad port
475 * we mark it as unhealthy.
477 if (physp_change_trap == TRUE) {
479 p_physp = get_physp_by_lid_and_num(sm,
488 OSM_LOG(sm->p_log, OSM_LOG_ERROR,
490 "Failed to find physical port by lid:%u num:%u\n",
491 cl_ntoh16(p_ntci->data_details.
493 p_ntci->data_details.
494 ntc_129_131.port_num);
496 /* When babbling port policy option is enabled and
497 Threshold for disabling a "babbling" port is exceeded */
500 && num_received >= 250
501 && disable_port(sm, p_physp) == 0)
504 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
505 "Marking unhealthy physical port by lid:%u num:%u\n",
506 cl_ntoh16(p_ntci->data_details.
508 p_ntci->data_details.
509 ntc_129_131.port_num);
510 /* check if the current state of the p_physp is healthy. If
511 it is - then this is a first change of state. Run a heavy sweep.
512 if it is not - no need to mark it again - just restart the timer. */
513 if (osm_physp_is_healthy(p_physp)) {
514 osm_physp_set_health(p_physp,
516 /* Make sure we sweep again - force a heavy sweep. */
517 /* The sweep should be done only after the re-registration, or
518 else we'll be losing track of the timer. */
519 run_heavy_sweep = TRUE;
521 /* If we are marking the port as unhealthy - we want to
522 keep this for a longer period of time than the
523 OSM_DEFAULT_TRAP_SUPRESSION_TIMEOUT. Use the
524 OSM_DEFAULT_UNHEALTHY_TIMEOUT */
525 event_wheel_timeout =
526 OSM_DEFAULT_UNHEALTHY_TIMEOUT;
531 /* restart the aging anyway */
532 /* If physp_change_trap is TRUE - then use a callback to unset the
533 healthy bit. If not - no need to use a callback. */
534 if (physp_change_trap == TRUE)
535 cl_event_wheel_reg(&sm->trap_aging_tracker, trap_key, cl_get_time_stamp() + event_wheel_timeout, osm_trap_rcv_aging_tracker_callback, /* no callback */
536 sm /* no context */ );
538 cl_event_wheel_reg(&sm->trap_aging_tracker, trap_key, cl_get_time_stamp() + event_wheel_timeout, NULL, /* no callback */
539 NULL /* no context */ );
541 /* If was already registered do nothing more */
542 if (num_received > 10 && run_heavy_sweep == FALSE) {
543 if (__print_num_received(num_received))
544 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
545 "Continuously received this trap %u times. Ignoring\n",
551 /* Check for node description update. IB Spec v1.2.1 pg 823 */
552 if ((p_ntci->data_details.ntc_144.local_changes & TRAP_144_MASK_OTHER_LOCAL_CHANGES) &&
553 (p_ntci->data_details.ntc_144.change_flgs & TRAP_144_MASK_NODE_DESCRIPTION_CHANGE)
555 OSM_LOG(sm->p_log, OSM_LOG_INFO, "Trap 144 Node description update\n");
558 CL_PLOCK_ACQUIRE(sm->p_lock);
559 osm_req_get_node_desc(sm, p_physp);
560 CL_PLOCK_RELEASE(sm->p_lock);
562 OSM_LOG(sm->p_log, OSM_LOG_ERROR,
563 "ERR 3812: No physical port found for "
564 "trap 144: \"node description update\"\n");
568 /* do a sweep if we received a trap */
569 if (sm->p_subn->opt.sweep_on_trap) {
570 /* if this is trap number 128 or run_heavy_sweep is TRUE - update the
571 force_single_heavy_sweep flag of the subnet.
572 Sweep also on traps 144/145 - these traps signal a change of a certain
573 port capability/system image guid.
574 TODO: In the future we can change this to just getting PortInfo on
575 this port instead of sweeping the entire subnet. */
576 if (ib_notice_is_generic(p_ntci) &&
577 ((cl_ntoh16(p_ntci->g_or_v.generic.trap_num) == 128) ||
578 (cl_ntoh16(p_ntci->g_or_v.generic.trap_num) == 144) ||
579 (cl_ntoh16(p_ntci->g_or_v.generic.trap_num) == 145) ||
581 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
582 "Forcing heavy sweep. Received trap:%u\n",
583 cl_ntoh16(p_ntci->g_or_v.generic.trap_num));
585 sm->p_subn->force_heavy_sweep = TRUE;
587 osm_sm_signal(sm, OSM_SIGNAL_SWEEP);
590 /* If we reached here due to trap 129/130/131 - do not need to do
591 the notice report. Just goto exit. We know this is the case
592 if physp_change_trap is TRUE. */
593 if (physp_change_trap == TRUE)
596 /* Add a call to osm_report_notice */
597 /* We are going to report the notice - so need to fix the IssuerGID
598 accordingly. See IBA 1.2 p.739 or IBA 1.1 p.653 for details. */
600 if (!tmp_madw.mad_addr.addr_type.gsi.global_route) {
601 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3806: "
602 "Received gsi trap with global_route FALSE. "
603 "Cannot update issuer_gid!\n");
606 memcpy(&(p_ntci->issuer_gid),
607 &(tmp_madw.mad_addr.addr_type.gsi.grh_info.src_gid),
610 /* Need to use the IssuerLID */
611 p_tbl = &sm->p_subn->port_lid_tbl;
613 CL_ASSERT(cl_ptr_vector_get_size(p_tbl) < 0x10000);
615 if ((uint16_t) cl_ptr_vector_get_size(p_tbl) <=
616 cl_ntoh16(source_lid)) {
617 /* the source lid is out of range */
618 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
619 "source lid is out of range:%u\n",
620 cl_ntoh16(source_lid));
624 p_port = cl_ptr_vector_get(p_tbl, cl_ntoh16(source_lid));
626 /* We have the lid - but no corresponding port */
627 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
628 "Cannot find port corresponding to lid:%u\n",
629 cl_ntoh16(source_lid));
634 p_ntci->issuer_gid.unicast.prefix =
635 sm->p_subn->opt.subnet_prefix;
636 p_ntci->issuer_gid.unicast.interface_id = p_port->guid;
639 /* we need a lock here as the InformInfo DB must be stable */
640 CL_PLOCK_ACQUIRE(sm->p_lock);
641 status = osm_report_notice(sm->p_log, sm->p_subn, p_ntci);
642 CL_PLOCK_RELEASE(sm->p_lock);
643 if (status != IB_SUCCESS) {
644 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3803: "
645 "Error sending trap reports (%s)\n",
646 ib_get_err_str(status));
651 OSM_LOG_EXIT(sm->p_log);
655 /**********************************************************************
656 CURRENTLY WE ARE NOT CREATING TRAPS - SO THIS CALL IS AN ERROR
657 **********************************************************************/
659 __osm_trap_rcv_process_sm(IN osm_sm_t * sm,
660 IN const osm_remote_sm_t * const p_sm)
662 /* const ib_sm_info_t* p_smi; */
664 OSM_LOG_ENTER(sm->p_log);
666 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3807: "
667 "This function is not supported yet\n");
669 OSM_LOG_EXIT(sm->p_log);
673 /**********************************************************************
674 CURRENTLY WE ARE NOT CREATING TRAPS - SO THIS CALL IN AN ERROR
675 **********************************************************************/
677 __osm_trap_rcv_process_response(IN osm_sm_t * sm,
678 IN const osm_madw_t * const p_madw)
681 OSM_LOG_ENTER(sm->p_log);
683 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3808: "
684 "This function is not supported yet\n");
686 OSM_LOG_EXIT(sm->p_log);
689 /**********************************************************************
690 **********************************************************************/
691 void osm_trap_rcv_process(IN void *context, IN void *data)
693 osm_sm_t *sm = context;
694 osm_madw_t *p_madw = data;
697 OSM_LOG_ENTER(sm->p_log);
701 p_smp = osm_madw_get_smp_ptr(p_madw);
704 Determine if this is a request for our own Trap
705 or if this is a response to our request for another
708 if (ib_smp_is_response(p_smp))
709 __osm_trap_rcv_process_response(sm, p_madw);
711 __osm_trap_rcv_process_request(sm, p_madw);
713 OSM_LOG_EXIT(sm->p_log);