2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
38 * Implementation of osm_pi_rcv_t.
39 * This object represents the PortInfo Receiver object.
40 * This object is part of the opensm family of objects.
45 #endif /* HAVE_CONFIG_H */
48 #include <iba/ib_types.h>
49 #include <complib/cl_qmap.h>
50 #include <complib/cl_passivelock.h>
51 #include <complib/cl_debug.h>
52 #include <vendor/osm_vendor_api.h>
53 #include <opensm/osm_madw.h>
54 #include <opensm/osm_log.h>
55 #include <opensm/osm_node.h>
56 #include <opensm/osm_subnet.h>
57 #include <opensm/osm_mad_pool.h>
58 #include <opensm/osm_msgdef.h>
59 #include <opensm/osm_helper.h>
60 #include <opensm/osm_pkey.h>
61 #include <opensm/osm_remote_sm.h>
62 #include <opensm/osm_opensm.h>
63 #include <opensm/osm_ucast_mgr.h>
65 /**********************************************************************
66 **********************************************************************/
68 __osm_pi_rcv_set_sm(IN osm_sm_t * sm,
69 IN osm_physp_t * const p_physp)
71 osm_bind_handle_t h_bind;
72 osm_dr_path_t *p_dr_path;
74 OSM_LOG_ENTER(sm->p_log);
76 OSM_LOG(sm->p_log, OSM_LOG_DEBUG,
77 "Setting IS_SM bit in port attributes\n");
79 p_dr_path = osm_physp_get_dr_path_ptr(p_physp);
80 h_bind = osm_dr_path_get_bind_handle(p_dr_path);
83 The 'IS_SM' bit isn't already set, so set it.
85 osm_vendor_set_sm(h_bind, TRUE);
87 OSM_LOG_EXIT(sm->p_log);
90 /**********************************************************************
91 **********************************************************************/
92 static void pi_rcv_check_and_fix_lid(osm_log_t *log, ib_port_info_t * const pi,
95 if (cl_ntoh16(pi->base_lid) > IB_LID_UCAST_END_HO) {
96 OSM_LOG(log, OSM_LOG_ERROR, "ERR 0F04: "
97 "Got invalid base LID %u from the network. "
98 "Corrected to %u.\n", cl_ntoh16(pi->base_lid),
99 cl_ntoh16(p->port_info.base_lid));
100 pi->base_lid = p->port_info.base_lid;
104 /**********************************************************************
105 **********************************************************************/
107 __osm_pi_rcv_process_endport(IN osm_sm_t * sm,
108 IN osm_physp_t * const p_physp,
109 IN const ib_port_info_t * const p_pi)
111 osm_madw_context_t context;
112 ib_api_status_t status;
113 ib_net64_t port_guid;
116 osm_remote_sm_t *p_sm;
118 OSM_LOG_ENTER(sm->p_log);
120 port_guid = osm_physp_get_port_guid(p_physp);
122 /* HACK extended port 0 should be handled too! */
123 if (osm_physp_get_port_num(p_physp) != 0) {
124 /* track the minimal endport MTU and rate */
125 mtu = ib_port_info_get_mtu_cap(p_pi);
126 if (mtu < sm->p_subn->min_ca_mtu) {
127 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
128 "Setting endport minimal MTU to:%u defined by port:0x%"
129 PRIx64 "\n", mtu, cl_ntoh64(port_guid));
130 sm->p_subn->min_ca_mtu = mtu;
133 rate = ib_port_info_compute_rate(p_pi);
134 if (rate < sm->p_subn->min_ca_rate) {
135 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
136 "Setting endport minimal rate to:%u defined by port:0x%"
137 PRIx64 "\n", rate, cl_ntoh64(port_guid));
138 sm->p_subn->min_ca_rate = rate;
142 if (port_guid == sm->p_subn->sm_port_guid) {
144 We received the PortInfo for our own port.
146 if (!(p_pi->capability_mask & IB_PORT_CAP_IS_SM))
148 Set the IS_SM bit to indicate our port hosts an SM.
150 __osm_pi_rcv_set_sm(sm, p_physp);
152 p_sm_tbl = &sm->p_subn->sm_guid_tbl;
153 if (p_pi->capability_mask & IB_PORT_CAP_IS_SM) {
155 * Before querying the SM - we want to make sure we
156 * clean its state, so if the querying fails we
157 * recognize that this SM is not active.
159 p_sm = (osm_remote_sm_t *) cl_qmap_get(p_sm_tbl, port_guid);
160 if (p_sm != (osm_remote_sm_t *) cl_qmap_end(p_sm_tbl))
162 p_sm->smi.pri_state = 0xF0 & p_sm->smi.pri_state;
163 if (sm->p_subn->opt.ignore_other_sm)
164 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
165 "Ignoring SM on port 0x%" PRIx64 "\n",
166 cl_ntoh64(port_guid));
168 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
169 "Detected another SM. Requesting SMInfo"
170 "\n\t\t\t\tPort 0x%" PRIx64 "\n",
171 cl_ntoh64(port_guid));
174 This port indicates it's an SM and
175 it's not our own port.
176 Acquire the SMInfo Attribute.
178 memset(&context, 0, sizeof(context));
179 context.smi_context.set_method = FALSE;
180 context.smi_context.port_guid = port_guid;
181 status = osm_req_get(sm,
182 osm_physp_get_dr_path_ptr
184 IB_MAD_ATTR_SM_INFO, 0,
188 if (status != IB_SUCCESS)
189 OSM_LOG(sm->p_log, OSM_LOG_ERROR,
191 "Failure requesting SMInfo (%s)\n",
192 ib_get_err_str(status));
195 p_sm = (osm_remote_sm_t *) cl_qmap_remove(p_sm_tbl, port_guid);
196 if (p_sm != (osm_remote_sm_t *) cl_qmap_end(p_sm_tbl))
201 OSM_LOG_EXIT(sm->p_log);
204 /**********************************************************************
205 The plock must be held before calling this function.
206 **********************************************************************/
208 __osm_pi_rcv_process_switch_port(IN osm_sm_t * sm,
209 IN osm_node_t * const p_node,
210 IN osm_physp_t * const p_physp,
211 IN ib_port_info_t * const p_pi)
213 ib_api_status_t status = IB_SUCCESS;
214 osm_madw_context_t context;
215 osm_physp_t *p_remote_physp;
216 osm_node_t *p_remote_node;
218 uint8_t remote_port_num;
221 OSM_LOG_ENTER(sm->p_log);
224 Check the state of the physical port.
225 If there appears to be something on the other end of the wire,
226 then ask for NodeInfo. Ignore the switch management port.
228 port_num = osm_physp_get_port_num(p_physp);
229 /* if in_sweep_hop_0 is TRUE, then this means the SM is on the switch,
230 and we got switchInfo of our local switch. Do not continue
231 probing through the switch. */
232 if (port_num != 0 && sm->p_subn->in_sweep_hop_0 == FALSE) {
233 switch (ib_port_info_get_port_state(p_pi)) {
235 p_remote_physp = osm_physp_get_remote(p_physp);
236 if (p_remote_physp) {
238 osm_physp_get_node_ptr(p_remote_physp);
240 osm_physp_get_port_num(p_remote_physp);
242 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
243 "Unlinking local node 0x%" PRIx64
245 "\n\t\t\t\tand remote node 0x%" PRIx64
247 cl_ntoh64(osm_node_get_node_guid
249 cl_ntoh64(osm_node_get_node_guid
253 if (sm->ucast_mgr.cache_valid)
254 osm_ucast_cache_add_link(&sm->ucast_mgr,
258 osm_node_unlink(p_node, (uint8_t) port_num,
260 (uint8_t) remote_port_num);
269 To avoid looping forever, only probe the port if it
270 is NOT the port that responded to the SMP.
272 Request node info from the other end of this link:
273 1) Copy the current path from the parent node.
274 2) Extend the path to the next hop thru this port.
275 3) Request node info with the new path
278 if (p_pi->local_port_num !=
279 osm_physp_get_port_num(p_physp)) {
280 path = *osm_physp_get_dr_path_ptr(p_physp);
282 osm_dr_path_extend(&path,
283 osm_physp_get_port_num
286 memset(&context, 0, sizeof(context));
287 context.ni_context.node_guid =
288 osm_node_get_node_guid(p_node);
289 context.ni_context.port_num =
290 osm_physp_get_port_num(p_physp);
292 status = osm_req_get(sm,
294 IB_MAD_ATTR_NODE_INFO,
299 if (status != IB_SUCCESS)
300 OSM_LOG(sm->p_log, OSM_LOG_ERROR,
302 "Failure initiating NodeInfo request (%s)\n",
303 ib_get_err_str(status));
305 OSM_LOG(sm->p_log, OSM_LOG_DEBUG,
306 "Skipping SMP responder port %u\n",
307 p_pi->local_port_num);
311 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 0F03: "
312 "Unknown link state = %u, port = %u\n",
313 ib_port_info_get_port_state(p_pi),
314 p_pi->local_port_num);
319 if (ib_port_info_get_port_state(p_pi) > IB_LINK_INIT && p_node->sw &&
320 p_node->sw->need_update == 1)
321 p_node->sw->need_update = 0;
323 if (p_physp->need_update)
324 sm->p_subn->ignore_existing_lfts = TRUE;
327 pi_rcv_check_and_fix_lid(sm->p_log, p_pi, p_physp);
330 Update the PortInfo attribute.
332 osm_physp_set_port_info(p_physp, p_pi);
335 /* Determine if base switch port 0 */
337 !ib_switch_info_is_enhanced_port0(&p_node->sw->switch_info))
338 /* PortState is not used on BSP0 but just in case it is DOWN */
339 p_physp->port_info = *p_pi;
340 __osm_pi_rcv_process_endport(sm, p_physp, p_pi);
343 OSM_LOG_EXIT(sm->p_log);
346 /**********************************************************************
347 **********************************************************************/
349 __osm_pi_rcv_process_ca_or_router_port(IN osm_sm_t * sm,
350 IN osm_node_t * const p_node,
351 IN osm_physp_t * const p_physp,
352 IN ib_port_info_t * const p_pi)
354 OSM_LOG_ENTER(sm->p_log);
356 UNUSED_PARAM(p_node);
358 pi_rcv_check_and_fix_lid(sm->p_log, p_pi, p_physp);
360 osm_physp_set_port_info(p_physp, p_pi);
362 __osm_pi_rcv_process_endport(sm, p_physp, p_pi);
364 OSM_LOG_EXIT(sm->p_log);
367 #define IBM_VENDOR_ID (0x5076)
368 /**********************************************************************
369 **********************************************************************/
370 static void get_pkey_table(IN osm_log_t * p_log,
372 IN osm_node_t * const p_node,
373 IN osm_physp_t * const p_physp)
376 osm_madw_context_t context;
377 ib_api_status_t status;
380 uint16_t block_num, max_blocks;
381 uint32_t attr_mod_ho;
383 OSM_LOG_ENTER(p_log);
385 path = *osm_physp_get_dr_path_ptr(p_physp);
387 context.pkey_context.node_guid = osm_node_get_node_guid(p_node);
388 context.pkey_context.port_guid = osm_physp_get_port_guid(p_physp);
389 context.pkey_context.set_method = FALSE;
391 port_num = p_physp->port_num;
393 if (!p_node->sw || port_num == 0)
394 /* The maximum blocks is defined by the node info partition cap for CA,
395 router, and switch management ports. */
397 (cl_ntoh16(p_node->node_info.partition_cap) +
398 IB_NUM_PKEY_ELEMENTS_IN_BLOCK - 1)
399 / IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
401 /* This is a switch, and not a management port. The maximum blocks
402 is defined in the switch info partition enforcement cap. */
404 /* Check for IBM eHCA firmware defect in reporting partition enforcement cap */
405 if (cl_ntoh32(ib_node_info_get_vendor_id(&p_node->node_info)) ==
407 p_node->sw->switch_info.enforce_cap = 0;
409 /* Bail out if this is a switch with no partition enforcement capability */
410 if (cl_ntoh16(p_node->sw->switch_info.enforce_cap) == 0)
413 max_blocks = (cl_ntoh16(p_node->sw->switch_info.enforce_cap) +
414 IB_NUM_PKEY_ELEMENTS_IN_BLOCK -
415 1) / IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
418 for (block_num = 0; block_num < max_blocks; block_num++) {
419 if (osm_node_get_type(p_node) != IB_NODE_TYPE_SWITCH)
420 attr_mod_ho = block_num;
422 attr_mod_ho = block_num | (port_num << 16);
423 status = osm_req_get(sm, &path, IB_MAD_ATTR_P_KEY_TABLE,
424 cl_hton32(attr_mod_ho),
425 CL_DISP_MSGID_NONE, &context);
427 if (status != IB_SUCCESS) {
428 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0F12: "
429 "Failure initiating PKeyTable request (%s)\n",
430 ib_get_err_str(status));
439 /**********************************************************************
440 **********************************************************************/
442 __osm_pi_rcv_get_pkey_slvl_vla_tables(IN osm_sm_t * sm,
443 IN osm_node_t * const p_node,
444 IN osm_physp_t * const p_physp)
446 OSM_LOG_ENTER(sm->p_log);
448 get_pkey_table(sm->p_log, sm, p_node, p_physp);
450 OSM_LOG_EXIT(sm->p_log);
453 /**********************************************************************
454 **********************************************************************/
456 osm_pi_rcv_process_set(IN osm_sm_t * sm, IN osm_node_t * const p_node,
457 IN const uint8_t port_num, IN osm_madw_t * const p_madw)
459 osm_physp_t *p_physp;
460 ib_net64_t port_guid;
462 ib_port_info_t *p_pi;
463 osm_pi_context_t *p_context;
464 osm_log_level_t level;
466 OSM_LOG_ENTER(sm->p_log);
468 p_context = osm_madw_get_pi_context_ptr(p_madw);
472 p_physp = osm_node_get_physp_ptr(p_node, port_num);
475 port_guid = osm_physp_get_port_guid(p_physp);
477 p_smp = osm_madw_get_smp_ptr(p_madw);
478 p_pi = (ib_port_info_t *) ib_smp_get_payload_ptr(p_smp);
480 /* check for error */
481 if (cl_ntoh16(p_smp->status) & 0x7fff) {
482 /* If port already ACTIVE, don't treat status 7 as error */
483 if (p_context->active_transition &&
484 (cl_ntoh16(p_smp->status) & 0x7fff) == 0x1c) {
485 level = OSM_LOG_INFO;
486 OSM_LOG(sm->p_log, OSM_LOG_INFO,
487 "Received error status 0x%x for SetResp() during ACTIVE transition\n",
488 cl_ntoh16(p_smp->status) & 0x7fff);
489 /* Should there be a subsequent Get to validate that port is ACTIVE ? */
491 level = OSM_LOG_ERROR;
492 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 0F10: "
493 "Received error status for SetResp()\n");
495 osm_dump_port_info(sm->p_log,
496 osm_node_get_node_guid(p_node),
497 port_guid, port_num, p_pi, level);
500 OSM_LOG(sm->p_log, OSM_LOG_DEBUG,
501 "Received logical SetResp() for GUID 0x%" PRIx64
503 "\n\t\t\t\tfor parent node GUID 0x%" PRIx64
504 " TID 0x%" PRIx64 "\n",
505 cl_ntoh64(port_guid), port_num,
506 cl_ntoh64(osm_node_get_node_guid(p_node)),
507 cl_ntoh64(p_smp->trans_id));
509 osm_physp_set_port_info(p_physp, p_pi);
511 OSM_LOG_EXIT(sm->p_log);
514 /**********************************************************************
515 **********************************************************************/
516 void osm_pi_rcv_process(IN void *context, IN void *data)
518 osm_sm_t *sm = context;
519 osm_madw_t *p_madw = data;
520 ib_port_info_t *p_pi;
523 osm_physp_t *p_physp;
524 osm_dr_path_t *p_dr_path;
526 osm_pi_context_t *p_context;
527 ib_net64_t port_guid;
528 ib_net64_t node_guid;
531 OSM_LOG_ENTER(sm->p_log);
536 p_smp = osm_madw_get_smp_ptr(p_madw);
537 p_context = osm_madw_get_pi_context_ptr(p_madw);
538 p_pi = (ib_port_info_t *) ib_smp_get_payload_ptr(p_smp);
540 CL_ASSERT(p_smp->attr_id == IB_MAD_ATTR_PORT_INFO);
542 port_num = (uint8_t) cl_ntoh32(p_smp->attr_mod);
544 port_guid = p_context->port_guid;
545 node_guid = p_context->node_guid;
547 osm_dump_port_info(sm->p_log,
548 node_guid, port_guid, port_num, p_pi, OSM_LOG_DEBUG);
550 /* On receipt of client reregister, clear the reregister bit so
551 reregistering won't be sent again and again */
552 if (ib_port_info_get_client_rereg(p_pi)) {
553 OSM_LOG(sm->p_log, OSM_LOG_DEBUG,
554 "Client reregister received on response\n");
555 ib_port_info_set_client_rereg(p_pi, 0);
559 we might get a response during a light sweep looking for a change in
560 the status of a remote port that did not respond in earlier sweeps.
561 So if the context of the Get was light_sweep - we do not need to
562 do anything with the response - just flag that we need a heavy sweep
564 if (p_context->light_sweep == TRUE) {
565 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
566 "Got light sweep response from remote port of parent node "
567 "GUID 0x%" PRIx64 " port 0x%016" PRIx64
568 ", Commencing heavy sweep\n",
569 cl_ntoh64(node_guid), cl_ntoh64(port_guid));
570 sm->p_subn->force_heavy_sweep = TRUE;
571 sm->p_subn->ignore_existing_lfts = TRUE;
575 CL_PLOCK_EXCL_ACQUIRE(sm->p_lock);
576 p_port = osm_get_port_by_guid(sm->p_subn, port_guid);
578 CL_PLOCK_RELEASE(sm->p_lock);
579 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 0F06: "
580 "No port object for port with GUID 0x%" PRIx64
581 "\n\t\t\t\tfor parent node GUID 0x%" PRIx64
582 ", TID 0x%" PRIx64 "\n",
583 cl_ntoh64(port_guid),
584 cl_ntoh64(node_guid), cl_ntoh64(p_smp->trans_id));
588 p_node = p_port->p_node;
592 If we were setting the PortInfo, then receiving
593 this attribute was not part of sweeping the subnet.
594 In this case, just update the PortInfo attribute.
596 In an unfortunate blunder, the IB spec defines the
597 return method for Set() as a GetResp(). Thus, we can't
598 use the method (what would have been SetResp()) to determine
599 our course of action. So, we have to carry this extra
600 boolean around to determine if we were doing Get() or Set().
602 if (p_context->set_method)
603 osm_pi_rcv_process_set(sm, p_node, port_num, p_madw);
605 p_port->discovery_count++;
608 This PortInfo arrived because we did a Get() method,
609 most likely due to a subnet sweep in progress.
611 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
612 "Discovered port num %u with GUID 0x%" PRIx64
613 " for parent node GUID 0x%" PRIx64
614 ", TID 0x%" PRIx64 "\n",
615 port_num, cl_ntoh64(port_guid),
616 cl_ntoh64(node_guid), cl_ntoh64(p_smp->trans_id));
618 p_physp = osm_node_get_physp_ptr(p_node, port_num);
621 Determine if we encountered a new Physical Port.
622 If so, initialize the new Physical Port then
623 continue processing as normal.
626 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
627 "Initializing port number %u\n", port_num);
628 p_physp = &p_node->physp_table[port_num];
629 osm_physp_init(p_physp,
633 osm_madw_get_bind_handle(p_madw),
634 p_smp->hop_count, p_smp->initial_path);
637 Update the directed route path to this port
638 in case the old path is no longer usable.
640 p_dr_path = osm_physp_get_dr_path_ptr(p_physp);
641 osm_dr_path_init(p_dr_path,
642 osm_madw_get_bind_handle(p_madw),
643 p_smp->hop_count, p_smp->initial_path);
646 /* if port just inited or reached INIT state (external reset)
647 request update for port related tables */
648 p_physp->need_update =
649 (ib_port_info_get_port_state(p_pi) == IB_LINK_INIT ||
650 p_physp->need_update > 1) ? 1 : 0;
652 switch (osm_node_get_type(p_node)) {
653 case IB_NODE_TYPE_CA:
654 case IB_NODE_TYPE_ROUTER:
655 __osm_pi_rcv_process_ca_or_router_port(sm,
659 case IB_NODE_TYPE_SWITCH:
660 __osm_pi_rcv_process_switch_port(sm,
661 p_node, p_physp, p_pi);
664 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 0F07: "
665 "Unknown node type %u with GUID 0x%" PRIx64
666 "\n", osm_node_get_type(p_node),
667 cl_ntoh64(node_guid));
672 Get the tables on the physp.
674 if (p_physp->need_update || sm->p_subn->need_update)
675 __osm_pi_rcv_get_pkey_slvl_vla_tables(sm, p_node,
680 CL_PLOCK_RELEASE(sm->p_lock);
684 Release the lock before jumping here!!
686 OSM_LOG_EXIT(sm->p_log);