2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
38 * Implementation of osm_physp_t.
39 * This object represents an Infiniband Port.
40 * This object is part of the opensm family of objects.
45 #endif /* HAVE_CONFIG_H */
49 #include <complib/cl_debug.h>
50 #include <iba/ib_types.h>
51 #include <opensm/osm_port.h>
52 #include <opensm/osm_node.h>
53 #include <opensm/osm_madw.h>
54 #include <opensm/osm_mcm_info.h>
55 #include <opensm/osm_switch.h>
57 /**********************************************************************
58 **********************************************************************/
59 void osm_physp_construct(IN osm_physp_t * const p_physp)
61 memset(p_physp, 0, sizeof(*p_physp));
62 osm_dr_path_construct(&p_physp->dr_path);
63 cl_ptr_vector_construct(&p_physp->slvl_by_port);
64 osm_pkey_tbl_construct(&p_physp->pkeys);
67 /**********************************************************************
68 **********************************************************************/
69 void osm_physp_destroy(IN osm_physp_t * const p_physp)
73 /* the physp might be uninitialized */
74 if (p_physp->port_guid) {
75 /* free the SL2VL Tables */
76 num_slvl = cl_ptr_vector_get_size(&p_physp->slvl_by_port);
77 for (i = 0; i < num_slvl; i++)
78 free(cl_ptr_vector_get(&p_physp->slvl_by_port, i));
79 cl_ptr_vector_destroy(&p_physp->slvl_by_port);
81 /* free the P_Key Tables */
82 osm_pkey_tbl_destroy(&p_physp->pkeys);
84 memset(p_physp, 0, sizeof(*p_physp));
85 osm_dr_path_construct(&p_physp->dr_path); /* clear dr_path */
89 /**********************************************************************
90 **********************************************************************/
92 osm_physp_init(IN osm_physp_t * const p_physp,
93 IN const ib_net64_t port_guid,
94 IN const uint8_t port_num,
95 IN const struct osm_node *const p_node,
96 IN const osm_bind_handle_t h_bind,
97 IN const uint8_t hop_count,
98 IN const uint8_t * const p_initial_path)
100 uint16_t num_slvl, i;
101 ib_slvl_table_t *p_slvl;
105 osm_physp_construct(p_physp);
106 p_physp->port_guid = port_guid;
107 p_physp->port_num = port_num;
108 p_physp->healthy = TRUE;
109 p_physp->need_update = 2;
110 p_physp->p_node = (struct osm_node *)p_node;
112 osm_dr_path_init(&p_physp->dr_path, h_bind, hop_count, p_initial_path);
114 /* allocate enough SL2VL tables */
115 if (osm_node_get_type(p_node) == IB_NODE_TYPE_SWITCH)
116 /* we need node num ports + 1 SL2VL tables */
117 num_slvl = osm_node_get_num_physp(p_node) + 1;
119 /* An end node - we need only one SL2VL */
122 cl_ptr_vector_init(&p_physp->slvl_by_port, num_slvl, 1);
123 for (i = 0; i < num_slvl; i++) {
124 p_slvl = (ib_slvl_table_t *) malloc(sizeof(ib_slvl_table_t));
127 memset(p_slvl, 0, sizeof(ib_slvl_table_t));
128 cl_ptr_vector_set(&p_physp->slvl_by_port, i, p_slvl);
131 /* initialize the pkey table */
132 osm_pkey_tbl_init(&p_physp->pkeys);
135 /**********************************************************************
136 **********************************************************************/
137 void osm_port_delete(IN OUT osm_port_t ** const pp_port)
139 /* cleanup all mcm recs attached */
140 osm_port_remove_all_mgrp(*pp_port);
145 /**********************************************************************
146 **********************************************************************/
148 osm_port_init(IN osm_port_t * const p_port,
149 IN const ib_node_info_t * p_ni,
150 IN osm_node_t * const p_parent_node)
152 ib_net64_t port_guid;
153 osm_physp_t *p_physp;
158 CL_ASSERT(p_parent_node);
160 memset(p_port, 0, sizeof(*p_port));
161 cl_qlist_init(&p_port->mcm_list);
162 p_port->p_node = (struct osm_node *)p_parent_node;
163 port_guid = p_ni->port_guid;
164 p_port->guid = port_guid;
165 port_num = p_ni->node_type == IB_NODE_TYPE_SWITCH ?
166 0 : ib_node_info_get_local_port_num(p_ni);
169 Get the pointers to the physical node objects "owned" by this
171 For switches, port '0' is owned; for HCA's and routers,
172 only the singular part that has this GUID is owned.
174 p_physp = osm_node_get_physp_ptr(p_parent_node, port_num);
175 CL_ASSERT(port_guid == osm_physp_get_port_guid(p_physp));
176 p_port->p_physp = p_physp;
179 /**********************************************************************
180 **********************************************************************/
181 osm_port_t *osm_port_new(IN const ib_node_info_t * p_ni,
182 IN osm_node_t * const p_parent_node)
186 p_port = malloc(sizeof(*p_port));
187 if (p_port != NULL) {
188 memset(p_port, 0, sizeof(*p_port));
189 osm_port_init(p_port, p_ni, p_parent_node);
195 /**********************************************************************
196 **********************************************************************/
198 osm_port_get_lid_range_ho(IN const osm_port_t * const p_port,
199 IN uint16_t * const p_min_lid,
200 IN uint16_t * const p_max_lid)
204 *p_min_lid = cl_ntoh16(osm_port_get_base_lid(p_port));
205 lmc = osm_port_get_lmc(p_port);
206 *p_max_lid = (uint16_t) (*p_min_lid + (1 << lmc) - 1);
209 /**********************************************************************
210 **********************************************************************/
212 osm_get_port_by_base_lid(IN const osm_subn_t * const p_subn,
213 IN const ib_net16_t lid,
214 IN OUT const osm_port_t ** const pp_port)
216 ib_api_status_t status;
222 /* Loop on lmc from 0 up through max LMC possible */
223 for (lmc = 0; lmc <= IB_PORT_LMC_MAX; lmc++) {
224 /* Calculate a base LID assuming this is the real LMC */
225 base_lid = cl_ntoh16(lid) & ~((1 << lmc) - 1);
227 /* Look for a match */
228 status = cl_ptr_vector_at(&p_subn->port_lid_tbl,
229 base_lid, (void **)pp_port);
230 if ((status == CL_SUCCESS) && (*pp_port != NULL)) {
231 /* Determine if base LID "tested" is the real base LID */
232 /* This is true if the LMC "tested" is the port's actual LMC */
233 if (lmc == osm_port_get_lmc(*pp_port)) {
240 status = IB_NOT_FOUND;
246 /**********************************************************************
247 **********************************************************************/
249 osm_port_add_mgrp(IN osm_port_t * const p_port, IN const ib_net16_t mlid)
251 ib_api_status_t status = IB_SUCCESS;
252 osm_mcm_info_t *p_mcm;
254 p_mcm = osm_mcm_info_new(mlid);
256 cl_qlist_insert_tail(&p_port->mcm_list,
257 (cl_list_item_t *) p_mcm);
259 status = IB_INSUFFICIENT_MEMORY;
264 /**********************************************************************
265 **********************************************************************/
267 __osm_port_mgrp_find_func(IN const cl_list_item_t * const p_list_item,
270 if (*((ib_net16_t *) context) == ((osm_mcm_info_t *) p_list_item)->mlid)
273 return (CL_NOT_FOUND);
276 /**********************************************************************
277 **********************************************************************/
279 osm_port_remove_mgrp(IN osm_port_t * const p_port, IN const ib_net16_t mlid)
281 cl_list_item_t *p_mcm;
283 p_mcm = cl_qlist_find_from_head(&p_port->mcm_list,
284 __osm_port_mgrp_find_func, &mlid);
286 if (p_mcm != cl_qlist_end(&p_port->mcm_list)) {
287 cl_qlist_remove_item(&p_port->mcm_list, p_mcm);
288 osm_mcm_info_delete((osm_mcm_info_t *) p_mcm);
292 /**********************************************************************
293 **********************************************************************/
294 void osm_port_remove_all_mgrp(IN osm_port_t * const p_port)
296 cl_list_item_t *p_mcm;
298 p_mcm = cl_qlist_remove_head(&p_port->mcm_list);
299 while (p_mcm != cl_qlist_end(&p_port->mcm_list)) {
300 osm_mcm_info_delete((osm_mcm_info_t *) p_mcm);
301 p_mcm = cl_qlist_remove_head(&p_port->mcm_list);
305 /**********************************************************************
306 **********************************************************************/
308 osm_physp_calc_link_mtu(IN osm_log_t * p_log, IN const osm_physp_t * p_physp)
310 const osm_physp_t *p_remote_physp;
314 OSM_LOG_ENTER(p_log);
316 p_remote_physp = osm_physp_get_remote(p_physp);
317 if (p_remote_physp) {
318 /* use the available MTU */
319 mtu = ib_port_info_get_mtu_cap(&p_physp->port_info);
322 ib_port_info_get_mtu_cap(&p_remote_physp->port_info);
324 OSM_LOG(p_log, OSM_LOG_DEBUG,
325 "Remote port 0x%016" PRIx64 " port = %u : "
326 "MTU = %u. This Port MTU: %u\n",
327 cl_ntoh64(osm_physp_get_port_guid(p_remote_physp)),
328 osm_physp_get_port_num(p_remote_physp),
331 if (mtu != remote_mtu) {
332 if (mtu > remote_mtu)
335 OSM_LOG(p_log, OSM_LOG_VERBOSE,
336 "MTU mismatch between ports."
337 "\n\t\t\t\tPort 0x%016" PRIx64 ", port %u"
338 " and port 0x%016" PRIx64 ", port %u."
339 "\n\t\t\t\tUsing lower MTU of %u\n",
340 cl_ntoh64(osm_physp_get_port_guid(p_physp)),
341 osm_physp_get_port_num(p_physp),
342 cl_ntoh64(osm_physp_get_port_guid(p_remote_physp)),
343 osm_physp_get_port_num(p_remote_physp),mtu);
346 mtu = ib_port_info_get_neighbor_mtu(&p_physp->port_info);
349 OSM_LOG(p_log, OSM_LOG_DEBUG, "ERR 4101: "
350 "Invalid MTU = 0. Forcing correction to 256\n");
358 /**********************************************************************
359 **********************************************************************/
361 osm_physp_calc_link_op_vls(IN osm_log_t * p_log,
362 IN const osm_subn_t * p_subn,
363 IN const osm_physp_t * p_physp)
365 const osm_physp_t *p_remote_physp;
367 uint8_t remote_op_vls;
369 OSM_LOG_ENTER(p_log);
371 p_remote_physp = osm_physp_get_remote(p_physp);
372 if (p_remote_physp) {
373 /* use the available VLCap */
374 op_vls = ib_port_info_get_vl_cap(&p_physp->port_info);
377 ib_port_info_get_vl_cap(&p_remote_physp->port_info);
379 OSM_LOG(p_log, OSM_LOG_DEBUG,
380 "Remote port 0x%016" PRIx64 " port = 0x%X : "
381 "VL_CAP = %u. This port VL_CAP = %u\n",
382 cl_ntoh64(osm_physp_get_port_guid(p_remote_physp)),
383 osm_physp_get_port_num(p_remote_physp),
384 remote_op_vls, op_vls);
386 if (op_vls != remote_op_vls) {
387 if (op_vls > remote_op_vls)
388 op_vls = remote_op_vls;
390 OSM_LOG(p_log, OSM_LOG_VERBOSE,
391 "OP_VLS mismatch between ports."
392 "\n\t\t\t\tPort 0x%016" PRIx64 ", port 0x%X"
393 " and port 0x%016" PRIx64 ", port 0x%X."
394 "\n\t\t\t\tUsing lower OP_VLS of %u\n",
395 cl_ntoh64(osm_physp_get_port_guid(p_physp)),
396 osm_physp_get_port_num(p_physp),
397 cl_ntoh64(osm_physp_get_port_guid(p_remote_physp)),
398 osm_physp_get_port_num(p_remote_physp), op_vls);
401 op_vls = ib_port_info_get_op_vls(&p_physp->port_info);
403 /* support user limitation of max_op_vls */
404 if (op_vls > p_subn->opt.max_op_vls)
405 op_vls = p_subn->opt.max_op_vls;
408 OSM_LOG(p_log, OSM_LOG_DEBUG, "ERR 4102: "
409 "Invalid OP_VLS = 0. Forcing correction to 1 (VL0)\n");
417 static inline uint64_t __osm_ptr_to_key(void const *p)
421 memcpy(&k, p, sizeof(void *));
425 static inline void *__osm_key_to_ptr(uint64_t k)
429 memcpy(&p, &k, sizeof(void *));
433 /**********************************************************************
434 Traverse the fabric from the SM node following the DR path given and
435 add every phys port traversed to the map. Avoid tracking the first and
436 last phys ports (going into the first switch and into the target port).
437 **********************************************************************/
439 __osm_physp_get_dr_physp_set(IN osm_log_t * p_log,
440 IN osm_subn_t const *p_subn,
441 IN osm_dr_path_t const *p_path,
442 OUT cl_map_t * p_physp_map)
445 osm_physp_t *p_physp;
448 cl_status_t status = CL_SUCCESS;
450 OSM_LOG_ENTER(p_log);
452 /* find the OSM node */
453 p_port = osm_get_port_by_guid(p_subn, p_subn->sm_port_guid);
455 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 4103: "
456 "Failed to find the SM own port by guid\n");
461 /* get the node of the SM */
462 p_node = p_port->p_node;
465 traverse the path adding the nodes to the table
466 start after the first dummy hop and stop just before the
469 for (hop = 1; hop < p_path->hop_count - 1; hop++) {
470 /* go out using the phys port of the path */
471 p_physp = osm_node_get_physp_ptr(p_node, p_path->path[hop]);
473 /* make sure we got a valid port and it has a remote port */
475 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 4104: "
476 "DR Traversal stopped on invalid port at hop:%u\n",
482 /* we track the ports we go out along the path */
484 cl_map_insert(p_physp_map, __osm_ptr_to_key(p_physp),
487 OSM_LOG(p_log, OSM_LOG_DEBUG,
488 "Traversed through node: 0x%016" PRIx64
490 cl_ntoh64(p_node->node_info.node_guid),
493 if (!(p_physp = osm_physp_get_remote(p_physp))) {
494 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 4106: "
495 "DR Traversal stopped on missing remote physp at hop:%u\n",
501 p_node = osm_physp_get_node_ptr(p_physp);
509 /**********************************************************************
510 **********************************************************************/
512 __osm_physp_update_new_dr_path(IN osm_physp_t const *p_dest_physp,
513 IN cl_map_t * p_visited_map,
514 IN osm_bind_handle_t * h_bind)
516 cl_list_t tmpPortsList;
517 osm_physp_t *p_physp, *p_src_physp = NULL;
518 uint8_t path_array[IB_SUBNET_PATH_HOPS_MAX];
520 osm_dr_path_t *p_dr_path;
522 cl_list_construct(&tmpPortsList);
523 cl_list_init(&tmpPortsList, 10);
525 cl_list_insert_head(&tmpPortsList, p_dest_physp);
526 /* get the output port where we need to come from */
527 p_physp = (osm_physp_t *) cl_map_get(p_visited_map,
528 __osm_ptr_to_key(p_dest_physp));
529 while (p_physp != NULL) {
530 cl_list_insert_head(&tmpPortsList, p_physp);
531 /* get the input port through where we reached the output port */
532 p_src_physp = p_physp;
533 p_physp = (osm_physp_t *) cl_map_get(p_visited_map,
534 __osm_ptr_to_key(p_physp));
535 /* if we reached a null p_physp - this means we are at the begining
536 of the path. Break. */
539 /* get the output port */
540 p_physp = (osm_physp_t *) cl_map_get(p_visited_map,
541 __osm_ptr_to_key(p_physp));
544 memset(path_array, 0, sizeof(path_array));
545 p_physp = (osm_physp_t *) cl_list_remove_head(&tmpPortsList);
546 while (p_physp != NULL) {
548 path_array[i] = p_physp->port_num;
549 p_physp = (osm_physp_t *) cl_list_remove_head(&tmpPortsList);
552 p_dr_path = osm_physp_get_dr_path_ptr(p_src_physp);
553 osm_dr_path_init(p_dr_path, h_bind, i, path_array);
556 cl_list_destroy(&tmpPortsList);
559 /**********************************************************************
560 **********************************************************************/
562 osm_physp_replace_dr_path_with_alternate_dr_path(IN osm_log_t * p_log,
563 IN osm_subn_t const *p_subn,
566 IN osm_bind_handle_t * h_bind)
569 cl_map_t visited_map;
570 osm_dr_path_t *p_dr_path;
571 cl_list_t *p_currPortsList;
572 cl_list_t *p_nextPortsList;
574 osm_physp_t *p_physp, *p_remote_physp;
575 ib_net64_t port_guid;
576 boolean_t next_list_is_full = TRUE, reached_dest = FALSE;
577 uint8_t num_ports, port_num;
579 p_nextPortsList = (cl_list_t *) malloc(sizeof(cl_list_t));
580 if (!p_nextPortsList)
584 initialize the map of all port participating in current dr path
585 not including first and last switches
587 cl_map_construct(&physp_map);
588 cl_map_init(&physp_map, 4);
589 cl_map_construct(&visited_map);
590 cl_map_init(&visited_map, 4);
591 p_dr_path = osm_physp_get_dr_path_ptr(p_dest_physp);
592 __osm_physp_get_dr_physp_set(p_log, p_subn, p_dr_path, &physp_map);
595 BFS from OSM port until we find the target physp but avoid
596 going through mapped ports
598 cl_list_construct(p_nextPortsList);
599 cl_list_init(p_nextPortsList, 10);
601 port_guid = p_subn->sm_port_guid;
603 CL_ASSERT(port_guid);
605 p_port = osm_get_port_by_guid(p_subn, port_guid);
607 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 4105: No SM port object\n");
612 HACK: We are assuming SM is running on HCA, so when getting the default
613 port we'll get the port connected to the rest of the subnet. If SM is
614 running on SWITCH - we should try to get a dr path from all switch ports.
616 p_physp = p_port->p_physp;
620 cl_list_insert_tail(p_nextPortsList, p_physp);
622 while (next_list_is_full == TRUE) {
623 next_list_is_full = FALSE;
624 p_currPortsList = p_nextPortsList;
625 p_nextPortsList = (cl_list_t *) malloc(sizeof(cl_list_t));
626 if (!p_nextPortsList) {
627 p_nextPortsList = p_currPortsList;
630 cl_list_construct(p_nextPortsList);
631 cl_list_init(p_nextPortsList, 10);
632 p_physp = (osm_physp_t *) cl_list_remove_head(p_currPortsList);
633 while (p_physp != NULL) {
634 /* If we are in a switch - need to go out through all
635 the other physical ports of the switch */
636 num_ports = osm_node_get_num_physp(p_physp->p_node);
638 for (port_num = 1; port_num < num_ports; port_num++) {
639 if (osm_node_get_type(p_physp->p_node) ==
642 osm_node_get_physp_ptr(p_physp->
646 /* this is HCA or router - the remote port is just the port connected
649 p_physp->p_remote_physp;
652 make sure that all of the following occurred:
653 1. The port isn't NULL
654 2. This is not the port we came from
655 3. The port is not in the physp_map
656 4. This port haven't been visited before
658 if (p_remote_physp &&
659 p_remote_physp != p_physp &&
660 cl_map_get(&physp_map,
661 __osm_ptr_to_key(p_remote_physp))
663 && cl_map_get(&visited_map,
665 (p_remote_physp)) == NULL) {
666 /* Insert the port into the visited_map, and save its source port */
667 cl_map_insert(&visited_map,
672 /* Is this the p_dest_physp? */
673 if (p_remote_physp == p_dest_physp) {
674 /* update the new dr path */
675 __osm_physp_update_new_dr_path
676 (p_dest_physp, &visited_map,
682 /* add the p_remote_physp to the nextPortsList */
683 cl_list_insert_tail(p_nextPortsList,
685 next_list_is_full = TRUE;
689 p_physp = (osm_physp_t *)
690 cl_list_remove_head(p_currPortsList);
691 if (reached_dest == TRUE) {
692 /* free the rest of the currPortsList */
693 while (p_physp != NULL)
694 p_physp = (osm_physp_t *)
697 /* free the nextPortsList, if items were added to it */
698 p_physp = (osm_physp_t *)
699 cl_list_remove_head(p_nextPortsList);
700 while (p_physp != NULL)
701 p_physp = (osm_physp_t *)
704 next_list_is_full = FALSE;
707 cl_list_destroy(p_currPortsList);
708 free(p_currPortsList);
713 cl_list_destroy(p_nextPortsList);
714 free(p_nextPortsList);
715 cl_map_destroy(&physp_map);
716 cl_map_destroy(&visited_map);
719 /**********************************************************************
720 **********************************************************************/
721 boolean_t osm_link_is_healthy(IN const osm_physp_t * const p_physp)
723 osm_physp_t *p_remote_physp;
726 p_remote_physp = p_physp->p_remote_physp;
727 if (p_remote_physp != NULL)
728 return ((p_physp->healthy) & (p_remote_physp->healthy));
729 /* the other side is not known - consider the link as healthy */
733 /**********************************************************************
734 **********************************************************************/
736 osm_physp_set_pkey_tbl(IN osm_log_t * p_log,
737 IN const osm_subn_t * p_subn,
738 IN osm_physp_t * const p_physp,
739 IN ib_pkey_table_t * p_pkey_tbl, IN uint16_t block_num)
743 CL_ASSERT(p_pkey_tbl);
745 (14.2.5.7) - the block number valid values are 0-2047, and are
746 further limited by the size of the P_Key table specified by
747 the PartitionCap on the node.
749 if (!p_physp->p_node->sw || p_physp->port_num == 0)
751 The maximum blocks is defined in the node info: partition cap
752 for CA, router, and switch management ports.
755 (cl_ntoh16(p_physp->p_node->node_info.partition_cap) +
756 IB_NUM_PKEY_ELEMENTS_IN_BLOCK - 1)
757 / IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
760 This is a switch, and not a management port. The maximum
761 blocks is defined in the switch info: partition enforcement
765 (cl_ntoh16(p_physp->p_node->sw->switch_info.enforce_cap) +
766 IB_NUM_PKEY_ELEMENTS_IN_BLOCK -
767 1) / IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
769 if (block_num >= max_blocks) {
770 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 4108: "
771 "Got illegal set for block number:%u "
772 "For GUID: %" PRIx64 " port number:%u\n",
774 cl_ntoh64(p_physp->p_node->node_info.node_guid),
779 osm_pkey_tbl_set(&p_physp->pkeys, block_num, p_pkey_tbl);