2 * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2002-2015 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
38 * Implementation of osm_physp_t.
39 * This object represents an Infiniband Port.
40 * This object is part of the opensm family of objects.
45 #endif /* HAVE_CONFIG_H */
49 #include <complib/cl_debug.h>
50 #include <iba/ib_types.h>
51 #include <opensm/osm_file_ids.h>
52 #define FILE_ID OSM_FILE_PORT_C
53 #include <opensm/osm_port.h>
54 #include <opensm/osm_node.h>
55 #include <opensm/osm_madw.h>
56 #include <opensm/osm_switch.h>
57 #include <opensm/osm_db_pack.h>
58 #include <opensm/osm_sm.h>
60 void osm_physp_construct(IN osm_physp_t * p_physp)
62 memset(p_physp, 0, sizeof(*p_physp));
63 osm_dr_path_construct(&p_physp->dr_path);
64 cl_ptr_vector_construct(&p_physp->slvl_by_port);
65 osm_pkey_tbl_construct(&p_physp->pkeys);
68 void osm_physp_destroy(IN osm_physp_t * p_physp)
72 /* the physp might be uninitialized */
73 if (p_physp->port_guid) {
75 free(p_physp->p_guids);
77 /* free the SL2VL Tables */
78 num_slvl = cl_ptr_vector_get_size(&p_physp->slvl_by_port);
79 for (i = 0; i < num_slvl; i++)
80 free(cl_ptr_vector_get(&p_physp->slvl_by_port, i));
81 cl_ptr_vector_destroy(&p_physp->slvl_by_port);
83 /* free the P_Key Tables */
84 osm_pkey_tbl_destroy(&p_physp->pkeys);
86 memset(p_physp, 0, sizeof(*p_physp));
87 osm_dr_path_construct(&p_physp->dr_path); /* clear dr_path */
91 void osm_physp_init(IN osm_physp_t * p_physp, IN ib_net64_t port_guid,
92 IN uint8_t port_num, IN const struct osm_node *p_node,
93 IN osm_bind_handle_t h_bind, IN uint8_t hop_count,
94 IN const uint8_t * p_initial_path)
97 ib_slvl_table_t *p_slvl;
101 osm_physp_construct(p_physp);
102 p_physp->port_guid = port_guid;
103 p_physp->port_num = port_num;
104 p_physp->healthy = TRUE;
105 p_physp->need_update = 2;
106 p_physp->p_node = (struct osm_node *)p_node;
108 osm_dr_path_init(&p_physp->dr_path, hop_count, p_initial_path);
110 /* allocate enough SL2VL tables */
111 if (osm_node_get_type(p_node) == IB_NODE_TYPE_SWITCH)
112 /* we need node num ports + 1 SL2VL tables */
113 num_slvl = osm_node_get_num_physp(p_node) + 1;
115 /* An end node - we need only one SL2VL */
118 cl_ptr_vector_init(&p_physp->slvl_by_port, num_slvl, 1);
119 for (i = 0; i < num_slvl; i++) {
120 p_slvl = (ib_slvl_table_t *) malloc(sizeof(ib_slvl_table_t));
123 memset(p_slvl, 0, sizeof(ib_slvl_table_t));
124 cl_ptr_vector_set(&p_physp->slvl_by_port, i, p_slvl);
127 /* initialize the pkey table */
128 osm_pkey_tbl_init(&p_physp->pkeys);
131 void osm_port_delete(IN OUT osm_port_t ** pp_port)
137 osm_port_t *osm_port_new(IN const ib_node_info_t * p_ni,
138 IN osm_node_t * p_parent_node)
141 ib_net64_t port_guid;
142 osm_physp_t *p_physp;
145 p_port = malloc(sizeof(*p_port));
149 memset(p_port, 0, sizeof(*p_port));
150 cl_qlist_init(&p_port->mcm_list);
151 p_port->p_node = (struct osm_node *)p_parent_node;
152 port_guid = p_ni->port_guid;
153 p_port->guid = port_guid;
154 port_num = p_ni->node_type == IB_NODE_TYPE_SWITCH ?
155 0 : ib_node_info_get_local_port_num(p_ni);
158 Get the pointers to the physical node objects "owned" by this
160 For switches, port '0' is owned; for HCA's and routers,
161 only the singular part that has this GUID is owned.
163 p_physp = osm_node_get_physp_ptr(p_parent_node, port_num);
167 CL_ASSERT(port_guid == osm_physp_get_port_guid(p_physp));
168 p_port->p_physp = p_physp;
173 void osm_port_get_lid_range_ho(IN const osm_port_t * p_port,
174 IN uint16_t * p_min_lid, IN uint16_t * p_max_lid)
178 *p_min_lid = cl_ntoh16(osm_port_get_base_lid(p_port));
179 lmc = osm_port_get_lmc(p_port);
180 *p_max_lid = (uint16_t) (*p_min_lid + (1 << lmc) - 1);
183 uint8_t osm_physp_calc_link_mtu(IN osm_log_t * p_log,
184 IN const osm_physp_t * p_physp,
185 IN uint8_t current_mtu)
187 const osm_physp_t *p_remote_physp;
191 OSM_LOG_ENTER(p_log);
193 p_remote_physp = osm_physp_get_remote(p_physp);
194 if (p_remote_physp) {
195 /* use the available MTU */
196 mtu = ib_port_info_get_mtu_cap(&p_physp->port_info);
199 ib_port_info_get_mtu_cap(&p_remote_physp->port_info);
201 OSM_LOG(p_log, OSM_LOG_DEBUG,
202 "Remote port 0x%016" PRIx64 " port = %u : "
203 "MTU = %u. This Port MTU: %u\n",
204 cl_ntoh64(osm_physp_get_port_guid(p_remote_physp)),
205 osm_physp_get_port_num(p_remote_physp),
208 if (mtu != remote_mtu) {
209 if (mtu > remote_mtu)
211 if (mtu != current_mtu)
212 OSM_LOG(p_log, OSM_LOG_VERBOSE,
213 "MTU mismatch between ports."
214 "\n\t\t\t\tPort 0x%016" PRIx64 ", port %u"
215 " and port 0x%016" PRIx64 ", port %u."
216 "\n\t\t\t\tUsing lower MTU of %u\n",
217 cl_ntoh64(osm_physp_get_port_guid(p_physp)),
218 osm_physp_get_port_num(p_physp),
219 cl_ntoh64(osm_physp_get_port_guid
221 osm_physp_get_port_num(p_remote_physp), mtu);
224 mtu = ib_port_info_get_neighbor_mtu(&p_physp->port_info);
227 OSM_LOG(p_log, OSM_LOG_DEBUG, "ERR 4101: "
228 "Invalid MTU = 0. Forcing correction to 256\n");
236 uint8_t osm_physp_calc_link_op_vls(IN osm_log_t * p_log,
237 IN const osm_subn_t * p_subn,
238 IN const osm_physp_t * p_physp,
239 IN uint8_t current_op_vls)
241 const osm_physp_t *p_remote_physp;
243 uint8_t remote_op_vls;
245 OSM_LOG_ENTER(p_log);
247 p_remote_physp = osm_physp_get_remote(p_physp);
248 if (p_remote_physp) {
249 /* use the available VLCap */
250 op_vls = ib_port_info_get_vl_cap(&p_physp->port_info);
253 ib_port_info_get_vl_cap(&p_remote_physp->port_info);
255 OSM_LOG(p_log, OSM_LOG_DEBUG,
256 "Remote port 0x%016" PRIx64 " port = 0x%X : "
257 "VL_CAP = %u. This port VL_CAP = %u\n",
258 cl_ntoh64(osm_physp_get_port_guid(p_remote_physp)),
259 osm_physp_get_port_num(p_remote_physp),
260 remote_op_vls, op_vls);
262 if (op_vls != remote_op_vls) {
263 if (op_vls > remote_op_vls)
264 op_vls = remote_op_vls;
265 if (op_vls != current_op_vls)
266 OSM_LOG(p_log, OSM_LOG_VERBOSE,
267 "OP_VLS mismatch between ports."
268 "\n\t\t\t\tPort 0x%016" PRIx64 ", port 0x%X"
269 " and port 0x%016" PRIx64 ", port 0x%X."
270 "\n\t\t\t\tUsing lower OP_VLS of %u\n",
271 cl_ntoh64(osm_physp_get_port_guid(p_physp)),
272 osm_physp_get_port_num(p_physp),
273 cl_ntoh64(osm_physp_get_port_guid
275 osm_physp_get_port_num(p_remote_physp), op_vls);
278 op_vls = ib_port_info_get_op_vls(&p_physp->port_info);
281 /* for non compliant implementations */
282 OSM_LOG(p_log, OSM_LOG_VERBOSE,
283 "Invalid OP_VLS = 0. Forcing correction to 1 (VL0)\n");
287 /* support user limitation of max_op_vls */
288 if (op_vls > p_subn->opt.max_op_vls)
289 op_vls = p_subn->opt.max_op_vls;
295 static inline uint64_t ptr_to_key(void const *p)
299 memcpy(&k, p, sizeof(void *));
303 static inline void *key_to_ptr(uint64_t k)
307 memcpy(&p, &k, sizeof(void *));
311 /**********************************************************************
312 Traverse the fabric from the SM node following the DR path given and
313 add every phys port traversed to the map. Avoid tracking the first and
314 last phys ports (going into the first switch and into the target port).
315 **********************************************************************/
316 static cl_status_t physp_get_dr_physp_set(IN osm_log_t * p_log,
317 IN osm_subn_t const *p_subn,
318 IN osm_dr_path_t const *p_path,
319 OUT cl_map_t * p_physp_map)
322 osm_physp_t *p_physp;
325 cl_status_t status = CL_SUCCESS;
327 OSM_LOG_ENTER(p_log);
329 /* find the OSM node */
330 p_port = osm_get_port_by_guid(p_subn, p_subn->sm_port_guid);
332 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 4103: "
333 "Failed to find the SM own port by guid\n");
338 /* get the node of the SM */
339 p_node = p_port->p_node;
342 traverse the path adding the nodes to the table
343 start after the first dummy hop and stop just before the
346 for (hop = 1; hop < p_path->hop_count - 1; hop++) {
347 /* go out using the phys port of the path */
348 p_physp = osm_node_get_physp_ptr(p_node, p_path->path[hop]);
350 /* make sure we got a valid port and it has a remote port */
352 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 4104: "
353 "DR Traversal stopped on invalid port at hop:%u\n",
359 /* we track the ports we go out along the path */
361 cl_map_insert(p_physp_map, ptr_to_key(p_physp), NULL);
363 OSM_LOG(p_log, OSM_LOG_DEBUG,
364 "Traversed through node: 0x%016" PRIx64
366 cl_ntoh64(p_node->node_info.node_guid),
369 if (!(p_physp = osm_physp_get_remote(p_physp))) {
370 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 4106: "
371 "DR Traversal stopped on missing remote physp at hop:%u\n",
377 p_node = osm_physp_get_node_ptr(p_physp);
385 static void physp_update_new_dr_path(IN osm_physp_t const *p_dest_physp,
386 IN cl_map_t * p_visited_map,
387 IN osm_bind_handle_t * h_bind)
389 cl_list_t tmpPortsList;
390 osm_physp_t *p_physp, *p_src_physp = NULL;
391 uint8_t path_array[IB_SUBNET_PATH_HOPS_MAX];
393 osm_dr_path_t *p_dr_path;
395 cl_list_construct(&tmpPortsList);
396 cl_list_init(&tmpPortsList, 10);
398 cl_list_insert_head(&tmpPortsList, p_dest_physp);
399 /* get the output port where we need to come from */
400 p_physp = (osm_physp_t *) cl_map_get(p_visited_map,
401 ptr_to_key(p_dest_physp));
402 while (p_physp != NULL) {
403 cl_list_insert_head(&tmpPortsList, p_physp);
404 /* get the input port through where we reached the output port */
405 p_src_physp = p_physp;
406 p_physp = (osm_physp_t *) cl_map_get(p_visited_map,
407 ptr_to_key(p_physp));
408 /* if we reached a null p_physp - this means we are at the begining
409 of the path. Break. */
412 /* get the output port */
413 p_physp = (osm_physp_t *) cl_map_get(p_visited_map,
414 ptr_to_key(p_physp));
417 memset(path_array, 0, sizeof(path_array));
418 p_physp = (osm_physp_t *) cl_list_remove_head(&tmpPortsList);
419 while (p_physp != NULL) {
421 path_array[i] = p_physp->port_num;
422 p_physp = (osm_physp_t *) cl_list_remove_head(&tmpPortsList);
425 p_dr_path = osm_physp_get_dr_path_ptr(p_src_physp);
426 osm_dr_path_init(p_dr_path, i, path_array);
429 cl_list_destroy(&tmpPortsList);
432 void osm_physp_replace_dr_path_with_alternate_dr_path(IN osm_log_t * p_log,
434 *p_subn, IN osm_physp_t const
436 IN osm_bind_handle_t *
440 cl_map_t visited_map;
441 osm_dr_path_t *p_dr_path;
442 cl_list_t *p_currPortsList;
443 cl_list_t *p_nextPortsList;
445 osm_physp_t *p_physp, *p_remote_physp;
446 ib_net64_t port_guid;
447 boolean_t next_list_is_full = TRUE, reached_dest = FALSE;
448 uint8_t num_ports, port_num;
450 p_nextPortsList = (cl_list_t *) malloc(sizeof(cl_list_t));
451 if (!p_nextPortsList)
455 initialize the map of all port participating in current dr path
456 not including first and last switches
458 cl_map_construct(&physp_map);
459 cl_map_init(&physp_map, 4);
460 cl_map_construct(&visited_map);
461 cl_map_init(&visited_map, 4);
462 p_dr_path = osm_physp_get_dr_path_ptr(p_dest_physp);
463 physp_get_dr_physp_set(p_log, p_subn, p_dr_path, &physp_map);
466 BFS from OSM port until we find the target physp but avoid
467 going through mapped ports
469 cl_list_construct(p_nextPortsList);
470 cl_list_init(p_nextPortsList, 10);
472 port_guid = p_subn->sm_port_guid;
474 CL_ASSERT(port_guid);
476 p_port = osm_get_port_by_guid(p_subn, port_guid);
478 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 4105: No SM port object\n");
483 HACK: We are assuming SM is running on HCA, so when getting the default
484 port we'll get the port connected to the rest of the subnet. If SM is
485 running on SWITCH - we should try to get a dr path from all switch ports.
487 p_physp = p_port->p_physp;
491 cl_list_insert_tail(p_nextPortsList, p_physp);
493 while (next_list_is_full == TRUE) {
494 next_list_is_full = FALSE;
495 p_currPortsList = p_nextPortsList;
496 p_nextPortsList = (cl_list_t *) malloc(sizeof(cl_list_t));
497 if (!p_nextPortsList) {
498 p_nextPortsList = p_currPortsList;
501 cl_list_construct(p_nextPortsList);
502 cl_list_init(p_nextPortsList, 10);
503 p_physp = (osm_physp_t *) cl_list_remove_head(p_currPortsList);
504 while (p_physp != NULL) {
505 /* If we are in a switch - need to go out through all
506 the other physical ports of the switch */
507 num_ports = osm_node_get_num_physp(p_physp->p_node);
509 for (port_num = 1; port_num < num_ports; port_num++) {
510 if (osm_node_get_type(p_physp->p_node) ==
513 osm_node_get_physp_ptr(p_physp->
517 /* this is HCA or router - the remote port is just the port connected
520 p_physp->p_remote_physp;
523 make sure that all of the following occurred:
524 1. The port isn't NULL
525 2. This is not the port we came from
526 3. The port is not in the physp_map
527 4. This port haven't been visited before
529 if (p_remote_physp &&
530 p_remote_physp != p_physp &&
531 cl_map_get(&physp_map,
532 ptr_to_key(p_remote_physp))
534 && cl_map_get(&visited_map,
536 (p_remote_physp)) == NULL) {
537 /* Insert the port into the visited_map, and save its source port */
538 cl_map_insert(&visited_map,
543 /* Is this the p_dest_physp? */
544 if (p_remote_physp == p_dest_physp) {
545 /* update the new dr path */
546 physp_update_new_dr_path
547 (p_dest_physp, &visited_map,
553 /* add the p_remote_physp to the nextPortsList */
554 cl_list_insert_tail(p_nextPortsList,
556 next_list_is_full = TRUE;
560 p_physp = (osm_physp_t *)
561 cl_list_remove_head(p_currPortsList);
562 if (reached_dest == TRUE) {
563 /* free the rest of the currPortsList */
564 while (p_physp != NULL)
565 p_physp = (osm_physp_t *)
568 /* free the nextPortsList, if items were added to it */
569 p_physp = (osm_physp_t *)
570 cl_list_remove_head(p_nextPortsList);
571 while (p_physp != NULL)
572 p_physp = (osm_physp_t *)
575 next_list_is_full = FALSE;
578 cl_list_destroy(p_currPortsList);
579 free(p_currPortsList);
584 cl_list_destroy(p_nextPortsList);
585 free(p_nextPortsList);
586 cl_map_destroy(&physp_map);
587 cl_map_destroy(&visited_map);
590 boolean_t osm_link_is_healthy(IN const osm_physp_t * p_physp)
592 osm_physp_t *p_remote_physp;
595 p_remote_physp = p_physp->p_remote_physp;
596 if (p_remote_physp != NULL)
597 return ((p_physp->healthy) & (p_remote_physp->healthy));
598 /* the other side is not known - consider the link as healthy */
602 void osm_physp_set_pkey_tbl(IN osm_log_t * p_log, IN const osm_subn_t * p_subn,
603 IN osm_physp_t * p_physp,
604 IN ib_pkey_table_t * p_pkey_tbl,
605 IN uint16_t block_num,
610 CL_ASSERT(p_pkey_tbl);
612 (14.2.5.7) - the block number valid values are 0-2047, and are
613 further limited by the size of the P_Key table specified by
614 the PartitionCap on the node.
616 if (!p_physp->p_node->sw || p_physp->port_num == 0)
618 The maximum blocks is defined in the node info: partition cap
619 for CA, router, and switch management ports.
622 (cl_ntoh16(p_physp->p_node->node_info.partition_cap) +
623 IB_NUM_PKEY_ELEMENTS_IN_BLOCK - 1)
624 / IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
627 This is a switch, and not a management port. The maximum
628 blocks is defined in the switch info: partition enforcement
632 (cl_ntoh16(p_physp->p_node->sw->switch_info.enforce_cap) +
633 IB_NUM_PKEY_ELEMENTS_IN_BLOCK -
634 1) / IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
636 if (block_num >= max_blocks) {
637 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 4108: "
638 "Got illegal update for block number:%u max:%u "
639 "for GUID: %" PRIx64 " port number:%u\n",
640 block_num, max_blocks,
641 cl_ntoh64(p_physp->p_node->node_info.node_guid),
646 /* decrement block received counter */
648 p_physp->pkeys.rcv_blocks_cnt--;
649 osm_pkey_tbl_set(&p_physp->pkeys, block_num, p_pkey_tbl,
650 p_subn->opt.allow_both_pkeys);
653 osm_alias_guid_t *osm_alias_guid_new(IN const ib_net64_t alias_guid,
654 IN osm_port_t *p_base_port)
656 osm_alias_guid_t *p_alias_guid;
658 p_alias_guid = calloc(1, sizeof(*p_alias_guid));
660 p_alias_guid->alias_guid = alias_guid;
661 p_alias_guid->p_base_port = p_base_port;
666 void osm_alias_guid_delete(IN OUT osm_alias_guid_t ** pp_alias_guid)
668 free(*pp_alias_guid);
669 *pp_alias_guid = NULL;
672 void osm_physp_set_port_info(IN osm_physp_t * p_physp,
673 IN const ib_port_info_t * p_pi,
674 IN const struct osm_sm * p_sm)
677 CL_ASSERT(osm_physp_is_valid(p_physp));
679 if (ib_port_info_get_port_state(p_pi) == IB_LINK_DOWN) {
680 /* If PortState is down, only copy PortState */
681 /* and PortPhysicalState per C14-24-2.1 */
682 ib_port_info_set_port_state(&p_physp->port_info, IB_LINK_DOWN);
683 ib_port_info_set_port_phys_state
684 (ib_port_info_get_port_phys_state(p_pi),
685 &p_physp->port_info);
687 p_physp->port_info = *p_pi;
689 /* The MKey in p_pi can only be considered valid if it's
690 * for a HCA/router or switch port 0, and it's either
691 * non-zero or the MKeyProtect bits are also zero.
693 if ((osm_node_get_type(p_physp->p_node) !=
694 IB_NODE_TYPE_SWITCH || p_physp->port_num == 0) &&
695 (p_pi->m_key != 0 || ib_port_info_get_mpb(p_pi) == 0))
696 osm_db_guid2mkey_set(p_sm->p_subn->p_g2m,
697 cl_ntoh64(p_physp->port_guid),
698 cl_ntoh64(p_pi->m_key));