2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2002-2008 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
38 * Implementation of osm_switch_t.
39 * This object represents an Infiniband switch.
40 * This object is part of the opensm family of objects.
45 #endif /* HAVE_CONFIG_H */
49 #include <complib/cl_math.h>
50 #include <iba/ib_types.h>
51 #include <opensm/osm_switch.h>
53 /**********************************************************************
54 **********************************************************************/
56 osm_switch_set_hops(IN osm_switch_t * const p_sw,
57 IN const uint16_t lid_ho,
58 IN const uint8_t port_num, IN const uint8_t num_hops)
60 if (lid_ho > p_sw->max_lid_ho)
62 if (!p_sw->hops[lid_ho]) {
63 p_sw->hops[lid_ho] = malloc(p_sw->num_ports);
64 if (!p_sw->hops[lid_ho])
66 memset(p_sw->hops[lid_ho], OSM_NO_PATH, p_sw->num_ports);
69 p_sw->hops[lid_ho][port_num] = num_hops;
70 if (p_sw->hops[lid_ho][0] > num_hops)
71 p_sw->hops[lid_ho][0] = num_hops;
76 /**********************************************************************
77 **********************************************************************/
78 static ib_api_status_t
79 osm_switch_init(IN osm_switch_t * const p_sw,
80 IN osm_node_t * const p_node,
81 IN const osm_madw_t * const p_madw)
83 ib_api_status_t status = IB_SUCCESS;
84 ib_switch_info_t *p_si;
89 p_smp = osm_madw_get_smp_ptr(p_madw);
90 p_si = (ib_switch_info_t *) ib_smp_get_payload_ptr(p_smp);
91 num_ports = osm_node_get_num_physp(p_node);
93 CL_ASSERT(p_smp->attr_id == IB_MAD_ATTR_SWITCH_INFO);
95 p_sw->p_node = p_node;
96 p_sw->switch_info = *p_si;
97 p_sw->num_ports = num_ports;
98 p_sw->need_update = 2;
100 /* Initiate the linear forwarding table */
102 if (!p_si->lin_cap) {
103 /* This switch does not support linear forwarding tables */
104 status = IB_UNSUPPORTED;
108 p_sw->lft = malloc(IB_LID_UCAST_END_HO + 1);
110 status = IB_INSUFFICIENT_MEMORY;
114 /* Initialize the table to OSM_NO_PATH, which is "invalid port" */
115 memset(p_sw->lft, OSM_NO_PATH, IB_LID_UCAST_END_HO + 1);
117 p_sw->p_prof = malloc(sizeof(*p_sw->p_prof) * num_ports);
118 if (p_sw->p_prof == NULL) {
119 status = IB_INSUFFICIENT_MEMORY;
123 memset(p_sw->p_prof, 0, sizeof(*p_sw->p_prof) * num_ports);
125 status = osm_mcast_tbl_init(&p_sw->mcast_tbl,
126 osm_node_get_num_physp(p_node),
127 cl_ntoh16(p_si->mcast_cap));
128 if (status != IB_SUCCESS)
131 for (port_num = 0; port_num < num_ports; port_num++)
132 osm_port_prof_construct(&p_sw->p_prof[port_num]);
138 /**********************************************************************
139 **********************************************************************/
140 void osm_switch_delete(IN OUT osm_switch_t ** const pp_sw)
142 osm_switch_t *p_sw = *pp_sw;
145 osm_mcast_tbl_destroy(&p_sw->mcast_tbl);
152 for (i = 0; i < p_sw->num_hops; i++)
161 /**********************************************************************
162 **********************************************************************/
163 osm_switch_t *osm_switch_new(IN osm_node_t * const p_node,
164 IN const osm_madw_t * const p_madw)
166 ib_api_status_t status;
172 p_sw = (osm_switch_t *) malloc(sizeof(*p_sw));
174 memset(p_sw, 0, sizeof(*p_sw));
175 status = osm_switch_init(p_sw, p_node, p_madw);
176 if (status != IB_SUCCESS)
177 osm_switch_delete(&p_sw);
183 /**********************************************************************
184 **********************************************************************/
186 osm_switch_get_lft_block(IN const osm_switch_t * const p_sw,
187 IN const uint16_t block_id,
188 OUT uint8_t * const p_block)
190 uint16_t base_lid_ho = block_id * IB_SMP_DATA_SIZE;
195 if (base_lid_ho > p_sw->max_lid_ho)
198 CL_ASSERT(base_lid_ho + IB_SMP_DATA_SIZE <= IB_LID_UCAST_END_HO);
199 memcpy(p_block, &(p_sw->lft[base_lid_ho]), IB_SMP_DATA_SIZE);
203 /**********************************************************************
204 **********************************************************************/
205 static struct osm_remote_node *
206 osm_switch_find_guid_common(IN const osm_switch_t * const p_sw,
207 IN struct osm_remote_guids_count *r,
209 IN int find_sys_guid,
210 IN int find_node_guid)
212 struct osm_remote_node *p_remote_guid = NULL;
213 osm_physp_t *p_physp;
214 osm_physp_t *p_rem_physp;
215 osm_node_t *p_rem_node;
222 p_physp = osm_node_get_physp_ptr(p_sw->p_node, port_num);
223 p_rem_physp = osm_physp_get_remote(p_physp);
224 p_rem_node = osm_physp_get_node_ptr(p_rem_physp);
225 sys_guid = p_rem_node->node_info.sys_guid;
226 node_guid = p_rem_node->node_info.node_guid;
228 for (i = 0; i < r->count; i++) {
230 || r->guids[i].node->node_info.sys_guid == sys_guid)
232 || r->guids[i].node->node_info.node_guid == node_guid)) {
233 p_remote_guid = &r->guids[i];
238 return p_remote_guid;
241 static struct osm_remote_node *
242 osm_switch_find_sys_guid_count(IN const osm_switch_t * const p_sw,
243 IN struct osm_remote_guids_count *r,
246 return osm_switch_find_guid_common(p_sw, r, port_num, 1, 0);
249 static struct osm_remote_node *
250 osm_switch_find_node_guid_count(IN const osm_switch_t * const p_sw,
251 IN struct osm_remote_guids_count *r,
254 return osm_switch_find_guid_common(p_sw, r, port_num, 0, 1);
257 /**********************************************************************
258 **********************************************************************/
260 osm_switch_recommend_path(IN const osm_switch_t * const p_sw,
261 IN osm_port_t * p_port,
262 IN const uint16_t lid_ho,
263 IN unsigned start_from,
264 IN const boolean_t ignore_existing,
265 IN const boolean_t dor)
268 We support an enhanced LMC aware routing mode:
269 In the case of LMC > 0, we can track the remote side
270 system and node for all of the lids of the target
271 and try and avoid routing again through the same
274 If this procedure is provided with the tracking array
275 and counter we can conduct this algorithm.
277 boolean_t routing_for_lmc = (p_port->priv != NULL);
283 uint32_t least_paths = 0xFFFFFFFF;
286 The follwing will track the least paths if the
287 route should go through a new system/node
289 uint32_t least_paths_other_sys = 0xFFFFFFFF;
290 uint32_t least_paths_other_nodes = 0xFFFFFFFF;
291 uint32_t least_forwarded_to = 0xFFFFFFFF;
292 uint32_t check_count;
293 uint8_t best_port = 0;
295 These vars track the best port if it connects to
296 not used system/node.
298 uint8_t best_port_other_sys = 0;
299 uint8_t best_port_other_node = 0;
300 boolean_t port_found = FALSE;
301 osm_physp_t *p_physp;
302 osm_physp_t *p_rem_physp;
303 osm_node_t *p_rem_node;
304 osm_node_t *p_rem_node_first = NULL;
305 struct osm_remote_node *p_remote_guid = NULL;
307 CL_ASSERT(lid_ho > 0);
309 if (p_port->p_node->sw) {
310 if (p_port->p_node->sw == p_sw)
312 base_lid = osm_port_get_base_lid(p_port);
314 p_physp = p_port->p_physp;
315 if (!p_physp || !p_physp->p_remote_physp ||
316 !p_physp->p_remote_physp->p_node->sw)
319 if (p_physp->p_remote_physp->p_node->sw == p_sw)
320 return p_physp->p_remote_physp->port_num;
322 osm_node_get_base_lid(p_physp->p_remote_physp->p_node, 0);
324 base_lid = cl_ntoh16(base_lid);
326 num_ports = p_sw->num_ports;
328 least_hops = osm_switch_get_least_hops(p_sw, base_lid);
329 if (least_hops == OSM_NO_PATH)
330 return (OSM_NO_PATH);
333 First, inquire with the forwarding table for an existing
334 route. If one is found, honor it unless:
335 1. the ignore existing flag is set.
336 2. the physical port is not a valid one or not healthy
337 3. the physical port has a remote port (the link is up)
338 4. the port has min-hops to the target (avoid loops)
340 if (!ignore_existing) {
341 port_num = osm_switch_get_port_by_lid(p_sw, lid_ho);
343 if (port_num != OSM_NO_PATH) {
344 CL_ASSERT(port_num < num_ports);
347 osm_node_get_physp_ptr(p_sw->p_node, port_num);
349 Don't be too trusting of the current forwarding table!
350 Verify that the port number is legal and that the
351 LID is reachable through this port.
353 if (p_physp && osm_physp_is_healthy(p_physp) &&
354 osm_physp_get_remote(p_physp)) {
356 osm_switch_get_hop_count(p_sw, base_lid,
359 If we aren't using pre-defined user routes
360 function, then we need to make sure that the
361 current path is the minimum one. In case of
362 having such a user function - this check will
363 not be done, and the old routing will be used.
364 Note: This means that it is the user's job to
365 clean all data in the forwarding tables that
366 he wants to be overridden by the minimum
369 if (hops == least_hops)
376 This algorithm selects a port based on a static load balanced
377 selection across equal hop-count ports.
378 There is lots of room for improved sophistication here,
379 possibly guided by user configuration info.
383 OpenSM routing is "local" - not considering a full lid to lid
384 path. As such we can not guarantee a path will not loop if we
385 do not always follow least hops.
386 So we must abort if not least hops.
389 /* port number starts with one and num_ports is 1 + num phys ports */
390 for (i = start_from; i < start_from + num_ports; i++) {
391 port_num = i%num_ports;
393 osm_switch_get_hop_count(p_sw, base_lid, port_num) !=
397 /* let us make sure it is not down or unhealthy */
398 p_physp = osm_node_get_physp_ptr(p_sw->p_node, port_num);
399 if (!p_physp || !osm_physp_is_healthy(p_physp) ||
401 we require all - non sma ports to be linked
404 !osm_physp_get_remote(p_physp))
408 We located a least-hop port, possibly one of many.
409 For this port, check the running total count of
410 the number of paths through this port. Select
411 the port routing the least number of paths.
414 osm_port_prof_path_count_get(&p_sw->p_prof[port_num]);
417 Advanced LMC routing requires tracking of the
418 best port by the node connected to the other side of
421 if (routing_for_lmc) {
422 /* Is the sys guid already used ? */
423 p_remote_guid = osm_switch_find_sys_guid_count(p_sw,
427 /* If not update the least hops for this case */
428 if (!p_remote_guid) {
429 if (check_count < least_paths_other_sys) {
430 least_paths_other_sys = check_count;
431 best_port_other_sys = port_num;
432 least_forwarded_to = 0;
434 } else { /* same sys found - try node */
435 /* Else is the node guid already used ? */
436 p_remote_guid = osm_switch_find_node_guid_count(p_sw,
440 /* If not update the least hops for this case */
442 && check_count < least_paths_other_nodes) {
443 least_paths_other_nodes = check_count;
444 best_port_other_node = port_num;
445 least_forwarded_to = 0;
447 /* else prior sys and node guid already used */
449 } /* same sys found */
452 /* routing for LMC mode */
454 the count is min but also lower then the max subscribed
456 if (check_count < least_paths) {
458 /* Get the Remote Node */
459 p_rem_physp = osm_physp_get_remote(p_physp);
461 osm_physp_get_node_ptr(p_rem_physp);
462 /* use the first dimension, but spread
463 * traffic out among the group of ports
464 * representing that dimension */
466 if (p_rem_node != p_rem_node_first)
469 p_rem_node_first = p_rem_node;
472 best_port = port_num;
473 least_paths = check_count;
476 && p_remote_guid->forwarded_to < least_forwarded_to)
477 least_forwarded_to = p_remote_guid->forwarded_to;
478 } else if (routing_for_lmc
480 && check_count == least_paths
481 && p_remote_guid->forwarded_to < least_forwarded_to) {
482 least_forwarded_to = p_remote_guid->forwarded_to;
483 best_port = port_num;
487 if (port_found == FALSE)
488 return (OSM_NO_PATH);
491 if we are in enhanced routing mode and the best port is not
494 if (routing_for_lmc && best_port) {
495 /* Select the least hop port of the non used sys first */
496 if (best_port_other_sys)
497 best_port = best_port_other_sys;
498 else if (best_port_other_node)
499 best_port = best_port_other_node;
505 /**********************************************************************
506 **********************************************************************/
507 void osm_switch_clear_hops(IN osm_switch_t * p_sw)
511 for (i = 0; i < p_sw->num_hops; i++)
513 memset(p_sw->hops[i], OSM_NO_PATH, p_sw->num_ports);
516 /**********************************************************************
517 **********************************************************************/
519 osm_switch_prepare_path_rebuild(IN osm_switch_t * p_sw, IN uint16_t max_lids)
524 for (i = 0; i < p_sw->num_ports; i++)
525 osm_port_prof_construct(&p_sw->p_prof[i]);
527 osm_switch_clear_hops(p_sw);
529 if (!p_sw->new_lft &&
530 !(p_sw->new_lft = malloc(IB_LID_UCAST_END_HO + 1)))
531 return IB_INSUFFICIENT_MEMORY;
533 memset(p_sw->new_lft, OSM_NO_PATH, IB_LID_UCAST_END_HO + 1);
536 hops = malloc((max_lids + 1) * sizeof(hops[0]));
539 memset(hops, 0, (max_lids + 1) * sizeof(hops[0]));
541 p_sw->num_hops = max_lids + 1;
542 } else if (max_lids + 1 > p_sw->num_hops) {
545 hops = malloc((max_lids + 1) * sizeof(hops[0]));
548 memcpy(hops, p_sw->hops, p_sw->num_hops * sizeof(hops[0]));
549 memset(hops + p_sw->num_hops, 0,
550 (max_lids + 1 - p_sw->num_hops) * sizeof(hops[0]));
551 old_hops = p_sw->hops;
553 p_sw->num_hops = max_lids + 1;
556 p_sw->max_lid_ho = max_lids;
561 /**********************************************************************
562 **********************************************************************/
564 osm_switch_get_port_least_hops(IN const osm_switch_t * const p_sw,
565 IN const osm_port_t * p_port)
569 if (p_port->p_node->sw) {
570 if (p_port->p_node->sw == p_sw)
572 lid = osm_node_get_base_lid(p_port->p_node, 0);
573 return osm_switch_get_least_hops(p_sw, cl_ntoh16(lid));
575 osm_physp_t *p = p_port->p_physp;
578 if (!p || !p->p_remote_physp || !p->p_remote_physp->p_node->sw)
580 if (p->p_remote_physp->p_node->sw == p_sw)
582 lid = osm_node_get_base_lid(p->p_remote_physp->p_node, 0);
583 hops = osm_switch_get_least_hops(p_sw, cl_ntoh16(lid));
584 return hops != OSM_NO_PATH ? hops + 1 : OSM_NO_PATH;
588 /**********************************************************************
589 **********************************************************************/
591 osm_switch_recommend_mcast_path(IN osm_switch_t * const p_sw,
592 IN osm_port_t * p_port,
593 IN uint16_t const mlid_ho,
594 IN boolean_t const ignore_existing)
602 CL_ASSERT(mlid_ho >= IB_LID_MCAST_START_HO);
604 if (p_port->p_node->sw) {
605 if (p_port->p_node->sw == p_sw)
607 base_lid = osm_port_get_base_lid(p_port);
609 osm_physp_t *p_physp = p_port->p_physp;
610 if (!p_physp || !p_physp->p_remote_physp ||
611 !p_physp->p_remote_physp->p_node->sw)
613 if (p_physp->p_remote_physp->p_node->sw == p_sw)
614 return p_physp->p_remote_physp->port_num;
616 osm_node_get_base_lid(p_physp->p_remote_physp->p_node, 0);
618 base_lid = cl_ntoh16(base_lid);
619 num_ports = p_sw->num_ports;
622 If the user wants us to ignore existing multicast routes,
623 then simply return the shortest hop count path to the
626 Otherwise, return the first port that has a path to the target,
627 picking from the ports that are already in the multicast group.
629 if (!ignore_existing) {
630 for (port_num = 1; port_num < num_ports; port_num++) {
631 if (!osm_mcast_tbl_is_port
632 (&p_sw->mcast_tbl, mlid_ho, port_num))
635 Don't be too trusting of the current forwarding table!
636 Verify that the LID is reachable through this port.
639 osm_switch_get_hop_count(p_sw, base_lid, port_num);
640 if (hops != OSM_NO_PATH)
646 Either no existing mcast paths reach this port or we are
647 ignoring existing paths.
649 Determine the best multicast path to the target. Note that this
650 algorithm is slightly different from the one used for unicast route
651 recommendation. In this case (multicast), we must NOT
652 perform any sort of load balancing. We MUST take the FIRST
653 port found that has <= the lowest hop count path. This prevents
654 more than one multicast path to the same remote switch which
655 prevents a multicast loop. Multicast loops are bad since the same
656 multicast packet will go around and around, inevitably creating
657 a black hole that will destroy the Earth in a firey conflagration.
659 least_hops = osm_switch_get_least_hops(p_sw, base_lid);
660 for (port_num = 1; port_num < num_ports; port_num++)
661 if (osm_switch_get_hop_count(p_sw, base_lid, port_num) ==
665 CL_ASSERT(port_num < num_ports);