2 * Copyright (c) 2004-2008 Voltaire Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 #endif /* HAVE_CONFIG_H */
45 #include <netinet/in.h>
49 #include <infiniband/common.h>
50 #include <infiniband/umad.h>
51 #include <infiniband/mad.h>
52 #include <infiniband/complib/cl_nodenamemap.h>
54 #include "ibdiag_common.h"
58 static char *node_type_str[] = {
66 static int timeout = 0; /* ms */
71 char *argv0 = "ibtracert";
73 static char *node_name_map_file = NULL;
74 static nn_map_t *node_name_map = NULL;
76 typedef struct Port Port;
77 typedef struct Switch Switch;
78 typedef struct Node Node;
111 uint64_t nodeguid; /* also portguid */
116 Node *nodesdist[MAXHOPS];
117 uint64_t target_portguid;
120 get_node(Node *node, Port *port, ib_portid_t *portid)
122 void *pi = port->portinfo, *ni = node->nodeinfo, *nd = node->nodedesc;
125 if (!smp_query(ni, portid, IB_ATTR_NODE_INFO, 0, timeout))
128 if (!smp_query(nd, portid, IB_ATTR_NODE_DESC, 0, timeout))
131 for (s = nd, e = s + 64; s < e; s++) {
138 if (!smp_query(pi, portid, IB_ATTR_PORT_INFO, 0, timeout))
141 mad_decode_field(ni, IB_NODE_GUID_F, &node->nodeguid);
142 mad_decode_field(ni, IB_NODE_TYPE_F, &node->type);
143 mad_decode_field(ni, IB_NODE_NPORTS_F, &node->numports);
145 mad_decode_field(ni, IB_NODE_PORT_GUID_F, &port->portguid);
146 mad_decode_field(ni, IB_NODE_LOCAL_PORT_F, &port->portnum);
147 mad_decode_field(pi, IB_PORT_LID_F, &port->lid);
148 mad_decode_field(pi, IB_PORT_LMC_F, &port->lmc);
149 mad_decode_field(pi, IB_PORT_STATE_F, &port->state);
151 DEBUG("portid %s: got node %" PRIx64 " '%s'", portid2str(portid), node->nodeguid, node->nodedesc);
156 switch_lookup(Switch *sw, ib_portid_t *portid, int lid)
158 void *si = sw->switchinfo, *fdb = sw->fdb;
160 if (!smp_query(si, portid, IB_ATTR_SWITCH_INFO, 0, timeout))
163 mad_decode_field(si, IB_SW_LINEAR_FDB_CAP_F, &sw->linearcap);
164 mad_decode_field(si, IB_SW_LINEAR_FDB_TOP_F, &sw->linearFDBtop);
166 if (lid > sw->linearcap && lid > sw->linearFDBtop)
169 if (!smp_query(fdb, portid, IB_ATTR_LINEARFORWTBL, lid / 64, timeout))
172 DEBUG("portid %s: forward lid %d to port %d",
173 portid2str(portid), lid, sw->fdb[lid % 64]);
174 return sw->fdb[lid % 64];
178 sameport(Port *a, Port *b)
180 return a->portguid == b->portguid || (force && a->lid == b->lid);
184 extend_dpath(ib_dr_path_t *path, int nextport)
186 if (path->cnt+2 >= sizeof(path->p))
189 path->p[path->cnt] = nextport;
194 dump_endnode(int dump, char *prompt, Node *node, Port *port)
196 char *nodename = NULL;
201 fprintf(f, "%s {0x%016" PRIx64 "}[%d]\n",
202 prompt, node->nodeguid,
203 node->type == IB_NODE_SWITCH ? 0 : port->portnum);
207 nodename = remap_node_name(node_name_map, node->nodeguid, node->nodedesc);
209 fprintf(f, "%s %s {0x%016" PRIx64 "} portnum %d lid %u-%u \"%s\"\n",
211 (node->type <= IB_NODE_MAX ? node_type_str[node->type] : "???"),
212 node->nodeguid, node->type == IB_NODE_SWITCH ? 0 : port->portnum,
213 port->lid, port->lid + (1 << port->lmc) - 1,
220 dump_route(int dump, Node *node, int outport, Port *port)
222 char *nodename = NULL;
224 if (!dump && !verbose)
227 nodename = remap_node_name(node_name_map, node->nodeguid, node->nodedesc);
230 fprintf(f, "[%d] -> {0x%016" PRIx64 "}[%d]\n",
231 outport, port->portguid, port->portnum);
233 fprintf(f, "[%d] -> %s port {0x%016" PRIx64 "}[%d] lid %u-%u \"%s\"\n",
235 (node->type <= IB_NODE_MAX ? node_type_str[node->type] : "???"),
236 port->portguid, port->portnum,
237 port->lid, port->lid + (1 << port->lmc) - 1,
244 find_route(ib_portid_t *from, ib_portid_t *to, int dump)
246 Node *node, fromnode, tonode, nextnode;
247 Port *port, fromport, toport, nextport;
249 int maxhops = MAXHOPS;
250 int portnum, outport;
252 DEBUG("from %s", portid2str(from));
254 if (get_node(&fromnode, &fromport, from) < 0 ||
255 get_node(&tonode, &toport, to) < 0) {
256 IBWARN("can't reach to/from ports");
260 toport.lid = to->lid;
261 IBWARN("Force: look for lid %d", to->lid);
266 portnum = port->portnum;
268 dump_endnode(dump, "From", node, port);
271 if (port->state != 4)
274 if (sameport(port, &toport))
278 if (node->type == IB_NODE_SWITCH) {
279 DEBUG("switch node");
280 if ((outport = switch_lookup(&sw, from, to->lid)) < 0 ||
281 outport > node->numports)
284 if (extend_dpath(&from->drpath, outport) < 0)
287 if (get_node(&nextnode, &nextport, from) < 0) {
288 IBWARN("can't reach port at %s", portid2str(from));
292 if (!sameport(&nextport, &toport))
295 break; /* found SMA port */
297 } else if ((node->type == IB_NODE_CA) ||
298 (node->type == IB_NODE_ROUTER)) {
301 DEBUG("ca or router node");
302 if (!sameport(port, &fromport)) {
303 IBWARN("can't continue: reached CA or router port %" PRIx64 ", lid %d",
304 port->portguid, port->lid);
307 /* we are at CA or router "from" - go one hop back to (hopefully) a switch */
308 if (from->drpath.cnt > 0) {
309 DEBUG("ca or router node - return back 1 hop");
313 if (portnum && extend_dpath(&from->drpath, portnum) < 0)
316 if (get_node(&nextnode, &nextport, from) < 0) {
317 IBWARN("can't reach port at %s", portid2str(from));
320 /* fix port num to be seen from the CA or router side */
322 nextport.portnum = from->drpath.p[from->drpath.cnt+1];
325 if (port->state != 4)
328 portnum = port->portnum;
329 dump_route(dump, node, outport, port);
333 IBWARN("no route found after %d hops", MAXHOPS);
336 dump_endnode(dump, "To", node, port);
340 IBWARN("Bad port state found: node \"%s\" port %d state %d",
341 clean_nodedesc(node->nodedesc), portnum, port->state);
344 IBWARN("Bad out port state found: node \"%s\" outport %d state %d",
345 clean_nodedesc(node->nodedesc), outport, port->state);
348 IBWARN("Bad forwarding table entry found at: node \"%s\" lid entry %d is %d (top %d)",
349 clean_nodedesc(node->nodedesc), to->lid, outport, sw.linearFDBtop);
352 IBWARN("Direct path too long!");
357 /**************************
361 #define HASHGUID(guid) ((uint32_t)(((uint32_t)(guid) * 101) ^ ((uint32_t)((guid) >> 32) * 103)))
365 insert_node(Node *new)
367 static Node *nodestbl[HTSZ];
368 int hash = HASHGUID(new->nodeguid) % HTSZ;
371 for (node = nodestbl[hash]; node; node = node->htnext)
372 if (node->nodeguid == new->nodeguid) {
373 DEBUG("node %" PRIx64 " already exists", new->nodeguid);
377 new->htnext = nodestbl[hash];
378 nodestbl[hash] = new;
384 get_port(Port *port, int portnum, ib_portid_t *portid)
389 port->portnum = portnum;
391 if (!smp_query(pi, portid, IB_ATTR_PORT_INFO, portnum, timeout))
394 mad_decode_field(pi, IB_PORT_LID_F, &port->lid);
395 mad_decode_field(pi, IB_PORT_LMC_F, &port->lmc);
396 mad_decode_field(pi, IB_PORT_STATE_F, &port->state);
397 mad_decode_field(pi, IB_PORT_PHYS_STATE_F, &port->physstate);
399 VERBOSE("portid %s portnum %d: lid %d state %d physstate %d",
400 portid2str(portid), portnum, port->lid, port->state, port->physstate);
405 link_port(Port *port, Node *node)
407 port->next = node->ports;
412 new_node(Node *node, Port *port, ib_portid_t *path, int dist)
414 if (port->portguid == target_portguid) {
415 node->dist = -1; /* tag as target */
416 link_port(port, node);
417 dump_endnode(verbose, "found target", node, port);
418 return 1; /* found; */
421 /* BFS search start with my self */
422 if (insert_node(node) < 0)
423 return -1; /* known switch */
425 VERBOSE("insert dist %d node %p port %d lid %d", dist, node, port->portnum, port->lid);
427 link_port(port, node);
431 node->dnext = nodesdist[dist];
432 nodesdist[dist] = node;
438 switch_mclookup(Node *node, ib_portid_t *portid, int mlid, char *map)
442 void *si = sw.switchinfo;
443 uint16_t *msets = (uint16_t *)mdb;
444 int maxsets, block, i, set;
448 if (!smp_query(si, portid, IB_ATTR_SWITCH_INFO, 0, timeout))
453 mad_decode_field(si, IB_SW_MCAST_FDB_CAP_F, &sw.mccap);
459 maxsets = (node->numports + 15) / 16; /* round up */
461 for (set = 0; set < maxsets; set++) {
462 if (!smp_query(mdb, portid, IB_ATTR_MULTICASTFORWTBL,
463 block | (set << 28), timeout))
466 for (i = 0; i < 16; i++, map++) {
467 uint16_t mask = ntohs(msets[mlid % 32]);
472 VERBOSE("Switch guid 0x%" PRIx64 ": mlid 0x%x is forwarded to port %d",
473 node->nodeguid, mlid + 0xc000, i + set * 16);
481 * Return 1 if found, 0 if not, -1 on errors.
484 find_mcpath(ib_portid_t *from, int mlid)
486 Node *node, *remotenode;
487 Port *port, *remoteport;
490 int dist = 0, leafport = 0;
493 DEBUG("from %s", portid2str(from));
495 if (!(node = calloc(1, sizeof(Node))))
496 IBERROR("out of memory");
498 if (!(port = calloc(1, sizeof(Port))))
499 IBERROR("out of memory");
501 if (get_node(node, port, from) < 0) {
502 IBWARN("can't reach node %s", portid2str(from));
506 node->upnode = 0; /* root */
507 if ((r = new_node(node, port, from, 0)) > 0) {
508 if (node->type != IB_NODE_SWITCH) {
509 IBWARN("ibtracert from CA to CA is unsupported");
510 return 0; /* ibtracert from host to itself is unsupported */
513 if (switch_mclookup(node, from, mlid, map) < 0 ||
519 for (dist = 0; dist < MAXHOPS; dist++) {
521 for (node = nodesdist[dist]; node; node = node->dnext) {
525 VERBOSE("dist %d node %p", dist, node);
526 dump_endnode(verbose, "processing", node, node->ports);
528 memset(map, 0, sizeof(map));
530 if (node->type != IB_NODE_SWITCH) {
533 leafport = path->drpath.p[path->drpath.cnt];
534 map[port->portnum] = 1;
535 node->upport = 0; /* starting here */
536 DEBUG("Starting from CA 0x%" PRIx64 " lid %d port %d (leafport %d)",
537 node->nodeguid, port->lid, port->portnum, leafport);
538 } else { /* switch */
540 /* if starting from a leaf port fix up port (up port) */
541 if (dist == 1 && leafport)
542 node->upport = leafport;
544 if (switch_mclookup(node, path, mlid, map) < 0) {
545 IBWARN("skipping bad Switch 0x%" PRIx64 "",
551 for (i = 1; i <= node->numports; i++) {
552 if (!map[i] || i == node->upport)
555 if (dist == 0 && leafport) {
556 if (from->drpath.cnt > 0)
559 if (!(port = calloc(1, sizeof(Port))))
560 IBERROR("out of memory");
562 if (get_port(port, i, path) < 0) {
563 IBWARN("can't reach node %s port %d", portid2str(path), i);
567 if (port->physstate != 5) { /* LinkUP */
573 link_port(port, node);
576 if (extend_dpath(&path->drpath, i) < 0)
580 if (!(remotenode = calloc(1, sizeof(Node))))
581 IBERROR("out of memory");
583 if (!(remoteport = calloc(1, sizeof(Port))))
584 IBERROR("out of memory");
586 if (get_node(remotenode, remoteport, path) < 0) {
587 IBWARN("NodeInfo on %s port %d failed, skipping port",
588 portid2str(path), i);
589 path->drpath.cnt--; /* restore path */
595 remotenode->upnode = node;
596 remotenode->upport = remoteport->portnum;
597 remoteport->remoteport = port;
599 if ((r = new_node(remotenode, remoteport, path, dist+1)) > 0)
603 dump_endnode(verbose, "new remote",
604 remotenode, remoteport);
605 else if (remotenode->type == IB_NODE_SWITCH)
606 dump_endnode(2, "ERR: circle discovered at",
607 remotenode, remoteport);
609 path->drpath.cnt--; /* restore path */
614 return 0; /* not found */
618 find_target_portguid(ib_portid_t *to)
623 if (get_node(&tonode, &toport, to) < 0) {
624 IBWARN("can't find to port\n");
628 return toport.portguid;
632 dump_mcpath(Node *node, int dumplevel)
634 char *nodename = NULL;
637 dump_mcpath(node->upnode, dumplevel);
639 nodename = remap_node_name(node_name_map, node->nodeguid, node->nodedesc);
642 printf("From %s 0x%" PRIx64 " port %d lid %u-%u \"%s\"\n",
643 (node->type <= IB_NODE_MAX ? node_type_str[node->type] : "???"),
644 node->nodeguid, node->ports->portnum, node->ports->lid,
645 node->ports->lid + (1 << node->ports->lmc) - 1,
652 printf("[%d] -> %s {0x%016" PRIx64 "}[%d]\n",
653 node->ports->remoteport->portnum,
654 (node->type <= IB_NODE_MAX ? node_type_str[node->type] : "???"),
655 node->nodeguid, node->upport);
657 printf("[%d] -> %s 0x%" PRIx64 "[%d] lid %u \"%s\"\n",
658 node->ports->remoteport->portnum,
659 (node->type <= IB_NODE_MAX ? node_type_str[node->type] : "???"),
660 node->nodeguid, node->upport,
661 node->ports->lid, nodename);
666 printf("To %s 0x%" PRIx64 " port %d lid %u-%u \"%s\"\n",
667 (node->type <= IB_NODE_MAX ? node_type_str[node->type] : "???"),
668 node->nodeguid, node->ports->portnum, node->ports->lid,
669 node->ports->lid + (1 << node->ports->lmc) - 1,
676 static int resolve_lid(ib_portid_t *portid, const void *srcport)
678 uint8_t portinfo[64];
681 if (!smp_query_via(portinfo, portid, IB_ATTR_PORT_INFO, 0, 0, srcport))
683 mad_decode_field(portinfo, IB_PORT_LID_F, &lid);
685 ib_portid_set(portid, lid, 0, 0);
695 if (!(basename = strrchr(argv0, '/')))
700 fprintf(stderr, "Usage: %s [-d(ebug) -v(erbose) -D(irect) -G(uids) -n(o_info) -C ca_name -P ca_port "
701 "-s smlid -t(imeout) timeout_ms -m mlid --node-name-map node-name-map ] <src-addr> <dest-addr>\n",
703 fprintf(stderr, "\n\tUnicast examples:\n");
704 fprintf(stderr, "\t\t%s 4 16\t\t\t# show path between lids 4 and 16\n", basename);
705 fprintf(stderr, "\t\t%s -n 4 16\t\t# same, but using simple output format\n", basename);
706 fprintf(stderr, "\t\t%s -G 0x8f1040396522d 0x002c9000100d051\t# use guid addresses\n", basename);
708 fprintf(stderr, "\n\tMulticast example:\n");
709 fprintf(stderr, "\t\t%s -m 0xc000 4 16\t# show multicast path of mlid 0xc000 between lids 4 and 16\n", basename);
714 main(int argc, char **argv)
716 int mgmt_classes[3] = {IB_SMI_CLASS, IB_SMI_DIRECT_CLASS, IB_SA_CLASS};
717 ib_portid_t my_portid = {0};
718 ib_portid_t src_portid = {0};
719 ib_portid_t dest_portid = {0};
720 ib_portid_t *sm_id = 0, sm_portid = {0};
721 int dumplevel = 2, dest_type = IB_DEST_LID, multicast = 0, mlid = 0;
727 static char const str_opts[] = "C:P:t:s:m:dvfDGnVhu";
728 static const struct option long_opts[] = {
731 { "debug", 0, 0, 'd'},
732 { "verbose", 0, 0, 'v'},
733 { "force", 0, 0, 'f'},
734 { "Direct", 0, 0, 'D'},
735 { "Guids", 0, 0, 'G'},
736 { "no_info", 0, 0, 'n'},
737 { "timeout", 1, 0, 't'},
740 { "Version", 0, 0, 'V'},
741 { "help", 0, 0, 'h'},
742 { "usage", 0, 0, 'u'},
743 { "node-name-map", 1, 0, 1},
752 int ch = getopt_long(argc, argv, str_opts, long_opts, NULL);
757 node_name_map_file = strdup(optarg);
763 ca_port = strtoul(optarg, 0, 0);
767 madrpc_show_errors(1);
772 dest_type = IB_DEST_DRPATH;
775 dest_type = IB_DEST_GUID;
779 mlid = strtoul(optarg, 0, 0);
788 if (ib_resolve_portid_str(&sm_portid, optarg, IB_DEST_LID, 0) < 0)
789 IBERROR("can't resolve SM destination port %s", optarg);
793 timeout = strtoul(optarg, 0, 0);
794 madrpc_set_timeout(timeout);
797 madrpc_show_errors(1);
801 fprintf(stderr, "%s %s\n", argv0, get_build_version() );
814 madrpc_init(ca, ca_port, mgmt_classes, 3);
815 node_name_map = open_node_name_map(node_name_map_file);
817 if (ib_resolve_portid_str(&src_portid, argv[0], dest_type, sm_id) < 0)
818 IBERROR("can't resolve source port %s", argv[0]);
820 if (ib_resolve_portid_str(&dest_portid, argv[1], dest_type, sm_id) < 0)
821 IBERROR("can't resolve destination port %s", argv[1]);
823 if (dest_type == IB_DEST_DRPATH) {
824 if (resolve_lid(&src_portid, NULL) < 0)
825 IBERROR("cannot resolve lid for port \'%s\'",
826 portid2str(&src_portid));
827 if (resolve_lid(&dest_portid, NULL) < 0)
828 IBERROR("cannot resolve lid for port \'%s\'",
829 portid2str(&dest_portid));
832 if (dest_portid.lid == 0 || src_portid.lid == 0) {
833 IBWARN("bad src/dest lid");
837 if (dest_type != IB_DEST_DRPATH) {
838 /* first find a direct path to the src port */
839 if (find_route(&my_portid, &src_portid, 0) < 0)
840 IBERROR("can't find a route to the src port");
842 src_portid = my_portid;
846 if (find_route(&src_portid, &dest_portid, dumplevel) < 0)
847 IBERROR("can't find a route from src to dest");
851 IBWARN("invalid MLID; must be 0xc000 or larger");
854 if (!(target_portguid = find_target_portguid(&dest_portid)))
855 IBERROR("can't reach target lid %d", dest_portid.lid);
857 if (!(endnode = find_mcpath(&src_portid, mlid)))
858 IBERROR("can't find a multicast route from src to dest");
860 /* dump multicast path */
861 dump_mcpath(endnode, dumplevel);
863 close_node_name_map(node_name_map);