2 * Copyright (c) 2004-2008 Voltaire Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 #endif /* HAVE_CONFIG_H */
42 #include <sys/types.h>
45 #include <sys/ioctl.h>
46 #include <netinet/in.h>
53 #define IB_OPENIB_OUI (0x001405)
55 #ifdef HAVE_VALGRIND_MEMCHECK_H
57 # include <valgrind/memcheck.h>
59 # ifndef VALGRIND_MAKE_MEM_DEFINED
60 # warning "Valgrind support requested, but VALGRIND_MAKE_MEM_DEFINED not available"
63 #endif /* HAVE_VALGRIND_MEMCHECK_H */
65 #ifndef VALGRIND_MAKE_MEM_DEFINED
66 # define VALGRIND_MAKE_MEM_DEFINED(addr,len)
69 typedef struct ib_user_mad_reg_req {
71 uint32_t method_mask[4];
74 uint8_t mgmt_class_version;
77 } ib_user_mad_reg_req_t;
79 #define TRACE if (umaddebug) IBWARN
80 #define DEBUG if (umaddebug) IBWARN
84 #define UMAD_DEV_FILE_SZ 256
86 static char *def_ca_name = "mthca0";
87 static int def_ca_port = 1;
89 static unsigned abi_version;
90 static unsigned new_user_mad_api;
92 /*************************************
96 find_cached_ca(char *ca_name, umad_ca_t *ca)
98 return 0; /* caching not implemented yet */
102 put_ca(umad_ca_t *ca)
104 return 0; /* caching not implemented yet */
108 release_port(umad_port_t *port)
112 port->pkeys_size = 0;
116 static int check_for_digit_name(const struct dirent *dent)
118 const char *p = dent->d_name;
119 while (*p && isdigit(*p))
125 get_port(char *ca_name, char *dir, int portnum, umad_port_t *port)
129 struct dirent **namelist = NULL;
132 strncpy(port->ca_name, ca_name, sizeof port->ca_name - 1);
133 port->portnum = portnum;
136 len = snprintf(port_dir, sizeof(port_dir), "%s/%d", dir, portnum);
137 if (len < 0 || len > sizeof(port_dir))
140 if (sys_read_uint(port_dir, SYS_PORT_LMC, &port->lmc) < 0)
142 if (sys_read_uint(port_dir, SYS_PORT_SMLID, &port->sm_lid) < 0)
144 if (sys_read_uint(port_dir, SYS_PORT_SMSL, &port->sm_sl) < 0)
146 if (sys_read_uint(port_dir, SYS_PORT_LID, &port->base_lid) < 0)
148 if (sys_read_uint(port_dir, SYS_PORT_STATE, &port->state) < 0)
150 if (sys_read_uint(port_dir, SYS_PORT_PHY_STATE, &port->phys_state) < 0)
152 if (sys_read_uint(port_dir, SYS_PORT_RATE, &port->rate) < 0)
154 if (sys_read_uint64(port_dir, SYS_PORT_CAPMASK, &port->capmask) < 0)
157 port->capmask = htonl(port->capmask);
159 if (sys_read_gid(port_dir, SYS_PORT_GID, gid) < 0)
162 memcpy(&port->gid_prefix, gid, sizeof port->gid_prefix);
163 memcpy(&port->port_guid, gid + 8, sizeof port->port_guid);
165 snprintf(port_dir + len, sizeof(port_dir) - len, "/pkeys");
166 ret = sys_scandir(port_dir, &namelist, check_for_digit_name, NULL);
168 IBWARN("no pkeys found for %s:%u (at dir %s)...",
169 port->ca_name, port->portnum, port_dir);
172 port->pkeys = calloc(ret, sizeof(port->pkeys[0]));
174 IBWARN("get_port: calloc failed: %s", strerror(errno));
177 for (i = 0; i < ret ; i++) {
179 idx = strtoul(namelist[i]->d_name, NULL, 0);
180 sys_read_uint(port_dir, namelist[i]->d_name, &val);
181 port->pkeys[idx] = val;
184 port->pkeys_size = ret;
187 port_dir[len] = '\0';
189 /* FIXME: handle gids */
195 for (i = 0; i < ret ; i++)
205 release_ca(umad_ca_t *ca)
209 for (i = 0; i <= ca->numports; i++) {
212 release_port(ca->ports[i]);
220 * if *port > 0, check ca[port] state. Otherwise set *port to
221 * the first port that is active, and if such is not found, to
222 * the first port that is link up and if none are linkup, then
223 * the first port that is not disabled. Otherwise return -1.
226 resolve_ca_port(char *ca_name, int *port)
229 int active = -1, up = -1;
232 TRACE("checking ca '%s'", ca_name);
234 if (umad_get_ca(ca_name, &ca) < 0)
237 if (ca.node_type == 2) {
238 *port = 0; /* switch sma port 0 */
242 if (*port > 0) { /* check only the port the user wants */
243 if (*port > ca.numports)
245 if (!ca.ports[*port])
247 if (ca.ports[*port]->state == 4)
249 if (ca.ports[*port]->phys_state != 3)
254 for (i = 0; i <= ca.numports; i++) {
255 DEBUG("checking port %d", i);
258 if (up < 0 && ca.ports[i]->phys_state == 5)
260 if (ca.ports[i]->state == 4) {
262 DEBUG("found active port %d", i);
267 if (active == -1 && up == -1) { /* no active or linkup port found */
268 for (i = 0; i <= ca.numports; i++) {
269 DEBUG("checking port %d", i);
272 if (ca.ports[i]->phys_state != 3) {
289 resolve_ca_name(char *ca_name, int *best_port)
291 static char names[UMAD_MAX_DEVICES][UMAD_CA_NAME_LEN];
292 int phys_found = -1, port_found = 0, port, port_type;
295 if (ca_name && (!best_port || *best_port))
299 if (resolve_ca_port(ca_name, best_port) < 0)
304 /* Get the list of CA names */
305 if ((n = umad_get_cas_names((void *)names, 20)) < 0)
308 /* Find the first existing CA with an active port */
309 for (caidx = 0; caidx < n; caidx++) {
310 TRACE("checking ca '%s'", names[caidx]);
312 port = best_port ? *best_port : 0;
313 if ((port_type = resolve_ca_port(names[caidx], &port)) < 0)
316 DEBUG("found ca %s with port %d type %d",
317 names[caidx], port, port_type);
322 DEBUG("found ca %s with active port %d",
324 return (char *)(names + caidx);
327 if (phys_found == -1) {
333 DEBUG("phys found %d on %s port %d",
334 phys_found, phys_found >=0 ? names[phys_found] : 0, port_found);
335 if (phys_found >= 0) {
337 *best_port = port_found;
338 return names[phys_found];
342 *best_port = def_ca_port;
347 get_ca(char *ca_name, umad_ca_t *ca)
353 struct dirent **namelist;
357 strncpy(ca->ca_name, ca_name, sizeof ca->ca_name);
359 snprintf(dir_name, sizeof(dir_name), "%s/%s", SYS_INFINIBAND,
362 if ((r = sys_read_uint(dir_name, SYS_NODE_TYPE, &ca->node_type)) < 0)
364 if (sys_read_string(dir_name, SYS_CA_FW_VERS, ca->fw_ver,
365 sizeof ca->fw_ver) < 0)
366 ca->fw_ver[0] = '\0';
367 if (sys_read_string(dir_name, SYS_CA_HW_VERS, ca->hw_ver,
368 sizeof ca->hw_ver) < 0)
369 ca->hw_ver[0] = '\0';
370 if ((r = sys_read_string(dir_name, SYS_CA_TYPE, ca->ca_type,
371 sizeof ca->ca_type)) < 0)
372 ca->ca_type[0] = '\0';
373 if ((r = sys_read_guid(dir_name, SYS_CA_NODE_GUID, &ca->node_guid)) < 0)
375 if ((r = sys_read_guid(dir_name, SYS_CA_SYS_GUID, &ca->system_guid)) < 0)
378 snprintf(dir_name, sizeof(dir_name), "%s/%s/%s",
379 SYS_INFINIBAND, ca->ca_name, SYS_CA_PORTS_DIR);
382 if (!(dir = opendir(dir_name)))
386 if ((r = sys_scandir(dir_name, &namelist, 0, alphasort)) < 0) {
387 ret = errno < 0 ? errno : -EIO;
393 memset(ca->ports, 0, sizeof ca->ports);
394 for (i = 0; i < r; i++) {
396 if (!strcmp(".", namelist[i]->d_name) ||
397 !strcmp("..", namelist[i]->d_name))
399 if (strcmp("0", namelist[i]->d_name) &&
400 ((portnum = atoi(namelist[i]->d_name)) <= 0 ||
401 portnum >= UMAD_CA_MAX_PORTS)) {
405 if (!(ca->ports[portnum] = calloc(1, sizeof(*ca->ports[portnum])))) {
409 if (get_port(ca_name, dir_name, portnum, ca->ports[portnum]) < 0) {
410 free(ca->ports[portnum]);
411 ca->ports[portnum] = NULL;
415 if (ca->numports < portnum)
416 ca->numports = portnum;
419 for (i = 0; i < r; i++)
430 for (i = 0; i < r; i++)
443 umad_id_to_dev(int umad_id, char *dev, unsigned *port)
448 snprintf(path, sizeof(path), SYS_INFINIBAND_MAD "/umad%d", umad_id);
450 if ((r = sys_read_string(path, SYS_IB_MAD_DEV, dev, UMAD_CA_NAME_LEN)) < 0)
453 if ((r = sys_read_uint(path, SYS_IB_MAD_PORT, port)) < 0)
460 dev_to_umad_id(char *dev, unsigned port)
462 char umad_dev[UMAD_CA_NAME_LEN];
466 for (id = 0; id < UMAD_MAX_PORTS; id++) {
467 if (umad_id_to_dev(id, umad_dev, &umad_port) < 0)
469 if (strncmp(dev, umad_dev, UMAD_CA_NAME_LEN))
471 if (port != umad_port)
474 DEBUG("mapped %s %d to %d", dev, port, id);
478 return -1; /* not found */
481 /*******************************
489 if (sys_read_uint(IB_UMAD_ABI_DIR, IB_UMAD_ABI_FILE, &abi_version) < 0) {
490 IBWARN("can't read ABI version from %s/%s (%m): is ib_umad module loaded?",
491 IB_UMAD_ABI_DIR, IB_UMAD_ABI_FILE);
494 if (abi_version < IB_UMAD_ABI_VERSION) {
495 IBWARN("wrong ABI version: %s/%s is %d but library minimal ABI is %d",
496 IB_UMAD_ABI_DIR, IB_UMAD_ABI_FILE, abi_version, IB_UMAD_ABI_VERSION);
506 /* FIXME - verify that all ports are closed */
510 static unsigned is_ib_type(char *ca_name)
515 snprintf(dir_name, sizeof(dir_name), "%s/%s", SYS_INFINIBAND, ca_name);
517 if (sys_read_uint(dir_name, SYS_NODE_TYPE, &type) < 0)
520 return type >= 1 && type <= 3 ? 1 : 0;
524 umad_get_cas_names(char cas[][UMAD_CA_NAME_LEN], int max)
526 struct dirent **namelist;
529 TRACE("max %d", max);
531 n = sys_scandir(SYS_INFINIBAND, &namelist, NULL, alphasort);
533 for (i = 0; i < n; i++) {
534 if (strcmp(namelist[i]->d_name, ".") &&
535 strcmp(namelist[i]->d_name, "..")) {
536 if (j < max && is_ib_type(namelist[i]->d_name))
537 strncpy(cas[j++], namelist[i]->d_name,
542 DEBUG("return %d cas", j);
544 /* Is this still needed ? */
545 strncpy((char *)cas, def_ca_name, UMAD_CA_NAME_LEN);
546 DEBUG("return 1 ca");
555 umad_get_ca_portguids(char *ca_name, uint64_t *portguids, int max)
560 TRACE("ca name %s max port guids %d", ca_name, max);
561 if (!(ca_name = resolve_ca_name(ca_name, 0)))
564 if (umad_get_ca(ca_name, &ca) < 0)
568 if (ca.numports + 1 > max) {
573 for (i = 0; i <= ca.numports; i++)
574 portguids[ports++] = ca.ports[i] ? ca.ports[i]->port_guid : 0;
578 DEBUG("%s: %d ports", ca_name, ports);
584 umad_get_issm_path(char *ca_name, int portnum, char path[], int max)
588 TRACE("ca %s port %d", ca_name, portnum);
590 if (!(ca_name = resolve_ca_name(ca_name, &portnum)))
593 if ((umad_id = dev_to_umad_id(ca_name, portnum)) < 0)
596 snprintf(path, max, "%s/issm%u", UMAD_DEV_DIR , umad_id);
602 umad_open_port(char *ca_name, int portnum)
604 char dev_file[UMAD_DEV_FILE_SZ];
607 TRACE("ca %s port %d", ca_name, portnum);
609 if (!(ca_name = resolve_ca_name(ca_name, &portnum)))
612 DEBUG("opening %s port %d", ca_name, portnum);
614 if ((umad_id = dev_to_umad_id(ca_name, portnum)) < 0)
617 snprintf(dev_file, sizeof(dev_file), "%s/umad%d",
618 UMAD_DEV_DIR , umad_id);
620 if ((fd = open(dev_file, O_RDWR|O_NONBLOCK)) < 0) {
621 DEBUG("open %s failed: %s", dev_file, strerror(errno));
625 if (abi_version > 5 || !ioctl(fd, IB_USER_MAD_ENABLE_PKEY, NULL))
626 new_user_mad_api = 1;
628 new_user_mad_api = 0;
630 DEBUG("opened %s fd %d portid %d", dev_file, fd, umad_id);
635 umad_get_ca(char *ca_name, umad_ca_t *ca)
639 TRACE("ca_name %s", ca_name);
640 if (!(ca_name = resolve_ca_name(ca_name, 0)))
643 if (find_cached_ca(ca_name, ca) > 0)
646 if ((r = get_ca(ca_name, ca)) < 0)
649 DEBUG("opened %s", ca_name);
654 umad_release_ca(umad_ca_t *ca)
658 TRACE("ca_name %s", ca->ca_name);
662 if ((r = release_ca(ca)) < 0)
665 DEBUG("releasing %s", ca->ca_name);
670 umad_get_port(char *ca_name, int portnum, umad_port_t *port)
674 TRACE("ca_name %s portnum %d", ca_name, portnum);
676 if (!(ca_name = resolve_ca_name(ca_name, &portnum)))
679 snprintf(dir_name, sizeof(dir_name), "%s/%s/%s",
680 SYS_INFINIBAND, ca_name, SYS_CA_PORTS_DIR);
682 return get_port(ca_name, dir_name, portnum, port);
686 umad_release_port(umad_port_t *port)
690 TRACE("port %s:%d", port->ca_name, port->portnum);
694 if ((r = release_port(port)) < 0)
697 DEBUG("releasing %s:%d", port->ca_name, port->portnum);
702 umad_close_port(int fd)
705 DEBUG("closed fd %d", fd);
710 umad_get_mad(void *umad)
712 return new_user_mad_api ? ((struct ib_user_mad *)umad)->data :
713 (void *)&((struct ib_user_mad *)umad)->addr.pkey_index;
719 return new_user_mad_api ? sizeof (struct ib_user_mad) :
720 sizeof(struct ib_user_mad) - 8;
724 umad_set_grh(void *umad, void *mad_addr)
726 struct ib_user_mad *mad = umad;
727 struct ib_mad_addr *addr = mad_addr;
730 mad->addr.grh_present = 1;
731 memcpy(mad->addr.gid, addr->gid, 16);
732 mad->addr.flow_label = htonl(addr->flow_label);
733 mad->addr.hop_limit = addr->hop_limit;
734 mad->addr.traffic_class = addr->traffic_class;
736 mad->addr.grh_present = 0;
741 umad_set_pkey(void *umad, int pkey_index)
743 struct ib_user_mad *mad = umad;
745 if (new_user_mad_api)
746 mad->addr.pkey_index = pkey_index;
752 umad_get_pkey(void *umad)
754 struct ib_user_mad *mad = umad;
756 if (new_user_mad_api)
757 return mad->addr.pkey_index;
763 umad_set_addr(void *umad, int dlid, int dqp, int sl, int qkey)
765 struct ib_user_mad *mad = umad;
767 TRACE("umad %p dlid %d dqp %d sl %d, qkey %x",
768 umad, dlid, dqp, sl, qkey);
769 mad->addr.qpn = htonl(dqp);
770 mad->addr.lid = htons(dlid);
771 mad->addr.qkey = htonl(qkey);
778 umad_set_addr_net(void *umad, int dlid, int dqp, int sl, int qkey)
780 struct ib_user_mad *mad = umad;
782 TRACE("umad %p dlid %d dqp %d sl %d qkey %x",
783 umad, ntohs(dlid), ntohl(dqp), sl, ntohl(qkey));
785 mad->addr.lid = dlid;
786 mad->addr.qkey = qkey;
793 umad_send(int fd, int agentid, void *umad, int length,
794 int timeout_ms, int retries)
796 struct ib_user_mad *mad = umad;
799 TRACE("fd %d agentid %d umad %p timeout %u",
800 fd, agentid, umad, timeout_ms);
803 mad->timeout_ms = timeout_ms;
804 mad->retries = retries;
805 mad->agent_id = agentid;
810 n = write(fd, mad, length + umad_size());
811 if (n == length + umad_size())
814 DEBUG("write returned %d != sizeof umad %zu + length %d (%m)",
815 n, umad_size(), length);
822 dev_poll(int fd, int timeout_ms)
828 ufds.events = POLLIN;
830 if ((n = poll(&ufds, 1, timeout_ms)) == 1)
840 umad_recv(int fd, void *umad, int *length, int timeout_ms)
842 struct ib_user_mad *mad = umad;
846 TRACE("fd %d umad %p timeout %u", fd, umad, timeout_ms);
848 if (!umad || !length) {
853 if (timeout_ms && (n = dev_poll(fd, timeout_ms)) < 0) {
859 n = read(fd, umad, umad_size() + *length);
861 VALGRIND_MAKE_MEM_DEFINED(umad, umad_size() + *length);
863 if ((n >= 0) && (n <= umad_size() + *length)) {
864 DEBUG("mad received by agent %d length %d", mad->agent_id, n);
866 *length = n - umad_size();
869 return mad->agent_id;
872 if (n == -EWOULDBLOCK) {
878 DEBUG("read returned %zu > sizeof umad %zu + length %d (%m)",
879 mad->length - umad_size(), umad_size(), *length);
881 *length = mad->length - umad_size();
888 umad_poll(int fd, int timeout_ms)
890 TRACE("fd %d timeout %u", fd, timeout_ms);
891 return dev_poll(fd, timeout_ms);
902 umad_register_oui(int fd, int mgmt_class, uint8_t rmpp_version,
903 uint8_t oui[3], long method_mask[])
905 struct ib_user_mad_reg_req req;
907 TRACE("fd %d mgmt_class %u rmpp_version %d oui 0x%x%x%x method_mask %p",
908 fd, mgmt_class, (int)rmpp_version, (int)oui[0], (int)oui[1],
909 (int)oui[2], method_mask);
911 if (mgmt_class < 0x30 || mgmt_class > 0x4f) {
912 DEBUG("mgmt class %d not in vendor range 2", mgmt_class);
917 req.mgmt_class = mgmt_class;
918 req.mgmt_class_version = 1;
919 memcpy(req.oui, oui, sizeof req.oui);
920 req.rmpp_version = rmpp_version;
923 memcpy(req.method_mask, method_mask, sizeof req.method_mask);
925 memset(req.method_mask, 0, sizeof req.method_mask);
927 VALGRIND_MAKE_MEM_DEFINED(&req, sizeof req);
929 if (!ioctl(fd, IB_USER_MAD_REGISTER_AGENT, (void *)&req)) {
930 DEBUG("fd %d registered to use agent %d qp %d class 0x%x oui %p",
931 fd, req.id, req.qpn, req.mgmt_class, oui);
932 return req.id; /* return agentid */
935 DEBUG("fd %d registering qp %d class 0x%x version %d oui %p failed: %m",
936 fd, req.qpn, req.mgmt_class, req.mgmt_class_version, oui);
941 umad_register(int fd, int mgmt_class, int mgmt_version,
942 uint8_t rmpp_version, long method_mask[])
944 struct ib_user_mad_reg_req req;
945 uint32_t oui = htonl(IB_OPENIB_OUI);
948 TRACE("fd %d mgmt_class %u mgmt_version %u rmpp_version %d method_mask %p",
949 fd, mgmt_class, mgmt_version, rmpp_version, method_mask);
951 req.qpn = qp = (mgmt_class == 0x1 || mgmt_class == 0x81) ? 0 : 1;
952 req.mgmt_class = mgmt_class;
953 req.mgmt_class_version = mgmt_version;
954 req.rmpp_version = rmpp_version;
957 memcpy(req.method_mask, method_mask, sizeof req.method_mask);
959 memset(req.method_mask, 0, sizeof req.method_mask);
961 memcpy(&req.oui, (char *)&oui + 1, sizeof req.oui);
963 VALGRIND_MAKE_MEM_DEFINED(&req, sizeof req);
965 if (!ioctl(fd, IB_USER_MAD_REGISTER_AGENT, (void *)&req)) {
966 DEBUG("fd %d registered to use agent %d qp %d",
968 return req.id; /* return agentid */
971 DEBUG("fd %d registering qp %d class 0x%x version %d failed: %m",
972 fd, qp, mgmt_class, mgmt_version);
977 umad_unregister(int fd, int agentid)
979 TRACE("fd %d unregistering agent %d", fd, agentid);
980 return ioctl(fd, IB_USER_MAD_UNREGISTER_AGENT, &agentid);
984 umad_status(void *umad)
986 struct ib_user_mad *mad = umad;
992 umad_get_mad_addr(void *umad)
994 struct ib_user_mad *mad = umad;
1000 umad_debug(int level)
1008 umad_addr_dump(ib_mad_addr_t *addr)
1010 #define HEX(x) ((x) < 10 ? '0' + (x) : 'a' + ((x) -10))
1014 for (i = 0; i < sizeof addr->gid; i++) {
1015 gid_str[i*2] = HEX(addr->gid[i] >> 4);
1016 gid_str[i*2+1] = HEX(addr->gid[i] & 0xf);
1019 IBWARN("qpn %d qkey 0x%x lid 0x%x sl %d\n"
1020 "grh_present %d gid_index %d hop_limit %d traffic_class %d flow_label 0x%x pkey_index 0x%x\n"
1022 ntohl(addr->qpn), ntohl(addr->qkey), ntohs(addr->lid), addr->sl,
1023 addr->grh_present, (int)addr->gid_index, (int)addr->hop_limit,
1024 (int)addr->traffic_class, addr->flow_label, addr->pkey_index,
1029 umad_dump(void *umad)
1031 struct ib_user_mad * mad = umad;
1033 IBWARN("agent id %d status %x timeout %d",
1034 mad->agent_id, mad->status, mad->timeout_ms);
1035 umad_addr_dump(&mad->addr);