2 * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2002-2010 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
5 * Copyright (c) 2009-2011 ZIH, TU Dresden, Federal Republic of Germany. All rights reserved.
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39 * Implementation of osm_opensm_t.
40 * This object represents the opensm super object.
41 * This object is part of the opensm family of objects.
46 #endif /* HAVE_CONFIG_H */
51 #include <complib/cl_dispatcher.h>
52 #include <complib/cl_passivelock.h>
53 #include <opensm/osm_file_ids.h>
54 #define FILE_ID OSM_FILE_OPENSM_C
55 #include <vendor/osm_vendor_api.h>
56 #include <opensm/osm_version.h>
57 #include <opensm/osm_base.h>
58 #include <opensm/osm_opensm.h>
59 #include <opensm/osm_log.h>
60 #include <opensm/osm_subnet.h>
61 #include <opensm/osm_sm.h>
62 #include <opensm/osm_vl15intf.h>
63 #include <opensm/osm_event_plugin.h>
64 #include <opensm/osm_congestion_control.h>
66 struct routing_engine_module {
68 int (*setup) (struct osm_routing_engine *, osm_opensm_t *);
71 extern int osm_ucast_minhop_setup(struct osm_routing_engine *, osm_opensm_t *);
72 extern int osm_ucast_updn_setup(struct osm_routing_engine *, osm_opensm_t *);
73 extern int osm_ucast_dnup_setup(struct osm_routing_engine *, osm_opensm_t *);
74 extern int osm_ucast_file_setup(struct osm_routing_engine *, osm_opensm_t *);
75 extern int osm_ucast_ftree_setup(struct osm_routing_engine *, osm_opensm_t *);
76 extern int osm_ucast_lash_setup(struct osm_routing_engine *, osm_opensm_t *);
77 extern int osm_ucast_dor_setup(struct osm_routing_engine *, osm_opensm_t *);
78 extern int osm_ucast_torus2QoS_setup(struct osm_routing_engine *, osm_opensm_t *);
79 extern int osm_ucast_sssp_setup(struct osm_routing_engine *, osm_opensm_t *);
80 extern int osm_ucast_dfsssp_setup(struct osm_routing_engine *, osm_opensm_t *);
82 const static struct routing_engine_module routing_modules[] = {
83 {"minhop", osm_ucast_minhop_setup},
84 {"updn", osm_ucast_updn_setup},
85 {"dnup", osm_ucast_dnup_setup},
86 {"file", osm_ucast_file_setup},
87 {"ftree", osm_ucast_ftree_setup},
88 {"lash", osm_ucast_lash_setup},
89 {"dor", osm_ucast_dor_setup},
90 {"torus-2QoS", osm_ucast_torus2QoS_setup},
91 {"dfsssp", osm_ucast_dfsssp_setup},
92 {"sssp", osm_ucast_sssp_setup},
96 const char *osm_routing_engine_type_str(IN osm_routing_engine_type_t type)
99 case OSM_ROUTING_ENGINE_TYPE_NONE:
101 case OSM_ROUTING_ENGINE_TYPE_MINHOP:
103 case OSM_ROUTING_ENGINE_TYPE_UPDN:
105 case OSM_ROUTING_ENGINE_TYPE_DNUP:
107 case OSM_ROUTING_ENGINE_TYPE_FILE:
109 case OSM_ROUTING_ENGINE_TYPE_FTREE:
111 case OSM_ROUTING_ENGINE_TYPE_LASH:
113 case OSM_ROUTING_ENGINE_TYPE_DOR:
115 case OSM_ROUTING_ENGINE_TYPE_TORUS_2QOS:
117 case OSM_ROUTING_ENGINE_TYPE_DFSSSP:
119 case OSM_ROUTING_ENGINE_TYPE_SSSP:
127 osm_routing_engine_type_t osm_routing_engine_type(IN const char *str)
129 /* For legacy reasons, consider a NULL pointer and the string
130 * "null" as the minhop routing engine.
132 if (!str || !strcasecmp(str, "null")
133 || !strcasecmp(str, "minhop"))
134 return OSM_ROUTING_ENGINE_TYPE_MINHOP;
135 else if (!strcasecmp(str, "none"))
136 return OSM_ROUTING_ENGINE_TYPE_NONE;
137 else if (!strcasecmp(str, "updn"))
138 return OSM_ROUTING_ENGINE_TYPE_UPDN;
139 else if (!strcasecmp(str, "dnup"))
140 return OSM_ROUTING_ENGINE_TYPE_DNUP;
141 else if (!strcasecmp(str, "file"))
142 return OSM_ROUTING_ENGINE_TYPE_FILE;
143 else if (!strcasecmp(str, "ftree"))
144 return OSM_ROUTING_ENGINE_TYPE_FTREE;
145 else if (!strcasecmp(str, "lash"))
146 return OSM_ROUTING_ENGINE_TYPE_LASH;
147 else if (!strcasecmp(str, "dor"))
148 return OSM_ROUTING_ENGINE_TYPE_DOR;
149 else if (!strcasecmp(str, "torus-2QoS"))
150 return OSM_ROUTING_ENGINE_TYPE_TORUS_2QOS;
151 else if (!strcasecmp(str, "sssp"))
152 return OSM_ROUTING_ENGINE_TYPE_SSSP;
153 else if (!strcasecmp(str, "dfsssp"))
154 return OSM_ROUTING_ENGINE_TYPE_DFSSSP;
156 return OSM_ROUTING_ENGINE_TYPE_UNKNOWN;
159 static void append_routing_engine(osm_opensm_t *osm,
160 struct osm_routing_engine *routing_engine)
162 struct osm_routing_engine *r;
164 routing_engine->next = NULL;
166 if (!osm->routing_engine_list) {
167 osm->routing_engine_list = routing_engine;
171 r = osm->routing_engine_list;
175 r->next = routing_engine;
178 static struct osm_routing_engine *setup_routing_engine(osm_opensm_t *osm,
181 struct osm_routing_engine *re;
182 const struct routing_engine_module *m;
184 if (!strcmp(name, "no_fallback")) {
185 osm->no_fallback_routing_engine = TRUE;
189 for (m = routing_modules; m->name && *m->name; m++) {
190 if (!strcmp(m->name, name)) {
191 re = malloc(sizeof(struct osm_routing_engine));
193 OSM_LOG(&osm->log, OSM_LOG_VERBOSE,
194 "memory allocation failed\n");
197 memset(re, 0, sizeof(struct osm_routing_engine));
200 re->type = osm_routing_engine_type(m->name);
201 if (m->setup(re, osm)) {
202 OSM_LOG(&osm->log, OSM_LOG_VERBOSE,
204 " engine \'%s\' failed\n", name);
208 OSM_LOG(&osm->log, OSM_LOG_DEBUG,
209 "\'%s\' routing engine set up\n", re->name);
210 if (re->type == OSM_ROUTING_ENGINE_TYPE_MINHOP)
211 osm->default_routing_engine = re;
216 OSM_LOG(&osm->log, OSM_LOG_ERROR,
217 "cannot find or setup routing engine \'%s\'\n", name);
221 static void setup_routing_engines(osm_opensm_t *osm, const char *engine_names)
223 char *name, *str, *p;
224 struct osm_routing_engine *re;
226 if (engine_names && *engine_names) {
227 str = strdup(engine_names);
228 name = strtok_r(str, ", \t\n", &p);
229 while (name && *name) {
230 re = setup_routing_engine(osm, name);
232 append_routing_engine(osm, re);
233 name = strtok_r(NULL, ", \t\n", &p);
237 if (!osm->default_routing_engine)
238 setup_routing_engine(osm, "minhop");
241 void osm_opensm_construct(IN osm_opensm_t * p_osm)
243 memset(p_osm, 0, sizeof(*p_osm));
244 p_osm->osm_version = OSM_VERSION;
245 osm_subn_construct(&p_osm->subn);
246 osm_db_construct(&p_osm->db);
247 osm_log_construct(&p_osm->log);
250 void osm_opensm_construct_finish(IN osm_opensm_t * p_osm)
252 osm_sm_construct(&p_osm->sm);
253 osm_sa_construct(&p_osm->sa);
254 osm_mad_pool_construct(&p_osm->mad_pool);
255 p_osm->mad_pool_constructed = TRUE;
256 osm_vl15_construct(&p_osm->vl15);
257 p_osm->vl15_constructed = TRUE;
260 static void destroy_routing_engines(osm_opensm_t *osm)
262 struct osm_routing_engine *r, *next;
264 next = osm->routing_engine_list;
269 r->destroy(r->context);
274 static void destroy_plugins(osm_opensm_t *osm)
277 /* remove from the list, and destroy it */
278 while (!cl_is_qlist_empty(&osm->plugin_list)){
279 p = (osm_epi_plugin_t *)cl_qlist_remove_head(&osm->plugin_list);
280 /* plugin is responsible for freeing its own resources */
285 void osm_opensm_destroy(IN osm_opensm_t * p_osm)
287 /* in case of shutdown through exit proc - no ^C */
288 osm_exit_flag = TRUE;
291 * First of all, clear the is_sm bit.
293 if (p_osm->sm.mad_ctrl.h_bind)
294 osm_vendor_set_sm(p_osm->sm.mad_ctrl.h_bind, FALSE);
296 #ifdef ENABLE_OSM_PERF_MGR
297 /* Shutdown the PerfMgr */
298 osm_perfmgr_shutdown(&p_osm->perfmgr);
299 #endif /* ENABLE_OSM_PERF_MGR */
301 osm_congestion_control_shutdown(&p_osm->cc);
304 * - make sure the SM sweeper thread exited
305 * - unbind from QP0 messages
307 osm_sm_shutdown(&p_osm->sm);
310 * - unbind from QP1 messages
312 osm_sa_shutdown(&p_osm->sa);
314 /* cleanup all messages on VL15 fifo that were not sent yet */
315 osm_vl15_shutdown(&p_osm->vl15, &p_osm->mad_pool);
317 /* shut down the dispatcher - so no new messages cross */
318 cl_disp_shutdown(&p_osm->disp);
319 if (p_osm->sa_set_disp_initialized)
320 cl_disp_shutdown(&p_osm->sa_set_disp);
323 if ((p_osm->sm.p_subn->sm_state == IB_SMINFO_STATE_MASTER) &&
324 p_osm->subn.opt.sa_db_dump)
325 osm_sa_db_file_dump(p_osm);
327 /* do the destruction in reverse order as init */
328 destroy_plugins(p_osm);
329 destroy_routing_engines(p_osm);
330 osm_sa_destroy(&p_osm->sa);
331 osm_sm_destroy(&p_osm->sm);
332 #ifdef ENABLE_OSM_PERF_MGR
333 osm_perfmgr_destroy(&p_osm->perfmgr);
334 #endif /* ENABLE_OSM_PERF_MGR */
335 osm_congestion_control_destroy(&p_osm->cc);
338 void osm_opensm_destroy_finish(IN osm_opensm_t * p_osm)
340 osm_db_destroy(&p_osm->db);
341 if (p_osm->vl15_constructed && p_osm->mad_pool_constructed)
342 osm_vl15_destroy(&p_osm->vl15, &p_osm->mad_pool);
343 if (p_osm->mad_pool_constructed)
344 osm_mad_pool_destroy(&p_osm->mad_pool);
345 p_osm->vl15_constructed = FALSE;
346 p_osm->mad_pool_constructed = FALSE;
347 osm_vendor_delete(&p_osm->p_vendor);
348 osm_subn_destroy(&p_osm->subn);
349 cl_disp_destroy(&p_osm->disp);
350 if (p_osm->sa_set_disp_initialized)
351 cl_disp_destroy(&p_osm->sa_set_disp);
352 #ifdef HAVE_LIBPTHREAD
353 pthread_cond_destroy(&p_osm->stats.cond);
354 pthread_mutex_destroy(&p_osm->stats.mutex);
356 cl_event_destroy(&p_osm->stats.event);
358 if (p_osm->node_name_map)
359 close_node_name_map(p_osm->node_name_map);
360 cl_plock_destroy(&p_osm->lock);
362 osm_log_destroy(&p_osm->log);
365 static void load_plugins(osm_opensm_t *osm, const char *plugin_names)
367 osm_epi_plugin_t *epi;
368 char *p_names, *name, *p;
370 p_names = strdup(plugin_names);
371 name = strtok_r(p_names, ", \t\n", &p);
372 while (name && *name) {
373 epi = osm_epi_construct(osm, name);
375 osm_log_v2(&osm->log, OSM_LOG_ERROR, FILE_ID,
376 "ERR 1000: cannot load plugin \'%s\'\n",
379 cl_qlist_insert_tail(&osm->plugin_list, &epi->list);
380 name = strtok_r(NULL, " \t\n", &p);
385 ib_api_status_t osm_opensm_init(IN osm_opensm_t * p_osm,
386 IN const osm_subn_opt_t * p_opt)
388 ib_api_status_t status;
390 /* Can't use log macros here, since we're initializing the log */
391 osm_opensm_construct(p_osm);
394 p_osm->log.daemon = 1;
396 status = osm_log_init_v2(&p_osm->log, p_opt->force_log_flush,
397 p_opt->log_flags, p_opt->log_file,
398 p_opt->log_max_size, p_opt->accum_log_file);
399 if (status != IB_SUCCESS)
401 p_osm->log.log_prefix = p_opt->log_prefix;
403 /* If there is a log level defined - add the OSM_VERSION to it */
404 osm_log_v2(&p_osm->log,
405 osm_log_get_level(&p_osm->log) & (OSM_LOG_SYS ^ 0xFF),
406 FILE_ID, "%s\n", p_osm->osm_version);
407 /* Write the OSM_VERSION to the SYS_LOG */
408 osm_log_v2(&p_osm->log, OSM_LOG_SYS, FILE_ID, "%s\n", p_osm->osm_version); /* Format Waived */
410 OSM_LOG(&p_osm->log, OSM_LOG_FUNCS, "[\n"); /* Format Waived */
412 status = cl_plock_init(&p_osm->lock);
413 if (status != IB_SUCCESS)
416 #ifdef HAVE_LIBPTHREAD
417 pthread_mutex_init(&p_osm->stats.mutex, NULL);
418 pthread_cond_init(&p_osm->stats.cond, NULL);
420 status = cl_event_init(&p_osm->stats.event, FALSE);
421 if (status != IB_SUCCESS)
425 if (p_opt->single_thread) {
426 OSM_LOG(&p_osm->log, OSM_LOG_INFO,
427 "Forcing single threaded dispatcher\n");
428 status = cl_disp_init(&p_osm->disp, 1, "opensm");
431 * Normal behavior is to initialize the dispatcher with
432 * one thread per CPU, as specified by a thread count of '0'.
434 status = cl_disp_init(&p_osm->disp, 0, "opensm");
436 if (status != IB_SUCCESS)
439 /* Unless OpenSM runs in single threaded mode, we create new single
440 * threaded dispatcher for SA Set and Delete requets.
442 p_osm->sa_set_disp_initialized = FALSE;
443 if (!p_opt->single_thread) {
444 status = cl_disp_init(&p_osm->sa_set_disp, 1, "subnadmin_set");
445 if (status != IB_SUCCESS)
447 p_osm->sa_set_disp_initialized = TRUE;
450 /* the DB is in use by subn so init before */
451 status = osm_db_init(&p_osm->db, &p_osm->log);
452 if (status != IB_SUCCESS)
455 status = osm_subn_init(&p_osm->subn, p_osm, p_opt);
456 if (status != IB_SUCCESS)
460 osm_vendor_new(&p_osm->log, p_opt->transaction_timeout);
461 if (p_osm->p_vendor == NULL)
462 status = IB_INSUFFICIENT_RESOURCES;
465 OSM_LOG(&p_osm->log, OSM_LOG_FUNCS, "]\n"); /* Format Waived */
469 ib_api_status_t osm_opensm_init_finish(IN osm_opensm_t * p_osm,
470 IN const osm_subn_opt_t * p_opt)
472 ib_api_status_t status;
474 osm_opensm_construct_finish(p_osm);
476 p_osm->subn.sm_port_guid = p_opt->guid;
478 status = osm_mad_pool_init(&p_osm->mad_pool);
479 if (status != IB_SUCCESS)
482 status = osm_vl15_init(&p_osm->vl15, p_osm->p_vendor,
483 &p_osm->log, &p_osm->stats, &p_osm->subn,
484 p_opt->max_wire_smps, p_opt->max_wire_smps2,
485 p_opt->max_smps_timeout);
486 if (status != IB_SUCCESS)
489 status = osm_sm_init(&p_osm->sm, &p_osm->subn, &p_osm->db,
490 p_osm->p_vendor, &p_osm->mad_pool, &p_osm->vl15,
491 &p_osm->log, &p_osm->stats, &p_osm->disp,
493 if (status != IB_SUCCESS)
496 status = osm_sa_init(&p_osm->sm, &p_osm->sa, &p_osm->subn,
497 p_osm->p_vendor, &p_osm->mad_pool, &p_osm->log,
498 &p_osm->stats, &p_osm->disp,
499 p_opt->single_thread ? NULL : &p_osm->sa_set_disp,
501 if (status != IB_SUCCESS)
504 cl_qlist_init(&p_osm->plugin_list);
506 if (p_opt->event_plugin_name)
507 load_plugins(p_osm, p_opt->event_plugin_name);
509 #ifdef ENABLE_OSM_PERF_MGR
510 status = osm_perfmgr_init(&p_osm->perfmgr, p_osm, p_opt);
511 if (status != IB_SUCCESS)
513 #endif /* ENABLE_OSM_PERF_MGR */
515 status = osm_congestion_control_init(&p_osm->cc,
517 if (status != IB_SUCCESS)
520 p_osm->no_fallback_routing_engine = FALSE;
522 setup_routing_engines(p_osm, p_opt->routing_engine_names);
524 p_osm->routing_engine_used = NULL /* OSM_ROUTING_ENGINE_TYPE_NONE */;
526 p_osm->node_name_map = open_node_name_map(p_opt->node_name_map_name);
529 OSM_LOG(&p_osm->log, OSM_LOG_FUNCS, "]\n"); /* Format Waived */
533 ib_api_status_t osm_opensm_bind(IN osm_opensm_t * p_osm, IN ib_net64_t guid)
535 ib_api_status_t status;
537 OSM_LOG_ENTER(&p_osm->log);
539 status = osm_sm_bind(&p_osm->sm, guid);
540 if (status != IB_SUCCESS)
543 status = osm_sa_bind(&p_osm->sa, guid);
544 if (status != IB_SUCCESS)
547 #ifdef ENABLE_OSM_PERF_MGR
548 status = osm_perfmgr_bind(&p_osm->perfmgr, guid);
549 if (status != IB_SUCCESS)
551 #endif /* ENABLE_OSM_PERF_MGR */
553 status = osm_congestion_control_bind(&p_osm->cc, guid);
554 if (status != IB_SUCCESS)
557 /* setting IS_SM in capability mask */
558 OSM_LOG(&p_osm->log, OSM_LOG_INFO, "Setting IS_SM on port 0x%016" PRIx64 "\n",
560 osm_vendor_set_sm(p_osm->sm.mad_ctrl.h_bind, TRUE);
563 OSM_LOG_EXIT(&p_osm->log);
567 void osm_opensm_report_event(osm_opensm_t *osm, osm_epi_event_id_t event_id,
570 cl_list_item_t *item;
572 for (item = cl_qlist_head(&osm->plugin_list);
573 !osm_exit_flag && item != cl_qlist_end(&osm->plugin_list);
574 item = cl_qlist_next(item)) {
575 osm_epi_plugin_t *p = (osm_epi_plugin_t *)item;
577 p->impl->report(p->plugin_data, event_id, event_data);