2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2002-2006 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
38 * Implementation of the P_Key Manager (Partititon Manager).
39 * This is part of the OpenSM.
44 #endif /* HAVE_CONFIG_H */
47 #include <iba/ib_types.h>
48 #include <complib/cl_qmap.h>
49 #include <complib/cl_debug.h>
50 #include <opensm/osm_node.h>
51 #include <opensm/osm_switch.h>
52 #include <opensm/osm_partition.h>
53 #include <opensm/osm_opensm.h>
55 /**********************************************************************
56 **********************************************************************/
58 The max number of pkey blocks for a physical port is located in
59 a different place for switch external ports (SwitchInfo) and the
60 rest of the ports (NodeInfo).
63 pkey_mgr_get_physp_max_blocks(IN const osm_subn_t * p_subn,
64 IN const osm_physp_t * p_physp)
66 osm_node_t *p_node = osm_physp_get_node_ptr(p_physp);
67 uint16_t num_pkeys = 0;
69 if (!p_node->sw || (osm_physp_get_port_num(p_physp) == 0))
70 num_pkeys = cl_ntoh16(p_node->node_info.partition_cap);
72 num_pkeys = cl_ntoh16(p_node->sw->switch_info.enforce_cap);
73 return ((num_pkeys + 31) / 32);
76 /**********************************************************************
77 **********************************************************************/
79 * Insert new pending pkey entry to the specific port pkey table
80 * pending pkeys. New entries are inserted at the back.
83 pkey_mgr_process_physical_port(IN osm_log_t * p_log,
85 IN const ib_net16_t pkey,
86 IN osm_physp_t * p_physp)
88 osm_node_t *p_node = osm_physp_get_node_ptr(p_physp);
89 osm_pkey_tbl_t *p_pkey_tbl;
90 ib_net16_t *p_orig_pkey;
92 osm_pending_pkey_t *p_pending;
94 p_pkey_tbl = &p_physp->pkeys;
95 p_pending = (osm_pending_pkey_t *) malloc(sizeof(osm_pending_pkey_t));
97 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0502: "
98 "Failed to allocate new pending pkey entry for node "
99 "0x%016" PRIx64 " port %u\n",
100 cl_ntoh64(osm_node_get_node_guid(p_node)),
101 osm_physp_get_port_num(p_physp));
104 p_pending->pkey = pkey;
105 p_orig_pkey = cl_map_get(&p_pkey_tbl->keys, ib_pkey_get_base(pkey));
107 p_pending->is_new = TRUE;
108 cl_qlist_insert_tail(&p_pkey_tbl->pending,
109 (cl_list_item_t *) p_pending);
112 CL_ASSERT(ib_pkey_get_base(*p_orig_pkey) ==
113 ib_pkey_get_base(pkey));
114 p_pending->is_new = FALSE;
115 if (osm_pkey_tbl_get_block_and_idx(p_pkey_tbl, p_orig_pkey,
117 &p_pending->index) !=
119 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0503: "
120 "Failed to obtain P_Key 0x%04x block and index for node "
121 "0x%016" PRIx64 " port %u\n",
122 ib_pkey_get_base(pkey),
123 cl_ntoh64(osm_node_get_node_guid(p_node)),
124 osm_physp_get_port_num(p_physp));
127 cl_qlist_insert_head(&p_pkey_tbl->pending,
128 (cl_list_item_t *) p_pending);
132 OSM_LOG(p_log, OSM_LOG_DEBUG,
133 "pkey 0x%04x was %s for node 0x%016" PRIx64 " port %u\n",
134 cl_ntoh16(pkey), stat,
135 cl_ntoh64(osm_node_get_node_guid(p_node)),
136 osm_physp_get_port_num(p_physp));
139 /**********************************************************************
140 **********************************************************************/
142 pkey_mgr_process_partition_table(osm_log_t * p_log, osm_sm_t * sm,
143 const osm_prtn_t * p_prtn,
144 const boolean_t full)
146 const cl_map_t *p_tbl =
147 full ? &p_prtn->full_guid_tbl : &p_prtn->part_guid_tbl;
148 cl_map_iterator_t i, i_next;
149 ib_net16_t pkey = p_prtn->pkey;
150 osm_physp_t *p_physp;
153 pkey |= cl_hton16(0x8000);
155 i_next = cl_map_head(p_tbl);
156 while (i_next != cl_map_end(p_tbl)) {
158 i_next = cl_map_next(i);
159 p_physp = cl_map_obj(i);
161 pkey_mgr_process_physical_port(p_log, sm, pkey,
166 /**********************************************************************
167 **********************************************************************/
168 static ib_api_status_t
169 pkey_mgr_update_pkey_entry(IN osm_sm_t * sm,
170 IN const osm_physp_t * p_physp,
171 IN const ib_pkey_table_t * block,
172 IN const uint16_t block_index)
174 osm_madw_context_t context;
175 osm_node_t *p_node = osm_physp_get_node_ptr(p_physp);
178 context.pkey_context.node_guid = osm_node_get_node_guid(p_node);
179 context.pkey_context.port_guid = osm_physp_get_port_guid(p_physp);
180 context.pkey_context.set_method = TRUE;
181 attr_mod = block_index;
182 if (osm_node_get_type(p_node) == IB_NODE_TYPE_SWITCH)
183 attr_mod |= osm_physp_get_port_num(p_physp) << 16;
184 return osm_req_set(sm, osm_physp_get_dr_path_ptr(p_physp),
185 (uint8_t *) block, sizeof(*block),
186 IB_MAD_ATTR_P_KEY_TABLE,
187 cl_hton32(attr_mod), CL_DISP_MSGID_NONE, &context);
190 /**********************************************************************
191 **********************************************************************/
193 pkey_mgr_enforce_partition(IN osm_log_t * p_log, osm_sm_t * sm,
194 IN osm_physp_t * p_physp, IN const boolean_t enforce)
196 osm_madw_context_t context;
197 uint8_t payload[IB_SMP_DATA_SIZE];
198 ib_port_info_t *p_pi;
199 ib_api_status_t status;
201 p_pi = &p_physp->port_info;
203 if ((p_pi->vl_enforce & 0xc) == (0xc) * (enforce == TRUE)) {
204 OSM_LOG(p_log, OSM_LOG_DEBUG,
205 "No need to update PortInfo for "
206 "node 0x%016" PRIx64 " port %u\n",
207 cl_ntoh64(osm_node_get_node_guid
208 (osm_physp_get_node_ptr(p_physp))),
209 osm_physp_get_port_num(p_physp));
213 memset(payload, 0, IB_SMP_DATA_SIZE);
214 memcpy(payload, p_pi, sizeof(ib_port_info_t));
216 p_pi = (ib_port_info_t *) payload;
218 p_pi->vl_enforce |= 0xc;
220 p_pi->vl_enforce &= ~0xc;
221 p_pi->state_info2 = 0;
222 ib_port_info_set_port_state(p_pi, IB_LINK_NO_CHANGE);
224 context.pi_context.node_guid =
225 osm_node_get_node_guid(osm_physp_get_node_ptr(p_physp));
226 context.pi_context.port_guid = osm_physp_get_port_guid(p_physp);
227 context.pi_context.set_method = TRUE;
228 context.pi_context.light_sweep = FALSE;
229 context.pi_context.active_transition = FALSE;
231 status = osm_req_set(sm, osm_physp_get_dr_path_ptr(p_physp),
232 payload, sizeof(payload),
233 IB_MAD_ATTR_PORT_INFO,
234 cl_hton32(osm_physp_get_port_num(p_physp)),
235 CL_DISP_MSGID_NONE, &context);
236 if (status != IB_SUCCESS) {
237 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0511: "
238 "Failed to set PortInfo for "
239 "node 0x%016" PRIx64 " port %u\n",
240 cl_ntoh64(osm_node_get_node_guid
241 (osm_physp_get_node_ptr(p_physp))),
242 osm_physp_get_port_num(p_physp));
245 OSM_LOG(p_log, OSM_LOG_DEBUG,
246 "Set PortInfo for node 0x%016" PRIx64 " port %u\n",
247 cl_ntoh64(osm_node_get_node_guid
248 (osm_physp_get_node_ptr(p_physp))),
249 osm_physp_get_port_num(p_physp));
254 /**********************************************************************
255 **********************************************************************/
256 static boolean_t pkey_mgr_update_port(osm_log_t * p_log, osm_sm_t * sm,
257 const osm_port_t * const p_port)
259 osm_physp_t *p_physp;
261 ib_pkey_table_t *block, *new_block;
262 osm_pkey_tbl_t *p_pkey_tbl;
263 uint16_t block_index;
265 uint16_t last_free_block_index = 0;
266 uint8_t last_free_pkey_index = 0;
267 uint16_t num_of_blocks;
268 uint16_t max_num_of_blocks;
269 ib_api_status_t status;
270 boolean_t ret_val = FALSE;
271 osm_pending_pkey_t *p_pending;
273 ib_pkey_table_t empty_block;
275 memset(&empty_block, 0, sizeof(ib_pkey_table_t));
277 p_physp = p_port->p_physp;
281 p_node = osm_physp_get_node_ptr(p_physp);
282 p_pkey_tbl = &p_physp->pkeys;
283 num_of_blocks = osm_pkey_tbl_get_num_blocks(p_pkey_tbl);
285 pkey_mgr_get_physp_max_blocks(sm->p_subn, p_physp);
286 if (p_pkey_tbl->max_blocks > max_num_of_blocks) {
287 OSM_LOG(p_log, OSM_LOG_INFO,
288 "Max number of blocks reduced from %u to %u "
289 "for node 0x%016" PRIx64 " port %u\n",
290 p_pkey_tbl->max_blocks, max_num_of_blocks,
291 cl_ntoh64(osm_node_get_node_guid(p_node)),
292 osm_physp_get_port_num(p_physp));
294 p_pkey_tbl->max_blocks = max_num_of_blocks;
296 osm_pkey_tbl_init_new_blocks(p_pkey_tbl);
297 p_pkey_tbl->used_blocks = 0;
300 process every pending pkey in order -
301 first must be "updated" last are "new"
304 (osm_pending_pkey_t *) cl_qlist_remove_head(&p_pkey_tbl->pending);
306 (osm_pending_pkey_t *) cl_qlist_end(&p_pkey_tbl->pending)) {
307 if (p_pending->is_new == FALSE) {
308 block_index = p_pending->block;
309 pkey_index = p_pending->index;
312 found = osm_pkey_find_next_free_entry(p_pkey_tbl,
313 &last_free_block_index,
314 &last_free_pkey_index);
316 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0504: "
317 "Failed to find empty space for new pkey 0x%04x "
318 "for node 0x%016" PRIx64 " port %u\n",
319 cl_ntoh16(p_pending->pkey),
320 cl_ntoh64(osm_node_get_node_guid
322 osm_physp_get_port_num(p_physp));
324 block_index = last_free_block_index;
325 pkey_index = last_free_pkey_index++;
331 osm_pkey_tbl_set_new_entry(p_pkey_tbl, block_index,
334 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0505: "
335 "Failed to set PKey 0x%04x in block %u idx %u "
336 "for node 0x%016" PRIx64 " port %u\n",
337 cl_ntoh16(p_pending->pkey), block_index,
339 cl_ntoh64(osm_node_get_node_guid
341 osm_physp_get_port_num(p_physp));
347 (osm_pending_pkey_t *) cl_qlist_remove_head(&p_pkey_tbl->
351 /* now look for changes and store */
352 for (block_index = 0; block_index < num_of_blocks; block_index++) {
353 block = osm_pkey_tbl_block_get(p_pkey_tbl, block_index);
354 new_block = osm_pkey_tbl_new_block_get(p_pkey_tbl, block_index);
356 new_block = &empty_block;
357 if (block && !memcmp(new_block, block, sizeof(*block)))
361 pkey_mgr_update_pkey_entry(sm, p_physp, new_block,
363 if (status == IB_SUCCESS) {
364 OSM_LOG(p_log, OSM_LOG_DEBUG,
365 "Updated pkey table block %d for node 0x%016"
366 PRIx64 " port %u\n", block_index,
367 cl_ntoh64(osm_node_get_node_guid(p_node)),
368 osm_physp_get_port_num(p_physp));
371 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0506: "
372 "pkey_mgr_update_pkey_entry() failed to update "
373 "pkey table block %d for node 0x%016" PRIx64
374 " port %u\n", block_index,
375 cl_ntoh64(osm_node_get_node_guid(p_node)),
376 osm_physp_get_port_num(p_physp));
383 /**********************************************************************
384 **********************************************************************/
386 pkey_mgr_update_peer_port(osm_log_t * p_log, osm_sm_t * sm,
387 const osm_subn_t * p_subn,
388 const osm_port_t * const p_port, boolean_t enforce)
390 osm_physp_t *p_physp, *peer;
392 ib_pkey_table_t *block, *peer_block;
393 const osm_pkey_tbl_t *p_pkey_tbl;
394 osm_pkey_tbl_t *p_peer_pkey_tbl;
395 uint16_t block_index;
396 uint16_t num_of_blocks;
397 uint16_t peer_max_blocks;
398 ib_api_status_t status = IB_SUCCESS;
399 boolean_t ret_val = FALSE;
400 boolean_t port_info_set = FALSE;
401 ib_pkey_table_t empty_block;
403 memset(&empty_block, 0, sizeof(ib_pkey_table_t));
405 p_physp = p_port->p_physp;
408 peer = osm_physp_get_remote(p_physp);
411 p_node = osm_physp_get_node_ptr(peer);
412 if (!p_node->sw || !p_node->sw->switch_info.enforce_cap)
415 p_pkey_tbl = osm_physp_get_pkey_tbl(p_physp);
416 p_peer_pkey_tbl = &peer->pkeys;
417 num_of_blocks = osm_pkey_tbl_get_num_blocks(p_pkey_tbl);
418 peer_max_blocks = pkey_mgr_get_physp_max_blocks(p_subn, peer);
419 if (peer_max_blocks < p_pkey_tbl->used_blocks) {
420 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0508: "
421 "Not enough pkey entries (%u < %u) on switch 0x%016"
422 PRIx64 " port %u. Clearing Enforcement bit\n",
423 peer_max_blocks, num_of_blocks,
424 cl_ntoh64(osm_node_get_node_guid(p_node)),
425 osm_physp_get_port_num(peer));
429 if (pkey_mgr_enforce_partition(p_log, sm, peer, enforce))
430 port_info_set = TRUE;
432 if (enforce == FALSE)
433 return port_info_set;
435 p_peer_pkey_tbl->used_blocks = p_pkey_tbl->used_blocks;
436 for (block_index = 0; block_index < p_pkey_tbl->used_blocks;
438 block = osm_pkey_tbl_new_block_get(p_pkey_tbl, block_index);
440 block = &empty_block;
443 osm_pkey_tbl_block_get(p_peer_pkey_tbl, block_index);
445 || memcmp(peer_block, block, sizeof(*peer_block))) {
447 pkey_mgr_update_pkey_entry(sm, peer, block,
449 if (status == IB_SUCCESS)
452 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0509: "
453 "pkey_mgr_update_pkey_entry() failed to update "
454 "pkey table block %d for node 0x%016"
455 PRIx64 " port %u\n", block_index,
456 cl_ntoh64(osm_node_get_node_guid
458 osm_physp_get_port_num(peer));
463 OSM_LOG(p_log, OSM_LOG_DEBUG,
464 "Pkey table was updated for node 0x%016" PRIx64
466 cl_ntoh64(osm_node_get_node_guid(p_node)),
467 osm_physp_get_port_num(peer));
474 /**********************************************************************
475 **********************************************************************/
476 osm_signal_t osm_pkey_mgr_process(IN osm_opensm_t * p_osm)
479 cl_map_item_t *p_next;
482 osm_signal_t signal = OSM_SIGNAL_DONE;
486 OSM_LOG_ENTER(&p_osm->log);
488 CL_PLOCK_EXCL_ACQUIRE(&p_osm->lock);
490 if (osm_prtn_make_partitions(&p_osm->log, &p_osm->subn) != IB_SUCCESS) {
491 OSM_LOG(&p_osm->log, OSM_LOG_ERROR, "ERR 0510: "
492 "osm_prtn_make_partitions() failed\n");
496 /* populate the pending pkey entries by scanning all partitions */
497 p_tbl = &p_osm->subn.prtn_pkey_tbl;
498 p_next = cl_qmap_head(p_tbl);
499 while (p_next != cl_qmap_end(p_tbl)) {
500 p_prtn = (osm_prtn_t *) p_next;
501 p_next = cl_qmap_next(p_next);
502 pkey_mgr_process_partition_table(&p_osm->log, &p_osm->sm,
504 pkey_mgr_process_partition_table(&p_osm->log, &p_osm->sm,
508 /* calculate and set new pkey tables */
509 p_tbl = &p_osm->subn.port_guid_tbl;
510 p_next = cl_qmap_head(p_tbl);
511 while (p_next != cl_qmap_end(p_tbl)) {
512 p_port = (osm_port_t *) p_next;
513 p_next = cl_qmap_next(p_next);
514 if (pkey_mgr_update_port(&p_osm->log, &p_osm->sm, p_port))
515 signal = OSM_SIGNAL_DONE_PENDING;
516 if ((osm_node_get_type(p_port->p_node) != IB_NODE_TYPE_SWITCH)
517 && pkey_mgr_update_peer_port(&p_osm->log, &p_osm->sm,
518 &p_osm->subn, p_port,
520 no_partition_enforcement))
521 signal = OSM_SIGNAL_DONE_PENDING;
525 CL_PLOCK_RELEASE(&p_osm->lock);
526 OSM_LOG_EXIT(&p_osm->log);