contrib/ofed/opensm/opensm/osm_pkey_mgr.c

   1 /*
   2  * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved.
   3  * Copyright (c) 2002-2011 Mellanox Technologies LTD. All rights reserved.
   4  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
   5  * Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * OpenIB.org BSD license below:
  12  *
  13  *     Redistribution and use in source and binary forms, with or
  14  *     without modification, are permitted provided that the following
  15  *     conditions are met:
  16  *
  17  *      - Redistributions of source code must retain the above
  18  *        copyright notice, this list of conditions and the following
  19  *        disclaimer.
  20  *
  21  *      - Redistributions in binary form must reproduce the above
  22  *        copyright notice, this list of conditions and the following
  23  *        disclaimer in the documentation and/or other materials
  24  *        provided with the distribution.
  25  *
  26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  33  * SOFTWARE.
  34  *
  35  */
  36
  37 /*
  38  * Abstract:
  39  * Implementation of the P_Key Manager (Partition Manager).
  40  * This is part of the OpenSM.
  41  */
  42
  43 #if HAVE_CONFIG_H
  44 #  include <config.h>
  45 #endif                          /* HAVE_CONFIG_H */
  46
  47 #include <string.h>
  48 #include <iba/ib_types.h>
  49 #include <complib/cl_qmap.h>
  50 #include <complib/cl_debug.h>
  51 #include <opensm/osm_file_ids.h>
  52 #define FILE_ID OSM_FILE_PKEY_MGR_C
  53 #include <opensm/osm_node.h>
  54 #include <opensm/osm_switch.h>
  55 #include <opensm/osm_partition.h>
  56 #include <opensm/osm_opensm.h>
  57
  58 static void clear_accum_pkey_index(osm_pkey_tbl_t * p_pkey_tbl,
  59                                    uint16_t pkey_index);
  60
  61 /*
  62   The max number of pkeys/pkey blocks for a physical port is located
  63   in a different place for switch external ports (SwitchInfo) and the
  64   rest of the ports (NodeInfo).
  65 */
  66 static uint16_t
  67 pkey_mgr_get_physp_max_pkeys(IN const osm_physp_t * p_physp)
  68 {
  69         osm_node_t *p_node = osm_physp_get_node_ptr(p_physp);
  70         uint16_t num_pkeys = 0;
  71
  72         if (!p_node->sw || (osm_physp_get_port_num(p_physp) == 0))
  73                 num_pkeys = cl_ntoh16(p_node->node_info.partition_cap);
  74         else
  75                 num_pkeys = cl_ntoh16(p_node->sw->switch_info.enforce_cap);
  76         return num_pkeys;
  77 }
  78
  79 static uint16_t
  80 pkey_mgr_get_physp_max_blocks(IN const osm_physp_t * p_physp)
  81 {
  82         return ((pkey_mgr_get_physp_max_pkeys(p_physp) + 31) / 32);
  83 }
  84
  85 /*
  86  * Insert new pending pkey entry to the specific port pkey table
  87  * pending pkeys. New entries are inserted at the back.
  88  */
  89 static void
  90 pkey_mgr_process_physical_port(IN osm_log_t * p_log,
  91                                IN osm_sm_t * sm,
  92                                IN const ib_net16_t pkey,
  93                                IN osm_physp_t * p_physp)
  94 {
  95         osm_node_t *p_node = osm_physp_get_node_ptr(p_physp);
  96         osm_pkey_tbl_t *p_pkey_tbl;
  97         ib_net16_t *p_orig_pkey;
  98         osm_pending_pkey_t *p_pending;
  99
 100         p_pkey_tbl = &p_physp->pkeys;
 101         p_pending = (osm_pending_pkey_t *) calloc(1, sizeof(osm_pending_pkey_t));
 102         if (!p_pending) {
 103                 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0502: "
 104                         "Failed to allocate new pending pkey entry for node "
 105                         "0x%016" PRIx64 " port %u\n",
 106                         cl_ntoh64(osm_node_get_node_guid(p_node)),
 107                         osm_physp_get_port_num(p_physp));
 108                 return;
 109         }
 110         p_pending->pkey = pkey;
 111         if (sm->p_subn->opt.allow_both_pkeys)
 112                 p_orig_pkey = cl_map_get(&p_pkey_tbl->keys, pkey);
 113         else
 114                 p_orig_pkey = cl_map_get(&p_pkey_tbl->keys,
 115                                          ib_pkey_get_base(pkey));
 116
 117         if (!p_orig_pkey) {
 118                 p_pending->is_new = TRUE;
 119         } else {
 120                 CL_ASSERT(ib_pkey_get_base(*p_orig_pkey) ==
 121                           ib_pkey_get_base(pkey));
 122                 p_pending->is_new = FALSE;
 123                 if (osm_pkey_tbl_get_block_and_idx(p_pkey_tbl, p_orig_pkey,
 124                                                    &p_pending->block,
 125                                                    &p_pending->index) !=
 126                     IB_SUCCESS) {
 127                         OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0503: "
 128                                 "Failed to obtain P_Key 0x%04x block and index "
 129                                 "for node 0x%016" PRIx64 " port %u\n",
 130                                 cl_ntoh16(ib_pkey_get_base(pkey)),
 131                                 cl_ntoh64(osm_node_get_node_guid(p_node)),
 132                                 osm_physp_get_port_num(p_physp));
 133                         free(p_pending);
 134                         return;
 135                 }
 136                 if (p_physp->pkeys.indx0_pkey) {
 137                         /*
 138                          * Remove the pkey that should be at index 0 from
 139                          * accum pkey if current position is not index 0
 140                          */
 141                         if (((sm->p_subn->opt.allow_both_pkeys &&
 142                               pkey == p_physp->pkeys.indx0_pkey) ||
 143                              (!sm->p_subn->opt.allow_both_pkeys &&
 144                               ib_pkey_get_base(pkey) == ib_pkey_get_base(p_physp->pkeys.indx0_pkey))) &&
 145                             (p_pending->block != 0 || p_pending->index != 0)) {
 146                                         p_pending->is_new = TRUE;
 147                                         clear_accum_pkey_index(p_pkey_tbl,
 148                                                                p_pending->block *
 149                                                                IB_NUM_PKEY_ELEMENTS_IN_BLOCK +
 150                                                                p_pending->index);
 151                         }
 152
 153                         if (p_pending->block == 0 && p_pending->index == 0) {
 154                                 /* Move the pkey away from index 0 */
 155                                 if ((sm->p_subn->opt.allow_both_pkeys &&
 156                                      pkey != p_physp->pkeys.indx0_pkey) ||
 157                                     (!sm->p_subn->opt.allow_both_pkeys &&
 158                                      ib_pkey_get_base(pkey) != ib_pkey_get_base(p_physp->pkeys.indx0_pkey))) {
 159                                         p_pending->is_new = TRUE;
 160                                         clear_accum_pkey_index(p_pkey_tbl, 0);
 161                                 }
 162                         }
 163                  } else {
 164                         /* If index 0 is occupied by non-default, it should reoccupied by pkey 0x7FFF */
 165                         if (p_pending->block == 0 && p_pending->index == 0) {
 166                                 if (ib_pkey_get_base(pkey) != IB_DEFAULT_PARTIAL_PKEY) {
 167                                         p_pending->is_new = TRUE;
 168                                         clear_accum_pkey_index(p_pkey_tbl, 0);
 169                                 }
 170                         /* Need to move default pkey to index 0 */
 171                         } else if ((sm->p_subn->opt.allow_both_pkeys &&
 172                                     pkey == IB_DEFAULT_PKEY) ||
 173                                    (!sm->p_subn->opt.allow_both_pkeys &&
 174                                     ib_pkey_get_base(pkey) == IB_DEFAULT_PARTIAL_PKEY)) {
 175                                         p_pending->is_new = TRUE;
 176                                         clear_accum_pkey_index(p_pkey_tbl,
 177                                                                p_pending->block *
 178                                                                IB_NUM_PKEY_ELEMENTS_IN_BLOCK +
 179                                                                p_pending->index);
 180                         }
 181                 }
 182
 183         }
 184         if (p_pending->is_new == TRUE)
 185                  cl_qlist_insert_tail(&p_pkey_tbl->pending,
 186                                       (cl_list_item_t *) p_pending);
 187         else
 188                 cl_qlist_insert_head(&p_pkey_tbl->pending,
 189                                      (cl_list_item_t *) p_pending);
 190
 191         OSM_LOG(p_log, OSM_LOG_DEBUG,
 192                 "pkey 0x%04x was %s for node 0x%016" PRIx64 " port %u\n",
 193                 cl_ntoh16(pkey), p_pending->is_new ? "inserted" : "updated",
 194                 cl_ntoh64(osm_node_get_node_guid(p_node)),
 195                 osm_physp_get_port_num(p_physp));
 196 }
 197
 198 static void
 199 pkey_mgr_process_partition_table(osm_log_t * p_log, osm_sm_t * sm,
 200                                  const osm_prtn_t * p_prtn,
 201                                  const boolean_t full)
 202 {
 203         const cl_map_t *p_tbl =
 204             full ? &p_prtn->full_guid_tbl : &p_prtn->part_guid_tbl;
 205         cl_map_iterator_t i, i_next;
 206         ib_net16_t pkey = p_prtn->pkey;
 207         osm_physp_t *p_physp;
 208
 209         if (full)
 210                 pkey |= cl_hton16(0x8000);
 211
 212         i_next = cl_map_head(p_tbl);
 213         while (i_next != cl_map_end(p_tbl)) {
 214                 i = i_next;
 215                 i_next = cl_map_next(i);
 216                 p_physp = cl_map_obj(i);
 217                 if (p_physp)
 218                         pkey_mgr_process_physical_port(p_log, sm, pkey,
 219                                                        p_physp);
 220         }
 221 }
 222
 223 static ib_api_status_t
 224 pkey_mgr_update_pkey_entry(IN osm_sm_t * sm,
 225                            IN const osm_physp_t * p_physp,
 226                            IN const ib_pkey_table_t * block,
 227                            IN const uint16_t block_index)
 228 {
 229         osm_madw_context_t context;
 230         osm_node_t *p_node = osm_physp_get_node_ptr(p_physp);
 231         osm_physp_t *physp0;
 232         uint32_t attr_mod;
 233         ib_net64_t m_key;
 234
 235         context.pkey_context.node_guid = osm_node_get_node_guid(p_node);
 236         context.pkey_context.port_guid = osm_physp_get_port_guid(p_physp);
 237         context.pkey_context.set_method = TRUE;
 238         attr_mod = block_index;
 239         if (osm_node_get_type(p_node) == IB_NODE_TYPE_SWITCH &&
 240             osm_physp_get_port_num(p_physp) != 0) {
 241                 attr_mod |= osm_physp_get_port_num(p_physp) << 16;
 242                 physp0 = osm_node_get_physp_ptr(p_node, 0);
 243                 m_key = ib_port_info_get_m_key(&physp0->port_info);
 244         } else
 245                 m_key = ib_port_info_get_m_key(&p_physp->port_info);
 246         return osm_req_set(sm, osm_physp_get_dr_path_ptr(p_physp),
 247                            (uint8_t *) block, sizeof(*block),
 248                            IB_MAD_ATTR_P_KEY_TABLE,
 249                            cl_hton32(attr_mod), FALSE, m_key,
 250                            CL_DISP_MSGID_NONE, &context);
 251 }
 252
 253 static ib_api_status_t
 254 pkey_mgr_enforce_partition(IN osm_log_t * p_log, osm_sm_t * sm,
 255                            IN osm_physp_t * p_physp,
 256                            IN osm_partition_enforce_type_enum enforce_type)
 257 {
 258         osm_madw_context_t context;
 259         uint8_t payload[IB_SMP_DATA_SIZE];
 260         ib_port_info_t *p_pi;
 261         ib_net64_t m_key;
 262         osm_physp_t *physp0;
 263         ib_api_status_t status;
 264         uint8_t enforce_bits;
 265
 266         p_pi = &p_physp->port_info;
 267
 268         if (enforce_type == OSM_PARTITION_ENFORCE_TYPE_BOTH)
 269                 enforce_bits = 0xc;
 270         else if (enforce_type == OSM_PARTITION_ENFORCE_TYPE_IN)
 271                 enforce_bits = 0x8;
 272         else
 273                 enforce_bits = 0x4;
 274
 275         if ((p_pi->vl_enforce & 0xc) == enforce_bits *
 276             (enforce_type != OSM_PARTITION_ENFORCE_TYPE_OFF)) {
 277                 OSM_LOG(p_log, OSM_LOG_DEBUG,
 278                         "No need to update PortInfo for "
 279                         "node 0x%016" PRIx64 " port %u (%s)\n",
 280                         cl_ntoh64(osm_node_get_node_guid
 281                                   (osm_physp_get_node_ptr(p_physp))),
 282                         osm_physp_get_port_num(p_physp),
 283                         p_physp->p_node->print_desc);
 284                 return IB_SUCCESS;
 285         }
 286
 287         memcpy(payload, p_pi, sizeof(ib_port_info_t));
 288
 289         p_pi = (ib_port_info_t *) payload;
 290         p_pi->vl_enforce &= ~0xc;
 291         if (enforce_type != OSM_PARTITION_ENFORCE_TYPE_OFF)
 292                 p_pi->vl_enforce |= enforce_bits;
 293
 294         p_pi->state_info2 = 0;
 295         ib_port_info_set_port_state(p_pi, IB_LINK_NO_CHANGE);
 296
 297         physp0 = osm_node_get_physp_ptr(p_physp->p_node, 0);
 298         m_key = ib_port_info_get_m_key(&physp0->port_info);
 299
 300         context.pi_context.node_guid =
 301             osm_node_get_node_guid(osm_physp_get_node_ptr(p_physp));
 302         context.pi_context.port_guid = osm_physp_get_port_guid(p_physp);
 303         context.pi_context.set_method = TRUE;
 304         context.pi_context.light_sweep = FALSE;
 305         context.pi_context.active_transition = FALSE;
 306         context.pi_context.client_rereg = FALSE;
 307
 308         status = osm_req_set(sm, osm_physp_get_dr_path_ptr(p_physp),
 309                              payload, sizeof(payload),
 310                              IB_MAD_ATTR_PORT_INFO,
 311                              cl_hton32(osm_physp_get_port_num(p_physp)),
 312                              FALSE, m_key,
 313                              CL_DISP_MSGID_NONE, &context);
 314         if (status != IB_SUCCESS)
 315                 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0511: "
 316                         "Failed to set PortInfo for "
 317                         "node 0x%016" PRIx64 " port %u (%s)\n",
 318                         cl_ntoh64(osm_node_get_node_guid
 319                                   (osm_physp_get_node_ptr(p_physp))),
 320                         osm_physp_get_port_num(p_physp),
 321                         p_physp->p_node->print_desc);
 322         else
 323                 OSM_LOG(p_log, OSM_LOG_DEBUG,
 324                         "Set PortInfo for node 0x%016" PRIx64 " port %u (%s)\n",
 325                         cl_ntoh64(osm_node_get_node_guid
 326                                   (osm_physp_get_node_ptr(p_physp))),
 327                         osm_physp_get_port_num(p_physp),
 328                         p_physp->p_node->print_desc);
 329         return status;
 330 }
 331
 332 static void clear_accum_pkey_index(osm_pkey_tbl_t * p_pkey_tbl,
 333                                    uint16_t pkey_index)
 334 {
 335         uint16_t pkey_idx_bias, pkey_idx;
 336         void *ptr;
 337         uintptr_t pkey_idx_ptr;
 338         cl_map_iterator_t map_iter, map_iter_temp;
 339
 340         map_iter = cl_map_head(&p_pkey_tbl->accum_pkeys);
 341
 342         pkey_idx_bias = pkey_index + 1; // adjust for pkey index bias in accum_pkeys
 343
 344         while (map_iter != cl_map_end(&p_pkey_tbl->accum_pkeys)) {
 345                 map_iter_temp = cl_map_next(map_iter);
 346                 ptr = (uint16_t *) cl_map_obj(map_iter);
 347                 CL_ASSERT(ptr);
 348                 pkey_idx_ptr = (uintptr_t) ptr;
 349                 pkey_idx = pkey_idx_ptr;
 350                 if (pkey_idx == pkey_idx_bias) {
 351                         cl_map_remove_item(&p_pkey_tbl->accum_pkeys, map_iter);
 352                         if (p_pkey_tbl->last_pkey_idx == pkey_idx)
 353                                 osm_pkey_find_last_accum_pkey_index(p_pkey_tbl);
 354                         break;
 355                 }
 356                 map_iter = map_iter_temp;
 357         }
 358 }
 359
 360 static int last_accum_pkey_index(osm_pkey_tbl_t * p_pkey_tbl,
 361                                  uint16_t * p_block_idx,
 362                                  uint8_t * p_pkey_idx)
 363 {
 364         if (p_pkey_tbl->last_pkey_idx) {
 365                 *p_block_idx = (p_pkey_tbl->last_pkey_idx - 1) / IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
 366                 *p_pkey_idx = (p_pkey_tbl->last_pkey_idx - 1) % IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
 367                 return 1;
 368         }
 369
 370         return 0;
 371 }
 372
 373 static int pkey_mgr_update_port(osm_log_t * p_log, osm_sm_t * sm,
 374                                 const osm_port_t * const p_port)
 375 {
 376         osm_physp_t *p_physp;
 377         osm_node_t *p_node;
 378         ib_pkey_table_t *block, *new_block;
 379         osm_pkey_tbl_t *p_pkey_tbl;
 380         uint16_t block_index;
 381         uint8_t pkey_index;
 382         uint16_t last_free_block_index = 0;
 383         uint8_t last_free_pkey_index = 0;
 384         uint16_t num_of_blocks;
 385         uint16_t max_num_of_blocks;
 386         ib_api_status_t status;
 387         osm_pending_pkey_t *p_pending;
 388         boolean_t found;
 389         ib_pkey_table_t empty_block;
 390         int ret = 0, full = 0;
 391         void *ptr;
 392         uintptr_t pkey_idx_ptr;
 393         uint16_t pkey_idx;
 394
 395         p_physp = p_port->p_physp;
 396         if (!p_physp)
 397                 return FALSE;
 398
 399         memset(&empty_block, 0, sizeof(ib_pkey_table_t));
 400
 401         p_node = osm_physp_get_node_ptr(p_physp);
 402         p_pkey_tbl = &p_physp->pkeys;
 403         num_of_blocks = osm_pkey_tbl_get_num_blocks(p_pkey_tbl);
 404         max_num_of_blocks = pkey_mgr_get_physp_max_blocks(p_physp);
 405         if (p_pkey_tbl->max_blocks > max_num_of_blocks) {
 406                 OSM_LOG(p_log, OSM_LOG_INFO,
 407                         "Max number of blocks reduced from %u to %u "
 408                         "for node 0x%016" PRIx64 " port %u (%s)\n",
 409                         p_pkey_tbl->max_blocks, max_num_of_blocks,
 410                         cl_ntoh64(osm_node_get_node_guid(p_node)),
 411                         osm_physp_get_port_num(p_physp),
 412                         p_physp->p_node->print_desc);
 413         }
 414         p_pkey_tbl->max_blocks = max_num_of_blocks;
 415
 416         osm_pkey_tbl_init_new_blocks(p_pkey_tbl);
 417         p_pkey_tbl->used_blocks = 0;
 418
 419         /*
 420            process every pending pkey in order -
 421            first must be "updated" last are "new"
 422          */
 423         p_pending =
 424             (osm_pending_pkey_t *) cl_qlist_remove_head(&p_pkey_tbl->pending);
 425         while (p_pending !=
 426                (osm_pending_pkey_t *) cl_qlist_end(&p_pkey_tbl->pending)) {
 427
 428                 found = FALSE;
 429                 ptr = NULL;
 430
 431                 if (p_pending->is_new == FALSE) {
 432                         block_index = p_pending->block;
 433                         pkey_index = p_pending->index;
 434                         found = TRUE;
 435                 } else {
 436                         ptr = cl_map_get(&p_pkey_tbl->accum_pkeys,p_pending->pkey);
 437                         if (ptr != NULL) {
 438                                 pkey_idx_ptr = (uintptr_t) ptr;
 439                                 pkey_idx = pkey_idx_ptr;
 440                                 pkey_idx--; /* adjust pkey index for bias */
 441                                 block_index = pkey_idx / IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
 442                                 pkey_index = pkey_idx % IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
 443
 444                                 if (((sm->p_subn->opt.allow_both_pkeys &&
 445                                       p_pending->pkey == p_physp->pkeys.indx0_pkey) ||
 446                                      (!sm->p_subn->opt.allow_both_pkeys &&
 447                                       ib_pkey_get_base(p_pending->pkey) == ib_pkey_get_base(p_physp->pkeys.indx0_pkey))) ||
 448                                     ((p_pending->pkey != p_physp->pkeys.indx0_pkey &&
 449                                       pkey_idx == 0))) {
 450                                         clear_accum_pkey_index(p_pkey_tbl, pkey_idx);
 451                                         cl_qlist_insert_tail(&p_pkey_tbl->pending,
 452                                                              (cl_list_item_t *)p_pending);
 453                                         p_pending =
 454                                             (osm_pending_pkey_t *) cl_qlist_remove_head(&p_pkey_tbl->pending);
 455                                         continue;
 456                                 } else
 457                                         found = TRUE;
 458                         }
 459
 460                         if (!found) {
 461                                 if (!p_pkey_tbl->indx0_pkey &&
 462                                     ((sm->p_subn->opt.allow_both_pkeys &&
 463                                       p_pending->pkey == IB_DEFAULT_PKEY) ||
 464                                      (!sm->p_subn->opt.allow_both_pkeys &&
 465                                       ib_pkey_get_base(p_pending->pkey) == IB_DEFAULT_PARTIAL_PKEY))) {
 466                                         block_index = 0;
 467                                         pkey_index = 0;
 468                                 } else if ((sm->p_subn->opt.allow_both_pkeys &&
 469                                             p_pending->pkey == p_pkey_tbl->indx0_pkey) ||
 470                                            (!sm->p_subn->opt.allow_both_pkeys &&
 471                                             ib_pkey_get_base(p_pending->pkey) ==
 472                                             ib_pkey_get_base(p_pkey_tbl->indx0_pkey))) {
 473                                         block_index = 0;
 474                                         pkey_index = 0;
 475                                 } else if (last_accum_pkey_index(p_pkey_tbl,
 476                                                                  &last_free_block_index,
 477                                                                  &last_free_pkey_index)) {
 478                                         block_index = last_free_block_index;
 479                                         pkey_index = last_free_pkey_index + 1;
 480                                         if (pkey_index >= IB_NUM_PKEY_ELEMENTS_IN_BLOCK) {
 481                                                 block_index++;
 482                                                 pkey_index -= IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
 483                                         }
 484                                 } else {
 485                                         block_index = 0;
 486                                         pkey_index = 1;
 487                                 }
 488
 489                                 if (block_index * IB_NUM_PKEY_ELEMENTS_IN_BLOCK + pkey_index >= pkey_mgr_get_physp_max_pkeys(p_physp)) {
 490                                         if ((sm->p_subn->opt.allow_both_pkeys &&
 491                                              p_pending->pkey != IB_DEFAULT_PKEY) ||
 492                                             (!sm->p_subn->opt.allow_both_pkeys &&
 493                                              ib_pkey_get_base(p_pending->pkey) != IB_DEFAULT_PARTIAL_PKEY)) {
 494                                                 last_free_block_index = 0;
 495                                                 last_free_pkey_index = 1;
 496                                                 found = osm_pkey_find_next_free_entry(p_pkey_tbl, &last_free_block_index, &last_free_pkey_index);
 497                                         } else
 498                                                 found = FALSE;
 499                                         if (!found)
 500                                                 full = 1;
 501                                         else {
 502                                                 block_index = last_free_block_index;
 503                                                 pkey_index = last_free_pkey_index;
 504                                                 if (block_index * IB_NUM_PKEY_ELEMENTS_IN_BLOCK + pkey_index >= pkey_mgr_get_physp_max_pkeys(p_physp)) {
 505                                                         full = 1;
 506                                                         found = FALSE;
 507                                                 } else {
 508                                                         OSM_LOG(p_log, OSM_LOG_INFO,
 509                                                                 "Reusing PKeyTable block index %u pkey index %u "
 510                                                                 "for pkey 0x%x on 0x%016" PRIx64 " port %u (%s)\n",
 511                                                                 block_index,
 512                                                                 pkey_index,
 513                                                                 cl_ntoh16(p_pending->pkey),
 514                                                                 cl_ntoh64(osm_node_get_node_guid(p_node)),
 515                                                                 osm_physp_get_port_num(p_physp),
 516                                                                 p_physp->p_node->print_desc);
 517
 518                                                         clear_accum_pkey_index(p_pkey_tbl, block_index * IB_NUM_PKEY_ELEMENTS_IN_BLOCK + pkey_index);
 519                                                 }
 520                                         }
 521                                         if (full)
 522                                                 OSM_LOG(p_log, OSM_LOG_ERROR,
 523                                                         "ERR 0512: "
 524                                                         "Failed to set PKey 0x%04x because Pkey table is full "
 525                                                         "for node 0x%016" PRIx64 " port %u (%s)\n",
 526                                                         cl_ntoh16(p_pending->pkey),
 527                                                         cl_ntoh64(osm_node_get_node_guid(p_node)),
 528                                                         osm_physp_get_port_num(p_physp),
 529                                                         p_physp->p_node->print_desc);
 530                                 } else
 531                                         found = TRUE;
 532                         }
 533                 }
 534
 535                 if (found) {
 536                         if (IB_SUCCESS !=
 537                             osm_pkey_tbl_set_new_entry(p_pkey_tbl, block_index,
 538                                                        pkey_index,
 539                                                        p_pending->pkey)) {
 540                                 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0505: "
 541                                         "Failed to set PKey 0x%04x in block %u idx %u "
 542                                         "for node 0x%016" PRIx64 " port %u (%s)\n",
 543                                         cl_ntoh16(p_pending->pkey), block_index,
 544                                         pkey_index,
 545                                         cl_ntoh64(osm_node_get_node_guid
 546                                                   (p_node)),
 547                                         osm_physp_get_port_num(p_physp),
 548                                         p_physp->p_node->print_desc);
 549                         }
 550                         if (ptr == NULL &&
 551                             CL_SUCCESS !=
 552                             osm_pkey_tbl_set_accum_pkeys(p_pkey_tbl,
 553                                                          p_pending->pkey,
 554                                                          block_index * IB_NUM_PKEY_ELEMENTS_IN_BLOCK + pkey_index)) {
 555                                 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0508: "
 556                                         "Failed to set accum_pkeys PKey 0x%04x "
 557                                         "in block %u idx %u for node 0x%016"
 558                                         PRIx64 " port %u (%s)\n",
 559                                         cl_ntoh16(p_pending->pkey), block_index,
 560                                         pkey_index,
 561                                         cl_ntoh64(osm_node_get_node_guid(p_node)),
 562                                         osm_physp_get_port_num(p_physp),
 563                                         p_physp->p_node->print_desc);
 564                         }
 565                 }
 566                 free(p_pending);
 567                 p_pending =
 568                     (osm_pending_pkey_t *) cl_qlist_remove_head(&p_pkey_tbl->
 569                                                                 pending);
 570         }
 571
 572         p_pkey_tbl->indx0_pkey = 0;
 573         /* now look for changes and store */
 574         for (block_index = 0; block_index < num_of_blocks; block_index++) {
 575                 block = osm_pkey_tbl_block_get(p_pkey_tbl, block_index);
 576                 new_block = osm_pkey_tbl_new_block_get(p_pkey_tbl, block_index);
 577                 if (!new_block)
 578                         new_block = &empty_block;
 579                 if (block && !memcmp(new_block, block, sizeof(*block)))
 580                         continue;
 581
 582                 status =
 583                     pkey_mgr_update_pkey_entry(sm, p_physp, new_block,
 584                                                block_index);
 585                 if (status == IB_SUCCESS)
 586                         OSM_LOG(p_log, OSM_LOG_DEBUG,
 587                                 "Updated pkey table block %u for node 0x%016"
 588                                 PRIx64 " port %u (%s)\n", block_index,
 589                                 cl_ntoh64(osm_node_get_node_guid(p_node)),
 590                                 osm_physp_get_port_num(p_physp),
 591                                 p_physp->p_node->print_desc);
 592                 else {
 593                         OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0506: "
 594                                 "pkey_mgr_update_pkey_entry() failed to update "
 595                                 "pkey table block %u for node 0x%016" PRIx64
 596                                 " port %u (%s)\n", block_index,
 597                                 cl_ntoh64(osm_node_get_node_guid(p_node)),
 598                                 osm_physp_get_port_num(p_physp),
 599                                 p_physp->p_node->print_desc);
 600                         ret = -1;
 601                 }
 602         }
 603
 604         return ret;
 605 }
 606
 607 static int last_used_pkey_index(const osm_port_t * const p_port,
 608                                 const osm_pkey_tbl_t * p_pkey_tbl,
 609                                 uint16_t * p_last_index)
 610 {
 611         ib_pkey_table_t *last_block;
 612         uint16_t index, last_index = 0;
 613
 614         CL_ASSERT(p_last_index);
 615
 616         last_block = osm_pkey_tbl_new_block_get(p_pkey_tbl,
 617                                                 p_pkey_tbl->used_blocks - 1);
 618         if (!last_block)
 619                 return 1;
 620
 621         if (p_pkey_tbl->used_blocks == p_pkey_tbl->max_blocks)
 622                 last_index = cl_ntoh16(p_port->p_node->node_info.partition_cap) % IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
 623         if (last_index == 0)
 624                 last_index = IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
 625         index = last_index;
 626         do {
 627                 index--;
 628                 if (!ib_pkey_is_invalid(last_block->pkey_entry[index]))
 629                         break;
 630         } while (index != 0);
 631
 632         *p_last_index = index;
 633         return 0;
 634 }
 635
 636 static int update_peer_block(osm_log_t * p_log, osm_sm_t * sm,
 637                              osm_physp_t * peer,
 638                              osm_pkey_tbl_t * p_peer_pkey_tbl,
 639                              ib_pkey_table_t * new_peer_block,
 640                              uint16_t peer_block_idx, osm_node_t * p_node)
 641 {
 642         int ret = 0;
 643         ib_pkey_table_t *peer_block;
 644
 645         peer_block = osm_pkey_tbl_block_get(p_peer_pkey_tbl, peer_block_idx);
 646         if (!peer_block ||
 647             memcmp(peer_block, new_peer_block, sizeof(*peer_block))) {
 648                 if (pkey_mgr_update_pkey_entry(sm, peer, new_peer_block,
 649                                                peer_block_idx) != IB_SUCCESS) {
 650                         OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0509: "
 651                                 "pkey_mgr_update_pkey_entry() failed to update "
 652                                 "pkey table block %u for node 0x%016"
 653                                 PRIx64 " port %u (%s)\n",
 654                                 peer_block_idx,
 655                                 cl_ntoh64(osm_node_get_node_guid(p_node)),
 656                                 osm_physp_get_port_num(peer),
 657                                 p_node->print_desc);
 658                         ret = -1;
 659                 }
 660         }
 661
 662         return ret;
 663 }
 664
 665 static int new_pkey_exists(osm_pkey_tbl_t * p_pkey_tbl, ib_net16_t pkey)
 666 {
 667         uint16_t num_blocks;
 668         uint16_t block_index;
 669         ib_pkey_table_t *block;
 670         uint16_t pkey_idx;
 671
 672         num_blocks = (uint16_t) cl_ptr_vector_get_size(&p_pkey_tbl->new_blocks);
 673         for (block_index = 0; block_index < num_blocks; block_index++) {
 674                 block = osm_pkey_tbl_new_block_get(p_pkey_tbl, block_index);
 675                 if (!block)
 676                         continue;
 677
 678                 for (pkey_idx = 0; pkey_idx < IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
 679                      pkey_idx++) {
 680                         if (block->pkey_entry[pkey_idx] == pkey)
 681                                 return 1;
 682                 }
 683         }
 684         return 0;
 685 }
 686
 687 static int pkey_mgr_update_peer_port(osm_log_t * p_log, osm_sm_t * sm,
 688                                      const osm_subn_t * p_subn,
 689                                      const osm_port_t * const p_port,
 690                                      osm_partition_enforce_type_enum enforce_type)
 691 {
 692         osm_physp_t *p_physp, *peer;
 693         osm_node_t *p_node;
 694         ib_pkey_table_t *block;
 695         const osm_pkey_tbl_t *p_pkey_tbl;
 696         osm_pkey_tbl_t *p_peer_pkey_tbl;
 697         uint16_t block_index, peer_block_idx;
 698         uint16_t peer_max_blocks;
 699         uint16_t last_index;
 700         ib_pkey_table_t new_peer_block;
 701         uint16_t pkey_idx, peer_pkey_idx;
 702         ib_net16_t pkey, full_pkey;
 703         int ret = 0, loop_exit = 0;
 704
 705         p_physp = p_port->p_physp;
 706         if (!p_physp)
 707                 return -1;
 708         peer = osm_physp_get_remote(p_physp);
 709         if (!peer)
 710                 return -1;
 711         p_node = osm_physp_get_node_ptr(peer);
 712         if (!p_node->sw || !p_node->sw->switch_info.enforce_cap)
 713                 return 0;
 714
 715         if (enforce_type == OSM_PARTITION_ENFORCE_TYPE_OFF) {
 716                 pkey_mgr_enforce_partition(p_log, sm, peer, OSM_PARTITION_ENFORCE_TYPE_OFF);
 717                 return ret;
 718         }
 719
 720         p_pkey_tbl = osm_physp_get_pkey_tbl(p_physp);
 721         peer_max_blocks = pkey_mgr_get_physp_max_blocks(peer);
 722         p_peer_pkey_tbl = &peer->pkeys;
 723         peer_block_idx = 0;
 724         peer_pkey_idx = 0;
 725         for (block_index = 0; block_index < p_pkey_tbl->used_blocks;
 726              block_index++) {
 727                 if (loop_exit)
 728                         break;
 729                 block = osm_pkey_tbl_new_block_get(p_pkey_tbl, block_index);
 730                 if (!block)
 731                         continue;
 732                 for (pkey_idx = 0; pkey_idx < IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
 733                      pkey_idx++) {
 734                         pkey = block->pkey_entry[pkey_idx];
 735                         if (ib_pkey_is_invalid(pkey))
 736                                 continue;
 737                         if (!ib_pkey_is_full_member(pkey)) {
 738                                 full_pkey = pkey | IB_PKEY_TYPE_MASK;
 739                                 if (new_pkey_exists(&p_physp->pkeys, full_pkey))
 740                                         continue;
 741                         }
 742                         new_peer_block.pkey_entry[peer_pkey_idx] = pkey;
 743                         if (peer_block_idx >= peer_max_blocks) {
 744                                 loop_exit = 1;
 745                                 break;
 746                         }
 747                         if (++peer_pkey_idx == IB_NUM_PKEY_ELEMENTS_IN_BLOCK) {
 748                                 if (update_peer_block(p_log, sm, peer,
 749                                                       p_peer_pkey_tbl,
 750                                                       &new_peer_block,
 751                                                       peer_block_idx, p_node))
 752                                         ret = -1;
 753                                 peer_pkey_idx = 0;
 754                                 peer_block_idx++;
 755                         }
 756                 }
 757         }
 758
 759         if (peer_block_idx < peer_max_blocks) {
 760                 if (peer_pkey_idx) {
 761                         /* Handle partial last block */
 762                         for (; peer_pkey_idx < IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
 763                              peer_pkey_idx++)
 764                                 new_peer_block.pkey_entry[peer_pkey_idx] = 0;
 765                         if (update_peer_block(p_log, sm, peer, p_peer_pkey_tbl,
 766                                               &new_peer_block, peer_block_idx,
 767                                               p_node))
 768                                 ret = -1;
 769                 } else
 770                         peer_block_idx--;
 771
 772                 p_peer_pkey_tbl->used_blocks = peer_block_idx + 1;
 773                 if (p_peer_pkey_tbl->used_blocks == peer_max_blocks) {
 774                         /* Is last used pkey index beyond switch peer port capacity ? */
 775                         if (!last_used_pkey_index(p_port, p_peer_pkey_tbl,
 776                                                   &last_index)) {
 777                                 last_index += peer_block_idx * IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
 778                                 if (cl_ntoh16(p_node->sw->switch_info.enforce_cap) <= last_index) {
 779                                         OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0507: "
 780                                                 "Not enough pkey entries (%u <= %u) on switch 0x%016"
 781                                                 PRIx64 " port %u (%s). Clearing Enforcement bit\n",
 782                                                 cl_ntoh16(p_node->sw->switch_info.enforce_cap),
 783                                                 last_index,
 784                                                 cl_ntoh64(osm_node_get_node_guid(p_node)),
 785                                                 osm_physp_get_port_num(peer),
 786                                                 p_node->print_desc);
 787                                         enforce_type = OSM_PARTITION_ENFORCE_TYPE_OFF;
 788                                         ret = -1;
 789                                 }
 790                         }
 791                 }
 792         } else {
 793                 p_peer_pkey_tbl->used_blocks = peer_max_blocks;
 794                 enforce_type = OSM_PARTITION_ENFORCE_TYPE_OFF;
 795         }
 796
 797         if (!ret)
 798                 OSM_LOG(p_log, OSM_LOG_DEBUG,
 799                         "Pkey table was successfully updated for node 0x%016"
 800                         PRIx64 " port %u (%s)\n",
 801                         cl_ntoh64(osm_node_get_node_guid(p_node)),
 802                         osm_physp_get_port_num(peer), p_node->print_desc);
 803
 804         if (pkey_mgr_enforce_partition(p_log, sm, peer, enforce_type))
 805                 ret = -1;
 806
 807         return ret;
 808 }
 809
 810 int osm_pkey_mgr_process(IN osm_opensm_t * p_osm)
 811 {
 812         cl_qmap_t *p_tbl;
 813         cl_map_item_t *p_next;
 814         osm_prtn_t *p_prtn;
 815         osm_port_t *p_port;
 816         osm_switch_t *p_sw;
 817         osm_physp_t *p_physp;
 818         osm_node_t *p_remote_node;
 819         uint8_t i;
 820         int ret = 0;
 821
 822         CL_ASSERT(p_osm);
 823
 824         OSM_LOG_ENTER(&p_osm->log);
 825
 826         CL_PLOCK_EXCL_ACQUIRE(&p_osm->lock);
 827
 828         if (osm_prtn_make_partitions(&p_osm->log, &p_osm->subn) != IB_SUCCESS) {
 829                 OSM_LOG(&p_osm->log, OSM_LOG_ERROR, "ERR 0510: "
 830                         "osm_prtn_make_partitions() failed\n");
 831                 ret = -1;
 832                 goto _err;
 833         }
 834
 835         /* populate the pending pkey entries by scanning all partitions */
 836         p_tbl = &p_osm->subn.prtn_pkey_tbl;
 837         p_next = cl_qmap_head(p_tbl);
 838         while (p_next != cl_qmap_end(p_tbl)) {
 839                 p_prtn = (osm_prtn_t *) p_next;
 840                 p_next = cl_qmap_next(p_next);
 841                 pkey_mgr_process_partition_table(&p_osm->log, &p_osm->sm,
 842                                                  p_prtn, FALSE);
 843                 pkey_mgr_process_partition_table(&p_osm->log, &p_osm->sm,
 844                                                  p_prtn, TRUE);
 845         }
 846
 847         /* calculate and set new pkey tables */
 848         p_tbl = &p_osm->subn.port_guid_tbl;
 849         p_next = cl_qmap_head(p_tbl);
 850         while (p_next != cl_qmap_end(p_tbl)) {
 851                 p_port = (osm_port_t *) p_next;
 852                 p_next = cl_qmap_next(p_next);
 853                 if (pkey_mgr_update_port(&p_osm->log, &p_osm->sm, p_port))
 854                         ret = -1;
 855                 if ((osm_node_get_type(p_port->p_node) != IB_NODE_TYPE_SWITCH)
 856                     && pkey_mgr_update_peer_port(&p_osm->log, &p_osm->sm,
 857                                                  &p_osm->subn, p_port,
 858                                                  p_osm->subn.opt.part_enforce_enum))
 859                         ret = -1;
 860         }
 861
 862         /* clear partition enforcement on inter-switch links */
 863         p_tbl = &p_osm->subn.sw_guid_tbl;
 864         p_next = cl_qmap_head(p_tbl);
 865         while (p_next != cl_qmap_end(p_tbl)) {
 866                 p_sw = (osm_switch_t *) p_next;
 867                 p_next = cl_qmap_next(p_next);
 868                 for (i = 1; i < p_sw->num_ports; i++) {
 869                         p_physp = osm_node_get_physp_ptr(p_sw->p_node, i);
 870                         if (p_physp && p_physp->p_remote_physp)
 871                                 p_remote_node = p_physp->p_remote_physp->p_node;
 872                         else
 873                                 continue;
 874
 875                         if (osm_node_get_type(p_remote_node) != IB_NODE_TYPE_SWITCH)
 876                                 continue;
 877
 878                         if(! (p_physp->port_info.vl_enforce & 0xc ))
 879                                 continue;
 880
 881                         /* clear partition enforcement */
 882                         if (pkey_mgr_enforce_partition(&p_osm->log, &p_osm->sm, p_physp, OSM_PARTITION_ENFORCE_TYPE_OFF))
 883                                 ret = -1;
 884                 }
 885         }
 886 _err:
 887         CL_PLOCK_RELEASE(&p_osm->lock);
 888         OSM_LOG_EXIT(&p_osm->log);
 889         return ret;
 890 }