contrib/ofed/management/opensm/opensm/osm_pkey_mgr.c

   1 /*
   2  * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
   3  * Copyright (c) 2002-2006 Mellanox Technologies LTD. All rights reserved.
   4  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenIB.org BSD license below:
  11  *
  12  *     Redistribution and use in source and binary forms, with or
  13  *     without modification, are permitted provided that the following
  14  *     conditions are met:
  15  *
  16  *      - Redistributions of source code must retain the above
  17  *        copyright notice, this list of conditions and the following
  18  *        disclaimer.
  19  *
  20  *      - Redistributions in binary form must reproduce the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer in the documentation and/or other materials
  23  *        provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  *
  34  */
  35
  36 /*
  37  * Abstract:
  38  * Implementation of the P_Key Manager (Partititon Manager).
  39  * This is part of the OpenSM.
  40  */
  41
  42 #if HAVE_CONFIG_H
  43 #  include <config.h>
  44 #endif                          /* HAVE_CONFIG_H */
  45
  46 #include <string.h>
  47 #include <iba/ib_types.h>
  48 #include <complib/cl_qmap.h>
  49 #include <complib/cl_debug.h>
  50 #include <opensm/osm_node.h>
  51 #include <opensm/osm_switch.h>
  52 #include <opensm/osm_partition.h>
  53 #include <opensm/osm_opensm.h>
  54
  55 /**********************************************************************
  56  **********************************************************************/
  57 /*
  58   The max number of pkey blocks for a physical port is located in
  59   a different place for switch external ports (SwitchInfo) and the
  60   rest of the ports (NodeInfo).
  61 */
  62 static uint16_t
  63 pkey_mgr_get_physp_max_blocks(IN const osm_subn_t * p_subn,
  64                               IN const osm_physp_t * p_physp)
  65 {
  66         osm_node_t *p_node = osm_physp_get_node_ptr(p_physp);
  67         uint16_t num_pkeys = 0;
  68
  69         if (!p_node->sw || (osm_physp_get_port_num(p_physp) == 0))
  70                 num_pkeys = cl_ntoh16(p_node->node_info.partition_cap);
  71         else
  72                 num_pkeys = cl_ntoh16(p_node->sw->switch_info.enforce_cap);
  73         return ((num_pkeys + 31) / 32);
  74 }
  75
  76 /**********************************************************************
  77  **********************************************************************/
  78 /*
  79  * Insert new pending pkey entry to the specific port pkey table
  80  * pending pkeys. New entries are inserted at the back.
  81  */
  82 static void
  83 pkey_mgr_process_physical_port(IN osm_log_t * p_log,
  84                                IN osm_sm_t * sm,
  85                                IN const ib_net16_t pkey,
  86                                IN osm_physp_t * p_physp)
  87 {
  88         osm_node_t *p_node = osm_physp_get_node_ptr(p_physp);
  89         osm_pkey_tbl_t *p_pkey_tbl;
  90         ib_net16_t *p_orig_pkey;
  91         char *stat = NULL;
  92         osm_pending_pkey_t *p_pending;
  93
  94         p_pkey_tbl = &p_physp->pkeys;
  95         p_pending = (osm_pending_pkey_t *) malloc(sizeof(osm_pending_pkey_t));
  96         if (!p_pending) {
  97                 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0502: "
  98                         "Failed to allocate new pending pkey entry for node "
  99                         "0x%016" PRIx64 " port %u\n",
 100                         cl_ntoh64(osm_node_get_node_guid(p_node)),
 101                         osm_physp_get_port_num(p_physp));
 102                 return;
 103         }
 104         p_pending->pkey = pkey;
 105         p_orig_pkey = cl_map_get(&p_pkey_tbl->keys, ib_pkey_get_base(pkey));
 106         if (!p_orig_pkey) {
 107                 p_pending->is_new = TRUE;
 108                 cl_qlist_insert_tail(&p_pkey_tbl->pending,
 109                                      (cl_list_item_t *) p_pending);
 110                 stat = "inserted";
 111         } else {
 112                 CL_ASSERT(ib_pkey_get_base(*p_orig_pkey) ==
 113                           ib_pkey_get_base(pkey));
 114                 p_pending->is_new = FALSE;
 115                 if (osm_pkey_tbl_get_block_and_idx(p_pkey_tbl, p_orig_pkey,
 116                                                    &p_pending->block,
 117                                                    &p_pending->index) !=
 118                     IB_SUCCESS) {
 119                         OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0503: "
 120                                 "Failed to obtain P_Key 0x%04x block and index for node "
 121                                 "0x%016" PRIx64 " port %u\n",
 122                                 ib_pkey_get_base(pkey),
 123                                 cl_ntoh64(osm_node_get_node_guid(p_node)),
 124                                 osm_physp_get_port_num(p_physp));
 125                         return;
 126                 }
 127                 cl_qlist_insert_head(&p_pkey_tbl->pending,
 128                                      (cl_list_item_t *) p_pending);
 129                 stat = "updated";
 130         }
 131
 132         OSM_LOG(p_log, OSM_LOG_DEBUG,
 133                 "pkey 0x%04x was %s for node 0x%016" PRIx64 " port %u\n",
 134                 cl_ntoh16(pkey), stat,
 135                 cl_ntoh64(osm_node_get_node_guid(p_node)),
 136                 osm_physp_get_port_num(p_physp));
 137 }
 138
 139 /**********************************************************************
 140  **********************************************************************/
 141 static void
 142 pkey_mgr_process_partition_table(osm_log_t * p_log, osm_sm_t * sm,
 143                                  const osm_prtn_t * p_prtn,
 144                                  const boolean_t full)
 145 {
 146         const cl_map_t *p_tbl =
 147             full ? &p_prtn->full_guid_tbl : &p_prtn->part_guid_tbl;
 148         cl_map_iterator_t i, i_next;
 149         ib_net16_t pkey = p_prtn->pkey;
 150         osm_physp_t *p_physp;
 151
 152         if (full)
 153                 pkey |= cl_hton16(0x8000);
 154
 155         i_next = cl_map_head(p_tbl);
 156         while (i_next != cl_map_end(p_tbl)) {
 157                 i = i_next;
 158                 i_next = cl_map_next(i);
 159                 p_physp = cl_map_obj(i);
 160                 if (p_physp)
 161                         pkey_mgr_process_physical_port(p_log, sm, pkey,
 162                                                        p_physp);
 163         }
 164 }
 165
 166 /**********************************************************************
 167  **********************************************************************/
 168 static ib_api_status_t
 169 pkey_mgr_update_pkey_entry(IN osm_sm_t * sm,
 170                            IN const osm_physp_t * p_physp,
 171                            IN const ib_pkey_table_t * block,
 172                            IN const uint16_t block_index)
 173 {
 174         osm_madw_context_t context;
 175         osm_node_t *p_node = osm_physp_get_node_ptr(p_physp);
 176         uint32_t attr_mod;
 177
 178         context.pkey_context.node_guid = osm_node_get_node_guid(p_node);
 179         context.pkey_context.port_guid = osm_physp_get_port_guid(p_physp);
 180         context.pkey_context.set_method = TRUE;
 181         attr_mod = block_index;
 182         if (osm_node_get_type(p_node) == IB_NODE_TYPE_SWITCH)
 183                 attr_mod |= osm_physp_get_port_num(p_physp) << 16;
 184         return osm_req_set(sm, osm_physp_get_dr_path_ptr(p_physp),
 185                            (uint8_t *) block, sizeof(*block),
 186                            IB_MAD_ATTR_P_KEY_TABLE,
 187                            cl_hton32(attr_mod), CL_DISP_MSGID_NONE, &context);
 188 }
 189
 190 /**********************************************************************
 191  **********************************************************************/
 192 static boolean_t
 193 pkey_mgr_enforce_partition(IN osm_log_t * p_log, osm_sm_t * sm,
 194                            IN osm_physp_t * p_physp, IN const boolean_t enforce)
 195 {
 196         osm_madw_context_t context;
 197         uint8_t payload[IB_SMP_DATA_SIZE];
 198         ib_port_info_t *p_pi;
 199         ib_api_status_t status;
 200
 201         p_pi = &p_physp->port_info;
 202
 203         if ((p_pi->vl_enforce & 0xc) == (0xc) * (enforce == TRUE)) {
 204                 OSM_LOG(p_log, OSM_LOG_DEBUG,
 205                         "No need to update PortInfo for "
 206                         "node 0x%016" PRIx64 " port %u\n",
 207                         cl_ntoh64(osm_node_get_node_guid
 208                                   (osm_physp_get_node_ptr(p_physp))),
 209                         osm_physp_get_port_num(p_physp));
 210                 return FALSE;
 211         }
 212
 213         memset(payload, 0, IB_SMP_DATA_SIZE);
 214         memcpy(payload, p_pi, sizeof(ib_port_info_t));
 215
 216         p_pi = (ib_port_info_t *) payload;
 217         if (enforce == TRUE)
 218                 p_pi->vl_enforce |= 0xc;
 219         else
 220                 p_pi->vl_enforce &= ~0xc;
 221         p_pi->state_info2 = 0;
 222         ib_port_info_set_port_state(p_pi, IB_LINK_NO_CHANGE);
 223
 224         context.pi_context.node_guid =
 225             osm_node_get_node_guid(osm_physp_get_node_ptr(p_physp));
 226         context.pi_context.port_guid = osm_physp_get_port_guid(p_physp);
 227         context.pi_context.set_method = TRUE;
 228         context.pi_context.light_sweep = FALSE;
 229         context.pi_context.active_transition = FALSE;
 230
 231         status = osm_req_set(sm, osm_physp_get_dr_path_ptr(p_physp),
 232                              payload, sizeof(payload),
 233                              IB_MAD_ATTR_PORT_INFO,
 234                              cl_hton32(osm_physp_get_port_num(p_physp)),
 235                              CL_DISP_MSGID_NONE, &context);
 236         if (status != IB_SUCCESS) {
 237                 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0511: "
 238                         "Failed to set PortInfo for "
 239                         "node 0x%016" PRIx64 " port %u\n",
 240                         cl_ntoh64(osm_node_get_node_guid
 241                                   (osm_physp_get_node_ptr(p_physp))),
 242                         osm_physp_get_port_num(p_physp));
 243                 return FALSE;
 244         } else {
 245                 OSM_LOG(p_log, OSM_LOG_DEBUG,
 246                         "Set PortInfo for node 0x%016" PRIx64 " port %u\n",
 247                         cl_ntoh64(osm_node_get_node_guid
 248                                   (osm_physp_get_node_ptr(p_physp))),
 249                         osm_physp_get_port_num(p_physp));
 250                 return TRUE;
 251         }
 252 }
 253
 254 /**********************************************************************
 255  **********************************************************************/
 256 static boolean_t pkey_mgr_update_port(osm_log_t * p_log, osm_sm_t * sm,
 257                                       const osm_port_t * const p_port)
 258 {
 259         osm_physp_t *p_physp;
 260         osm_node_t *p_node;
 261         ib_pkey_table_t *block, *new_block;
 262         osm_pkey_tbl_t *p_pkey_tbl;
 263         uint16_t block_index;
 264         uint8_t pkey_index;
 265         uint16_t last_free_block_index = 0;
 266         uint8_t last_free_pkey_index = 0;
 267         uint16_t num_of_blocks;
 268         uint16_t max_num_of_blocks;
 269         ib_api_status_t status;
 270         boolean_t ret_val = FALSE;
 271         osm_pending_pkey_t *p_pending;
 272         boolean_t found;
 273         ib_pkey_table_t empty_block;
 274
 275         memset(&empty_block, 0, sizeof(ib_pkey_table_t));
 276
 277         p_physp = p_port->p_physp;
 278         if (!p_physp)
 279                 return FALSE;
 280
 281         p_node = osm_physp_get_node_ptr(p_physp);
 282         p_pkey_tbl = &p_physp->pkeys;
 283         num_of_blocks = osm_pkey_tbl_get_num_blocks(p_pkey_tbl);
 284         max_num_of_blocks =
 285             pkey_mgr_get_physp_max_blocks(sm->p_subn, p_physp);
 286         if (p_pkey_tbl->max_blocks > max_num_of_blocks) {
 287                 OSM_LOG(p_log, OSM_LOG_INFO,
 288                         "Max number of blocks reduced from %u to %u "
 289                         "for node 0x%016" PRIx64 " port %u\n",
 290                         p_pkey_tbl->max_blocks, max_num_of_blocks,
 291                         cl_ntoh64(osm_node_get_node_guid(p_node)),
 292                         osm_physp_get_port_num(p_physp));
 293         }
 294         p_pkey_tbl->max_blocks = max_num_of_blocks;
 295
 296         osm_pkey_tbl_init_new_blocks(p_pkey_tbl);
 297         p_pkey_tbl->used_blocks = 0;
 298
 299         /*
 300            process every pending pkey in order -
 301            first must be "updated" last are "new"
 302          */
 303         p_pending =
 304             (osm_pending_pkey_t *) cl_qlist_remove_head(&p_pkey_tbl->pending);
 305         while (p_pending !=
 306                (osm_pending_pkey_t *) cl_qlist_end(&p_pkey_tbl->pending)) {
 307                 if (p_pending->is_new == FALSE) {
 308                         block_index = p_pending->block;
 309                         pkey_index = p_pending->index;
 310                         found = TRUE;
 311                 } else {
 312                         found = osm_pkey_find_next_free_entry(p_pkey_tbl,
 313                                                               &last_free_block_index,
 314                                                               &last_free_pkey_index);
 315                         if (!found) {
 316                                 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0504: "
 317                                         "Failed to find empty space for new pkey 0x%04x "
 318                                         "for node 0x%016" PRIx64 " port %u\n",
 319                                         cl_ntoh16(p_pending->pkey),
 320                                         cl_ntoh64(osm_node_get_node_guid
 321                                                   (p_node)),
 322                                         osm_physp_get_port_num(p_physp));
 323                         } else {
 324                                 block_index = last_free_block_index;
 325                                 pkey_index = last_free_pkey_index++;
 326                         }
 327                 }
 328
 329                 if (found) {
 330                         if (IB_SUCCESS !=
 331                             osm_pkey_tbl_set_new_entry(p_pkey_tbl, block_index,
 332                                                        pkey_index,
 333                                                        p_pending->pkey)) {
 334                                 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0505: "
 335                                         "Failed to set PKey 0x%04x in block %u idx %u "
 336                                         "for node 0x%016" PRIx64 " port %u\n",
 337                                         cl_ntoh16(p_pending->pkey), block_index,
 338                                         pkey_index,
 339                                         cl_ntoh64(osm_node_get_node_guid
 340                                                   (p_node)),
 341                                         osm_physp_get_port_num(p_physp));
 342                         }
 343                 }
 344
 345                 free(p_pending);
 346                 p_pending =
 347                     (osm_pending_pkey_t *) cl_qlist_remove_head(&p_pkey_tbl->
 348                                                                 pending);
 349         }
 350
 351         /* now look for changes and store */
 352         for (block_index = 0; block_index < num_of_blocks; block_index++) {
 353                 block = osm_pkey_tbl_block_get(p_pkey_tbl, block_index);
 354                 new_block = osm_pkey_tbl_new_block_get(p_pkey_tbl, block_index);
 355                 if (!new_block)
 356                         new_block = &empty_block;
 357                 if (block && !memcmp(new_block, block, sizeof(*block)))
 358                         continue;
 359
 360                 status =
 361                     pkey_mgr_update_pkey_entry(sm, p_physp, new_block,
 362                                                block_index);
 363                 if (status == IB_SUCCESS) {
 364                         OSM_LOG(p_log, OSM_LOG_DEBUG,
 365                                 "Updated pkey table block %d for node 0x%016"
 366                                 PRIx64 " port %u\n", block_index,
 367                                 cl_ntoh64(osm_node_get_node_guid(p_node)),
 368                                 osm_physp_get_port_num(p_physp));
 369                         ret_val = TRUE;
 370                 } else {
 371                         OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0506: "
 372                                 "pkey_mgr_update_pkey_entry() failed to update "
 373                                 "pkey table block %d for node 0x%016" PRIx64
 374                                 " port %u\n", block_index,
 375                                 cl_ntoh64(osm_node_get_node_guid(p_node)),
 376                                 osm_physp_get_port_num(p_physp));
 377                 }
 378         }
 379
 380         return ret_val;
 381 }
 382
 383 /**********************************************************************
 384  **********************************************************************/
 385 static boolean_t
 386 pkey_mgr_update_peer_port(osm_log_t * p_log, osm_sm_t * sm,
 387                           const osm_subn_t * p_subn,
 388                           const osm_port_t * const p_port, boolean_t enforce)
 389 {
 390         osm_physp_t *p_physp, *peer;
 391         osm_node_t *p_node;
 392         ib_pkey_table_t *block, *peer_block;
 393         const osm_pkey_tbl_t *p_pkey_tbl;
 394         osm_pkey_tbl_t *p_peer_pkey_tbl;
 395         uint16_t block_index;
 396         uint16_t num_of_blocks;
 397         uint16_t peer_max_blocks;
 398         ib_api_status_t status = IB_SUCCESS;
 399         boolean_t ret_val = FALSE;
 400         boolean_t port_info_set = FALSE;
 401         ib_pkey_table_t empty_block;
 402
 403         memset(&empty_block, 0, sizeof(ib_pkey_table_t));
 404
 405         p_physp = p_port->p_physp;
 406         if (!p_physp)
 407                 return FALSE;
 408         peer = osm_physp_get_remote(p_physp);
 409         if (!peer)
 410                 return FALSE;
 411         p_node = osm_physp_get_node_ptr(peer);
 412         if (!p_node->sw || !p_node->sw->switch_info.enforce_cap)
 413                 return FALSE;
 414
 415         p_pkey_tbl = osm_physp_get_pkey_tbl(p_physp);
 416         p_peer_pkey_tbl = &peer->pkeys;
 417         num_of_blocks = osm_pkey_tbl_get_num_blocks(p_pkey_tbl);
 418         peer_max_blocks = pkey_mgr_get_physp_max_blocks(p_subn, peer);
 419         if (peer_max_blocks < p_pkey_tbl->used_blocks) {
 420                 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0508: "
 421                         "Not enough pkey entries (%u < %u) on switch 0x%016"
 422                         PRIx64 " port %u. Clearing Enforcement bit\n",
 423                         peer_max_blocks, num_of_blocks,
 424                         cl_ntoh64(osm_node_get_node_guid(p_node)),
 425                         osm_physp_get_port_num(peer));
 426                 enforce = FALSE;
 427         }
 428
 429         if (pkey_mgr_enforce_partition(p_log, sm, peer, enforce))
 430                 port_info_set = TRUE;
 431
 432         if (enforce == FALSE)
 433                 return port_info_set;
 434
 435         p_peer_pkey_tbl->used_blocks = p_pkey_tbl->used_blocks;
 436         for (block_index = 0; block_index < p_pkey_tbl->used_blocks;
 437              block_index++) {
 438                 block = osm_pkey_tbl_new_block_get(p_pkey_tbl, block_index);
 439                 if (!block)
 440                         block = &empty_block;
 441
 442                 peer_block =
 443                     osm_pkey_tbl_block_get(p_peer_pkey_tbl, block_index);
 444                 if (!peer_block
 445                     || memcmp(peer_block, block, sizeof(*peer_block))) {
 446                         status =
 447                             pkey_mgr_update_pkey_entry(sm, peer, block,
 448                                                        block_index);
 449                         if (status == IB_SUCCESS)
 450                                 ret_val = TRUE;
 451                         else
 452                                 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0509: "
 453                                         "pkey_mgr_update_pkey_entry() failed to update "
 454                                         "pkey table block %d for node 0x%016"
 455                                         PRIx64 " port %u\n", block_index,
 456                                         cl_ntoh64(osm_node_get_node_guid
 457                                                   (p_node)),
 458                                         osm_physp_get_port_num(peer));
 459                 }
 460         }
 461
 462         if (ret_val)
 463                 OSM_LOG(p_log, OSM_LOG_DEBUG,
 464                         "Pkey table was updated for node 0x%016" PRIx64
 465                         " port %u\n",
 466                         cl_ntoh64(osm_node_get_node_guid(p_node)),
 467                         osm_physp_get_port_num(peer));
 468
 469         if (port_info_set)
 470                 return TRUE;
 471         return ret_val;
 472 }
 473
 474 /**********************************************************************
 475  **********************************************************************/
 476 osm_signal_t osm_pkey_mgr_process(IN osm_opensm_t * p_osm)
 477 {
 478         cl_qmap_t *p_tbl;
 479         cl_map_item_t *p_next;
 480         osm_prtn_t *p_prtn;
 481         osm_port_t *p_port;
 482         osm_signal_t signal = OSM_SIGNAL_DONE;
 483
 484         CL_ASSERT(p_osm);
 485
 486         OSM_LOG_ENTER(&p_osm->log);
 487
 488         CL_PLOCK_EXCL_ACQUIRE(&p_osm->lock);
 489
 490         if (osm_prtn_make_partitions(&p_osm->log, &p_osm->subn) != IB_SUCCESS) {
 491                 OSM_LOG(&p_osm->log, OSM_LOG_ERROR, "ERR 0510: "
 492                         "osm_prtn_make_partitions() failed\n");
 493                 goto _err;
 494         }
 495
 496         /* populate the pending pkey entries by scanning all partitions */
 497         p_tbl = &p_osm->subn.prtn_pkey_tbl;
 498         p_next = cl_qmap_head(p_tbl);
 499         while (p_next != cl_qmap_end(p_tbl)) {
 500                 p_prtn = (osm_prtn_t *) p_next;
 501                 p_next = cl_qmap_next(p_next);
 502                 pkey_mgr_process_partition_table(&p_osm->log, &p_osm->sm,
 503                                                  p_prtn, FALSE);
 504                 pkey_mgr_process_partition_table(&p_osm->log, &p_osm->sm,
 505                                                  p_prtn, TRUE);
 506         }
 507
 508         /* calculate and set new pkey tables */
 509         p_tbl = &p_osm->subn.port_guid_tbl;
 510         p_next = cl_qmap_head(p_tbl);
 511         while (p_next != cl_qmap_end(p_tbl)) {
 512                 p_port = (osm_port_t *) p_next;
 513                 p_next = cl_qmap_next(p_next);
 514                 if (pkey_mgr_update_port(&p_osm->log, &p_osm->sm, p_port))
 515                         signal = OSM_SIGNAL_DONE_PENDING;
 516                 if ((osm_node_get_type(p_port->p_node) != IB_NODE_TYPE_SWITCH)
 517                     && pkey_mgr_update_peer_port(&p_osm->log, &p_osm->sm,
 518                                                  &p_osm->subn, p_port,
 519                                                  !p_osm->subn.opt.
 520                                                  no_partition_enforcement))
 521                         signal = OSM_SIGNAL_DONE_PENDING;
 522         }
 523
 524 _err:
 525         CL_PLOCK_RELEASE(&p_osm->lock);
 526         OSM_LOG_EXIT(&p_osm->log);
 527         return (signal);
 528 }