]> CyberLeo.Net >> Repos - FreeBSD/releng/10.0.git/blob - contrib/ofed/management/opensm/libvendor/osm_vendor_ibumad.c
- Copy stable/10 (r259064) to releng/10.0 as part of the
[FreeBSD/releng/10.0.git] / contrib / ofed / management / opensm / libvendor / osm_vendor_ibumad.c
1 /*
2  * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
3  * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
4  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  *
34  */
35
36 /*
37  * Abstract:
38  *    Implementation of osm_vendor_t (for umad).
39  * This object represents the OpenIB vendor layer.
40  * This object is part of the opensm family of objects.
41  *
42  * Environment:
43  *    Linux User Mode
44  *
45  */
46
47 #if HAVE_CONFIG_H
48 #  include <config.h>
49 #endif                          /* HAVE_CONFIG_H */
50
51 #ifdef OSM_VENDOR_INTF_OPENIB
52
53 #include <unistd.h>
54 #include <stdlib.h>
55 #include <fcntl.h>
56 #include <errno.h>
57
58 #include <iba/ib_types.h>
59 #include <complib/cl_qlist.h>
60 #include <complib/cl_math.h>
61 #include <complib/cl_debug.h>
62 #include <opensm/osm_madw.h>
63 #include <opensm/osm_log.h>
64 #include <opensm/osm_mad_pool.h>
65 #include <opensm/osm_helper.h>
66 #include <vendor/osm_vendor_api.h>
67
68 /****s* OpenSM: Vendor UMAD/osm_umad_bind_info_t
69  * NAME
70  *   osm_umad_bind_info_t
71  *
72  * DESCRIPTION
73  *    Structure containing bind information.
74  *
75  * SYNOPSIS
76  */
77 typedef struct _osm_umad_bind_info {
78         osm_vendor_t *p_vend;
79         void *client_context;
80         osm_mad_pool_t *p_mad_pool;
81         osm_vend_mad_recv_callback_t mad_recv_callback;
82         osm_vend_mad_send_err_callback_t send_err_callback;
83         ib_net64_t port_guid;
84         int port_id;
85         int agent_id;
86         int agent_id1;          /* SMI requires two agents */
87 } osm_umad_bind_info_t;
88
89 typedef struct _umad_receiver {
90         pthread_t tid;
91         osm_vendor_t *p_vend;
92         osm_log_t *p_log;
93 } umad_receiver_t;
94
95 static void osm_vendor_close_port(osm_vendor_t * const p_vend);
96
97 static void clear_madw(osm_vendor_t * p_vend)
98 {
99         umad_match_t *m, *e, *old_m;
100         ib_net64_t old_tid;
101
102         OSM_LOG_ENTER(p_vend->p_log);
103         pthread_mutex_lock(&p_vend->match_tbl_mutex);
104         for (m = p_vend->mtbl.tbl, e = m + p_vend->mtbl.max; m < e; m++) {
105                 if (m->tid) {
106                         old_m = m;
107                         old_tid = m->tid;
108                         m->tid = 0;
109                         osm_mad_pool_put(((osm_umad_bind_info_t
110                                            *) ((osm_madw_t *) m->v)->h_bind)->
111                                          p_mad_pool, m->v);
112                         pthread_mutex_unlock(&p_vend->match_tbl_mutex);
113                         OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 5401: "
114                                 "evicting entry %p (tid was 0x%" PRIx64 ")\n",
115                                 old_m, old_tid);
116                         goto Exit;
117                 }
118         }
119         pthread_mutex_unlock(&p_vend->match_tbl_mutex);
120
121 Exit:
122         OSM_LOG_EXIT(p_vend->p_log);
123 }
124
125 static osm_madw_t *get_madw(osm_vendor_t * p_vend, ib_net64_t * tid)
126 {
127         umad_match_t *m, *e;
128         ib_net64_t mtid = (*tid & CL_HTON64(0x00000000ffffffffllu));
129         osm_madw_t *res;
130
131         /*
132          * Since mtid == 0 is the empty key, we should not
133          * waste time looking for it
134          */
135         if (mtid == 0)
136                 return 0;
137
138         pthread_mutex_lock(&p_vend->match_tbl_mutex);
139         for (m = p_vend->mtbl.tbl, e = m + p_vend->mtbl.max; m < e; m++) {
140                 if (m->tid == mtid) {
141                         m->tid = 0;
142                         *tid = mtid;
143                         res = m->v;
144                         pthread_mutex_unlock(&p_vend->match_tbl_mutex);
145                         return res;
146                 }
147         }
148
149         pthread_mutex_unlock(&p_vend->match_tbl_mutex);
150         return 0;
151 }
152
153 static void
154 put_madw(osm_vendor_t * p_vend, osm_madw_t * p_madw, ib_net64_t tid)
155 {
156         umad_match_t *m, *e, *old_lru, *lru = 0;
157         osm_madw_t *p_req_madw;
158         osm_umad_bind_info_t *p_bind;
159         ib_net64_t old_tid;
160         uint32_t oldest = ~0;
161
162         pthread_mutex_lock(&p_vend->match_tbl_mutex);
163         for (m = p_vend->mtbl.tbl, e = m + p_vend->mtbl.max; m < e; m++) {
164                 if (m->tid == 0) {
165                         m->tid = tid;
166                         m->v = p_madw;
167                         m->version =
168                             cl_atomic_inc((atomic32_t *) & p_vend->mtbl.
169                                           last_version);
170                         pthread_mutex_unlock(&p_vend->match_tbl_mutex);
171                         return;
172                 }
173                 if (oldest > m->version) {
174                         oldest = m->version;
175                         lru = m;
176                 }
177         }
178
179         old_lru = lru;
180         old_tid = lru->tid;
181         p_req_madw = old_lru->v;
182         p_bind = p_req_madw->h_bind;
183         p_req_madw->status = IB_CANCELED;
184         pthread_mutex_lock(&p_vend->cb_mutex);
185         (*p_bind->send_err_callback) (p_bind->client_context, p_req_madw);
186         pthread_mutex_unlock(&p_vend->cb_mutex);
187         lru->tid = tid;
188         lru->v = p_madw;
189         lru->version =
190             cl_atomic_inc((atomic32_t *) & p_vend->mtbl.last_version);
191         pthread_mutex_unlock(&p_vend->match_tbl_mutex);
192         OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 5402: "
193                 "evicting entry %p (tid was 0x%" PRIx64 ")\n", old_lru,
194                 cl_ntoh64(old_tid));
195 }
196
197 static void
198 ib_mad_addr_conv(ib_user_mad_t * umad, osm_mad_addr_t * osm_mad_addr,
199                  int is_smi)
200 {
201         ib_mad_addr_t *ib_mad_addr = umad_get_mad_addr(umad);
202         osm_mad_addr->dest_lid = ib_mad_addr->lid;
203         osm_mad_addr->path_bits = ib_mad_addr->path_bits;
204         osm_mad_addr->static_rate = 0;
205
206         if (is_smi) {
207                 osm_mad_addr->addr_type.smi.source_lid = osm_mad_addr->dest_lid;
208                 osm_mad_addr->addr_type.smi.port_num = 255;     /* not used */
209                 return;
210         }
211
212         osm_mad_addr->addr_type.gsi.remote_qp = ib_mad_addr->qpn;
213         osm_mad_addr->addr_type.gsi.remote_qkey = ib_mad_addr->qkey;
214         osm_mad_addr->addr_type.gsi.pkey_ix = umad_get_pkey(umad);
215         osm_mad_addr->addr_type.gsi.service_level = ib_mad_addr->sl;
216         osm_mad_addr->addr_type.gsi.global_route = 0;   /* FIXME: handle GRH */
217         memset(&osm_mad_addr->addr_type.gsi.grh_info, 0,
218                sizeof osm_mad_addr->addr_type.gsi.grh_info);
219 }
220
221 static void *swap_mad_bufs(osm_madw_t * p_madw, void *umad)
222 {
223         void *old;
224
225         old = p_madw->vend_wrap.umad;
226         p_madw->vend_wrap.umad = umad;
227         p_madw->p_mad = umad_get_mad(umad);
228
229         return old;
230 }
231
232 static void unlock_mutex(void *arg)
233 {
234         pthread_mutex_unlock(arg);
235 }
236
237 static void *umad_receiver(void *p_ptr)
238 {
239         umad_receiver_t *const p_ur = (umad_receiver_t *) p_ptr;
240         osm_vendor_t *p_vend = p_ur->p_vend;
241         osm_umad_bind_info_t *p_bind;
242         ib_mad_addr_t *ib_mad_addr;
243         osm_mad_addr_t osm_addr;
244         osm_madw_t *p_madw, *p_req_madw;
245         ib_mad_t *mad;
246         void *umad = 0;
247         int mad_agent, length;
248
249         OSM_LOG_ENTER(p_ur->p_log);
250
251         for (;;) {
252                 if (!umad &&
253                     !(umad = umad_alloc(1, umad_size() + MAD_BLOCK_SIZE))) {
254                         OSM_LOG(p_ur->p_log, OSM_LOG_ERROR, "ERR 5403: "
255                                 "can't alloc MAD sized umad\n");
256                         break;
257                 }
258
259                 length = MAD_BLOCK_SIZE;
260                 if ((mad_agent = umad_recv(p_vend->umad_port_id, umad,
261                                            &length, -1)) < 0) {
262                         if (length <= MAD_BLOCK_SIZE) {
263                                 OSM_LOG(p_ur->p_log, OSM_LOG_ERROR, "ERR 5404: "
264                                         "recv error on MAD sized umad (%m)\n");
265                                 continue;
266                         } else {
267                                 umad_free(umad);
268                                 /* Need a larger buffer for RMPP */
269                                 umad = umad_alloc(1, umad_size() + length);
270                                 if (!umad) {
271                                         OSM_LOG(p_ur->p_log, OSM_LOG_ERROR,
272                                                 "ERR 5405: "
273                                                 "can't alloc umad length %d\n",
274                                                 length);
275                                         continue;
276                                 }
277
278                                 if ((mad_agent = umad_recv(p_vend->umad_port_id,
279                                                            umad, &length,
280                                                            -1)) < 0) {
281                                         OSM_LOG(p_ur->p_log, OSM_LOG_ERROR,
282                                                 "ERR 5406: "
283                                                 "recv error on umad length %d (%m)\n",
284                                                 length);
285                                         continue;
286                                 }
287                         }
288                 }
289
290                 if (mad_agent >= UMAD_CA_MAX_AGENTS ||
291                     !(p_bind = p_vend->agents[mad_agent])) {
292                         OSM_LOG(p_ur->p_log, OSM_LOG_ERROR, "ERR 5407: "
293                                 "invalid mad agent %d - dropping\n", mad_agent);
294                         continue;
295                 }
296
297                 mad = (ib_mad_t *) umad_get_mad(umad);
298                 ib_mad_addr = umad_get_mad_addr(umad);
299
300                 ib_mad_addr_conv(umad, &osm_addr,
301                                  mad->mgmt_class == IB_MCLASS_SUBN_LID ||
302                                  mad->mgmt_class == IB_MCLASS_SUBN_DIR);
303
304                 if (!(p_madw = osm_mad_pool_get(p_bind->p_mad_pool,
305                                                 (osm_bind_handle_t) p_bind,
306                                                 MAX(length, MAD_BLOCK_SIZE),
307                                                 &osm_addr))) {
308                         OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 5408: "
309                                 "request for a new madw failed -- dropping packet\n");
310                         continue;
311                 }
312
313                 /* Need to fix up MAD size if short RMPP packet */
314                 if (length < MAD_BLOCK_SIZE)
315                         p_madw->mad_size = length;
316
317                 /*
318                  * Avoid copying by swapping mad buf pointers.
319                  * Do not use umad after this line of code.
320                  */
321                 umad = swap_mad_bufs(p_madw, umad);
322
323                 /* if status != 0 then we are handling recv timeout on send */
324                 if (umad_status(p_madw->vend_wrap.umad)) {
325                         OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 5409: "
326                                 "send completed with error"
327                                 " (method=0x%X attr=0x%X trans_id=0x%" PRIx64
328                                 ") -- dropping\n",
329                                 mad->method, cl_ntoh16(mad->attr_id),
330                                 cl_ntoh64(mad->trans_id));
331                         if (mad->mgmt_class != IB_MCLASS_SUBN_DIR) {
332                                 /* LID routed */
333                                 OSM_LOG(p_vend->p_log, OSM_LOG_ERROR,
334                                         "ERR 5410: class 0x%x LID 0x%x\n",
335                                         mad->mgmt_class,
336                                         cl_ntoh16(ib_mad_addr->lid));
337                         } else {
338                                 ib_smp_t *smp;
339
340                                 /* Direct routed SMP */
341                                 smp = (ib_smp_t *) mad;
342                                 OSM_LOG(p_vend->p_log, OSM_LOG_ERROR,
343                                         "ERR 5411: DR SMP Hop Ptr: 0x%X\n",
344                                         smp->hop_ptr);
345                                 osm_dump_smp_dr_path(p_vend->p_log, smp,
346                                                      OSM_LOG_ERROR);
347                         }
348
349                         if (!(p_req_madw = get_madw(p_vend, &mad->trans_id))) {
350                                 OSM_LOG(p_vend->p_log, OSM_LOG_ERROR,
351                                         "ERR 5412: "
352                                         "Failed to obtain request madw for timed out MAD"
353                                         "(method=0x%X attr=0x%X tid=0x%"PRIx64") -- dropping\n",
354                                         mad->method, cl_ntoh16(mad->attr_id),
355                                         cl_ntoh64(mad->trans_id));
356                         } else {
357                                 p_req_madw->status = IB_TIMEOUT;
358                                 /* cb frees req_madw */
359                                 pthread_mutex_lock(&p_vend->cb_mutex);
360                                 pthread_cleanup_push(unlock_mutex,
361                                                      &p_vend->cb_mutex);
362                                 (*p_bind->send_err_callback) (p_bind->
363                                                               client_context,
364                                                               p_req_madw);
365                                 pthread_cleanup_pop(1);
366                         }
367
368                         osm_mad_pool_put(p_bind->p_mad_pool, p_madw);
369                         continue;
370                 }
371
372                 p_req_madw = 0;
373                 if (ib_mad_is_response(mad) &&
374                     !(p_req_madw = get_madw(p_vend, &mad->trans_id))) {
375                         OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 5413: "
376                                 "Failed to obtain request madw for received MAD"
377                                 "(method=0x%X attr=0x%X tid=0x%"PRIx64") -- dropping\n",
378                                 mad->method, cl_ntoh16((mad)->attr_id),
379                                 cl_ntoh64(mad->trans_id));
380                         osm_mad_pool_put(p_bind->p_mad_pool, p_madw);
381                         continue;
382                 }
383 #ifndef VENDOR_RMPP_SUPPORT
384                 if ((mad->mgmt_class != IB_MCLASS_SUBN_DIR) &&
385                     (mad->mgmt_class != IB_MCLASS_SUBN_LID) &&
386                     (ib_rmpp_is_flag_set((ib_rmpp_mad_t *) mad,
387                                          IB_RMPP_FLAG_ACTIVE))) {
388                         OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 5414: "
389                                 "class 0x%x method 0x%x RMPP version %d type "
390                                 "%d flags 0x%x received -- dropping\n",
391                                 mad->mgmt_class, mad->method,
392                                 ((ib_rmpp_mad_t *) mad)->rmpp_version,
393                                 ((ib_rmpp_mad_t *) mad)->rmpp_type,
394                                 ((ib_rmpp_mad_t *) mad)->rmpp_flags);
395                         osm_mad_pool_put(p_bind->p_mad_pool, p_madw);
396                         continue;
397                 }
398 #endif
399
400                 /* call the CB */
401                 pthread_mutex_lock(&p_vend->cb_mutex);
402                 pthread_cleanup_push(unlock_mutex, &p_vend->cb_mutex);
403                 (*p_bind->mad_recv_callback) (p_madw, p_bind->client_context,
404                                               p_req_madw);
405                 pthread_cleanup_pop(1);
406         }
407
408         OSM_LOG_EXIT(p_vend->p_log);
409         return NULL;
410 }
411
412 static int umad_receiver_start(osm_vendor_t * p_vend)
413 {
414         umad_receiver_t *p_ur = p_vend->receiver;
415
416         p_ur->p_vend = p_vend;
417         p_ur->p_log = p_vend->p_log;
418
419         if (pthread_create(&p_ur->tid, NULL, umad_receiver, p_ur) < 0)
420                 return -1;
421
422         return 0;
423 }
424
425 static void umad_receiver_stop(umad_receiver_t * p_ur)
426 {
427         pthread_cancel(p_ur->tid);
428         pthread_join(p_ur->tid, NULL);
429         p_ur->tid = 0;
430         p_ur->p_vend = NULL;
431         p_ur->p_log = NULL;
432 }
433
434 /**********************************************************************
435  **********************************************************************/
436 ib_api_status_t
437 osm_vendor_init(IN osm_vendor_t * const p_vend,
438                 IN osm_log_t * const p_log, IN const uint32_t timeout)
439 {
440         char *max = NULL;
441         int r, n_cas;
442
443         OSM_LOG_ENTER(p_log);
444
445         p_vend->p_log = p_log;
446         p_vend->timeout = timeout;
447         p_vend->max_retries = OSM_DEFAULT_RETRY_COUNT;
448         pthread_mutex_init(&p_vend->cb_mutex, NULL);
449         pthread_mutex_init(&p_vend->match_tbl_mutex, NULL);
450         p_vend->umad_port_id = -1;
451         p_vend->issmfd = -1;
452
453         /*
454          * Open our instance of UMAD.
455          */
456         if ((r = umad_init()) < 0) {
457                 OSM_LOG(p_vend->p_log, OSM_LOG_ERROR,
458                         "ERR 5415: Error opening UMAD\n");
459         }
460
461         if ((n_cas = umad_get_cas_names(p_vend->ca_names,
462                                         OSM_UMAD_MAX_CAS)) < 0) {
463                 OSM_LOG(p_vend->p_log, OSM_LOG_ERROR,
464                         "ERR 5416: umad_get_cas_names failed\n");
465                 r = n_cas;
466                 goto Exit;
467         }
468
469         p_vend->ca_count = n_cas;
470         p_vend->mtbl.max = DEFAULT_OSM_UMAD_MAX_PENDING;
471
472         if ((max = getenv("OSM_UMAD_MAX_PENDING")) != NULL) {
473                 int tmp = strtol(max, NULL, 0);
474                 if (tmp > 0)
475                         p_vend->mtbl.max = tmp;
476                 else
477                         OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "Error:"
478                                 "OSM_UMAD_MAX_PENDING=%d is invalid",
479                                 tmp);
480         }
481
482         OSM_LOG(p_vend->p_log, OSM_LOG_INFO, "%d pending umads specified\n",
483                 p_vend->mtbl.max);
484
485         p_vend->mtbl.tbl = calloc(p_vend->mtbl.max, sizeof(*(p_vend->mtbl.tbl)));
486         if (!p_vend->mtbl.tbl) {
487                 OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "Error:"
488                         "failed to allocate vendor match table\n");
489                 r = IB_INSUFFICIENT_MEMORY;
490                 goto Exit;
491         }
492
493 Exit:
494         OSM_LOG_EXIT(p_log);
495         return (r);
496 }
497
498 /**********************************************************************
499  **********************************************************************/
500 osm_vendor_t *osm_vendor_new(IN osm_log_t * const p_log,
501                              IN const uint32_t timeout)
502 {
503         osm_vendor_t *p_vend = NULL;
504
505         OSM_LOG_ENTER(p_log);
506
507         if (!timeout) {
508                 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 5433: "
509                         "transaction timeout cannot be 0\n");
510                 goto Exit;
511         }
512
513         p_vend = malloc(sizeof(*p_vend));
514         if (p_vend == NULL) {
515                 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 5417: "
516                         "Unable to allocate vendor object\n");
517                 goto Exit;
518         }
519
520         memset(p_vend, 0, sizeof(*p_vend));
521
522         if (osm_vendor_init(p_vend, p_log, timeout) < 0) {
523                 free(p_vend);
524                 p_vend = NULL;
525         }
526
527 Exit:
528         OSM_LOG_EXIT(p_log);
529         return (p_vend);
530 }
531
532 /**********************************************************************
533  **********************************************************************/
534 void osm_vendor_delete(IN osm_vendor_t ** const pp_vend)
535 {
536         osm_vendor_close_port(*pp_vend);
537
538         clear_madw(*pp_vend);
539         /* make sure all ports are closed */
540         umad_done();
541
542         pthread_mutex_destroy(&(*pp_vend)->cb_mutex);
543         pthread_mutex_destroy(&(*pp_vend)->match_tbl_mutex);
544         free((*pp_vend)->mtbl.tbl);
545         free(*pp_vend);
546         *pp_vend = NULL;
547 }
548
549 /**********************************************************************
550  **********************************************************************/
551 ib_api_status_t
552 osm_vendor_get_all_port_attr(IN osm_vendor_t * const p_vend,
553                              IN ib_port_attr_t * const p_attr_array,
554                              IN uint32_t * const p_num_ports)
555 {
556         umad_ca_t ca;
557         ib_port_attr_t *attr = p_attr_array;
558         unsigned done = 0;
559         int r, i, j;
560
561         OSM_LOG_ENTER(p_vend->p_log);
562
563         CL_ASSERT(p_vend && p_num_ports);
564
565         if (!*p_num_ports) {
566                 r = IB_INVALID_PARAMETER;
567                 OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 5418: "
568                         "Ports in should be > 0\n");
569                 goto Exit;
570         }
571
572         if (!p_attr_array) {
573                 r = IB_INSUFFICIENT_MEMORY;
574                 *p_num_ports = 0;
575                 goto Exit;
576         }
577
578         for (i = 0; i < p_vend->ca_count && !done; i++) {
579                 /*
580                  * For each CA, retrieve the port guids
581                  */
582                 if (umad_get_ca(p_vend->ca_names[i], &ca) == 0) {
583                         if (ca.node_type < 1 || ca.node_type > 3)
584                                 continue;
585                         for (j = 0; j <= ca.numports; j++) {
586                                 if (!ca.ports[j])
587                                         continue;
588                                 attr->port_guid = ca.ports[j]->port_guid;
589                                 attr->lid = ca.ports[j]->base_lid;
590                                 attr->port_num = ca.ports[j]->portnum;
591                                 attr->sm_lid = ca.ports[j]->sm_lid;
592                                 attr->link_state = ca.ports[j]->state;
593                                 attr++;
594                                 if (attr - p_attr_array > *p_num_ports) {
595                                         done = 1;
596                                         break;
597                                 }
598                         }
599                         umad_release_ca(&ca);
600                 }
601         }
602
603         *p_num_ports = attr - p_attr_array;
604         r = 0;
605
606 Exit:
607         OSM_LOG_EXIT(p_vend->p_log);
608         return r;
609 }
610
611 /**********************************************************************
612  **********************************************************************/
613 static int
614 osm_vendor_open_port(IN osm_vendor_t * const p_vend,
615                      IN const ib_net64_t port_guid)
616 {
617         ib_net64_t portguids[OSM_UMAD_MAX_PORTS_PER_CA + 1];
618         umad_ca_t umad_ca;
619         int i = 0, umad_port_id = -1;
620         char *name;
621         int ca, r;
622
623         CL_ASSERT(p_vend);
624
625         OSM_LOG_ENTER(p_vend->p_log);
626
627         if (p_vend->umad_port_id >= 0) {
628                 umad_port_id = p_vend->umad_port_id;
629                 goto Exit;
630         }
631
632         if (!port_guid) {
633                 name = NULL;
634                 i = 0;
635                 goto _found;
636         }
637
638         for (ca = 0; ca < p_vend->ca_count; ca++) {
639                 if ((r = umad_get_ca_portguids(p_vend->ca_names[ca],
640                                                portguids,
641                                                OSM_UMAD_MAX_CAS)) < 0) {
642                         OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 5421: "
643                                 "Unable to get CA %s port guids (%s)\n",
644                                 p_vend->ca_names[ca], strerror(r));
645                         goto Exit;
646                 }
647                 for (i = 0; i < r; i++)
648                         if (port_guid == portguids[i]) {
649                                 name = p_vend->ca_names[ca];
650                                 goto _found;
651                         }
652         }
653
654         /*
655          * No local CA owns this guid!
656          */
657         OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 5422: "
658                 "Unable to find requested CA guid 0x%" PRIx64 "\n",
659                 cl_ntoh64(port_guid));
660         goto Exit;
661
662 _found:
663         /* Validate that node is an IB node type (not iWARP) */
664         if (umad_get_ca(name, &umad_ca) < 0) {
665                 OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 542A: "
666                         "umad_get_ca() failed\n");
667                 goto Exit;
668         }
669
670         if (umad_ca.node_type < 1 || umad_ca.node_type > 3) {
671                 OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 542D: "
672                         "Type %d of node \'%s\' is not an IB node type\n",
673                         umad_ca.node_type, umad_ca.ca_name);
674                 fprintf(stderr,
675                         "Type %d of node \'%s\' is not an IB node type\n",
676                         umad_ca.node_type, umad_ca.ca_name);
677                 umad_release_ca(&umad_ca);
678                 goto Exit;
679         }
680         umad_release_ca(&umad_ca);
681
682         /* Port found, try to open it */
683         if (umad_get_port(name, i, &p_vend->umad_port) < 0) {
684                 OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 542B: "
685                         "umad_get_port() failed\n");
686                 goto Exit;
687         }
688
689         if ((umad_port_id = umad_open_port(p_vend->umad_port.ca_name,
690                                            p_vend->umad_port.portnum)) < 0) {
691                 OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 542C: "
692                         "umad_open_port() failed\n");
693                 goto Exit;
694         }
695
696         p_vend->umad_port_id = umad_port_id;
697
698         /* start receiver thread */
699         if (!(p_vend->receiver = calloc(1, sizeof(umad_receiver_t)))) {
700                 OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 5423: "
701                         "Unable to alloc receiver struct\n");
702                 umad_close_port(umad_port_id);
703                 umad_release_port(&p_vend->umad_port);
704                 p_vend->umad_port.port_guid = 0;
705                 p_vend->umad_port_id = umad_port_id = -1;
706                 goto Exit;
707         }
708         if (umad_receiver_start(p_vend) != 0) {
709                 OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 5420: "
710                         "umad_receiver_init failed\n");
711                 umad_close_port(umad_port_id);
712                 umad_release_port(&p_vend->umad_port);
713                 p_vend->umad_port.port_guid = 0;
714                 p_vend->umad_port_id = umad_port_id = -1;
715         }
716
717 Exit:
718         OSM_LOG_EXIT(p_vend->p_log);
719         return umad_port_id;
720 }
721
722 static void osm_vendor_close_port(osm_vendor_t * const p_vend)
723 {
724         umad_receiver_t *p_ur;
725         int i;
726
727         p_ur = p_vend->receiver;
728         p_vend->receiver = NULL;
729         if (p_ur) {
730                 umad_receiver_stop(p_ur);
731                 free(p_ur);
732         }
733
734         if (p_vend->umad_port_id >= 0) {
735                 for (i = 0; i < UMAD_CA_MAX_AGENTS; i++)
736                         if (p_vend->agents[i])
737                                 umad_unregister(p_vend->umad_port_id, i);
738                 umad_close_port(p_vend->umad_port_id);
739                 umad_release_port(&p_vend->umad_port);
740                 p_vend->umad_port.port_guid = 0;
741                 p_vend->umad_port_id = -1;
742         }
743 }
744
745 static int set_bit(int nr, void *method_mask)
746 {
747         long mask, *addr = method_mask;
748         int retval;
749
750         addr += nr / (8 * sizeof(long));
751         mask = 1L << (nr % (8 * sizeof(long)));
752         retval = (mask & *addr) != 0;
753         *addr |= mask;
754         return retval;
755 }
756
757 /**********************************************************************
758  **********************************************************************/
759 osm_bind_handle_t
760 osm_vendor_bind(IN osm_vendor_t * const p_vend,
761                 IN osm_bind_info_t * const p_user_bind,
762                 IN osm_mad_pool_t * const p_mad_pool,
763                 IN osm_vend_mad_recv_callback_t mad_recv_callback,
764                 IN osm_vend_mad_send_err_callback_t send_err_callback,
765                 IN void *context)
766 {
767         ib_net64_t port_guid;
768         osm_umad_bind_info_t *p_bind = 0;
769         long method_mask[16 / sizeof(long)];
770         int umad_port_id;
771         uint8_t rmpp_version;
772
773         OSM_LOG_ENTER(p_vend->p_log);
774
775         CL_ASSERT(p_user_bind);
776         CL_ASSERT(p_mad_pool);
777         CL_ASSERT(mad_recv_callback);
778         CL_ASSERT(send_err_callback);
779
780         port_guid = p_user_bind->port_guid;
781
782         OSM_LOG(p_vend->p_log, OSM_LOG_INFO,
783                 "Binding to port 0x%" PRIx64 "\n", cl_ntoh64(port_guid));
784
785         if ((umad_port_id = osm_vendor_open_port(p_vend, port_guid)) < 0) {
786                 OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 5424: "
787                         "Unable to open port 0x%" PRIx64 "\n",
788                         cl_ntoh64(port_guid));
789                 goto Exit;
790         }
791
792         if (umad_get_issm_path(p_vend->umad_port.ca_name,
793                                p_vend->umad_port.portnum,
794                                p_vend->issm_path,
795                                sizeof(p_vend->issm_path)) < 0) {
796                 OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 542E: "
797                         "Cannot resolve issm path for port %s:%u\n",
798                         p_vend->umad_port.ca_name, p_vend->umad_port.portnum);
799                 goto Exit;
800         }
801
802         if (!(p_bind = malloc(sizeof(*p_bind)))) {
803                 OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 5425: "
804                         "Unable to allocate internal bind object\n");
805                 goto Exit;
806         }
807
808         memset(p_bind, 0, sizeof(*p_bind));
809         p_bind->p_vend = p_vend;
810         p_bind->port_id = umad_port_id;
811         p_bind->client_context = context;
812         p_bind->mad_recv_callback = mad_recv_callback;
813         p_bind->send_err_callback = send_err_callback;
814         p_bind->p_mad_pool = p_mad_pool;
815         p_bind->port_guid = port_guid;
816
817         memset(method_mask, 0, sizeof method_mask);
818         if (p_user_bind->is_responder) {
819                 set_bit(IB_MAD_METHOD_GET, &method_mask);
820                 set_bit(IB_MAD_METHOD_SET, &method_mask);
821                 if (p_user_bind->mad_class == IB_MCLASS_SUBN_ADM) {
822                         set_bit(IB_MAD_METHOD_GETTABLE, &method_mask);
823                         set_bit(IB_MAD_METHOD_DELETE, &method_mask);
824 #ifdef DUAL_SIDED_RMPP
825                         set_bit(IB_MAD_METHOD_GETMULTI, &method_mask);
826 #endif
827                         /* Add in IB_MAD_METHOD_GETTRACETABLE */
828                         /* when supported by OpenSM */
829                 }
830         }
831         if (p_user_bind->is_report_processor)
832                 set_bit(IB_MAD_METHOD_REPORT, &method_mask);
833         if (p_user_bind->is_trap_processor) {
834                 set_bit(IB_MAD_METHOD_TRAP, &method_mask);
835                 set_bit(IB_MAD_METHOD_TRAP_REPRESS, &method_mask);
836         }
837 #ifndef VENDOR_RMPP_SUPPORT
838         rmpp_version = 0;
839 #else
840         /* If SA class, set rmpp_version */
841         if (p_user_bind->mad_class == IB_MCLASS_SUBN_ADM)
842                 rmpp_version = 1;
843         else
844                 rmpp_version = 0;
845 #endif
846
847         if ((p_bind->agent_id = umad_register(p_vend->umad_port_id,
848                                               p_user_bind->mad_class,
849                                               p_user_bind->class_version,
850                                               rmpp_version, method_mask)) < 0) {
851                 OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 5426: "
852                         "Unable to register class %u version %u\n",
853                         p_user_bind->mad_class, p_user_bind->class_version);
854                 free(p_bind);
855                 p_bind = 0;
856                 goto Exit;
857         }
858
859         if (p_bind->agent_id >= UMAD_CA_MAX_AGENTS ||
860             p_vend->agents[p_bind->agent_id]) {
861                 OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 5427: "
862                         "bad agent id %u or duplicate agent for class %u vers %u\n",
863                         p_bind->agent_id, p_user_bind->mad_class,
864                         p_user_bind->class_version);
865                 free(p_bind);
866                 p_bind = 0;
867                 goto Exit;
868         }
869
870         p_vend->agents[p_bind->agent_id] = p_bind;
871
872         /* If Subn Directed Route class, register Subn LID routed class */
873         if (p_user_bind->mad_class == IB_MCLASS_SUBN_DIR) {
874                 if ((p_bind->agent_id1 = umad_register(p_vend->umad_port_id,
875                                                        IB_MCLASS_SUBN_LID,
876                                                        p_user_bind->
877                                                        class_version, 0,
878                                                        method_mask)) < 0) {
879                         OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 5428: "
880                                 "Unable to register class 1 version %u\n",
881                                 p_user_bind->class_version);
882                         free(p_bind);
883                         p_bind = 0;
884                         goto Exit;
885                 }
886
887                 if (p_bind->agent_id1 >= UMAD_CA_MAX_AGENTS ||
888                     p_vend->agents[p_bind->agent_id1]) {
889                         OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 5429: "
890                                 "bad agent id %u or duplicate agent for class 1 vers %u\n",
891                                 p_bind->agent_id1, p_user_bind->class_version);
892                         free(p_bind);
893                         p_bind = 0;
894                         goto Exit;
895                 }
896
897                 p_vend->agents[p_bind->agent_id1] = p_bind;
898         }
899
900 Exit:
901         OSM_LOG_EXIT(p_vend->p_log);
902         return ((osm_bind_handle_t) p_bind);
903 }
904
905 /**********************************************************************
906  **********************************************************************/
907 static void
908 __osm_vendor_recv_dummy_cb(IN osm_madw_t * p_madw,
909                            IN void *bind_context, IN osm_madw_t * p_req_madw)
910 {
911 #ifdef _DEBUG_
912         fprintf(stderr,
913                 "__osm_vendor_recv_dummy_cb: Ignoring received MAD after osm_vendor_unbind\n");
914 #endif
915 }
916
917 /**********************************************************************
918  **********************************************************************/
919 static void
920 __osm_vendor_send_err_dummy_cb(IN void *bind_context,
921                                IN osm_madw_t * p_req_madw)
922 {
923 #ifdef _DEBUG_
924         fprintf(stderr,
925                 "__osm_vendor_send_err_dummy_cb: Ignoring send error after osm_vendor_unbind\n");
926 #endif
927 }
928
929 /**********************************************************************
930  **********************************************************************/
931 void osm_vendor_unbind(IN osm_bind_handle_t h_bind)
932 {
933         osm_umad_bind_info_t *p_bind = (osm_umad_bind_info_t *) h_bind;
934         osm_vendor_t *p_vend = p_bind->p_vend;
935
936         OSM_LOG_ENTER(p_vend->p_log);
937
938         pthread_mutex_lock(&p_vend->cb_mutex);
939         p_bind->mad_recv_callback = __osm_vendor_recv_dummy_cb;
940         p_bind->send_err_callback = __osm_vendor_send_err_dummy_cb;
941         pthread_mutex_unlock(&p_vend->cb_mutex);
942
943         OSM_LOG_EXIT(p_vend->p_log);
944 }
945
946 /**********************************************************************
947  **********************************************************************/
948 ib_mad_t *osm_vendor_get(IN osm_bind_handle_t h_bind,
949                          IN const uint32_t mad_size,
950                          IN osm_vend_wrap_t * const p_vw)
951 {
952         osm_umad_bind_info_t *p_bind = (osm_umad_bind_info_t *) h_bind;
953         osm_vendor_t *p_vend = p_bind->p_vend;
954
955         OSM_LOG_ENTER(p_vend->p_log);
956
957         OSM_LOG(p_vend->p_log, OSM_LOG_DEBUG,
958                 "Acquiring UMAD for p_madw = %p, size = %u\n", p_vw, mad_size);
959         CL_ASSERT(p_vw);
960         p_vw->size = mad_size;
961         p_vw->umad = umad_alloc(1, mad_size + umad_size());
962
963         /* track locally */
964         p_vw->h_bind = h_bind;
965
966         OSM_LOG(p_vend->p_log, OSM_LOG_DEBUG,
967                 "Acquired UMAD %p, size = %u\n", p_vw->umad, p_vw->size);
968
969         OSM_LOG_EXIT(p_vend->p_log);
970         return umad_get_mad(p_vw->umad);
971 }
972
973 /**********************************************************************
974  **********************************************************************/
975 void
976 osm_vendor_put(IN osm_bind_handle_t h_bind, IN osm_vend_wrap_t * const p_vw)
977 {
978         osm_umad_bind_info_t *p_bind = (osm_umad_bind_info_t *) h_bind;
979         osm_vendor_t *p_vend = p_bind->p_vend;
980         osm_madw_t *p_madw;
981
982         OSM_LOG_ENTER(p_vend->p_log);
983
984         CL_ASSERT(p_vw);
985
986         OSM_LOG(p_vend->p_log, OSM_LOG_DEBUG, "Retiring UMAD %p\n", p_vw->umad);
987
988         /*
989          * We moved the removal of the transaction to immediately after
990          * it was looked up.
991          */
992
993         /* free the mad but the wrapper is part of the madw object */
994         umad_free(p_vw->umad);
995         p_vw->umad = 0;
996         p_madw = PARENT_STRUCT(p_vw, osm_madw_t, vend_wrap);
997         p_madw->p_mad = NULL;
998
999         OSM_LOG_EXIT(p_vend->p_log);
1000 }
1001
1002 /**********************************************************************
1003  **********************************************************************/
1004 ib_api_status_t
1005 osm_vendor_send(IN osm_bind_handle_t h_bind,
1006                 IN osm_madw_t * const p_madw, IN boolean_t const resp_expected)
1007 {
1008         osm_umad_bind_info_t *const p_bind = h_bind;
1009         osm_vendor_t *const p_vend = p_bind->p_vend;
1010         osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw);
1011         osm_mad_addr_t *const p_mad_addr = osm_madw_get_mad_addr_ptr(p_madw);
1012         ib_mad_t *const p_mad = osm_madw_get_mad_ptr(p_madw);
1013         ib_sa_mad_t *const p_sa = (ib_sa_mad_t *) p_mad;
1014         int ret = -1;
1015         int is_rmpp = 0;
1016         uint32_t sent_mad_size;
1017 #ifndef VENDOR_RMPP_SUPPORT
1018         uint32_t paylen = 0;
1019 #endif
1020
1021         OSM_LOG_ENTER(p_vend->p_log);
1022
1023         CL_ASSERT(p_vw->h_bind == h_bind);
1024         CL_ASSERT(p_mad == umad_get_mad(p_vw->umad));
1025
1026         if (p_mad->mgmt_class == IB_MCLASS_SUBN_DIR) {
1027                 umad_set_addr_net(p_vw->umad, 0xffff, 0, 0, 0);
1028                 umad_set_grh(p_vw->umad, 0);
1029                 goto Resp;
1030         }
1031         if (p_mad->mgmt_class == IB_MCLASS_SUBN_LID) {
1032                 umad_set_addr_net(p_vw->umad, p_mad_addr->dest_lid, 0, 0, 0);
1033                 umad_set_grh(p_vw->umad, 0);
1034                 goto Resp;
1035         }
1036         /* GSI classes */
1037         umad_set_addr_net(p_vw->umad, p_mad_addr->dest_lid,
1038                           p_mad_addr->addr_type.gsi.remote_qp,
1039                           p_mad_addr->addr_type.gsi.service_level,
1040                           IB_QP1_WELL_KNOWN_Q_KEY);
1041         umad_set_grh(p_vw->umad, 0);    /* FIXME: GRH support */
1042         umad_set_pkey(p_vw->umad, p_mad_addr->addr_type.gsi.pkey_ix);
1043         if (ib_class_is_rmpp(p_mad->mgmt_class)) {      /* RMPP GSI classes     FIXME: no GRH */
1044                 if (!ib_rmpp_is_flag_set((ib_rmpp_mad_t *) p_sa,
1045                                          IB_RMPP_FLAG_ACTIVE)) {
1046                         /* Clear RMPP header when RMPP not ACTIVE */
1047                         p_sa->rmpp_version = 0;
1048                         p_sa->rmpp_type = 0;
1049                         p_sa->rmpp_flags = 0;
1050                         p_sa->rmpp_status = 0;
1051 #ifdef VENDOR_RMPP_SUPPORT
1052                 } else
1053                         is_rmpp = 1;
1054                 OSM_LOG(p_vend->p_log, OSM_LOG_VERBOSE, "RMPP %d length %d\n",
1055                         ib_rmpp_is_flag_set((ib_rmpp_mad_t *) p_sa,
1056                                             IB_RMPP_FLAG_ACTIVE),
1057                         p_madw->mad_size);
1058 #else
1059                 } else {
1060                         p_sa->rmpp_version = 1;
1061                         p_sa->seg_num = cl_ntoh32(1);   /* first DATA is seg 1 */
1062                         p_sa->rmpp_flags |= (uint8_t) 0x70;     /* RRespTime of 14 (high 5 bits) */
1063                         p_sa->rmpp_status = 0;
1064                         paylen = p_madw->mad_size - IB_SA_MAD_HDR_SIZE;
1065                         paylen += (IB_SA_MAD_HDR_SIZE - MAD_RMPP_HDR_SIZE);
1066                         p_sa->paylen_newwin = cl_ntoh32(paylen);
1067                 }
1068 #endif
1069         }
1070
1071 Resp:
1072         if (resp_expected)
1073                 put_madw(p_vend, p_madw, p_mad->trans_id);
1074
1075 #ifdef VENDOR_RMPP_SUPPORT
1076         sent_mad_size = p_madw->mad_size;
1077 #else
1078         sent_mad_size = is_rmpp ? p_madw->mad_size - IB_SA_MAD_HDR_SIZE :
1079             p_madw->mad_size;
1080 #endif
1081         if ((ret = umad_send(p_bind->port_id, p_bind->agent_id, p_vw->umad,
1082                              sent_mad_size,
1083                              resp_expected ? p_vend->timeout : 0,
1084                              p_vend->max_retries)) < 0) {
1085                 OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 5430: "
1086                         "Send p_madw = %p of size %d failed %d (%m)\n",
1087                         p_madw, sent_mad_size, ret);
1088                 if (resp_expected) {
1089                         get_madw(p_vend, &p_mad->trans_id);     /* remove from aging table */
1090                         p_madw->status = IB_ERROR;
1091                         pthread_mutex_lock(&p_vend->cb_mutex);
1092                         (*p_bind->send_err_callback) (p_bind->client_context, p_madw);  /* cb frees madw */
1093                         pthread_mutex_unlock(&p_vend->cb_mutex);
1094                 } else
1095                         osm_mad_pool_put(p_bind->p_mad_pool, p_madw);
1096                 goto Exit;
1097         }
1098
1099         if (!resp_expected)
1100                 osm_mad_pool_put(p_bind->p_mad_pool, p_madw);
1101
1102         OSM_LOG(p_vend->p_log, OSM_LOG_DEBUG, "Completed sending %s p_madw = %p\n",
1103                 resp_expected ? "request" : "response or unsolicited", p_madw);
1104 Exit:
1105         OSM_LOG_EXIT(p_vend->p_log);
1106         return (ret);
1107 }
1108
1109 /**********************************************************************
1110  **********************************************************************/
1111 ib_api_status_t osm_vendor_local_lid_change(IN osm_bind_handle_t h_bind)
1112 {
1113         osm_umad_bind_info_t *p_bind = (osm_umad_bind_info_t *) h_bind;
1114         osm_vendor_t *p_vend = p_bind->p_vend;
1115
1116         OSM_LOG_ENTER(p_vend->p_log);
1117         ;
1118         OSM_LOG_EXIT(p_vend->p_log);
1119         return (0);
1120 }
1121
1122 /**********************************************************************
1123  **********************************************************************/
1124 void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val)
1125 {
1126         osm_umad_bind_info_t *p_bind = (osm_umad_bind_info_t *) h_bind;
1127         osm_vendor_t *p_vend = p_bind->p_vend;
1128
1129         OSM_LOG_ENTER(p_vend->p_log);
1130         if (TRUE == is_sm_val) {
1131                 p_vend->issmfd = open(p_vend->issm_path, O_NONBLOCK);
1132                 if (p_vend->issmfd < 0) {
1133                         OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 5431: "
1134                                 "setting IS_SM capmask: cannot open file "
1135                                 "\'%s\': %s\n",
1136                                 p_vend->issm_path, strerror(errno));
1137                         p_vend->issmfd = -1;
1138                 }
1139         } else if (p_vend->issmfd != -1) {
1140                 if (0 != close(p_vend->issmfd))
1141                         OSM_LOG(p_vend->p_log, OSM_LOG_ERROR, "ERR 5432: "
1142                                 "clearing IS_SM capmask: cannot close: %s\n",
1143                                 strerror(errno));
1144                 p_vend->issmfd = -1;
1145         }
1146         OSM_LOG_EXIT(p_vend->p_log);
1147 }
1148
1149 void osm_vendor_set_debug(IN osm_vendor_t * const p_vend, IN int32_t level)
1150 {
1151         umad_debug(level);
1152 }
1153
1154 #endif                          /* OSM_VENDOR_INTF_OPENIB */