2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Intel Corporation. All rights reserved.
4 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
5 * Copyright (c) 2005 Voltaire, Inc. All rights reserved.
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 * $Id: cache.c 1349 2004-12-16 21:09:43Z roland $
38 #include <sys/cdefs.h>
39 __FBSDID("$FreeBSD$");
41 #include <sys/types.h>
43 #include <sys/param.h>
44 #include <sys/systm.h>
45 #include <sys/malloc.h>
46 #include <sys/kernel.h>
47 #include <sys/taskqueue.h>
49 #include <sys/rwlock.h>
50 #include <sys/mutex.h>
51 #include <sys/module.h>
52 #include <sys/syslog.h>
56 #include <sys/condvar.h>
57 #include <sys/socket.h>
58 #include <sys/condvar.h>
61 #include <contrib/rdma/ib_cache.h>
63 #include "core_priv.h"
65 struct ib_pkey_cache {
72 union ib_gid table[0];
75 struct ib_update_work {
77 struct ib_device *device;
81 static inline int start_port(struct ib_device *device)
83 return (device->node_type == RDMA_NODE_IB_SWITCH) ? 0 : 1;
86 static inline int end_port(struct ib_device *device)
88 return (device->node_type == RDMA_NODE_IB_SWITCH) ?
89 0 : device->phys_port_cnt;
92 int ib_get_cached_gid(struct ib_device *device,
97 struct ib_gid_cache *cache;
100 if (port_num < start_port(device) || port_num > end_port(device))
103 mtx_lock(&device->cache.lock);
105 cache = device->cache.gid_cache[port_num - start_port(device)];
107 if (index < 0 || index >= cache->table_len)
110 *gid = cache->table[index];
112 mtx_unlock(&device->cache.lock);
117 int ib_find_cached_gid(struct ib_device *device,
122 struct ib_gid_cache *cache;
130 mtx_lock(&device->cache.lock);
132 for (p = 0; p <= end_port(device) - start_port(device); ++p) {
133 cache = device->cache.gid_cache[p];
134 for (i = 0; i < cache->table_len; ++i) {
135 if (!memcmp(gid, &cache->table[i], sizeof *gid)) {
136 *port_num = p + start_port(device);
145 mtx_unlock(&device->cache.lock);
150 int ib_get_cached_pkey(struct ib_device *device,
155 struct ib_pkey_cache *cache;
158 if (port_num < start_port(device) || port_num > end_port(device))
161 mtx_lock(&device->cache.lock);
163 cache = device->cache.pkey_cache[port_num - start_port(device)];
165 if (index < 0 || index >= cache->table_len)
168 *pkey = cache->table[index];
170 mtx_unlock(&device->cache.lock);
175 int ib_find_cached_pkey(struct ib_device *device,
180 struct ib_pkey_cache *cache;
184 if (port_num < start_port(device) || port_num > end_port(device))
187 mtx_lock(&device->cache.lock);
189 cache = device->cache.pkey_cache[port_num - start_port(device)];
193 for (i = 0; i < cache->table_len; ++i)
194 if ((cache->table[i] & 0x7fff) == (pkey & 0x7fff)) {
200 mtx_unlock(&device->cache.lock);
205 int ib_get_cached_lmc(struct ib_device *device,
211 if (port_num < start_port(device) || port_num > end_port(device))
214 mtx_lock(&device->cache.lock);
215 *lmc = device->cache.lmc_cache[port_num - start_port(device)];
216 mtx_unlock(&device->cache.lock);
221 static void ib_cache_update(struct ib_device *device,
224 struct ib_port_attr *tprops = NULL;
225 struct ib_pkey_cache *pkey_cache = NULL, *old_pkey_cache;
226 struct ib_gid_cache *gid_cache = NULL, *old_gid_cache;
230 tprops = malloc(sizeof *tprops, M_DEVBUF, M_NOWAIT);
234 ret = ib_query_port(device, port, tprops);
236 log(LOG_WARNING, "ib_query_port failed (%d) for %s\n",
241 pkey_cache = malloc(sizeof *pkey_cache + tprops->pkey_tbl_len *
242 sizeof *pkey_cache->table, M_DEVBUF, M_NOWAIT);
246 pkey_cache->table_len = tprops->pkey_tbl_len;
248 gid_cache = malloc(sizeof *gid_cache + tprops->gid_tbl_len *
249 sizeof *gid_cache->table, M_DEVBUF, M_NOWAIT);
253 gid_cache->table_len = tprops->gid_tbl_len;
255 for (i = 0; i < pkey_cache->table_len; ++i) {
256 ret = ib_query_pkey(device, port, i, pkey_cache->table + i);
258 log(LOG_WARNING, "ib_query_pkey failed (%d) for %s (index %d)\n",
259 ret, device->name, i);
264 for (i = 0; i < gid_cache->table_len; ++i) {
265 ret = ib_query_gid(device, port, i, gid_cache->table + i);
267 log(LOG_WARNING, "ib_query_gid failed (%d) for %s (index %d)\n",
268 ret, device->name, i);
273 mtx_lock(&device->cache.lock);
275 old_pkey_cache = device->cache.pkey_cache[port - start_port(device)];
276 old_gid_cache = device->cache.gid_cache [port - start_port(device)];
278 device->cache.pkey_cache[port - start_port(device)] = pkey_cache;
279 device->cache.gid_cache [port - start_port(device)] = gid_cache;
281 device->cache.lmc_cache[port - start_port(device)] = tprops->lmc;
283 mtx_unlock(&device->cache.lock);
285 free(old_pkey_cache, M_DEVBUF);
286 free(old_gid_cache, M_DEVBUF);
287 free(tprops, M_DEVBUF);
291 free(pkey_cache, M_DEVBUF);
292 free(gid_cache, M_DEVBUF);
293 free(tprops, M_DEVBUF);
296 static void ib_cache_task(void *context, int pending)
298 struct ib_update_work *work = context;
300 ib_cache_update(work->device, work->port_num);
301 free(work, M_DEVBUF);
304 static void ib_cache_event(struct ib_event_handler *handler,
305 struct ib_event *event)
307 struct ib_update_work *work;
309 if (event->event == IB_EVENT_PORT_ERR ||
310 event->event == IB_EVENT_PORT_ACTIVE ||
311 event->event == IB_EVENT_LID_CHANGE ||
312 event->event == IB_EVENT_PKEY_CHANGE ||
313 event->event == IB_EVENT_SM_CHANGE ||
314 event->event == IB_EVENT_CLIENT_REREGISTER) {
315 work = malloc(sizeof *work, M_DEVBUF, M_NOWAIT);
317 TASK_INIT(&work->task, 0, ib_cache_task, work);
318 work->device = event->device;
319 work->port_num = event->element.port_num;
320 taskqueue_enqueue(taskqueue_thread, &work->task);
325 static void ib_cache_setup_one(struct ib_device *device)
329 mtx_init(&device->cache.lock, "ib device cache", NULL,
332 device->cache.pkey_cache =
333 malloc(sizeof *device->cache.pkey_cache *
334 (end_port(device) - start_port(device) + 1), M_DEVBUF,
336 device->cache.gid_cache =
337 malloc(sizeof *device->cache.gid_cache *
338 (end_port(device) - start_port(device) + 1), M_DEVBUF,
341 device->cache.lmc_cache = malloc(sizeof *device->cache.lmc_cache *
343 start_port(device) + 1),
346 if (!device->cache.pkey_cache || !device->cache.gid_cache ||
347 !device->cache.lmc_cache) {
348 log(LOG_WARNING, "Couldn't allocate cache "
349 "for %s\n", device->name);
353 for (p = 0; p <= end_port(device) - start_port(device); ++p) {
354 device->cache.pkey_cache[p] = NULL;
355 device->cache.gid_cache [p] = NULL;
356 ib_cache_update(device, p + start_port(device));
359 INIT_IB_EVENT_HANDLER(&device->cache.event_handler,
360 device, ib_cache_event);
361 if (ib_register_event_handler(&device->cache.event_handler))
367 for (p = 0; p <= end_port(device) - start_port(device); ++p) {
368 free(device->cache.pkey_cache[p], M_DEVBUF);
369 free(device->cache.gid_cache[p], M_DEVBUF);
373 free(device->cache.pkey_cache, M_DEVBUF);
374 free(device->cache.gid_cache, M_DEVBUF);
375 free(device->cache.lmc_cache, M_DEVBUF);
378 static void ib_cache_cleanup_one(struct ib_device *device)
382 ib_unregister_event_handler(&device->cache.event_handler);
384 flush_scheduled_work();
387 for (p = 0; p <= end_port(device) - start_port(device); ++p) {
388 free(device->cache.pkey_cache[p], M_DEVBUF);
389 free(device->cache.gid_cache[p], M_DEVBUF);
392 free(device->cache.pkey_cache, M_DEVBUF);
393 free(device->cache.gid_cache, M_DEVBUF);
394 free(device->cache.lmc_cache, M_DEVBUF);
397 static struct ib_client cache_client = {
399 .add = ib_cache_setup_one,
400 .remove = ib_cache_cleanup_one
403 int ib_cache_setup(void)
405 return ib_register_client(&cache_client);
408 void ib_cache_cleanup(void)
410 ib_unregister_client(&cache_client);