]> CyberLeo.Net >> Repos - FreeBSD/FreeBSD.git/blob - sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_ib.c
MFV r321673:
[FreeBSD/FreeBSD.git] / sys / ofed / drivers / infiniband / ulp / ipoib / ipoib_ib.c
1 /*
2  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
3  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
4  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5  * Copyright (c) 2004, 2005 Voltaire, Inc. All rights reserved.
6  *
7  * This software is available to you under a choice of one of two
8  * licenses.  You may choose to be licensed under the terms of the GNU
9  * General Public License (GPL) Version 2, available from the file
10  * COPYING in the main directory of this source tree, or the
11  * OpenIB.org BSD license below:
12  *
13  *     Redistribution and use in source and binary forms, with or
14  *     without modification, are permitted provided that the following
15  *     conditions are met:
16  *
17  *      - Redistributions of source code must retain the above
18  *        copyright notice, this list of conditions and the following
19  *        disclaimer.
20  *
21  *      - Redistributions in binary form must reproduce the above
22  *        copyright notice, this list of conditions and the following
23  *        disclaimer in the documentation and/or other materials
24  *        provided with the distribution.
25  *
26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33  * SOFTWARE.
34  */
35
36 #include "ipoib.h"
37
38 #include <rdma/ib_cache.h>
39
40 #include <security/mac/mac_framework.h>
41
42 #include <linux/delay.h>
43 #include <linux/dma-mapping.h>
44
45 #ifdef CONFIG_INFINIBAND_IPOIB_DEBUG_DATA
46 static int data_debug_level;
47
48 module_param(data_debug_level, int, 0644);
49 MODULE_PARM_DESC(data_debug_level,
50                  "Enable data path debug tracing if > 0");
51 #endif
52
53 static DEFINE_MUTEX(pkey_mutex);
54
55 struct ipoib_ah *ipoib_create_ah(struct ipoib_dev_priv *priv,
56                                  struct ib_pd *pd, struct ib_ah_attr *attr)
57 {
58         struct ipoib_ah *ah;
59
60         ah = kmalloc(sizeof *ah, GFP_KERNEL);
61         if (!ah)
62                 return NULL;
63
64         ah->priv      = priv;
65         ah->last_send = 0;
66         kref_init(&ah->ref);
67
68         ah->ah = ib_create_ah(pd, attr);
69         if (IS_ERR(ah->ah)) {
70                 kfree(ah);
71                 ah = NULL;
72         } else
73                 ipoib_dbg(priv, "Created ah %p\n", ah->ah);
74
75         return ah;
76 }
77
78 void ipoib_free_ah(struct kref *kref)
79 {
80         struct ipoib_ah *ah = container_of(kref, struct ipoib_ah, ref);
81         struct ipoib_dev_priv *priv = ah->priv;
82
83         unsigned long flags;
84
85         spin_lock_irqsave(&priv->lock, flags);
86         list_add_tail(&ah->list, &priv->dead_ahs);
87         spin_unlock_irqrestore(&priv->lock, flags);
88 }
89
90 void
91 ipoib_dma_unmap_rx(struct ipoib_dev_priv *priv, struct ipoib_rx_buf *rx_req)
92 {
93         struct mbuf *m;
94         int i;
95
96         for (i = 0, m = rx_req->mb; m != NULL; m = m->m_next, i++)
97                 ib_dma_unmap_single(priv->ca, rx_req->mapping[i], m->m_len,
98                     DMA_FROM_DEVICE);
99 }
100
101 void
102 ipoib_dma_mb(struct ipoib_dev_priv *priv, struct mbuf *mb, unsigned int length)
103 {
104
105         m_adj(mb, -(mb->m_pkthdr.len - length));
106 }
107
108 struct mbuf *
109 ipoib_alloc_map_mb(struct ipoib_dev_priv *priv, struct ipoib_rx_buf *rx_req,
110     int size)
111 {
112         struct mbuf *mb, *m;
113         int i, j;
114
115         rx_req->mb = NULL;
116         mb = m_getm2(NULL, size, M_NOWAIT, MT_DATA, M_PKTHDR);
117         if (mb == NULL)
118                 return (NULL);
119         for (i = 0, m = mb; m != NULL; m = m->m_next, i++) {
120                 m->m_len = M_SIZE(m);
121                 mb->m_pkthdr.len += m->m_len;
122                 rx_req->mapping[i] = ib_dma_map_single(priv->ca,
123                     mtod(m, void *), m->m_len, DMA_FROM_DEVICE);
124                 if (unlikely(ib_dma_mapping_error(priv->ca,
125                     rx_req->mapping[i])))
126                         goto error;
127
128         }
129         rx_req->mb = mb;
130         return (mb);
131 error:
132         for (j = 0, m = mb; j < i; m = m->m_next, j++)
133                 ib_dma_unmap_single(priv->ca, rx_req->mapping[j], m->m_len,
134                     DMA_FROM_DEVICE);
135         m_freem(mb);
136         return (NULL);
137
138 }
139
140 static int ipoib_ib_post_receive(struct ipoib_dev_priv *priv, int id)
141 {
142         struct ipoib_rx_buf *rx_req;
143         struct ib_recv_wr *bad_wr;
144         struct mbuf *m;
145         int ret;
146         int i;
147
148         rx_req = &priv->rx_ring[id];
149         for (m = rx_req->mb, i = 0; m != NULL; m = m->m_next, i++) {
150                 priv->rx_sge[i].addr = rx_req->mapping[i];
151                 priv->rx_sge[i].length = m->m_len;
152         }
153         priv->rx_wr.num_sge = i;
154         priv->rx_wr.wr_id = id | IPOIB_OP_RECV;
155
156         ret = ib_post_recv(priv->qp, &priv->rx_wr, &bad_wr);
157         if (unlikely(ret)) {
158                 ipoib_warn(priv, "receive failed for buf %d (%d)\n", id, ret);
159                 ipoib_dma_unmap_rx(priv, &priv->rx_ring[id]);
160                 m_freem(priv->rx_ring[id].mb);
161                 priv->rx_ring[id].mb = NULL;
162         }
163
164         return ret;
165 }
166
167 static struct mbuf *
168 ipoib_alloc_rx_mb(struct ipoib_dev_priv *priv, int id)
169 {
170
171         return ipoib_alloc_map_mb(priv, &priv->rx_ring[id],
172             priv->max_ib_mtu + IB_GRH_BYTES);
173 }
174
175 static int ipoib_ib_post_receives(struct ipoib_dev_priv *priv)
176 {
177         int i;
178
179         for (i = 0; i < ipoib_recvq_size; ++i) {
180                 if (!ipoib_alloc_rx_mb(priv, i)) {
181                         ipoib_warn(priv, "failed to allocate receive buffer %d\n", i);
182                         return -ENOMEM;
183                 }
184                 if (ipoib_ib_post_receive(priv, i)) {
185                         ipoib_warn(priv, "ipoib_ib_post_receive failed for buf %d\n", i);
186                         return -EIO;
187                 }
188         }
189
190         return 0;
191 }
192
193 static void
194 ipoib_ib_handle_rx_wc(struct ipoib_dev_priv *priv, struct ib_wc *wc)
195 {
196         struct ipoib_rx_buf saverx;
197         unsigned int wr_id = wc->wr_id & ~IPOIB_OP_RECV;
198         struct ifnet *dev = priv->dev;
199         struct ipoib_header *eh;
200         struct mbuf *mb;
201
202         ipoib_dbg_data(priv, "recv completion: id %d, status: %d\n",
203                        wr_id, wc->status);
204
205         if (unlikely(wr_id >= ipoib_recvq_size)) {
206                 ipoib_warn(priv, "recv completion event with wrid %d (> %d)\n",
207                            wr_id, ipoib_recvq_size);
208                 return;
209         }
210
211         mb  = priv->rx_ring[wr_id].mb;
212
213         if (unlikely(wc->status != IB_WC_SUCCESS)) {
214                 if (wc->status != IB_WC_WR_FLUSH_ERR) {
215                         ipoib_warn(priv, "failed recv event "
216                                    "(status=%d, wrid=%d vend_err %x)\n",
217                                    wc->status, wr_id, wc->vendor_err);
218                         goto repost;
219                 }
220                 if (mb) {
221                         ipoib_dma_unmap_rx(priv, &priv->rx_ring[wr_id]);
222                         m_freem(mb);
223                         priv->rx_ring[wr_id].mb = NULL;
224                 }
225                 return;
226         }
227
228         /*
229          * Drop packets that this interface sent, ie multicast packets
230          * that the HCA has replicated.
231          */
232         if (wc->slid == priv->local_lid && wc->src_qp == priv->qp->qp_num)
233                 goto repost;
234
235         memcpy(&saverx, &priv->rx_ring[wr_id], sizeof(saverx));
236         /*
237          * If we can't allocate a new RX buffer, dump
238          * this packet and reuse the old buffer.
239          */
240         if (unlikely(!ipoib_alloc_rx_mb(priv, wr_id))) {
241                 memcpy(&priv->rx_ring[wr_id], &saverx, sizeof(saverx));
242                 if_inc_counter(dev, IFCOUNTER_IQDROPS, 1);
243                 goto repost;
244         }
245
246         ipoib_dbg_data(priv, "received %d bytes, SLID 0x%04x\n",
247                        wc->byte_len, wc->slid);
248
249         ipoib_dma_unmap_rx(priv, &saverx);
250         ipoib_dma_mb(priv, mb, wc->byte_len);
251
252         if_inc_counter(dev, IFCOUNTER_IPACKETS, 1);
253         if_inc_counter(dev, IFCOUNTER_IBYTES, mb->m_pkthdr.len);
254         mb->m_pkthdr.rcvif = dev;
255         m_adj(mb, sizeof(struct ib_grh) - INFINIBAND_ALEN);
256         eh = mtod(mb, struct ipoib_header *);
257         bzero(eh->hwaddr, 4);   /* Zero the queue pair, only dgid is in grh */
258
259         if (test_bit(IPOIB_FLAG_CSUM, &priv->flags) && likely(wc->csum_ok))
260                 mb->m_pkthdr.csum_flags = CSUM_IP_CHECKED | CSUM_IP_VALID;
261
262         dev->if_input(dev, mb);
263
264 repost:
265         if (unlikely(ipoib_ib_post_receive(priv, wr_id)))
266                 ipoib_warn(priv, "ipoib_ib_post_receive failed "
267                            "for buf %d\n", wr_id);
268 }
269
270 int ipoib_dma_map_tx(struct ib_device *ca, struct ipoib_tx_buf *tx_req, int max)
271 {
272         struct mbuf *mb = tx_req->mb;
273         u64 *mapping = tx_req->mapping;
274         struct mbuf *m, *p;
275         int error;
276         int i;
277
278         for (m = mb, p = NULL, i = 0; m != NULL; p = m, m = m->m_next, i++) {
279                 if (m->m_len != 0)
280                         continue;
281                 if (p == NULL)
282                         panic("ipoib_dma_map_tx: First mbuf empty\n");
283                 p->m_next = m_free(m);
284                 m = p;
285                 i--;
286         }
287         i--;
288         if (i >= max) {
289                 tx_req->mb = mb = m_defrag(mb, M_NOWAIT);
290                 if (mb == NULL)
291                         return -EIO;
292                 for (m = mb, i = 0; m != NULL; m = m->m_next, i++);
293                 if (i >= max)
294                         return -EIO;
295         }
296         error = 0;
297         for (m = mb, i = 0; m != NULL; m = m->m_next, i++) {
298                 mapping[i] = ib_dma_map_single(ca, mtod(m, void *),
299                                                m->m_len, DMA_TO_DEVICE);
300                 if (unlikely(ib_dma_mapping_error(ca, mapping[i]))) {
301                         error = -EIO;
302                         break;
303                 }
304         }
305         if (error) {
306                 int end;
307
308                 end = i;
309                 for (m = mb, i = 0; i < end; m = m->m_next, i++)
310                         ib_dma_unmap_single(ca, mapping[i], m->m_len,
311                                             DMA_TO_DEVICE);
312         }
313         return error;
314 }
315
316 void ipoib_dma_unmap_tx(struct ib_device *ca, struct ipoib_tx_buf *tx_req)
317 {
318         struct mbuf *mb = tx_req->mb;
319         u64 *mapping = tx_req->mapping;
320         struct mbuf *m;
321         int i;
322
323         for (m = mb, i = 0; m != NULL; m = m->m_next, i++)
324                 ib_dma_unmap_single(ca, mapping[i], m->m_len, DMA_TO_DEVICE);
325 }
326
327 static void ipoib_ib_handle_tx_wc(struct ipoib_dev_priv *priv, struct ib_wc *wc)
328 {
329         struct ifnet *dev = priv->dev;
330         unsigned int wr_id = wc->wr_id;
331         struct ipoib_tx_buf *tx_req;
332
333         ipoib_dbg_data(priv, "send completion: id %d, status: %d\n",
334                        wr_id, wc->status);
335
336         if (unlikely(wr_id >= ipoib_sendq_size)) {
337                 ipoib_warn(priv, "send completion event with wrid %d (> %d)\n",
338                            wr_id, ipoib_sendq_size);
339                 return;
340         }
341
342         tx_req = &priv->tx_ring[wr_id];
343
344         ipoib_dma_unmap_tx(priv->ca, tx_req);
345
346         if_inc_counter(dev, IFCOUNTER_OPACKETS, 1);
347
348         m_freem(tx_req->mb);
349
350         ++priv->tx_tail;
351         if (unlikely(--priv->tx_outstanding == ipoib_sendq_size >> 1) &&
352             (dev->if_drv_flags & IFF_DRV_OACTIVE) &&
353             test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags))
354                 dev->if_drv_flags &= ~IFF_DRV_OACTIVE;
355
356         if (wc->status != IB_WC_SUCCESS &&
357             wc->status != IB_WC_WR_FLUSH_ERR)
358                 ipoib_warn(priv, "failed send event "
359                            "(status=%d, wrid=%d vend_err %x)\n",
360                            wc->status, wr_id, wc->vendor_err);
361 }
362
363 int
364 ipoib_poll_tx(struct ipoib_dev_priv *priv)
365 {
366         int n, i;
367
368         n = ib_poll_cq(priv->send_cq, MAX_SEND_CQE, priv->send_wc);
369         for (i = 0; i < n; ++i) {
370                 struct ib_wc *wc = priv->send_wc + i;
371                 if (wc->wr_id & IPOIB_OP_CM)
372                         ipoib_cm_handle_tx_wc(priv, wc);
373                 else
374                         ipoib_ib_handle_tx_wc(priv, wc);
375         }
376
377         return n == MAX_SEND_CQE;
378 }
379
380 static void
381 ipoib_poll(struct ipoib_dev_priv *priv)
382 {
383         int n, i;
384
385 poll_more:
386         spin_lock(&priv->drain_lock);
387         for (;;) {
388                 n = ib_poll_cq(priv->recv_cq, IPOIB_NUM_WC, priv->ibwc);
389                 for (i = 0; i < n; i++) {
390                         struct ib_wc *wc = priv->ibwc + i;
391
392                         if ((wc->wr_id & IPOIB_OP_RECV) == 0)
393                                 panic("ipoib_poll: Bad wr_id 0x%jX\n",
394                                     (intmax_t)wc->wr_id);
395                         if (wc->wr_id & IPOIB_OP_CM)
396                                 ipoib_cm_handle_rx_wc(priv, wc);
397                         else
398                                 ipoib_ib_handle_rx_wc(priv, wc);
399                 }
400
401                 if (n != IPOIB_NUM_WC)
402                         break;
403         }
404         spin_unlock(&priv->drain_lock);
405
406         if (ib_req_notify_cq(priv->recv_cq,
407             IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS))
408                 goto poll_more;
409 }
410
411 void ipoib_ib_completion(struct ib_cq *cq, void *dev_ptr)
412 {
413         struct ipoib_dev_priv *priv = dev_ptr;
414
415         ipoib_poll(priv);
416 }
417
418 static void drain_tx_cq(struct ipoib_dev_priv *priv)
419 {
420         struct ifnet *dev = priv->dev;
421
422         spin_lock(&priv->lock);
423         while (ipoib_poll_tx(priv))
424                 ; /* nothing */
425
426         if (dev->if_drv_flags & IFF_DRV_OACTIVE)
427                 mod_timer(&priv->poll_timer, jiffies + 1);
428
429         spin_unlock(&priv->lock);
430 }
431
432 void ipoib_send_comp_handler(struct ib_cq *cq, void *dev_ptr)
433 {
434         struct ipoib_dev_priv *priv = dev_ptr;
435
436         mod_timer(&priv->poll_timer, jiffies);
437 }
438
439 static inline int
440 post_send(struct ipoib_dev_priv *priv, unsigned int wr_id,
441     struct ib_ah *address, u32 qpn, struct ipoib_tx_buf *tx_req, void *head,
442     int hlen)
443 {
444         struct ib_send_wr *bad_wr;
445         struct mbuf *mb = tx_req->mb;
446         u64 *mapping = tx_req->mapping;
447         struct mbuf *m;
448         int i;
449
450         for (m = mb, i = 0; m != NULL; m = m->m_next, i++) {
451                 priv->tx_sge[i].addr         = mapping[i];
452                 priv->tx_sge[i].length       = m->m_len;
453         }
454         priv->tx_wr.num_sge          = i;
455         priv->tx_wr.wr_id            = wr_id;
456         priv->tx_wr.wr.ud.remote_qpn = qpn;
457         priv->tx_wr.wr.ud.ah         = address;
458
459
460         if (head) {
461                 priv->tx_wr.wr.ud.mss    = 0; /* XXX mb_shinfo(mb)->gso_size; */
462                 priv->tx_wr.wr.ud.header = head;
463                 priv->tx_wr.wr.ud.hlen   = hlen;
464                 priv->tx_wr.opcode       = IB_WR_LSO;
465         } else
466                 priv->tx_wr.opcode       = IB_WR_SEND;
467
468         return ib_post_send(priv->qp, &priv->tx_wr, &bad_wr);
469 }
470
471 void
472 ipoib_send(struct ipoib_dev_priv *priv, struct mbuf *mb,
473     struct ipoib_ah *address, u32 qpn)
474 {
475         struct ifnet *dev = priv->dev;
476         struct ipoib_tx_buf *tx_req;
477         int hlen;
478         void *phead;
479
480         if (unlikely(priv->tx_outstanding > MAX_SEND_CQE))
481                 while (ipoib_poll_tx(priv))
482                         ; /* nothing */
483
484         m_adj(mb, sizeof (struct ipoib_pseudoheader));
485         if (0 /* XXX segment offload mb_is_gso(mb) */) {
486                 /* XXX hlen = mb_transport_offset(mb) + tcp_hdrlen(mb); */
487                 phead = mtod(mb, void *);
488                 if (mb->m_len < hlen) {
489                         ipoib_warn(priv, "linear data too small\n");
490                         if_inc_counter(dev, IFCOUNTER_OERRORS, 1);
491                         m_freem(mb);
492                         return;
493                 }
494                 m_adj(mb, hlen);
495         } else {
496                 if (unlikely(mb->m_pkthdr.len - IPOIB_ENCAP_LEN > priv->mcast_mtu)) {
497                         ipoib_warn(priv, "packet len %d (> %d) too long to send, dropping\n",
498                                    mb->m_pkthdr.len, priv->mcast_mtu);
499                         if_inc_counter(dev, IFCOUNTER_OERRORS, 1);
500                         ipoib_cm_mb_too_long(priv, mb, priv->mcast_mtu);
501                         return;
502                 }
503                 phead = NULL;
504                 hlen  = 0;
505         }
506
507         ipoib_dbg_data(priv, "sending packet, length=%d address=%p qpn=0x%06x\n",
508                        mb->m_pkthdr.len, address, qpn);
509
510         /*
511          * We put the mb into the tx_ring _before_ we call post_send()
512          * because it's entirely possible that the completion handler will
513          * run before we execute anything after the post_send().  That
514          * means we have to make sure everything is properly recorded and
515          * our state is consistent before we call post_send().
516          */
517         tx_req = &priv->tx_ring[priv->tx_head & (ipoib_sendq_size - 1)];
518         tx_req->mb = mb;
519         if (unlikely(ipoib_dma_map_tx(priv->ca, tx_req, IPOIB_UD_TX_SG))) {
520                 if_inc_counter(dev, IFCOUNTER_OERRORS, 1);
521                 if (tx_req->mb)
522                         m_freem(tx_req->mb);
523                 return;
524         }
525
526         if (mb->m_pkthdr.csum_flags & (CSUM_IP|CSUM_TCP|CSUM_UDP))
527                 priv->tx_wr.send_flags |= IB_SEND_IP_CSUM;
528         else
529                 priv->tx_wr.send_flags &= ~IB_SEND_IP_CSUM;
530
531         if (++priv->tx_outstanding == ipoib_sendq_size) {
532                 ipoib_dbg(priv, "TX ring full, stopping kernel net queue\n");
533                 if (ib_req_notify_cq(priv->send_cq, IB_CQ_NEXT_COMP))
534                         ipoib_warn(priv, "request notify on send CQ failed\n");
535                 dev->if_drv_flags |= IFF_DRV_OACTIVE;
536         }
537
538         if (unlikely(post_send(priv,
539             priv->tx_head & (ipoib_sendq_size - 1), address->ah, qpn,
540             tx_req, phead, hlen))) {
541                 ipoib_warn(priv, "post_send failed\n");
542                 if_inc_counter(dev, IFCOUNTER_OERRORS, 1);
543                 --priv->tx_outstanding;
544                 ipoib_dma_unmap_tx(priv->ca, tx_req);
545                 m_freem(mb);
546                 if (dev->if_drv_flags & IFF_DRV_OACTIVE)
547                         dev->if_drv_flags &= ~IFF_DRV_OACTIVE;
548         } else {
549                 address->last_send = priv->tx_head;
550                 ++priv->tx_head;
551         }
552 }
553
554 static void __ipoib_reap_ah(struct ipoib_dev_priv *priv)
555 {
556         struct ipoib_ah *ah, *tah;
557         LIST_HEAD(remove_list);
558         unsigned long flags;
559
560         spin_lock_irqsave(&priv->lock, flags);
561
562         list_for_each_entry_safe(ah, tah, &priv->dead_ahs, list)
563                 if ((int) priv->tx_tail - (int) ah->last_send >= 0) {
564                         list_del(&ah->list);
565                         ib_destroy_ah(ah->ah);
566                         kfree(ah);
567                 }
568
569         spin_unlock_irqrestore(&priv->lock, flags);
570 }
571
572 void ipoib_reap_ah(struct work_struct *work)
573 {
574         struct ipoib_dev_priv *priv =
575                 container_of(work, struct ipoib_dev_priv, ah_reap_task.work);
576
577         __ipoib_reap_ah(priv);
578
579         if (!test_bit(IPOIB_STOP_REAPER, &priv->flags))
580                 queue_delayed_work(ipoib_workqueue, &priv->ah_reap_task,
581                                    HZ);
582 }
583
584 static void ipoib_ah_dev_cleanup(struct ipoib_dev_priv *priv)
585 {
586         unsigned long begin;
587
588         begin = jiffies;
589
590         while (!list_empty(&priv->dead_ahs)) {
591                 __ipoib_reap_ah(priv);
592
593                 if (time_after(jiffies, begin + HZ)) {
594                         ipoib_warn(priv, "timing out; will leak address handles\n");
595                         break;
596                 }
597
598                 msleep(1);
599         }
600 }
601
602 static void ipoib_ib_tx_timer_func(unsigned long ctx)
603 {
604         drain_tx_cq((struct ipoib_dev_priv *)ctx);
605 }
606
607 int ipoib_ib_dev_open(struct ipoib_dev_priv *priv)
608 {
609         int ret;
610
611         if (ib_find_pkey(priv->ca, priv->port, priv->pkey, &priv->pkey_index)) {
612                 ipoib_warn(priv, "P_Key 0x%04x not found\n", priv->pkey);
613                 clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
614                 return -1;
615         }
616         set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
617
618         ret = ipoib_init_qp(priv);
619         if (ret) {
620                 ipoib_warn(priv, "ipoib_init_qp returned %d\n", ret);
621                 return -1;
622         }
623
624         ret = ipoib_ib_post_receives(priv);
625         if (ret) {
626                 ipoib_warn(priv, "ipoib_ib_post_receives returned %d\n", ret);
627                 ipoib_ib_dev_stop(priv, 1);
628                 return -1;
629         }
630
631         ret = ipoib_cm_dev_open(priv);
632         if (ret) {
633                 ipoib_warn(priv, "ipoib_cm_dev_open returned %d\n", ret);
634                 ipoib_ib_dev_stop(priv, 1);
635                 return -1;
636         }
637
638         clear_bit(IPOIB_STOP_REAPER, &priv->flags);
639         queue_delayed_work(ipoib_workqueue, &priv->ah_reap_task, HZ);
640
641         return 0;
642 }
643
644 static void ipoib_pkey_dev_check_presence(struct ipoib_dev_priv *priv)
645 {
646         u16 pkey_index = 0;
647
648         if (ib_find_pkey(priv->ca, priv->port, priv->pkey, &pkey_index))
649                 clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
650         else
651                 set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
652 }
653
654 int ipoib_ib_dev_up(struct ipoib_dev_priv *priv)
655 {
656
657         ipoib_pkey_dev_check_presence(priv);
658
659         if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags)) {
660                 ipoib_dbg(priv, "PKEY is not assigned.\n");
661                 return 0;
662         }
663
664         set_bit(IPOIB_FLAG_OPER_UP, &priv->flags);
665
666         return ipoib_mcast_start_thread(priv);
667 }
668
669 int ipoib_ib_dev_down(struct ipoib_dev_priv *priv, int flush)
670 {
671
672         ipoib_dbg(priv, "downing ib_dev\n");
673
674         clear_bit(IPOIB_FLAG_OPER_UP, &priv->flags);
675         if_link_state_change(priv->dev, LINK_STATE_DOWN);
676
677         /* Shutdown the P_Key thread if still active */
678         if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags)) {
679                 mutex_lock(&pkey_mutex);
680                 set_bit(IPOIB_PKEY_STOP, &priv->flags);
681                 cancel_delayed_work(&priv->pkey_poll_task);
682                 mutex_unlock(&pkey_mutex);
683                 if (flush)
684                         flush_workqueue(ipoib_workqueue);
685         }
686
687         ipoib_mcast_stop_thread(priv, flush);
688         ipoib_mcast_dev_flush(priv);
689
690         ipoib_flush_paths(priv);
691
692         return 0;
693 }
694
695 static int recvs_pending(struct ipoib_dev_priv *priv)
696 {
697         int pending = 0;
698         int i;
699
700         for (i = 0; i < ipoib_recvq_size; ++i)
701                 if (priv->rx_ring[i].mb)
702                         ++pending;
703
704         return pending;
705 }
706
707 void ipoib_drain_cq(struct ipoib_dev_priv *priv)
708 {
709         int i, n;
710
711         spin_lock(&priv->drain_lock);
712         do {
713                 n = ib_poll_cq(priv->recv_cq, IPOIB_NUM_WC, priv->ibwc);
714                 for (i = 0; i < n; ++i) {
715                         /*
716                          * Convert any successful completions to flush
717                          * errors to avoid passing packets up the
718                          * stack after bringing the device down.
719                          */
720                         if (priv->ibwc[i].status == IB_WC_SUCCESS)
721                                 priv->ibwc[i].status = IB_WC_WR_FLUSH_ERR;
722
723                         if ((priv->ibwc[i].wr_id & IPOIB_OP_RECV) == 0)
724                                 panic("ipoib_drain_cq:  Bad wrid 0x%jX\n",
725                                     (intmax_t)priv->ibwc[i].wr_id);
726                         if (priv->ibwc[i].wr_id & IPOIB_OP_CM)
727                                 ipoib_cm_handle_rx_wc(priv, priv->ibwc + i);
728                         else
729                                 ipoib_ib_handle_rx_wc(priv, priv->ibwc + i);
730                 }
731         } while (n == IPOIB_NUM_WC);
732         spin_unlock(&priv->drain_lock);
733
734         spin_lock(&priv->lock);
735         while (ipoib_poll_tx(priv))
736                 ; /* nothing */
737
738         spin_unlock(&priv->lock);
739 }
740
741 int ipoib_ib_dev_stop(struct ipoib_dev_priv *priv, int flush)
742 {
743         struct ib_qp_attr qp_attr;
744         unsigned long begin;
745         struct ipoib_tx_buf *tx_req;
746         int i;
747
748         ipoib_cm_dev_stop(priv);
749
750         /*
751          * Move our QP to the error state and then reinitialize in
752          * when all work requests have completed or have been flushed.
753          */
754         qp_attr.qp_state = IB_QPS_ERR;
755         if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE))
756                 ipoib_warn(priv, "Failed to modify QP to ERROR state\n");
757
758         /* Wait for all sends and receives to complete */
759         begin = jiffies;
760
761         while (priv->tx_head != priv->tx_tail || recvs_pending(priv)) {
762                 if (time_after(jiffies, begin + 5 * HZ)) {
763                         ipoib_warn(priv, "timing out; %d sends %d receives not completed\n",
764                                    priv->tx_head - priv->tx_tail, recvs_pending(priv));
765
766                         /*
767                          * assume the HW is wedged and just free up
768                          * all our pending work requests.
769                          */
770                         while ((int) priv->tx_tail - (int) priv->tx_head < 0) {
771                                 tx_req = &priv->tx_ring[priv->tx_tail &
772                                                         (ipoib_sendq_size - 1)];
773                                 ipoib_dma_unmap_tx(priv->ca, tx_req);
774                                 m_freem(tx_req->mb);
775                                 ++priv->tx_tail;
776                                 --priv->tx_outstanding;
777                         }
778
779                         for (i = 0; i < ipoib_recvq_size; ++i) {
780                                 struct ipoib_rx_buf *rx_req;
781
782                                 rx_req = &priv->rx_ring[i];
783                                 if (!rx_req->mb)
784                                         continue;
785                                 ipoib_dma_unmap_rx(priv, &priv->rx_ring[i]);
786                                 m_freem(rx_req->mb);
787                                 rx_req->mb = NULL;
788                         }
789
790                         goto timeout;
791                 }
792
793                 ipoib_drain_cq(priv);
794
795                 msleep(1);
796         }
797
798         ipoib_dbg(priv, "All sends and receives done.\n");
799
800 timeout:
801         del_timer_sync(&priv->poll_timer);
802         qp_attr.qp_state = IB_QPS_RESET;
803         if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE))
804                 ipoib_warn(priv, "Failed to modify QP to RESET state\n");
805
806         /* Wait for all AHs to be reaped */
807         set_bit(IPOIB_STOP_REAPER, &priv->flags);
808         cancel_delayed_work(&priv->ah_reap_task);
809         if (flush)
810                 flush_workqueue(ipoib_workqueue);
811
812         ipoib_ah_dev_cleanup(priv);
813
814         ib_req_notify_cq(priv->recv_cq, IB_CQ_NEXT_COMP);
815
816         return 0;
817 }
818
819 int ipoib_ib_dev_init(struct ipoib_dev_priv *priv, struct ib_device *ca, int port)
820 {
821         struct ifnet *dev = priv->dev;
822
823         priv->ca = ca;
824         priv->port = port;
825         priv->qp = NULL;
826
827         if (ipoib_transport_dev_init(priv, ca)) {
828                 printk(KERN_WARNING "%s: ipoib_transport_dev_init failed\n", ca->name);
829                 return -ENODEV;
830         }
831
832         setup_timer(&priv->poll_timer, ipoib_ib_tx_timer_func,
833                     (unsigned long) priv);
834
835         if (dev->if_flags & IFF_UP) {
836                 if (ipoib_ib_dev_open(priv)) {
837                         ipoib_transport_dev_cleanup(priv);
838                         return -ENODEV;
839                 }
840         }
841
842         return 0;
843 }
844
845 static void __ipoib_ib_dev_flush(struct ipoib_dev_priv *priv,
846                                 enum ipoib_flush_level level)
847 {
848         struct ipoib_dev_priv *cpriv;
849         u16 new_index;
850
851         mutex_lock(&priv->vlan_mutex);
852
853         /*
854          * Flush any child interfaces too -- they might be up even if
855          * the parent is down.
856          */
857         list_for_each_entry(cpriv, &priv->child_intfs, list)
858                 __ipoib_ib_dev_flush(cpriv, level);
859
860         mutex_unlock(&priv->vlan_mutex);
861
862         if (!test_bit(IPOIB_FLAG_INITIALIZED, &priv->flags)) {
863                 ipoib_dbg(priv, "Not flushing - IPOIB_FLAG_INITIALIZED not set.\n");
864                 return;
865         }
866
867         if (!test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags)) {
868                 ipoib_dbg(priv, "Not flushing - IPOIB_FLAG_ADMIN_UP not set.\n");
869                 return;
870         }
871
872         if (level == IPOIB_FLUSH_HEAVY) {
873                 if (ib_find_pkey(priv->ca, priv->port, priv->pkey, &new_index)) {
874                         clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
875                         ipoib_ib_dev_down(priv, 0);
876                         ipoib_ib_dev_stop(priv, 0);
877                         if (ipoib_pkey_dev_delay_open(priv))
878                                 return;
879                 }
880
881                 /* restart QP only if P_Key index is changed */
882                 if (test_and_set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags) &&
883                     new_index == priv->pkey_index) {
884                         ipoib_dbg(priv, "Not flushing - P_Key index not changed.\n");
885                         return;
886                 }
887                 priv->pkey_index = new_index;
888         }
889
890         if (level == IPOIB_FLUSH_LIGHT) {
891                 ipoib_mark_paths_invalid(priv);
892                 ipoib_mcast_dev_flush(priv);
893         }
894
895         if (level >= IPOIB_FLUSH_NORMAL)
896                 ipoib_ib_dev_down(priv, 0);
897
898         if (level == IPOIB_FLUSH_HEAVY) {
899                 ipoib_ib_dev_stop(priv, 0);
900                 ipoib_ib_dev_open(priv);
901         }
902
903         /*
904          * The device could have been brought down between the start and when
905          * we get here, don't bring it back up if it's not configured up
906          */
907         if (test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags)) {
908                 if (level >= IPOIB_FLUSH_NORMAL)
909                         ipoib_ib_dev_up(priv);
910                 ipoib_mcast_restart_task(&priv->restart_task);
911         }
912 }
913
914 void ipoib_ib_dev_flush_light(struct work_struct *work)
915 {
916         struct ipoib_dev_priv *priv =
917                 container_of(work, struct ipoib_dev_priv, flush_light);
918
919         __ipoib_ib_dev_flush(priv, IPOIB_FLUSH_LIGHT);
920 }
921
922 void ipoib_ib_dev_flush_normal(struct work_struct *work)
923 {
924         struct ipoib_dev_priv *priv =
925                 container_of(work, struct ipoib_dev_priv, flush_normal);
926
927         __ipoib_ib_dev_flush(priv, IPOIB_FLUSH_NORMAL);
928 }
929
930 void ipoib_ib_dev_flush_heavy(struct work_struct *work)
931 {
932         struct ipoib_dev_priv *priv =
933                 container_of(work, struct ipoib_dev_priv, flush_heavy);
934
935         __ipoib_ib_dev_flush(priv, IPOIB_FLUSH_HEAVY);
936 }
937
938 void ipoib_ib_dev_cleanup(struct ipoib_dev_priv *priv)
939 {
940
941         ipoib_dbg(priv, "cleaning up ib_dev\n");
942
943         ipoib_mcast_stop_thread(priv, 1);
944         ipoib_mcast_dev_flush(priv);
945
946         ipoib_ah_dev_cleanup(priv);
947         ipoib_transport_dev_cleanup(priv);
948 }
949
950 /*
951  * Delayed P_Key Assigment Interim Support
952  *
953  * The following is initial implementation of delayed P_Key assigment
954  * mechanism. It is using the same approach implemented for the multicast
955  * group join. The single goal of this implementation is to quickly address
956  * Bug #2507. This implementation will probably be removed when the P_Key
957  * change async notification is available.
958  */
959
960 void ipoib_pkey_poll(struct work_struct *work)
961 {
962         struct ipoib_dev_priv *priv =
963                 container_of(work, struct ipoib_dev_priv, pkey_poll_task.work);
964
965         ipoib_pkey_dev_check_presence(priv);
966
967         if (test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags))
968                 ipoib_open(priv);
969         else {
970                 mutex_lock(&pkey_mutex);
971                 if (!test_bit(IPOIB_PKEY_STOP, &priv->flags))
972                         queue_delayed_work(ipoib_workqueue,
973                                            &priv->pkey_poll_task,
974                                            HZ);
975                 mutex_unlock(&pkey_mutex);
976         }
977 }
978
979 int ipoib_pkey_dev_delay_open(struct ipoib_dev_priv *priv)
980 {
981
982         /* Look for the interface pkey value in the IB Port P_Key table and */
983         /* set the interface pkey assigment flag                            */
984         ipoib_pkey_dev_check_presence(priv);
985
986         /* P_Key value not assigned yet - start polling */
987         if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags)) {
988                 mutex_lock(&pkey_mutex);
989                 clear_bit(IPOIB_PKEY_STOP, &priv->flags);
990                 queue_delayed_work(ipoib_workqueue,
991                                    &priv->pkey_poll_task,
992                                    HZ);
993                 mutex_unlock(&pkey_mutex);
994                 return 1;
995         }
996
997         return 0;
998 }