sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cq.c

   1
   2 /**************************************************************************
   3
   4 Copyright (c) 2007, Chelsio Inc.
   5 All rights reserved.
   6
   7 Redistribution and use in source and binary forms, with or without
   8 modification, are permitted provided that the following conditions are met:
   9
  10  1. Redistributions of source code must retain the above copyright notice,
  11     this list of conditions and the following disclaimer.
  12
  13  2. Neither the name of the Chelsio Corporation nor the names of its
  14     contributors may be used to endorse or promote products derived from
  15     this software without specific prior written permission.
  16
  17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  18 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  21 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  22 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  23 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  24 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  25 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  26 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27 POSSIBILITY OF SUCH DAMAGE.
  28
  29 ***************************************************************************/
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34 #include <sys/systm.h>
  35 #include <sys/kernel.h>
  36 #include <sys/bus.h>
  37 #include <sys/module.h>
  38 #include <sys/pciio.h>
  39 #include <sys/conf.h>
  40 #include <machine/bus.h>
  41 #include <machine/resource.h>
  42 #include <sys/bus_dma.h>
  43 #include <sys/rman.h>
  44 #include <sys/ioccom.h>
  45 #include <sys/mbuf.h>
  46 #include <sys/mutex.h>
  47 #include <sys/rwlock.h>
  48 #include <sys/linker.h>
  49 #include <sys/firmware.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/smp.h>
  53 #include <sys/sysctl.h>
  54 #include <sys/syslog.h>
  55 #include <sys/queue.h>
  56 #include <sys/taskqueue.h>
  57 #include <sys/proc.h>
  58 #include <sys/queue.h>
  59 #include <sys/libkern.h>
  60
  61 #include <netinet/in.h>
  62 #include <contrib/rdma/ib_verbs.h>
  63 #include <contrib/rdma/ib_umem.h>
  64 #include <contrib/rdma/ib_user_verbs.h>
  65
  66 #include <cxgb_include.h>
  67 #include <ulp/iw_cxgb/iw_cxgb_wr.h>
  68 #include <ulp/iw_cxgb/iw_cxgb_hal.h>
  69 #include <ulp/iw_cxgb/iw_cxgb_provider.h>
  70 #include <ulp/iw_cxgb/iw_cxgb_cm.h>
  71 #include <ulp/iw_cxgb/iw_cxgb.h>
  72 #include <ulp/iw_cxgb/iw_cxgb_resource.h>
  73 #include <ulp/iw_cxgb/iw_cxgb_user.h>
  74
  75 /*
  76  * Get one cq entry from cxio and map it to openib.
  77  *
  78  * Returns:
  79  *      0                       cqe returned
  80  *      -ENOBUFS                EMPTY;
  81  *      -EAGAIN                 caller must try again
  82  *      any other neg errno     fatal error
  83  */
  84 static int iwch_poll_cq_one(struct iwch_dev *rhp, struct iwch_cq *chp,
  85                             struct ib_wc *wc)
  86 {
  87         struct iwch_qp *qhp = NULL;
  88         struct t3_cqe cqe, *rd_cqe;
  89         struct t3_wq *wq;
  90         u32 credit = 0;
  91         u8 cqe_flushed;
  92         u64 cookie;
  93         int ret = 1;
  94
  95         rd_cqe = cxio_next_cqe(&chp->cq);
  96
  97         if (!rd_cqe)
  98                 return 0;
  99
 100         qhp = get_qhp(rhp, CQE_QPID(*rd_cqe));
 101         if (!qhp)
 102                 wq = NULL;
 103         else {
 104                 mtx_lock(&qhp->lock);
 105                 wq = &(qhp->wq);
 106         }
 107         ret = cxio_poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie,
 108                                    &credit);
 109         if (t3a_device(chp->rhp) && credit) {
 110                 CTR3(KTR_IW_CXGB, "%s updating %d cq credits on id %d", __FUNCTION__,
 111                      credit, chp->cq.cqid);
 112                 cxio_hal_cq_op(&rhp->rdev, &chp->cq, CQ_CREDIT_UPDATE, credit);
 113         }
 114
 115         if (ret) {
 116                 ret = -EAGAIN;
 117                 goto out;
 118         }
 119         ret = 1;
 120
 121         wc->wr_id = cookie;
 122         wc->qp = &qhp->ibqp;
 123         wc->vendor_err = CQE_STATUS(cqe);
 124
 125         CTR4(KTR_IW_CXGB, "iwch_poll_cq_one qpid 0x%x type %d opcode %d status 0x%x",
 126              CQE_QPID(cqe), CQE_TYPE(cqe),
 127              CQE_OPCODE(cqe), CQE_STATUS(cqe));
 128         CTR3(KTR_IW_CXGB, "wrid hi 0x%x lo 0x%x cookie 0x%llx",
 129              CQE_WRID_HI(cqe), CQE_WRID_LOW(cqe), (unsigned long long) cookie);
 130
 131         if (CQE_TYPE(cqe) == 0) {
 132                 if (!CQE_STATUS(cqe))
 133                         wc->byte_len = CQE_LEN(cqe);
 134                 else
 135                         wc->byte_len = 0;
 136                 wc->opcode = IB_WC_RECV;
 137         } else {
 138                 switch (CQE_OPCODE(cqe)) {
 139                 case T3_RDMA_WRITE:
 140                         wc->opcode = IB_WC_RDMA_WRITE;
 141                         break;
 142                 case T3_READ_REQ:
 143                         wc->opcode = IB_WC_RDMA_READ;
 144                         wc->byte_len = CQE_LEN(cqe);
 145                         break;
 146                 case T3_SEND:
 147                 case T3_SEND_WITH_SE:
 148                         wc->opcode = IB_WC_SEND;
 149                         break;
 150                 case T3_BIND_MW:
 151                         wc->opcode = IB_WC_BIND_MW;
 152                         break;
 153
 154                 /* these aren't supported yet */
 155                 case T3_SEND_WITH_INV:
 156                 case T3_SEND_WITH_SE_INV:
 157                 case T3_LOCAL_INV:
 158                 case T3_FAST_REGISTER:
 159                 default:
 160                         log(LOG_ERR, "Unexpected opcode %d "
 161                                "in the CQE received for QPID=0x%0x\n",
 162                                CQE_OPCODE(cqe), CQE_QPID(cqe));
 163                         ret = -EINVAL;
 164                         goto out;
 165                 }
 166         }
 167
 168         if (cqe_flushed)
 169                 wc->status = IB_WC_WR_FLUSH_ERR;
 170         else {
 171
 172                 switch (CQE_STATUS(cqe)) {
 173                 case TPT_ERR_SUCCESS:
 174                         wc->status = IB_WC_SUCCESS;
 175                         break;
 176                 case TPT_ERR_STAG:
 177                         wc->status = IB_WC_LOC_ACCESS_ERR;
 178                         break;
 179                 case TPT_ERR_PDID:
 180                         wc->status = IB_WC_LOC_PROT_ERR;
 181                         break;
 182                 case TPT_ERR_QPID:
 183                 case TPT_ERR_ACCESS:
 184                         wc->status = IB_WC_LOC_ACCESS_ERR;
 185                         break;
 186                 case TPT_ERR_WRAP:
 187                         wc->status = IB_WC_GENERAL_ERR;
 188                         break;
 189                 case TPT_ERR_BOUND:
 190                         wc->status = IB_WC_LOC_LEN_ERR;
 191                         break;
 192                 case TPT_ERR_INVALIDATE_SHARED_MR:
 193                 case TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND:
 194                         wc->status = IB_WC_MW_BIND_ERR;
 195                         break;
 196                 case TPT_ERR_CRC:
 197                 case TPT_ERR_MARKER:
 198                 case TPT_ERR_PDU_LEN_ERR:
 199                 case TPT_ERR_OUT_OF_RQE:
 200                 case TPT_ERR_DDP_VERSION:
 201                 case TPT_ERR_RDMA_VERSION:
 202                 case TPT_ERR_DDP_QUEUE_NUM:
 203                 case TPT_ERR_MSN:
 204                 case TPT_ERR_TBIT:
 205                 case TPT_ERR_MO:
 206                 case TPT_ERR_MSN_RANGE:
 207                 case TPT_ERR_IRD_OVERFLOW:
 208                 case TPT_ERR_OPCODE:
 209                         wc->status = IB_WC_FATAL_ERR;
 210                         break;
 211                 case TPT_ERR_SWFLUSH:
 212                         wc->status = IB_WC_WR_FLUSH_ERR;
 213                         break;
 214                 default:
 215                         log(LOG_ERR, "Unexpected cqe_status 0x%x for "
 216                                "QPID=0x%0x\n", CQE_STATUS(cqe), CQE_QPID(cqe));
 217                         ret = -EINVAL;
 218                 }
 219         }
 220 out:
 221         if (wq)
 222                 mtx_unlock(&qhp->lock);
 223         return ret;
 224 }
 225
 226 int iwch_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 227 {
 228         struct iwch_dev *rhp;
 229         struct iwch_cq *chp;
 230         int npolled;
 231         int err = 0;
 232
 233         chp = to_iwch_cq(ibcq);
 234         rhp = chp->rhp;
 235
 236         mtx_lock(&chp->lock);
 237         for (npolled = 0; npolled < num_entries; ++npolled) {
 238 #ifdef DEBUG
 239                 int i=0;
 240 #endif
 241
 242                 /*
 243                  * Because T3 can post CQEs that are _not_ associated
 244                  * with a WR, we might have to poll again after removing
 245                  * one of these.
 246                  */
 247                 do {
 248                         err = iwch_poll_cq_one(rhp, chp, wc + npolled);
 249 #ifdef DEBUG
 250                         PANIC_IF(++i > 1000);
 251 #endif
 252                 } while (err == -EAGAIN);
 253                 if (err <= 0)
 254                         break;
 255         }
 256         mtx_unlock(&chp->lock);
 257
 258         if (err < 0) {
 259                 return err;
 260         } else {
 261                 return npolled;
 262         }
 263 }
 264