sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cq.c

   1
   2 /**************************************************************************
   3
   4 Copyright (c) 2007, Chelsio Inc.
   5 All rights reserved.
   6
   7 Redistribution and use in source and binary forms, with or without
   8 modification, are permitted provided that the following conditions are met:
   9
  10  1. Redistributions of source code must retain the above copyright notice,
  11     this list of conditions and the following disclaimer.
  12
  13  2. Neither the name of the Chelsio Corporation nor the names of its
  14     contributors may be used to endorse or promote products derived from
  15     this software without specific prior written permission.
  16
  17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  18 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  21 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  22 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  23 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  24 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  25 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  26 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27 POSSIBILITY OF SUCH DAMAGE.
  28
  29 ***************************************************************************/
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include "opt_inet.h"
  34
  35 #ifdef TCP_OFFLOAD
  36 #include <sys/param.h>
  37 #include <sys/systm.h>
  38 #include <sys/kernel.h>
  39 #include <sys/bus.h>
  40 #include <sys/pciio.h>
  41 #include <sys/conf.h>
  42 #include <machine/bus.h>
  43 #include <machine/resource.h>
  44 #include <sys/bus_dma.h>
  45 #include <sys/rman.h>
  46 #include <sys/ioccom.h>
  47 #include <sys/mbuf.h>
  48 #include <sys/mutex.h>
  49 #include <sys/rwlock.h>
  50 #include <sys/linker.h>
  51 #include <sys/firmware.h>
  52 #include <sys/socket.h>
  53 #include <sys/sockio.h>
  54 #include <sys/smp.h>
  55 #include <sys/sysctl.h>
  56 #include <sys/syslog.h>
  57 #include <sys/queue.h>
  58 #include <sys/taskqueue.h>
  59 #include <sys/proc.h>
  60 #include <sys/queue.h>
  61 #include <sys/libkern.h>
  62
  63 #include <netinet/in.h>
  64 #include <rdma/ib_verbs.h>
  65 #include <rdma/ib_umem.h>
  66 #include <rdma/ib_user_verbs.h>
  67 #include <linux/idr.h>
  68 #include <ulp/iw_cxgb/iw_cxgb_ib_intfc.h>
  69
  70 #include <cxgb_include.h>
  71 #include <ulp/iw_cxgb/iw_cxgb_wr.h>
  72 #include <ulp/iw_cxgb/iw_cxgb_hal.h>
  73 #include <ulp/iw_cxgb/iw_cxgb_provider.h>
  74 #include <ulp/iw_cxgb/iw_cxgb_cm.h>
  75 #include <ulp/iw_cxgb/iw_cxgb.h>
  76 #include <ulp/iw_cxgb/iw_cxgb_resource.h>
  77 #include <ulp/iw_cxgb/iw_cxgb_user.h>
  78
  79 /*
  80  * Get one cq entry from cxio and map it to openib.
  81  *
  82  * Returns:
  83  *      0                       cqe returned
  84  *      -ENOBUFS                EMPTY;
  85  *      -EAGAIN                 caller must try again
  86  *      any other neg errno     fatal error
  87  */
  88 static int iwch_poll_cq_one(struct iwch_dev *rhp, struct iwch_cq *chp,
  89                             struct ib_wc *wc)
  90 {
  91         struct iwch_qp *qhp = NULL;
  92         struct t3_cqe cqe, *rd_cqe;
  93         struct t3_wq *wq;
  94         u32 credit = 0;
  95         u8 cqe_flushed;
  96         u64 cookie;
  97         int ret = 1;
  98
  99         rd_cqe = cxio_next_cqe(&chp->cq);
 100
 101         if (!rd_cqe)
 102                 return 0;
 103
 104         qhp = get_qhp(rhp, CQE_QPID(*rd_cqe));
 105         if (!qhp)
 106                 wq = NULL;
 107         else {
 108                 mtx_lock(&qhp->lock);
 109                 wq = &(qhp->wq);
 110         }
 111         ret = cxio_poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie,
 112                                    &credit);
 113         if (t3a_device(chp->rhp) && credit) {
 114                 CTR3(KTR_IW_CXGB, "%s updating %d cq credits on id %d", __FUNCTION__,
 115                      credit, chp->cq.cqid);
 116                 cxio_hal_cq_op(&rhp->rdev, &chp->cq, CQ_CREDIT_UPDATE, credit);
 117         }
 118
 119         if (ret) {
 120                 ret = -EAGAIN;
 121                 goto out;
 122         }
 123         ret = 1;
 124
 125         wc->wr_id = cookie;
 126         wc->qp = &qhp->ibqp;
 127         wc->vendor_err = CQE_STATUS(cqe);
 128
 129         CTR4(KTR_IW_CXGB, "iwch_poll_cq_one qpid 0x%x type %d opcode %d status 0x%x",
 130              CQE_QPID(cqe), CQE_TYPE(cqe),
 131              CQE_OPCODE(cqe), CQE_STATUS(cqe));
 132         CTR3(KTR_IW_CXGB, "wrid hi 0x%x lo 0x%x cookie 0x%llx",
 133              CQE_WRID_HI(cqe), CQE_WRID_LOW(cqe), (unsigned long long) cookie);
 134
 135         if (CQE_TYPE(cqe) == 0) {
 136                 if (!CQE_STATUS(cqe))
 137                         wc->byte_len = CQE_LEN(cqe);
 138                 else
 139                         wc->byte_len = 0;
 140                 wc->opcode = IB_WC_RECV;
 141         } else {
 142                 switch (CQE_OPCODE(cqe)) {
 143                 case T3_RDMA_WRITE:
 144                         wc->opcode = IB_WC_RDMA_WRITE;
 145                         break;
 146                 case T3_READ_REQ:
 147                         wc->opcode = IB_WC_RDMA_READ;
 148                         wc->byte_len = CQE_LEN(cqe);
 149                         break;
 150                 case T3_SEND:
 151                 case T3_SEND_WITH_SE:
 152                         wc->opcode = IB_WC_SEND;
 153                         break;
 154                 case T3_BIND_MW:
 155                         wc->opcode = IB_WC_BIND_MW;
 156                         break;
 157
 158                 /* these aren't supported yet */
 159                 case T3_SEND_WITH_INV:
 160                 case T3_SEND_WITH_SE_INV:
 161                 case T3_LOCAL_INV:
 162                 case T3_FAST_REGISTER:
 163                 default:
 164                         log(LOG_ERR, "Unexpected opcode %d "
 165                                "in the CQE received for QPID=0x%0x\n",
 166                                CQE_OPCODE(cqe), CQE_QPID(cqe));
 167                         ret = -EINVAL;
 168                         goto out;
 169                 }
 170         }
 171
 172         if (cqe_flushed)
 173                 wc->status = IB_WC_WR_FLUSH_ERR;
 174         else {
 175
 176                 switch (CQE_STATUS(cqe)) {
 177                 case TPT_ERR_SUCCESS:
 178                         wc->status = IB_WC_SUCCESS;
 179                         break;
 180                 case TPT_ERR_STAG:
 181                         wc->status = IB_WC_LOC_ACCESS_ERR;
 182                         break;
 183                 case TPT_ERR_PDID:
 184                         wc->status = IB_WC_LOC_PROT_ERR;
 185                         break;
 186                 case TPT_ERR_QPID:
 187                 case TPT_ERR_ACCESS:
 188                         wc->status = IB_WC_LOC_ACCESS_ERR;
 189                         break;
 190                 case TPT_ERR_WRAP:
 191                         wc->status = IB_WC_GENERAL_ERR;
 192                         break;
 193                 case TPT_ERR_BOUND:
 194                         wc->status = IB_WC_LOC_LEN_ERR;
 195                         break;
 196                 case TPT_ERR_INVALIDATE_SHARED_MR:
 197                 case TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND:
 198                         wc->status = IB_WC_MW_BIND_ERR;
 199                         break;
 200                 case TPT_ERR_CRC:
 201                 case TPT_ERR_MARKER:
 202                 case TPT_ERR_PDU_LEN_ERR:
 203                 case TPT_ERR_OUT_OF_RQE:
 204                 case TPT_ERR_DDP_VERSION:
 205                 case TPT_ERR_RDMA_VERSION:
 206                 case TPT_ERR_DDP_QUEUE_NUM:
 207                 case TPT_ERR_MSN:
 208                 case TPT_ERR_TBIT:
 209                 case TPT_ERR_MO:
 210                 case TPT_ERR_MSN_RANGE:
 211                 case TPT_ERR_IRD_OVERFLOW:
 212                 case TPT_ERR_OPCODE:
 213                         wc->status = IB_WC_FATAL_ERR;
 214                         break;
 215                 case TPT_ERR_SWFLUSH:
 216                         wc->status = IB_WC_WR_FLUSH_ERR;
 217                         break;
 218                 default:
 219                         log(LOG_ERR, "Unexpected cqe_status 0x%x for "
 220                                "QPID=0x%0x\n", CQE_STATUS(cqe), CQE_QPID(cqe));
 221                         ret = -EINVAL;
 222                 }
 223         }
 224 out:
 225         if (wq)
 226                 mtx_unlock(&qhp->lock);
 227         return ret;
 228 }
 229
 230 int iwch_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 231 {
 232         struct iwch_dev *rhp;
 233         struct iwch_cq *chp;
 234         int npolled;
 235         int err = 0;
 236
 237         chp = to_iwch_cq(ibcq);
 238         rhp = chp->rhp;
 239
 240         mtx_lock(&chp->lock);
 241         for (npolled = 0; npolled < num_entries; ++npolled) {
 242 #ifdef DEBUG
 243                 int i=0;
 244 #endif
 245
 246                 /*
 247                  * Because T3 can post CQEs that are _not_ associated
 248                  * with a WR, we might have to poll again after removing
 249                  * one of these.
 250                  */
 251                 do {
 252                         err = iwch_poll_cq_one(rhp, chp, wc + npolled);
 253 #ifdef DEBUG
 254                         PANIC_IF(++i > 1000);
 255 #endif
 256                 } while (err == -EAGAIN);
 257                 if (err <= 0)
 258                         break;
 259         }
 260         mtx_unlock(&chp->lock);
 261
 262         if (err < 0) {
 263                 return err;
 264         } else {
 265                 return npolled;
 266         }
 267 }
 268 #endif