sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cq.c

   1
   2 /**************************************************************************
   3
   4 Copyright (c) 2007, Chelsio Inc.
   5 All rights reserved.
   6
   7 Redistribution and use in source and binary forms, with or without
   8 modification, are permitted provided that the following conditions are met:
   9
  10  1. Redistributions of source code must retain the above copyright notice,
  11     this list of conditions and the following disclaimer.
  12
  13  2. Neither the name of the Chelsio Corporation nor the names of its
  14     contributors may be used to endorse or promote products derived from
  15     this software without specific prior written permission.
  16
  17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  18 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  21 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  22 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  23 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  24 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  25 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  26 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27 POSSIBILITY OF SUCH DAMAGE.
  28
  29 ***************************************************************************/
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34 #include <sys/systm.h>
  35 #include <sys/kernel.h>
  36 #include <sys/bus.h>
  37 #include <sys/module.h>
  38 #include <sys/pciio.h>
  39 #include <sys/conf.h>
  40 #include <machine/bus.h>
  41 #include <machine/resource.h>
  42 #include <sys/bus_dma.h>
  43 #include <sys/rman.h>
  44 #include <sys/ioccom.h>
  45 #include <sys/mbuf.h>
  46 #include <sys/mutex.h>
  47 #include <sys/rwlock.h>
  48 #include <sys/linker.h>
  49 #include <sys/firmware.h>
  50 #include <sys/socket.h>
  51 #include <sys/sockio.h>
  52 #include <sys/smp.h>
  53 #include <sys/sysctl.h>
  54 #include <sys/syslog.h>
  55 #include <sys/queue.h>
  56 #include <sys/taskqueue.h>
  57 #include <sys/proc.h>
  58 #include <sys/queue.h>
  59 #include <sys/libkern.h>
  60
  61 #include <netinet/in.h>
  62 #include <contrib/rdma/ib_verbs.h>
  63 #include <contrib/rdma/ib_umem.h>
  64 #include <contrib/rdma/ib_user_verbs.h>
  65
  66
  67 #ifdef CONFIG_DEFINED
  68 #include <cxgb_include.h>
  69 #include <ulp/iw_cxgb/iw_cxgb_wr.h>
  70 #include <ulp/iw_cxgb/iw_cxgb_hal.h>
  71 #include <ulp/iw_cxgb/iw_cxgb_provider.h>
  72 #include <ulp/iw_cxgb/iw_cxgb_cm.h>
  73 #include <ulp/iw_cxgb/iw_cxgb.h>
  74 #include <ulp/iw_cxgb/iw_cxgb_resource.h>
  75 #include <ulp/iw_cxgb/iw_cxgb_user.h>
  76 #else
  77 #include <dev/cxgb/cxgb_include.h>
  78 #include <dev/cxgb/ulp/iw_cxgb/iw_cxgb_wr.h>
  79 #include <dev/cxgb/ulp/iw_cxgb/iw_cxgb_hal.h>
  80 #include <dev/cxgb/ulp/iw_cxgb/iw_cxgb_provider.h>
  81 #include <dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.h>
  82 #include <dev/cxgb/ulp/iw_cxgb/iw_cxgb.h>
  83 #include <dev/cxgb/ulp/iw_cxgb/iw_cxgb_resource.h>
  84 #include <dev/cxgb/ulp/iw_cxgb/iw_cxgb_user.h>
  85 #endif
  86
  87 /*
  88  * Get one cq entry from cxio and map it to openib.
  89  *
  90  * Returns:
  91  *      0                       cqe returned
  92  *      -ENOBUFS                EMPTY;
  93  *      -EAGAIN                 caller must try again
  94  *      any other neg errno     fatal error
  95  */
  96 static int iwch_poll_cq_one(struct iwch_dev *rhp, struct iwch_cq *chp,
  97                             struct ib_wc *wc)
  98 {
  99         struct iwch_qp *qhp = NULL;
 100         struct t3_cqe cqe, *rd_cqe;
 101         struct t3_wq *wq;
 102         u32 credit = 0;
 103         u8 cqe_flushed;
 104         u64 cookie;
 105         int ret = 1;
 106
 107         rd_cqe = cxio_next_cqe(&chp->cq);
 108
 109         if (!rd_cqe)
 110                 return 0;
 111
 112         qhp = get_qhp(rhp, CQE_QPID(*rd_cqe));
 113         if (!qhp)
 114                 wq = NULL;
 115         else {
 116                 mtx_lock(&qhp->lock);
 117                 wq = &(qhp->wq);
 118         }
 119         ret = cxio_poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie,
 120                                    &credit);
 121         if (t3a_device(chp->rhp) && credit) {
 122                 CTR3(KTR_IW_CXGB, "%s updating %d cq credits on id %d", __FUNCTION__,
 123                      credit, chp->cq.cqid);
 124                 cxio_hal_cq_op(&rhp->rdev, &chp->cq, CQ_CREDIT_UPDATE, credit);
 125         }
 126
 127         if (ret) {
 128                 ret = -EAGAIN;
 129                 goto out;
 130         }
 131         ret = 1;
 132
 133         wc->wr_id = cookie;
 134         wc->qp = &qhp->ibqp;
 135         wc->vendor_err = CQE_STATUS(cqe);
 136
 137         CTR4(KTR_IW_CXGB, "iwch_poll_cq_one qpid 0x%x type %d opcode %d status 0x%x",
 138              CQE_QPID(cqe), CQE_TYPE(cqe),
 139              CQE_OPCODE(cqe), CQE_STATUS(cqe));
 140         CTR3(KTR_IW_CXGB, "wrid hi 0x%x lo 0x%x cookie 0x%llx",
 141              CQE_WRID_HI(cqe), CQE_WRID_LOW(cqe), (unsigned long long) cookie);
 142
 143         if (CQE_TYPE(cqe) == 0) {
 144                 if (!CQE_STATUS(cqe))
 145                         wc->byte_len = CQE_LEN(cqe);
 146                 else
 147                         wc->byte_len = 0;
 148                 wc->opcode = IB_WC_RECV;
 149         } else {
 150                 switch (CQE_OPCODE(cqe)) {
 151                 case T3_RDMA_WRITE:
 152                         wc->opcode = IB_WC_RDMA_WRITE;
 153                         break;
 154                 case T3_READ_REQ:
 155                         wc->opcode = IB_WC_RDMA_READ;
 156                         wc->byte_len = CQE_LEN(cqe);
 157                         break;
 158                 case T3_SEND:
 159                 case T3_SEND_WITH_SE:
 160                         wc->opcode = IB_WC_SEND;
 161                         break;
 162                 case T3_BIND_MW:
 163                         wc->opcode = IB_WC_BIND_MW;
 164                         break;
 165
 166                 /* these aren't supported yet */
 167                 case T3_SEND_WITH_INV:
 168                 case T3_SEND_WITH_SE_INV:
 169                 case T3_LOCAL_INV:
 170                 case T3_FAST_REGISTER:
 171                 default:
 172                         log(LOG_ERR, "Unexpected opcode %d "
 173                                "in the CQE received for QPID=0x%0x\n",
 174                                CQE_OPCODE(cqe), CQE_QPID(cqe));
 175                         ret = -EINVAL;
 176                         goto out;
 177                 }
 178         }
 179
 180         if (cqe_flushed)
 181                 wc->status = IB_WC_WR_FLUSH_ERR;
 182         else {
 183
 184                 switch (CQE_STATUS(cqe)) {
 185                 case TPT_ERR_SUCCESS:
 186                         wc->status = IB_WC_SUCCESS;
 187                         break;
 188                 case TPT_ERR_STAG:
 189                         wc->status = IB_WC_LOC_ACCESS_ERR;
 190                         break;
 191                 case TPT_ERR_PDID:
 192                         wc->status = IB_WC_LOC_PROT_ERR;
 193                         break;
 194                 case TPT_ERR_QPID:
 195                 case TPT_ERR_ACCESS:
 196                         wc->status = IB_WC_LOC_ACCESS_ERR;
 197                         break;
 198                 case TPT_ERR_WRAP:
 199                         wc->status = IB_WC_GENERAL_ERR;
 200                         break;
 201                 case TPT_ERR_BOUND:
 202                         wc->status = IB_WC_LOC_LEN_ERR;
 203                         break;
 204                 case TPT_ERR_INVALIDATE_SHARED_MR:
 205                 case TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND:
 206                         wc->status = IB_WC_MW_BIND_ERR;
 207                         break;
 208                 case TPT_ERR_CRC:
 209                 case TPT_ERR_MARKER:
 210                 case TPT_ERR_PDU_LEN_ERR:
 211                 case TPT_ERR_OUT_OF_RQE:
 212                 case TPT_ERR_DDP_VERSION:
 213                 case TPT_ERR_RDMA_VERSION:
 214                 case TPT_ERR_DDP_QUEUE_NUM:
 215                 case TPT_ERR_MSN:
 216                 case TPT_ERR_TBIT:
 217                 case TPT_ERR_MO:
 218                 case TPT_ERR_MSN_RANGE:
 219                 case TPT_ERR_IRD_OVERFLOW:
 220                 case TPT_ERR_OPCODE:
 221                         wc->status = IB_WC_FATAL_ERR;
 222                         break;
 223                 case TPT_ERR_SWFLUSH:
 224                         wc->status = IB_WC_WR_FLUSH_ERR;
 225                         break;
 226                 default:
 227                         log(LOG_ERR, "Unexpected cqe_status 0x%x for "
 228                                "QPID=0x%0x\n", CQE_STATUS(cqe), CQE_QPID(cqe));
 229                         ret = -EINVAL;
 230                 }
 231         }
 232 out:
 233         if (wq)
 234                 mtx_unlock(&qhp->lock);
 235         return ret;
 236 }
 237
 238 int iwch_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 239 {
 240         struct iwch_dev *rhp;
 241         struct iwch_cq *chp;
 242         int npolled;
 243         int err = 0;
 244
 245         chp = to_iwch_cq(ibcq);
 246         rhp = chp->rhp;
 247
 248         mtx_lock(&chp->lock);
 249         for (npolled = 0; npolled < num_entries; ++npolled) {
 250 #ifdef DEBUG
 251                 int i=0;
 252 #endif
 253
 254                 /*
 255                  * Because T3 can post CQEs that are _not_ associated
 256                  * with a WR, we might have to poll again after removing
 257                  * one of these.
 258                  */
 259                 do {
 260                         err = iwch_poll_cq_one(rhp, chp, wc + npolled);
 261 #ifdef DEBUG
 262                         PANIC_IF(++i > 1000);
 263 #endif
 264                 } while (err == -EAGAIN);
 265                 if (err <= 0)
 266                         break;
 267         }
 268         mtx_unlock(&chp->lock);
 269
 270         if (err < 0) {
 271                 return err;
 272         } else {
 273                 return npolled;
 274         }
 275 }
 276