sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_hal.c

   1
   2 /**************************************************************************
   3
   4 Copyright (c) 2007, Chelsio Inc.
   5 All rights reserved.
   6
   7 Redistribution and use in source and binary forms, with or without
   8 modification, are permitted provided that the following conditions are met:
   9
  10  1. Redistributions of source code must retain the above copyright notice,
  11     this list of conditions and the following disclaimer.
  12
  13  2. Neither the name of the Chelsio Corporation nor the names of its
  14     contributors may be used to endorse or promote products derived from
  15     this software without specific prior written permission.
  16
  17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  18 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  21 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  22 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  23 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  24 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  25 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  26 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27 POSSIBILITY OF SUCH DAMAGE.
  28
  29 ***************************************************************************/
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34 #include <sys/systm.h>
  35 #include <sys/kernel.h>
  36 #include <sys/bus.h>
  37 #include <sys/module.h>
  38 #include <sys/pciio.h>
  39 #include <sys/conf.h>
  40 #include <machine/bus.h>
  41 #include <machine/resource.h>
  42 #include <sys/bus_dma.h>
  43 #include <sys/rman.h>
  44 #include <sys/ioccom.h>
  45 #include <sys/mbuf.h>
  46 #include <sys/rwlock.h>
  47 #include <sys/linker.h>
  48 #include <sys/firmware.h>
  49 #include <sys/socket.h>
  50 #include <sys/sockio.h>
  51 #include <sys/smp.h>
  52 #include <sys/sysctl.h>
  53 #include <sys/syslog.h>
  54 #include <sys/queue.h>
  55 #include <sys/taskqueue.h>
  56 #include <sys/proc.h>
  57 #include <sys/queue.h>
  58
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61
  62 #include <netinet/in.h>
  63
  64 #include <contrib/rdma/ib_verbs.h>
  65
  66
  67 #ifdef CONFIG_DEFINED
  68 #include <cxgb_include.h>
  69 #include <ulp/iw_cxgb/iw_cxgb_wr.h>
  70 #include <ulp/iw_cxgb/iw_cxgb_hal.h>
  71 #include <ulp/iw_cxgb/iw_cxgb_provider.h>
  72 #include <ulp/iw_cxgb/iw_cxgb_cm.h>
  73 #include <ulp/iw_cxgb/iw_cxgb.h>
  74 #include <ulp/iw_cxgb/iw_cxgb_resource.h>
  75 #else
  76 #include <dev/cxgb/cxgb_include.h>
  77 #include <dev/cxgb/ulp/iw_cxgb/iw_cxgb_wr.h>
  78 #include <dev/cxgb/ulp/iw_cxgb/iw_cxgb_hal.h>
  79 #include <dev/cxgb/ulp/iw_cxgb/iw_cxgb_provider.h>
  80 #include <dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.h>
  81 #include <dev/cxgb/ulp/iw_cxgb/iw_cxgb.h>
  82 #include <dev/cxgb/ulp/iw_cxgb/iw_cxgb_resource.h>
  83 #endif
  84
  85 static TAILQ_HEAD( ,cxio_rdev) rdev_list;
  86 static cxio_hal_ev_callback_func_t cxio_ev_cb = NULL;
  87
  88 static struct cxio_rdev *
  89 cxio_hal_find_rdev_by_name(char *dev_name)
  90 {
  91         struct cxio_rdev *rdev;
  92
  93         TAILQ_FOREACH(rdev, &rdev_list, entry)
  94                 if (!strcmp(rdev->dev_name, dev_name))
  95                         return rdev;
  96         return NULL;
  97 }
  98
  99 struct cxio_rdev *
 100 cxio_hal_find_rdev_by_t3cdev(struct t3cdev *tdev)
 101 {
 102         struct cxio_rdev *rdev;
 103
 104         TAILQ_FOREACH(rdev, &rdev_list, entry)
 105                 if (rdev->t3cdev_p == tdev)
 106                         return rdev;
 107         return NULL;
 108 }
 109
 110 int
 111 cxio_hal_cq_op(struct cxio_rdev *rdev_p, struct t3_cq *cq,
 112                    enum t3_cq_opcode op, u32 credit)
 113 {
 114         int ret;
 115         struct t3_cqe *cqe;
 116         u32 rptr;
 117
 118         struct rdma_cq_op setup;
 119         setup.id = cq->cqid;
 120         setup.credits = (op == CQ_CREDIT_UPDATE) ? credit : 0;
 121         setup.op = op;
 122         ret = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_OP, &setup);
 123
 124         if ((ret < 0) || (op == CQ_CREDIT_UPDATE))
 125                 return (ret);
 126
 127         /*
 128          * If the rearm returned an index other than our current index,
 129          * then there might be CQE's in flight (being DMA'd).  We must wait
 130          * here for them to complete or the consumer can miss a notification.
 131          */
 132         if (Q_PTR2IDX((cq->rptr), cq->size_log2) != ret) {
 133                 int i=0;
 134
 135                 rptr = cq->rptr;
 136
 137                 /*
 138                  * Keep the generation correct by bumping rptr until it
 139                  * matches the index returned by the rearm - 1.
 140                  */
 141                 while (Q_PTR2IDX((rptr+1), cq->size_log2) != ret)
 142                         rptr++;
 143
 144                 /*
 145                  * Now rptr is the index for the (last) cqe that was
 146                  * in-flight at the time the HW rearmed the CQ.  We
 147                  * spin until that CQE is valid.
 148                  */
 149                 cqe = cq->queue + Q_PTR2IDX(rptr, cq->size_log2);
 150                 while (!CQ_VLD_ENTRY(rptr, cq->size_log2, cqe)) {
 151                         DELAY(1);
 152                         if (i++ > 1000000) {
 153                                 PANIC_IF(1);
 154                                 log(LOG_ERR, "%s: stalled rnic\n",
 155                                        rdev_p->dev_name);
 156                                 return (-EIO);
 157                         }
 158                 }
 159
 160                 return 1;
 161         }
 162
 163         return 0;
 164 }
 165
 166 static int
 167 cxio_hal_clear_cq_ctx(struct cxio_rdev *rdev_p, u32 cqid)
 168 {
 169         struct rdma_cq_setup setup;
 170         setup.id = cqid;
 171         setup.base_addr = 0;    /* NULL address */
 172         setup.size = 0;         /* disaable the CQ */
 173         setup.credits = 0;
 174         setup.credit_thres = 0;
 175         setup.ovfl_mode = 0;
 176         return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));
 177 }
 178
 179 static int
 180 cxio_hal_clear_qp_ctx(struct cxio_rdev *rdev_p, u32 qpid)
 181 {
 182         u64 sge_cmd;
 183         struct t3_modify_qp_wr *wqe;
 184         struct mbuf *m = m_gethdr(MT_DATA, M_NOWAIT);
 185         if (m == NULL) {
 186                 CTR1(KTR_IW_CXGB, "%s m_gethdr failed", __FUNCTION__);
 187                 return (-ENOMEM);
 188         }
 189         wqe = mtod(m, struct t3_modify_qp_wr *);
 190         m->m_len = m->m_pkthdr.len = sizeof(*wqe);
 191         memset(wqe, 0, sizeof(*wqe));
 192         build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD, 3, 0, qpid, 7);
 193         wqe->flags = htobe32(MODQP_WRITE_EC);
 194         sge_cmd = qpid << 8 | 3;
 195         wqe->sge_cmd = htobe64(sge_cmd);
 196         m_set_priority(m, CPL_PRIORITY_CONTROL);
 197         m_set_sgl(m, NULL);
 198         m_set_sgllen(m, 0);
 199         return (cxgb_ofld_send(rdev_p->t3cdev_p, m));
 200 }
 201
 202 int
 203 cxio_create_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)
 204 {
 205         struct rdma_cq_setup setup;
 206         int size = (1UL << (cq->size_log2)) * sizeof(struct t3_cqe);
 207
 208         cq->cqid = cxio_hal_get_cqid(rdev_p->rscp);
 209         if (!cq->cqid)
 210                 return (-ENOMEM);
 211         cq->sw_queue = malloc(size, M_DEVBUF, M_NOWAIT|M_ZERO);
 212         if (!cq->sw_queue)
 213                 return (-ENOMEM);
 214 #if 0
 215         cq->queue = dma_alloc_coherent(rdev_p->rnic_info.pdev,
 216                                              (1UL << (cq->size_log2)) *
 217                                              sizeof(struct t3_cqe),
 218                                              &(cq->dma_addr), M_NOWAIT);
 219 #else
 220         cq->queue = contigmalloc((1UL << (cq->size_log2))*sizeof(struct t3_cqe),
 221             M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 4096, 0);
 222         if (cq->queue)
 223                 cq->dma_addr = vtophys(cq->queue);
 224         else {
 225                 free(cq->sw_queue, M_DEVBUF);
 226                 return (-ENOMEM);
 227         }
 228 #endif
 229
 230 #ifdef notyet
 231         pci_unmap_addr_set(cq, mapping, cq->dma_addr);
 232 #endif
 233         memset(cq->queue, 0, size);
 234         setup.id = cq->cqid;
 235         setup.base_addr = (u64) (cq->dma_addr);
 236         setup.size = 1UL << cq->size_log2;
 237         setup.credits = 65535;
 238         setup.credit_thres = 1;
 239         if (rdev_p->t3cdev_p->type != T3A)
 240                 setup.ovfl_mode = 0;
 241         else
 242                 setup.ovfl_mode = 1;
 243         return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));
 244 }
 245
 246 int
 247 cxio_resize_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)
 248 {
 249         struct rdma_cq_setup setup;
 250         setup.id = cq->cqid;
 251         setup.base_addr = (u64) (cq->dma_addr);
 252         setup.size = 1UL << cq->size_log2;
 253         setup.credits = setup.size;
 254         setup.credit_thres = setup.size;        /* TBD: overflow recovery */
 255         setup.ovfl_mode = 1;
 256         return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));
 257 }
 258
 259 static u32
 260 get_qpid(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)
 261 {
 262         struct cxio_qpid *entry;
 263         u32 qpid;
 264         int i;
 265
 266         mtx_lock(&uctx->lock);
 267         if (!TAILQ_EMPTY(&uctx->qpids)) {
 268
 269                 entry = TAILQ_FIRST(&uctx->qpids);
 270                 TAILQ_REMOVE(&uctx->qpids, entry, entry);
 271                 qpid = entry->qpid;
 272                 free(entry, M_DEVBUF);
 273         } else {
 274                 qpid = cxio_hal_get_qpid(rdev_p->rscp);
 275                 if (!qpid)
 276                         goto out;
 277                 for (i = qpid+1; i & rdev_p->qpmask; i++) {
 278                         entry = malloc(sizeof *entry, M_DEVBUF, M_NOWAIT);
 279                         if (!entry)
 280                                 break;
 281                         entry->qpid = i;
 282                         TAILQ_INSERT_TAIL(&uctx->qpids, entry, entry);
 283                 }
 284         }
 285 out:
 286         mtx_unlock(&uctx->lock);
 287         CTR2(KTR_IW_CXGB, "%s qpid 0x%x", __FUNCTION__, qpid);
 288         return qpid;
 289 }
 290
 291 static void
 292 put_qpid(struct cxio_rdev *rdev_p, u32 qpid,
 293                      struct cxio_ucontext *uctx)
 294 {
 295         struct cxio_qpid *entry;
 296
 297         entry = malloc(sizeof *entry, M_DEVBUF, M_NOWAIT);
 298         CTR2(KTR_IW_CXGB, "%s qpid 0x%x", __FUNCTION__, qpid);
 299         entry->qpid = qpid;
 300         mtx_lock(&uctx->lock);
 301         TAILQ_INSERT_TAIL(&uctx->qpids, entry, entry);
 302         mtx_unlock(&uctx->lock);
 303 }
 304
 305 void
 306 cxio_release_ucontext(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)
 307 {
 308         struct cxio_qpid *pos, *tmp;
 309
 310         mtx_lock(&uctx->lock);
 311         TAILQ_FOREACH_SAFE(pos, &uctx->qpids, entry, tmp) {
 312                 TAILQ_REMOVE(&uctx->qpids, pos, entry);
 313                 if (!(pos->qpid & rdev_p->qpmask))
 314                         cxio_hal_put_qpid(rdev_p->rscp, pos->qpid);
 315                 free(pos, M_DEVBUF);
 316         }
 317         mtx_unlock(&uctx->lock);
 318 }
 319
 320 void
 321 cxio_init_ucontext(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)
 322 {
 323         TAILQ_INIT(&uctx->qpids);
 324         mtx_init(&uctx->lock, "cxio uctx", NULL, MTX_DEF|MTX_DUPOK);
 325 }
 326
 327 int
 328 cxio_create_qp(struct cxio_rdev *rdev_p, u32 kernel_domain,
 329     struct t3_wq *wq, struct cxio_ucontext *uctx)
 330 {
 331         int depth = 1UL << wq->size_log2;
 332         int rqsize = 1UL << wq->rq_size_log2;
 333
 334         wq->qpid = get_qpid(rdev_p, uctx);
 335         if (!wq->qpid)
 336                 return (-ENOMEM);
 337
 338         wq->rq = malloc(depth * sizeof(u64), M_DEVBUF, M_NOWAIT|M_ZERO);
 339         if (!wq->rq)
 340                 goto err1;
 341
 342         wq->rq_addr = cxio_hal_rqtpool_alloc(rdev_p, rqsize);
 343         if (!wq->rq_addr)
 344                 goto err2;
 345
 346         wq->sq = malloc(depth * sizeof(struct t3_swsq), M_DEVBUF, M_NOWAIT|M_ZERO);
 347         if (!wq->sq)
 348                 goto err3;
 349 #if 0
 350         wq->queue = dma_alloc_coherent(rdev_p->rnic_info.pdev,
 351                                              depth * sizeof(union t3_wr),
 352                                              &(wq->dma_addr), M_NOWAIT);
 353 #else
 354         wq->queue = contigmalloc(depth *sizeof(union t3_wr),
 355             M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 4096, 0);
 356         if (wq->queue)
 357                 wq->dma_addr = vtophys(wq->queue);
 358
 359 #endif
 360         if (!wq->queue)
 361                 goto err4;
 362
 363         memset(wq->queue, 0, depth * sizeof(union t3_wr));
 364 #ifdef notyet
 365         pci_unmap_addr_set(wq, mapping, wq->dma_addr);
 366 #endif
 367         wq->doorbell = rdev_p->rnic_info.kdb_addr;
 368         if (!kernel_domain)
 369                 wq->udb = (u64)rdev_p->rnic_info.udbell_physbase +
 370                                         (wq->qpid << rdev_p->qpshift);
 371         CTR4(KTR_IW_CXGB, "%s qpid 0x%x doorbell 0x%p udb 0x%llx", __FUNCTION__,
 372              wq->qpid, wq->doorbell, (unsigned long long) wq->udb);
 373         return 0;
 374 err4:
 375         free(wq->sq, M_DEVBUF);
 376 err3:
 377         cxio_hal_rqtpool_free(rdev_p, wq->rq_addr, rqsize);
 378 err2:
 379         free(wq->rq, M_DEVBUF);
 380 err1:
 381         put_qpid(rdev_p, wq->qpid, uctx);
 382         return (-ENOMEM);
 383 }
 384
 385 int
 386 cxio_destroy_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)
 387 {
 388         int err;
 389         err = cxio_hal_clear_cq_ctx(rdev_p, cq->cqid);
 390         free(cq->sw_queue, M_DEVBUF);
 391 #if 0
 392         dma_free_coherent(&(rdev_p->rnic_info.pdev),
 393             (1UL << (cq->size_log2))
 394                           * sizeof(struct t3_cqe), cq->queue,
 395             /* pci_unmap_addr(cq, mapping)*/  0);
 396 #else
 397         contigfree(cq->queue,(1UL << (cq->size_log2))
 398             * sizeof(struct t3_cqe), M_DEVBUF);
 399 #endif
 400         cxio_hal_put_cqid(rdev_p->rscp, cq->cqid);
 401         return err;
 402 }
 403
 404 int
 405 cxio_destroy_qp(struct cxio_rdev *rdev_p, struct t3_wq *wq,
 406     struct cxio_ucontext *uctx)
 407 {
 408
 409 #if 0
 410         dma_free_coherent(&(rdev_p->rnic_info.pdev),
 411                           (1UL << (wq->size_log2))
 412                           * sizeof(union t3_wr), wq->queue,
 413             /* pci_unmap_addr(wq, mapping)*/ 0);
 414 #else
 415         contigfree(wq->queue, (1UL << (wq->size_log2))
 416             * sizeof(union t3_wr), M_DEVBUF);
 417 #endif
 418         free(wq->sq, M_DEVBUF);
 419         cxio_hal_rqtpool_free(rdev_p, wq->rq_addr, (1UL << wq->rq_size_log2));
 420         free(wq->rq, M_DEVBUF);
 421         put_qpid(rdev_p, wq->qpid, uctx);
 422         return 0;
 423 }
 424
 425 static void
 426 insert_recv_cqe(struct t3_wq *wq, struct t3_cq *cq)
 427 {
 428         struct t3_cqe cqe;
 429
 430         CTR5(KTR_IW_CXGB, "%s wq %p cq %p sw_rptr 0x%x sw_wptr 0x%x", __FUNCTION__,
 431              wq, cq, cq->sw_rptr, cq->sw_wptr);
 432         memset(&cqe, 0, sizeof(cqe));
 433         cqe.header = htobe32(V_CQE_STATUS(TPT_ERR_SWFLUSH) |
 434                                  V_CQE_OPCODE(T3_SEND) |
 435                                  V_CQE_TYPE(0) |
 436                                  V_CQE_SWCQE(1) |
 437                                  V_CQE_QPID(wq->qpid) |
 438                                  V_CQE_GENBIT(Q_GENBIT(cq->sw_wptr,
 439                                                        cq->size_log2)));
 440         *(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2)) = cqe;
 441         cq->sw_wptr++;
 442 }
 443
 444 void
 445 cxio_flush_rq(struct t3_wq *wq, struct t3_cq *cq, int count)
 446 {
 447         u32 ptr;
 448
 449         CTR3(KTR_IW_CXGB, "%s wq %p cq %p", __FUNCTION__, wq, cq);
 450
 451         /* flush RQ */
 452         CTR4(KTR_IW_CXGB, "%s rq_rptr %u rq_wptr %u skip count %u", __FUNCTION__,
 453             wq->rq_rptr, wq->rq_wptr, count);
 454         ptr = wq->rq_rptr + count;
 455         while (ptr++ != wq->rq_wptr)
 456                 insert_recv_cqe(wq, cq);
 457 }
 458
 459 static void
 460 insert_sq_cqe(struct t3_wq *wq, struct t3_cq *cq,
 461                           struct t3_swsq *sqp)
 462 {
 463         struct t3_cqe cqe;
 464
 465         CTR5(KTR_IW_CXGB, "%s wq %p cq %p sw_rptr 0x%x sw_wptr 0x%x", __FUNCTION__,
 466              wq, cq, cq->sw_rptr, cq->sw_wptr);
 467         memset(&cqe, 0, sizeof(cqe));
 468         cqe.header = htobe32(V_CQE_STATUS(TPT_ERR_SWFLUSH) |
 469                                  V_CQE_OPCODE(sqp->opcode) |
 470                                  V_CQE_TYPE(1) |
 471                                  V_CQE_SWCQE(1) |
 472                                  V_CQE_QPID(wq->qpid) |
 473                                  V_CQE_GENBIT(Q_GENBIT(cq->sw_wptr,
 474                                                        cq->size_log2)));
 475         cqe.u.scqe.wrid_hi = sqp->sq_wptr;
 476
 477         *(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2)) = cqe;
 478         cq->sw_wptr++;
 479 }
 480
 481 void
 482 cxio_flush_sq(struct t3_wq *wq, struct t3_cq *cq, int count)
 483 {
 484         __u32 ptr;
 485         struct t3_swsq *sqp = wq->sq + Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2);
 486
 487         ptr = wq->sq_rptr + count;
 488         sqp += count;
 489         while (ptr != wq->sq_wptr) {
 490                 insert_sq_cqe(wq, cq, sqp);
 491                 sqp++;
 492                 ptr++;
 493         }
 494 }
 495
 496 /*
 497  * Move all CQEs from the HWCQ into the SWCQ.
 498  */
 499 void
 500 cxio_flush_hw_cq(struct t3_cq *cq)
 501 {
 502         struct t3_cqe *cqe, *swcqe;
 503
 504         CTR3(KTR_IW_CXGB, "%s cq %p cqid 0x%x", __FUNCTION__, cq, cq->cqid);
 505         cqe = cxio_next_hw_cqe(cq);
 506         while (cqe) {
 507                 CTR3(KTR_IW_CXGB, "%s flushing hwcq rptr 0x%x to swcq wptr 0x%x",
 508                      __FUNCTION__, cq->rptr, cq->sw_wptr);
 509                 swcqe = cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2);
 510                 *swcqe = *cqe;
 511                 swcqe->header |= htobe32(V_CQE_SWCQE(1));
 512                 cq->sw_wptr++;
 513                 cq->rptr++;
 514                 cqe = cxio_next_hw_cqe(cq);
 515         }
 516 }
 517
 518 static int cqe_completes_wr(struct t3_cqe *cqe, struct t3_wq *wq)
 519 {
 520         if (CQE_OPCODE(*cqe) == T3_TERMINATE)
 521                 return 0;
 522
 523         if ((CQE_OPCODE(*cqe) == T3_RDMA_WRITE) && RQ_TYPE(*cqe))
 524                 return 0;
 525
 526         if ((CQE_OPCODE(*cqe) == T3_READ_RESP) && SQ_TYPE(*cqe))
 527                 return 0;
 528
 529         if ((CQE_OPCODE(*cqe) == T3_SEND) && RQ_TYPE(*cqe) &&
 530             Q_EMPTY(wq->rq_rptr, wq->rq_wptr))
 531                 return 0;
 532
 533         return 1;
 534 }
 535
 536 void
 537 cxio_count_scqes(struct t3_cq *cq, struct t3_wq *wq, int *count)
 538 {
 539         struct t3_cqe *cqe;
 540         u32 ptr;
 541
 542         *count = 0;
 543         ptr = cq->sw_rptr;
 544         while (!Q_EMPTY(ptr, cq->sw_wptr)) {
 545                 cqe = cq->sw_queue + (Q_PTR2IDX(ptr, cq->size_log2));
 546                 if ((SQ_TYPE(*cqe) || (CQE_OPCODE(*cqe) == T3_READ_RESP)) &&
 547                     (CQE_QPID(*cqe) == wq->qpid))
 548                         (*count)++;
 549                 ptr++;
 550         }
 551         CTR3(KTR_IW_CXGB, "%s cq %p count %d", __FUNCTION__, cq, *count);
 552 }
 553
 554 void
 555 cxio_count_rcqes(struct t3_cq *cq, struct t3_wq *wq, int *count)
 556 {
 557         struct t3_cqe *cqe;
 558         u32 ptr;
 559
 560         *count = 0;
 561         CTR2(KTR_IW_CXGB, "%s count zero %d", __FUNCTION__, *count);
 562         ptr = cq->sw_rptr;
 563         while (!Q_EMPTY(ptr, cq->sw_wptr)) {
 564                 cqe = cq->sw_queue + (Q_PTR2IDX(ptr, cq->size_log2));
 565                 if (RQ_TYPE(*cqe) && (CQE_OPCODE(*cqe) != T3_READ_RESP) &&
 566                     (CQE_QPID(*cqe) == wq->qpid) && cqe_completes_wr(cqe, wq))
 567                         (*count)++;
 568                 ptr++;
 569         }
 570         CTR3(KTR_IW_CXGB, "%s cq %p count %d", __FUNCTION__, cq, *count);
 571 }
 572
 573 static int
 574 cxio_hal_init_ctrl_cq(struct cxio_rdev *rdev_p)
 575 {
 576         struct rdma_cq_setup setup;
 577         setup.id = 0;
 578         setup.base_addr = 0;    /* NULL address */
 579         setup.size = 1;         /* enable the CQ */
 580         setup.credits = 0;
 581
 582         /* force SGE to redirect to RspQ and interrupt */
 583         setup.credit_thres = 0;
 584         setup.ovfl_mode = 1;
 585         return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));
 586 }
 587
 588 static int
 589 cxio_hal_init_ctrl_qp(struct cxio_rdev *rdev_p)
 590 {
 591         int err;
 592         u64 sge_cmd, ctx0, ctx1;
 593         u64 base_addr;
 594         struct t3_modify_qp_wr *wqe;
 595         struct mbuf *m;
 596
 597         m = m_gethdr(MT_DATA, M_NOWAIT);
 598         if (m == NULL) {
 599                 CTR1(KTR_IW_CXGB, "%s m_gethdr failed", __FUNCTION__);
 600                 return (-ENOMEM);
 601         }
 602         err = cxio_hal_init_ctrl_cq(rdev_p);
 603         if (err) {
 604                 CTR2(KTR_IW_CXGB, "%s err %d initializing ctrl_cq", __FUNCTION__, err);
 605                 goto err;
 606         }
 607 #if 0
 608         rdev_p->ctrl_qp.workq = dma_alloc_coherent(
 609                 rdev_p->rnic_info.pdev,
 610                     (1 << T3_CTRL_QP_SIZE_LOG2) *
 611                     sizeof(union t3_wr),
 612                     &(rdev_p->ctrl_qp.dma_addr),
 613                     M_NOWAIT);
 614 #else
 615         rdev_p->ctrl_qp.workq = contigmalloc((1 << T3_CTRL_QP_SIZE_LOG2)
 616             *sizeof(union t3_wr), M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 4096, 0);
 617         if (rdev_p->ctrl_qp.workq)
 618                 rdev_p->ctrl_qp.dma_addr = vtophys(rdev_p->ctrl_qp.workq);
 619
 620 #endif
 621
 622         if (!rdev_p->ctrl_qp.workq) {
 623                 CTR1(KTR_IW_CXGB, "%s dma_alloc_coherent failed", __FUNCTION__);
 624                 err = -ENOMEM;
 625                 goto err;
 626         }
 627 #if 0
 628         pci_unmap_addr_set(&rdev_p->ctrl_qp, mapping,
 629                            rdev_p->ctrl_qp.dma_addr);
 630 #endif
 631         rdev_p->ctrl_qp.doorbell = (void /*__iomem */ *)rdev_p->rnic_info.kdb_addr;
 632         memset(rdev_p->ctrl_qp.workq, 0,
 633                (1 << T3_CTRL_QP_SIZE_LOG2) * sizeof(union t3_wr));
 634
 635         mtx_init(&rdev_p->ctrl_qp.lock, "ctl-qp lock", NULL, MTX_DEF|MTX_DUPOK);
 636
 637         /* update HW Ctrl QP context */
 638         base_addr = rdev_p->ctrl_qp.dma_addr;
 639         base_addr >>= 12;
 640         ctx0 = (V_EC_SIZE((1 << T3_CTRL_QP_SIZE_LOG2)) |
 641                 V_EC_BASE_LO((u32) base_addr & 0xffff));
 642         ctx0 <<= 32;
 643         ctx0 |= V_EC_CREDITS(FW_WR_NUM);
 644         base_addr >>= 16;
 645         ctx1 = (u32) base_addr;
 646         base_addr >>= 32;
 647         ctx1 |= ((u64) (V_EC_BASE_HI((u32) base_addr & 0xf) | V_EC_RESPQ(0) |
 648                         V_EC_TYPE(0) | V_EC_GEN(1) |
 649                         V_EC_UP_TOKEN(T3_CTL_QP_TID) | F_EC_VALID)) << 32;
 650         wqe = mtod(m, struct t3_modify_qp_wr *);
 651         m->m_len = m->m_pkthdr.len = sizeof(*wqe);
 652         memset(wqe, 0, sizeof(*wqe));
 653         build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD, 3, 0,
 654                        T3_CTL_QP_TID, 7);
 655         wqe->flags = htobe32(MODQP_WRITE_EC);
 656         sge_cmd = (3ULL << 56) | FW_RI_SGEEC_START << 8 | 3;
 657         wqe->sge_cmd = htobe64(sge_cmd);
 658         wqe->ctx1 = htobe64(ctx1);
 659         wqe->ctx0 = htobe64(ctx0);
 660         CTR3(KTR_IW_CXGB, "CtrlQP dma_addr 0x%llx workq %p size %d",
 661              (unsigned long long) rdev_p->ctrl_qp.dma_addr,
 662              rdev_p->ctrl_qp.workq, 1 << T3_CTRL_QP_SIZE_LOG2);
 663         m_set_priority(m, CPL_PRIORITY_CONTROL);
 664         m_set_sgl(m, NULL);
 665         m_set_sgllen(m, 0);
 666         return (cxgb_ofld_send(rdev_p->t3cdev_p, m));
 667 err:
 668         m_free(m);
 669         return err;
 670 }
 671
 672 static int
 673 cxio_hal_destroy_ctrl_qp(struct cxio_rdev *rdev_p)
 674 {
 675 #if 0
 676
 677         dma_free_coherent(&(rdev_p->rnic_info.pdev),
 678                           (1UL << T3_CTRL_QP_SIZE_LOG2)
 679                           * sizeof(union t3_wr), rdev_p->ctrl_qp.workq,
 680             /* pci_unmap_addr(&rdev_p->ctrl_qp, mapping)*/ 0);
 681 #else
 682         contigfree(rdev_p->ctrl_qp.workq,(1UL << T3_CTRL_QP_SIZE_LOG2)
 683             * sizeof(union t3_wr), M_DEVBUF);
 684 #endif
 685         return cxio_hal_clear_qp_ctx(rdev_p, T3_CTRL_QP_ID);
 686 }
 687
 688 /* write len bytes of data into addr (32B aligned address)
 689  * If data is NULL, clear len byte of memory to zero.
 690  * caller aquires the ctrl_qp lock before the call
 691  */
 692 static int
 693 cxio_hal_ctrl_qp_write_mem(struct cxio_rdev *rdev_p, u32 addr,
 694                                       u32 len, void *data, int completion)
 695 {
 696         u32 i, nr_wqe, copy_len;
 697         u8 *copy_data;
 698         u8 wr_len, utx_len;     /* lenght in 8 byte flit */
 699         enum t3_wr_flags flag;
 700         __be64 *wqe;
 701         u64 utx_cmd;
 702         addr &= 0x7FFFFFF;
 703         nr_wqe = len % 96 ? len / 96 + 1 : len / 96;    /* 96B max per WQE */
 704         CTR6(KTR_IW_CXGB, "cxio_hal_ctrl_qp_write_mem wptr 0x%x rptr 0x%x len %d, nr_wqe %d data %p addr 0x%0x",
 705              rdev_p->ctrl_qp.wptr, rdev_p->ctrl_qp.rptr, len,
 706              nr_wqe, data, addr);
 707         utx_len = 3;            /* in 32B unit */
 708         for (i = 0; i < nr_wqe; i++) {
 709                 if (Q_FULL(rdev_p->ctrl_qp.rptr, rdev_p->ctrl_qp.wptr,
 710                            T3_CTRL_QP_SIZE_LOG2)) {
 711                         CTR4(KTR_IW_CXGB, "%s ctrl_qp full wtpr 0x%0x rptr 0x%0x, "
 712                              "wait for more space i %d", __FUNCTION__,
 713                              rdev_p->ctrl_qp.wptr, rdev_p->ctrl_qp.rptr, i);
 714                         if (cxio_wait(&rdev_p->ctrl_qp,
 715                                 &rdev_p->ctrl_qp.lock,
 716                                 !Q_FULL(rdev_p->ctrl_qp.rptr,
 717                                         rdev_p->ctrl_qp.wptr,
 718                                         T3_CTRL_QP_SIZE_LOG2))) {
 719                                 CTR1(KTR_IW_CXGB, "%s ctrl_qp workq interrupted",
 720                                      __FUNCTION__);
 721                                 return (-ERESTART);
 722                         }
 723                         CTR2(KTR_IW_CXGB, "%s ctrl_qp wakeup, continue posting work request "
 724                              "i %d", __FUNCTION__, i);
 725                 }
 726                 wqe = (__be64 *)(rdev_p->ctrl_qp.workq + (rdev_p->ctrl_qp.wptr %
 727                                                 (1 << T3_CTRL_QP_SIZE_LOG2)));
 728                 flag = 0;
 729                 if (i == (nr_wqe - 1)) {
 730                         /* last WQE */
 731                         flag = completion ? T3_COMPLETION_FLAG : 0;
 732                         if (len % 32)
 733                                 utx_len = len / 32 + 1;
 734                         else
 735                                 utx_len = len / 32;
 736                 }
 737
 738                 /*
 739                  * Force a CQE to return the credit to the workq in case
 740                  * we posted more than half the max QP size of WRs
 741                  */
 742                 if ((i != 0) &&
 743                     (i % (((1 << T3_CTRL_QP_SIZE_LOG2)) >> 1) == 0)) {
 744                         flag = T3_COMPLETION_FLAG;
 745                         CTR2(KTR_IW_CXGB, "%s force completion at i %d", __FUNCTION__, i);
 746                 }
 747
 748                 /* build the utx mem command */
 749                 wqe += (sizeof(struct t3_bypass_wr) >> 3);
 750                 utx_cmd = (T3_UTX_MEM_WRITE << 28) | (addr + i * 3);
 751                 utx_cmd <<= 32;
 752                 utx_cmd |= (utx_len << 28) | ((utx_len << 2) + 1);
 753                 *wqe = htobe64(utx_cmd);
 754                 wqe++;
 755                 copy_data = (u8 *) data + i * 96;
 756                 copy_len = len > 96 ? 96 : len;
 757
 758                 /* clear memory content if data is NULL */
 759                 if (data)
 760                         memcpy(wqe, copy_data, copy_len);
 761                 else
 762                         memset(wqe, 0, copy_len);
 763                 if (copy_len % 32)
 764                         memset(((u8 *) wqe) + copy_len, 0,
 765                                32 - (copy_len % 32));
 766                 wr_len = ((sizeof(struct t3_bypass_wr)) >> 3) + 1 +
 767                          (utx_len << 2);
 768                 wqe = (__be64 *)(rdev_p->ctrl_qp.workq + (rdev_p->ctrl_qp.wptr %
 769                               (1 << T3_CTRL_QP_SIZE_LOG2)));
 770
 771                 /* wptr in the WRID[31:0] */
 772                 ((union t3_wrid *)(wqe+1))->id0.low = rdev_p->ctrl_qp.wptr;
 773
 774                 /*
 775                  * This must be the last write with a memory barrier
 776                  * for the genbit
 777                  */
 778                 build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_BP, flag,
 779                                Q_GENBIT(rdev_p->ctrl_qp.wptr,
 780                                         T3_CTRL_QP_SIZE_LOG2), T3_CTRL_QP_ID,
 781                                wr_len);
 782                 if (flag == T3_COMPLETION_FLAG)
 783                         ring_doorbell(rdev_p->ctrl_qp.doorbell, T3_CTRL_QP_ID);
 784
 785                 len -= 96;
 786                 rdev_p->ctrl_qp.wptr++;
 787         }
 788         return 0;
 789 }
 790
 791 /* IN: stag key, pdid, perm, zbva, to, len, page_size, pbl, and pbl_size
 792  * OUT: stag index, actual pbl_size, pbl_addr allocated.
 793  * TBD: shared memory region support
 794  */
 795 static int
 796 __cxio_tpt_op(struct cxio_rdev *rdev_p, u32 reset_tpt_entry,
 797                          u32 *stag, u8 stag_state, u32 pdid,
 798                          enum tpt_mem_type type, enum tpt_mem_perm perm,
 799                          u32 zbva, u64 to, u32 len, u8 page_size, __be64 *pbl,
 800                          u32 *pbl_size, u32 *pbl_addr)
 801 {
 802         int err;
 803         struct tpt_entry tpt;
 804         u32 stag_idx;
 805         u32 wptr;
 806         int rereg = (*stag != T3_STAG_UNSET);
 807
 808         stag_state = stag_state > 0;
 809         stag_idx = (*stag) >> 8;
 810
 811         if ((!reset_tpt_entry) && !(*stag != T3_STAG_UNSET)) {
 812                 stag_idx = cxio_hal_get_stag(rdev_p->rscp);
 813                 if (!stag_idx)
 814                         return (-ENOMEM);
 815                 *stag = (stag_idx << 8) | ((*stag) & 0xFF);
 816         }
 817         CTR5(KTR_IW_CXGB, "%s stag_state 0x%0x type 0x%0x pdid 0x%0x, stag_idx 0x%x",
 818              __FUNCTION__, stag_state, type, pdid, stag_idx);
 819
 820         if (reset_tpt_entry)
 821                 cxio_hal_pblpool_free(rdev_p, *pbl_addr, *pbl_size << 3);
 822         else if (!rereg) {
 823                 *pbl_addr = cxio_hal_pblpool_alloc(rdev_p, *pbl_size << 3);
 824                 if (!*pbl_addr) {
 825                         return (-ENOMEM);
 826                 }
 827         }
 828
 829         mtx_lock(&rdev_p->ctrl_qp.lock);
 830
 831         /* write PBL first if any - update pbl only if pbl list exist */
 832         if (pbl) {
 833
 834                 CTR4(KTR_IW_CXGB, "%s *pdb_addr 0x%x, pbl_base 0x%x, pbl_size %d",
 835                      __FUNCTION__, *pbl_addr, rdev_p->rnic_info.pbl_base,
 836                      *pbl_size);
 837                 err = cxio_hal_ctrl_qp_write_mem(rdev_p,
 838                                 (*pbl_addr >> 5),
 839                                 (*pbl_size << 3), pbl, 0);
 840                 if (err)
 841                         goto ret;
 842         }
 843
 844         /* write TPT entry */
 845         if (reset_tpt_entry)
 846                 memset(&tpt, 0, sizeof(tpt));
 847         else {
 848                 tpt.valid_stag_pdid = htobe32(F_TPT_VALID |
 849                                 V_TPT_STAG_KEY((*stag) & M_TPT_STAG_KEY) |
 850                                 V_TPT_STAG_STATE(stag_state) |
 851                                 V_TPT_STAG_TYPE(type) | V_TPT_PDID(pdid));
 852                 PANIC_IF(page_size >= 28);
 853                 tpt.flags_pagesize_qpid = htobe32(V_TPT_PERM(perm) |
 854                                 F_TPT_MW_BIND_ENABLE |
 855                                 V_TPT_ADDR_TYPE((zbva ? TPT_ZBTO : TPT_VATO)) |
 856                                 V_TPT_PAGE_SIZE(page_size));
 857                 tpt.rsvd_pbl_addr = reset_tpt_entry ? 0 :
 858                                     htobe32(V_TPT_PBL_ADDR(PBL_OFF(rdev_p, *pbl_addr)>>3));
 859                 tpt.len = htobe32(len);
 860                 tpt.va_hi = htobe32((u32) (to >> 32));
 861                 tpt.va_low_or_fbo = htobe32((u32) (to & 0xFFFFFFFFULL));
 862                 tpt.rsvd_bind_cnt_or_pstag = 0;
 863                 tpt.rsvd_pbl_size = reset_tpt_entry ? 0 :
 864                                   htobe32(V_TPT_PBL_SIZE((*pbl_size) >> 2));
 865         }
 866         err = cxio_hal_ctrl_qp_write_mem(rdev_p,
 867                                        stag_idx +
 868                                        (rdev_p->rnic_info.tpt_base >> 5),
 869                                        sizeof(tpt), &tpt, 1);
 870
 871         /* release the stag index to free pool */
 872         if (reset_tpt_entry)
 873                 cxio_hal_put_stag(rdev_p->rscp, stag_idx);
 874 ret:
 875         wptr = rdev_p->ctrl_qp.wptr;
 876         mtx_unlock(&rdev_p->ctrl_qp.lock);
 877         if (!err)
 878                 if (cxio_wait(&rdev_p->ctrl_qp,
 879                         &rdev_p->ctrl_qp.lock,
 880                         SEQ32_GE(rdev_p->ctrl_qp.rptr, wptr)))
 881                         return (-ERESTART);
 882         return err;
 883 }
 884
 885 int
 886 cxio_register_phys_mem(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid,
 887                            enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len,
 888                            u8 page_size, __be64 *pbl, u32 *pbl_size,
 889                            u32 *pbl_addr)
 890 {
 891         *stag = T3_STAG_UNSET;
 892         return __cxio_tpt_op(rdev_p, 0, stag, 1, pdid, TPT_NON_SHARED_MR, perm,
 893                              zbva, to, len, page_size, pbl, pbl_size, pbl_addr);
 894 }
 895
 896 int
 897 cxio_reregister_phys_mem(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid,
 898                            enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len,
 899                            u8 page_size, __be64 *pbl, u32 *pbl_size,
 900                            u32 *pbl_addr)
 901 {
 902         return __cxio_tpt_op(rdev_p, 0, stag, 1, pdid, TPT_NON_SHARED_MR, perm,
 903                              zbva, to, len, page_size, pbl, pbl_size, pbl_addr);
 904 }
 905
 906 int
 907 cxio_dereg_mem(struct cxio_rdev *rdev_p, u32 stag, u32 pbl_size,
 908                    u32 pbl_addr)
 909 {
 910         return __cxio_tpt_op(rdev_p, 1, &stag, 0, 0, 0, 0, 0, 0ULL, 0, 0, NULL,
 911                              &pbl_size, &pbl_addr);
 912 }
 913
 914 int
 915 cxio_allocate_window(struct cxio_rdev *rdev_p, u32 * stag, u32 pdid)
 916 {
 917         u32 pbl_size = 0;
 918         *stag = T3_STAG_UNSET;
 919         return __cxio_tpt_op(rdev_p, 0, stag, 0, pdid, TPT_MW, 0, 0, 0ULL, 0, 0,
 920                              NULL, &pbl_size, NULL);
 921 }
 922
 923 int
 924 cxio_deallocate_window(struct cxio_rdev *rdev_p, u32 stag)
 925 {
 926         return __cxio_tpt_op(rdev_p, 1, &stag, 0, 0, 0, 0, 0, 0ULL, 0, 0, NULL,
 927                              NULL, NULL);
 928 }
 929
 930 int
 931 cxio_rdma_init(struct cxio_rdev *rdev_p, struct t3_rdma_init_attr *attr)
 932 {
 933         struct t3_rdma_init_wr *wqe;
 934         struct mbuf *m = m_gethdr(MT_DATA, M_NOWAIT);
 935         if (m == NULL)
 936                 return (-ENOMEM);
 937         CTR2(KTR_IW_CXGB, "%s rdev_p %p", __FUNCTION__, rdev_p);
 938         wqe = mtod(m, struct t3_rdma_init_wr *);
 939         m->m_len = m->m_pkthdr.len = sizeof(*wqe);
 940         wqe->wrh.op_seop_flags = htobe32(V_FW_RIWR_OP(T3_WR_INIT));
 941         wqe->wrh.gen_tid_len = htobe32(V_FW_RIWR_TID(attr->tid) |
 942                                            V_FW_RIWR_LEN(sizeof(*wqe) >> 3));
 943         wqe->wrid.id1 = 0;
 944         wqe->qpid = htobe32(attr->qpid);
 945         wqe->pdid = htobe32(attr->pdid);
 946         wqe->scqid = htobe32(attr->scqid);
 947         wqe->rcqid = htobe32(attr->rcqid);
 948         wqe->rq_addr = htobe32(attr->rq_addr - rdev_p->rnic_info.rqt_base);
 949         wqe->rq_size = htobe32(attr->rq_size);
 950         wqe->mpaattrs = attr->mpaattrs;
 951         wqe->qpcaps = attr->qpcaps;
 952         wqe->ulpdu_size = htobe16(attr->tcp_emss);
 953         wqe->flags = htobe32(attr->flags);
 954         wqe->ord = htobe32(attr->ord);
 955         wqe->ird = htobe32(attr->ird);
 956         wqe->qp_dma_addr = htobe64(attr->qp_dma_addr);
 957         wqe->qp_dma_size = htobe32(attr->qp_dma_size);
 958         wqe->irs = htobe32(attr->irs);
 959         m_set_priority(m, 0);   /* 0=>ToeQ; 1=>CtrlQ */
 960         m_set_sgl(m, NULL);
 961         m_set_sgllen(m, 0);
 962         return (cxgb_ofld_send(rdev_p->t3cdev_p, m));
 963 }
 964
 965 void
 966 cxio_register_ev_cb(cxio_hal_ev_callback_func_t ev_cb)
 967 {
 968         cxio_ev_cb = ev_cb;
 969 }
 970
 971 void
 972 cxio_unregister_ev_cb(cxio_hal_ev_callback_func_t ev_cb)
 973 {
 974         cxio_ev_cb = NULL;
 975 }
 976
 977 static int
 978 cxio_hal_ev_handler(struct t3cdev *t3cdev_p, struct mbuf *m)
 979 {
 980         static int cnt;
 981         struct cxio_rdev *rdev_p = NULL;
 982         struct respQ_msg_t *rsp_msg = (struct respQ_msg_t *) m->m_data;
 983
 984         CTR6(KTR_IW_CXGB, "%s cq_id 0x%x cq_ptr 0x%x genbit %0x overflow %0x an %0x",
 985              __FUNCTION__, RSPQ_CQID(rsp_msg), RSPQ_CQPTR(rsp_msg),
 986              RSPQ_GENBIT(rsp_msg), RSPQ_OVERFLOW(rsp_msg), RSPQ_AN(rsp_msg));
 987         CTR4(KTR_IW_CXGB, "se %0x notify %0x cqbranch %0x creditth %0x",
 988              RSPQ_SE(rsp_msg), RSPQ_NOTIFY(rsp_msg), RSPQ_CQBRANCH(rsp_msg),
 989              RSPQ_CREDIT_THRESH(rsp_msg));
 990         CTR4(KTR_IW_CXGB, "CQE: QPID 0x%0x type 0x%0x status 0x%0x opcode %d",
 991              CQE_QPID(rsp_msg->cqe),
 992              CQE_TYPE(rsp_msg->cqe), CQE_STATUS(rsp_msg->cqe),
 993              CQE_OPCODE(rsp_msg->cqe));
 994         CTR3(KTR_IW_CXGB, "len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x",
 995              CQE_LEN(rsp_msg->cqe), CQE_WRID_HI(rsp_msg->cqe), CQE_WRID_LOW(rsp_msg->cqe));
 996         rdev_p = (struct cxio_rdev *)t3cdev_p->ulp;
 997         if (!rdev_p) {
 998                 CTR2(KTR_IW_CXGB, "%s called by t3cdev %p with null ulp", __FUNCTION__,
 999                      t3cdev_p);
1000                 return 0;
1001         }
1002         if (CQE_QPID(rsp_msg->cqe) == T3_CTRL_QP_ID) {
1003                 mtx_lock(&rdev_p->ctrl_qp.lock);
1004                 rdev_p->ctrl_qp.rptr = CQE_WRID_LOW(rsp_msg->cqe) + 1;
1005                 wakeup(&rdev_p->ctrl_qp);
1006                 mtx_unlock(&rdev_p->ctrl_qp.lock);
1007                 m_free(m);
1008         } else if (CQE_QPID(rsp_msg->cqe) == 0xfff8)
1009                 m_free(m);
1010         else if (cxio_ev_cb)
1011                 (*cxio_ev_cb) (rdev_p, m);
1012         else
1013                 m_free(m);
1014         cnt++;
1015         return 0;
1016 }
1017
1018 /* Caller takes care of locking if needed */
1019 int
1020 cxio_rdev_open(struct cxio_rdev *rdev_p)
1021 {
1022         struct ifnet *ifp;
1023         int err = 0;
1024
1025         if (strlen(rdev_p->dev_name)) {
1026                 if (cxio_hal_find_rdev_by_name(rdev_p->dev_name)) {
1027                         return (-EBUSY);
1028                 }
1029                 ifp = rdev_p->ifp;
1030                 if (ifp == NULL)
1031                         return (-EINVAL);
1032                 if_free(ifp);
1033         } else if (rdev_p->t3cdev_p) {
1034                 if (cxio_hal_find_rdev_by_t3cdev(rdev_p->t3cdev_p))
1035                         return (-EBUSY);
1036                 ifp = rdev_p->t3cdev_p->lldev;
1037                 strncpy(rdev_p->dev_name, rdev_p->t3cdev_p->name,
1038                         T3_MAX_DEV_NAME_LEN);
1039         } else {
1040                 CTR1(KTR_IW_CXGB, "%s t3cdev_p or dev_name must be set", __FUNCTION__);
1041                 return (-EINVAL);
1042         }
1043
1044         TAILQ_INSERT_TAIL(&rdev_list, rdev_p, entry);
1045
1046         CTR2(KTR_IW_CXGB, "%s opening rnic dev %s", __FUNCTION__, rdev_p->dev_name);
1047         memset(&rdev_p->ctrl_qp, 0, sizeof(rdev_p->ctrl_qp));
1048         if (!rdev_p->t3cdev_p)
1049                 rdev_p->t3cdev_p = T3CDEV(ifp);
1050         rdev_p->t3cdev_p->ulp = (void *) rdev_p;
1051         err = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_GET_PARAMS,
1052                                          &(rdev_p->rnic_info));
1053         if (err) {
1054                 log(LOG_ERR, "%s t3cdev_p(%p)->ctl returned error %d.\n",
1055                      __FUNCTION__, rdev_p->t3cdev_p, err);
1056                 goto err1;
1057         }
1058         err = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, GET_PORTS,
1059                                     &(rdev_p->port_info));
1060         if (err) {
1061                 log(LOG_ERR, "%s t3cdev_p(%p)->ctl returned error %d.\n",
1062                      __FUNCTION__, rdev_p->t3cdev_p, err);
1063                 goto err1;
1064         }
1065
1066         /*
1067          * qpshift is the number of bits to shift the qpid left in order
1068          * to get the correct address of the doorbell for that qp.
1069          */
1070         cxio_init_ucontext(rdev_p, &rdev_p->uctx);
1071         rdev_p->qpshift = PAGE_SHIFT -
1072                           ilog2(65536 >>
1073                                     ilog2(rdev_p->rnic_info.udbell_len >>
1074                                               PAGE_SHIFT));
1075         rdev_p->qpnr = rdev_p->rnic_info.udbell_len >> PAGE_SHIFT;
1076         rdev_p->qpmask = (65536 >> ilog2(rdev_p->qpnr)) - 1;
1077         CTR4(KTR_IW_CXGB, "cxio_rdev_open rnic %s info: tpt_base 0x%0x tpt_top 0x%0x num stags %d",
1078              rdev_p->dev_name, rdev_p->rnic_info.tpt_base,
1079              rdev_p->rnic_info.tpt_top, cxio_num_stags(rdev_p));
1080         CTR4(KTR_IW_CXGB, "pbl_base 0x%0x pbl_top 0x%0x rqt_base 0x%0x, rqt_top 0x%0x",
1081              rdev_p->rnic_info.pbl_base,
1082              rdev_p->rnic_info.pbl_top, rdev_p->rnic_info.rqt_base,
1083              rdev_p->rnic_info.rqt_top);
1084         CTR6(KTR_IW_CXGB, "udbell_len 0x%0x udbell_physbase 0x%lx kdb_addr %p qpshift %lu "
1085              "qpnr %d qpmask 0x%x",
1086              rdev_p->rnic_info.udbell_len,
1087              rdev_p->rnic_info.udbell_physbase, rdev_p->rnic_info.kdb_addr,
1088              rdev_p->qpshift, rdev_p->qpnr, rdev_p->qpmask);
1089
1090         err = cxio_hal_init_ctrl_qp(rdev_p);
1091         if (err) {
1092                 log(LOG_ERR, "%s error %d initializing ctrl_qp.\n",
1093                        __FUNCTION__, err);
1094                 goto err1;
1095         }
1096         err = cxio_hal_init_resource(rdev_p, cxio_num_stags(rdev_p), 0,
1097                                      0, T3_MAX_NUM_QP, T3_MAX_NUM_CQ,
1098                                      T3_MAX_NUM_PD);
1099         if (err) {
1100                 log(LOG_ERR, "%s error %d initializing hal resources.\n",
1101                        __FUNCTION__, err);
1102                 goto err2;
1103         }
1104         err = cxio_hal_pblpool_create(rdev_p);
1105         if (err) {
1106                 log(LOG_ERR, "%s error %d initializing pbl mem pool.\n",
1107                        __FUNCTION__, err);
1108                 goto err3;
1109         }
1110         err = cxio_hal_rqtpool_create(rdev_p);
1111         if (err) {
1112                 log(LOG_ERR, "%s error %d initializing rqt mem pool.\n",
1113                        __FUNCTION__, err);
1114                 goto err4;
1115         }
1116         return 0;
1117 err4:
1118         cxio_hal_pblpool_destroy(rdev_p);
1119 err3:
1120         cxio_hal_destroy_resource(rdev_p->rscp);
1121 err2:
1122         cxio_hal_destroy_ctrl_qp(rdev_p);
1123 err1:
1124         TAILQ_REMOVE(&rdev_list, rdev_p, entry);
1125         return err;
1126 }
1127
1128 void
1129 cxio_rdev_close(struct cxio_rdev *rdev_p)
1130 {
1131         if (rdev_p) {
1132                 cxio_hal_pblpool_destroy(rdev_p);
1133                 cxio_hal_rqtpool_destroy(rdev_p);
1134                 TAILQ_REMOVE(&rdev_list, rdev_p, entry);
1135                 rdev_p->t3cdev_p->ulp = NULL;
1136                 cxio_hal_destroy_ctrl_qp(rdev_p);
1137                 cxio_hal_destroy_resource(rdev_p->rscp);
1138         }
1139 }
1140
1141 int
1142 cxio_hal_init(void)
1143 {
1144         TAILQ_INIT(&rdev_list);
1145 #ifdef needed
1146         if (cxio_hal_init_rhdl_resource(T3_MAX_NUM_RI))
1147                 return (-ENOMEM);
1148 #endif
1149         t3_register_cpl_handler(CPL_ASYNC_NOTIF, cxio_hal_ev_handler);
1150         return 0;
1151 }
1152
1153 void
1154 cxio_hal_exit(void)
1155 {
1156         struct cxio_rdev *rdev, *tmp;
1157
1158         t3_register_cpl_handler(CPL_ASYNC_NOTIF, NULL);
1159         TAILQ_FOREACH_SAFE(rdev, &rdev_list, entry, tmp)
1160                 cxio_rdev_close(rdev);
1161 #ifdef needed
1162         cxio_hal_destroy_rhdl_resource();
1163 #endif
1164 }
1165
1166 static void
1167 flush_completed_wrs(struct t3_wq *wq, struct t3_cq *cq)
1168 {
1169         struct t3_swsq *sqp;
1170         __u32 ptr = wq->sq_rptr;
1171         int count = Q_COUNT(wq->sq_rptr, wq->sq_wptr);
1172
1173         sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2);
1174         while (count--)
1175                 if (!sqp->signaled) {
1176                         ptr++;
1177                         sqp = wq->sq + Q_PTR2IDX(ptr,  wq->sq_size_log2);
1178                 } else if (sqp->complete) {
1179
1180                         /*
1181                          * Insert this completed cqe into the swcq.
1182                          */
1183                         CTR3(KTR_IW_CXGB, "%s moving cqe into swcq sq idx %ld cq idx %ld",
1184                              __FUNCTION__, Q_PTR2IDX(ptr,  wq->sq_size_log2),
1185                              Q_PTR2IDX(cq->sw_wptr, cq->size_log2));
1186                         sqp->cqe.header |= htonl(V_CQE_SWCQE(1));
1187                         *(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2))
1188                                 = sqp->cqe;
1189                         cq->sw_wptr++;
1190                         sqp->signaled = 0;
1191                         break;
1192                 } else
1193                         break;
1194 }
1195
1196 static void
1197 create_read_req_cqe(struct t3_wq *wq, struct t3_cqe *hw_cqe,
1198                                 struct t3_cqe *read_cqe)
1199 {
1200         read_cqe->u.scqe.wrid_hi = wq->oldest_read->sq_wptr;
1201         read_cqe->len = wq->oldest_read->read_len;
1202         read_cqe->header = htonl(V_CQE_QPID(CQE_QPID(*hw_cqe)) |
1203                                  V_CQE_SWCQE(SW_CQE(*hw_cqe)) |
1204                                  V_CQE_OPCODE(T3_READ_REQ) |
1205                                  V_CQE_TYPE(1));
1206 }
1207
1208 /*
1209  * Return a ptr to the next read wr in the SWSQ or NULL.
1210  */
1211 static void
1212 advance_oldest_read(struct t3_wq *wq)
1213 {
1214
1215         u32 rptr = wq->oldest_read - wq->sq + 1;
1216         u32 wptr = Q_PTR2IDX(wq->sq_wptr, wq->sq_size_log2);
1217
1218         while (Q_PTR2IDX(rptr, wq->sq_size_log2) != wptr) {
1219                 wq->oldest_read = wq->sq + Q_PTR2IDX(rptr, wq->sq_size_log2);
1220
1221                 if (wq->oldest_read->opcode == T3_READ_REQ)
1222                         return;
1223                 rptr++;
1224         }
1225         wq->oldest_read = NULL;
1226 }
1227
1228 /*
1229  * cxio_poll_cq
1230  *
1231  * Caller must:
1232  *     check the validity of the first CQE,
1233  *     supply the wq assicated with the qpid.
1234  *
1235  * credit: cq credit to return to sge.
1236  * cqe_flushed: 1 iff the CQE is flushed.
1237  * cqe: copy of the polled CQE.
1238  *
1239  * return value:
1240  *     0       CQE returned,
1241  *    -1       CQE skipped, try again.
1242  */
1243 int
1244 cxio_poll_cq(struct t3_wq *wq, struct t3_cq *cq, struct t3_cqe *cqe,
1245                      u8 *cqe_flushed, u64 *cookie, u32 *credit)
1246 {
1247         int ret = 0;
1248         struct t3_cqe *hw_cqe, read_cqe;
1249
1250         *cqe_flushed = 0;
1251         *credit = 0;
1252         hw_cqe = cxio_next_cqe(cq);
1253
1254         CTR5(KTR_IW_CXGB, "cxio_poll_cq CQE OOO %d qpid 0x%0x genbit %d type %d status 0x%0x",
1255              CQE_OOO(*hw_cqe), CQE_QPID(*hw_cqe),
1256              CQE_GENBIT(*hw_cqe), CQE_TYPE(*hw_cqe), CQE_STATUS(*hw_cqe));
1257         CTR4(KTR_IW_CXGB, "opcode 0x%0x len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x",
1258              CQE_OPCODE(*hw_cqe), CQE_LEN(*hw_cqe), CQE_WRID_HI(*hw_cqe),
1259              CQE_WRID_LOW(*hw_cqe));
1260
1261         /*
1262          * skip cqe's not affiliated with a QP.
1263          */
1264         if (wq == NULL) {
1265                 ret = -1;
1266                 goto skip_cqe;
1267         }
1268
1269         /*
1270          * Gotta tweak READ completions:
1271          *      1) the cqe doesn't contain the sq_wptr from the wr.
1272          *      2) opcode not reflected from the wr.
1273          *      3) read_len not reflected from the wr.
1274          *      4) cq_type is RQ_TYPE not SQ_TYPE.
1275          */
1276         if (RQ_TYPE(*hw_cqe) && (CQE_OPCODE(*hw_cqe) == T3_READ_RESP)) {
1277
1278                 /*
1279                  * Don't write to the HWCQ, so create a new read req CQE
1280                  * in local memory.
1281                  */
1282                 create_read_req_cqe(wq, hw_cqe, &read_cqe);
1283                 hw_cqe = &read_cqe;
1284                 advance_oldest_read(wq);
1285         }
1286
1287         /*
1288          * T3A: Discard TERMINATE CQEs.
1289          */
1290         if (CQE_OPCODE(*hw_cqe) == T3_TERMINATE) {
1291                 ret = -1;
1292                 wq->error = 1;
1293                 goto skip_cqe;
1294         }
1295
1296         if (CQE_STATUS(*hw_cqe) || wq->error) {
1297                 *cqe_flushed = wq->error;
1298                 wq->error = 1;
1299
1300                 /*
1301                  * T3A inserts errors into the CQE.  We cannot return
1302                  * these as work completions.
1303                  */
1304                 /* incoming write failures */
1305                 if ((CQE_OPCODE(*hw_cqe) == T3_RDMA_WRITE)
1306                      && RQ_TYPE(*hw_cqe)) {
1307                         ret = -1;
1308                         goto skip_cqe;
1309                 }
1310                 /* incoming read request failures */
1311                 if ((CQE_OPCODE(*hw_cqe) == T3_READ_RESP) && SQ_TYPE(*hw_cqe)) {
1312                         ret = -1;
1313                         goto skip_cqe;
1314                 }
1315
1316                 /* incoming SEND with no receive posted failures */
1317                 if ((CQE_OPCODE(*hw_cqe) == T3_SEND) && RQ_TYPE(*hw_cqe) &&
1318                     Q_EMPTY(wq->rq_rptr, wq->rq_wptr)) {
1319                         ret = -1;
1320                         goto skip_cqe;
1321                 }
1322                 goto proc_cqe;
1323         }
1324
1325         /*
1326          * RECV completion.
1327          */
1328         if (RQ_TYPE(*hw_cqe)) {
1329
1330                 /*
1331                  * HW only validates 4 bits of MSN.  So we must validate that
1332                  * the MSN in the SEND is the next expected MSN.  If its not,
1333                  * then we complete this with TPT_ERR_MSN and mark the wq in
1334                  * error.
1335                  */
1336                 if (__predict_false((CQE_WRID_MSN(*hw_cqe) != (wq->rq_rptr + 1)))) {
1337                         wq->error = 1;
1338                         hw_cqe->header |= htonl(V_CQE_STATUS(TPT_ERR_MSN));
1339                         goto proc_cqe;
1340                 }
1341                 goto proc_cqe;
1342         }
1343
1344         /*
1345          * If we get here its a send completion.
1346          *
1347          * Handle out of order completion. These get stuffed
1348          * in the SW SQ. Then the SW SQ is walked to move any
1349          * now in-order completions into the SW CQ.  This handles
1350          * 2 cases:
1351          *      1) reaping unsignaled WRs when the first subsequent
1352          *         signaled WR is completed.
1353          *      2) out of order read completions.
1354          */
1355         if (!SW_CQE(*hw_cqe) && (CQE_WRID_SQ_WPTR(*hw_cqe) != wq->sq_rptr)) {
1356                 struct t3_swsq *sqp;
1357
1358                 CTR2(KTR_IW_CXGB, "%s out of order completion going in swsq at idx %ld",
1359                      __FUNCTION__,
1360                      Q_PTR2IDX(CQE_WRID_SQ_WPTR(*hw_cqe), wq->sq_size_log2));
1361                 sqp = wq->sq +
1362                       Q_PTR2IDX(CQE_WRID_SQ_WPTR(*hw_cqe), wq->sq_size_log2);
1363                 sqp->cqe = *hw_cqe;
1364                 sqp->complete = 1;
1365                 ret = -1;
1366                 goto flush_wq;
1367         }
1368
1369 proc_cqe:
1370         *cqe = *hw_cqe;
1371
1372         /*
1373          * Reap the associated WR(s) that are freed up with this
1374          * completion.
1375          */
1376         if (SQ_TYPE(*hw_cqe)) {
1377                 wq->sq_rptr = CQE_WRID_SQ_WPTR(*hw_cqe);
1378                 CTR2(KTR_IW_CXGB, "%s completing sq idx %ld", __FUNCTION__,
1379                      Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2));
1380                 *cookie = (wq->sq +
1381                            Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2))->wr_id;
1382                 wq->sq_rptr++;
1383         } else {
1384                 CTR2(KTR_IW_CXGB, "%s completing rq idx %ld", __FUNCTION__,
1385                      Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2));
1386                 *cookie = *(wq->rq + Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2));
1387                 wq->rq_rptr++;
1388         }
1389
1390 flush_wq:
1391         /*
1392          * Flush any completed cqes that are now in-order.
1393          */
1394         flush_completed_wrs(wq, cq);
1395
1396 skip_cqe:
1397         if (SW_CQE(*hw_cqe)) {
1398                 CTR4(KTR_IW_CXGB, "%s cq %p cqid 0x%x skip sw cqe sw_rptr 0x%x",
1399                      __FUNCTION__, cq, cq->cqid, cq->sw_rptr);
1400                 ++cq->sw_rptr;
1401         } else {
1402                 CTR4(KTR_IW_CXGB, "%s cq %p cqid 0x%x skip hw cqe rptr 0x%x",
1403                      __FUNCTION__, cq, cq->cqid, cq->rptr);
1404                 ++cq->rptr;
1405
1406                 /*
1407                  * T3A: compute credits.
1408                  */
1409                 if (((cq->rptr - cq->wptr) > (1 << (cq->size_log2 - 1)))
1410                     || ((cq->rptr - cq->wptr) >= 128)) {
1411                         *credit = cq->rptr - cq->wptr;
1412                         cq->wptr = cq->rptr;
1413                 }
1414         }
1415         return ret;
1416 }
1417
1418