sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_hal.c

   1
   2 /**************************************************************************
   3
   4 Copyright (c) 2007, Chelsio Inc.
   5 All rights reserved.
   6
   7 Redistribution and use in source and binary forms, with or without
   8 modification, are permitted provided that the following conditions are met:
   9
  10  1. Redistributions of source code must retain the above copyright notice,
  11     this list of conditions and the following disclaimer.
  12
  13  2. Neither the name of the Chelsio Corporation nor the names of its
  14     contributors may be used to endorse or promote products derived from
  15     this software without specific prior written permission.
  16
  17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  18 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  21 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  22 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  23 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  24 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  25 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  26 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27 POSSIBILITY OF SUCH DAMAGE.
  28
  29 ***************************************************************************/
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34 #include <sys/systm.h>
  35 #include <sys/kernel.h>
  36 #include <sys/bus.h>
  37 #include <sys/module.h>
  38 #include <sys/pciio.h>
  39 #include <sys/conf.h>
  40 #include <machine/bus.h>
  41 #include <machine/resource.h>
  42 #include <sys/bus_dma.h>
  43 #include <sys/rman.h>
  44 #include <sys/ioccom.h>
  45 #include <sys/mbuf.h>
  46 #include <sys/rwlock.h>
  47 #include <sys/linker.h>
  48 #include <sys/firmware.h>
  49 #include <sys/socket.h>
  50 #include <sys/sockio.h>
  51 #include <sys/smp.h>
  52 #include <sys/sysctl.h>
  53 #include <sys/syslog.h>
  54 #include <sys/queue.h>
  55 #include <sys/taskqueue.h>
  56 #include <sys/proc.h>
  57 #include <sys/queue.h>
  58
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61
  62 #include <netinet/in.h>
  63
  64 #include <contrib/rdma/ib_verbs.h>
  65
  66 #include <cxgb_include.h>
  67 #include <ulp/tom/cxgb_l2t.h>
  68 #include <ulp/iw_cxgb/iw_cxgb_wr.h>
  69 #include <ulp/iw_cxgb/iw_cxgb_hal.h>
  70 #include <ulp/iw_cxgb/iw_cxgb_provider.h>
  71 #include <ulp/iw_cxgb/iw_cxgb_cm.h>
  72 #include <ulp/iw_cxgb/iw_cxgb.h>
  73 #include <ulp/iw_cxgb/iw_cxgb_resource.h>
  74
  75 static TAILQ_HEAD( ,cxio_rdev) rdev_list;
  76 static cxio_hal_ev_callback_func_t cxio_ev_cb = NULL;
  77
  78 static struct cxio_rdev *
  79 cxio_hal_find_rdev_by_name(char *dev_name)
  80 {
  81         struct cxio_rdev *rdev;
  82
  83         TAILQ_FOREACH(rdev, &rdev_list, entry)
  84                 if (!strcmp(rdev->dev_name, dev_name))
  85                         return rdev;
  86         return NULL;
  87 }
  88
  89 struct cxio_rdev *
  90 cxio_hal_find_rdev_by_t3cdev(struct t3cdev *tdev)
  91 {
  92         struct cxio_rdev *rdev;
  93
  94         TAILQ_FOREACH(rdev, &rdev_list, entry)
  95                 if (rdev->t3cdev_p == tdev)
  96                         return rdev;
  97         return NULL;
  98 }
  99
 100 int
 101 cxio_hal_cq_op(struct cxio_rdev *rdev_p, struct t3_cq *cq,
 102                    enum t3_cq_opcode op, u32 credit)
 103 {
 104         int ret;
 105         struct t3_cqe *cqe;
 106         u32 rptr;
 107
 108         struct rdma_cq_op setup;
 109         setup.id = cq->cqid;
 110         setup.credits = (op == CQ_CREDIT_UPDATE) ? credit : 0;
 111         setup.op = op;
 112         ret = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_OP, &setup);
 113
 114         if ((ret < 0) || (op == CQ_CREDIT_UPDATE))
 115                 return (ret);
 116
 117         /*
 118          * If the rearm returned an index other than our current index,
 119          * then there might be CQE's in flight (being DMA'd).  We must wait
 120          * here for them to complete or the consumer can miss a notification.
 121          */
 122         if (Q_PTR2IDX((cq->rptr), cq->size_log2) != ret) {
 123                 int i=0;
 124
 125                 rptr = cq->rptr;
 126
 127                 /*
 128                  * Keep the generation correct by bumping rptr until it
 129                  * matches the index returned by the rearm - 1.
 130                  */
 131                 while (Q_PTR2IDX((rptr+1), cq->size_log2) != ret)
 132                         rptr++;
 133
 134                 /*
 135                  * Now rptr is the index for the (last) cqe that was
 136                  * in-flight at the time the HW rearmed the CQ.  We
 137                  * spin until that CQE is valid.
 138                  */
 139                 cqe = cq->queue + Q_PTR2IDX(rptr, cq->size_log2);
 140                 while (!CQ_VLD_ENTRY(rptr, cq->size_log2, cqe)) {
 141                         DELAY(1);
 142                         if (i++ > 1000000) {
 143                                 PANIC_IF(1);
 144                                 log(LOG_ERR, "%s: stalled rnic\n",
 145                                        rdev_p->dev_name);
 146                                 return (-EIO);
 147                         }
 148                 }
 149
 150                 return 1;
 151         }
 152
 153         return 0;
 154 }
 155
 156 static int
 157 cxio_hal_clear_cq_ctx(struct cxio_rdev *rdev_p, u32 cqid)
 158 {
 159         struct rdma_cq_setup setup;
 160         setup.id = cqid;
 161         setup.base_addr = 0;    /* NULL address */
 162         setup.size = 0;         /* disaable the CQ */
 163         setup.credits = 0;
 164         setup.credit_thres = 0;
 165         setup.ovfl_mode = 0;
 166         return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));
 167 }
 168
 169 static int
 170 cxio_hal_clear_qp_ctx(struct cxio_rdev *rdev_p, u32 qpid)
 171 {
 172         u64 sge_cmd;
 173         struct t3_modify_qp_wr *wqe;
 174         struct mbuf *m = m_gethdr(MT_DATA, M_NOWAIT);
 175         if (m == NULL) {
 176                 CTR1(KTR_IW_CXGB, "%s m_gethdr failed", __FUNCTION__);
 177                 return (-ENOMEM);
 178         }
 179         wqe = mtod(m, struct t3_modify_qp_wr *);
 180         m->m_len = m->m_pkthdr.len = sizeof(*wqe);
 181         memset(wqe, 0, sizeof(*wqe));
 182         build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD, 3, 0, qpid, 7);
 183         wqe->flags = htobe32(MODQP_WRITE_EC);
 184         sge_cmd = qpid << 8 | 3;
 185         wqe->sge_cmd = htobe64(sge_cmd);
 186         m_set_priority(m, CPL_PRIORITY_CONTROL);
 187         m_set_sgl(m, NULL);
 188         m_set_sgllen(m, 0);
 189         return (cxgb_ofld_send(rdev_p->t3cdev_p, m));
 190 }
 191
 192 int
 193 cxio_create_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)
 194 {
 195         struct rdma_cq_setup setup;
 196         int size = (1UL << (cq->size_log2)) * sizeof(struct t3_cqe);
 197
 198         cq->cqid = cxio_hal_get_cqid(rdev_p->rscp);
 199         if (!cq->cqid)
 200                 return (-ENOMEM);
 201         cq->sw_queue = malloc(size, M_DEVBUF, M_NOWAIT|M_ZERO);
 202         if (!cq->sw_queue)
 203                 return (-ENOMEM);
 204 #if 0
 205         cq->queue = dma_alloc_coherent(rdev_p->rnic_info.pdev,
 206                                              (1UL << (cq->size_log2)) *
 207                                              sizeof(struct t3_cqe),
 208                                              &(cq->dma_addr), M_NOWAIT);
 209 #else
 210         cq->queue = contigmalloc((1UL << (cq->size_log2))*sizeof(struct t3_cqe),
 211             M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 4096, 0);
 212         if (cq->queue)
 213                 cq->dma_addr = vtophys(cq->queue);
 214         else {
 215                 free(cq->sw_queue, M_DEVBUF);
 216                 return (-ENOMEM);
 217         }
 218 #endif
 219
 220 #ifdef notyet
 221         pci_unmap_addr_set(cq, mapping, cq->dma_addr);
 222 #endif
 223         memset(cq->queue, 0, size);
 224         setup.id = cq->cqid;
 225         setup.base_addr = (u64) (cq->dma_addr);
 226         setup.size = 1UL << cq->size_log2;
 227         setup.credits = 65535;
 228         setup.credit_thres = 1;
 229         if (rdev_p->t3cdev_p->type != T3A)
 230                 setup.ovfl_mode = 0;
 231         else
 232                 setup.ovfl_mode = 1;
 233         return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));
 234 }
 235
 236 int
 237 cxio_resize_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)
 238 {
 239         struct rdma_cq_setup setup;
 240         setup.id = cq->cqid;
 241         setup.base_addr = (u64) (cq->dma_addr);
 242         setup.size = 1UL << cq->size_log2;
 243         setup.credits = setup.size;
 244         setup.credit_thres = setup.size;        /* TBD: overflow recovery */
 245         setup.ovfl_mode = 1;
 246         return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));
 247 }
 248
 249 static u32
 250 get_qpid(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)
 251 {
 252         struct cxio_qpid *entry;
 253         u32 qpid;
 254         int i;
 255
 256         mtx_lock(&uctx->lock);
 257         if (!TAILQ_EMPTY(&uctx->qpids)) {
 258
 259                 entry = TAILQ_FIRST(&uctx->qpids);
 260                 TAILQ_REMOVE(&uctx->qpids, entry, entry);
 261                 qpid = entry->qpid;
 262                 free(entry, M_DEVBUF);
 263         } else {
 264                 qpid = cxio_hal_get_qpid(rdev_p->rscp);
 265                 if (!qpid)
 266                         goto out;
 267                 for (i = qpid+1; i & rdev_p->qpmask; i++) {
 268                         entry = malloc(sizeof *entry, M_DEVBUF, M_NOWAIT);
 269                         if (!entry)
 270                                 break;
 271                         entry->qpid = i;
 272                         TAILQ_INSERT_TAIL(&uctx->qpids, entry, entry);
 273                 }
 274         }
 275 out:
 276         mtx_unlock(&uctx->lock);
 277         CTR2(KTR_IW_CXGB, "%s qpid 0x%x", __FUNCTION__, qpid);
 278         return qpid;
 279 }
 280
 281 static void
 282 put_qpid(struct cxio_rdev *rdev_p, u32 qpid,
 283                      struct cxio_ucontext *uctx)
 284 {
 285         struct cxio_qpid *entry;
 286
 287         entry = malloc(sizeof *entry, M_DEVBUF, M_NOWAIT);
 288         CTR2(KTR_IW_CXGB, "%s qpid 0x%x", __FUNCTION__, qpid);
 289         entry->qpid = qpid;
 290         mtx_lock(&uctx->lock);
 291         TAILQ_INSERT_TAIL(&uctx->qpids, entry, entry);
 292         mtx_unlock(&uctx->lock);
 293 }
 294
 295 void
 296 cxio_release_ucontext(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)
 297 {
 298         struct cxio_qpid *pos, *tmp;
 299
 300         mtx_lock(&uctx->lock);
 301         TAILQ_FOREACH_SAFE(pos, &uctx->qpids, entry, tmp) {
 302                 TAILQ_REMOVE(&uctx->qpids, pos, entry);
 303                 if (!(pos->qpid & rdev_p->qpmask))
 304                         cxio_hal_put_qpid(rdev_p->rscp, pos->qpid);
 305                 free(pos, M_DEVBUF);
 306         }
 307         mtx_unlock(&uctx->lock);
 308 }
 309
 310 void
 311 cxio_init_ucontext(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx)
 312 {
 313         TAILQ_INIT(&uctx->qpids);
 314         mtx_init(&uctx->lock, "cxio uctx", NULL, MTX_DEF|MTX_DUPOK);
 315 }
 316
 317 int
 318 cxio_create_qp(struct cxio_rdev *rdev_p, u32 kernel_domain,
 319     struct t3_wq *wq, struct cxio_ucontext *uctx)
 320 {
 321         int depth = 1UL << wq->size_log2;
 322         int rqsize = 1UL << wq->rq_size_log2;
 323
 324         wq->qpid = get_qpid(rdev_p, uctx);
 325         if (!wq->qpid)
 326                 return (-ENOMEM);
 327
 328         wq->rq = malloc(depth * sizeof(u64), M_DEVBUF, M_NOWAIT|M_ZERO);
 329         if (!wq->rq)
 330                 goto err1;
 331
 332         wq->rq_addr = cxio_hal_rqtpool_alloc(rdev_p, rqsize);
 333         if (!wq->rq_addr)
 334                 goto err2;
 335
 336         wq->sq = malloc(depth * sizeof(struct t3_swsq), M_DEVBUF, M_NOWAIT|M_ZERO);
 337         if (!wq->sq)
 338                 goto err3;
 339 #if 0
 340         wq->queue = dma_alloc_coherent(rdev_p->rnic_info.pdev,
 341                                              depth * sizeof(union t3_wr),
 342                                              &(wq->dma_addr), M_NOWAIT);
 343 #else
 344         wq->queue = contigmalloc(depth *sizeof(union t3_wr),
 345             M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 4096, 0);
 346         if (wq->queue)
 347                 wq->dma_addr = vtophys(wq->queue);
 348
 349 #endif
 350         if (!wq->queue)
 351                 goto err4;
 352
 353         memset(wq->queue, 0, depth * sizeof(union t3_wr));
 354 #ifdef notyet
 355         pci_unmap_addr_set(wq, mapping, wq->dma_addr);
 356 #endif
 357         wq->doorbell = rdev_p->rnic_info.kdb_addr;
 358         if (!kernel_domain)
 359                 wq->udb = (u64)rdev_p->rnic_info.udbell_physbase +
 360                                         (wq->qpid << rdev_p->qpshift);
 361         CTR4(KTR_IW_CXGB, "%s qpid 0x%x doorbell 0x%p udb 0x%llx", __FUNCTION__,
 362              wq->qpid, wq->doorbell, (unsigned long long) wq->udb);
 363         return 0;
 364 err4:
 365         free(wq->sq, M_DEVBUF);
 366 err3:
 367         cxio_hal_rqtpool_free(rdev_p, wq->rq_addr, rqsize);
 368 err2:
 369         free(wq->rq, M_DEVBUF);
 370 err1:
 371         put_qpid(rdev_p, wq->qpid, uctx);
 372         return (-ENOMEM);
 373 }
 374
 375 int
 376 cxio_destroy_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq)
 377 {
 378         int err;
 379         err = cxio_hal_clear_cq_ctx(rdev_p, cq->cqid);
 380         free(cq->sw_queue, M_DEVBUF);
 381 #if 0
 382         dma_free_coherent(&(rdev_p->rnic_info.pdev),
 383             (1UL << (cq->size_log2))
 384                           * sizeof(struct t3_cqe), cq->queue,
 385             /* pci_unmap_addr(cq, mapping)*/  0);
 386 #else
 387         contigfree(cq->queue,(1UL << (cq->size_log2))
 388             * sizeof(struct t3_cqe), M_DEVBUF);
 389 #endif
 390         cxio_hal_put_cqid(rdev_p->rscp, cq->cqid);
 391         return err;
 392 }
 393
 394 int
 395 cxio_destroy_qp(struct cxio_rdev *rdev_p, struct t3_wq *wq,
 396     struct cxio_ucontext *uctx)
 397 {
 398
 399 #if 0
 400         dma_free_coherent(&(rdev_p->rnic_info.pdev),
 401                           (1UL << (wq->size_log2))
 402                           * sizeof(union t3_wr), wq->queue,
 403             /* pci_unmap_addr(wq, mapping)*/ 0);
 404 #else
 405         contigfree(wq->queue, (1UL << (wq->size_log2))
 406             * sizeof(union t3_wr), M_DEVBUF);
 407 #endif
 408         free(wq->sq, M_DEVBUF);
 409         cxio_hal_rqtpool_free(rdev_p, wq->rq_addr, (1UL << wq->rq_size_log2));
 410         free(wq->rq, M_DEVBUF);
 411         put_qpid(rdev_p, wq->qpid, uctx);
 412         return 0;
 413 }
 414
 415 static void
 416 insert_recv_cqe(struct t3_wq *wq, struct t3_cq *cq)
 417 {
 418         struct t3_cqe cqe;
 419
 420         CTR5(KTR_IW_CXGB, "%s wq %p cq %p sw_rptr 0x%x sw_wptr 0x%x", __FUNCTION__,
 421              wq, cq, cq->sw_rptr, cq->sw_wptr);
 422         memset(&cqe, 0, sizeof(cqe));
 423         cqe.header = htobe32(V_CQE_STATUS(TPT_ERR_SWFLUSH) |
 424                                  V_CQE_OPCODE(T3_SEND) |
 425                                  V_CQE_TYPE(0) |
 426                                  V_CQE_SWCQE(1) |
 427                                  V_CQE_QPID(wq->qpid) |
 428                                  V_CQE_GENBIT(Q_GENBIT(cq->sw_wptr,
 429                                                        cq->size_log2)));
 430         *(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2)) = cqe;
 431         cq->sw_wptr++;
 432 }
 433
 434 void
 435 cxio_flush_rq(struct t3_wq *wq, struct t3_cq *cq, int count)
 436 {
 437         u32 ptr;
 438
 439         CTR3(KTR_IW_CXGB, "%s wq %p cq %p", __FUNCTION__, wq, cq);
 440
 441         /* flush RQ */
 442         CTR4(KTR_IW_CXGB, "%s rq_rptr %u rq_wptr %u skip count %u", __FUNCTION__,
 443             wq->rq_rptr, wq->rq_wptr, count);
 444         ptr = wq->rq_rptr + count;
 445         while (ptr++ != wq->rq_wptr)
 446                 insert_recv_cqe(wq, cq);
 447 }
 448
 449 static void
 450 insert_sq_cqe(struct t3_wq *wq, struct t3_cq *cq,
 451                           struct t3_swsq *sqp)
 452 {
 453         struct t3_cqe cqe;
 454
 455         CTR5(KTR_IW_CXGB, "%s wq %p cq %p sw_rptr 0x%x sw_wptr 0x%x", __FUNCTION__,
 456              wq, cq, cq->sw_rptr, cq->sw_wptr);
 457         memset(&cqe, 0, sizeof(cqe));
 458         cqe.header = htobe32(V_CQE_STATUS(TPT_ERR_SWFLUSH) |
 459                                  V_CQE_OPCODE(sqp->opcode) |
 460                                  V_CQE_TYPE(1) |
 461                                  V_CQE_SWCQE(1) |
 462                                  V_CQE_QPID(wq->qpid) |
 463                                  V_CQE_GENBIT(Q_GENBIT(cq->sw_wptr,
 464                                                        cq->size_log2)));
 465         cqe.u.scqe.wrid_hi = sqp->sq_wptr;
 466
 467         *(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2)) = cqe;
 468         cq->sw_wptr++;
 469 }
 470
 471 void
 472 cxio_flush_sq(struct t3_wq *wq, struct t3_cq *cq, int count)
 473 {
 474         __u32 ptr;
 475         struct t3_swsq *sqp = wq->sq + Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2);
 476
 477         ptr = wq->sq_rptr + count;
 478         sqp += count;
 479         while (ptr != wq->sq_wptr) {
 480                 insert_sq_cqe(wq, cq, sqp);
 481                 sqp++;
 482                 ptr++;
 483         }
 484 }
 485
 486 /*
 487  * Move all CQEs from the HWCQ into the SWCQ.
 488  */
 489 void
 490 cxio_flush_hw_cq(struct t3_cq *cq)
 491 {
 492         struct t3_cqe *cqe, *swcqe;
 493
 494         CTR3(KTR_IW_CXGB, "%s cq %p cqid 0x%x", __FUNCTION__, cq, cq->cqid);
 495         cqe = cxio_next_hw_cqe(cq);
 496         while (cqe) {
 497                 CTR3(KTR_IW_CXGB, "%s flushing hwcq rptr 0x%x to swcq wptr 0x%x",
 498                      __FUNCTION__, cq->rptr, cq->sw_wptr);
 499                 swcqe = cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2);
 500                 *swcqe = *cqe;
 501                 swcqe->header |= htobe32(V_CQE_SWCQE(1));
 502                 cq->sw_wptr++;
 503                 cq->rptr++;
 504                 cqe = cxio_next_hw_cqe(cq);
 505         }
 506 }
 507
 508 static int cqe_completes_wr(struct t3_cqe *cqe, struct t3_wq *wq)
 509 {
 510         if (CQE_OPCODE(*cqe) == T3_TERMINATE)
 511                 return 0;
 512
 513         if ((CQE_OPCODE(*cqe) == T3_RDMA_WRITE) && RQ_TYPE(*cqe))
 514                 return 0;
 515
 516         if ((CQE_OPCODE(*cqe) == T3_READ_RESP) && SQ_TYPE(*cqe))
 517                 return 0;
 518
 519         if ((CQE_OPCODE(*cqe) == T3_SEND) && RQ_TYPE(*cqe) &&
 520             Q_EMPTY(wq->rq_rptr, wq->rq_wptr))
 521                 return 0;
 522
 523         return 1;
 524 }
 525
 526 void
 527 cxio_count_scqes(struct t3_cq *cq, struct t3_wq *wq, int *count)
 528 {
 529         struct t3_cqe *cqe;
 530         u32 ptr;
 531
 532         *count = 0;
 533         ptr = cq->sw_rptr;
 534         while (!Q_EMPTY(ptr, cq->sw_wptr)) {
 535                 cqe = cq->sw_queue + (Q_PTR2IDX(ptr, cq->size_log2));
 536                 if ((SQ_TYPE(*cqe) || (CQE_OPCODE(*cqe) == T3_READ_RESP)) &&
 537                     (CQE_QPID(*cqe) == wq->qpid))
 538                         (*count)++;
 539                 ptr++;
 540         }
 541         CTR3(KTR_IW_CXGB, "%s cq %p count %d", __FUNCTION__, cq, *count);
 542 }
 543
 544 void
 545 cxio_count_rcqes(struct t3_cq *cq, struct t3_wq *wq, int *count)
 546 {
 547         struct t3_cqe *cqe;
 548         u32 ptr;
 549
 550         *count = 0;
 551         CTR2(KTR_IW_CXGB, "%s count zero %d", __FUNCTION__, *count);
 552         ptr = cq->sw_rptr;
 553         while (!Q_EMPTY(ptr, cq->sw_wptr)) {
 554                 cqe = cq->sw_queue + (Q_PTR2IDX(ptr, cq->size_log2));
 555                 if (RQ_TYPE(*cqe) && (CQE_OPCODE(*cqe) != T3_READ_RESP) &&
 556                     (CQE_QPID(*cqe) == wq->qpid) && cqe_completes_wr(cqe, wq))
 557                         (*count)++;
 558                 ptr++;
 559         }
 560         CTR3(KTR_IW_CXGB, "%s cq %p count %d", __FUNCTION__, cq, *count);
 561 }
 562
 563 static int
 564 cxio_hal_init_ctrl_cq(struct cxio_rdev *rdev_p)
 565 {
 566         struct rdma_cq_setup setup;
 567         setup.id = 0;
 568         setup.base_addr = 0;    /* NULL address */
 569         setup.size = 1;         /* enable the CQ */
 570         setup.credits = 0;
 571
 572         /* force SGE to redirect to RspQ and interrupt */
 573         setup.credit_thres = 0;
 574         setup.ovfl_mode = 1;
 575         return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup));
 576 }
 577
 578 static int
 579 cxio_hal_init_ctrl_qp(struct cxio_rdev *rdev_p)
 580 {
 581         int err;
 582         u64 sge_cmd, ctx0, ctx1;
 583         u64 base_addr;
 584         struct t3_modify_qp_wr *wqe;
 585         struct mbuf *m;
 586
 587         m = m_gethdr(MT_DATA, M_NOWAIT);
 588         if (m == NULL) {
 589                 CTR1(KTR_IW_CXGB, "%s m_gethdr failed", __FUNCTION__);
 590                 return (-ENOMEM);
 591         }
 592         err = cxio_hal_init_ctrl_cq(rdev_p);
 593         if (err) {
 594                 CTR2(KTR_IW_CXGB, "%s err %d initializing ctrl_cq", __FUNCTION__, err);
 595                 goto err;
 596         }
 597 #if 0
 598         rdev_p->ctrl_qp.workq = dma_alloc_coherent(
 599                 rdev_p->rnic_info.pdev,
 600                     (1 << T3_CTRL_QP_SIZE_LOG2) *
 601                     sizeof(union t3_wr),
 602                     &(rdev_p->ctrl_qp.dma_addr),
 603                     M_NOWAIT);
 604 #else
 605         rdev_p->ctrl_qp.workq = contigmalloc((1 << T3_CTRL_QP_SIZE_LOG2)
 606             *sizeof(union t3_wr), M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 4096, 0);
 607         if (rdev_p->ctrl_qp.workq)
 608                 rdev_p->ctrl_qp.dma_addr = vtophys(rdev_p->ctrl_qp.workq);
 609
 610 #endif
 611
 612         if (!rdev_p->ctrl_qp.workq) {
 613                 CTR1(KTR_IW_CXGB, "%s dma_alloc_coherent failed", __FUNCTION__);
 614                 err = -ENOMEM;
 615                 goto err;
 616         }
 617 #if 0
 618         pci_unmap_addr_set(&rdev_p->ctrl_qp, mapping,
 619                            rdev_p->ctrl_qp.dma_addr);
 620 #endif
 621         rdev_p->ctrl_qp.doorbell = (void /*__iomem */ *)rdev_p->rnic_info.kdb_addr;
 622         memset(rdev_p->ctrl_qp.workq, 0,
 623                (1 << T3_CTRL_QP_SIZE_LOG2) * sizeof(union t3_wr));
 624
 625         mtx_init(&rdev_p->ctrl_qp.lock, "ctl-qp lock", NULL, MTX_DEF|MTX_DUPOK);
 626
 627         /* update HW Ctrl QP context */
 628         base_addr = rdev_p->ctrl_qp.dma_addr;
 629         base_addr >>= 12;
 630         ctx0 = (V_EC_SIZE((1 << T3_CTRL_QP_SIZE_LOG2)) |
 631                 V_EC_BASE_LO((u32) base_addr & 0xffff));
 632         ctx0 <<= 32;
 633         ctx0 |= V_EC_CREDITS(FW_WR_NUM);
 634         base_addr >>= 16;
 635         ctx1 = (u32) base_addr;
 636         base_addr >>= 32;
 637         ctx1 |= ((u64) (V_EC_BASE_HI((u32) base_addr & 0xf) | V_EC_RESPQ(0) |
 638                         V_EC_TYPE(0) | V_EC_GEN(1) |
 639                         V_EC_UP_TOKEN(T3_CTL_QP_TID) | F_EC_VALID)) << 32;
 640         wqe = mtod(m, struct t3_modify_qp_wr *);
 641         m->m_len = m->m_pkthdr.len = sizeof(*wqe);
 642         memset(wqe, 0, sizeof(*wqe));
 643         build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD, 3, 0,
 644                        T3_CTL_QP_TID, 7);
 645         wqe->flags = htobe32(MODQP_WRITE_EC);
 646         sge_cmd = (3ULL << 56) | FW_RI_SGEEC_START << 8 | 3;
 647         wqe->sge_cmd = htobe64(sge_cmd);
 648         wqe->ctx1 = htobe64(ctx1);
 649         wqe->ctx0 = htobe64(ctx0);
 650         CTR3(KTR_IW_CXGB, "CtrlQP dma_addr 0x%llx workq %p size %d",
 651              (unsigned long long) rdev_p->ctrl_qp.dma_addr,
 652              rdev_p->ctrl_qp.workq, 1 << T3_CTRL_QP_SIZE_LOG2);
 653         m_set_priority(m, CPL_PRIORITY_CONTROL);
 654         m_set_sgl(m, NULL);
 655         m_set_sgllen(m, 0);
 656         return (cxgb_ofld_send(rdev_p->t3cdev_p, m));
 657 err:
 658         m_free(m);
 659         return err;
 660 }
 661
 662 static int
 663 cxio_hal_destroy_ctrl_qp(struct cxio_rdev *rdev_p)
 664 {
 665 #if 0
 666
 667         dma_free_coherent(&(rdev_p->rnic_info.pdev),
 668                           (1UL << T3_CTRL_QP_SIZE_LOG2)
 669                           * sizeof(union t3_wr), rdev_p->ctrl_qp.workq,
 670             /* pci_unmap_addr(&rdev_p->ctrl_qp, mapping)*/ 0);
 671 #else
 672         contigfree(rdev_p->ctrl_qp.workq,(1UL << T3_CTRL_QP_SIZE_LOG2)
 673             * sizeof(union t3_wr), M_DEVBUF);
 674 #endif
 675         return cxio_hal_clear_qp_ctx(rdev_p, T3_CTRL_QP_ID);
 676 }
 677
 678 /* write len bytes of data into addr (32B aligned address)
 679  * If data is NULL, clear len byte of memory to zero.
 680  * caller aquires the ctrl_qp lock before the call
 681  */
 682 static int
 683 cxio_hal_ctrl_qp_write_mem(struct cxio_rdev *rdev_p, u32 addr,
 684                                       u32 len, void *data, int completion)
 685 {
 686         u32 i, nr_wqe, copy_len;
 687         u8 *copy_data;
 688         u8 wr_len, utx_len;     /* lenght in 8 byte flit */
 689         enum t3_wr_flags flag;
 690         __be64 *wqe;
 691         u64 utx_cmd;
 692         addr &= 0x7FFFFFF;
 693         nr_wqe = len % 96 ? len / 96 + 1 : len / 96;    /* 96B max per WQE */
 694         CTR6(KTR_IW_CXGB, "cxio_hal_ctrl_qp_write_mem wptr 0x%x rptr 0x%x len %d, nr_wqe %d data %p addr 0x%0x",
 695              rdev_p->ctrl_qp.wptr, rdev_p->ctrl_qp.rptr, len,
 696              nr_wqe, data, addr);
 697         utx_len = 3;            /* in 32B unit */
 698         for (i = 0; i < nr_wqe; i++) {
 699                 if (Q_FULL(rdev_p->ctrl_qp.rptr, rdev_p->ctrl_qp.wptr,
 700                            T3_CTRL_QP_SIZE_LOG2)) {
 701                         CTR4(KTR_IW_CXGB, "%s ctrl_qp full wtpr 0x%0x rptr 0x%0x, "
 702                              "wait for more space i %d", __FUNCTION__,
 703                              rdev_p->ctrl_qp.wptr, rdev_p->ctrl_qp.rptr, i);
 704                         if (cxio_wait(&rdev_p->ctrl_qp,
 705                                 &rdev_p->ctrl_qp.lock,
 706                                 !Q_FULL(rdev_p->ctrl_qp.rptr,
 707                                         rdev_p->ctrl_qp.wptr,
 708                                         T3_CTRL_QP_SIZE_LOG2))) {
 709                                 CTR1(KTR_IW_CXGB, "%s ctrl_qp workq interrupted",
 710                                      __FUNCTION__);
 711                                 return (-ERESTART);
 712                         }
 713                         CTR2(KTR_IW_CXGB, "%s ctrl_qp wakeup, continue posting work request "
 714                              "i %d", __FUNCTION__, i);
 715                 }
 716                 wqe = (__be64 *)(rdev_p->ctrl_qp.workq + (rdev_p->ctrl_qp.wptr %
 717                                                 (1 << T3_CTRL_QP_SIZE_LOG2)));
 718                 flag = 0;
 719                 if (i == (nr_wqe - 1)) {
 720                         /* last WQE */
 721                         flag = completion ? T3_COMPLETION_FLAG : 0;
 722                         if (len % 32)
 723                                 utx_len = len / 32 + 1;
 724                         else
 725                                 utx_len = len / 32;
 726                 }
 727
 728                 /*
 729                  * Force a CQE to return the credit to the workq in case
 730                  * we posted more than half the max QP size of WRs
 731                  */
 732                 if ((i != 0) &&
 733                     (i % (((1 << T3_CTRL_QP_SIZE_LOG2)) >> 1) == 0)) {
 734                         flag = T3_COMPLETION_FLAG;
 735                         CTR2(KTR_IW_CXGB, "%s force completion at i %d", __FUNCTION__, i);
 736                 }
 737
 738                 /* build the utx mem command */
 739                 wqe += (sizeof(struct t3_bypass_wr) >> 3);
 740                 utx_cmd = (T3_UTX_MEM_WRITE << 28) | (addr + i * 3);
 741                 utx_cmd <<= 32;
 742                 utx_cmd |= (utx_len << 28) | ((utx_len << 2) + 1);
 743                 *wqe = htobe64(utx_cmd);
 744                 wqe++;
 745                 copy_data = (u8 *) data + i * 96;
 746                 copy_len = len > 96 ? 96 : len;
 747
 748                 /* clear memory content if data is NULL */
 749                 if (data)
 750                         memcpy(wqe, copy_data, copy_len);
 751                 else
 752                         memset(wqe, 0, copy_len);
 753                 if (copy_len % 32)
 754                         memset(((u8 *) wqe) + copy_len, 0,
 755                                32 - (copy_len % 32));
 756                 wr_len = ((sizeof(struct t3_bypass_wr)) >> 3) + 1 +
 757                          (utx_len << 2);
 758                 wqe = (__be64 *)(rdev_p->ctrl_qp.workq + (rdev_p->ctrl_qp.wptr %
 759                               (1 << T3_CTRL_QP_SIZE_LOG2)));
 760
 761                 /* wptr in the WRID[31:0] */
 762                 ((union t3_wrid *)(wqe+1))->id0.low = rdev_p->ctrl_qp.wptr;
 763
 764                 /*
 765                  * This must be the last write with a memory barrier
 766                  * for the genbit
 767                  */
 768                 build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_BP, flag,
 769                                Q_GENBIT(rdev_p->ctrl_qp.wptr,
 770                                         T3_CTRL_QP_SIZE_LOG2), T3_CTRL_QP_ID,
 771                                wr_len);
 772                 if (flag == T3_COMPLETION_FLAG)
 773                         ring_doorbell(rdev_p->ctrl_qp.doorbell, T3_CTRL_QP_ID);
 774
 775                 len -= 96;
 776                 rdev_p->ctrl_qp.wptr++;
 777         }
 778         return 0;
 779 }
 780
 781 /* IN: stag key, pdid, perm, zbva, to, len, page_size, pbl, and pbl_size
 782  * OUT: stag index, actual pbl_size, pbl_addr allocated.
 783  * TBD: shared memory region support
 784  */
 785 static int
 786 __cxio_tpt_op(struct cxio_rdev *rdev_p, u32 reset_tpt_entry,
 787                          u32 *stag, u8 stag_state, u32 pdid,
 788                          enum tpt_mem_type type, enum tpt_mem_perm perm,
 789                          u32 zbva, u64 to, u32 len, u8 page_size, __be64 *pbl,
 790                          u32 *pbl_size, u32 *pbl_addr)
 791 {
 792         int err;
 793         struct tpt_entry tpt;
 794         u32 stag_idx;
 795         u32 wptr;
 796         int rereg = (*stag != T3_STAG_UNSET);
 797
 798         stag_state = stag_state > 0;
 799         stag_idx = (*stag) >> 8;
 800
 801         if ((!reset_tpt_entry) && !(*stag != T3_STAG_UNSET)) {
 802                 stag_idx = cxio_hal_get_stag(rdev_p->rscp);
 803                 if (!stag_idx)
 804                         return (-ENOMEM);
 805                 *stag = (stag_idx << 8) | ((*stag) & 0xFF);
 806         }
 807         CTR5(KTR_IW_CXGB, "%s stag_state 0x%0x type 0x%0x pdid 0x%0x, stag_idx 0x%x",
 808              __FUNCTION__, stag_state, type, pdid, stag_idx);
 809
 810         if (reset_tpt_entry)
 811                 cxio_hal_pblpool_free(rdev_p, *pbl_addr, *pbl_size << 3);
 812         else if (!rereg) {
 813                 *pbl_addr = cxio_hal_pblpool_alloc(rdev_p, *pbl_size << 3);
 814                 if (!*pbl_addr) {
 815                         return (-ENOMEM);
 816                 }
 817         }
 818
 819         mtx_lock(&rdev_p->ctrl_qp.lock);
 820
 821         /* write PBL first if any - update pbl only if pbl list exist */
 822         if (pbl) {
 823
 824                 CTR4(KTR_IW_CXGB, "%s *pdb_addr 0x%x, pbl_base 0x%x, pbl_size %d",
 825                      __FUNCTION__, *pbl_addr, rdev_p->rnic_info.pbl_base,
 826                      *pbl_size);
 827                 err = cxio_hal_ctrl_qp_write_mem(rdev_p,
 828                                 (*pbl_addr >> 5),
 829                                 (*pbl_size << 3), pbl, 0);
 830                 if (err)
 831                         goto ret;
 832         }
 833
 834         /* write TPT entry */
 835         if (reset_tpt_entry)
 836                 memset(&tpt, 0, sizeof(tpt));
 837         else {
 838                 tpt.valid_stag_pdid = htobe32(F_TPT_VALID |
 839                                 V_TPT_STAG_KEY((*stag) & M_TPT_STAG_KEY) |
 840                                 V_TPT_STAG_STATE(stag_state) |
 841                                 V_TPT_STAG_TYPE(type) | V_TPT_PDID(pdid));
 842                 PANIC_IF(page_size >= 28);
 843                 tpt.flags_pagesize_qpid = htobe32(V_TPT_PERM(perm) |
 844                                 F_TPT_MW_BIND_ENABLE |
 845                                 V_TPT_ADDR_TYPE((zbva ? TPT_ZBTO : TPT_VATO)) |
 846                                 V_TPT_PAGE_SIZE(page_size));
 847                 tpt.rsvd_pbl_addr = reset_tpt_entry ? 0 :
 848                                     htobe32(V_TPT_PBL_ADDR(PBL_OFF(rdev_p, *pbl_addr)>>3));
 849                 tpt.len = htobe32(len);
 850                 tpt.va_hi = htobe32((u32) (to >> 32));
 851                 tpt.va_low_or_fbo = htobe32((u32) (to & 0xFFFFFFFFULL));
 852                 tpt.rsvd_bind_cnt_or_pstag = 0;
 853                 tpt.rsvd_pbl_size = reset_tpt_entry ? 0 :
 854                                   htobe32(V_TPT_PBL_SIZE((*pbl_size) >> 2));
 855         }
 856         err = cxio_hal_ctrl_qp_write_mem(rdev_p,
 857                                        stag_idx +
 858                                        (rdev_p->rnic_info.tpt_base >> 5),
 859                                        sizeof(tpt), &tpt, 1);
 860
 861         /* release the stag index to free pool */
 862         if (reset_tpt_entry)
 863                 cxio_hal_put_stag(rdev_p->rscp, stag_idx);
 864 ret:
 865         wptr = rdev_p->ctrl_qp.wptr;
 866         mtx_unlock(&rdev_p->ctrl_qp.lock);
 867         if (!err)
 868                 if (cxio_wait(&rdev_p->ctrl_qp,
 869                         &rdev_p->ctrl_qp.lock,
 870                         SEQ32_GE(rdev_p->ctrl_qp.rptr, wptr)))
 871                         return (-ERESTART);
 872         return err;
 873 }
 874
 875 int
 876 cxio_register_phys_mem(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid,
 877                            enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len,
 878                            u8 page_size, __be64 *pbl, u32 *pbl_size,
 879                            u32 *pbl_addr)
 880 {
 881         *stag = T3_STAG_UNSET;
 882         return __cxio_tpt_op(rdev_p, 0, stag, 1, pdid, TPT_NON_SHARED_MR, perm,
 883                              zbva, to, len, page_size, pbl, pbl_size, pbl_addr);
 884 }
 885
 886 int
 887 cxio_reregister_phys_mem(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid,
 888                            enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len,
 889                            u8 page_size, __be64 *pbl, u32 *pbl_size,
 890                            u32 *pbl_addr)
 891 {
 892         return __cxio_tpt_op(rdev_p, 0, stag, 1, pdid, TPT_NON_SHARED_MR, perm,
 893                              zbva, to, len, page_size, pbl, pbl_size, pbl_addr);
 894 }
 895
 896 int
 897 cxio_dereg_mem(struct cxio_rdev *rdev_p, u32 stag, u32 pbl_size,
 898                    u32 pbl_addr)
 899 {
 900         return __cxio_tpt_op(rdev_p, 1, &stag, 0, 0, 0, 0, 0, 0ULL, 0, 0, NULL,
 901                              &pbl_size, &pbl_addr);
 902 }
 903
 904 int
 905 cxio_allocate_window(struct cxio_rdev *rdev_p, u32 * stag, u32 pdid)
 906 {
 907         u32 pbl_size = 0;
 908         *stag = T3_STAG_UNSET;
 909         return __cxio_tpt_op(rdev_p, 0, stag, 0, pdid, TPT_MW, 0, 0, 0ULL, 0, 0,
 910                              NULL, &pbl_size, NULL);
 911 }
 912
 913 int
 914 cxio_deallocate_window(struct cxio_rdev *rdev_p, u32 stag)
 915 {
 916         return __cxio_tpt_op(rdev_p, 1, &stag, 0, 0, 0, 0, 0, 0ULL, 0, 0, NULL,
 917                              NULL, NULL);
 918 }
 919
 920 int
 921 cxio_rdma_init(struct cxio_rdev *rdev_p, struct t3_rdma_init_attr *attr)
 922 {
 923         struct t3_rdma_init_wr *wqe;
 924         struct mbuf *m = m_gethdr(MT_DATA, M_NOWAIT);
 925         if (m == NULL)
 926                 return (-ENOMEM);
 927         CTR2(KTR_IW_CXGB, "%s rdev_p %p", __FUNCTION__, rdev_p);
 928         wqe = mtod(m, struct t3_rdma_init_wr *);
 929         m->m_len = m->m_pkthdr.len = sizeof(*wqe);
 930         wqe->wrh.op_seop_flags = htobe32(V_FW_RIWR_OP(T3_WR_INIT));
 931         wqe->wrh.gen_tid_len = htobe32(V_FW_RIWR_TID(attr->tid) |
 932                                            V_FW_RIWR_LEN(sizeof(*wqe) >> 3));
 933         wqe->wrid.id1 = 0;
 934         wqe->qpid = htobe32(attr->qpid);
 935         wqe->pdid = htobe32(attr->pdid);
 936         wqe->scqid = htobe32(attr->scqid);
 937         wqe->rcqid = htobe32(attr->rcqid);
 938         wqe->rq_addr = htobe32(attr->rq_addr - rdev_p->rnic_info.rqt_base);
 939         wqe->rq_size = htobe32(attr->rq_size);
 940         wqe->mpaattrs = attr->mpaattrs;
 941         wqe->qpcaps = attr->qpcaps;
 942         wqe->ulpdu_size = htobe16(attr->tcp_emss);
 943         wqe->flags = htobe32(attr->flags);
 944         wqe->ord = htobe32(attr->ord);
 945         wqe->ird = htobe32(attr->ird);
 946         wqe->qp_dma_addr = htobe64(attr->qp_dma_addr);
 947         wqe->qp_dma_size = htobe32(attr->qp_dma_size);
 948         wqe->irs = htobe32(attr->irs);
 949         m_set_priority(m, 0);   /* 0=>ToeQ; 1=>CtrlQ */
 950         m_set_sgl(m, NULL);
 951         m_set_sgllen(m, 0);
 952         return (cxgb_ofld_send(rdev_p->t3cdev_p, m));
 953 }
 954
 955 void
 956 cxio_register_ev_cb(cxio_hal_ev_callback_func_t ev_cb)
 957 {
 958         cxio_ev_cb = ev_cb;
 959 }
 960
 961 void
 962 cxio_unregister_ev_cb(cxio_hal_ev_callback_func_t ev_cb)
 963 {
 964         cxio_ev_cb = NULL;
 965 }
 966
 967 static int
 968 cxio_hal_ev_handler(struct t3cdev *t3cdev_p, struct mbuf *m)
 969 {
 970         static int cnt;
 971         struct cxio_rdev *rdev_p = NULL;
 972         struct respQ_msg_t *rsp_msg = (struct respQ_msg_t *) m->m_data;
 973
 974         CTR6(KTR_IW_CXGB, "%s cq_id 0x%x cq_ptr 0x%x genbit %0x overflow %0x an %0x",
 975              __FUNCTION__, RSPQ_CQID(rsp_msg), RSPQ_CQPTR(rsp_msg),
 976              RSPQ_GENBIT(rsp_msg), RSPQ_OVERFLOW(rsp_msg), RSPQ_AN(rsp_msg));
 977         CTR4(KTR_IW_CXGB, "se %0x notify %0x cqbranch %0x creditth %0x",
 978              RSPQ_SE(rsp_msg), RSPQ_NOTIFY(rsp_msg), RSPQ_CQBRANCH(rsp_msg),
 979              RSPQ_CREDIT_THRESH(rsp_msg));
 980         CTR4(KTR_IW_CXGB, "CQE: QPID 0x%0x type 0x%0x status 0x%0x opcode %d",
 981              CQE_QPID(rsp_msg->cqe),
 982              CQE_TYPE(rsp_msg->cqe), CQE_STATUS(rsp_msg->cqe),
 983              CQE_OPCODE(rsp_msg->cqe));
 984         CTR3(KTR_IW_CXGB, "len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x",
 985              CQE_LEN(rsp_msg->cqe), CQE_WRID_HI(rsp_msg->cqe), CQE_WRID_LOW(rsp_msg->cqe));
 986         rdev_p = (struct cxio_rdev *)t3cdev_p->ulp;
 987         if (!rdev_p) {
 988                 CTR2(KTR_IW_CXGB, "%s called by t3cdev %p with null ulp", __FUNCTION__,
 989                      t3cdev_p);
 990                 return 0;
 991         }
 992         if (CQE_QPID(rsp_msg->cqe) == T3_CTRL_QP_ID) {
 993                 mtx_lock(&rdev_p->ctrl_qp.lock);
 994                 rdev_p->ctrl_qp.rptr = CQE_WRID_LOW(rsp_msg->cqe) + 1;
 995                 wakeup(&rdev_p->ctrl_qp);
 996                 mtx_unlock(&rdev_p->ctrl_qp.lock);
 997                 m_free(m);
 998         } else if (CQE_QPID(rsp_msg->cqe) == 0xfff8)
 999                 m_free(m);
1000         else if (cxio_ev_cb)
1001                 (*cxio_ev_cb) (rdev_p, m);
1002         else
1003                 m_free(m);
1004         cnt++;
1005         return 0;
1006 }
1007
1008 /* Caller takes care of locking if needed */
1009 int
1010 cxio_rdev_open(struct cxio_rdev *rdev_p)
1011 {
1012         struct ifnet *ifp;
1013         int err = 0;
1014
1015         if (strlen(rdev_p->dev_name)) {
1016                 if (cxio_hal_find_rdev_by_name(rdev_p->dev_name)) {
1017                         return (-EBUSY);
1018                 }
1019                 ifp = rdev_p->ifp;
1020                 if (ifp == NULL)
1021                         return (-EINVAL);
1022                 if_free(ifp);
1023         } else if (rdev_p->t3cdev_p) {
1024                 if (cxio_hal_find_rdev_by_t3cdev(rdev_p->t3cdev_p))
1025                         return (-EBUSY);
1026                 ifp = rdev_p->t3cdev_p->lldev;
1027                 strncpy(rdev_p->dev_name, rdev_p->t3cdev_p->name,
1028                         T3_MAX_DEV_NAME_LEN);
1029         } else {
1030                 CTR1(KTR_IW_CXGB, "%s t3cdev_p or dev_name must be set", __FUNCTION__);
1031                 return (-EINVAL);
1032         }
1033
1034         TAILQ_INSERT_TAIL(&rdev_list, rdev_p, entry);
1035
1036         CTR2(KTR_IW_CXGB, "%s opening rnic dev %s", __FUNCTION__, rdev_p->dev_name);
1037         memset(&rdev_p->ctrl_qp, 0, sizeof(rdev_p->ctrl_qp));
1038         if (!rdev_p->t3cdev_p)
1039                 rdev_p->t3cdev_p = T3CDEV(ifp);
1040         rdev_p->t3cdev_p->ulp = (void *) rdev_p;
1041         err = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_GET_PARAMS,
1042                                          &(rdev_p->rnic_info));
1043         if (err) {
1044                 log(LOG_ERR, "%s t3cdev_p(%p)->ctl returned error %d.\n",
1045                      __FUNCTION__, rdev_p->t3cdev_p, err);
1046                 goto err1;
1047         }
1048         err = rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, GET_PORTS,
1049                                     &(rdev_p->port_info));
1050         if (err) {
1051                 log(LOG_ERR, "%s t3cdev_p(%p)->ctl returned error %d.\n",
1052                      __FUNCTION__, rdev_p->t3cdev_p, err);
1053                 goto err1;
1054         }
1055
1056         /*
1057          * qpshift is the number of bits to shift the qpid left in order
1058          * to get the correct address of the doorbell for that qp.
1059          */
1060         cxio_init_ucontext(rdev_p, &rdev_p->uctx);
1061         rdev_p->qpshift = PAGE_SHIFT -
1062                           ilog2(65536 >>
1063                                     ilog2(rdev_p->rnic_info.udbell_len >>
1064                                               PAGE_SHIFT));
1065         rdev_p->qpnr = rdev_p->rnic_info.udbell_len >> PAGE_SHIFT;
1066         rdev_p->qpmask = (65536 >> ilog2(rdev_p->qpnr)) - 1;
1067         CTR4(KTR_IW_CXGB, "cxio_rdev_open rnic %s info: tpt_base 0x%0x tpt_top 0x%0x num stags %d",
1068              rdev_p->dev_name, rdev_p->rnic_info.tpt_base,
1069              rdev_p->rnic_info.tpt_top, cxio_num_stags(rdev_p));
1070         CTR4(KTR_IW_CXGB, "pbl_base 0x%0x pbl_top 0x%0x rqt_base 0x%0x, rqt_top 0x%0x",
1071              rdev_p->rnic_info.pbl_base,
1072              rdev_p->rnic_info.pbl_top, rdev_p->rnic_info.rqt_base,
1073              rdev_p->rnic_info.rqt_top);
1074         CTR6(KTR_IW_CXGB, "udbell_len 0x%0x udbell_physbase 0x%lx kdb_addr %p qpshift %lu "
1075              "qpnr %d qpmask 0x%x",
1076              rdev_p->rnic_info.udbell_len,
1077              rdev_p->rnic_info.udbell_physbase, rdev_p->rnic_info.kdb_addr,
1078              rdev_p->qpshift, rdev_p->qpnr, rdev_p->qpmask);
1079
1080         err = cxio_hal_init_ctrl_qp(rdev_p);
1081         if (err) {
1082                 log(LOG_ERR, "%s error %d initializing ctrl_qp.\n",
1083                        __FUNCTION__, err);
1084                 goto err1;
1085         }
1086         err = cxio_hal_init_resource(rdev_p, cxio_num_stags(rdev_p), 0,
1087                                      0, T3_MAX_NUM_QP, T3_MAX_NUM_CQ,
1088                                      T3_MAX_NUM_PD);
1089         if (err) {
1090                 log(LOG_ERR, "%s error %d initializing hal resources.\n",
1091                        __FUNCTION__, err);
1092                 goto err2;
1093         }
1094         err = cxio_hal_pblpool_create(rdev_p);
1095         if (err) {
1096                 log(LOG_ERR, "%s error %d initializing pbl mem pool.\n",
1097                        __FUNCTION__, err);
1098                 goto err3;
1099         }
1100         err = cxio_hal_rqtpool_create(rdev_p);
1101         if (err) {
1102                 log(LOG_ERR, "%s error %d initializing rqt mem pool.\n",
1103                        __FUNCTION__, err);
1104                 goto err4;
1105         }
1106         return 0;
1107 err4:
1108         cxio_hal_pblpool_destroy(rdev_p);
1109 err3:
1110         cxio_hal_destroy_resource(rdev_p->rscp);
1111 err2:
1112         cxio_hal_destroy_ctrl_qp(rdev_p);
1113 err1:
1114         TAILQ_REMOVE(&rdev_list, rdev_p, entry);
1115         return err;
1116 }
1117
1118 void
1119 cxio_rdev_close(struct cxio_rdev *rdev_p)
1120 {
1121         if (rdev_p) {
1122                 cxio_hal_pblpool_destroy(rdev_p);
1123                 cxio_hal_rqtpool_destroy(rdev_p);
1124                 TAILQ_REMOVE(&rdev_list, rdev_p, entry);
1125                 rdev_p->t3cdev_p->ulp = NULL;
1126                 cxio_hal_destroy_ctrl_qp(rdev_p);
1127                 cxio_hal_destroy_resource(rdev_p->rscp);
1128         }
1129 }
1130
1131 int
1132 cxio_hal_init(void)
1133 {
1134         TAILQ_INIT(&rdev_list);
1135 #ifdef needed
1136         if (cxio_hal_init_rhdl_resource(T3_MAX_NUM_RI))
1137                 return (-ENOMEM);
1138 #endif
1139         t3_register_cpl_handler(CPL_ASYNC_NOTIF, cxio_hal_ev_handler);
1140         return 0;
1141 }
1142
1143 void
1144 cxio_hal_exit(void)
1145 {
1146         struct cxio_rdev *rdev, *tmp;
1147
1148         t3_register_cpl_handler(CPL_ASYNC_NOTIF, NULL);
1149         TAILQ_FOREACH_SAFE(rdev, &rdev_list, entry, tmp)
1150                 cxio_rdev_close(rdev);
1151 #ifdef needed
1152         cxio_hal_destroy_rhdl_resource();
1153 #endif
1154 }
1155
1156 static void
1157 flush_completed_wrs(struct t3_wq *wq, struct t3_cq *cq)
1158 {
1159         struct t3_swsq *sqp;
1160         __u32 ptr = wq->sq_rptr;
1161         int count = Q_COUNT(wq->sq_rptr, wq->sq_wptr);
1162
1163         sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2);
1164         while (count--)
1165                 if (!sqp->signaled) {
1166                         ptr++;
1167                         sqp = wq->sq + Q_PTR2IDX(ptr,  wq->sq_size_log2);
1168                 } else if (sqp->complete) {
1169
1170                         /*
1171                          * Insert this completed cqe into the swcq.
1172                          */
1173                         CTR3(KTR_IW_CXGB, "%s moving cqe into swcq sq idx %ld cq idx %ld",
1174                              __FUNCTION__, Q_PTR2IDX(ptr,  wq->sq_size_log2),
1175                              Q_PTR2IDX(cq->sw_wptr, cq->size_log2));
1176                         sqp->cqe.header |= htonl(V_CQE_SWCQE(1));
1177                         *(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2))
1178                                 = sqp->cqe;
1179                         cq->sw_wptr++;
1180                         sqp->signaled = 0;
1181                         break;
1182                 } else
1183                         break;
1184 }
1185
1186 static void
1187 create_read_req_cqe(struct t3_wq *wq, struct t3_cqe *hw_cqe,
1188                                 struct t3_cqe *read_cqe)
1189 {
1190         read_cqe->u.scqe.wrid_hi = wq->oldest_read->sq_wptr;
1191         read_cqe->len = wq->oldest_read->read_len;
1192         read_cqe->header = htonl(V_CQE_QPID(CQE_QPID(*hw_cqe)) |
1193                                  V_CQE_SWCQE(SW_CQE(*hw_cqe)) |
1194                                  V_CQE_OPCODE(T3_READ_REQ) |
1195                                  V_CQE_TYPE(1));
1196 }
1197
1198 /*
1199  * Return a ptr to the next read wr in the SWSQ or NULL.
1200  */
1201 static void
1202 advance_oldest_read(struct t3_wq *wq)
1203 {
1204
1205         u32 rptr = wq->oldest_read - wq->sq + 1;
1206         u32 wptr = Q_PTR2IDX(wq->sq_wptr, wq->sq_size_log2);
1207
1208         while (Q_PTR2IDX(rptr, wq->sq_size_log2) != wptr) {
1209                 wq->oldest_read = wq->sq + Q_PTR2IDX(rptr, wq->sq_size_log2);
1210
1211                 if (wq->oldest_read->opcode == T3_READ_REQ)
1212                         return;
1213                 rptr++;
1214         }
1215         wq->oldest_read = NULL;
1216 }
1217
1218 /*
1219  * cxio_poll_cq
1220  *
1221  * Caller must:
1222  *     check the validity of the first CQE,
1223  *     supply the wq assicated with the qpid.
1224  *
1225  * credit: cq credit to return to sge.
1226  * cqe_flushed: 1 iff the CQE is flushed.
1227  * cqe: copy of the polled CQE.
1228  *
1229  * return value:
1230  *     0       CQE returned,
1231  *    -1       CQE skipped, try again.
1232  */
1233 int
1234 cxio_poll_cq(struct t3_wq *wq, struct t3_cq *cq, struct t3_cqe *cqe,
1235                      u8 *cqe_flushed, u64 *cookie, u32 *credit)
1236 {
1237         int ret = 0;
1238         struct t3_cqe *hw_cqe, read_cqe;
1239
1240         *cqe_flushed = 0;
1241         *credit = 0;
1242         hw_cqe = cxio_next_cqe(cq);
1243
1244         CTR5(KTR_IW_CXGB, "cxio_poll_cq CQE OOO %d qpid 0x%0x genbit %d type %d status 0x%0x",
1245              CQE_OOO(*hw_cqe), CQE_QPID(*hw_cqe),
1246              CQE_GENBIT(*hw_cqe), CQE_TYPE(*hw_cqe), CQE_STATUS(*hw_cqe));
1247         CTR4(KTR_IW_CXGB, "opcode 0x%0x len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x",
1248              CQE_OPCODE(*hw_cqe), CQE_LEN(*hw_cqe), CQE_WRID_HI(*hw_cqe),
1249              CQE_WRID_LOW(*hw_cqe));
1250
1251         /*
1252          * skip cqe's not affiliated with a QP.
1253          */
1254         if (wq == NULL) {
1255                 ret = -1;
1256                 goto skip_cqe;
1257         }
1258
1259         /*
1260          * Gotta tweak READ completions:
1261          *      1) the cqe doesn't contain the sq_wptr from the wr.
1262          *      2) opcode not reflected from the wr.
1263          *      3) read_len not reflected from the wr.
1264          *      4) cq_type is RQ_TYPE not SQ_TYPE.
1265          */
1266         if (RQ_TYPE(*hw_cqe) && (CQE_OPCODE(*hw_cqe) == T3_READ_RESP)) {
1267
1268                 /*
1269                  * Don't write to the HWCQ, so create a new read req CQE
1270                  * in local memory.
1271                  */
1272                 create_read_req_cqe(wq, hw_cqe, &read_cqe);
1273                 hw_cqe = &read_cqe;
1274                 advance_oldest_read(wq);
1275         }
1276
1277         /*
1278          * T3A: Discard TERMINATE CQEs.
1279          */
1280         if (CQE_OPCODE(*hw_cqe) == T3_TERMINATE) {
1281                 ret = -1;
1282                 wq->error = 1;
1283                 goto skip_cqe;
1284         }
1285
1286         if (CQE_STATUS(*hw_cqe) || wq->error) {
1287                 *cqe_flushed = wq->error;
1288                 wq->error = 1;
1289
1290                 /*
1291                  * T3A inserts errors into the CQE.  We cannot return
1292                  * these as work completions.
1293                  */
1294                 /* incoming write failures */
1295                 if ((CQE_OPCODE(*hw_cqe) == T3_RDMA_WRITE)
1296                      && RQ_TYPE(*hw_cqe)) {
1297                         ret = -1;
1298                         goto skip_cqe;
1299                 }
1300                 /* incoming read request failures */
1301                 if ((CQE_OPCODE(*hw_cqe) == T3_READ_RESP) && SQ_TYPE(*hw_cqe)) {
1302                         ret = -1;
1303                         goto skip_cqe;
1304                 }
1305
1306                 /* incoming SEND with no receive posted failures */
1307                 if ((CQE_OPCODE(*hw_cqe) == T3_SEND) && RQ_TYPE(*hw_cqe) &&
1308                     Q_EMPTY(wq->rq_rptr, wq->rq_wptr)) {
1309                         ret = -1;
1310                         goto skip_cqe;
1311                 }
1312                 goto proc_cqe;
1313         }
1314
1315         /*
1316          * RECV completion.
1317          */
1318         if (RQ_TYPE(*hw_cqe)) {
1319
1320                 /*
1321                  * HW only validates 4 bits of MSN.  So we must validate that
1322                  * the MSN in the SEND is the next expected MSN.  If its not,
1323                  * then we complete this with TPT_ERR_MSN and mark the wq in
1324                  * error.
1325                  */
1326                 if (__predict_false((CQE_WRID_MSN(*hw_cqe) != (wq->rq_rptr + 1)))) {
1327                         wq->error = 1;
1328                         hw_cqe->header |= htonl(V_CQE_STATUS(TPT_ERR_MSN));
1329                         goto proc_cqe;
1330                 }
1331                 goto proc_cqe;
1332         }
1333
1334         /*
1335          * If we get here its a send completion.
1336          *
1337          * Handle out of order completion. These get stuffed
1338          * in the SW SQ. Then the SW SQ is walked to move any
1339          * now in-order completions into the SW CQ.  This handles
1340          * 2 cases:
1341          *      1) reaping unsignaled WRs when the first subsequent
1342          *         signaled WR is completed.
1343          *      2) out of order read completions.
1344          */
1345         if (!SW_CQE(*hw_cqe) && (CQE_WRID_SQ_WPTR(*hw_cqe) != wq->sq_rptr)) {
1346                 struct t3_swsq *sqp;
1347
1348                 CTR2(KTR_IW_CXGB, "%s out of order completion going in swsq at idx %ld",
1349                      __FUNCTION__,
1350                      Q_PTR2IDX(CQE_WRID_SQ_WPTR(*hw_cqe), wq->sq_size_log2));
1351                 sqp = wq->sq +
1352                       Q_PTR2IDX(CQE_WRID_SQ_WPTR(*hw_cqe), wq->sq_size_log2);
1353                 sqp->cqe = *hw_cqe;
1354                 sqp->complete = 1;
1355                 ret = -1;
1356                 goto flush_wq;
1357         }
1358
1359 proc_cqe:
1360         *cqe = *hw_cqe;
1361
1362         /*
1363          * Reap the associated WR(s) that are freed up with this
1364          * completion.
1365          */
1366         if (SQ_TYPE(*hw_cqe)) {
1367                 wq->sq_rptr = CQE_WRID_SQ_WPTR(*hw_cqe);
1368                 CTR2(KTR_IW_CXGB, "%s completing sq idx %ld", __FUNCTION__,
1369                      Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2));
1370                 *cookie = (wq->sq +
1371                            Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2))->wr_id;
1372                 wq->sq_rptr++;
1373         } else {
1374                 CTR2(KTR_IW_CXGB, "%s completing rq idx %ld", __FUNCTION__,
1375                      Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2));
1376                 *cookie = *(wq->rq + Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2));
1377                 wq->rq_rptr++;
1378         }
1379
1380 flush_wq:
1381         /*
1382          * Flush any completed cqes that are now in-order.
1383          */
1384         flush_completed_wrs(wq, cq);
1385
1386 skip_cqe:
1387         if (SW_CQE(*hw_cqe)) {
1388                 CTR4(KTR_IW_CXGB, "%s cq %p cqid 0x%x skip sw cqe sw_rptr 0x%x",
1389                      __FUNCTION__, cq, cq->cqid, cq->sw_rptr);
1390                 ++cq->sw_rptr;
1391         } else {
1392                 CTR4(KTR_IW_CXGB, "%s cq %p cqid 0x%x skip hw cqe rptr 0x%x",
1393                      __FUNCTION__, cq, cq->cqid, cq->rptr);
1394                 ++cq->rptr;
1395
1396                 /*
1397                  * T3A: compute credits.
1398                  */
1399                 if (((cq->rptr - cq->wptr) > (1 << (cq->size_log2 - 1)))
1400                     || ((cq->rptr - cq->wptr) >= 128)) {
1401                         *credit = cq->rptr - cq->wptr;
1402                         cq->wptr = cq->rptr;
1403                 }
1404         }
1405         return ret;
1406 }
1407
1408