sys/dev/mlx5/mlx5_ib/mlx5_ib_cq.c

   1 /*-
   2  * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
  14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  16  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
  17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  23  * SUCH DAMAGE.
  24  *
  25  * $FreeBSD$
  26  */
  27
  28 #include <linux/compiler.h>
  29 #include <linux/kref.h>
  30 #include <rdma/ib_umem.h>
  31 #include <rdma/ib_user_verbs.h>
  32 #include "mlx5_ib.h"
  33 #include "user.h"
  34
  35 static void mlx5_ib_cq_comp(struct mlx5_core_cq *cq)
  36 {
  37         struct ib_cq *ibcq = &to_mibcq(cq)->ibcq;
  38
  39         ibcq->comp_handler(ibcq, ibcq->cq_context);
  40 }
  41
  42 static void mlx5_ib_cq_event(struct mlx5_core_cq *mcq, int type)
  43 {
  44         struct mlx5_ib_cq *cq = container_of(mcq, struct mlx5_ib_cq, mcq);
  45         struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device);
  46         struct ib_cq *ibcq = &cq->ibcq;
  47         struct ib_event event;
  48
  49         if (type != MLX5_EVENT_TYPE_CQ_ERROR) {
  50                 mlx5_ib_warn(dev, "Unexpected event type %d on CQ %06x\n",
  51                              type, mcq->cqn);
  52                 return;
  53         }
  54
  55         if (ibcq->event_handler) {
  56                 event.device     = &dev->ib_dev;
  57                 event.event      = IB_EVENT_CQ_ERR;
  58                 event.element.cq = ibcq;
  59                 ibcq->event_handler(&event, ibcq->cq_context);
  60         }
  61 }
  62
  63 static void *get_cqe_from_buf(struct mlx5_ib_cq_buf *buf, int n, int size)
  64 {
  65         return mlx5_buf_offset(&buf->buf, n * size);
  66 }
  67
  68 static void *get_cqe(struct mlx5_ib_cq *cq, int n)
  69 {
  70         return get_cqe_from_buf(&cq->buf, n, cq->mcq.cqe_sz);
  71 }
  72
  73 static u8 sw_ownership_bit(int n, int nent)
  74 {
  75         return (n & nent) ? 1 : 0;
  76 }
  77
  78 static void *get_sw_cqe(struct mlx5_ib_cq *cq, int n)
  79 {
  80         void *cqe = get_cqe(cq, n & cq->ibcq.cqe);
  81         struct mlx5_cqe64 *cqe64;
  82
  83         cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64;
  84
  85         if (likely((cqe64->op_own) >> 4 != MLX5_CQE_INVALID) &&
  86             !((cqe64->op_own & MLX5_CQE_OWNER_MASK) ^ !!(n & (cq->ibcq.cqe + 1)))) {
  87                 return cqe;
  88         } else {
  89                 return NULL;
  90         }
  91 }
  92
  93 static void *next_cqe_sw(struct mlx5_ib_cq *cq)
  94 {
  95         return get_sw_cqe(cq, cq->mcq.cons_index);
  96 }
  97
  98 static enum ib_wc_opcode get_umr_comp(struct mlx5_ib_wq *wq, int idx)
  99 {
 100         switch (wq->swr_ctx[idx].wr_data) {
 101         case IB_WR_LOCAL_INV:
 102                 return IB_WC_LOCAL_INV;
 103
 104         case IB_WR_FAST_REG_MR:
 105                 return IB_WC_FAST_REG_MR;
 106
 107         default:
 108                 printf("mlx5_ib: WARN: ""unknown completion status\n");
 109                 return 0;
 110         }
 111 }
 112
 113 static void handle_good_req(struct ib_wc *wc, struct mlx5_cqe64 *cqe,
 114                             struct mlx5_ib_wq *wq, int idx)
 115 {
 116         wc->wc_flags = 0;
 117         switch (be32_to_cpu(cqe->sop_drop_qpn) >> 24) {
 118         case MLX5_OPCODE_RDMA_WRITE_IMM:
 119                 wc->wc_flags |= IB_WC_WITH_IMM;
 120         case MLX5_OPCODE_RDMA_WRITE:
 121                 wc->opcode    = IB_WC_RDMA_WRITE;
 122                 break;
 123         case MLX5_OPCODE_SEND_IMM:
 124                 wc->wc_flags |= IB_WC_WITH_IMM;
 125         case MLX5_OPCODE_NOP:
 126         case MLX5_OPCODE_SEND:
 127         case MLX5_OPCODE_SEND_INVAL:
 128                 wc->opcode    = IB_WC_SEND;
 129                 break;
 130         case MLX5_OPCODE_RDMA_READ:
 131                 wc->opcode    = IB_WC_RDMA_READ;
 132                 wc->byte_len  = be32_to_cpu(cqe->byte_cnt);
 133                 break;
 134         case MLX5_OPCODE_ATOMIC_CS:
 135                 wc->opcode    = IB_WC_COMP_SWAP;
 136                 wc->byte_len  = 8;
 137                 break;
 138         case MLX5_OPCODE_ATOMIC_FA:
 139                 wc->opcode    = IB_WC_FETCH_ADD;
 140                 wc->byte_len  = 8;
 141                 break;
 142         case MLX5_OPCODE_ATOMIC_MASKED_CS:
 143                 wc->opcode    = IB_WC_MASKED_COMP_SWAP;
 144                 wc->byte_len  = 8;
 145                 break;
 146         case MLX5_OPCODE_ATOMIC_MASKED_FA:
 147                 wc->opcode    = IB_WC_MASKED_FETCH_ADD;
 148                 wc->byte_len  = 8;
 149                 break;
 150         case MLX5_OPCODE_BIND_MW:
 151                 wc->opcode    = IB_WC_BIND_MW;
 152                 break;
 153         case MLX5_OPCODE_UMR:
 154                 wc->opcode = get_umr_comp(wq, idx);
 155                 break;
 156         }
 157 }
 158
 159 enum {
 160         MLX5_GRH_IN_BUFFER = 1,
 161         MLX5_GRH_IN_CQE    = 2,
 162 };
 163
 164 static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe,
 165                              struct mlx5_ib_qp *qp)
 166 {
 167         struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.device);
 168         struct mlx5_ib_srq *srq;
 169         struct mlx5_ib_wq *wq;
 170         u16 wqe_ctr;
 171         u8 g;
 172 #if defined(DX_ROCE_V1_5) || defined(DX_WINDOWS)
 173         u8 udp_header_valid;
 174 #endif
 175
 176         if (qp->ibqp.srq || qp->ibqp.xrcd) {
 177                 struct mlx5_core_srq *msrq = NULL;
 178
 179                 if (qp->ibqp.xrcd) {
 180                         msrq = mlx5_core_get_srq(dev->mdev,
 181                                                  be32_to_cpu(cqe->srqn));
 182                         srq = to_mibsrq(msrq);
 183                 } else {
 184                         srq = to_msrq(qp->ibqp.srq);
 185                 }
 186                 if (srq) {
 187                         wqe_ctr = be16_to_cpu(cqe->wqe_counter);
 188                         wc->wr_id = srq->wrid[wqe_ctr];
 189                         mlx5_ib_free_srq_wqe(srq, wqe_ctr);
 190                         if (msrq && atomic_dec_and_test(&msrq->refcount))
 191                                 complete(&msrq->free);
 192                 }
 193         } else {
 194                 wq        = &qp->rq;
 195                 wc->wr_id = wq->rwr_ctx[wq->tail & (wq->wqe_cnt - 1)].wrid;
 196                 ++wq->tail;
 197         }
 198         wc->byte_len = be32_to_cpu(cqe->byte_cnt);
 199
 200         switch (cqe->op_own >> 4) {
 201         case MLX5_CQE_RESP_WR_IMM:
 202                 wc->opcode      = IB_WC_RECV_RDMA_WITH_IMM;
 203                 wc->wc_flags    = IB_WC_WITH_IMM;
 204                 wc->ex.imm_data = cqe->imm_inval_pkey;
 205                 break;
 206         case MLX5_CQE_RESP_SEND:
 207                 wc->opcode   = IB_WC_RECV;
 208                 wc->wc_flags = 0;
 209                 break;
 210         case MLX5_CQE_RESP_SEND_IMM:
 211                 wc->opcode      = IB_WC_RECV;
 212                 wc->wc_flags    = IB_WC_WITH_IMM;
 213                 wc->ex.imm_data = cqe->imm_inval_pkey;
 214                 break;
 215         case MLX5_CQE_RESP_SEND_INV:
 216                 wc->opcode      = IB_WC_RECV;
 217                 wc->wc_flags    = IB_WC_WITH_INVALIDATE;
 218                 wc->ex.invalidate_rkey = be32_to_cpu(cqe->imm_inval_pkey);
 219                 break;
 220         }
 221         wc->slid           = be16_to_cpu(cqe->slid);
 222         wc->sl             = (be32_to_cpu(cqe->flags_rqpn) >> 24) & 0xf;
 223         wc->src_qp         = be32_to_cpu(cqe->flags_rqpn) & 0xffffff;
 224         wc->dlid_path_bits = cqe->ml_path;
 225         g = (be32_to_cpu(cqe->flags_rqpn) >> 28) & 3;
 226         wc->wc_flags |= g ? IB_WC_GRH : 0;
 227         wc->pkey_index     = be32_to_cpu(cqe->imm_inval_pkey) & 0xffff;
 228
 229 #if defined(DX_ROCE_V1_5) || defined(DX_WINDOWS)
 230         udp_header_valid = wc->sl & 0x8;
 231         if (udp_header_valid)
 232                 wc->wc_flags |= IB_WC_WITH_UDP_HDR;
 233
 234 #endif
 235 }
 236
 237 static void dump_cqe(struct mlx5_ib_dev *dev, struct mlx5_err_cqe *cqe)
 238 {
 239         __be32 *p = (__be32 *)cqe;
 240         int i;
 241
 242         mlx5_ib_warn(dev, "dump error cqe\n");
 243         for (i = 0; i < sizeof(*cqe) / 16; i++, p += 4)
 244                 printf("mlx5_ib: INFO: ""%08x %08x %08x %08x\n", be32_to_cpu(p[0]), be32_to_cpu(p[1]), be32_to_cpu(p[2]), be32_to_cpu(p[3]));
 245 }
 246
 247 static void mlx5_handle_error_cqe(struct mlx5_ib_dev *dev,
 248                                   struct mlx5_err_cqe *cqe,
 249                                   struct ib_wc *wc)
 250 {
 251         int dump = 1;
 252
 253         switch (cqe->syndrome) {
 254         case MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR:
 255                 wc->status = IB_WC_LOC_LEN_ERR;
 256                 break;
 257         case MLX5_CQE_SYNDROME_LOCAL_QP_OP_ERR:
 258                 wc->status = IB_WC_LOC_QP_OP_ERR;
 259                 break;
 260         case MLX5_CQE_SYNDROME_LOCAL_PROT_ERR:
 261                 wc->status = IB_WC_LOC_PROT_ERR;
 262                 break;
 263         case MLX5_CQE_SYNDROME_WR_FLUSH_ERR:
 264                 dump = 0;
 265                 wc->status = IB_WC_WR_FLUSH_ERR;
 266                 break;
 267         case MLX5_CQE_SYNDROME_MW_BIND_ERR:
 268                 wc->status = IB_WC_MW_BIND_ERR;
 269                 break;
 270         case MLX5_CQE_SYNDROME_BAD_RESP_ERR:
 271                 wc->status = IB_WC_BAD_RESP_ERR;
 272                 break;
 273         case MLX5_CQE_SYNDROME_LOCAL_ACCESS_ERR:
 274                 wc->status = IB_WC_LOC_ACCESS_ERR;
 275                 break;
 276         case MLX5_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR:
 277                 wc->status = IB_WC_REM_INV_REQ_ERR;
 278                 break;
 279         case MLX5_CQE_SYNDROME_REMOTE_ACCESS_ERR:
 280                 wc->status = IB_WC_REM_ACCESS_ERR;
 281                 break;
 282         case MLX5_CQE_SYNDROME_REMOTE_OP_ERR:
 283                 wc->status = IB_WC_REM_OP_ERR;
 284                 break;
 285         case MLX5_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:
 286                 wc->status = IB_WC_RETRY_EXC_ERR;
 287                 dump = 0;
 288                 break;
 289         case MLX5_CQE_SYNDROME_RNR_RETRY_EXC_ERR:
 290                 wc->status = IB_WC_RNR_RETRY_EXC_ERR;
 291                 dump = 0;
 292                 break;
 293         case MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR:
 294                 wc->status = IB_WC_REM_ABORT_ERR;
 295                 break;
 296         default:
 297                 wc->status = IB_WC_GENERAL_ERR;
 298                 break;
 299         }
 300
 301         wc->vendor_err = cqe->vendor_err_synd;
 302         if (dump)
 303                 dump_cqe(dev, cqe);
 304 }
 305
 306 static int is_atomic_response(struct mlx5_ib_qp *qp, u16 idx)
 307 {
 308         /* TBD: waiting decision
 309         */
 310         return 0;
 311 }
 312
 313 static void *mlx5_get_atomic_laddr(struct mlx5_ib_qp *qp, u16 idx)
 314 {
 315         struct mlx5_wqe_data_seg *dpseg;
 316         void *addr;
 317
 318         dpseg = mlx5_get_send_wqe(qp, idx) + sizeof(struct mlx5_wqe_ctrl_seg) +
 319                 sizeof(struct mlx5_wqe_raddr_seg) +
 320                 sizeof(struct mlx5_wqe_atomic_seg);
 321         addr = (void *)(uintptr_t)be64_to_cpu(dpseg->addr);
 322         return addr;
 323 }
 324
 325 static void handle_atomic(struct mlx5_ib_qp *qp, struct mlx5_cqe64 *cqe64,
 326                           u16 idx)
 327 {
 328         void *addr;
 329         int byte_count;
 330         int i;
 331
 332         if (!is_atomic_response(qp, idx))
 333                 return;
 334
 335         byte_count = be32_to_cpu(cqe64->byte_cnt);
 336         addr = mlx5_get_atomic_laddr(qp, idx);
 337
 338         if (byte_count == 4) {
 339                 *(u32 *)addr = be32_to_cpu(*((__be32 *)addr));
 340         } else {
 341                 for (i = 0; i < byte_count; i += 8) {
 342                         *(u64 *)addr = be64_to_cpu(*((__be64 *)addr));
 343                         addr += 8;
 344                 }
 345         }
 346
 347         return;
 348 }
 349
 350 static void handle_atomics(struct mlx5_ib_qp *qp, struct mlx5_cqe64 *cqe64,
 351                            u16 tail, u16 head)
 352 {
 353         u16 idx;
 354
 355         do {
 356                 idx = tail & (qp->sq.wqe_cnt - 1);
 357                 handle_atomic(qp, cqe64, idx);
 358                 if (idx == head)
 359                         break;
 360
 361                 tail = qp->sq.swr_ctx[idx].w_list.next;
 362         } while (1);
 363         tail = qp->sq.swr_ctx[idx].w_list.next;
 364         qp->sq.last_poll = tail;
 365 }
 366
 367 static void free_cq_buf(struct mlx5_ib_dev *dev, struct mlx5_ib_cq_buf *buf)
 368 {
 369         mlx5_buf_free(dev->mdev, &buf->buf);
 370 }
 371
 372 static void sw_send_comp(struct mlx5_ib_qp *qp, int num_entries,
 373                          struct ib_wc *wc, int *npolled)
 374 {
 375         struct mlx5_ib_wq *wq;
 376         unsigned cur;
 377         unsigned idx;
 378         int np;
 379         int i;
 380
 381         wq = &qp->sq;
 382         cur = wq->head - wq->tail;
 383         np = *npolled;
 384
 385         if (cur == 0)
 386                 return;
 387
 388         for (i = 0;  i < cur && np < num_entries; i++) {
 389                 idx = wq->last_poll & (wq->wqe_cnt - 1);
 390                 wc->wr_id = wq->swr_ctx[idx].wrid;
 391                 wc->status = IB_WC_WR_FLUSH_ERR;
 392                 wc->vendor_err = MLX5_CQE_SYNDROME_WR_FLUSH_ERR;
 393                 wq->tail++;
 394                 np++;
 395                 wc->qp = &qp->ibqp;
 396                 wc++;
 397                 wq->last_poll = wq->swr_ctx[idx].w_list.next;
 398         }
 399         *npolled = np;
 400 }
 401
 402 static void sw_recv_comp(struct mlx5_ib_qp *qp, int num_entries,
 403                          struct ib_wc *wc, int *npolled)
 404 {
 405         struct mlx5_ib_wq *wq;
 406         unsigned cur;
 407         int np;
 408         int i;
 409
 410         wq = &qp->rq;
 411         cur = wq->head - wq->tail;
 412         np = *npolled;
 413
 414         if (cur == 0)
 415                 return;
 416
 417         for (i = 0;  i < cur && np < num_entries; i++) {
 418                 wc->wr_id = wq->rwr_ctx[wq->tail & (wq->wqe_cnt - 1)].wrid;
 419                 wc->status = IB_WC_WR_FLUSH_ERR;
 420                 wc->vendor_err = MLX5_CQE_SYNDROME_WR_FLUSH_ERR;
 421                 wq->tail++;
 422                 np++;
 423                 wc->qp = &qp->ibqp;
 424                 wc++;
 425         }
 426         *npolled = np;
 427 }
 428
 429 static void mlx5_ib_poll_sw_comp(struct mlx5_ib_cq *cq, int num_entries,
 430                                  struct ib_wc *wc, int *npolled)
 431 {
 432         struct mlx5_ib_qp *qp;
 433
 434         *npolled = 0;
 435         /* Find uncompleted WQEs belonging to that cq and retrun mmics ones */
 436         list_for_each_entry(qp, &cq->list_send_qp, cq_send_list) {
 437                 sw_send_comp(qp, num_entries, wc + *npolled, npolled);
 438                 if (*npolled >= num_entries)
 439                         return;
 440         }
 441
 442         list_for_each_entry(qp, &cq->list_recv_qp, cq_recv_list) {
 443                 sw_recv_comp(qp, num_entries, wc + *npolled, npolled);
 444                 if (*npolled >= num_entries)
 445                         return;
 446         }
 447 }
 448
 449 static inline u32 mlx5_ib_base_mkey(const u32 key)
 450 {
 451         return key & 0xffffff00u;
 452 }
 453
 454 static int mlx5_poll_one(struct mlx5_ib_cq *cq,
 455                          struct mlx5_ib_qp **cur_qp,
 456                          struct ib_wc *wc)
 457 {
 458         struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device);
 459         struct mlx5_err_cqe *err_cqe;
 460         struct mlx5_cqe64 *cqe64;
 461         struct mlx5_core_qp *mqp;
 462         struct mlx5_ib_wq *wq;
 463         struct mlx5_sig_err_cqe *sig_err_cqe;
 464         struct mlx5_core_mr *mmr;
 465         struct mlx5_ib_mr *mr;
 466         unsigned long flags;
 467         u8 opcode;
 468         u32 qpn;
 469         u16 wqe_ctr;
 470         void *cqe;
 471         int idx;
 472
 473 repoll:
 474         cqe = next_cqe_sw(cq);
 475         if (!cqe)
 476                 return -EAGAIN;
 477
 478         cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64;
 479
 480         ++cq->mcq.cons_index;
 481
 482         /* Make sure we read CQ entry contents after we've checked the
 483          * ownership bit.
 484          */
 485         rmb();
 486
 487         opcode = cqe64->op_own >> 4;
 488         if (unlikely(opcode == MLX5_CQE_RESIZE_CQ)) {
 489                 if (likely(cq->resize_buf)) {
 490                         free_cq_buf(dev, &cq->buf);
 491                         cq->buf = *cq->resize_buf;
 492                         kfree(cq->resize_buf);
 493                         cq->resize_buf = NULL;
 494                         goto repoll;
 495                 } else {
 496                         mlx5_ib_warn(dev, "unexpected resize cqe\n");
 497                 }
 498         }
 499
 500         qpn = ntohl(cqe64->sop_drop_qpn) & 0xffffff;
 501         if (!*cur_qp || (qpn != (*cur_qp)->ibqp.qp_num)) {
 502                 /* We do not have to take the QP table lock here,
 503                  * because CQs will be locked while QPs are removed
 504                  * from the table.
 505                  */
 506                 mqp = __mlx5_qp_lookup(dev->mdev, qpn);
 507                 if (unlikely(!mqp)) {
 508                         mlx5_ib_warn(dev, "CQE@CQ %06x for unknown QPN %6x\n",
 509                                      cq->mcq.cqn, qpn);
 510                         return -EINVAL;
 511                 }
 512
 513                 *cur_qp = to_mibqp(mqp);
 514         }
 515
 516         wc->qp  = &(*cur_qp)->ibqp;
 517         switch (opcode) {
 518         case MLX5_CQE_REQ:
 519                 wq = &(*cur_qp)->sq;
 520                 wqe_ctr = be16_to_cpu(cqe64->wqe_counter);
 521                 idx = wqe_ctr & (wq->wqe_cnt - 1);
 522                 handle_good_req(wc, cqe64, wq, idx);
 523                 handle_atomics(*cur_qp, cqe64, wq->last_poll, idx);
 524                 wc->wr_id = wq->swr_ctx[idx].wrid;
 525                 wq->tail = wq->swr_ctx[idx].wqe_head + 1;
 526                 if (unlikely(wq->swr_ctx[idx].w_list.opcode &
 527                     MLX5_OPCODE_SIGNATURE_CANCELED))
 528                         wc->status = IB_WC_GENERAL_ERR;
 529                 else
 530                         wc->status = IB_WC_SUCCESS;
 531                 break;
 532         case MLX5_CQE_RESP_WR_IMM:
 533         case MLX5_CQE_RESP_SEND:
 534         case MLX5_CQE_RESP_SEND_IMM:
 535         case MLX5_CQE_RESP_SEND_INV:
 536                 handle_responder(wc, cqe64, *cur_qp);
 537                 wc->status = IB_WC_SUCCESS;
 538                 break;
 539         case MLX5_CQE_RESIZE_CQ:
 540                 break;
 541         case MLX5_CQE_REQ_ERR:
 542         case MLX5_CQE_RESP_ERR:
 543                 err_cqe = (struct mlx5_err_cqe *)cqe64;
 544                 mlx5_handle_error_cqe(dev, err_cqe, wc);
 545                 mlx5_ib_dbg(dev, "%s error cqe on cqn 0x%x:\n",
 546                             opcode == MLX5_CQE_REQ_ERR ?
 547                             "Requestor" : "Responder", cq->mcq.cqn);
 548                 mlx5_ib_dbg(dev, "syndrome 0x%x, vendor syndrome 0x%x\n",
 549                             err_cqe->syndrome, err_cqe->vendor_err_synd);
 550                 if (opcode == MLX5_CQE_REQ_ERR) {
 551                         wq = &(*cur_qp)->sq;
 552                         wqe_ctr = be16_to_cpu(cqe64->wqe_counter);
 553                         idx = wqe_ctr & (wq->wqe_cnt - 1);
 554                         wc->wr_id = wq->swr_ctx[idx].wrid;
 555                         wq->tail = wq->swr_ctx[idx].wqe_head + 1;
 556                 } else {
 557                         struct mlx5_ib_srq *srq;
 558
 559                         if ((*cur_qp)->ibqp.srq) {
 560                                 srq = to_msrq((*cur_qp)->ibqp.srq);
 561                                 wqe_ctr = be16_to_cpu(cqe64->wqe_counter);
 562                                 wc->wr_id = srq->wrid[wqe_ctr];
 563                                 mlx5_ib_free_srq_wqe(srq, wqe_ctr);
 564                         } else {
 565                                 wq = &(*cur_qp)->rq;
 566                                 wc->wr_id = wq->rwr_ctx[wq->tail & (wq->wqe_cnt - 1)].wrid;
 567                                 ++wq->tail;
 568                         }
 569                 }
 570                 break;
 571         case MLX5_CQE_SIG_ERR:
 572                 sig_err_cqe = (struct mlx5_sig_err_cqe *)cqe64;
 573
 574                 spin_lock_irqsave(&dev->mdev->priv.mr_table.lock, flags);
 575                 mmr = __mlx5_mr_lookup(dev->mdev,
 576                                          mlx5_ib_base_mkey(be32_to_cpu(sig_err_cqe->mkey)));
 577                 if (unlikely(!mmr)) {
 578                         spin_unlock_irqrestore(&dev->mdev->priv.mr_table.lock, flags);
 579                         mlx5_ib_warn(dev, "CQE@CQ %06x for unknown MR %6x\n",
 580                                      cq->mcq.cqn, be32_to_cpu(sig_err_cqe->mkey));
 581                         return -EINVAL;
 582                 }
 583
 584                 mr = to_mibmr(mmr);
 585                 mr->sig->sig_err_exists = true;
 586                 mr->sig->sigerr_count++;
 587
 588                 mlx5_ib_warn(dev, "CQN: 0x%x Got SIGERR\n", cq->mcq.cqn);
 589
 590                 spin_unlock_irqrestore(&dev->mdev->priv.mr_table.lock, flags);
 591                 goto repoll;
 592         }
 593
 594         return 0;
 595 }
 596
 597 int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 598 {
 599         struct mlx5_ib_cq *cq = to_mcq(ibcq);
 600         struct mlx5_ib_qp *cur_qp = NULL;
 601         struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device);
 602         struct mlx5_core_dev *mdev = dev->mdev;
 603         unsigned long flags;
 604         int npolled;
 605         int err = 0;
 606
 607         spin_lock_irqsave(&cq->lock, flags);
 608         if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
 609                 mlx5_ib_poll_sw_comp(cq, num_entries, wc, &npolled);
 610                 goto out;
 611         }
 612
 613         for (npolled = 0; npolled < num_entries; npolled++) {
 614                 err = mlx5_poll_one(cq, &cur_qp, wc + npolled);
 615                 if (err)
 616                         break;
 617         }
 618
 619         if (npolled)
 620                 mlx5_cq_set_ci(&cq->mcq);
 621 out:
 622         spin_unlock_irqrestore(&cq->lock, flags);
 623
 624         if (err == 0 || err == -EAGAIN)
 625                 return npolled;
 626         else
 627                 return err;
 628 }
 629
 630 int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 631 {
 632         struct mlx5_core_dev *mdev = to_mdev(ibcq->device)->mdev;
 633         void __iomem *uar_page = mdev->priv.uuari.uars[0].map;
 634
 635
 636         mlx5_cq_arm(&to_mcq(ibcq)->mcq,
 637                     (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
 638                     MLX5_CQ_DB_REQ_NOT_SOL : MLX5_CQ_DB_REQ_NOT,
 639                     uar_page,
 640                     MLX5_GET_DOORBELL_LOCK(&mdev->priv.cq_uar_lock),
 641                     to_mcq(ibcq)->mcq.cons_index);
 642
 643         return 0;
 644 }
 645
 646 static int alloc_cq_buf(struct mlx5_ib_dev *dev, struct mlx5_ib_cq_buf *buf,
 647                         int nent, int cqe_size)
 648 {
 649         int err;
 650
 651         err = mlx5_buf_alloc(dev->mdev, nent * cqe_size,
 652                              PAGE_SIZE * 2, &buf->buf);
 653         if (err) {
 654                 mlx5_ib_err(dev, "alloc failed\n");
 655                 return err;
 656         }
 657
 658         buf->cqe_size = cqe_size;
 659         buf->nent = nent;
 660
 661         return 0;
 662 }
 663
 664 static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
 665                           struct ib_ucontext *context, struct mlx5_ib_cq *cq,
 666                           int entries, struct mlx5_create_cq_mbox_in **cqb,
 667                           int *cqe_size, int *index, int *inlen)
 668 {
 669         struct mlx5_exp_ib_create_cq ucmd;
 670         size_t ucmdlen;
 671         int page_shift;
 672         int npages;
 673         int ncont;
 674         int err;
 675
 676         memset(&ucmd, 0, sizeof(ucmd));
 677
 678         ucmdlen =
 679                 (udata->inlen - sizeof(struct ib_uverbs_cmd_hdr) <
 680                 sizeof(struct mlx5_ib_create_cq)) ?
 681                 (sizeof(struct mlx5_ib_create_cq) - sizeof(ucmd.reserved)) :
 682                 sizeof(struct mlx5_ib_create_cq);
 683
 684         if (ib_copy_from_udata(&ucmd, udata, ucmdlen)) {
 685                 mlx5_ib_err(dev, "copy failed\n");
 686                 return -EFAULT;
 687         }
 688
 689         if (ucmdlen == sizeof(ucmd) && ucmd.reserved != 0) {
 690                 mlx5_ib_err(dev, "command corrupted\n");
 691                 return -EINVAL;
 692         }
 693
 694         if (ucmd.cqe_size != 64 && ucmd.cqe_size != 128) {
 695                 mlx5_ib_warn(dev, "wrong CQE size %d\n", ucmd.cqe_size);
 696                 return -EINVAL;
 697         }
 698
 699         *cqe_size = ucmd.cqe_size;
 700
 701         cq->buf.umem = ib_umem_get(context, ucmd.buf_addr,
 702                                    entries * ucmd.cqe_size,
 703                                    IB_ACCESS_LOCAL_WRITE, 1);
 704         if (IS_ERR(cq->buf.umem)) {
 705                 err = PTR_ERR(cq->buf.umem);
 706                 return err;
 707         }
 708
 709         err = mlx5_ib_db_map_user(to_mucontext(context), ucmd.db_addr,
 710                                   &cq->db);
 711         if (err) {
 712                 mlx5_ib_warn(dev, "map failed\n");
 713                 goto err_umem;
 714         }
 715
 716         mlx5_ib_cont_pages(cq->buf.umem, ucmd.buf_addr, &npages, &page_shift,
 717                            &ncont, NULL);
 718         mlx5_ib_dbg(dev, "addr 0x%llx, size %u, npages %d, page_shift %d, ncont %d\n",
 719                     (unsigned long long)ucmd.buf_addr, entries * ucmd.cqe_size,
 720                     npages, page_shift, ncont);
 721
 722         *inlen = sizeof(**cqb) + sizeof(*(*cqb)->pas) * ncont;
 723         *cqb = mlx5_vzalloc(*inlen);
 724         if (!*cqb) {
 725                 err = -ENOMEM;
 726                 goto err_db;
 727         }
 728
 729         mlx5_ib_populate_pas(dev, cq->buf.umem, page_shift, (*cqb)->pas, 0);
 730         (*cqb)->ctx.log_pg_sz = page_shift - MLX5_ADAPTER_PAGE_SHIFT;
 731
 732         *index = to_mucontext(context)->uuari.uars[0].index;
 733
 734         if (*cqe_size == 64 && MLX5_CAP_GEN(dev->mdev, cqe_compression)) {
 735                 if (ucmd.exp_data.cqe_comp_en == 1 &&
 736                     (ucmd.exp_data.comp_mask & MLX5_EXP_CREATE_CQ_MASK_CQE_COMP_EN)) {
 737                         MLX5_SET(cqc, &(*cqb)->ctx, cqe_compression_en, 1);
 738                         if (ucmd.exp_data.cqe_comp_recv_type ==
 739                             MLX5_IB_CQE_FORMAT_CSUM &&
 740                             (ucmd.exp_data.comp_mask &
 741                             MLX5_EXP_CREATE_CQ_MASK_CQE_COMP_RECV_TYPE))
 742                                 MLX5_SET(cqc, &(*cqb)->ctx, mini_cqe_res_format,
 743                                          MLX5_IB_CQE_FORMAT_CSUM);
 744                 }
 745         }
 746
 747         return 0;
 748
 749 err_db:
 750         mlx5_ib_db_unmap_user(to_mucontext(context), &cq->db);
 751
 752 err_umem:
 753         ib_umem_release(cq->buf.umem);
 754         return err;
 755 }
 756
 757 static void destroy_cq_user(struct mlx5_ib_cq *cq, struct ib_ucontext *context)
 758 {
 759
 760         mlx5_ib_db_unmap_user(to_mucontext(context), &cq->db);
 761         ib_umem_release(cq->buf.umem);
 762 }
 763
 764 static void init_cq_buf(struct mlx5_ib_cq *cq, struct mlx5_ib_cq_buf *buf)
 765 {
 766         int i;
 767         void *cqe;
 768         struct mlx5_cqe64 *cqe64;
 769
 770         for (i = 0; i < buf->nent; i++) {
 771                 cqe = get_cqe_from_buf(buf, i, buf->cqe_size);
 772                 cqe64 = buf->cqe_size == 64 ? cqe : cqe + 64;
 773                 cqe64->op_own = MLX5_CQE_INVALID << 4;
 774         }
 775 }
 776
 777 static int create_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
 778                             int entries, int cqe_size,
 779                             struct mlx5_create_cq_mbox_in **cqb,
 780                             int *index, int *inlen)
 781 {
 782         int err;
 783
 784         err = mlx5_db_alloc(dev->mdev, &cq->db);
 785         if (err)
 786                 return err;
 787
 788         cq->mcq.set_ci_db  = cq->db.db;
 789         cq->mcq.arm_db     = cq->db.db + 1;
 790         cq->mcq.cqe_sz = cqe_size;
 791
 792         err = alloc_cq_buf(dev, &cq->buf, entries, cqe_size);
 793         if (err)
 794                 goto err_db;
 795
 796         init_cq_buf(cq, &cq->buf);
 797
 798         *inlen = sizeof(**cqb) + sizeof(*(*cqb)->pas) * cq->buf.buf.npages;
 799         *cqb = mlx5_vzalloc(*inlen);
 800         if (!*cqb) {
 801                 err = -ENOMEM;
 802                 goto err_buf;
 803         }
 804         mlx5_fill_page_array(&cq->buf.buf, (*cqb)->pas);
 805
 806         (*cqb)->ctx.log_pg_sz = cq->buf.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT;
 807         *index = dev->mdev->priv.uuari.uars[0].index;
 808
 809         return 0;
 810
 811 err_buf:
 812         free_cq_buf(dev, &cq->buf);
 813
 814 err_db:
 815         mlx5_db_free(dev->mdev, &cq->db);
 816         return err;
 817 }
 818
 819 static void destroy_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq)
 820 {
 821         free_cq_buf(dev, &cq->buf);
 822         mlx5_db_free(dev->mdev, &cq->db);
 823 }
 824
 825 struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
 826                                 int entries, int vector,
 827                                 struct ib_ucontext *context,
 828                                 struct ib_udata *udata)
 829 {
 830         struct mlx5_create_cq_mbox_in *cqb = NULL;
 831         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 832         struct mlx5_ib_cq *cq;
 833         int uninitialized_var(index);
 834         int uninitialized_var(inlen);
 835         int cqe_size;
 836         int irqn;
 837         int eqn;
 838         int err;
 839
 840         if (entries < 0 || roundup_pow_of_two(entries + 1) >
 841             (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz))) {
 842                 mlx5_ib_warn(dev, "wrong entries number %d(%ld), max %d\n",
 843                              entries, roundup_pow_of_two(entries + 1),
 844                              1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz));
 845                 return ERR_PTR(-EINVAL);
 846         }
 847
 848         entries = roundup_pow_of_two(entries + 1);
 849
 850         cq = kzalloc(sizeof(*cq), GFP_KERNEL);
 851         if (!cq)
 852                 return ERR_PTR(-ENOMEM);
 853
 854         cq->ibcq.cqe = entries - 1;
 855         mutex_init(&cq->resize_mutex);
 856         spin_lock_init(&cq->lock);
 857         cq->resize_buf = NULL;
 858         cq->resize_umem = NULL;
 859
 860         INIT_LIST_HEAD(&cq->list_send_qp);
 861         INIT_LIST_HEAD(&cq->list_recv_qp);
 862
 863         if (context) {
 864                 err = create_cq_user(dev, udata, context, cq, entries,
 865                                      &cqb, &cqe_size, &index, &inlen);
 866                 if (err)
 867                         goto err_create;
 868         } else {
 869                 cqe_size = (cache_line_size() >= 128 ? 128 : 64);
 870                 err = create_cq_kernel(dev, cq, entries, cqe_size, &cqb,
 871                                        &index, &inlen);
 872                 if (err)
 873                         goto err_create;
 874         }
 875
 876         cq->cqe_size = cqe_size;
 877         cqb->ctx.cqe_sz_flags = cqe_sz_to_mlx_sz(cqe_size) << 5;
 878         cqb->ctx.log_sz_usr_page = cpu_to_be32((ilog2(entries) << 24) | index);
 879         err = mlx5_vector2eqn(dev->mdev, vector, &eqn, &irqn);
 880         if (err)
 881                 goto err_cqb;
 882
 883         cqb->ctx.c_eqn = cpu_to_be16(eqn);
 884         cqb->ctx.db_record_addr = cpu_to_be64(cq->db.dma);
 885
 886         err = mlx5_core_create_cq(dev->mdev, &cq->mcq, cqb, inlen);
 887         if (err)
 888                 goto err_cqb;
 889
 890         mlx5_ib_dbg(dev, "cqn 0x%x\n", cq->mcq.cqn);
 891         cq->mcq.irqn = irqn;
 892         cq->mcq.comp  = mlx5_ib_cq_comp;
 893         cq->mcq.event = mlx5_ib_cq_event;
 894
 895         if (context)
 896                 if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) {
 897                         err = -EFAULT;
 898                         goto err_cmd;
 899                 }
 900
 901
 902         kvfree(cqb);
 903         return &cq->ibcq;
 904
 905 err_cmd:
 906         mlx5_core_destroy_cq(dev->mdev, &cq->mcq);
 907
 908 err_cqb:
 909         kvfree(cqb);
 910         if (context)
 911                 destroy_cq_user(cq, context);
 912         else
 913                 destroy_cq_kernel(dev, cq);
 914
 915 err_create:
 916         kfree(cq);
 917
 918         return ERR_PTR(err);
 919 }
 920
 921
 922 int mlx5_ib_destroy_cq(struct ib_cq *cq)
 923 {
 924         struct mlx5_ib_dev *dev = to_mdev(cq->device);
 925         struct mlx5_ib_cq *mcq = to_mcq(cq);
 926         struct ib_ucontext *context = NULL;
 927
 928         if (cq->uobject)
 929                 context = cq->uobject->context;
 930
 931         mlx5_core_destroy_cq(dev->mdev, &mcq->mcq);
 932         if (context)
 933                 destroy_cq_user(mcq, context);
 934         else
 935                 destroy_cq_kernel(dev, mcq);
 936
 937         kfree(mcq);
 938
 939         return 0;
 940 }
 941
 942 static int is_equal_rsn(struct mlx5_cqe64 *cqe64, u32 rsn)
 943 {
 944         return rsn == (ntohl(cqe64->sop_drop_qpn) & 0xffffff);
 945 }
 946
 947 void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 rsn, struct mlx5_ib_srq *srq)
 948 {
 949         struct mlx5_cqe64 *cqe64, *dest64;
 950         void *cqe, *dest;
 951         u32 prod_index;
 952         int nfreed = 0;
 953         u8 owner_bit;
 954
 955         if (!cq)
 956                 return;
 957
 958         /* First we need to find the current producer index, so we
 959          * know where to start cleaning from.  It doesn't matter if HW
 960          * adds new entries after this loop -- the QP we're worried
 961          * about is already in RESET, so the new entries won't come
 962          * from our QP and therefore don't need to be checked.
 963          */
 964         for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); prod_index++)
 965                 if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe)
 966                         break;
 967
 968         /* Now sweep backwards through the CQ, removing CQ entries
 969          * that match our QP by copying older entries on top of them.
 970          */
 971         while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) {
 972                 cqe = get_cqe(cq, prod_index & cq->ibcq.cqe);
 973                 cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64;
 974                 if (is_equal_rsn(cqe64, rsn)) {
 975                         if (srq && (ntohl(cqe64->srqn) & 0xffffff))
 976                                 mlx5_ib_free_srq_wqe(srq, be16_to_cpu(cqe64->wqe_counter));
 977                         ++nfreed;
 978                 } else if (nfreed) {
 979                         dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe);
 980                         dest64 = (cq->mcq.cqe_sz == 64) ? dest : dest + 64;
 981                         owner_bit = dest64->op_own & MLX5_CQE_OWNER_MASK;
 982                         memcpy(dest, cqe, cq->mcq.cqe_sz);
 983                         dest64->op_own = owner_bit |
 984                                 (dest64->op_own & ~MLX5_CQE_OWNER_MASK);
 985                 }
 986         }
 987
 988         if (nfreed) {
 989                 cq->mcq.cons_index += nfreed;
 990                 /* Make sure update of buffer contents is done before
 991                  * updating consumer index.
 992                  */
 993                 wmb();
 994                 mlx5_cq_set_ci(&cq->mcq);
 995         }
 996 }
 997
 998 void mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq)
 999 {
1000         if (!cq)
1001                 return;
1002
1003         spin_lock_irq(&cq->lock);
1004         __mlx5_ib_cq_clean(cq, qpn, srq);
1005         spin_unlock_irq(&cq->lock);
1006 }
1007
1008 int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
1009 {
1010         struct mlx5_modify_cq_mbox_in *in;
1011         struct mlx5_ib_dev *dev = to_mdev(cq->device);
1012         struct mlx5_ib_cq *mcq = to_mcq(cq);
1013
1014         int err;
1015         u32 fsel = 0;
1016
1017         in = kzalloc(sizeof(*in), GFP_KERNEL);
1018         if (!in)
1019                 return -ENOMEM;
1020
1021         in->cqn = cpu_to_be32(mcq->mcq.cqn);
1022         if (1) {
1023                 if (MLX5_CAP_GEN(dev->mdev, cq_moderation)) {
1024                         fsel |= (MLX5_CQ_MODIFY_PERIOD | MLX5_CQ_MODIFY_COUNT);
1025                         if (cq_period & 0xf000) {
1026                                 /* A value higher than 0xfff is required, better
1027                                  * use the largest value possible. */
1028                                 cq_period = 0xfff;
1029                                 printf("mlx5_ib: INFO: ""period supported is limited to 12 bits\n");
1030                         }
1031
1032                         in->ctx.cq_period = cpu_to_be16(cq_period);
1033                         in->ctx.cq_max_count = cpu_to_be16(cq_count);
1034                 } else {
1035                         err = -ENOSYS;
1036                         goto out;
1037                 }
1038         }
1039         in->field_select = cpu_to_be32(fsel);
1040         err = mlx5_core_modify_cq(dev->mdev, &mcq->mcq, in, sizeof(*in));
1041
1042 out:
1043         kfree(in);
1044         if (err)
1045                 mlx5_ib_warn(dev, "modify cq 0x%x failed\n", mcq->mcq.cqn);
1046
1047         return err;
1048 }
1049
1050 static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
1051                        int entries, struct ib_udata *udata, int *npas,
1052                        int *page_shift, int *cqe_size)
1053 {
1054         struct mlx5_ib_resize_cq ucmd;
1055         struct ib_umem *umem;
1056         int err;
1057         int npages;
1058         struct ib_ucontext *context = cq->buf.umem->context;
1059
1060         err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd));
1061         if (err)
1062                 return err;
1063
1064         if (ucmd.reserved0 || ucmd.reserved1)
1065                 return -EINVAL;
1066
1067         umem = ib_umem_get(context, ucmd.buf_addr, entries * ucmd.cqe_size,
1068                            IB_ACCESS_LOCAL_WRITE, 1);
1069         if (IS_ERR(umem)) {
1070                 err = PTR_ERR(umem);
1071                 return err;
1072         }
1073
1074         mlx5_ib_cont_pages(umem, ucmd.buf_addr, &npages, page_shift,
1075                            npas, NULL);
1076
1077         cq->resize_umem = umem;
1078         *cqe_size = ucmd.cqe_size;
1079
1080         return 0;
1081 }
1082
1083 static void un_resize_user(struct mlx5_ib_cq *cq)
1084 {
1085         ib_umem_release(cq->resize_umem);
1086 }
1087
1088 static int resize_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
1089                          int entries, int cqe_size)
1090 {
1091         int err;
1092
1093         cq->resize_buf = kzalloc(sizeof(*cq->resize_buf), GFP_KERNEL);
1094         if (!cq->resize_buf)
1095                 return -ENOMEM;
1096
1097         err = alloc_cq_buf(dev, cq->resize_buf, entries, cqe_size);
1098         if (err)
1099                 goto ex;
1100
1101         init_cq_buf(cq, cq->resize_buf);
1102
1103         return 0;
1104
1105 ex:
1106         kfree(cq->resize_buf);
1107         return err;
1108 }
1109
1110 static void un_resize_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq)
1111 {
1112         free_cq_buf(dev, cq->resize_buf);
1113         cq->resize_buf = NULL;
1114 }
1115
1116 static int copy_resize_cqes(struct mlx5_ib_cq *cq)
1117 {
1118         struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device);
1119         struct mlx5_cqe64 *scqe64;
1120         struct mlx5_cqe64 *dcqe64;
1121         void *start_cqe;
1122         void *scqe;
1123         void *dcqe;
1124         int ssize;
1125         int dsize;
1126         int i;
1127         u8 sw_own;
1128
1129         ssize = cq->buf.cqe_size;
1130         dsize = cq->resize_buf->cqe_size;
1131         if (ssize != dsize) {
1132                 mlx5_ib_warn(dev, "resize from different cqe size is not supported\n");
1133                 return -EINVAL;
1134         }
1135
1136         i = cq->mcq.cons_index;
1137         scqe = get_sw_cqe(cq, i);
1138         scqe64 = ssize == 64 ? scqe : scqe + 64;
1139         start_cqe = scqe;
1140         if (!scqe) {
1141                 mlx5_ib_warn(dev, "expected cqe in sw ownership\n");
1142                 return -EINVAL;
1143         }
1144
1145         while ((scqe64->op_own >> 4) != MLX5_CQE_RESIZE_CQ) {
1146                 dcqe = get_cqe_from_buf(cq->resize_buf,
1147                                         (i + 1) & (cq->resize_buf->nent),
1148                                         dsize);
1149                 dcqe64 = dsize == 64 ? dcqe : dcqe + 64;
1150                 sw_own = sw_ownership_bit(i + 1, cq->resize_buf->nent);
1151                 memcpy(dcqe, scqe, dsize);
1152                 dcqe64->op_own = (dcqe64->op_own & ~MLX5_CQE_OWNER_MASK) | sw_own;
1153
1154                 ++i;
1155                 scqe = get_sw_cqe(cq, i);
1156                 scqe64 = ssize == 64 ? scqe : scqe + 64;
1157                 if (!scqe) {
1158                         mlx5_ib_warn(dev, "expected cqe in sw ownership\n");
1159                         return -EINVAL;
1160                 }
1161
1162                 if (scqe == start_cqe) {
1163                         printf("mlx5_ib: WARN: ""resize CQ failed to get resize CQE, CQN 0x%x\n", cq->mcq.cqn);
1164                         return -ENOMEM;
1165                 }
1166         }
1167         ++cq->mcq.cons_index;
1168         return 0;
1169 }
1170
1171 int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)
1172 {
1173         struct mlx5_ib_dev *dev = to_mdev(ibcq->device);
1174         struct mlx5_ib_cq *cq = to_mcq(ibcq);
1175         struct mlx5_modify_cq_mbox_in *in;
1176         int err;
1177         int npas;
1178         int page_shift;
1179         int inlen;
1180         int uninitialized_var(cqe_size);
1181         unsigned long flags;
1182
1183         if (!MLX5_CAP_GEN(dev->mdev, cq_resize)) {
1184                 mlx5_ib_warn(dev, "Firmware does not support resize CQ\n");
1185                 return -ENOSYS;
1186         }
1187
1188         if (entries < 1 || roundup_pow_of_two(entries + 1) >
1189             (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz))) {
1190                 mlx5_ib_warn(dev, "wrong entries number %d(%ld), max %d\n",
1191                              entries, roundup_pow_of_two(entries + 1),
1192                              1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz));
1193                 return -EINVAL;
1194         }
1195
1196         entries = roundup_pow_of_two(entries + 1);
1197
1198         if (entries == ibcq->cqe + 1)
1199                 return 0;
1200
1201         mutex_lock(&cq->resize_mutex);
1202         if (udata) {
1203                 err = resize_user(dev, cq, entries, udata, &npas, &page_shift,
1204                                   &cqe_size);
1205         } else {
1206                 cqe_size = 64;
1207                 err = resize_kernel(dev, cq, entries, cqe_size);
1208                 if (!err) {
1209                         npas = cq->resize_buf->buf.npages;
1210                         page_shift = cq->resize_buf->buf.page_shift;
1211                 }
1212         }
1213
1214         if (err) {
1215                 mlx5_ib_warn(dev, "resize failed: %d\n", err);
1216                 goto ex;
1217         }
1218
1219         inlen = sizeof(*in) + npas * sizeof(in->pas[0]);
1220         in = mlx5_vzalloc(inlen);
1221         if (!in) {
1222                 err = -ENOMEM;
1223                 goto ex_resize;
1224         }
1225
1226         if (udata)
1227                 mlx5_ib_populate_pas(dev, cq->resize_umem, page_shift,
1228                                      in->pas, 0);
1229         else
1230                 mlx5_fill_page_array(&cq->resize_buf->buf, in->pas);
1231
1232         in->field_select = cpu_to_be32(MLX5_MODIFY_CQ_MASK_LOG_SIZE  |
1233                                        MLX5_MODIFY_CQ_MASK_PG_OFFSET |
1234                                        MLX5_MODIFY_CQ_MASK_PG_SIZE);
1235         in->ctx.log_pg_sz = page_shift - MLX5_ADAPTER_PAGE_SHIFT;
1236         in->ctx.cqe_sz_flags = cqe_sz_to_mlx_sz(cqe_size) << 5;
1237         in->ctx.page_offset = 0;
1238         in->ctx.log_sz_usr_page = cpu_to_be32(ilog2(entries) << 24);
1239         in->hdr.opmod = cpu_to_be16(MLX5_CQ_OPMOD_RESIZE);
1240         in->cqn = cpu_to_be32(cq->mcq.cqn);
1241
1242         err = mlx5_core_modify_cq(dev->mdev, &cq->mcq, in, inlen);
1243         if (err) {
1244                 mlx5_ib_warn(dev, "modify cq failed: %d\n", err);
1245                 goto ex_alloc;
1246         }
1247
1248         if (udata) {
1249                 cq->ibcq.cqe = entries - 1;
1250                 ib_umem_release(cq->buf.umem);
1251                 cq->buf.umem = cq->resize_umem;
1252                 cq->resize_umem = NULL;
1253         } else {
1254                 struct mlx5_ib_cq_buf tbuf;
1255                 int resized = 0;
1256
1257                 spin_lock_irqsave(&cq->lock, flags);
1258                 if (cq->resize_buf) {
1259                         err = copy_resize_cqes(cq);
1260                         if (!err) {
1261                                 tbuf = cq->buf;
1262                                 cq->buf = *cq->resize_buf;
1263                                 kfree(cq->resize_buf);
1264                                 cq->resize_buf = NULL;
1265                                 resized = 1;
1266                         }
1267                 }
1268                 cq->ibcq.cqe = entries - 1;
1269                 spin_unlock_irqrestore(&cq->lock, flags);
1270                 if (resized)
1271                         free_cq_buf(dev, &tbuf);
1272         }
1273         mutex_unlock(&cq->resize_mutex);
1274
1275         kvfree(in);
1276         return 0;
1277
1278 ex_alloc:
1279         kvfree(in);
1280
1281 ex_resize:
1282         if (udata)
1283                 un_resize_user(cq);
1284         else
1285                 un_resize_kernel(dev, cq);
1286 ex:
1287         mutex_unlock(&cq->resize_mutex);
1288         return err;
1289 }
1290
1291 int mlx5_ib_get_cqe_size(struct mlx5_ib_dev *dev, struct ib_cq *ibcq)
1292 {
1293         struct mlx5_ib_cq *cq;
1294
1295         if (!ibcq)
1296                 return 128;
1297
1298         cq = to_mcq(ibcq);
1299         return cq->cqe_size;
1300 }