sys/dev/mlx5/mlx5_ib/mlx5_ib_cq.c

   1 /*-
   2  * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
  14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  16  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
  17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  23  * SUCH DAMAGE.
  24  *
  25  * $FreeBSD$
  26  */
  27
  28 #include <linux/kref.h>
  29 #include <rdma/ib_umem.h>
  30 #include <rdma/ib_user_verbs.h>
  31 #include "mlx5_ib.h"
  32 #include "user.h"
  33
  34 static void mlx5_ib_cq_comp(struct mlx5_core_cq *cq)
  35 {
  36         struct ib_cq *ibcq = &to_mibcq(cq)->ibcq;
  37
  38         ibcq->comp_handler(ibcq, ibcq->cq_context);
  39 }
  40
  41 static void mlx5_ib_cq_event(struct mlx5_core_cq *mcq, int type)
  42 {
  43         struct mlx5_ib_cq *cq = container_of(mcq, struct mlx5_ib_cq, mcq);
  44         struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device);
  45         struct ib_cq *ibcq = &cq->ibcq;
  46         struct ib_event event;
  47
  48         if (type != MLX5_EVENT_TYPE_CQ_ERROR) {
  49                 mlx5_ib_warn(dev, "Unexpected event type %d on CQ %06x\n",
  50                              type, mcq->cqn);
  51                 return;
  52         }
  53
  54         if (ibcq->event_handler) {
  55                 event.device     = &dev->ib_dev;
  56                 event.event      = IB_EVENT_CQ_ERR;
  57                 event.element.cq = ibcq;
  58                 ibcq->event_handler(&event, ibcq->cq_context);
  59         }
  60 }
  61
  62 static void *get_cqe_from_buf(struct mlx5_ib_cq_buf *buf, int n, int size)
  63 {
  64         return mlx5_buf_offset(&buf->buf, n * size);
  65 }
  66
  67 static void *get_cqe(struct mlx5_ib_cq *cq, int n)
  68 {
  69         return get_cqe_from_buf(&cq->buf, n, cq->mcq.cqe_sz);
  70 }
  71
  72 static u8 sw_ownership_bit(int n, int nent)
  73 {
  74         return (n & nent) ? 1 : 0;
  75 }
  76
  77 static void *get_sw_cqe(struct mlx5_ib_cq *cq, int n)
  78 {
  79         void *cqe = get_cqe(cq, n & cq->ibcq.cqe);
  80         struct mlx5_cqe64 *cqe64;
  81
  82         cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64;
  83
  84         if (likely((cqe64->op_own) >> 4 != MLX5_CQE_INVALID) &&
  85             !((cqe64->op_own & MLX5_CQE_OWNER_MASK) ^ !!(n & (cq->ibcq.cqe + 1)))) {
  86                 return cqe;
  87         } else {
  88                 return NULL;
  89         }
  90 }
  91
  92 static void *next_cqe_sw(struct mlx5_ib_cq *cq)
  93 {
  94         return get_sw_cqe(cq, cq->mcq.cons_index);
  95 }
  96
  97 static enum ib_wc_opcode get_umr_comp(struct mlx5_ib_wq *wq, int idx)
  98 {
  99         switch (wq->swr_ctx[idx].wr_data) {
 100         case IB_WR_LOCAL_INV:
 101                 return IB_WC_LOCAL_INV;
 102
 103         case IB_WR_FAST_REG_MR:
 104                 return IB_WC_FAST_REG_MR;
 105
 106         default:
 107                 printf("mlx5_ib: WARN: ""unknown completion status\n");
 108                 return 0;
 109         }
 110 }
 111
 112 static void handle_good_req(struct ib_wc *wc, struct mlx5_cqe64 *cqe,
 113                             struct mlx5_ib_wq *wq, int idx)
 114 {
 115         wc->wc_flags = 0;
 116         switch (be32_to_cpu(cqe->sop_drop_qpn) >> 24) {
 117         case MLX5_OPCODE_RDMA_WRITE_IMM:
 118                 wc->wc_flags |= IB_WC_WITH_IMM;
 119         case MLX5_OPCODE_RDMA_WRITE:
 120                 wc->opcode    = IB_WC_RDMA_WRITE;
 121                 break;
 122         case MLX5_OPCODE_SEND_IMM:
 123                 wc->wc_flags |= IB_WC_WITH_IMM;
 124         case MLX5_OPCODE_NOP:
 125         case MLX5_OPCODE_SEND:
 126         case MLX5_OPCODE_SEND_INVAL:
 127                 wc->opcode    = IB_WC_SEND;
 128                 break;
 129         case MLX5_OPCODE_RDMA_READ:
 130                 wc->opcode    = IB_WC_RDMA_READ;
 131                 wc->byte_len  = be32_to_cpu(cqe->byte_cnt);
 132                 break;
 133         case MLX5_OPCODE_ATOMIC_CS:
 134                 wc->opcode    = IB_WC_COMP_SWAP;
 135                 wc->byte_len  = 8;
 136                 break;
 137         case MLX5_OPCODE_ATOMIC_FA:
 138                 wc->opcode    = IB_WC_FETCH_ADD;
 139                 wc->byte_len  = 8;
 140                 break;
 141         case MLX5_OPCODE_ATOMIC_MASKED_CS:
 142                 wc->opcode    = IB_WC_MASKED_COMP_SWAP;
 143                 wc->byte_len  = 8;
 144                 break;
 145         case MLX5_OPCODE_ATOMIC_MASKED_FA:
 146                 wc->opcode    = IB_WC_MASKED_FETCH_ADD;
 147                 wc->byte_len  = 8;
 148                 break;
 149         case MLX5_OPCODE_BIND_MW:
 150                 wc->opcode    = IB_WC_BIND_MW;
 151                 break;
 152         case MLX5_OPCODE_UMR:
 153                 wc->opcode = get_umr_comp(wq, idx);
 154                 break;
 155         }
 156 }
 157
 158 enum {
 159         MLX5_GRH_IN_BUFFER = 1,
 160         MLX5_GRH_IN_CQE    = 2,
 161 };
 162
 163 static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe,
 164                              struct mlx5_ib_qp *qp)
 165 {
 166         struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.device);
 167         struct mlx5_ib_srq *srq;
 168         struct mlx5_ib_wq *wq;
 169         u16 wqe_ctr;
 170         u8 g;
 171 #if defined(DX_ROCE_V1_5) || defined(DX_WINDOWS)
 172         u8 udp_header_valid;
 173 #endif
 174
 175         if (qp->ibqp.srq || qp->ibqp.xrcd) {
 176                 struct mlx5_core_srq *msrq = NULL;
 177
 178                 if (qp->ibqp.xrcd) {
 179                         msrq = mlx5_core_get_srq(dev->mdev,
 180                                                  be32_to_cpu(cqe->srqn));
 181                         srq = to_mibsrq(msrq);
 182                 } else {
 183                         srq = to_msrq(qp->ibqp.srq);
 184                 }
 185                 if (srq) {
 186                         wqe_ctr = be16_to_cpu(cqe->wqe_counter);
 187                         wc->wr_id = srq->wrid[wqe_ctr];
 188                         mlx5_ib_free_srq_wqe(srq, wqe_ctr);
 189                         if (msrq && atomic_dec_and_test(&msrq->refcount))
 190                                 complete(&msrq->free);
 191                 }
 192         } else {
 193                 wq        = &qp->rq;
 194                 wc->wr_id = wq->rwr_ctx[wq->tail & (wq->wqe_cnt - 1)].wrid;
 195                 ++wq->tail;
 196         }
 197         wc->byte_len = be32_to_cpu(cqe->byte_cnt);
 198
 199         switch (cqe->op_own >> 4) {
 200         case MLX5_CQE_RESP_WR_IMM:
 201                 wc->opcode      = IB_WC_RECV_RDMA_WITH_IMM;
 202                 wc->wc_flags    = IB_WC_WITH_IMM;
 203                 wc->ex.imm_data = cqe->imm_inval_pkey;
 204                 break;
 205         case MLX5_CQE_RESP_SEND:
 206                 wc->opcode   = IB_WC_RECV;
 207                 wc->wc_flags = 0;
 208                 break;
 209         case MLX5_CQE_RESP_SEND_IMM:
 210                 wc->opcode      = IB_WC_RECV;
 211                 wc->wc_flags    = IB_WC_WITH_IMM;
 212                 wc->ex.imm_data = cqe->imm_inval_pkey;
 213                 break;
 214         case MLX5_CQE_RESP_SEND_INV:
 215                 wc->opcode      = IB_WC_RECV;
 216                 wc->wc_flags    = IB_WC_WITH_INVALIDATE;
 217                 wc->ex.invalidate_rkey = be32_to_cpu(cqe->imm_inval_pkey);
 218                 break;
 219         }
 220         wc->slid           = be16_to_cpu(cqe->slid);
 221         wc->sl             = (be32_to_cpu(cqe->flags_rqpn) >> 24) & 0xf;
 222         wc->src_qp         = be32_to_cpu(cqe->flags_rqpn) & 0xffffff;
 223         wc->dlid_path_bits = cqe->ml_path;
 224         g = (be32_to_cpu(cqe->flags_rqpn) >> 28) & 3;
 225         wc->wc_flags |= g ? IB_WC_GRH : 0;
 226         wc->pkey_index     = be32_to_cpu(cqe->imm_inval_pkey) & 0xffff;
 227
 228 #if defined(DX_ROCE_V1_5) || defined(DX_WINDOWS)
 229         udp_header_valid = wc->sl & 0x8;
 230         if (udp_header_valid)
 231                 wc->wc_flags |= IB_WC_WITH_UDP_HDR;
 232
 233 #endif
 234 }
 235
 236 static void dump_cqe(struct mlx5_ib_dev *dev, struct mlx5_err_cqe *cqe)
 237 {
 238         __be32 *p = (__be32 *)cqe;
 239         int i;
 240
 241         mlx5_ib_warn(dev, "dump error cqe\n");
 242         for (i = 0; i < sizeof(*cqe) / 16; i++, p += 4)
 243                 printf("mlx5_ib: INFO: ""%08x %08x %08x %08x\n", be32_to_cpu(p[0]), be32_to_cpu(p[1]), be32_to_cpu(p[2]), be32_to_cpu(p[3]));
 244 }
 245
 246 static void mlx5_handle_error_cqe(struct mlx5_ib_dev *dev,
 247                                   struct mlx5_err_cqe *cqe,
 248                                   struct ib_wc *wc)
 249 {
 250         int dump = 1;
 251
 252         switch (cqe->syndrome) {
 253         case MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR:
 254                 wc->status = IB_WC_LOC_LEN_ERR;
 255                 break;
 256         case MLX5_CQE_SYNDROME_LOCAL_QP_OP_ERR:
 257                 wc->status = IB_WC_LOC_QP_OP_ERR;
 258                 break;
 259         case MLX5_CQE_SYNDROME_LOCAL_PROT_ERR:
 260                 wc->status = IB_WC_LOC_PROT_ERR;
 261                 break;
 262         case MLX5_CQE_SYNDROME_WR_FLUSH_ERR:
 263                 dump = 0;
 264                 wc->status = IB_WC_WR_FLUSH_ERR;
 265                 break;
 266         case MLX5_CQE_SYNDROME_MW_BIND_ERR:
 267                 wc->status = IB_WC_MW_BIND_ERR;
 268                 break;
 269         case MLX5_CQE_SYNDROME_BAD_RESP_ERR:
 270                 wc->status = IB_WC_BAD_RESP_ERR;
 271                 break;
 272         case MLX5_CQE_SYNDROME_LOCAL_ACCESS_ERR:
 273                 wc->status = IB_WC_LOC_ACCESS_ERR;
 274                 break;
 275         case MLX5_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR:
 276                 wc->status = IB_WC_REM_INV_REQ_ERR;
 277                 break;
 278         case MLX5_CQE_SYNDROME_REMOTE_ACCESS_ERR:
 279                 wc->status = IB_WC_REM_ACCESS_ERR;
 280                 break;
 281         case MLX5_CQE_SYNDROME_REMOTE_OP_ERR:
 282                 wc->status = IB_WC_REM_OP_ERR;
 283                 break;
 284         case MLX5_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:
 285                 wc->status = IB_WC_RETRY_EXC_ERR;
 286                 dump = 0;
 287                 break;
 288         case MLX5_CQE_SYNDROME_RNR_RETRY_EXC_ERR:
 289                 wc->status = IB_WC_RNR_RETRY_EXC_ERR;
 290                 dump = 0;
 291                 break;
 292         case MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR:
 293                 wc->status = IB_WC_REM_ABORT_ERR;
 294                 break;
 295         default:
 296                 wc->status = IB_WC_GENERAL_ERR;
 297                 break;
 298         }
 299
 300         wc->vendor_err = cqe->vendor_err_synd;
 301         if (dump)
 302                 dump_cqe(dev, cqe);
 303 }
 304
 305 static int is_atomic_response(struct mlx5_ib_qp *qp, u16 idx)
 306 {
 307         /* TBD: waiting decision
 308         */
 309         return 0;
 310 }
 311
 312 static void *mlx5_get_atomic_laddr(struct mlx5_ib_qp *qp, u16 idx)
 313 {
 314         struct mlx5_wqe_data_seg *dpseg;
 315         void *addr;
 316
 317         dpseg = mlx5_get_send_wqe(qp, idx) + sizeof(struct mlx5_wqe_ctrl_seg) +
 318                 sizeof(struct mlx5_wqe_raddr_seg) +
 319                 sizeof(struct mlx5_wqe_atomic_seg);
 320         addr = (void *)(uintptr_t)be64_to_cpu(dpseg->addr);
 321         return addr;
 322 }
 323
 324 static void handle_atomic(struct mlx5_ib_qp *qp, struct mlx5_cqe64 *cqe64,
 325                           u16 idx)
 326 {
 327         void *addr;
 328         int byte_count;
 329         int i;
 330
 331         if (!is_atomic_response(qp, idx))
 332                 return;
 333
 334         byte_count = be32_to_cpu(cqe64->byte_cnt);
 335         addr = mlx5_get_atomic_laddr(qp, idx);
 336
 337         if (byte_count == 4) {
 338                 *(u32 *)addr = be32_to_cpu(*((__be32 *)addr));
 339         } else {
 340                 for (i = 0; i < byte_count; i += 8) {
 341                         *(u64 *)addr = be64_to_cpu(*((__be64 *)addr));
 342                         addr += 8;
 343                 }
 344         }
 345
 346         return;
 347 }
 348
 349 static void handle_atomics(struct mlx5_ib_qp *qp, struct mlx5_cqe64 *cqe64,
 350                            u16 tail, u16 head)
 351 {
 352         u16 idx;
 353
 354         do {
 355                 idx = tail & (qp->sq.wqe_cnt - 1);
 356                 handle_atomic(qp, cqe64, idx);
 357                 if (idx == head)
 358                         break;
 359
 360                 tail = qp->sq.swr_ctx[idx].w_list.next;
 361         } while (1);
 362         tail = qp->sq.swr_ctx[idx].w_list.next;
 363         qp->sq.last_poll = tail;
 364 }
 365
 366 static void free_cq_buf(struct mlx5_ib_dev *dev, struct mlx5_ib_cq_buf *buf)
 367 {
 368         mlx5_buf_free(dev->mdev, &buf->buf);
 369 }
 370
 371 static void sw_send_comp(struct mlx5_ib_qp *qp, int num_entries,
 372                          struct ib_wc *wc, int *npolled)
 373 {
 374         struct mlx5_ib_wq *wq;
 375         unsigned cur;
 376         unsigned idx;
 377         int np;
 378         int i;
 379
 380         wq = &qp->sq;
 381         cur = wq->head - wq->tail;
 382         np = *npolled;
 383
 384         if (cur == 0)
 385                 return;
 386
 387         for (i = 0;  i < cur && np < num_entries; i++) {
 388                 idx = wq->last_poll & (wq->wqe_cnt - 1);
 389                 wc->wr_id = wq->swr_ctx[idx].wrid;
 390                 wc->status = IB_WC_WR_FLUSH_ERR;
 391                 wc->vendor_err = MLX5_CQE_SYNDROME_WR_FLUSH_ERR;
 392                 wq->tail++;
 393                 np++;
 394                 wc->qp = &qp->ibqp;
 395                 wc++;
 396                 wq->last_poll = wq->swr_ctx[idx].w_list.next;
 397         }
 398         *npolled = np;
 399 }
 400
 401 static void sw_recv_comp(struct mlx5_ib_qp *qp, int num_entries,
 402                          struct ib_wc *wc, int *npolled)
 403 {
 404         struct mlx5_ib_wq *wq;
 405         unsigned cur;
 406         int np;
 407         int i;
 408
 409         wq = &qp->rq;
 410         cur = wq->head - wq->tail;
 411         np = *npolled;
 412
 413         if (cur == 0)
 414                 return;
 415
 416         for (i = 0;  i < cur && np < num_entries; i++) {
 417                 wc->wr_id = wq->rwr_ctx[wq->tail & (wq->wqe_cnt - 1)].wrid;
 418                 wc->status = IB_WC_WR_FLUSH_ERR;
 419                 wc->vendor_err = MLX5_CQE_SYNDROME_WR_FLUSH_ERR;
 420                 wq->tail++;
 421                 np++;
 422                 wc->qp = &qp->ibqp;
 423                 wc++;
 424         }
 425         *npolled = np;
 426 }
 427
 428 static void mlx5_ib_poll_sw_comp(struct mlx5_ib_cq *cq, int num_entries,
 429                                  struct ib_wc *wc, int *npolled)
 430 {
 431         struct mlx5_ib_qp *qp;
 432
 433         *npolled = 0;
 434         /* Find uncompleted WQEs belonging to that cq and retrun mmics ones */
 435         list_for_each_entry(qp, &cq->list_send_qp, cq_send_list) {
 436                 sw_send_comp(qp, num_entries, wc + *npolled, npolled);
 437                 if (*npolled >= num_entries)
 438                         return;
 439         }
 440
 441         list_for_each_entry(qp, &cq->list_recv_qp, cq_recv_list) {
 442                 sw_recv_comp(qp, num_entries, wc + *npolled, npolled);
 443                 if (*npolled >= num_entries)
 444                         return;
 445         }
 446 }
 447
 448 static inline u32 mlx5_ib_base_mkey(const u32 key)
 449 {
 450         return key & 0xffffff00u;
 451 }
 452
 453 static int mlx5_poll_one(struct mlx5_ib_cq *cq,
 454                          struct mlx5_ib_qp **cur_qp,
 455                          struct ib_wc *wc)
 456 {
 457         struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device);
 458         struct mlx5_err_cqe *err_cqe;
 459         struct mlx5_cqe64 *cqe64;
 460         struct mlx5_core_qp *mqp;
 461         struct mlx5_ib_wq *wq;
 462         struct mlx5_sig_err_cqe *sig_err_cqe;
 463         struct mlx5_core_mr *mmr;
 464         struct mlx5_ib_mr *mr;
 465         unsigned long flags;
 466         u8 opcode;
 467         u32 qpn;
 468         u16 wqe_ctr;
 469         void *cqe;
 470         int idx;
 471
 472 repoll:
 473         cqe = next_cqe_sw(cq);
 474         if (!cqe)
 475                 return -EAGAIN;
 476
 477         cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64;
 478
 479         ++cq->mcq.cons_index;
 480
 481         /* Make sure we read CQ entry contents after we've checked the
 482          * ownership bit.
 483          */
 484         rmb();
 485
 486         opcode = cqe64->op_own >> 4;
 487         if (unlikely(opcode == MLX5_CQE_RESIZE_CQ)) {
 488                 if (likely(cq->resize_buf)) {
 489                         free_cq_buf(dev, &cq->buf);
 490                         cq->buf = *cq->resize_buf;
 491                         kfree(cq->resize_buf);
 492                         cq->resize_buf = NULL;
 493                         goto repoll;
 494                 } else {
 495                         mlx5_ib_warn(dev, "unexpected resize cqe\n");
 496                 }
 497         }
 498
 499         qpn = ntohl(cqe64->sop_drop_qpn) & 0xffffff;
 500         if (!*cur_qp || (qpn != (*cur_qp)->ibqp.qp_num)) {
 501                 /* We do not have to take the QP table lock here,
 502                  * because CQs will be locked while QPs are removed
 503                  * from the table.
 504                  */
 505                 mqp = __mlx5_qp_lookup(dev->mdev, qpn);
 506                 if (unlikely(!mqp)) {
 507                         mlx5_ib_warn(dev, "CQE@CQ %06x for unknown QPN %6x\n",
 508                                      cq->mcq.cqn, qpn);
 509                         return -EINVAL;
 510                 }
 511
 512                 *cur_qp = to_mibqp(mqp);
 513         }
 514
 515         wc->qp  = &(*cur_qp)->ibqp;
 516         switch (opcode) {
 517         case MLX5_CQE_REQ:
 518                 wq = &(*cur_qp)->sq;
 519                 wqe_ctr = be16_to_cpu(cqe64->wqe_counter);
 520                 idx = wqe_ctr & (wq->wqe_cnt - 1);
 521                 handle_good_req(wc, cqe64, wq, idx);
 522                 handle_atomics(*cur_qp, cqe64, wq->last_poll, idx);
 523                 wc->wr_id = wq->swr_ctx[idx].wrid;
 524                 wq->tail = wq->swr_ctx[idx].wqe_head + 1;
 525                 if (unlikely(wq->swr_ctx[idx].w_list.opcode &
 526                     MLX5_OPCODE_SIGNATURE_CANCELED))
 527                         wc->status = IB_WC_GENERAL_ERR;
 528                 else
 529                         wc->status = IB_WC_SUCCESS;
 530                 break;
 531         case MLX5_CQE_RESP_WR_IMM:
 532         case MLX5_CQE_RESP_SEND:
 533         case MLX5_CQE_RESP_SEND_IMM:
 534         case MLX5_CQE_RESP_SEND_INV:
 535                 handle_responder(wc, cqe64, *cur_qp);
 536                 wc->status = IB_WC_SUCCESS;
 537                 break;
 538         case MLX5_CQE_RESIZE_CQ:
 539                 break;
 540         case MLX5_CQE_REQ_ERR:
 541         case MLX5_CQE_RESP_ERR:
 542                 err_cqe = (struct mlx5_err_cqe *)cqe64;
 543                 mlx5_handle_error_cqe(dev, err_cqe, wc);
 544                 mlx5_ib_dbg(dev, "%s error cqe on cqn 0x%x:\n",
 545                             opcode == MLX5_CQE_REQ_ERR ?
 546                             "Requestor" : "Responder", cq->mcq.cqn);
 547                 mlx5_ib_dbg(dev, "syndrome 0x%x, vendor syndrome 0x%x\n",
 548                             err_cqe->syndrome, err_cqe->vendor_err_synd);
 549                 if (opcode == MLX5_CQE_REQ_ERR) {
 550                         wq = &(*cur_qp)->sq;
 551                         wqe_ctr = be16_to_cpu(cqe64->wqe_counter);
 552                         idx = wqe_ctr & (wq->wqe_cnt - 1);
 553                         wc->wr_id = wq->swr_ctx[idx].wrid;
 554                         wq->tail = wq->swr_ctx[idx].wqe_head + 1;
 555                 } else {
 556                         struct mlx5_ib_srq *srq;
 557
 558                         if ((*cur_qp)->ibqp.srq) {
 559                                 srq = to_msrq((*cur_qp)->ibqp.srq);
 560                                 wqe_ctr = be16_to_cpu(cqe64->wqe_counter);
 561                                 wc->wr_id = srq->wrid[wqe_ctr];
 562                                 mlx5_ib_free_srq_wqe(srq, wqe_ctr);
 563                         } else {
 564                                 wq = &(*cur_qp)->rq;
 565                                 wc->wr_id = wq->rwr_ctx[wq->tail & (wq->wqe_cnt - 1)].wrid;
 566                                 ++wq->tail;
 567                         }
 568                 }
 569                 break;
 570         case MLX5_CQE_SIG_ERR:
 571                 sig_err_cqe = (struct mlx5_sig_err_cqe *)cqe64;
 572
 573                 spin_lock_irqsave(&dev->mdev->priv.mr_table.lock, flags);
 574                 mmr = __mlx5_mr_lookup(dev->mdev,
 575                                          mlx5_ib_base_mkey(be32_to_cpu(sig_err_cqe->mkey)));
 576                 if (unlikely(!mmr)) {
 577                         spin_unlock_irqrestore(&dev->mdev->priv.mr_table.lock, flags);
 578                         mlx5_ib_warn(dev, "CQE@CQ %06x for unknown MR %6x\n",
 579                                      cq->mcq.cqn, be32_to_cpu(sig_err_cqe->mkey));
 580                         return -EINVAL;
 581                 }
 582
 583                 mr = to_mibmr(mmr);
 584                 mr->sig->sig_err_exists = true;
 585                 mr->sig->sigerr_count++;
 586
 587                 mlx5_ib_warn(dev, "CQN: 0x%x Got SIGERR\n", cq->mcq.cqn);
 588
 589                 spin_unlock_irqrestore(&dev->mdev->priv.mr_table.lock, flags);
 590                 goto repoll;
 591         }
 592
 593         return 0;
 594 }
 595
 596 int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 597 {
 598         struct mlx5_ib_cq *cq = to_mcq(ibcq);
 599         struct mlx5_ib_qp *cur_qp = NULL;
 600         struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device);
 601         struct mlx5_core_dev *mdev = dev->mdev;
 602         unsigned long flags;
 603         int npolled;
 604         int err = 0;
 605
 606         spin_lock_irqsave(&cq->lock, flags);
 607         if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
 608                 mlx5_ib_poll_sw_comp(cq, num_entries, wc, &npolled);
 609                 goto out;
 610         }
 611
 612         for (npolled = 0; npolled < num_entries; npolled++) {
 613                 err = mlx5_poll_one(cq, &cur_qp, wc + npolled);
 614                 if (err)
 615                         break;
 616         }
 617
 618         if (npolled)
 619                 mlx5_cq_set_ci(&cq->mcq);
 620 out:
 621         spin_unlock_irqrestore(&cq->lock, flags);
 622
 623         if (err == 0 || err == -EAGAIN)
 624                 return npolled;
 625         else
 626                 return err;
 627 }
 628
 629 int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 630 {
 631         struct mlx5_core_dev *mdev = to_mdev(ibcq->device)->mdev;
 632         void __iomem *uar_page = mdev->priv.uuari.uars[0].map;
 633
 634
 635         mlx5_cq_arm(&to_mcq(ibcq)->mcq,
 636                     (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
 637                     MLX5_CQ_DB_REQ_NOT_SOL : MLX5_CQ_DB_REQ_NOT,
 638                     uar_page,
 639                     MLX5_GET_DOORBELL_LOCK(&mdev->priv.cq_uar_lock),
 640                     to_mcq(ibcq)->mcq.cons_index);
 641
 642         return 0;
 643 }
 644
 645 static int alloc_cq_buf(struct mlx5_ib_dev *dev, struct mlx5_ib_cq_buf *buf,
 646                         int nent, int cqe_size)
 647 {
 648         int err;
 649
 650         err = mlx5_buf_alloc(dev->mdev, nent * cqe_size,
 651                              PAGE_SIZE * 2, &buf->buf);
 652         if (err) {
 653                 mlx5_ib_err(dev, "alloc failed\n");
 654                 return err;
 655         }
 656
 657         buf->cqe_size = cqe_size;
 658         buf->nent = nent;
 659
 660         return 0;
 661 }
 662
 663 static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
 664                           struct ib_ucontext *context, struct mlx5_ib_cq *cq,
 665                           int entries, struct mlx5_create_cq_mbox_in **cqb,
 666                           int *cqe_size, int *index, int *inlen)
 667 {
 668         struct mlx5_exp_ib_create_cq ucmd;
 669         size_t ucmdlen;
 670         int page_shift;
 671         int npages;
 672         int ncont;
 673         int err;
 674
 675         memset(&ucmd, 0, sizeof(ucmd));
 676
 677         ucmdlen =
 678                 (udata->inlen - sizeof(struct ib_uverbs_cmd_hdr) <
 679                 sizeof(struct mlx5_ib_create_cq)) ?
 680                 (sizeof(struct mlx5_ib_create_cq) - sizeof(ucmd.reserved)) :
 681                 sizeof(struct mlx5_ib_create_cq);
 682
 683         if (ib_copy_from_udata(&ucmd, udata, ucmdlen)) {
 684                 mlx5_ib_err(dev, "copy failed\n");
 685                 return -EFAULT;
 686         }
 687
 688         if (ucmdlen == sizeof(ucmd) && ucmd.reserved != 0) {
 689                 mlx5_ib_err(dev, "command corrupted\n");
 690                 return -EINVAL;
 691         }
 692
 693         if (ucmd.cqe_size != 64 && ucmd.cqe_size != 128) {
 694                 mlx5_ib_warn(dev, "wrong CQE size %d\n", ucmd.cqe_size);
 695                 return -EINVAL;
 696         }
 697
 698         *cqe_size = ucmd.cqe_size;
 699
 700         cq->buf.umem = ib_umem_get(context, ucmd.buf_addr,
 701                                    entries * ucmd.cqe_size,
 702                                    IB_ACCESS_LOCAL_WRITE, 1);
 703         if (IS_ERR(cq->buf.umem)) {
 704                 err = PTR_ERR(cq->buf.umem);
 705                 return err;
 706         }
 707
 708         err = mlx5_ib_db_map_user(to_mucontext(context), ucmd.db_addr,
 709                                   &cq->db);
 710         if (err) {
 711                 mlx5_ib_warn(dev, "map failed\n");
 712                 goto err_umem;
 713         }
 714
 715         mlx5_ib_cont_pages(cq->buf.umem, ucmd.buf_addr, &npages, &page_shift,
 716                            &ncont, NULL);
 717         mlx5_ib_dbg(dev, "addr 0x%llx, size %u, npages %d, page_shift %d, ncont %d\n",
 718                     (unsigned long long)ucmd.buf_addr, entries * ucmd.cqe_size,
 719                     npages, page_shift, ncont);
 720
 721         *inlen = sizeof(**cqb) + sizeof(*(*cqb)->pas) * ncont;
 722         *cqb = mlx5_vzalloc(*inlen);
 723         if (!*cqb) {
 724                 err = -ENOMEM;
 725                 goto err_db;
 726         }
 727
 728         mlx5_ib_populate_pas(dev, cq->buf.umem, page_shift, (*cqb)->pas, 0);
 729         (*cqb)->ctx.log_pg_sz = page_shift - MLX5_ADAPTER_PAGE_SHIFT;
 730
 731         *index = to_mucontext(context)->uuari.uars[0].index;
 732
 733         if (*cqe_size == 64 && MLX5_CAP_GEN(dev->mdev, cqe_compression)) {
 734                 if (ucmd.exp_data.cqe_comp_en == 1 &&
 735                     (ucmd.exp_data.comp_mask & MLX5_EXP_CREATE_CQ_MASK_CQE_COMP_EN)) {
 736                         MLX5_SET(cqc, &(*cqb)->ctx, cqe_compression_en, 1);
 737                         if (ucmd.exp_data.cqe_comp_recv_type ==
 738                             MLX5_IB_CQE_FORMAT_CSUM &&
 739                             (ucmd.exp_data.comp_mask &
 740                             MLX5_EXP_CREATE_CQ_MASK_CQE_COMP_RECV_TYPE))
 741                                 MLX5_SET(cqc, &(*cqb)->ctx, mini_cqe_res_format,
 742                                          MLX5_IB_CQE_FORMAT_CSUM);
 743                 }
 744         }
 745
 746         return 0;
 747
 748 err_db:
 749         mlx5_ib_db_unmap_user(to_mucontext(context), &cq->db);
 750
 751 err_umem:
 752         ib_umem_release(cq->buf.umem);
 753         return err;
 754 }
 755
 756 static void destroy_cq_user(struct mlx5_ib_cq *cq, struct ib_ucontext *context)
 757 {
 758
 759         mlx5_ib_db_unmap_user(to_mucontext(context), &cq->db);
 760         ib_umem_release(cq->buf.umem);
 761 }
 762
 763 static void init_cq_buf(struct mlx5_ib_cq *cq, struct mlx5_ib_cq_buf *buf)
 764 {
 765         int i;
 766         void *cqe;
 767         struct mlx5_cqe64 *cqe64;
 768
 769         for (i = 0; i < buf->nent; i++) {
 770                 cqe = get_cqe_from_buf(buf, i, buf->cqe_size);
 771                 cqe64 = buf->cqe_size == 64 ? cqe : cqe + 64;
 772                 cqe64->op_own = MLX5_CQE_INVALID << 4;
 773         }
 774 }
 775
 776 static int create_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
 777                             int entries, int cqe_size,
 778                             struct mlx5_create_cq_mbox_in **cqb,
 779                             int *index, int *inlen)
 780 {
 781         int err;
 782
 783         err = mlx5_db_alloc(dev->mdev, &cq->db);
 784         if (err)
 785                 return err;
 786
 787         cq->mcq.set_ci_db  = cq->db.db;
 788         cq->mcq.arm_db     = cq->db.db + 1;
 789         cq->mcq.cqe_sz = cqe_size;
 790
 791         err = alloc_cq_buf(dev, &cq->buf, entries, cqe_size);
 792         if (err)
 793                 goto err_db;
 794
 795         init_cq_buf(cq, &cq->buf);
 796
 797         *inlen = sizeof(**cqb) + sizeof(*(*cqb)->pas) * cq->buf.buf.npages;
 798         *cqb = mlx5_vzalloc(*inlen);
 799         if (!*cqb) {
 800                 err = -ENOMEM;
 801                 goto err_buf;
 802         }
 803         mlx5_fill_page_array(&cq->buf.buf, (*cqb)->pas);
 804
 805         (*cqb)->ctx.log_pg_sz = cq->buf.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT;
 806         *index = dev->mdev->priv.uuari.uars[0].index;
 807
 808         return 0;
 809
 810 err_buf:
 811         free_cq_buf(dev, &cq->buf);
 812
 813 err_db:
 814         mlx5_db_free(dev->mdev, &cq->db);
 815         return err;
 816 }
 817
 818 static void destroy_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq)
 819 {
 820         free_cq_buf(dev, &cq->buf);
 821         mlx5_db_free(dev->mdev, &cq->db);
 822 }
 823
 824 struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
 825                                 struct ib_cq_init_attr *attr,
 826                                 struct ib_ucontext *context,
 827                                 struct ib_udata *udata)
 828 {
 829         struct mlx5_create_cq_mbox_in *cqb = NULL;
 830         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 831         struct mlx5_ib_cq *cq;
 832         int entries = attr->cqe;
 833         int vector = attr->comp_vector;
 834         int uninitialized_var(index);
 835         int uninitialized_var(inlen);
 836         int cqe_size;
 837         int irqn;
 838         int eqn;
 839         int err;
 840
 841         if (entries < 0 || roundup_pow_of_two(entries + 1) >
 842             (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz))) {
 843                 mlx5_ib_warn(dev, "wrong entries number %d(%ld), max %d\n",
 844                              entries, roundup_pow_of_two(entries + 1),
 845                              1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz));
 846                 return ERR_PTR(-EINVAL);
 847         }
 848
 849         entries = roundup_pow_of_two(entries + 1);
 850
 851         cq = kzalloc(sizeof(*cq), GFP_KERNEL);
 852         if (!cq)
 853                 return ERR_PTR(-ENOMEM);
 854
 855         cq->ibcq.cqe = entries - 1;
 856         mutex_init(&cq->resize_mutex);
 857         spin_lock_init(&cq->lock);
 858         cq->resize_buf = NULL;
 859         cq->resize_umem = NULL;
 860
 861         INIT_LIST_HEAD(&cq->list_send_qp);
 862         INIT_LIST_HEAD(&cq->list_recv_qp);
 863
 864         if (context) {
 865                 err = create_cq_user(dev, udata, context, cq, entries,
 866                                      &cqb, &cqe_size, &index, &inlen);
 867                 if (err)
 868                         goto err_create;
 869         } else {
 870                 cqe_size = (cache_line_size() >= 128 ? 128 : 64);
 871                 err = create_cq_kernel(dev, cq, entries, cqe_size, &cqb,
 872                                        &index, &inlen);
 873                 if (err)
 874                         goto err_create;
 875         }
 876
 877         cq->cqe_size = cqe_size;
 878         cqb->ctx.cqe_sz_flags = cqe_sz_to_mlx_sz(cqe_size) << 5;
 879         cqb->ctx.log_sz_usr_page = cpu_to_be32((ilog2(entries) << 24) | index);
 880         err = mlx5_vector2eqn(dev->mdev, vector, &eqn, &irqn);
 881         if (err)
 882                 goto err_cqb;
 883
 884         cqb->ctx.c_eqn = cpu_to_be16(eqn);
 885         cqb->ctx.db_record_addr = cpu_to_be64(cq->db.dma);
 886
 887         err = mlx5_core_create_cq(dev->mdev, &cq->mcq, cqb, inlen);
 888         if (err)
 889                 goto err_cqb;
 890
 891         mlx5_ib_dbg(dev, "cqn 0x%x\n", cq->mcq.cqn);
 892         cq->mcq.irqn = irqn;
 893         cq->mcq.comp  = mlx5_ib_cq_comp;
 894         cq->mcq.event = mlx5_ib_cq_event;
 895
 896         if (context)
 897                 if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) {
 898                         err = -EFAULT;
 899                         goto err_cmd;
 900                 }
 901
 902
 903         kvfree(cqb);
 904         return &cq->ibcq;
 905
 906 err_cmd:
 907         mlx5_core_destroy_cq(dev->mdev, &cq->mcq);
 908
 909 err_cqb:
 910         kvfree(cqb);
 911         if (context)
 912                 destroy_cq_user(cq, context);
 913         else
 914                 destroy_cq_kernel(dev, cq);
 915
 916 err_create:
 917         kfree(cq);
 918
 919         return ERR_PTR(err);
 920 }
 921
 922
 923 int mlx5_ib_destroy_cq(struct ib_cq *cq)
 924 {
 925         struct mlx5_ib_dev *dev = to_mdev(cq->device);
 926         struct mlx5_ib_cq *mcq = to_mcq(cq);
 927         struct ib_ucontext *context = NULL;
 928
 929         if (cq->uobject)
 930                 context = cq->uobject->context;
 931
 932         mlx5_core_destroy_cq(dev->mdev, &mcq->mcq);
 933         if (context)
 934                 destroy_cq_user(mcq, context);
 935         else
 936                 destroy_cq_kernel(dev, mcq);
 937
 938         kfree(mcq);
 939
 940         return 0;
 941 }
 942
 943 static int is_equal_rsn(struct mlx5_cqe64 *cqe64, u32 rsn)
 944 {
 945         return rsn == (ntohl(cqe64->sop_drop_qpn) & 0xffffff);
 946 }
 947
 948 void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 rsn, struct mlx5_ib_srq *srq)
 949 {
 950         struct mlx5_cqe64 *cqe64, *dest64;
 951         void *cqe, *dest;
 952         u32 prod_index;
 953         int nfreed = 0;
 954         u8 owner_bit;
 955
 956         if (!cq)
 957                 return;
 958
 959         /* First we need to find the current producer index, so we
 960          * know where to start cleaning from.  It doesn't matter if HW
 961          * adds new entries after this loop -- the QP we're worried
 962          * about is already in RESET, so the new entries won't come
 963          * from our QP and therefore don't need to be checked.
 964          */
 965         for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); prod_index++)
 966                 if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe)
 967                         break;
 968
 969         /* Now sweep backwards through the CQ, removing CQ entries
 970          * that match our QP by copying older entries on top of them.
 971          */
 972         while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) {
 973                 cqe = get_cqe(cq, prod_index & cq->ibcq.cqe);
 974                 cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64;
 975                 if (is_equal_rsn(cqe64, rsn)) {
 976                         if (srq && (ntohl(cqe64->srqn) & 0xffffff))
 977                                 mlx5_ib_free_srq_wqe(srq, be16_to_cpu(cqe64->wqe_counter));
 978                         ++nfreed;
 979                 } else if (nfreed) {
 980                         dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe);
 981                         dest64 = (cq->mcq.cqe_sz == 64) ? dest : dest + 64;
 982                         owner_bit = dest64->op_own & MLX5_CQE_OWNER_MASK;
 983                         memcpy(dest, cqe, cq->mcq.cqe_sz);
 984                         dest64->op_own = owner_bit |
 985                                 (dest64->op_own & ~MLX5_CQE_OWNER_MASK);
 986                 }
 987         }
 988
 989         if (nfreed) {
 990                 cq->mcq.cons_index += nfreed;
 991                 /* Make sure update of buffer contents is done before
 992                  * updating consumer index.
 993                  */
 994                 wmb();
 995                 mlx5_cq_set_ci(&cq->mcq);
 996         }
 997 }
 998
 999 void mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq)
1000 {
1001         if (!cq)
1002                 return;
1003
1004         spin_lock_irq(&cq->lock);
1005         __mlx5_ib_cq_clean(cq, qpn, srq);
1006         spin_unlock_irq(&cq->lock);
1007 }
1008
1009 int mlx5_ib_modify_cq(struct ib_cq *cq, struct ib_cq_attr *attr, int cq_attr_mask)
1010 {
1011         struct mlx5_modify_cq_mbox_in *in;
1012         struct mlx5_ib_dev *dev = to_mdev(cq->device);
1013         struct mlx5_ib_cq *mcq = to_mcq(cq);
1014         u16 cq_count = attr->moderation.cq_count;
1015         u16 cq_period = attr->moderation.cq_period;
1016
1017         int err;
1018         u32 fsel = 0;
1019
1020         in = kzalloc(sizeof(*in), GFP_KERNEL);
1021         if (!in)
1022                 return -ENOMEM;
1023
1024         in->cqn = cpu_to_be32(mcq->mcq.cqn);
1025         if (cq_attr_mask & IB_CQ_MODERATION) {
1026                 if (MLX5_CAP_GEN(dev->mdev, cq_moderation)) {
1027                         fsel |= (MLX5_CQ_MODIFY_PERIOD | MLX5_CQ_MODIFY_COUNT);
1028                         if (cq_period & 0xf000) {
1029                                 /* A value higher than 0xfff is required, better
1030                                  * use the largest value possible. */
1031                                 cq_period = 0xfff;
1032                                 printf("mlx5_ib: INFO: ""period supported is limited to 12 bits\n");
1033                         }
1034
1035                         in->ctx.cq_period = cpu_to_be16(cq_period);
1036                         in->ctx.cq_max_count = cpu_to_be16(cq_count);
1037                 } else {
1038                         err = -ENOSYS;
1039                         goto out;
1040                 }
1041         }
1042         in->field_select = cpu_to_be32(fsel);
1043         err = mlx5_core_modify_cq(dev->mdev, &mcq->mcq, in, sizeof(*in));
1044
1045 out:
1046         kfree(in);
1047         if (err)
1048                 mlx5_ib_warn(dev, "modify cq 0x%x failed\n", mcq->mcq.cqn);
1049
1050         return err;
1051 }
1052
1053 static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
1054                        int entries, struct ib_udata *udata, int *npas,
1055                        int *page_shift, int *cqe_size)
1056 {
1057         struct mlx5_ib_resize_cq ucmd;
1058         struct ib_umem *umem;
1059         int err;
1060         int npages;
1061         struct ib_ucontext *context = cq->buf.umem->context;
1062
1063         err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd));
1064         if (err)
1065                 return err;
1066
1067         if (ucmd.reserved0 || ucmd.reserved1)
1068                 return -EINVAL;
1069
1070         umem = ib_umem_get(context, ucmd.buf_addr, entries * ucmd.cqe_size,
1071                            IB_ACCESS_LOCAL_WRITE, 1);
1072         if (IS_ERR(umem)) {
1073                 err = PTR_ERR(umem);
1074                 return err;
1075         }
1076
1077         mlx5_ib_cont_pages(umem, ucmd.buf_addr, &npages, page_shift,
1078                            npas, NULL);
1079
1080         cq->resize_umem = umem;
1081         *cqe_size = ucmd.cqe_size;
1082
1083         return 0;
1084 }
1085
1086 static void un_resize_user(struct mlx5_ib_cq *cq)
1087 {
1088         ib_umem_release(cq->resize_umem);
1089 }
1090
1091 static int resize_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
1092                          int entries, int cqe_size)
1093 {
1094         int err;
1095
1096         cq->resize_buf = kzalloc(sizeof(*cq->resize_buf), GFP_KERNEL);
1097         if (!cq->resize_buf)
1098                 return -ENOMEM;
1099
1100         err = alloc_cq_buf(dev, cq->resize_buf, entries, cqe_size);
1101         if (err)
1102                 goto ex;
1103
1104         init_cq_buf(cq, cq->resize_buf);
1105
1106         return 0;
1107
1108 ex:
1109         kfree(cq->resize_buf);
1110         return err;
1111 }
1112
1113 static void un_resize_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq)
1114 {
1115         free_cq_buf(dev, cq->resize_buf);
1116         cq->resize_buf = NULL;
1117 }
1118
1119 static int copy_resize_cqes(struct mlx5_ib_cq *cq)
1120 {
1121         struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device);
1122         struct mlx5_cqe64 *scqe64;
1123         struct mlx5_cqe64 *dcqe64;
1124         void *start_cqe;
1125         void *scqe;
1126         void *dcqe;
1127         int ssize;
1128         int dsize;
1129         int i;
1130         u8 sw_own;
1131
1132         ssize = cq->buf.cqe_size;
1133         dsize = cq->resize_buf->cqe_size;
1134         if (ssize != dsize) {
1135                 mlx5_ib_warn(dev, "resize from different cqe size is not supported\n");
1136                 return -EINVAL;
1137         }
1138
1139         i = cq->mcq.cons_index;
1140         scqe = get_sw_cqe(cq, i);
1141         scqe64 = ssize == 64 ? scqe : scqe + 64;
1142         start_cqe = scqe;
1143         if (!scqe) {
1144                 mlx5_ib_warn(dev, "expected cqe in sw ownership\n");
1145                 return -EINVAL;
1146         }
1147
1148         while ((scqe64->op_own >> 4) != MLX5_CQE_RESIZE_CQ) {
1149                 dcqe = get_cqe_from_buf(cq->resize_buf,
1150                                         (i + 1) & (cq->resize_buf->nent),
1151                                         dsize);
1152                 dcqe64 = dsize == 64 ? dcqe : dcqe + 64;
1153                 sw_own = sw_ownership_bit(i + 1, cq->resize_buf->nent);
1154                 memcpy(dcqe, scqe, dsize);
1155                 dcqe64->op_own = (dcqe64->op_own & ~MLX5_CQE_OWNER_MASK) | sw_own;
1156
1157                 ++i;
1158                 scqe = get_sw_cqe(cq, i);
1159                 scqe64 = ssize == 64 ? scqe : scqe + 64;
1160                 if (!scqe) {
1161                         mlx5_ib_warn(dev, "expected cqe in sw ownership\n");
1162                         return -EINVAL;
1163                 }
1164
1165                 if (scqe == start_cqe) {
1166                         printf("mlx5_ib: WARN: ""resize CQ failed to get resize CQE, CQN 0x%x\n", cq->mcq.cqn);
1167                         return -ENOMEM;
1168                 }
1169         }
1170         ++cq->mcq.cons_index;
1171         return 0;
1172 }
1173
1174 int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)
1175 {
1176         struct mlx5_ib_dev *dev = to_mdev(ibcq->device);
1177         struct mlx5_ib_cq *cq = to_mcq(ibcq);
1178         struct mlx5_modify_cq_mbox_in *in;
1179         int err;
1180         int npas;
1181         int page_shift;
1182         int inlen;
1183         int uninitialized_var(cqe_size);
1184         unsigned long flags;
1185
1186         if (!MLX5_CAP_GEN(dev->mdev, cq_resize)) {
1187                 mlx5_ib_warn(dev, "Firmware does not support resize CQ\n");
1188                 return -ENOSYS;
1189         }
1190
1191         if (entries < 1 || roundup_pow_of_two(entries + 1) >
1192             (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz))) {
1193                 mlx5_ib_warn(dev, "wrong entries number %d(%ld), max %d\n",
1194                              entries, roundup_pow_of_two(entries + 1),
1195                              1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz));
1196                 return -EINVAL;
1197         }
1198
1199         entries = roundup_pow_of_two(entries + 1);
1200
1201         if (entries == ibcq->cqe + 1)
1202                 return 0;
1203
1204         mutex_lock(&cq->resize_mutex);
1205         if (udata) {
1206                 err = resize_user(dev, cq, entries, udata, &npas, &page_shift,
1207                                   &cqe_size);
1208         } else {
1209                 cqe_size = 64;
1210                 err = resize_kernel(dev, cq, entries, cqe_size);
1211                 if (!err) {
1212                         npas = cq->resize_buf->buf.npages;
1213                         page_shift = cq->resize_buf->buf.page_shift;
1214                 }
1215         }
1216
1217         if (err) {
1218                 mlx5_ib_warn(dev, "resize failed: %d\n", err);
1219                 goto ex;
1220         }
1221
1222         inlen = sizeof(*in) + npas * sizeof(in->pas[0]);
1223         in = mlx5_vzalloc(inlen);
1224         if (!in) {
1225                 err = -ENOMEM;
1226                 goto ex_resize;
1227         }
1228
1229         if (udata)
1230                 mlx5_ib_populate_pas(dev, cq->resize_umem, page_shift,
1231                                      in->pas, 0);
1232         else
1233                 mlx5_fill_page_array(&cq->resize_buf->buf, in->pas);
1234
1235         in->field_select = cpu_to_be32(MLX5_MODIFY_CQ_MASK_LOG_SIZE  |
1236                                        MLX5_MODIFY_CQ_MASK_PG_OFFSET |
1237                                        MLX5_MODIFY_CQ_MASK_PG_SIZE);
1238         in->ctx.log_pg_sz = page_shift - MLX5_ADAPTER_PAGE_SHIFT;
1239         in->ctx.cqe_sz_flags = cqe_sz_to_mlx_sz(cqe_size) << 5;
1240         in->ctx.page_offset = 0;
1241         in->ctx.log_sz_usr_page = cpu_to_be32(ilog2(entries) << 24);
1242         in->hdr.opmod = cpu_to_be16(MLX5_CQ_OPMOD_RESIZE);
1243         in->cqn = cpu_to_be32(cq->mcq.cqn);
1244
1245         err = mlx5_core_modify_cq(dev->mdev, &cq->mcq, in, inlen);
1246         if (err) {
1247                 mlx5_ib_warn(dev, "modify cq failed: %d\n", err);
1248                 goto ex_alloc;
1249         }
1250
1251         if (udata) {
1252                 cq->ibcq.cqe = entries - 1;
1253                 ib_umem_release(cq->buf.umem);
1254                 cq->buf.umem = cq->resize_umem;
1255                 cq->resize_umem = NULL;
1256         } else {
1257                 struct mlx5_ib_cq_buf tbuf;
1258                 int resized = 0;
1259
1260                 spin_lock_irqsave(&cq->lock, flags);
1261                 if (cq->resize_buf) {
1262                         err = copy_resize_cqes(cq);
1263                         if (!err) {
1264                                 tbuf = cq->buf;
1265                                 cq->buf = *cq->resize_buf;
1266                                 kfree(cq->resize_buf);
1267                                 cq->resize_buf = NULL;
1268                                 resized = 1;
1269                         }
1270                 }
1271                 cq->ibcq.cqe = entries - 1;
1272                 spin_unlock_irqrestore(&cq->lock, flags);
1273                 if (resized)
1274                         free_cq_buf(dev, &tbuf);
1275         }
1276         mutex_unlock(&cq->resize_mutex);
1277
1278         kvfree(in);
1279         return 0;
1280
1281 ex_alloc:
1282         kvfree(in);
1283
1284 ex_resize:
1285         if (udata)
1286                 un_resize_user(cq);
1287         else
1288                 un_resize_kernel(dev, cq);
1289 ex:
1290         mutex_unlock(&cq->resize_mutex);
1291         return err;
1292 }
1293
1294 int mlx5_ib_get_cqe_size(struct mlx5_ib_dev *dev, struct ib_cq *ibcq)
1295 {
1296         struct mlx5_ib_cq *cq;
1297
1298         if (!ibcq)
1299                 return 128;
1300
1301         cq = to_mcq(ibcq);
1302         return cq->cqe_size;
1303 }