sys/dev/mlx5/mlx5_fpga/mlx5fpga_xfer.c

   1 /*-
   2  * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  * $FreeBSD$
  33  */
  34
  35 #include <dev/mlx5/mlx5_fpga/xfer.h>
  36 #include <dev/mlx5/mlx5_fpga/conn.h>
  37
  38 struct xfer_state {
  39         const struct mlx5_fpga_transaction *xfer;
  40         /* Total transactions */
  41         unsigned int start_count;
  42         unsigned int done_count;
  43         unsigned int error_count;
  44         u8 status;
  45         /* Inflight transactions */
  46         unsigned int budget;
  47         unsigned int inflight_count;
  48         /* Chunking state */
  49         size_t pos;
  50         spinlock_t lock; /* Protects all members of this struct */
  51 };
  52
  53 struct xfer_transaction {
  54         struct xfer_state *xfer_state;
  55         struct mlx5_fpga_transaction transaction;
  56 };
  57
  58 static void trans_complete(const struct mlx5_fpga_transaction *complete,
  59                            u8 status);
  60
  61 static void xfer_complete(struct xfer_state *xfer_state)
  62 {
  63         const struct mlx5_fpga_transaction *xfer = xfer_state->xfer;
  64         u8 status = xfer_state->status;
  65
  66         kfree(xfer_state);
  67         xfer->complete1(xfer, status);
  68 }
  69
  70 /* Xfer state spin lock must be locked */
  71 static int exec_more(struct xfer_state *xfer_state)
  72 {
  73         struct xfer_transaction *xfer_trans;
  74         size_t left, cur_size, page_size;
  75         u64 pos_addr, ddr_base;
  76         u8 *pos_data;
  77         int ret = 0;
  78
  79         ddr_base = mlx5_fpga_ddr_base_get(xfer_state->xfer->conn->fdev);
  80         page_size = (xfer_state->xfer->addr + xfer_state->pos < ddr_base) ?
  81                     sizeof(u32) : (1 << MLX5_FPGA_TRANSACTION_SEND_PAGE_BITS);
  82
  83         do {
  84                 if (xfer_state->status != IB_WC_SUCCESS) {
  85                         ret = -EIO;
  86                         break;
  87                 }
  88
  89                 left = xfer_state->xfer->size - xfer_state->pos;
  90                 if (!left)
  91                         break;
  92
  93                 xfer_trans = kzalloc(sizeof(*xfer_trans), GFP_ATOMIC);
  94                 if (!xfer_trans) {
  95                         ret = -ENOMEM;
  96                         break;
  97                 }
  98
  99                 pos_addr = xfer_state->xfer->addr + xfer_state->pos;
 100                 pos_data = xfer_state->xfer->data + xfer_state->pos;
 101
 102                 /* Determine largest possible transaction at this point */
 103                 cur_size = page_size - (pos_addr & (page_size - 1));
 104                 if (cur_size > MLX5_FPGA_TRANSACTION_MAX_SIZE)
 105                         cur_size = MLX5_FPGA_TRANSACTION_MAX_SIZE;
 106                 if (cur_size > left)
 107                         cur_size = left;
 108
 109                 xfer_trans->xfer_state = xfer_state;
 110                 xfer_trans->transaction.addr = pos_addr;
 111                 xfer_trans->transaction.complete1 = trans_complete;
 112                 xfer_trans->transaction.conn = xfer_state->xfer->conn;
 113                 xfer_trans->transaction.data = pos_data;
 114                 xfer_trans->transaction.direction = xfer_state->xfer->direction;
 115                 xfer_trans->transaction.size = cur_size;
 116
 117                 xfer_state->start_count++;
 118                 xfer_state->inflight_count++;
 119                 mlx5_fpga_dbg(xfer_state->xfer->conn->fdev, "Starting %zu bytes at %p done; %u started %u inflight %u done %u error\n",
 120                               xfer_trans->transaction.size,
 121                               xfer_trans->transaction.data,
 122                               xfer_state->start_count,
 123                               xfer_state->inflight_count,
 124                               xfer_state->done_count,
 125                               xfer_state->error_count);
 126                 ret = mlx5_fpga_trans_exec(&xfer_trans->transaction);
 127                 if (ret) {
 128                         xfer_state->start_count--;
 129                         xfer_state->inflight_count--;
 130                         if (ret == -EBUSY)
 131                                 ret = 0;
 132
 133                         if (ret) {
 134                                 mlx5_fpga_warn(xfer_state->xfer->conn->fdev, "Transfer failed to start transaction: %d. %u started %u done %u error\n",
 135                                                ret, xfer_state->start_count,
 136                                                xfer_state->done_count,
 137                                                xfer_state->error_count);
 138                                 xfer_state->status = IB_WC_GENERAL_ERR;
 139                         }
 140                         kfree(xfer_trans);
 141                         break;
 142                 }
 143                 xfer_state->pos += cur_size;
 144                 if (xfer_state->inflight_count >= xfer_state->budget)
 145                         break;
 146         } while (cur_size != left);
 147
 148         return ret;
 149 }
 150
 151 static void trans_complete(const struct mlx5_fpga_transaction *complete,
 152                            u8 status)
 153 {
 154         struct xfer_transaction *xfer_trans;
 155         struct xfer_state *xfer_state;
 156         unsigned long flags;
 157         bool done = false;
 158         int ret;
 159
 160         xfer_trans = container_of(complete, struct xfer_transaction,
 161                                   transaction);
 162         xfer_state = xfer_trans->xfer_state;
 163         mlx5_fpga_dbg(complete->conn->fdev, "Transaction %zu bytes at %p done, status %u; %u started %u inflight %u done %u error\n",
 164                       xfer_trans->transaction.size,
 165                       xfer_trans->transaction.data, status,
 166                       xfer_state->start_count, xfer_state->inflight_count,
 167                       xfer_state->done_count, xfer_state->error_count);
 168         kfree(xfer_trans);
 169
 170         spin_lock_irqsave(&xfer_state->lock, flags);
 171
 172         if (status != IB_WC_SUCCESS) {
 173                 xfer_state->error_count++;
 174                 mlx5_fpga_warn(complete->conn->fdev, "Transaction failed during transfer. %u started %u inflight %u done %u error\n",
 175                                xfer_state->start_count,
 176                                xfer_state->inflight_count,
 177                                xfer_state->done_count, xfer_state->error_count);
 178                 if (xfer_state->status == IB_WC_SUCCESS)
 179                         xfer_state->status = status;
 180         } else {
 181                 xfer_state->done_count++;
 182         }
 183         ret = exec_more(xfer_state);
 184
 185         xfer_state->inflight_count--;
 186         if (!xfer_state->inflight_count)
 187                 done = true;
 188
 189         spin_unlock_irqrestore(&xfer_state->lock, flags);
 190
 191         if (done)
 192                 xfer_complete(xfer_state);
 193 }
 194
 195 int mlx5_fpga_xfer_exec(const struct mlx5_fpga_transaction *xfer)
 196 {
 197         u64 base = mlx5_fpga_ddr_base_get(xfer->conn->fdev);
 198         u64 size = mlx5_fpga_ddr_size_get(xfer->conn->fdev);
 199         struct xfer_state *xfer_state;
 200         unsigned long flags;
 201         bool done = false;
 202         int ret = 0;
 203
 204         if (xfer->addr + xfer->size > base + size) {
 205                 mlx5_fpga_warn(xfer->conn->fdev, "Transfer ends at %jx outside of DDR range %jx\n",
 206                                (uintmax_t)(xfer->addr + xfer->size), (uintmax_t)(base + size));
 207                 return -EINVAL;
 208         }
 209
 210         if (xfer->addr & MLX5_FPGA_TRANSACTION_SEND_ALIGN_BITS) {
 211                 mlx5_fpga_warn(xfer->conn->fdev, "Transfer address %jx not aligned\n",
 212                                (uintmax_t)xfer->addr);
 213                 return -EINVAL;
 214         }
 215
 216         if (xfer->size & MLX5_FPGA_TRANSACTION_SEND_ALIGN_BITS) {
 217                 mlx5_fpga_warn(xfer->conn->fdev, "Transfer size %zu not aligned\n",
 218                                xfer->size);
 219                 return -EINVAL;
 220         }
 221
 222         if (xfer->size < 1) {
 223                 mlx5_fpga_warn(xfer->conn->fdev, "Empty transfer size %zu not allowed\n",
 224                                xfer->size);
 225                 return -EINVAL;
 226         }
 227
 228         xfer_state = kzalloc(sizeof(*xfer_state), GFP_KERNEL);
 229         xfer_state->xfer = xfer;
 230         xfer_state->status = IB_WC_SUCCESS;
 231         xfer_state->budget = 7;
 232         spin_lock_init(&xfer_state->lock);
 233         spin_lock_irqsave(&xfer_state->lock, flags);
 234
 235         ret = exec_more(xfer_state);
 236         if (ret && (xfer_state->start_count == 0))
 237                 done = true;
 238
 239         spin_unlock_irqrestore(&xfer_state->lock, flags);
 240
 241         if (done)
 242                 xfer_complete(xfer_state);
 243         return ret;
 244 }