sys/contrib/openzfs/module/os/linux/zfs/qat_crypt.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * This file represents the QAT implementation of checksums and encryption.
  24  * Internally, QAT shares the same cryptographic instances for both of these
  25  * operations, so the code has been combined here. QAT data compression uses
  26  * compression instances, so that code is separated into qat_compress.c
  27  */
  28
  29 #if defined(_KERNEL) && defined(HAVE_QAT)
  30 #include <linux/slab.h>
  31 #include <linux/vmalloc.h>
  32 #include <linux/pagemap.h>
  33 #include <linux/completion.h>
  34 #include <sys/zfs_context.h>
  35 #include <sys/zio_crypt.h>
  36 #include "lac/cpa_cy_im.h"
  37 #include "lac/cpa_cy_common.h"
  38 #include <sys/qat.h>
  39
  40 /*
  41  * Max instances in a QAT device, each instance is a channel to submit
  42  * jobs to QAT hardware, this is only for pre-allocating instances
  43  * and session arrays; the actual number of instances are defined in
  44  * the QAT driver's configure file.
  45  */
  46 #define QAT_CRYPT_MAX_INSTANCES         48
  47
  48 #define MAX_PAGE_NUM                    1024
  49
  50 static Cpa32U inst_num = 0;
  51 static Cpa16U num_inst = 0;
  52 static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES];
  53 static boolean_t qat_cy_init_done = B_FALSE;
  54 int zfs_qat_encrypt_disable = 0;
  55 int zfs_qat_checksum_disable = 0;
  56
  57 typedef struct cy_callback {
  58         CpaBoolean verify_result;
  59         struct completion complete;
  60 } cy_callback_t;
  61
  62 static void
  63 symcallback(void *p_callback, CpaStatus status, const CpaCySymOp operation,
  64     void *op_data, CpaBufferList *buf_list_dst, CpaBoolean verify)
  65 {
  66         cy_callback_t *cb = p_callback;
  67
  68         if (cb != NULL) {
  69                 /* indicate that the function has been called */
  70                 cb->verify_result = verify;
  71                 complete(&cb->complete);
  72         }
  73 }
  74
  75 boolean_t
  76 qat_crypt_use_accel(size_t s_len)
  77 {
  78         return (!zfs_qat_encrypt_disable &&
  79             qat_cy_init_done &&
  80             s_len >= QAT_MIN_BUF_SIZE &&
  81             s_len <= QAT_MAX_BUF_SIZE);
  82 }
  83
  84 boolean_t
  85 qat_checksum_use_accel(size_t s_len)
  86 {
  87         return (!zfs_qat_checksum_disable &&
  88             qat_cy_init_done &&
  89             s_len >= QAT_MIN_BUF_SIZE &&
  90             s_len <= QAT_MAX_BUF_SIZE);
  91 }
  92
  93 void
  94 qat_cy_clean(void)
  95 {
  96         for (Cpa16U i = 0; i < num_inst; i++)
  97                 cpaCyStopInstance(cy_inst_handles[i]);
  98
  99         num_inst = 0;
 100         qat_cy_init_done = B_FALSE;
 101 }
 102
 103 int
 104 qat_cy_init(void)
 105 {
 106         CpaStatus status = CPA_STATUS_FAIL;
 107
 108         if (qat_cy_init_done)
 109                 return (0);
 110
 111         status = cpaCyGetNumInstances(&num_inst);
 112         if (status != CPA_STATUS_SUCCESS)
 113                 return (-1);
 114
 115         /* if the user has configured no QAT encryption units just return */
 116         if (num_inst == 0)
 117                 return (0);
 118
 119         if (num_inst > QAT_CRYPT_MAX_INSTANCES)
 120                 num_inst = QAT_CRYPT_MAX_INSTANCES;
 121
 122         status = cpaCyGetInstances(num_inst, &cy_inst_handles[0]);
 123         if (status != CPA_STATUS_SUCCESS)
 124                 return (-1);
 125
 126         for (Cpa16U i = 0; i < num_inst; i++) {
 127                 status = cpaCySetAddressTranslation(cy_inst_handles[i],
 128                     (void *)virt_to_phys);
 129                 if (status != CPA_STATUS_SUCCESS)
 130                         goto error;
 131
 132                 status = cpaCyStartInstance(cy_inst_handles[i]);
 133                 if (status != CPA_STATUS_SUCCESS)
 134                         goto error;
 135         }
 136
 137         qat_cy_init_done = B_TRUE;
 138         return (0);
 139
 140 error:
 141         qat_cy_clean();
 142         return (-1);
 143 }
 144
 145 void
 146 qat_cy_fini(void)
 147 {
 148         if (!qat_cy_init_done)
 149                 return;
 150
 151         qat_cy_clean();
 152 }
 153
 154 static CpaStatus
 155 qat_init_crypt_session_ctx(qat_encrypt_dir_t dir, CpaInstanceHandle inst_handle,
 156     CpaCySymSessionCtx **cy_session_ctx, crypto_key_t *key,
 157     Cpa64U crypt, Cpa32U aad_len)
 158 {
 159         CpaStatus status = CPA_STATUS_SUCCESS;
 160         Cpa32U ctx_size;
 161         Cpa32U ciper_algorithm;
 162         Cpa32U hash_algorithm;
 163         CpaCySymSessionSetupData sd = { 0 };
 164
 165         if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_CCM) {
 166                 return (CPA_STATUS_FAIL);
 167         } else {
 168                 ciper_algorithm = CPA_CY_SYM_CIPHER_AES_GCM;
 169                 hash_algorithm = CPA_CY_SYM_HASH_AES_GCM;
 170         }
 171
 172         sd.cipherSetupData.cipherAlgorithm = ciper_algorithm;
 173         sd.cipherSetupData.pCipherKey = key->ck_data;
 174         sd.cipherSetupData.cipherKeyLenInBytes = key->ck_length / 8;
 175         sd.hashSetupData.hashAlgorithm = hash_algorithm;
 176         sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_AUTH;
 177         sd.hashSetupData.digestResultLenInBytes = ZIO_DATA_MAC_LEN;
 178         sd.hashSetupData.authModeSetupData.aadLenInBytes = aad_len;
 179         sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
 180         sd.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
 181         sd.digestIsAppended = CPA_FALSE;
 182         sd.verifyDigest = CPA_FALSE;
 183
 184         if (dir == QAT_ENCRYPT) {
 185                 sd.cipherSetupData.cipherDirection =
 186                     CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
 187                 sd.algChainOrder =
 188                     CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
 189         } else {
 190                 ASSERT3U(dir, ==, QAT_DECRYPT);
 191                 sd.cipherSetupData.cipherDirection =
 192                     CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT;
 193                 sd.algChainOrder =
 194                     CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
 195         }
 196
 197         status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
 198         if (status != CPA_STATUS_SUCCESS)
 199                 return (status);
 200
 201         status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
 202         if (status != CPA_STATUS_SUCCESS)
 203                 return (status);
 204
 205         status = cpaCySymInitSession(inst_handle, symcallback, &sd,
 206             *cy_session_ctx);
 207         if (status != CPA_STATUS_SUCCESS) {
 208                 QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
 209                 return (status);
 210         }
 211
 212         return (CPA_STATUS_SUCCESS);
 213 }
 214
 215 static CpaStatus
 216 qat_init_checksum_session_ctx(CpaInstanceHandle inst_handle,
 217     CpaCySymSessionCtx **cy_session_ctx, Cpa64U cksum)
 218 {
 219         CpaStatus status = CPA_STATUS_SUCCESS;
 220         Cpa32U ctx_size;
 221         Cpa32U hash_algorithm;
 222         CpaCySymSessionSetupData sd = { 0 };
 223
 224         /*
 225          * ZFS's SHA512 checksum is actually SHA512/256, which uses
 226          * a different IV from standard SHA512. QAT does not support
 227          * SHA512/256, so we can only support SHA256.
 228          */
 229         if (cksum == ZIO_CHECKSUM_SHA256)
 230                 hash_algorithm = CPA_CY_SYM_HASH_SHA256;
 231         else
 232                 return (CPA_STATUS_FAIL);
 233
 234         sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
 235         sd.symOperation = CPA_CY_SYM_OP_HASH;
 236         sd.hashSetupData.hashAlgorithm = hash_algorithm;
 237         sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_PLAIN;
 238         sd.hashSetupData.digestResultLenInBytes = sizeof (zio_cksum_t);
 239         sd.digestIsAppended = CPA_FALSE;
 240         sd.verifyDigest = CPA_FALSE;
 241
 242         status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
 243         if (status != CPA_STATUS_SUCCESS)
 244                 return (status);
 245
 246         status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
 247         if (status != CPA_STATUS_SUCCESS)
 248                 return (status);
 249
 250         status = cpaCySymInitSession(inst_handle, symcallback, &sd,
 251             *cy_session_ctx);
 252         if (status != CPA_STATUS_SUCCESS) {
 253                 QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
 254                 return (status);
 255         }
 256
 257         return (CPA_STATUS_SUCCESS);
 258 }
 259
 260 static CpaStatus
 261 qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle, uint32_t nr_bufs,
 262     CpaBufferList *src, CpaBufferList *dst)
 263 {
 264         CpaStatus status = CPA_STATUS_SUCCESS;
 265         Cpa32U meta_size = 0;
 266
 267         status = cpaCyBufferListGetMetaSize(inst_handle, nr_bufs, &meta_size);
 268         if (status != CPA_STATUS_SUCCESS)
 269                 return (status);
 270
 271         status = QAT_PHYS_CONTIG_ALLOC(&src->pPrivateMetaData, meta_size);
 272         if (status != CPA_STATUS_SUCCESS)
 273                 goto error;
 274
 275         if (src != dst) {
 276                 status = QAT_PHYS_CONTIG_ALLOC(&dst->pPrivateMetaData,
 277                     meta_size);
 278                 if (status != CPA_STATUS_SUCCESS)
 279                         goto error;
 280         }
 281
 282         return (CPA_STATUS_SUCCESS);
 283
 284 error:
 285         QAT_PHYS_CONTIG_FREE(src->pPrivateMetaData);
 286         if (src != dst)
 287                 QAT_PHYS_CONTIG_FREE(dst->pPrivateMetaData);
 288
 289         return (status);
 290 }
 291
 292 int
 293 qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
 294     uint8_t *aad_buf, uint32_t aad_len, uint8_t *iv_buf, uint8_t *digest_buf,
 295     crypto_key_t *key, uint64_t crypt, uint32_t enc_len)
 296 {
 297         CpaStatus status = CPA_STATUS_SUCCESS;
 298         Cpa16U i;
 299         CpaInstanceHandle cy_inst_handle;
 300         Cpa16U nr_bufs = (enc_len >> PAGE_SHIFT) + 2;
 301         Cpa32U bytes_left = 0;
 302         Cpa8S *data = NULL;
 303         CpaCySymSessionCtx *cy_session_ctx = NULL;
 304         cy_callback_t cb;
 305         CpaCySymOpData op_data = { 0 };
 306         CpaBufferList src_buffer_list = { 0 };
 307         CpaBufferList dst_buffer_list = { 0 };
 308         CpaFlatBuffer *flat_src_buf_array = NULL;
 309         CpaFlatBuffer *flat_src_buf = NULL;
 310         CpaFlatBuffer *flat_dst_buf_array = NULL;
 311         CpaFlatBuffer *flat_dst_buf = NULL;
 312         struct page *in_pages[MAX_PAGE_NUM];
 313         struct page *out_pages[MAX_PAGE_NUM];
 314         Cpa32U in_page_num = 0;
 315         Cpa32U out_page_num = 0;
 316         Cpa32U in_page_off = 0;
 317         Cpa32U out_page_off = 0;
 318
 319         if (dir == QAT_ENCRYPT) {
 320                 QAT_STAT_BUMP(encrypt_requests);
 321                 QAT_STAT_INCR(encrypt_total_in_bytes, enc_len);
 322         } else {
 323                 QAT_STAT_BUMP(decrypt_requests);
 324                 QAT_STAT_INCR(decrypt_total_in_bytes, enc_len);
 325         }
 326
 327         i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
 328         cy_inst_handle = cy_inst_handles[i];
 329
 330         status = qat_init_crypt_session_ctx(dir, cy_inst_handle,
 331             &cy_session_ctx, key, crypt, aad_len);
 332         if (status != CPA_STATUS_SUCCESS) {
 333                 /* don't count CCM as a failure since it's not supported */
 334                 if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_GCM)
 335                         QAT_STAT_BUMP(crypt_fails);
 336                 return (status);
 337         }
 338
 339         /*
 340          * We increment nr_bufs by 2 to allow us to handle non
 341          * page-aligned buffer addresses and buffers whose sizes
 342          * are not divisible by PAGE_SIZE.
 343          */
 344         status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
 345             &src_buffer_list, &dst_buffer_list);
 346         if (status != CPA_STATUS_SUCCESS)
 347                 goto fail;
 348
 349         status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
 350             nr_bufs * sizeof (CpaFlatBuffer));
 351         if (status != CPA_STATUS_SUCCESS)
 352                 goto fail;
 353         status = QAT_PHYS_CONTIG_ALLOC(&flat_dst_buf_array,
 354             nr_bufs * sizeof (CpaFlatBuffer));
 355         if (status != CPA_STATUS_SUCCESS)
 356                 goto fail;
 357         status = QAT_PHYS_CONTIG_ALLOC(&op_data.pDigestResult,
 358             ZIO_DATA_MAC_LEN);
 359         if (status != CPA_STATUS_SUCCESS)
 360                 goto fail;
 361         status = QAT_PHYS_CONTIG_ALLOC(&op_data.pIv,
 362             ZIO_DATA_IV_LEN);
 363         if (status != CPA_STATUS_SUCCESS)
 364                 goto fail;
 365         if (aad_len > 0) {
 366                 status = QAT_PHYS_CONTIG_ALLOC(&op_data.pAdditionalAuthData,
 367                     aad_len);
 368                 if (status != CPA_STATUS_SUCCESS)
 369                         goto fail;
 370                 bcopy(aad_buf, op_data.pAdditionalAuthData, aad_len);
 371         }
 372
 373         bytes_left = enc_len;
 374         data = src_buf;
 375         flat_src_buf = flat_src_buf_array;
 376         while (bytes_left > 0) {
 377                 in_page_off = ((long)data & ~PAGE_MASK);
 378                 in_pages[in_page_num] = qat_mem_to_page(data);
 379                 flat_src_buf->pData = kmap(in_pages[in_page_num]) + in_page_off;
 380                 flat_src_buf->dataLenInBytes =
 381                     min((long)PAGE_SIZE - in_page_off, (long)bytes_left);
 382                 data += flat_src_buf->dataLenInBytes;
 383                 bytes_left -= flat_src_buf->dataLenInBytes;
 384                 flat_src_buf++;
 385                 in_page_num++;
 386         }
 387         src_buffer_list.pBuffers = flat_src_buf_array;
 388         src_buffer_list.numBuffers = in_page_num;
 389
 390         bytes_left = enc_len;
 391         data = dst_buf;
 392         flat_dst_buf = flat_dst_buf_array;
 393         while (bytes_left > 0) {
 394                 out_page_off = ((long)data & ~PAGE_MASK);
 395                 out_pages[out_page_num] = qat_mem_to_page(data);
 396                 flat_dst_buf->pData = kmap(out_pages[out_page_num]) +
 397                     out_page_off;
 398                 flat_dst_buf->dataLenInBytes =
 399                     min((long)PAGE_SIZE - out_page_off, (long)bytes_left);
 400                 data += flat_dst_buf->dataLenInBytes;
 401                 bytes_left -= flat_dst_buf->dataLenInBytes;
 402                 flat_dst_buf++;
 403                 out_page_num++;
 404         }
 405         dst_buffer_list.pBuffers = flat_dst_buf_array;
 406         dst_buffer_list.numBuffers = out_page_num;
 407
 408         op_data.sessionCtx = cy_session_ctx;
 409         op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
 410         op_data.cryptoStartSrcOffsetInBytes = 0;
 411         op_data.messageLenToCipherInBytes = 0;
 412         op_data.hashStartSrcOffsetInBytes = 0;
 413         op_data.messageLenToHashInBytes = 0;
 414         op_data.messageLenToCipherInBytes = enc_len;
 415         op_data.ivLenInBytes = ZIO_DATA_IV_LEN;
 416         bcopy(iv_buf, op_data.pIv, ZIO_DATA_IV_LEN);
 417         /* if dir is QAT_DECRYPT, copy digest_buf to pDigestResult */
 418         if (dir == QAT_DECRYPT)
 419                 bcopy(digest_buf, op_data.pDigestResult, ZIO_DATA_MAC_LEN);
 420
 421         cb.verify_result = CPA_FALSE;
 422         init_completion(&cb.complete);
 423         status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
 424             &src_buffer_list, &dst_buffer_list, NULL);
 425         if (status != CPA_STATUS_SUCCESS)
 426                 goto fail;
 427
 428         /* we now wait until the completion of the operation. */
 429         wait_for_completion(&cb.complete);
 430
 431         if (cb.verify_result == CPA_FALSE) {
 432                 status = CPA_STATUS_FAIL;
 433                 goto fail;
 434         }
 435
 436         if (dir == QAT_ENCRYPT) {
 437                 /* if dir is QAT_ENCRYPT, save pDigestResult to digest_buf */
 438                 bcopy(op_data.pDigestResult, digest_buf, ZIO_DATA_MAC_LEN);
 439                 QAT_STAT_INCR(encrypt_total_out_bytes, enc_len);
 440         } else {
 441                 QAT_STAT_INCR(decrypt_total_out_bytes, enc_len);
 442         }
 443
 444 fail:
 445         if (status != CPA_STATUS_SUCCESS)
 446                 QAT_STAT_BUMP(crypt_fails);
 447
 448         for (i = 0; i < in_page_num; i++)
 449                 kunmap(in_pages[i]);
 450         for (i = 0; i < out_page_num; i++)
 451                 kunmap(out_pages[i]);
 452
 453         cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
 454         if (aad_len > 0)
 455                 QAT_PHYS_CONTIG_FREE(op_data.pAdditionalAuthData);
 456         QAT_PHYS_CONTIG_FREE(op_data.pIv);
 457         QAT_PHYS_CONTIG_FREE(op_data.pDigestResult);
 458         QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
 459         QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData);
 460         QAT_PHYS_CONTIG_FREE(cy_session_ctx);
 461         QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
 462         QAT_PHYS_CONTIG_FREE(flat_dst_buf_array);
 463
 464         return (status);
 465 }
 466
 467 int
 468 qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp)
 469 {
 470         CpaStatus status;
 471         Cpa16U i;
 472         CpaInstanceHandle cy_inst_handle;
 473         Cpa16U nr_bufs = (size >> PAGE_SHIFT) + 2;
 474         Cpa32U bytes_left = 0;
 475         Cpa8S *data = NULL;
 476         CpaCySymSessionCtx *cy_session_ctx = NULL;
 477         cy_callback_t cb;
 478         Cpa8U *digest_buffer = NULL;
 479         CpaCySymOpData op_data = { 0 };
 480         CpaBufferList src_buffer_list = { 0 };
 481         CpaFlatBuffer *flat_src_buf_array = NULL;
 482         CpaFlatBuffer *flat_src_buf = NULL;
 483         struct page *in_pages[MAX_PAGE_NUM];
 484         Cpa32U page_num = 0;
 485         Cpa32U page_off = 0;
 486
 487         QAT_STAT_BUMP(cksum_requests);
 488         QAT_STAT_INCR(cksum_total_in_bytes, size);
 489
 490         i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
 491         cy_inst_handle = cy_inst_handles[i];
 492
 493         status = qat_init_checksum_session_ctx(cy_inst_handle,
 494             &cy_session_ctx, cksum);
 495         if (status != CPA_STATUS_SUCCESS) {
 496                 /* don't count unsupported checksums as a failure */
 497                 if (cksum == ZIO_CHECKSUM_SHA256 ||
 498                     cksum == ZIO_CHECKSUM_SHA512)
 499                         QAT_STAT_BUMP(cksum_fails);
 500                 return (status);
 501         }
 502
 503         /*
 504          * We increment nr_bufs by 2 to allow us to handle non
 505          * page-aligned buffer addresses and buffers whose sizes
 506          * are not divisible by PAGE_SIZE.
 507          */
 508         status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
 509             &src_buffer_list, &src_buffer_list);
 510         if (status != CPA_STATUS_SUCCESS)
 511                 goto fail;
 512
 513         status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
 514             nr_bufs * sizeof (CpaFlatBuffer));
 515         if (status != CPA_STATUS_SUCCESS)
 516                 goto fail;
 517         status = QAT_PHYS_CONTIG_ALLOC(&digest_buffer,
 518             sizeof (zio_cksum_t));
 519         if (status != CPA_STATUS_SUCCESS)
 520                 goto fail;
 521
 522         bytes_left = size;
 523         data = buf;
 524         flat_src_buf = flat_src_buf_array;
 525         while (bytes_left > 0) {
 526                 page_off = ((long)data & ~PAGE_MASK);
 527                 in_pages[page_num] = qat_mem_to_page(data);
 528                 flat_src_buf->pData = kmap(in_pages[page_num]) + page_off;
 529                 flat_src_buf->dataLenInBytes =
 530                     min((long)PAGE_SIZE - page_off, (long)bytes_left);
 531                 data += flat_src_buf->dataLenInBytes;
 532                 bytes_left -= flat_src_buf->dataLenInBytes;
 533                 flat_src_buf++;
 534                 page_num++;
 535         }
 536         src_buffer_list.pBuffers = flat_src_buf_array;
 537         src_buffer_list.numBuffers = page_num;
 538
 539         op_data.sessionCtx = cy_session_ctx;
 540         op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
 541         op_data.hashStartSrcOffsetInBytes = 0;
 542         op_data.messageLenToHashInBytes = size;
 543         op_data.pDigestResult = digest_buffer;
 544
 545         cb.verify_result = CPA_FALSE;
 546         init_completion(&cb.complete);
 547         status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
 548             &src_buffer_list, &src_buffer_list, NULL);
 549         if (status != CPA_STATUS_SUCCESS)
 550                 goto fail;
 551
 552         /* we now wait until the completion of the operation. */
 553         wait_for_completion(&cb.complete);
 554
 555         if (cb.verify_result == CPA_FALSE) {
 556                 status = CPA_STATUS_FAIL;
 557                 goto fail;
 558         }
 559
 560         bcopy(digest_buffer, zcp, sizeof (zio_cksum_t));
 561
 562 fail:
 563         if (status != CPA_STATUS_SUCCESS)
 564                 QAT_STAT_BUMP(cksum_fails);
 565
 566         for (i = 0; i < page_num; i++)
 567                 kunmap(in_pages[i]);
 568
 569         cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
 570         QAT_PHYS_CONTIG_FREE(digest_buffer);
 571         QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
 572         QAT_PHYS_CONTIG_FREE(cy_session_ctx);
 573         QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
 574
 575         return (status);
 576 }
 577
 578 static int
 579 param_set_qat_encrypt(const char *val, zfs_kernel_param_t *kp)
 580 {
 581         int ret;
 582         int *pvalue = kp->arg;
 583         ret = param_set_int(val, kp);
 584         if (ret)
 585                 return (ret);
 586         /*
 587          * zfs_qat_encrypt_disable = 0: enable qat encrypt
 588          * try to initialize qat instance if it has not been done
 589          */
 590         if (*pvalue == 0 && !qat_cy_init_done) {
 591                 ret = qat_cy_init();
 592                 if (ret != 0) {
 593                         zfs_qat_encrypt_disable = 1;
 594                         return (ret);
 595                 }
 596         }
 597         return (ret);
 598 }
 599
 600 static int
 601 param_set_qat_checksum(const char *val, zfs_kernel_param_t *kp)
 602 {
 603         int ret;
 604         int *pvalue = kp->arg;
 605         ret = param_set_int(val, kp);
 606         if (ret)
 607                 return (ret);
 608         /*
 609          * set_checksum_param_ops = 0: enable qat checksum
 610          * try to initialize qat instance if it has not been done
 611          */
 612         if (*pvalue == 0 && !qat_cy_init_done) {
 613                 ret = qat_cy_init();
 614                 if (ret != 0) {
 615                         zfs_qat_checksum_disable = 1;
 616                         return (ret);
 617                 }
 618         }
 619         return (ret);
 620 }
 621
 622 module_param_call(zfs_qat_encrypt_disable, param_set_qat_encrypt,
 623     param_get_int, &zfs_qat_encrypt_disable, 0644);
 624 MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Enable/Disable QAT encryption");
 625
 626 module_param_call(zfs_qat_checksum_disable, param_set_qat_checksum,
 627     param_get_int, &zfs_qat_checksum_disable, 0644);
 628 MODULE_PARM_DESC(zfs_qat_checksum_disable, "Enable/Disable QAT checksumming");
 629
 630 #endif