sys/dev/qat/qatvar.h

   1 /* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */
   2 /*      $NetBSD: qatvar.h,v 1.2 2020/03/14 18:08:39 ad Exp $    */
   3
   4 /*
   5  * Copyright (c) 2019 Internet Initiative Japan, Inc.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  18  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  19  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  20  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  21  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  22  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  23  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  24  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  25  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  26  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27  * POSSIBILITY OF SUCH DAMAGE.
  28  */
  29
  30 /*
  31  *   Copyright(c) 2007-2019 Intel Corporation. All rights reserved.
  32  *
  33  *   Redistribution and use in source and binary forms, with or without
  34  *   modification, are permitted provided that the following conditions
  35  *   are met:
  36  *
  37  *     * Redistributions of source code must retain the above copyright
  38  *       notice, this list of conditions and the following disclaimer.
  39  *     * Redistributions in binary form must reproduce the above copyright
  40  *       notice, this list of conditions and the following disclaimer in
  41  *       the documentation and/or other materials provided with the
  42  *       distribution.
  43  *     * Neither the name of Intel Corporation nor the names of its
  44  *       contributors may be used to endorse or promote products derived
  45  *       from this software without specific prior written permission.
  46  *
  47  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  48  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  49  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  50  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  51  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  52  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  53  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  54  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  55  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  56  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  57  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  58  */
  59
  60 /* $FreeBSD$ */
  61
  62 #ifndef _DEV_PCI_QATVAR_H_
  63 #define _DEV_PCI_QATVAR_H_
  64
  65 #include <sys/counter.h>
  66 #include <sys/malloc.h>
  67
  68 #include <opencrypto/cryptodev.h>
  69
  70 #define QAT_NSYMREQ     256
  71 #define QAT_NSYMCOOKIE  ((QAT_NSYMREQ * 2 + 1) * 2)
  72 #define QAT_NASYMREQ    64
  73 #define QAT_BATCH_SUBMIT_FREE_SPACE     2
  74
  75 #define QAT_EV_NAME_SIZE                32
  76 #define QAT_RING_NAME_SIZE              32
  77
  78 #define QAT_MAXSEG                      32      /* max segments for sg dma */
  79 #define QAT_MAXLEN                      65535   /* IP_MAXPACKET */
  80
  81 #define QAT_HB_INTERVAL                 500     /* heartbeat msec */
  82 #define QAT_SSM_WDT                     100
  83
  84 enum qat_chip_type {
  85         QAT_CHIP_C2XXX = 0,     /* NanoQAT: Atom C2000 */
  86         QAT_CHIP_C2XXX_IOV,
  87         QAT_CHIP_C3XXX,         /* Atom C3000 */
  88         QAT_CHIP_C3XXX_IOV,
  89         QAT_CHIP_C62X,
  90         QAT_CHIP_C62X_IOV,
  91         QAT_CHIP_D15XX,
  92         QAT_CHIP_D15XX_IOV,
  93         QAT_CHIP_DH895XCC,
  94         QAT_CHIP_DH895XCC_IOV,
  95 };
  96
  97 enum qat_sku {
  98         QAT_SKU_UNKNOWN = 0,
  99         QAT_SKU_1,
 100         QAT_SKU_2,
 101         QAT_SKU_3,
 102         QAT_SKU_4,
 103         QAT_SKU_VF,
 104 };
 105
 106 enum qat_ae_status {
 107         QAT_AE_ENABLED = 1,
 108         QAT_AE_ACTIVE,
 109         QAT_AE_DISABLED
 110 };
 111
 112 #define TIMEOUT_AE_RESET        100
 113 #define TIMEOUT_AE_CHECK        10000
 114 #define TIMEOUT_AE_CSR          500
 115 #define AE_EXEC_CYCLE           20
 116
 117 #define QAT_UOF_MAX_PAGE                1
 118 #define QAT_UOF_MAX_PAGE_REGION         1
 119
 120 struct qat_dmamem {
 121         bus_dma_tag_t qdm_dma_tag;
 122         bus_dmamap_t qdm_dma_map;
 123         bus_size_t qdm_dma_size;
 124         bus_dma_segment_t qdm_dma_seg;
 125         void *qdm_dma_vaddr;
 126 };
 127
 128 /* Valid internal ring size values */
 129 #define QAT_RING_SIZE_128 0x01
 130 #define QAT_RING_SIZE_256 0x02
 131 #define QAT_RING_SIZE_512 0x03
 132 #define QAT_RING_SIZE_4K 0x06
 133 #define QAT_RING_SIZE_16K 0x08
 134 #define QAT_RING_SIZE_4M 0x10
 135 #define QAT_MIN_RING_SIZE QAT_RING_SIZE_128
 136 #define QAT_MAX_RING_SIZE QAT_RING_SIZE_4M
 137 #define QAT_DEFAULT_RING_SIZE QAT_RING_SIZE_16K
 138
 139 /* Valid internal msg size values */
 140 #define QAT_MSG_SIZE_32 0x01
 141 #define QAT_MSG_SIZE_64 0x02
 142 #define QAT_MSG_SIZE_128 0x04
 143 #define QAT_MIN_MSG_SIZE QAT_MSG_SIZE_32
 144 #define QAT_MAX_MSG_SIZE QAT_MSG_SIZE_128
 145
 146 /* Size to bytes conversion macros for ring and msg size values */
 147 #define QAT_MSG_SIZE_TO_BYTES(SIZE) (SIZE << 5)
 148 #define QAT_BYTES_TO_MSG_SIZE(SIZE) (SIZE >> 5)
 149 #define QAT_SIZE_TO_RING_SIZE_IN_BYTES(SIZE) ((1 << (SIZE - 1)) << 7)
 150 #define QAT_RING_SIZE_IN_BYTES_TO_SIZE(SIZE) ((1 << (SIZE - 1)) >> 7)
 151
 152 /* Minimum ring bufer size for memory allocation */
 153 #define QAT_RING_SIZE_BYTES_MIN(SIZE) \
 154         ((SIZE < QAT_SIZE_TO_RING_SIZE_IN_BYTES(QAT_RING_SIZE_4K)) ? \
 155                 QAT_SIZE_TO_RING_SIZE_IN_BYTES(QAT_RING_SIZE_4K) : SIZE)
 156 #define QAT_RING_SIZE_MODULO(SIZE) (SIZE + 0x6)
 157 #define QAT_SIZE_TO_POW(SIZE) ((((SIZE & 0x4) >> 1) | ((SIZE & 0x4) >> 2) | \
 158                                 SIZE) & ~0x4)
 159 /* Max outstanding requests */
 160 #define QAT_MAX_INFLIGHTS(RING_SIZE, MSG_SIZE) \
 161         ((((1 << (RING_SIZE - 1)) << 3) >> QAT_SIZE_TO_POW(MSG_SIZE)) - 1)
 162
 163 #define QAT_RING_PATTERN                0x7f
 164
 165 struct qat_softc;
 166
 167 typedef int (*qat_cb_t)(struct qat_softc *, void *, void *);
 168
 169 struct qat_ring {
 170         struct mtx qr_ring_mtx;   /* Lock per ring */
 171         bool qr_need_wakeup;
 172         void *qr_ring_vaddr;
 173         uint32_t * volatile qr_inflight;        /* tx/rx shared */
 174         uint32_t qr_head;
 175         uint32_t qr_tail;
 176         uint8_t qr_msg_size;
 177         uint8_t qr_ring_size;
 178         uint32_t qr_ring;       /* ring number in bank */
 179         uint32_t qr_bank;       /* bank number in device */
 180         uint32_t qr_ring_id;
 181         uint32_t qr_ring_mask;
 182         qat_cb_t qr_cb;
 183         void *qr_cb_arg;
 184         struct qat_dmamem qr_dma;
 185         bus_addr_t qr_ring_paddr;
 186
 187         const char *qr_name;
 188 };
 189
 190 struct qat_bank {
 191         struct qat_softc *qb_sc;        /* back pointer to softc */
 192         uint32_t qb_intr_mask;          /* current interrupt mask */
 193         uint32_t qb_allocated_rings;    /* current allocated ring bitfiled */
 194         uint32_t qb_coalescing_time;    /* timer in nano sec, 0: disabled */
 195 #define COALESCING_TIME_INTERVAL_DEFAULT        10000
 196 #define COALESCING_TIME_INTERVAL_MIN            500
 197 #define COALESCING_TIME_INTERVAL_MAX            0xfffff
 198         uint32_t qb_bank;               /* bank index */
 199         struct mtx qb_bank_mtx;
 200         struct resource *qb_ih;
 201         void *qb_ih_cookie;
 202
 203         struct qat_ring qb_et_rings[MAX_RING_PER_BANK];
 204
 205 };
 206
 207 struct qat_ap_bank {
 208         uint32_t qab_nf_mask;
 209         uint32_t qab_nf_dest;
 210         uint32_t qab_ne_mask;
 211         uint32_t qab_ne_dest;
 212 };
 213
 214 struct qat_ae_page {
 215         struct qat_ae_page *qap_next;
 216         struct qat_uof_page *qap_page;
 217         struct qat_ae_region *qap_region;
 218         u_int qap_flags;
 219 };
 220
 221 #define QAT_AE_PAGA_FLAG_WAITING        (1 << 0)
 222
 223 struct qat_ae_region {
 224         struct qat_ae_page *qar_loaded_page;
 225         STAILQ_HEAD(, qat_ae_page) qar_waiting_pages;
 226 };
 227
 228 struct qat_ae_slice {
 229         u_int qas_assigned_ctx_mask;
 230         struct qat_ae_region qas_regions[QAT_UOF_MAX_PAGE_REGION];
 231         struct qat_ae_page qas_pages[QAT_UOF_MAX_PAGE];
 232         struct qat_ae_page *qas_cur_pages[MAX_AE_CTX];
 233         struct qat_uof_image *qas_image;
 234 };
 235
 236 #define QAT_AE(sc, ae)                  \
 237                 ((sc)->sc_ae[ae])
 238
 239 struct qat_ae {
 240         u_int qae_state;                /* AE state */
 241         u_int qae_ustore_size;          /* free micro-store address */
 242         u_int qae_free_addr;            /* free micro-store address */
 243         u_int qae_free_size;            /* free micro-store size */
 244         u_int qae_live_ctx_mask;        /* live context mask */
 245         u_int qae_ustore_dram_addr;     /* mirco-store DRAM address */
 246         u_int qae_reload_size;          /* reloadable code size */
 247
 248         /* aefw */
 249         u_int qae_num_slices;
 250         struct qat_ae_slice qae_slices[MAX_AE_CTX];
 251         u_int qae_reloc_ustore_dram;    /* reloadable ustore-dram address */
 252         u_int qae_effect_ustore_size;   /* effective AE ustore size */
 253         u_int qae_shareable_ustore;
 254 };
 255
 256 struct qat_mof {
 257         void *qmf_sym;                  /* SYM_OBJS in sc_fw_mof */
 258         size_t qmf_sym_size;
 259         void *qmf_uof_objs;             /* UOF_OBJS in sc_fw_mof */
 260         size_t qmf_uof_objs_size;
 261         void *qmf_suof_objs;            /* SUOF_OBJS in sc_fw_mof */
 262         size_t qmf_suof_objs_size;
 263 };
 264
 265 struct qat_ae_batch_init {
 266         u_int qabi_ae;
 267         u_int qabi_addr;
 268         u_int *qabi_value;
 269         u_int qabi_size;
 270         STAILQ_ENTRY(qat_ae_batch_init) qabi_next;
 271 };
 272
 273 STAILQ_HEAD(qat_ae_batch_init_list, qat_ae_batch_init);
 274
 275 /* overwritten struct uof_uword_block */
 276 struct qat_uof_uword_block {
 277         u_int quub_start_addr;          /* start address */
 278         u_int quub_num_words;           /* number of microwords */
 279         uint64_t quub_micro_words;      /* pointer to the uwords */
 280 };
 281
 282 struct qat_uof_page {
 283         u_int qup_page_num;             /* page number */
 284         u_int qup_def_page;             /* default page */
 285         u_int qup_page_region;          /* region of page */
 286         u_int qup_beg_vaddr;            /* begin virtual address */
 287         u_int qup_beg_paddr;            /* begin physical address */
 288
 289         u_int qup_num_uc_var;           /* num of uC var in array */
 290         struct uof_uword_fixup *qup_uc_var;
 291                                         /* array of import variables */
 292         u_int qup_num_imp_var;          /* num of import var in array */
 293         struct uof_import_var *qup_imp_var;
 294                                         /* array of import variables */
 295         u_int qup_num_imp_expr;         /* num of import expr in array */
 296         struct uof_uword_fixup *qup_imp_expr;
 297                                         /* array of import expressions */
 298         u_int qup_num_neigh_reg;        /* num of neigh-reg in array */
 299         struct uof_uword_fixup *qup_neigh_reg;
 300                                         /* array of neigh-reg assignments */
 301         u_int qup_num_micro_words;      /* number of microwords in the seg */
 302
 303         u_int qup_num_uw_blocks;        /* number of uword blocks */
 304         struct qat_uof_uword_block *qup_uw_blocks;
 305                                         /* array of uword blocks */
 306 };
 307
 308 struct qat_uof_image {
 309         struct uof_image *qui_image;            /* image pointer */
 310         struct qat_uof_page qui_pages[QAT_UOF_MAX_PAGE];
 311                                                 /* array of pages */
 312
 313         u_int qui_num_ae_reg;                   /* num of registers */
 314         struct uof_ae_reg *qui_ae_reg;          /* array of registers */
 315
 316         u_int qui_num_init_reg_sym;             /* num of reg/sym init values */
 317         struct uof_init_reg_sym *qui_init_reg_sym;
 318                                         /* array of reg/sym init values */
 319
 320         u_int qui_num_sbreak;                   /* num of sbreak values */
 321         struct qui_sbreak *qui_sbreak;          /* array of sbreak values */
 322
 323         u_int qui_num_uwords_used;
 324                                 /* highest uword addressreferenced + 1 */
 325 };
 326
 327 struct qat_aefw_uof {
 328         size_t qafu_size;                       /* uof size */
 329         struct uof_obj_hdr *qafu_obj_hdr;       /* UOF_OBJS */
 330
 331         void *qafu_str_tab;
 332         size_t qafu_str_tab_size;
 333
 334         u_int qafu_num_init_mem;
 335         struct uof_init_mem *qafu_init_mem;
 336         size_t qafu_init_mem_size;
 337
 338         struct uof_var_mem_seg *qafu_var_mem_seg;
 339
 340         struct qat_ae_batch_init_list qafu_lm_init[MAX_AE];
 341         size_t qafu_num_lm_init[MAX_AE];
 342         size_t qafu_num_lm_init_inst[MAX_AE];
 343
 344         u_int qafu_num_imgs;                    /* number of uof image */
 345         struct qat_uof_image qafu_imgs[MAX_NUM_AE * MAX_AE_CTX];
 346                                                 /* uof images */
 347 };
 348
 349 #define QAT_SERVICE_CRYPTO_A            (1 << 0)
 350 #define QAT_SERVICE_CRYPTO_B            (1 << 1)
 351
 352 struct qat_admin_rings {
 353         uint32_t qadr_active_aes_per_accel;
 354         uint8_t qadr_srv_mask[MAX_AE_PER_ACCEL];
 355
 356         struct qat_dmamem qadr_dma;
 357         struct fw_init_ring_table *qadr_master_ring_tbl;
 358         struct fw_init_ring_table *qadr_cya_ring_tbl;
 359         struct fw_init_ring_table *qadr_cyb_ring_tbl;
 360
 361         struct qat_ring *qadr_admin_tx;
 362         struct qat_ring *qadr_admin_rx;
 363 };
 364
 365 struct qat_accel_init_cb {
 366         int qaic_status;
 367 };
 368
 369 struct qat_admin_comms {
 370         struct qat_dmamem qadc_dma;
 371         struct qat_dmamem qadc_const_tbl_dma;
 372         struct qat_dmamem qadc_hb_dma;
 373 };
 374
 375 #define QAT_PID_MINOR_REV 0xf
 376 #define QAT_PID_MAJOR_REV (0xf << 4)
 377
 378 struct qat_suof_image {
 379         char *qsi_simg_buf;
 380         u_long qsi_simg_len;
 381         char *qsi_css_header;
 382         char *qsi_css_key;
 383         char *qsi_css_signature;
 384         char *qsi_css_simg;
 385         u_long qsi_simg_size;
 386         u_int qsi_ae_num;
 387         u_int qsi_ae_mask;
 388         u_int qsi_fw_type;
 389         u_long qsi_simg_name;
 390         u_long qsi_appmeta_data;
 391         struct qat_dmamem qsi_dma;
 392 };
 393
 394 struct qat_aefw_suof {
 395         u_int qafs_file_id;
 396         u_int qafs_check_sum;
 397         char qafs_min_ver;
 398         char qafs_maj_ver;
 399         char qafs_fw_type;
 400         char *qafs_suof_buf;
 401         u_int qafs_suof_size;
 402         char *qafs_sym_str;
 403         u_int qafs_sym_size;
 404         u_int qafs_num_simgs;
 405         struct qat_suof_image *qafs_simg;
 406 };
 407
 408 enum qat_sym_hash_algorithm {
 409         QAT_SYM_HASH_NONE = 0,
 410         QAT_SYM_HASH_MD5 = 1,
 411         QAT_SYM_HASH_SHA1 = 2,
 412         QAT_SYM_HASH_SHA224 = 3,
 413         QAT_SYM_HASH_SHA256 = 4,
 414         QAT_SYM_HASH_SHA384 = 5,
 415         QAT_SYM_HASH_SHA512 = 6,
 416         QAT_SYM_HASH_AES_XCBC = 7,
 417         QAT_SYM_HASH_AES_CCM = 8,
 418         QAT_SYM_HASH_AES_GCM = 9,
 419         QAT_SYM_HASH_KASUMI_F9 = 10,
 420         QAT_SYM_HASH_SNOW3G_UIA2 = 11,
 421         QAT_SYM_HASH_AES_CMAC = 12,
 422         QAT_SYM_HASH_AES_GMAC = 13,
 423         QAT_SYM_HASH_AES_CBC_MAC = 14,
 424 };
 425
 426 #define QAT_HASH_MD5_BLOCK_SIZE                 64
 427 #define QAT_HASH_MD5_DIGEST_SIZE                16
 428 #define QAT_HASH_MD5_STATE_SIZE                 16
 429 #define QAT_HASH_SHA1_BLOCK_SIZE                64
 430 #define QAT_HASH_SHA1_DIGEST_SIZE               20
 431 #define QAT_HASH_SHA1_STATE_SIZE                20
 432 #define QAT_HASH_SHA224_BLOCK_SIZE              64
 433 #define QAT_HASH_SHA224_DIGEST_SIZE             28
 434 #define QAT_HASH_SHA224_STATE_SIZE              32
 435 #define QAT_HASH_SHA256_BLOCK_SIZE              64
 436 #define QAT_HASH_SHA256_DIGEST_SIZE             32
 437 #define QAT_HASH_SHA256_STATE_SIZE              32
 438 #define QAT_HASH_SHA384_BLOCK_SIZE              128
 439 #define QAT_HASH_SHA384_DIGEST_SIZE             48
 440 #define QAT_HASH_SHA384_STATE_SIZE              64
 441 #define QAT_HASH_SHA512_BLOCK_SIZE              128
 442 #define QAT_HASH_SHA512_DIGEST_SIZE             64
 443 #define QAT_HASH_SHA512_STATE_SIZE              64
 444 #define QAT_HASH_XCBC_PRECOMP_KEY_NUM           3
 445 #define QAT_HASH_XCBC_MAC_BLOCK_SIZE            16
 446 #define QAT_HASH_XCBC_MAC_128_DIGEST_SIZE       16
 447 #define QAT_HASH_CMAC_BLOCK_SIZE                16
 448 #define QAT_HASH_CMAC_128_DIGEST_SIZE           16
 449 #define QAT_HASH_AES_CCM_BLOCK_SIZE             16
 450 #define QAT_HASH_AES_CCM_DIGEST_SIZE            16
 451 #define QAT_HASH_AES_GCM_BLOCK_SIZE             16
 452 #define QAT_HASH_AES_GCM_DIGEST_SIZE            16
 453 #define QAT_HASH_AES_GCM_STATE_SIZE             16
 454 #define QAT_HASH_KASUMI_F9_BLOCK_SIZE           8
 455 #define QAT_HASH_KASUMI_F9_DIGEST_SIZE          4
 456 #define QAT_HASH_SNOW3G_UIA2_BLOCK_SIZE         8
 457 #define QAT_HASH_SNOW3G_UIA2_DIGEST_SIZE        4
 458 #define QAT_HASH_AES_CBC_MAC_BLOCK_SIZE         16
 459 #define QAT_HASH_AES_CBC_MAC_DIGEST_SIZE        16
 460 #define QAT_HASH_AES_GCM_ICV_SIZE_8             8
 461 #define QAT_HASH_AES_GCM_ICV_SIZE_12            12
 462 #define QAT_HASH_AES_GCM_ICV_SIZE_16            16
 463 #define QAT_HASH_AES_CCM_ICV_SIZE_MIN           4
 464 #define QAT_HASH_AES_CCM_ICV_SIZE_MAX           16
 465 #define QAT_HASH_IPAD_BYTE                      0x36
 466 #define QAT_HASH_OPAD_BYTE                      0x5c
 467 #define QAT_HASH_IPAD_4_BYTES                   0x36363636
 468 #define QAT_HASH_OPAD_4_BYTES                   0x5c5c5c5c
 469 #define QAT_HASH_KASUMI_F9_KEY_MODIFIER_4_BYTES 0xAAAAAAAA
 470
 471 #define QAT_SYM_XCBC_STATE_SIZE         ((QAT_HASH_XCBC_MAC_BLOCK_SIZE) * 3)
 472 #define QAT_SYM_CMAC_STATE_SIZE         ((QAT_HASH_CMAC_BLOCK_SIZE) * 3)
 473
 474 struct qat_sym_hash_alg_info {
 475         uint32_t qshai_digest_len;              /* Digest length in bytes */
 476         uint32_t qshai_block_len;               /* Block length in bytes */
 477         uint32_t qshai_state_size;              /* size of above state in bytes */
 478         const uint8_t *qshai_init_state;        /* Initial state */
 479
 480         const struct auth_hash *qshai_sah;      /* software auth hash */
 481         uint32_t qshai_state_offset;            /* offset to state in *_CTX */
 482         uint32_t qshai_state_word;
 483 };
 484
 485 struct qat_sym_hash_qat_info {
 486         uint32_t qshqi_algo_enc;        /* QAT Algorithm encoding */
 487         uint32_t qshqi_auth_counter;    /* Counter value for Auth */
 488         uint32_t qshqi_state1_len;      /* QAT state1 length in bytes */
 489         uint32_t qshqi_state2_len;      /* QAT state2 length in bytes */
 490 };
 491
 492 struct qat_sym_hash_def {
 493         const struct qat_sym_hash_alg_info *qshd_alg;
 494         const struct qat_sym_hash_qat_info *qshd_qat;
 495 };
 496
 497 #define QAT_SYM_REQ_PARAMS_SIZE_MAX                     (24 + 32)
 498 /* Reserve enough space for cipher and authentication request params */
 499 /* Basis of values are guaranteed in qat_hw*var.h with CTASSERT */
 500
 501 #define QAT_SYM_REQ_PARAMS_SIZE_PADDED                  \
 502                 roundup(QAT_SYM_REQ_PARAMS_SIZE_MAX, QAT_OPTIMAL_ALIGN)
 503 /* Pad out to 64-byte multiple to ensure optimal alignment of next field */
 504
 505 #define QAT_SYM_KEY_TLS_PREFIX_SIZE                     (128)
 506 /* Hash Prefix size in bytes for TLS (128 = MAX = SHA2 (384, 512)*/
 507
 508 #define QAT_SYM_KEY_MAX_HASH_STATE_BUFFER               \
 509                 (QAT_SYM_KEY_TLS_PREFIX_SIZE * 2)
 510 /* hash state prefix buffer structure that holds the maximum sized secret */
 511
 512 #define QAT_SYM_HASH_BUFFER_LEN                 QAT_HASH_SHA512_STATE_SIZE
 513 /* Buffer length to hold 16 byte MD5 key and 20 byte SHA1 key */
 514
 515 #define QAT_GCM_AAD_SIZE_MAX            240
 516 /* Maximum AAD size */
 517
 518 #define QAT_AES_GCM_AAD_ALIGN           16
 519
 520 struct qat_sym_bulk_cookie {
 521         uint8_t qsbc_req_params_buf[QAT_SYM_REQ_PARAMS_SIZE_PADDED];
 522         /* memory block reserved for request params
 523          * NOTE: Field must be correctly aligned in memory for access by QAT
 524          * engine */
 525         struct qat_crypto *qsbc_crypto;
 526         struct qat_session *qsbc_session;
 527         /* Session context */
 528         void *qsbc_cb_tag;
 529         /* correlator supplied by the client */
 530         uint8_t qsbc_msg[QAT_MSG_SIZE_TO_BYTES(QAT_MAX_MSG_SIZE)];
 531         /* QAT request message */
 532 } __aligned(QAT_OPTIMAL_ALIGN);
 533
 534 /* Basis of values are guaranteed in qat_hw*var.h with CTASSERT */
 535 #define HASH_CONTENT_DESC_SIZE          176
 536 #define CIPHER_CONTENT_DESC_SIZE        64
 537
 538 #define CONTENT_DESC_MAX_SIZE   roundup(                                \
 539                 HASH_CONTENT_DESC_SIZE + CIPHER_CONTENT_DESC_SIZE,      \
 540                 QAT_OPTIMAL_ALIGN)
 541
 542 struct qat_sym_cookie {
 543         union qat_sym_cookie_u {
 544                 /* should be 64byte aligned */
 545                 struct qat_sym_bulk_cookie qsc_bulk_cookie;
 546                                                 /* symmetric bulk cookie */
 547 #ifdef notyet
 548                 struct qat_sym_key_cookie qsc_key_cookie;
 549                                                 /* symmetric key cookie */
 550                 struct qat_sym_nrbg_cookie qsc_nrbg_cookie;
 551                                                 /* symmetric NRBG cookie */
 552 #endif
 553         } u;
 554
 555         /* should be 64-byte aligned */
 556         struct buffer_list_desc qsc_buf_list;
 557         struct flat_buffer_desc qsc_flat_bufs[QAT_MAXSEG]; /* should be here */
 558
 559         bus_dmamap_t qsc_self_dmamap;   /* self DMA mapping and
 560                                            end of DMA region */
 561         bus_dma_tag_t qsc_self_dma_tag;
 562
 563         uint8_t qsc_iv_buf[EALG_MAX_BLOCK_LEN];
 564         uint8_t qsc_auth_res[QAT_SYM_HASH_BUFFER_LEN];
 565         uint8_t qsc_gcm_aad[QAT_GCM_AAD_SIZE_MAX];
 566         uint8_t qsc_content_desc[CONTENT_DESC_MAX_SIZE];
 567         struct cryptodesc *qsc_enc;
 568         struct cryptodesc *qsc_mac;
 569
 570         bus_dmamap_t qsc_buf_dmamap;    /* qsc_flat_bufs DMA mapping */
 571         bus_dma_tag_t qsc_buf_dma_tag;
 572         void *qsc_buf;
 573
 574         bus_addr_t qsc_bulk_req_params_buf_paddr;
 575         bus_addr_t qsc_buffer_list_desc_paddr;
 576         bus_addr_t qsc_iv_buf_paddr;
 577         bus_addr_t qsc_auth_res_paddr;
 578         bus_addr_t qsc_gcm_aad_paddr;
 579         bus_addr_t qsc_content_desc_paddr;
 580 };
 581
 582 CTASSERT(offsetof(struct qat_sym_cookie,
 583     u.qsc_bulk_cookie.qsbc_req_params_buf) % QAT_OPTIMAL_ALIGN == 0);
 584 CTASSERT(offsetof(struct qat_sym_cookie, qsc_buf_list) % QAT_OPTIMAL_ALIGN == 0);
 585
 586 #define MAX_CIPHER_SETUP_BLK_SZ                                         \
 587                 (sizeof(struct hw_cipher_config) +                      \
 588                 2 * HW_KASUMI_KEY_SZ + 2 * HW_KASUMI_BLK_SZ)
 589 #define MAX_HASH_SETUP_BLK_SZ   sizeof(union hw_auth_algo_blk)
 590
 591 struct qat_crypto_desc {
 592         uint8_t qcd_content_desc[CONTENT_DESC_MAX_SIZE]; /* must be first */
 593         /* using only for qat 1.5 */
 594         uint8_t qcd_hash_state_prefix_buf[QAT_GCM_AAD_SIZE_MAX];
 595
 596         bus_addr_t qcd_desc_paddr;
 597         bus_addr_t qcd_hash_state_paddr;
 598
 599         enum fw_slice qcd_slices[MAX_FW_SLICE + 1];
 600         enum fw_la_cmd_id qcd_cmd_id;
 601         enum hw_cipher_dir qcd_cipher_dir;
 602
 603         /* content desc info */
 604         uint8_t qcd_hdr_sz;             /* in quad words */
 605         uint8_t qcd_hw_blk_sz;          /* in quad words */
 606         uint32_t qcd_cipher_offset;
 607         uint32_t qcd_auth_offset;
 608         /* hash info */
 609         uint8_t qcd_state_storage_sz;   /* in quad words */
 610         uint32_t qcd_gcm_aad_sz_offset1;
 611         uint32_t qcd_gcm_aad_sz_offset2;
 612         /* cipher info */
 613         uint16_t qcd_cipher_blk_sz;     /* in bytes */
 614         uint16_t qcd_auth_sz;           /* in bytes */
 615
 616         uint8_t qcd_req_cache[QAT_MSG_SIZE_TO_BYTES(QAT_MAX_MSG_SIZE)];
 617 } __aligned(QAT_OPTIMAL_ALIGN);
 618
 619 /* should be aligned to 64bytes */
 620 struct qat_session {
 621         struct qat_crypto_desc *qs_dec_desc;    /* should be at top of struct*/
 622         /* decrypt or auth then decrypt or auth */
 623
 624         struct qat_crypto_desc *qs_enc_desc;
 625         /* encrypt or encrypt then auth */
 626
 627         struct qat_dmamem qs_desc_mem;
 628
 629         enum hw_cipher_algo qs_cipher_algo;
 630         enum hw_cipher_mode qs_cipher_mode;
 631         enum hw_auth_algo qs_auth_algo;
 632         enum hw_auth_mode qs_auth_mode;
 633
 634         const uint8_t *qs_cipher_key;
 635         int qs_cipher_klen;
 636         const uint8_t *qs_auth_key;
 637         int qs_auth_klen;
 638         int qs_auth_mlen;
 639         int qs_ivlen;
 640
 641         uint32_t qs_status;
 642 #define QAT_SESSION_STATUS_ACTIVE       (1 << 0)
 643 #define QAT_SESSION_STATUS_FREEING      (1 << 1)
 644         uint32_t qs_inflight;
 645         int qs_aad_length;
 646         bool qs_need_wakeup;
 647
 648         struct mtx qs_session_mtx;
 649 };
 650
 651 struct qat_crypto_bank {
 652         uint16_t qcb_bank;
 653
 654         struct qat_ring *qcb_sym_tx;
 655         struct qat_ring *qcb_sym_rx;
 656
 657         struct qat_dmamem qcb_symck_dmamems[QAT_NSYMCOOKIE];
 658         struct qat_sym_cookie *qcb_symck_free[QAT_NSYMCOOKIE];
 659         uint32_t qcb_symck_free_count;
 660
 661         struct mtx qcb_bank_mtx;
 662
 663         char qcb_ring_names[2][QAT_RING_NAME_SIZE];     /* sym tx,rx */
 664 };
 665
 666 struct qat_crypto {
 667         struct qat_softc *qcy_sc;
 668         uint32_t qcy_bank_mask;
 669         uint16_t qcy_num_banks;
 670
 671         int32_t qcy_cid;                /* OpenCrypto driver ID */
 672
 673         struct qat_crypto_bank *qcy_banks; /* array of qat_crypto_bank */
 674
 675         uint32_t qcy_session_free_count;
 676
 677         struct mtx qcy_crypto_mtx;
 678 };
 679
 680 struct qat_hw {
 681         int8_t qhw_sram_bar_id;
 682         int8_t qhw_misc_bar_id;
 683         int8_t qhw_etr_bar_id;
 684
 685         bus_size_t qhw_cap_global_offset;
 686         bus_size_t qhw_ae_offset;
 687         bus_size_t qhw_ae_local_offset;
 688         bus_size_t qhw_etr_bundle_size;
 689
 690         /* crypto processing callbacks */
 691         size_t qhw_crypto_opaque_offset;
 692         void (*qhw_crypto_setup_req_params)(struct qat_crypto_bank *,
 693             struct qat_session *, struct qat_crypto_desc const *,
 694             struct qat_sym_cookie *, struct cryptodesc *, struct cryptodesc *);
 695         void (*qhw_crypto_setup_desc)(struct qat_crypto *, struct qat_session *,
 696             struct qat_crypto_desc *);
 697
 698         uint8_t qhw_num_banks;                  /* max number of banks */
 699         uint8_t qhw_num_ap_banks;               /* max number of AutoPush banks */
 700         uint8_t qhw_num_rings_per_bank;         /* rings per bank */
 701         uint8_t qhw_num_accel;                  /* max number of accelerators */
 702         uint8_t qhw_num_engines;                /* max number of accelerator engines */
 703         uint8_t qhw_tx_rx_gap;
 704         uint32_t qhw_tx_rings_mask;
 705         uint32_t qhw_clock_per_sec;
 706         bool qhw_fw_auth;
 707         uint32_t qhw_fw_req_size;
 708         uint32_t qhw_fw_resp_size;
 709
 710         uint8_t qhw_ring_sym_tx;
 711         uint8_t qhw_ring_sym_rx;
 712         uint8_t qhw_ring_asym_tx;
 713         uint8_t qhw_ring_asym_rx;
 714
 715         /* MSIx */
 716         uint32_t qhw_msix_ae_vec_gap;   /* gap to ae vec from bank */
 717
 718         const char *qhw_mof_fwname;
 719         const char *qhw_mmp_fwname;
 720
 721         uint32_t qhw_prod_type;         /* cpu type */
 722
 723         /* setup callbacks */
 724         uint32_t (*qhw_get_accel_mask)(struct qat_softc *);
 725         uint32_t (*qhw_get_ae_mask)(struct qat_softc *);
 726         enum qat_sku (*qhw_get_sku)(struct qat_softc *);
 727         uint32_t (*qhw_get_accel_cap)(struct qat_softc *);
 728         const char *(*qhw_get_fw_uof_name)(struct qat_softc *);
 729         void (*qhw_enable_intr)(struct qat_softc *);
 730         void (*qhw_init_etr_intr)(struct qat_softc *, int);
 731         int (*qhw_init_admin_comms)(struct qat_softc *);
 732         int (*qhw_send_admin_init)(struct qat_softc *);
 733         int (*qhw_init_arb)(struct qat_softc *);
 734         void (*qhw_get_arb_mapping)(struct qat_softc *, const uint32_t **);
 735         void (*qhw_enable_error_correction)(struct qat_softc *);
 736         int (*qhw_check_uncorrectable_error)(struct qat_softc *);
 737         void (*qhw_print_err_registers)(struct qat_softc *);
 738         void (*qhw_disable_error_interrupts)(struct qat_softc *);
 739         int (*qhw_check_slice_hang)(struct qat_softc *);
 740         int (*qhw_set_ssm_wdtimer)(struct qat_softc *);
 741 };
 742
 743
 744 /* sc_flags */
 745 #define QAT_FLAG_ESRAM_ENABLE_AUTO_INIT (1 << 0)
 746 #define QAT_FLAG_SHRAM_WAIT_READY       (1 << 1)
 747
 748 /* sc_accel_cap */
 749 #define QAT_ACCEL_CAP_CRYPTO_SYMMETRIC  (1 << 0)
 750 #define QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC (1 << 1)
 751 #define QAT_ACCEL_CAP_CIPHER            (1 << 2)
 752 #define QAT_ACCEL_CAP_AUTHENTICATION    (1 << 3)
 753 #define QAT_ACCEL_CAP_REGEX             (1 << 4)
 754 #define QAT_ACCEL_CAP_COMPRESSION       (1 << 5)
 755 #define QAT_ACCEL_CAP_LZS_COMPRESSION   (1 << 6)
 756 #define QAT_ACCEL_CAP_RANDOM_NUMBER     (1 << 7)
 757 #define QAT_ACCEL_CAP_ZUC               (1 << 8)
 758 #define QAT_ACCEL_CAP_SHA3              (1 << 9)
 759 #define QAT_ACCEL_CAP_KPT               (1 << 10)
 760
 761 #define QAT_ACCEL_CAP_BITS      \
 762         "\177\020"      \
 763         "b\x0a"         "KPT\0" \
 764         "b\x09"         "SHA3\0" \
 765         "b\x08"         "ZUC\0" \
 766         "b\x07"         "RANDOM_NUMBER\0" \
 767         "b\x06"         "LZS_COMPRESSION\0" \
 768         "b\x05"         "COMPRESSION\0" \
 769         "b\x04"         "REGEX\0" \
 770         "b\x03"         "AUTHENTICATION\0" \
 771         "b\x02"         "CIPHER\0" \
 772         "b\x01"         "CRYPTO_ASYMMETRIC\0" \
 773         "b\x00"         "CRYPTO_SYMMETRIC\0"
 774
 775 #define QAT_HI_PRIO_RING_WEIGHT         0xfc
 776 #define QAT_LO_PRIO_RING_WEIGHT         0xfe
 777 #define QAT_DEFAULT_RING_WEIGHT         0xff
 778 #define QAT_DEFAULT_PVL                 0
 779
 780 struct firmware;
 781 struct resource;
 782
 783 struct qat_softc {
 784         device_t sc_dev;
 785
 786         struct resource *sc_res[MAX_BARS];
 787         int sc_rid[MAX_BARS];
 788         bus_space_tag_t sc_csrt[MAX_BARS];
 789         bus_space_handle_t sc_csrh[MAX_BARS];
 790
 791         uint32_t sc_ae_num;
 792         uint32_t sc_ae_mask;
 793
 794         struct qat_crypto sc_crypto;            /* crypto services */
 795
 796         struct qat_hw sc_hw;
 797
 798         uint8_t sc_rev;
 799         enum qat_sku sc_sku;
 800         uint32_t sc_flags;
 801
 802         uint32_t sc_accel_num;
 803         uint32_t sc_accel_mask;
 804         uint32_t sc_accel_cap;
 805
 806         struct qat_admin_rings sc_admin_rings;  /* use only for qat 1.5 */
 807         struct qat_admin_comms sc_admin_comms;  /* use only for qat 1.7 */
 808
 809         /* ETR */
 810         struct qat_bank *sc_etr_banks;          /* array of etr banks */
 811         struct qat_ap_bank *sc_etr_ap_banks;    /* array of etr auto push banks */
 812
 813         /* AE */
 814         struct qat_ae sc_ae[MAX_NUM_AE];
 815
 816         /* Interrupt */
 817         struct resource *sc_ih;                 /* ae cluster ih */
 818         void *sc_ih_cookie;                     /* ae cluster ih cookie */
 819
 820         /* Counters */
 821         counter_u64_t sc_gcm_aad_restarts;
 822         counter_u64_t sc_gcm_aad_updates;
 823         counter_u64_t sc_ring_full_restarts;
 824         counter_u64_t sc_sym_alloc_failures;
 825
 826         /* Firmware */
 827         void *sc_fw_mof;                        /* mof data */
 828         size_t sc_fw_mof_size;                  /* mof size */
 829         struct qat_mof sc_mof;                  /* mof sections */
 830
 831         const char *sc_fw_uof_name;             /* uof/suof name in mof */
 832
 833         void *sc_fw_uof;                        /* uof head */
 834         size_t sc_fw_uof_size;                  /* uof size */
 835         struct qat_aefw_uof sc_aefw_uof;        /* UOF_OBJS in uof */
 836
 837         void *sc_fw_suof;                       /* suof head */
 838         size_t sc_fw_suof_size;                 /* suof size */
 839         struct qat_aefw_suof sc_aefw_suof;      /* suof context */
 840
 841         void *sc_fw_mmp;                        /* mmp data */
 842         size_t sc_fw_mmp_size;                  /* mmp size */
 843 };
 844
 845 static inline void
 846 qat_bar_write_4(struct qat_softc *sc, int baroff, bus_size_t offset,
 847     uint32_t value)
 848 {
 849
 850         MPASS(baroff >= 0 && baroff < MAX_BARS);
 851
 852         bus_space_write_4(sc->sc_csrt[baroff],
 853             sc->sc_csrh[baroff], offset, value);
 854 }
 855
 856 static inline uint32_t
 857 qat_bar_read_4(struct qat_softc *sc, int baroff, bus_size_t offset)
 858 {
 859
 860         MPASS(baroff >= 0 && baroff < MAX_BARS);
 861
 862         return bus_space_read_4(sc->sc_csrt[baroff],
 863             sc->sc_csrh[baroff], offset);
 864 }
 865
 866 static inline void
 867 qat_misc_write_4(struct qat_softc *sc, bus_size_t offset, uint32_t value)
 868 {
 869
 870         qat_bar_write_4(sc, sc->sc_hw.qhw_misc_bar_id, offset, value);
 871 }
 872
 873 static inline uint32_t
 874 qat_misc_read_4(struct qat_softc *sc, bus_size_t offset)
 875 {
 876
 877         return qat_bar_read_4(sc, sc->sc_hw.qhw_misc_bar_id, offset);
 878 }
 879
 880 static inline void
 881 qat_misc_read_write_or_4(struct qat_softc *sc, bus_size_t offset,
 882     uint32_t value)
 883 {
 884         uint32_t reg;
 885
 886         reg = qat_misc_read_4(sc, offset);
 887         reg |= value;
 888         qat_misc_write_4(sc, offset, reg);
 889 }
 890
 891 static inline void
 892 qat_misc_read_write_and_4(struct qat_softc *sc, bus_size_t offset,
 893     uint32_t mask)
 894 {
 895         uint32_t reg;
 896
 897         reg = qat_misc_read_4(sc, offset);
 898         reg &= mask;
 899         qat_misc_write_4(sc, offset, reg);
 900 }
 901
 902 static inline void
 903 qat_etr_write_4(struct qat_softc *sc, bus_size_t offset, uint32_t value)
 904 {
 905
 906         qat_bar_write_4(sc, sc->sc_hw.qhw_etr_bar_id, offset, value);
 907 }
 908
 909 static inline uint32_t
 910 qat_etr_read_4(struct qat_softc *sc, bus_size_t offset)
 911 {
 912
 913         return qat_bar_read_4(sc, sc->sc_hw.qhw_etr_bar_id, offset);
 914 }
 915
 916 static inline void
 917 qat_ae_local_write_4(struct qat_softc *sc, u_char ae, bus_size_t offset,
 918         uint32_t value)
 919 {
 920
 921         offset = __SHIFTIN(ae & sc->sc_ae_mask, AE_LOCAL_AE_MASK) |
 922             (offset & AE_LOCAL_CSR_MASK);
 923
 924         qat_misc_write_4(sc, sc->sc_hw.qhw_ae_local_offset + offset,
 925             value);
 926 }
 927
 928 static inline uint32_t
 929 qat_ae_local_read_4(struct qat_softc *sc, u_char ae, bus_size_t offset)
 930 {
 931
 932         offset = __SHIFTIN(ae & sc->sc_ae_mask, AE_LOCAL_AE_MASK) |
 933             (offset & AE_LOCAL_CSR_MASK);
 934
 935         return qat_misc_read_4(sc, sc->sc_hw.qhw_ae_local_offset + offset);
 936 }
 937
 938 static inline void
 939 qat_ae_xfer_write_4(struct qat_softc *sc, u_char ae, bus_size_t offset,
 940         uint32_t value)
 941 {
 942         offset = __SHIFTIN(ae & sc->sc_ae_mask, AE_XFER_AE_MASK) |
 943             __SHIFTIN(offset, AE_XFER_CSR_MASK);
 944
 945         qat_misc_write_4(sc, sc->sc_hw.qhw_ae_offset + offset, value);
 946 }
 947
 948 static inline void
 949 qat_cap_global_write_4(struct qat_softc *sc, bus_size_t offset, uint32_t value)
 950 {
 951
 952         qat_misc_write_4(sc, sc->sc_hw.qhw_cap_global_offset + offset, value);
 953 }
 954
 955 static inline uint32_t
 956 qat_cap_global_read_4(struct qat_softc *sc, bus_size_t offset)
 957 {
 958
 959         return qat_misc_read_4(sc, sc->sc_hw.qhw_cap_global_offset + offset);
 960 }
 961
 962
 963 static inline void
 964 qat_etr_bank_write_4(struct qat_softc *sc, int bank,
 965         bus_size_t offset, uint32_t value)
 966 {
 967
 968         qat_etr_write_4(sc, sc->sc_hw.qhw_etr_bundle_size * bank + offset,
 969             value);
 970 }
 971
 972 static inline uint32_t
 973 qat_etr_bank_read_4(struct qat_softc *sc, int bank,
 974         bus_size_t offset)
 975 {
 976
 977         return qat_etr_read_4(sc,
 978             sc->sc_hw.qhw_etr_bundle_size * bank + offset);
 979 }
 980
 981 static inline void
 982 qat_etr_ap_bank_write_4(struct qat_softc *sc, int ap_bank,
 983         bus_size_t offset, uint32_t value)
 984 {
 985
 986         qat_etr_write_4(sc, ETR_AP_BANK_OFFSET * ap_bank + offset, value);
 987 }
 988
 989 static inline uint32_t
 990 qat_etr_ap_bank_read_4(struct qat_softc *sc, int ap_bank,
 991         bus_size_t offset)
 992 {
 993
 994         return qat_etr_read_4(sc, ETR_AP_BANK_OFFSET * ap_bank + offset);
 995 }
 996
 997
 998 static inline void
 999 qat_etr_bank_ring_write_4(struct qat_softc *sc, int bank, int ring,
1000         bus_size_t offset, uint32_t value)
1001 {
1002
1003         qat_etr_bank_write_4(sc, bank, (ring << 2) + offset, value);
1004 }
1005
1006 static inline uint32_t
1007 qat_etr_bank_ring_read_4(struct qat_softc *sc, int bank, int ring,
1008         bus_size_t offset)
1009 {
1010
1011         return qat_etr_bank_read_4(sc, bank, (ring << 2) * offset);
1012 }
1013
1014 static inline void
1015 qat_etr_bank_ring_base_write_8(struct qat_softc *sc, int bank, int ring,
1016         uint64_t value)
1017 {
1018         uint32_t lo, hi;
1019
1020         lo = (uint32_t)(value & 0xffffffff);
1021         hi = (uint32_t)((value & 0xffffffff00000000ULL) >> 32);
1022         qat_etr_bank_ring_write_4(sc, bank, ring, ETR_RING_LBASE, lo);
1023         qat_etr_bank_ring_write_4(sc, bank, ring, ETR_RING_UBASE, hi);
1024 }
1025
1026 static inline void
1027 qat_arb_ringsrvarben_write_4(struct qat_softc *sc, int index, uint32_t value)
1028 {
1029
1030         qat_etr_write_4(sc, ARB_RINGSRVARBEN_OFFSET +
1031             (ARB_REG_SLOT * index), value);
1032 }
1033
1034 static inline void
1035 qat_arb_sarconfig_write_4(struct qat_softc *sc, int index, uint32_t value)
1036 {
1037
1038         qat_etr_write_4(sc, ARB_OFFSET +
1039             (ARB_REG_SIZE * index), value);
1040 }
1041
1042 static inline void
1043 qat_arb_wrk_2_ser_map_write_4(struct qat_softc *sc, int index, uint32_t value)
1044 {
1045
1046         qat_etr_write_4(sc, ARB_OFFSET + ARB_WRK_2_SER_MAP_OFFSET +
1047             (ARB_REG_SIZE * index), value);
1048 }
1049
1050 void *          qat_alloc_mem(size_t);
1051 void            qat_free_mem(void *);
1052 void            qat_free_dmamem(struct qat_softc *, struct qat_dmamem *);
1053 int             qat_alloc_dmamem(struct qat_softc *, struct qat_dmamem *, int,
1054                     bus_size_t, bus_size_t);
1055
1056 int             qat_etr_setup_ring(struct qat_softc *, int, uint32_t, uint32_t,
1057                     uint32_t, qat_cb_t, void *, const char *,
1058                     struct qat_ring **);
1059 int             qat_etr_put_msg(struct qat_softc *, struct qat_ring *,
1060                     uint32_t *);
1061
1062 void            qat_memcpy_htobe64(void *, const void *, size_t);
1063 void            qat_memcpy_htobe32(void *, const void *, size_t);
1064 void            qat_memcpy_htobe(void *, const void *, size_t, uint32_t);
1065 void            qat_crypto_gmac_precompute(const struct qat_crypto_desc *,
1066                     const uint8_t *key, int klen,
1067                     const struct qat_sym_hash_def *, uint8_t *);
1068 void            qat_crypto_hmac_precompute(const struct qat_crypto_desc *,
1069                     const uint8_t *, int, const struct qat_sym_hash_def *,
1070                     uint8_t *, uint8_t *);
1071 uint16_t        qat_crypto_load_cipher_session(const struct qat_crypto_desc *,
1072                     const struct qat_session *);
1073 uint16_t        qat_crypto_load_auth_session(const struct qat_crypto_desc *,
1074                     const struct qat_session *,
1075                     struct qat_sym_hash_def const **);
1076
1077 #endif