sys/opencrypto/ktls_ocf.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause
   3  *
   4  * Copyright (c) 2019 Netflix Inc.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include <sys/param.h>
  32 #include <sys/systm.h>
  33 #include <sys/counter.h>
  34 #include <sys/endian.h>
  35 #include <sys/kernel.h>
  36 #include <sys/ktls.h>
  37 #include <sys/lock.h>
  38 #include <sys/malloc.h>
  39 #include <sys/mbuf.h>
  40 #include <sys/module.h>
  41 #include <sys/mutex.h>
  42 #include <sys/sysctl.h>
  43 #include <sys/uio.h>
  44 #include <vm/vm.h>
  45 #include <vm/pmap.h>
  46 #include <vm/vm_param.h>
  47 #include <netinet/in.h>
  48 #include <opencrypto/cryptodev.h>
  49 #include <opencrypto/ktls.h>
  50
  51 struct ktls_ocf_sw {
  52         /* Encrypt a single outbound TLS record. */
  53         int     (*encrypt)(struct ktls_ocf_encrypt_state *state,
  54             struct ktls_session *tls, struct mbuf *m,
  55             struct iovec *outiov, int outiovcnt);
  56
  57         /* Re-encrypt a received TLS record that is partially decrypted. */
  58         int     (*recrypt)(struct ktls_session *tls,
  59             const struct tls_record_layer *hdr, struct mbuf *m,
  60             uint64_t seqno);
  61
  62         /* Decrypt a received TLS record. */
  63         int     (*decrypt)(struct ktls_session *tls,
  64             const struct tls_record_layer *hdr, struct mbuf *m,
  65             uint64_t seqno, int *trailer_len);
  66 };
  67
  68 struct ktls_ocf_session {
  69         const struct ktls_ocf_sw *sw;
  70         crypto_session_t sid;
  71         crypto_session_t mac_sid;
  72         crypto_session_t recrypt_sid;
  73         struct mtx lock;
  74         int mac_len;
  75         bool implicit_iv;
  76
  77         /* Only used for TLS 1.0 with the implicit IV. */
  78 #ifdef INVARIANTS
  79         bool in_progress;
  80         uint64_t next_seqno;
  81 #endif
  82         char iv[AES_BLOCK_LEN];
  83 };
  84
  85 struct ocf_operation {
  86         struct ktls_ocf_session *os;
  87         bool done;
  88 };
  89
  90 static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS");
  91
  92 SYSCTL_DECL(_kern_ipc_tls);
  93 SYSCTL_DECL(_kern_ipc_tls_stats);
  94
  95 static SYSCTL_NODE(_kern_ipc_tls_stats, OID_AUTO, ocf,
  96     CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
  97     "Kernel TLS offload via OCF stats");
  98
  99 static COUNTER_U64_DEFINE_EARLY(ocf_tls10_cbc_encrypts);
 100 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls10_cbc_encrypts,
 101     CTLFLAG_RD, &ocf_tls10_cbc_encrypts,
 102     "Total number of OCF TLS 1.0 CBC encryption operations");
 103
 104 static COUNTER_U64_DEFINE_EARLY(ocf_tls11_cbc_encrypts);
 105 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls11_cbc_encrypts,
 106     CTLFLAG_RD, &ocf_tls11_cbc_encrypts,
 107     "Total number of OCF TLS 1.1/1.2 CBC encryption operations");
 108
 109 static COUNTER_U64_DEFINE_EARLY(ocf_tls12_gcm_decrypts);
 110 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_decrypts,
 111     CTLFLAG_RD, &ocf_tls12_gcm_decrypts,
 112     "Total number of OCF TLS 1.2 GCM decryption operations");
 113
 114 static COUNTER_U64_DEFINE_EARLY(ocf_tls12_gcm_encrypts);
 115 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_encrypts,
 116     CTLFLAG_RD, &ocf_tls12_gcm_encrypts,
 117     "Total number of OCF TLS 1.2 GCM encryption operations");
 118
 119 static COUNTER_U64_DEFINE_EARLY(ocf_tls12_gcm_recrypts);
 120 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_recrypts,
 121     CTLFLAG_RD, &ocf_tls12_gcm_recrypts,
 122     "Total number of OCF TLS 1.2 GCM re-encryption operations");
 123
 124 static COUNTER_U64_DEFINE_EARLY(ocf_tls12_chacha20_decrypts);
 125 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_chacha20_decrypts,
 126     CTLFLAG_RD, &ocf_tls12_chacha20_decrypts,
 127     "Total number of OCF TLS 1.2 Chacha20-Poly1305 decryption operations");
 128
 129 static COUNTER_U64_DEFINE_EARLY(ocf_tls12_chacha20_encrypts);
 130 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_chacha20_encrypts,
 131     CTLFLAG_RD, &ocf_tls12_chacha20_encrypts,
 132     "Total number of OCF TLS 1.2 Chacha20-Poly1305 encryption operations");
 133
 134 static COUNTER_U64_DEFINE_EARLY(ocf_tls13_gcm_decrypts);
 135 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_decrypts,
 136     CTLFLAG_RD, &ocf_tls13_gcm_decrypts,
 137     "Total number of OCF TLS 1.3 GCM decryption operations");
 138
 139 static COUNTER_U64_DEFINE_EARLY(ocf_tls13_gcm_encrypts);
 140 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_encrypts,
 141     CTLFLAG_RD, &ocf_tls13_gcm_encrypts,
 142     "Total number of OCF TLS 1.3 GCM encryption operations");
 143
 144 static COUNTER_U64_DEFINE_EARLY(ocf_tls13_gcm_recrypts);
 145 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_recrypts,
 146     CTLFLAG_RD, &ocf_tls13_gcm_recrypts,
 147     "Total number of OCF TLS 1.3 GCM re-encryption operations");
 148
 149 static COUNTER_U64_DEFINE_EARLY(ocf_tls13_chacha20_decrypts);
 150 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_chacha20_decrypts,
 151     CTLFLAG_RD, &ocf_tls13_chacha20_decrypts,
 152     "Total number of OCF TLS 1.3 Chacha20-Poly1305 decryption operations");
 153
 154 static COUNTER_U64_DEFINE_EARLY(ocf_tls13_chacha20_encrypts);
 155 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_chacha20_encrypts,
 156     CTLFLAG_RD, &ocf_tls13_chacha20_encrypts,
 157     "Total number of OCF TLS 1.3 Chacha20-Poly1305 encryption operations");
 158
 159 static COUNTER_U64_DEFINE_EARLY(ocf_inplace);
 160 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, inplace,
 161     CTLFLAG_RD, &ocf_inplace,
 162     "Total number of OCF in-place operations");
 163
 164 static COUNTER_U64_DEFINE_EARLY(ocf_separate_output);
 165 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, separate_output,
 166     CTLFLAG_RD, &ocf_separate_output,
 167     "Total number of OCF operations with a separate output buffer");
 168
 169 static COUNTER_U64_DEFINE_EARLY(ocf_retries);
 170 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, retries, CTLFLAG_RD,
 171     &ocf_retries,
 172     "Number of OCF encryption operation retries");
 173
 174 static int
 175 ktls_ocf_callback_sync(struct cryptop *crp __unused)
 176 {
 177         return (0);
 178 }
 179
 180 static int
 181 ktls_ocf_callback_async(struct cryptop *crp)
 182 {
 183         struct ocf_operation *oo;
 184
 185         oo = crp->crp_opaque;
 186         mtx_lock(&oo->os->lock);
 187         oo->done = true;
 188         mtx_unlock(&oo->os->lock);
 189         wakeup(oo);
 190         return (0);
 191 }
 192
 193 static int
 194 ktls_ocf_dispatch(struct ktls_ocf_session *os, struct cryptop *crp)
 195 {
 196         struct ocf_operation oo;
 197         int error;
 198         bool async;
 199
 200         oo.os = os;
 201         oo.done = false;
 202
 203         crp->crp_opaque = &oo;
 204         for (;;) {
 205                 async = !CRYPTO_SESS_SYNC(crp->crp_session);
 206                 crp->crp_callback = async ? ktls_ocf_callback_async :
 207                     ktls_ocf_callback_sync;
 208
 209                 error = crypto_dispatch(crp);
 210                 if (error)
 211                         break;
 212                 if (async) {
 213                         mtx_lock(&os->lock);
 214                         while (!oo.done)
 215                                 mtx_sleep(&oo, &os->lock, 0, "ocfktls", 0);
 216                         mtx_unlock(&os->lock);
 217                 }
 218
 219                 if (crp->crp_etype != EAGAIN) {
 220                         error = crp->crp_etype;
 221                         break;
 222                 }
 223
 224                 crp->crp_etype = 0;
 225                 crp->crp_flags &= ~CRYPTO_F_DONE;
 226                 oo.done = false;
 227                 counter_u64_add(ocf_retries, 1);
 228         }
 229         return (error);
 230 }
 231
 232 static int
 233 ktls_ocf_dispatch_async_cb(struct cryptop *crp)
 234 {
 235         struct ktls_ocf_encrypt_state *state;
 236         int error;
 237
 238         state = crp->crp_opaque;
 239         if (crp->crp_etype == EAGAIN) {
 240                 crp->crp_etype = 0;
 241                 crp->crp_flags &= ~CRYPTO_F_DONE;
 242                 counter_u64_add(ocf_retries, 1);
 243                 error = crypto_dispatch(crp);
 244                 if (error != 0) {
 245                         crypto_destroyreq(crp);
 246                         ktls_encrypt_cb(state, error);
 247                 }
 248                 return (0);
 249         }
 250
 251         error = crp->crp_etype;
 252         crypto_destroyreq(crp);
 253         ktls_encrypt_cb(state, error);
 254         return (0);
 255 }
 256
 257 static int
 258 ktls_ocf_dispatch_async(struct ktls_ocf_encrypt_state *state,
 259     struct cryptop *crp)
 260 {
 261         int error;
 262
 263         crp->crp_opaque = state;
 264         crp->crp_callback = ktls_ocf_dispatch_async_cb;
 265         error = crypto_dispatch(crp);
 266         if (error != 0)
 267                 crypto_destroyreq(crp);
 268         return (error);
 269 }
 270
 271 static int
 272 ktls_ocf_tls_cbc_encrypt(struct ktls_ocf_encrypt_state *state,
 273     struct ktls_session *tls, struct mbuf *m, struct iovec *outiov,
 274     int outiovcnt)
 275 {
 276         const struct tls_record_layer *hdr;
 277         struct uio *uio;
 278         struct tls_mac_data *ad;
 279         struct cryptop *crp;
 280         struct ktls_ocf_session *os;
 281         struct iovec iov[m->m_epg_npgs + 2];
 282         u_int pgoff;
 283         int i, error;
 284         uint16_t tls_comp_len;
 285         uint8_t pad;
 286
 287         MPASS(outiovcnt + 1 <= nitems(iov));
 288
 289         os = tls->ocf_session;
 290         hdr = (const struct tls_record_layer *)m->m_epg_hdr;
 291         crp = &state->crp;
 292         uio = &state->uio;
 293         MPASS(tls->sync_dispatch);
 294
 295 #ifdef INVARIANTS
 296         if (os->implicit_iv) {
 297                 mtx_lock(&os->lock);
 298                 KASSERT(!os->in_progress,
 299                     ("concurrent implicit IV encryptions"));
 300                 if (os->next_seqno != m->m_epg_seqno) {
 301                         printf("KTLS CBC: TLS records out of order.  "
 302                             "Expected %ju, got %ju\n",
 303                             (uintmax_t)os->next_seqno,
 304                             (uintmax_t)m->m_epg_seqno);
 305                         mtx_unlock(&os->lock);
 306                         return (EINVAL);
 307                 }
 308                 os->in_progress = true;
 309                 mtx_unlock(&os->lock);
 310         }
 311 #endif
 312
 313         /* Payload length. */
 314         tls_comp_len = m->m_len - (m->m_epg_hdrlen + m->m_epg_trllen);
 315
 316         /* Initialize the AAD. */
 317         ad = &state->mac;
 318         ad->seq = htobe64(m->m_epg_seqno);
 319         ad->type = hdr->tls_type;
 320         ad->tls_vmajor = hdr->tls_vmajor;
 321         ad->tls_vminor = hdr->tls_vminor;
 322         ad->tls_length = htons(tls_comp_len);
 323
 324         /* First, compute the MAC. */
 325         iov[0].iov_base = ad;
 326         iov[0].iov_len = sizeof(*ad);
 327         pgoff = m->m_epg_1st_off;
 328         for (i = 0; i < m->m_epg_npgs; i++, pgoff = 0) {
 329                 iov[i + 1].iov_base = (void *)PHYS_TO_DMAP(m->m_epg_pa[i] +
 330                     pgoff);
 331                 iov[i + 1].iov_len = m_epg_pagelen(m, i, pgoff);
 332         }
 333         iov[m->m_epg_npgs + 1].iov_base = m->m_epg_trail;
 334         iov[m->m_epg_npgs + 1].iov_len = os->mac_len;
 335         uio->uio_iov = iov;
 336         uio->uio_iovcnt = m->m_epg_npgs + 2;
 337         uio->uio_offset = 0;
 338         uio->uio_segflg = UIO_SYSSPACE;
 339         uio->uio_td = curthread;
 340         uio->uio_resid = sizeof(*ad) + tls_comp_len + os->mac_len;
 341
 342         crypto_initreq(crp, os->mac_sid);
 343         crp->crp_payload_start = 0;
 344         crp->crp_payload_length = sizeof(*ad) + tls_comp_len;
 345         crp->crp_digest_start = crp->crp_payload_length;
 346         crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST;
 347         crp->crp_flags = CRYPTO_F_CBIMM;
 348         crypto_use_uio(crp, uio);
 349         error = ktls_ocf_dispatch(os, crp);
 350
 351         crypto_destroyreq(crp);
 352         if (error) {
 353 #ifdef INVARIANTS
 354                 if (os->implicit_iv) {
 355                         mtx_lock(&os->lock);
 356                         os->in_progress = false;
 357                         mtx_unlock(&os->lock);
 358                 }
 359 #endif
 360                 return (error);
 361         }
 362
 363         /* Second, add the padding. */
 364         pad = m->m_epg_trllen - os->mac_len - 1;
 365         for (i = 0; i < pad + 1; i++)
 366                 m->m_epg_trail[os->mac_len + i] = pad;
 367
 368         /* Finally, encrypt the record. */
 369         crypto_initreq(crp, os->sid);
 370         crp->crp_payload_start = m->m_epg_hdrlen;
 371         crp->crp_payload_length = tls_comp_len + m->m_epg_trllen;
 372         KASSERT(crp->crp_payload_length % AES_BLOCK_LEN == 0,
 373             ("invalid encryption size"));
 374         crypto_use_single_mbuf(crp, m);
 375         crp->crp_op = CRYPTO_OP_ENCRYPT;
 376         crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
 377         if (os->implicit_iv)
 378                 memcpy(crp->crp_iv, os->iv, AES_BLOCK_LEN);
 379         else
 380                 memcpy(crp->crp_iv, hdr + 1, AES_BLOCK_LEN);
 381
 382         if (outiov != NULL) {
 383                 uio->uio_iov = outiov;
 384                 uio->uio_iovcnt = outiovcnt;
 385                 uio->uio_offset = 0;
 386                 uio->uio_segflg = UIO_SYSSPACE;
 387                 uio->uio_td = curthread;
 388                 uio->uio_resid = crp->crp_payload_length;
 389                 crypto_use_output_uio(crp, uio);
 390         }
 391
 392         if (os->implicit_iv)
 393                 counter_u64_add(ocf_tls10_cbc_encrypts, 1);
 394         else
 395                 counter_u64_add(ocf_tls11_cbc_encrypts, 1);
 396         if (outiov != NULL)
 397                 counter_u64_add(ocf_separate_output, 1);
 398         else
 399                 counter_u64_add(ocf_inplace, 1);
 400         error = ktls_ocf_dispatch(os, crp);
 401
 402         crypto_destroyreq(crp);
 403
 404         if (os->implicit_iv) {
 405                 KASSERT(os->mac_len + pad + 1 >= AES_BLOCK_LEN,
 406                     ("trailer too short to read IV"));
 407                 memcpy(os->iv, m->m_epg_trail + m->m_epg_trllen - AES_BLOCK_LEN,
 408                     AES_BLOCK_LEN);
 409 #ifdef INVARIANTS
 410                 mtx_lock(&os->lock);
 411                 os->next_seqno = m->m_epg_seqno + 1;
 412                 os->in_progress = false;
 413                 mtx_unlock(&os->lock);
 414 #endif
 415         }
 416         return (error);
 417 }
 418
 419 static const struct ktls_ocf_sw ktls_ocf_tls_cbc_sw = {
 420         .encrypt = ktls_ocf_tls_cbc_encrypt
 421 };
 422
 423 static int
 424 ktls_ocf_tls12_aead_encrypt(struct ktls_ocf_encrypt_state *state,
 425     struct ktls_session *tls, struct mbuf *m, struct iovec *outiov,
 426     int outiovcnt)
 427 {
 428         const struct tls_record_layer *hdr;
 429         struct uio *uio;
 430         struct tls_aead_data *ad;
 431         struct cryptop *crp;
 432         struct ktls_ocf_session *os;
 433         int error;
 434         uint16_t tls_comp_len;
 435
 436         os = tls->ocf_session;
 437         hdr = (const struct tls_record_layer *)m->m_epg_hdr;
 438         crp = &state->crp;
 439         uio = &state->uio;
 440
 441         crypto_initreq(crp, os->sid);
 442
 443         /* Setup the IV. */
 444         if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {
 445                 memcpy(crp->crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN);
 446                 memcpy(crp->crp_iv + TLS_AEAD_GCM_LEN, hdr + 1,
 447                     sizeof(uint64_t));
 448         } else {
 449                 /*
 450                  * Chacha20-Poly1305 constructs the IV for TLS 1.2
 451                  * identically to constructing the IV for AEAD in TLS
 452                  * 1.3.
 453                  */
 454                 memcpy(crp->crp_iv, tls->params.iv, tls->params.iv_len);
 455                 *(uint64_t *)(crp->crp_iv + 4) ^= htobe64(m->m_epg_seqno);
 456         }
 457
 458         /* Setup the AAD. */
 459         ad = &state->aead;
 460         tls_comp_len = m->m_len - (m->m_epg_hdrlen + m->m_epg_trllen);
 461         ad->seq = htobe64(m->m_epg_seqno);
 462         ad->type = hdr->tls_type;
 463         ad->tls_vmajor = hdr->tls_vmajor;
 464         ad->tls_vminor = hdr->tls_vminor;
 465         ad->tls_length = htons(tls_comp_len);
 466         crp->crp_aad = ad;
 467         crp->crp_aad_length = sizeof(*ad);
 468
 469         /* Set fields for input payload. */
 470         crypto_use_single_mbuf(crp, m);
 471         crp->crp_payload_start = m->m_epg_hdrlen;
 472         crp->crp_payload_length = tls_comp_len;
 473
 474         if (outiov != NULL) {
 475                 crp->crp_digest_start = crp->crp_payload_length;
 476
 477                 uio->uio_iov = outiov;
 478                 uio->uio_iovcnt = outiovcnt;
 479                 uio->uio_offset = 0;
 480                 uio->uio_segflg = UIO_SYSSPACE;
 481                 uio->uio_td = curthread;
 482                 uio->uio_resid = crp->crp_payload_length + tls->params.tls_tlen;
 483                 crypto_use_output_uio(crp, uio);
 484         } else
 485                 crp->crp_digest_start = crp->crp_payload_start +
 486                     crp->crp_payload_length;
 487
 488         crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST;
 489         crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
 490         if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
 491                 counter_u64_add(ocf_tls12_gcm_encrypts, 1);
 492         else
 493                 counter_u64_add(ocf_tls12_chacha20_encrypts, 1);
 494         if (outiov != NULL)
 495                 counter_u64_add(ocf_separate_output, 1);
 496         else
 497                 counter_u64_add(ocf_inplace, 1);
 498         if (tls->sync_dispatch) {
 499                 error = ktls_ocf_dispatch(os, crp);
 500                 crypto_destroyreq(crp);
 501         } else
 502                 error = ktls_ocf_dispatch_async(state, crp);
 503         return (error);
 504 }
 505
 506 static int
 507 ktls_ocf_tls12_aead_decrypt(struct ktls_session *tls,
 508     const struct tls_record_layer *hdr, struct mbuf *m, uint64_t seqno,
 509     int *trailer_len)
 510 {
 511         struct tls_aead_data ad;
 512         struct cryptop crp;
 513         struct ktls_ocf_session *os;
 514         int error;
 515         uint16_t tls_comp_len;
 516
 517         os = tls->ocf_session;
 518
 519         crypto_initreq(&crp, os->sid);
 520
 521         /* Setup the IV. */
 522         if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {
 523                 memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN);
 524                 memcpy(crp.crp_iv + TLS_AEAD_GCM_LEN, hdr + 1,
 525                     sizeof(uint64_t));
 526         } else {
 527                 /*
 528                  * Chacha20-Poly1305 constructs the IV for TLS 1.2
 529                  * identically to constructing the IV for AEAD in TLS
 530                  * 1.3.
 531                  */
 532                 memcpy(crp.crp_iv, tls->params.iv, tls->params.iv_len);
 533                 *(uint64_t *)(crp.crp_iv + 4) ^= htobe64(seqno);
 534         }
 535
 536         /* Setup the AAD. */
 537         if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
 538                 tls_comp_len = ntohs(hdr->tls_length) -
 539                     (AES_GMAC_HASH_LEN + sizeof(uint64_t));
 540         else
 541                 tls_comp_len = ntohs(hdr->tls_length) - POLY1305_HASH_LEN;
 542         ad.seq = htobe64(seqno);
 543         ad.type = hdr->tls_type;
 544         ad.tls_vmajor = hdr->tls_vmajor;
 545         ad.tls_vminor = hdr->tls_vminor;
 546         ad.tls_length = htons(tls_comp_len);
 547         crp.crp_aad = &ad;
 548         crp.crp_aad_length = sizeof(ad);
 549
 550         crp.crp_payload_start = tls->params.tls_hlen;
 551         crp.crp_payload_length = tls_comp_len;
 552         crp.crp_digest_start = crp.crp_payload_start + crp.crp_payload_length;
 553
 554         crp.crp_op = CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST;
 555         crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
 556         crypto_use_mbuf(&crp, m);
 557
 558         if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
 559                 counter_u64_add(ocf_tls12_gcm_decrypts, 1);
 560         else
 561                 counter_u64_add(ocf_tls12_chacha20_decrypts, 1);
 562         error = ktls_ocf_dispatch(os, &crp);
 563
 564         crypto_destroyreq(&crp);
 565         *trailer_len = tls->params.tls_tlen;
 566         return (error);
 567 }
 568
 569 /*
 570  * Reconstruct encrypted mbuf data in input buffer.
 571  */
 572 static void
 573 ktls_ocf_recrypt_fixup(struct mbuf *m, u_int skip, u_int len, char *buf)
 574 {
 575         const char *src = buf;
 576         u_int todo;
 577
 578         while (skip >= m->m_len) {
 579                 skip -= m->m_len;
 580                 m = m->m_next;
 581         }
 582
 583         while (len > 0) {
 584                 todo = m->m_len - skip;
 585                 if (todo > len)
 586                         todo = len;
 587
 588                 if (m->m_flags & M_DECRYPTED)
 589                         memcpy(mtod(m, char *) + skip, src, todo);
 590                 src += todo;
 591                 len -= todo;
 592                 skip = 0;
 593                 m = m->m_next;
 594         }
 595 }
 596
 597 static int
 598 ktls_ocf_tls12_aead_recrypt(struct ktls_session *tls,
 599     const struct tls_record_layer *hdr, struct mbuf *m,
 600     uint64_t seqno)
 601 {
 602         struct cryptop crp;
 603         struct ktls_ocf_session *os;
 604         char *buf;
 605         u_int payload_len;
 606         int error;
 607
 608         os = tls->ocf_session;
 609
 610         crypto_initreq(&crp, os->recrypt_sid);
 611
 612         KASSERT(tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16,
 613             ("%s: only AES-GCM is supported", __func__));
 614
 615         /* Setup the IV. */
 616         memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN);
 617         memcpy(crp.crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, sizeof(uint64_t));
 618         be32enc(crp.crp_iv + AES_GCM_IV_LEN, 2);
 619
 620         payload_len = ntohs(hdr->tls_length) -
 621             (AES_GMAC_HASH_LEN + sizeof(uint64_t));
 622         crp.crp_op = CRYPTO_OP_ENCRYPT;
 623         crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
 624         crypto_use_mbuf(&crp, m);
 625         crp.crp_payload_start = tls->params.tls_hlen;
 626         crp.crp_payload_length = payload_len;
 627
 628         buf = malloc(payload_len, M_KTLS_OCF, M_WAITOK);
 629         crypto_use_output_buf(&crp, buf, payload_len);
 630
 631         counter_u64_add(ocf_tls12_gcm_recrypts, 1);
 632         error = ktls_ocf_dispatch(os, &crp);
 633
 634         crypto_destroyreq(&crp);
 635
 636         if (error == 0)
 637                 ktls_ocf_recrypt_fixup(m, tls->params.tls_hlen, payload_len,
 638                     buf);
 639
 640         free(buf, M_KTLS_OCF);
 641         return (error);
 642 }
 643
 644 static const struct ktls_ocf_sw ktls_ocf_tls12_aead_sw = {
 645         .encrypt = ktls_ocf_tls12_aead_encrypt,
 646         .recrypt = ktls_ocf_tls12_aead_recrypt,
 647         .decrypt = ktls_ocf_tls12_aead_decrypt,
 648 };
 649
 650 static int
 651 ktls_ocf_tls13_aead_encrypt(struct ktls_ocf_encrypt_state *state,
 652     struct ktls_session *tls, struct mbuf *m, struct iovec *outiov,
 653     int outiovcnt)
 654 {
 655         const struct tls_record_layer *hdr;
 656         struct uio *uio;
 657         struct tls_aead_data_13 *ad;
 658         struct cryptop *crp;
 659         struct ktls_ocf_session *os;
 660         int error;
 661
 662         os = tls->ocf_session;
 663         hdr = (const struct tls_record_layer *)m->m_epg_hdr;
 664         crp = &state->crp;
 665         uio = &state->uio;
 666
 667         crypto_initreq(crp, os->sid);
 668
 669         /* Setup the nonce. */
 670         memcpy(crp->crp_iv, tls->params.iv, tls->params.iv_len);
 671         *(uint64_t *)(crp->crp_iv + 4) ^= htobe64(m->m_epg_seqno);
 672
 673         /* Setup the AAD. */
 674         ad = &state->aead13;
 675         ad->type = hdr->tls_type;
 676         ad->tls_vmajor = hdr->tls_vmajor;
 677         ad->tls_vminor = hdr->tls_vminor;
 678         ad->tls_length = hdr->tls_length;
 679         crp->crp_aad = ad;
 680         crp->crp_aad_length = sizeof(*ad);
 681
 682         /* Set fields for input payload. */
 683         crypto_use_single_mbuf(crp, m);
 684         crp->crp_payload_start = m->m_epg_hdrlen;
 685         crp->crp_payload_length = m->m_len -
 686             (m->m_epg_hdrlen + m->m_epg_trllen);
 687
 688         /* Store the record type as the first byte of the trailer. */
 689         m->m_epg_trail[0] = m->m_epg_record_type;
 690         crp->crp_payload_length++;
 691
 692         if (outiov != NULL) {
 693                 crp->crp_digest_start = crp->crp_payload_length;
 694
 695                 uio->uio_iov = outiov;
 696                 uio->uio_iovcnt = outiovcnt;
 697                 uio->uio_offset = 0;
 698                 uio->uio_segflg = UIO_SYSSPACE;
 699                 uio->uio_td = curthread;
 700                 uio->uio_resid = m->m_len - m->m_epg_hdrlen;
 701                 crypto_use_output_uio(crp, uio);
 702         } else
 703                 crp->crp_digest_start = crp->crp_payload_start +
 704                     crp->crp_payload_length;
 705
 706         crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST;
 707         crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
 708
 709         if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
 710                 counter_u64_add(ocf_tls13_gcm_encrypts, 1);
 711         else
 712                 counter_u64_add(ocf_tls13_chacha20_encrypts, 1);
 713         if (outiov != NULL)
 714                 counter_u64_add(ocf_separate_output, 1);
 715         else
 716                 counter_u64_add(ocf_inplace, 1);
 717         if (tls->sync_dispatch) {
 718                 error = ktls_ocf_dispatch(os, crp);
 719                 crypto_destroyreq(crp);
 720         } else
 721                 error = ktls_ocf_dispatch_async(state, crp);
 722         return (error);
 723 }
 724
 725 static int
 726 ktls_ocf_tls13_aead_decrypt(struct ktls_session *tls,
 727     const struct tls_record_layer *hdr, struct mbuf *m, uint64_t seqno,
 728     int *trailer_len)
 729 {
 730         struct tls_aead_data_13 ad;
 731         struct cryptop crp;
 732         struct ktls_ocf_session *os;
 733         int error;
 734         u_int tag_len;
 735
 736         os = tls->ocf_session;
 737
 738         tag_len = tls->params.tls_tlen - 1;
 739
 740         /* Payload must contain at least one byte for the record type. */
 741         if (ntohs(hdr->tls_length) < tag_len + 1)
 742                 return (EBADMSG);
 743
 744         crypto_initreq(&crp, os->sid);
 745
 746         /* Setup the nonce. */
 747         memcpy(crp.crp_iv, tls->params.iv, tls->params.iv_len);
 748         *(uint64_t *)(crp.crp_iv + 4) ^= htobe64(seqno);
 749
 750         /* Setup the AAD. */
 751         ad.type = hdr->tls_type;
 752         ad.tls_vmajor = hdr->tls_vmajor;
 753         ad.tls_vminor = hdr->tls_vminor;
 754         ad.tls_length = hdr->tls_length;
 755         crp.crp_aad = &ad;
 756         crp.crp_aad_length = sizeof(ad);
 757
 758         crp.crp_payload_start = tls->params.tls_hlen;
 759         crp.crp_payload_length = ntohs(hdr->tls_length) - tag_len;
 760         crp.crp_digest_start = crp.crp_payload_start + crp.crp_payload_length;
 761
 762         crp.crp_op = CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST;
 763         crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
 764         crypto_use_mbuf(&crp, m);
 765
 766         if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
 767                 counter_u64_add(ocf_tls13_gcm_decrypts, 1);
 768         else
 769                 counter_u64_add(ocf_tls13_chacha20_decrypts, 1);
 770         error = ktls_ocf_dispatch(os, &crp);
 771
 772         crypto_destroyreq(&crp);
 773         *trailer_len = tag_len;
 774         return (error);
 775 }
 776
 777 static int
 778 ktls_ocf_tls13_aead_recrypt(struct ktls_session *tls,
 779     const struct tls_record_layer *hdr, struct mbuf *m,
 780     uint64_t seqno)
 781 {
 782         struct cryptop crp;
 783         struct ktls_ocf_session *os;
 784         char *buf;
 785         u_int payload_len;
 786         int error;
 787
 788         os = tls->ocf_session;
 789
 790         crypto_initreq(&crp, os->recrypt_sid);
 791
 792         KASSERT(tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16,
 793             ("%s: only AES-GCM is supported", __func__));
 794
 795         /* Setup the IV. */
 796         memcpy(crp.crp_iv, tls->params.iv, tls->params.iv_len);
 797         *(uint64_t *)(crp.crp_iv + 4) ^= htobe64(seqno);
 798         be32enc(crp.crp_iv + 12, 2);
 799
 800         payload_len = ntohs(hdr->tls_length) - AES_GMAC_HASH_LEN;
 801         crp.crp_op = CRYPTO_OP_ENCRYPT;
 802         crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
 803         crypto_use_mbuf(&crp, m);
 804         crp.crp_payload_start = tls->params.tls_hlen;
 805         crp.crp_payload_length = payload_len;
 806
 807         buf = malloc(payload_len, M_KTLS_OCF, M_WAITOK);
 808         crypto_use_output_buf(&crp, buf, payload_len);
 809
 810         counter_u64_add(ocf_tls13_gcm_recrypts, 1);
 811         error = ktls_ocf_dispatch(os, &crp);
 812
 813         crypto_destroyreq(&crp);
 814
 815         if (error == 0)
 816                 ktls_ocf_recrypt_fixup(m, tls->params.tls_hlen, payload_len,
 817                     buf);
 818
 819         free(buf, M_KTLS_OCF);
 820         return (error);
 821 }
 822
 823 static const struct ktls_ocf_sw ktls_ocf_tls13_aead_sw = {
 824         .encrypt = ktls_ocf_tls13_aead_encrypt,
 825         .recrypt = ktls_ocf_tls13_aead_recrypt,
 826         .decrypt = ktls_ocf_tls13_aead_decrypt,
 827 };
 828
 829 void
 830 ktls_ocf_free(struct ktls_session *tls)
 831 {
 832         struct ktls_ocf_session *os;
 833
 834         os = tls->ocf_session;
 835         crypto_freesession(os->sid);
 836         crypto_freesession(os->mac_sid);
 837         crypto_freesession(os->recrypt_sid);
 838         mtx_destroy(&os->lock);
 839         zfree(os, M_KTLS_OCF);
 840 }
 841
 842 int
 843 ktls_ocf_try(struct socket *so, struct ktls_session *tls, int direction)
 844 {
 845         struct crypto_session_params csp, mac_csp, recrypt_csp;
 846         struct ktls_ocf_session *os;
 847         int error, mac_len;
 848
 849         memset(&csp, 0, sizeof(csp));
 850         memset(&mac_csp, 0, sizeof(mac_csp));
 851         mac_csp.csp_mode = CSP_MODE_NONE;
 852         mac_len = 0;
 853         memset(&recrypt_csp, 0, sizeof(mac_csp));
 854         recrypt_csp.csp_mode = CSP_MODE_NONE;
 855
 856         switch (tls->params.cipher_algorithm) {
 857         case CRYPTO_AES_NIST_GCM_16:
 858                 switch (tls->params.cipher_key_len) {
 859                 case 128 / 8:
 860                 case 256 / 8:
 861                         break;
 862                 default:
 863                         return (EINVAL);
 864                 }
 865
 866                 /* Only TLS 1.2 and 1.3 are supported. */
 867                 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||
 868                     tls->params.tls_vminor < TLS_MINOR_VER_TWO ||
 869                     tls->params.tls_vminor > TLS_MINOR_VER_THREE)
 870                         return (EPROTONOSUPPORT);
 871
 872                 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD;
 873                 csp.csp_mode = CSP_MODE_AEAD;
 874                 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16;
 875                 csp.csp_cipher_key = tls->params.cipher_key;
 876                 csp.csp_cipher_klen = tls->params.cipher_key_len;
 877                 csp.csp_ivlen = AES_GCM_IV_LEN;
 878
 879                 recrypt_csp.csp_flags |= CSP_F_SEPARATE_OUTPUT;
 880                 recrypt_csp.csp_mode = CSP_MODE_CIPHER;
 881                 recrypt_csp.csp_cipher_alg = CRYPTO_AES_ICM;
 882                 recrypt_csp.csp_cipher_key = tls->params.cipher_key;
 883                 recrypt_csp.csp_cipher_klen = tls->params.cipher_key_len;
 884                 recrypt_csp.csp_ivlen = AES_BLOCK_LEN;
 885                 break;
 886         case CRYPTO_AES_CBC:
 887                 switch (tls->params.cipher_key_len) {
 888                 case 128 / 8:
 889                 case 256 / 8:
 890                         break;
 891                 default:
 892                         return (EINVAL);
 893                 }
 894
 895                 switch (tls->params.auth_algorithm) {
 896                 case CRYPTO_SHA1_HMAC:
 897                         mac_len = SHA1_HASH_LEN;
 898                         break;
 899                 case CRYPTO_SHA2_256_HMAC:
 900                         mac_len = SHA2_256_HASH_LEN;
 901                         break;
 902                 case CRYPTO_SHA2_384_HMAC:
 903                         mac_len = SHA2_384_HASH_LEN;
 904                         break;
 905                 default:
 906                         return (EINVAL);
 907                 }
 908
 909                 /* Only TLS 1.0-1.2 are supported. */
 910                 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||
 911                     tls->params.tls_vminor < TLS_MINOR_VER_ZERO ||
 912                     tls->params.tls_vminor > TLS_MINOR_VER_TWO)
 913                         return (EPROTONOSUPPORT);
 914
 915                 /* AES-CBC is not supported for receive. */
 916                 if (direction == KTLS_RX)
 917                         return (EPROTONOSUPPORT);
 918
 919                 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT;
 920                 csp.csp_mode = CSP_MODE_CIPHER;
 921                 csp.csp_cipher_alg = CRYPTO_AES_CBC;
 922                 csp.csp_cipher_key = tls->params.cipher_key;
 923                 csp.csp_cipher_klen = tls->params.cipher_key_len;
 924                 csp.csp_ivlen = AES_BLOCK_LEN;
 925
 926                 mac_csp.csp_flags |= CSP_F_SEPARATE_OUTPUT;
 927                 mac_csp.csp_mode = CSP_MODE_DIGEST;
 928                 mac_csp.csp_auth_alg = tls->params.auth_algorithm;
 929                 mac_csp.csp_auth_key = tls->params.auth_key;
 930                 mac_csp.csp_auth_klen = tls->params.auth_key_len;
 931                 break;
 932         case CRYPTO_CHACHA20_POLY1305:
 933                 switch (tls->params.cipher_key_len) {
 934                 case 256 / 8:
 935                         break;
 936                 default:
 937                         return (EINVAL);
 938                 }
 939
 940                 /* Only TLS 1.2 and 1.3 are supported. */
 941                 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||
 942                     tls->params.tls_vminor < TLS_MINOR_VER_TWO ||
 943                     tls->params.tls_vminor > TLS_MINOR_VER_THREE)
 944                         return (EPROTONOSUPPORT);
 945
 946                 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD;
 947                 csp.csp_mode = CSP_MODE_AEAD;
 948                 csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305;
 949                 csp.csp_cipher_key = tls->params.cipher_key;
 950                 csp.csp_cipher_klen = tls->params.cipher_key_len;
 951                 csp.csp_ivlen = CHACHA20_POLY1305_IV_LEN;
 952                 break;
 953         default:
 954                 return (EPROTONOSUPPORT);
 955         }
 956
 957         os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT | M_ZERO);
 958         if (os == NULL)
 959                 return (ENOMEM);
 960
 961         error = crypto_newsession(&os->sid, &csp,
 962             CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE);
 963         if (error) {
 964                 free(os, M_KTLS_OCF);
 965                 return (error);
 966         }
 967
 968         if (mac_csp.csp_mode != CSP_MODE_NONE) {
 969                 error = crypto_newsession(&os->mac_sid, &mac_csp,
 970                     CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE);
 971                 if (error) {
 972                         crypto_freesession(os->sid);
 973                         free(os, M_KTLS_OCF);
 974                         return (error);
 975                 }
 976                 os->mac_len = mac_len;
 977         }
 978
 979         if (recrypt_csp.csp_mode != CSP_MODE_NONE) {
 980                 error = crypto_newsession(&os->recrypt_sid, &recrypt_csp,
 981                     CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE);
 982                 if (error) {
 983                         crypto_freesession(os->sid);
 984                         free(os, M_KTLS_OCF);
 985                         return (error);
 986                 }
 987         }
 988
 989         mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF);
 990         tls->ocf_session = os;
 991         if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16 ||
 992             tls->params.cipher_algorithm == CRYPTO_CHACHA20_POLY1305) {
 993                 if (tls->params.tls_vminor == TLS_MINOR_VER_THREE)
 994                         os->sw = &ktls_ocf_tls13_aead_sw;
 995                 else
 996                         os->sw = &ktls_ocf_tls12_aead_sw;
 997         } else {
 998                 os->sw = &ktls_ocf_tls_cbc_sw;
 999                 if (tls->params.tls_vminor == TLS_MINOR_VER_ZERO) {
1000                         os->implicit_iv = true;
1001                         memcpy(os->iv, tls->params.iv, AES_BLOCK_LEN);
1002 #ifdef INVARIANTS
1003                         os->next_seqno = tls->next_seqno;
1004 #endif
1005                 }
1006         }
1007
1008         /*
1009          * AES-CBC is always synchronous currently.  Asynchronous
1010          * operation would require multiple callbacks and an additional
1011          * iovec array in ktls_ocf_encrypt_state.
1012          */
1013         tls->sync_dispatch = CRYPTO_SESS_SYNC(os->sid) ||
1014             tls->params.cipher_algorithm == CRYPTO_AES_CBC;
1015         return (0);
1016 }
1017
1018 int
1019 ktls_ocf_encrypt(struct ktls_ocf_encrypt_state *state,
1020     struct ktls_session *tls, struct mbuf *m, struct iovec *outiov,
1021     int outiovcnt)
1022 {
1023         return (tls->ocf_session->sw->encrypt(state, tls, m, outiov,
1024             outiovcnt));
1025 }
1026
1027 int
1028 ktls_ocf_decrypt(struct ktls_session *tls, const struct tls_record_layer *hdr,
1029     struct mbuf *m, uint64_t seqno, int *trailer_len)
1030 {
1031         return (tls->ocf_session->sw->decrypt(tls, hdr, m, seqno, trailer_len));
1032 }
1033
1034 int
1035 ktls_ocf_recrypt(struct ktls_session *tls, const struct tls_record_layer *hdr,
1036     struct mbuf *m, uint64_t seqno)
1037 {
1038         return (tls->ocf_session->sw->recrypt(tls, hdr, m, seqno));
1039 }
1040
1041 bool
1042 ktls_ocf_recrypt_supported(struct ktls_session *tls)
1043 {
1044         return (tls->ocf_session->sw->recrypt != NULL &&
1045             tls->ocf_session->recrypt_sid != NULL);
1046 }