sys/opencrypto/crypto.c

   1 /*-
   2  * Copyright (c) 2002-2006 Sam Leffler.  All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  14  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  15  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  16  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  17  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  18  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  19  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  20  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  21  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  22  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  23  */
  24
  25 #include <sys/cdefs.h>
  26 __FBSDID("$FreeBSD$");
  27
  28 /*
  29  * Cryptographic Subsystem.
  30  *
  31  * This code is derived from the Openbsd Cryptographic Framework (OCF)
  32  * that has the copyright shown below.  Very little of the original
  33  * code remains.
  34  */
  35
  36 /*-
  37  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
  38  *
  39  * This code was written by Angelos D. Keromytis in Athens, Greece, in
  40  * February 2000. Network Security Technologies Inc. (NSTI) kindly
  41  * supported the development of this code.
  42  *
  43  * Copyright (c) 2000, 2001 Angelos D. Keromytis
  44  *
  45  * Permission to use, copy, and modify this software with or without fee
  46  * is hereby granted, provided that this entire notice is included in
  47  * all source code copies of any software which is or includes a copy or
  48  * modification of this software.
  49  *
  50  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
  51  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
  52  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
  53  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
  54  * PURPOSE.
  55  */
  56
  57 #define CRYPTO_TIMING                           /* enable timing support */
  58
  59 #include "opt_compat.h"
  60 #include "opt_ddb.h"
  61
  62 #include <sys/param.h>
  63 #include <sys/systm.h>
  64 #include <sys/eventhandler.h>
  65 #include <sys/kernel.h>
  66 #include <sys/kthread.h>
  67 #include <sys/linker.h>
  68 #include <sys/lock.h>
  69 #include <sys/module.h>
  70 #include <sys/mutex.h>
  71 #include <sys/malloc.h>
  72 #include <sys/mbuf.h>
  73 #include <sys/proc.h>
  74 #include <sys/refcount.h>
  75 #include <sys/sdt.h>
  76 #include <sys/smp.h>
  77 #include <sys/sysctl.h>
  78 #include <sys/taskqueue.h>
  79 #include <sys/uio.h>
  80
  81 #include <ddb/ddb.h>
  82
  83 #include <vm/uma.h>
  84 #include <crypto/intake.h>
  85 #include <opencrypto/cryptodev.h>
  86 #include <opencrypto/xform_auth.h>
  87 #include <opencrypto/xform_enc.h>
  88
  89 #include <sys/kobj.h>
  90 #include <sys/bus.h>
  91 #include "cryptodev_if.h"
  92
  93 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
  94 #include <machine/pcb.h>
  95 #endif
  96
  97 SDT_PROVIDER_DEFINE(opencrypto);
  98
  99 /*
 100  * Crypto drivers register themselves by allocating a slot in the
 101  * crypto_drivers table with crypto_get_driverid() and then registering
 102  * each asym algorithm they support with crypto_kregister().
 103  */
 104 static  struct mtx crypto_drivers_mtx;          /* lock on driver table */
 105 #define CRYPTO_DRIVER_LOCK()    mtx_lock(&crypto_drivers_mtx)
 106 #define CRYPTO_DRIVER_UNLOCK()  mtx_unlock(&crypto_drivers_mtx)
 107 #define CRYPTO_DRIVER_ASSERT()  mtx_assert(&crypto_drivers_mtx, MA_OWNED)
 108
 109 /*
 110  * Crypto device/driver capabilities structure.
 111  *
 112  * Synchronization:
 113  * (d) - protected by CRYPTO_DRIVER_LOCK()
 114  * (q) - protected by CRYPTO_Q_LOCK()
 115  * Not tagged fields are read-only.
 116  */
 117 struct cryptocap {
 118         device_t        cc_dev;
 119         uint32_t        cc_hid;
 120         u_int32_t       cc_sessions;            /* (d) # of sessions */
 121         u_int32_t       cc_koperations;         /* (d) # os asym operations */
 122         u_int8_t        cc_kalg[CRK_ALGORITHM_MAX + 1];
 123
 124         int             cc_flags;               /* (d) flags */
 125 #define CRYPTOCAP_F_CLEANUP     0x80000000      /* needs resource cleanup */
 126         int             cc_qblocked;            /* (q) symmetric q blocked */
 127         int             cc_kqblocked;           /* (q) asymmetric q blocked */
 128         size_t          cc_session_size;
 129         volatile int    cc_refs;
 130 };
 131
 132 static  struct cryptocap **crypto_drivers = NULL;
 133 static  int crypto_drivers_size = 0;
 134
 135 struct crypto_session {
 136         struct cryptocap *cap;
 137         void *softc;
 138         struct crypto_session_params csp;
 139 };
 140
 141 /*
 142  * There are two queues for crypto requests; one for symmetric (e.g.
 143  * cipher) operations and one for asymmetric (e.g. MOD)operations.
 144  * A single mutex is used to lock access to both queues.  We could
 145  * have one per-queue but having one simplifies handling of block/unblock
 146  * operations.
 147  */
 148 static  int crp_sleep = 0;
 149 static  TAILQ_HEAD(cryptop_q ,cryptop) crp_q;           /* request queues */
 150 static  TAILQ_HEAD(,cryptkop) crp_kq;
 151 static  struct mtx crypto_q_mtx;
 152 #define CRYPTO_Q_LOCK()         mtx_lock(&crypto_q_mtx)
 153 #define CRYPTO_Q_UNLOCK()       mtx_unlock(&crypto_q_mtx)
 154
 155 SYSCTL_NODE(_kern, OID_AUTO, crypto, CTLFLAG_RW, 0,
 156     "In-kernel cryptography");
 157
 158 /*
 159  * Taskqueue used to dispatch the crypto requests
 160  * that have the CRYPTO_F_ASYNC flag
 161  */
 162 static struct taskqueue *crypto_tq;
 163
 164 /*
 165  * Crypto seq numbers are operated on with modular arithmetic
 166  */
 167 #define CRYPTO_SEQ_GT(a,b)      ((int)((a)-(b)) > 0)
 168
 169 struct crypto_ret_worker {
 170         struct mtx crypto_ret_mtx;
 171
 172         TAILQ_HEAD(,cryptop) crp_ordered_ret_q; /* ordered callback queue for symetric jobs */
 173         TAILQ_HEAD(,cryptop) crp_ret_q;         /* callback queue for symetric jobs */
 174         TAILQ_HEAD(,cryptkop) crp_ret_kq;       /* callback queue for asym jobs */
 175
 176         u_int32_t reorder_ops;          /* total ordered sym jobs received */
 177         u_int32_t reorder_cur_seq;      /* current sym job dispatched */
 178
 179         struct proc *cryptoretproc;
 180 };
 181 static struct crypto_ret_worker *crypto_ret_workers = NULL;
 182
 183 #define CRYPTO_RETW(i)          (&crypto_ret_workers[i])
 184 #define CRYPTO_RETW_ID(w)       ((w) - crypto_ret_workers)
 185 #define FOREACH_CRYPTO_RETW(w) \
 186         for (w = crypto_ret_workers; w < crypto_ret_workers + crypto_workers_num; ++w)
 187
 188 #define CRYPTO_RETW_LOCK(w)     mtx_lock(&w->crypto_ret_mtx)
 189 #define CRYPTO_RETW_UNLOCK(w)   mtx_unlock(&w->crypto_ret_mtx)
 190 #define CRYPTO_RETW_EMPTY(w) \
 191         (TAILQ_EMPTY(&w->crp_ret_q) && TAILQ_EMPTY(&w->crp_ret_kq) && TAILQ_EMPTY(&w->crp_ordered_ret_q))
 192
 193 static int crypto_workers_num = 0;
 194 SYSCTL_INT(_kern_crypto, OID_AUTO, num_workers, CTLFLAG_RDTUN,
 195            &crypto_workers_num, 0,
 196            "Number of crypto workers used to dispatch crypto jobs");
 197 #ifdef COMPAT_FREEBSD12
 198 SYSCTL_INT(_kern, OID_AUTO, crypto_workers_num, CTLFLAG_RDTUN,
 199            &crypto_workers_num, 0,
 200            "Number of crypto workers used to dispatch crypto jobs");
 201 #endif
 202
 203 static  uma_zone_t cryptop_zone;
 204 static  uma_zone_t cryptoses_zone;
 205
 206 int     crypto_userasymcrypto = 1;
 207 SYSCTL_INT(_kern_crypto, OID_AUTO, asym_enable, CTLFLAG_RW,
 208            &crypto_userasymcrypto, 0,
 209            "Enable user-mode access to asymmetric crypto support");
 210 #ifdef COMPAT_FREEBSD12
 211 SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW,
 212            &crypto_userasymcrypto, 0,
 213            "Enable/disable user-mode access to asymmetric crypto support");
 214 #endif
 215
 216 int     crypto_devallowsoft = 0;
 217 SYSCTL_INT(_kern_crypto, OID_AUTO, allow_soft, CTLFLAG_RW,
 218            &crypto_devallowsoft, 0,
 219            "Enable use of software crypto by /dev/crypto");
 220 #ifdef COMPAT_FREEBSD12
 221 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW,
 222            &crypto_devallowsoft, 0,
 223            "Enable/disable use of software crypto by /dev/crypto");
 224 #endif
 225
 226 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
 227
 228 static  void crypto_proc(void);
 229 static  struct proc *cryptoproc;
 230 static  void crypto_ret_proc(struct crypto_ret_worker *ret_worker);
 231 static  void crypto_destroy(void);
 232 static  int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint);
 233 static  int crypto_kinvoke(struct cryptkop *krp);
 234 static  void crypto_task_invoke(void *ctx, int pending);
 235 static void crypto_batch_enqueue(struct cryptop *crp);
 236
 237 static  struct cryptostats cryptostats;
 238 SYSCTL_STRUCT(_kern_crypto, OID_AUTO, stats, CTLFLAG_RW, &cryptostats,
 239             cryptostats, "Crypto system statistics");
 240
 241 #ifdef CRYPTO_TIMING
 242 static  int crypto_timing = 0;
 243 SYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW,
 244            &crypto_timing, 0, "Enable/disable crypto timing support");
 245 #endif
 246
 247 /* Try to avoid directly exposing the key buffer as a symbol */
 248 static struct keybuf *keybuf;
 249
 250 static struct keybuf empty_keybuf = {
 251         .kb_nents = 0
 252 };
 253
 254 /* Obtain the key buffer from boot metadata */
 255 static void
 256 keybuf_init(void)
 257 {
 258         caddr_t kmdp;
 259
 260         kmdp = preload_search_by_type("elf kernel");
 261
 262         if (kmdp == NULL)
 263                 kmdp = preload_search_by_type("elf64 kernel");
 264
 265         keybuf = (struct keybuf *)preload_search_info(kmdp,
 266             MODINFO_METADATA | MODINFOMD_KEYBUF);
 267
 268         if (keybuf == NULL)
 269                 keybuf = &empty_keybuf;
 270 }
 271
 272 /* It'd be nice if we could store these in some kind of secure memory... */
 273 struct keybuf * get_keybuf(void) {
 274
 275         return (keybuf);
 276 }
 277
 278 static struct cryptocap *
 279 cap_ref(struct cryptocap *cap)
 280 {
 281
 282         refcount_acquire(&cap->cc_refs);
 283         return (cap);
 284 }
 285
 286 static void
 287 cap_rele(struct cryptocap *cap)
 288 {
 289
 290         if (refcount_release(&cap->cc_refs) == 0)
 291                 return;
 292
 293         KASSERT(cap->cc_sessions == 0,
 294             ("freeing crypto driver with active sessions"));
 295         KASSERT(cap->cc_koperations == 0,
 296             ("freeing crypto driver with active key operations"));
 297
 298         free(cap, M_CRYPTO_DATA);
 299 }
 300
 301 static int
 302 crypto_init(void)
 303 {
 304         struct crypto_ret_worker *ret_worker;
 305         int error;
 306
 307         mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table",
 308                 MTX_DEF|MTX_QUIET);
 309
 310         TAILQ_INIT(&crp_q);
 311         TAILQ_INIT(&crp_kq);
 312         mtx_init(&crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF);
 313
 314         cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop),
 315                                     0, 0, 0, 0,
 316                                     UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
 317         cryptoses_zone = uma_zcreate("crypto_session",
 318             sizeof(struct crypto_session), NULL, NULL, NULL, NULL,
 319             UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
 320
 321         if (cryptop_zone == NULL || cryptoses_zone == NULL) {
 322                 printf("crypto_init: cannot setup crypto zones\n");
 323                 error = ENOMEM;
 324                 goto bad;
 325         }
 326
 327         crypto_drivers_size = CRYPTO_DRIVERS_INITIAL;
 328         crypto_drivers = malloc(crypto_drivers_size *
 329             sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO);
 330         if (crypto_drivers == NULL) {
 331                 printf("crypto_init: cannot setup crypto drivers\n");
 332                 error = ENOMEM;
 333                 goto bad;
 334         }
 335
 336         if (crypto_workers_num < 1 || crypto_workers_num > mp_ncpus)
 337                 crypto_workers_num = mp_ncpus;
 338
 339         crypto_tq = taskqueue_create("crypto", M_WAITOK|M_ZERO,
 340                                 taskqueue_thread_enqueue, &crypto_tq);
 341         if (crypto_tq == NULL) {
 342                 printf("crypto init: cannot setup crypto taskqueue\n");
 343                 error = ENOMEM;
 344                 goto bad;
 345         }
 346
 347         taskqueue_start_threads(&crypto_tq, crypto_workers_num, PRI_MIN_KERN,
 348                 "crypto");
 349
 350         error = kproc_create((void (*)(void *)) crypto_proc, NULL,
 351                     &cryptoproc, 0, 0, "crypto");
 352         if (error) {
 353                 printf("crypto_init: cannot start crypto thread; error %d",
 354                         error);
 355                 goto bad;
 356         }
 357
 358         crypto_ret_workers = malloc(crypto_workers_num * sizeof(struct crypto_ret_worker),
 359                         M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
 360         if (crypto_ret_workers == NULL) {
 361                 error = ENOMEM;
 362                 printf("crypto_init: cannot allocate ret workers\n");
 363                 goto bad;
 364         }
 365
 366
 367         FOREACH_CRYPTO_RETW(ret_worker) {
 368                 TAILQ_INIT(&ret_worker->crp_ordered_ret_q);
 369                 TAILQ_INIT(&ret_worker->crp_ret_q);
 370                 TAILQ_INIT(&ret_worker->crp_ret_kq);
 371
 372                 ret_worker->reorder_ops = 0;
 373                 ret_worker->reorder_cur_seq = 0;
 374
 375                 mtx_init(&ret_worker->crypto_ret_mtx, "crypto", "crypto return queues", MTX_DEF);
 376
 377                 error = kproc_create((void (*)(void *)) crypto_ret_proc, ret_worker,
 378                                 &ret_worker->cryptoretproc, 0, 0, "crypto returns %td", CRYPTO_RETW_ID(ret_worker));
 379                 if (error) {
 380                         printf("crypto_init: cannot start cryptoret thread; error %d",
 381                                 error);
 382                         goto bad;
 383                 }
 384         }
 385
 386         keybuf_init();
 387
 388         return 0;
 389 bad:
 390         crypto_destroy();
 391         return error;
 392 }
 393
 394 /*
 395  * Signal a crypto thread to terminate.  We use the driver
 396  * table lock to synchronize the sleep/wakeups so that we
 397  * are sure the threads have terminated before we release
 398  * the data structures they use.  See crypto_finis below
 399  * for the other half of this song-and-dance.
 400  */
 401 static void
 402 crypto_terminate(struct proc **pp, void *q)
 403 {
 404         struct proc *p;
 405
 406         mtx_assert(&crypto_drivers_mtx, MA_OWNED);
 407         p = *pp;
 408         *pp = NULL;
 409         if (p) {
 410                 wakeup_one(q);
 411                 PROC_LOCK(p);           /* NB: insure we don't miss wakeup */
 412                 CRYPTO_DRIVER_UNLOCK(); /* let crypto_finis progress */
 413                 msleep(p, &p->p_mtx, PWAIT, "crypto_destroy", 0);
 414                 PROC_UNLOCK(p);
 415                 CRYPTO_DRIVER_LOCK();
 416         }
 417 }
 418
 419 static void
 420 hmac_init_pad(struct auth_hash *axf, const char *key, int klen, void *auth_ctx,
 421     uint8_t padval)
 422 {
 423         uint8_t hmac_key[HMAC_MAX_BLOCK_LEN];
 424         u_int i;
 425
 426         KASSERT(axf->blocksize <= sizeof(hmac_key),
 427             ("Invalid HMAC block size %d", axf->blocksize));
 428
 429         /*
 430          * If the key is larger than the block size, use the digest of
 431          * the key as the key instead.
 432          */
 433         memset(hmac_key, 0, sizeof(hmac_key));
 434         if (klen > axf->blocksize) {
 435                 axf->Init(auth_ctx);
 436                 axf->Update(auth_ctx, key, klen);
 437                 axf->Final(hmac_key, auth_ctx);
 438                 klen = axf->hashsize;
 439         } else
 440                 memcpy(hmac_key, key, klen);
 441
 442         for (i = 0; i < axf->blocksize; i++)
 443                 hmac_key[i] ^= padval;
 444
 445         axf->Init(auth_ctx);
 446         axf->Update(auth_ctx, hmac_key, axf->blocksize);
 447 }
 448
 449 void
 450 hmac_init_ipad(struct auth_hash *axf, const char *key, int klen,
 451     void *auth_ctx)
 452 {
 453
 454         hmac_init_pad(axf, key, klen, auth_ctx, HMAC_IPAD_VAL);
 455 }
 456
 457 void
 458 hmac_init_opad(struct auth_hash *axf, const char *key, int klen,
 459     void *auth_ctx)
 460 {
 461
 462         hmac_init_pad(axf, key, klen, auth_ctx, HMAC_OPAD_VAL);
 463 }
 464
 465 static void
 466 crypto_destroy(void)
 467 {
 468         struct crypto_ret_worker *ret_worker;
 469         int i;
 470
 471         /*
 472          * Terminate any crypto threads.
 473          */
 474         if (crypto_tq != NULL)
 475                 taskqueue_drain_all(crypto_tq);
 476         CRYPTO_DRIVER_LOCK();
 477         crypto_terminate(&cryptoproc, &crp_q);
 478         FOREACH_CRYPTO_RETW(ret_worker)
 479                 crypto_terminate(&ret_worker->cryptoretproc, &ret_worker->crp_ret_q);
 480         CRYPTO_DRIVER_UNLOCK();
 481
 482         /* XXX flush queues??? */
 483
 484         /*
 485          * Reclaim dynamically allocated resources.
 486          */
 487         for (i = 0; i < crypto_drivers_size; i++) {
 488                 if (crypto_drivers[i] != NULL)
 489                         cap_rele(crypto_drivers[i]);
 490         }
 491         free(crypto_drivers, M_CRYPTO_DATA);
 492
 493         if (cryptoses_zone != NULL)
 494                 uma_zdestroy(cryptoses_zone);
 495         if (cryptop_zone != NULL)
 496                 uma_zdestroy(cryptop_zone);
 497         mtx_destroy(&crypto_q_mtx);
 498         FOREACH_CRYPTO_RETW(ret_worker)
 499                 mtx_destroy(&ret_worker->crypto_ret_mtx);
 500         free(crypto_ret_workers, M_CRYPTO_DATA);
 501         if (crypto_tq != NULL)
 502                 taskqueue_free(crypto_tq);
 503         mtx_destroy(&crypto_drivers_mtx);
 504 }
 505
 506 uint32_t
 507 crypto_ses2hid(crypto_session_t crypto_session)
 508 {
 509         return (crypto_session->cap->cc_hid);
 510 }
 511
 512 uint32_t
 513 crypto_ses2caps(crypto_session_t crypto_session)
 514 {
 515         return (crypto_session->cap->cc_flags & 0xff000000);
 516 }
 517
 518 void *
 519 crypto_get_driver_session(crypto_session_t crypto_session)
 520 {
 521         return (crypto_session->softc);
 522 }
 523
 524 const struct crypto_session_params *
 525 crypto_get_params(crypto_session_t crypto_session)
 526 {
 527         return (&crypto_session->csp);
 528 }
 529
 530 struct auth_hash *
 531 crypto_auth_hash(const struct crypto_session_params *csp)
 532 {
 533
 534         switch (csp->csp_auth_alg) {
 535         case CRYPTO_SHA1_HMAC:
 536                 return (&auth_hash_hmac_sha1);
 537         case CRYPTO_SHA2_224_HMAC:
 538                 return (&auth_hash_hmac_sha2_224);
 539         case CRYPTO_SHA2_256_HMAC:
 540                 return (&auth_hash_hmac_sha2_256);
 541         case CRYPTO_SHA2_384_HMAC:
 542                 return (&auth_hash_hmac_sha2_384);
 543         case CRYPTO_SHA2_512_HMAC:
 544                 return (&auth_hash_hmac_sha2_512);
 545         case CRYPTO_NULL_HMAC:
 546                 return (&auth_hash_null);
 547         case CRYPTO_RIPEMD160_HMAC:
 548                 return (&auth_hash_hmac_ripemd_160);
 549         case CRYPTO_SHA1:
 550                 return (&auth_hash_sha1);
 551         case CRYPTO_SHA2_224:
 552                 return (&auth_hash_sha2_224);
 553         case CRYPTO_SHA2_256:
 554                 return (&auth_hash_sha2_256);
 555         case CRYPTO_SHA2_384:
 556                 return (&auth_hash_sha2_384);
 557         case CRYPTO_SHA2_512:
 558                 return (&auth_hash_sha2_512);
 559         case CRYPTO_AES_NIST_GMAC:
 560                 switch (csp->csp_auth_klen) {
 561                 case 128 / 8:
 562                         return (&auth_hash_nist_gmac_aes_128);
 563                 case 192 / 8:
 564                         return (&auth_hash_nist_gmac_aes_192);
 565                 case 256 / 8:
 566                         return (&auth_hash_nist_gmac_aes_256);
 567                 default:
 568                         return (NULL);
 569                 }
 570         case CRYPTO_BLAKE2B:
 571                 return (&auth_hash_blake2b);
 572         case CRYPTO_BLAKE2S:
 573                 return (&auth_hash_blake2s);
 574         case CRYPTO_POLY1305:
 575                 return (&auth_hash_poly1305);
 576         case CRYPTO_AES_CCM_CBC_MAC:
 577                 switch (csp->csp_auth_klen) {
 578                 case 128 / 8:
 579                         return (&auth_hash_ccm_cbc_mac_128);
 580                 case 192 / 8:
 581                         return (&auth_hash_ccm_cbc_mac_192);
 582                 case 256 / 8:
 583                         return (&auth_hash_ccm_cbc_mac_256);
 584                 default:
 585                         return (NULL);
 586                 }
 587         default:
 588                 return (NULL);
 589         }
 590 }
 591
 592 struct enc_xform *
 593 crypto_cipher(const struct crypto_session_params *csp)
 594 {
 595
 596         switch (csp->csp_cipher_alg) {
 597         case CRYPTO_RIJNDAEL128_CBC:
 598                 return (&enc_xform_rijndael128);
 599         case CRYPTO_AES_XTS:
 600                 return (&enc_xform_aes_xts);
 601         case CRYPTO_AES_ICM:
 602                 return (&enc_xform_aes_icm);
 603         case CRYPTO_AES_NIST_GCM_16:
 604                 return (&enc_xform_aes_nist_gcm);
 605         case CRYPTO_CAMELLIA_CBC:
 606                 return (&enc_xform_camellia);
 607         case CRYPTO_NULL_CBC:
 608                 return (&enc_xform_null);
 609         case CRYPTO_CHACHA20:
 610                 return (&enc_xform_chacha20);
 611         case CRYPTO_AES_CCM_16:
 612                 return (&enc_xform_ccm);
 613         default:
 614                 return (NULL);
 615         }
 616 }
 617
 618 static struct cryptocap *
 619 crypto_checkdriver(u_int32_t hid)
 620 {
 621
 622         return (hid >= crypto_drivers_size ? NULL : crypto_drivers[hid]);
 623 }
 624
 625 /*
 626  * Select a driver for a new session that supports the specified
 627  * algorithms and, optionally, is constrained according to the flags.
 628  */
 629 static struct cryptocap *
 630 crypto_select_driver(const struct crypto_session_params *csp, int flags)
 631 {
 632         struct cryptocap *cap, *best;
 633         int best_match, error, hid;
 634
 635         CRYPTO_DRIVER_ASSERT();
 636
 637         best = NULL;
 638         for (hid = 0; hid < crypto_drivers_size; hid++) {
 639                 /*
 640                  * If there is no driver for this slot, or the driver
 641                  * is not appropriate (hardware or software based on
 642                  * match), then skip.
 643                  */
 644                 cap = crypto_drivers[hid];
 645                 if (cap == NULL ||
 646                     (cap->cc_flags & flags) == 0)
 647                         continue;
 648
 649                 error = CRYPTODEV_PROBESESSION(cap->cc_dev, csp);
 650                 if (error >= 0)
 651                         continue;
 652
 653                 /*
 654                  * Use the driver with the highest probe value.
 655                  * Hardware drivers use a higher probe value than
 656                  * software.  In case of a tie, prefer the driver with
 657                  * the fewest active sessions.
 658                  */
 659                 if (best == NULL || error > best_match ||
 660                     (error == best_match &&
 661                     cap->cc_sessions < best->cc_sessions)) {
 662                         best = cap;
 663                         best_match = error;
 664                 }
 665         }
 666         return best;
 667 }
 668
 669 static enum alg_type {
 670         ALG_NONE = 0,
 671         ALG_CIPHER,
 672         ALG_DIGEST,
 673         ALG_KEYED_DIGEST,
 674         ALG_COMPRESSION,
 675         ALG_AEAD
 676 } alg_types[] = {
 677         [CRYPTO_SHA1_HMAC] = ALG_KEYED_DIGEST,
 678         [CRYPTO_RIPEMD160_HMAC] = ALG_KEYED_DIGEST,
 679         [CRYPTO_AES_CBC] = ALG_CIPHER,
 680         [CRYPTO_SHA1] = ALG_DIGEST,
 681         [CRYPTO_NULL_HMAC] = ALG_DIGEST,
 682         [CRYPTO_NULL_CBC] = ALG_CIPHER,
 683         [CRYPTO_DEFLATE_COMP] = ALG_COMPRESSION,
 684         [CRYPTO_SHA2_256_HMAC] = ALG_KEYED_DIGEST,
 685         [CRYPTO_SHA2_384_HMAC] = ALG_KEYED_DIGEST,
 686         [CRYPTO_SHA2_512_HMAC] = ALG_KEYED_DIGEST,
 687         [CRYPTO_CAMELLIA_CBC] = ALG_CIPHER,
 688         [CRYPTO_AES_XTS] = ALG_CIPHER,
 689         [CRYPTO_AES_ICM] = ALG_CIPHER,
 690         [CRYPTO_AES_NIST_GMAC] = ALG_KEYED_DIGEST,
 691         [CRYPTO_AES_NIST_GCM_16] = ALG_AEAD,
 692         [CRYPTO_BLAKE2B] = ALG_KEYED_DIGEST,
 693         [CRYPTO_BLAKE2S] = ALG_KEYED_DIGEST,
 694         [CRYPTO_CHACHA20] = ALG_CIPHER,
 695         [CRYPTO_SHA2_224_HMAC] = ALG_KEYED_DIGEST,
 696         [CRYPTO_RIPEMD160] = ALG_DIGEST,
 697         [CRYPTO_SHA2_224] = ALG_DIGEST,
 698         [CRYPTO_SHA2_256] = ALG_DIGEST,
 699         [CRYPTO_SHA2_384] = ALG_DIGEST,
 700         [CRYPTO_SHA2_512] = ALG_DIGEST,
 701         [CRYPTO_POLY1305] = ALG_KEYED_DIGEST,
 702         [CRYPTO_AES_CCM_CBC_MAC] = ALG_KEYED_DIGEST,
 703         [CRYPTO_AES_CCM_16] = ALG_AEAD,
 704 };
 705
 706 static enum alg_type
 707 alg_type(int alg)
 708 {
 709
 710         if (alg < nitems(alg_types))
 711                 return (alg_types[alg]);
 712         return (ALG_NONE);
 713 }
 714
 715 static bool
 716 alg_is_compression(int alg)
 717 {
 718
 719         return (alg_type(alg) == ALG_COMPRESSION);
 720 }
 721
 722 static bool
 723 alg_is_cipher(int alg)
 724 {
 725
 726         return (alg_type(alg) == ALG_CIPHER);
 727 }
 728
 729 static bool
 730 alg_is_digest(int alg)
 731 {
 732
 733         return (alg_type(alg) == ALG_DIGEST ||
 734             alg_type(alg) == ALG_KEYED_DIGEST);
 735 }
 736
 737 static bool
 738 alg_is_keyed_digest(int alg)
 739 {
 740
 741         return (alg_type(alg) == ALG_KEYED_DIGEST);
 742 }
 743
 744 static bool
 745 alg_is_aead(int alg)
 746 {
 747
 748         return (alg_type(alg) == ALG_AEAD);
 749 }
 750
 751 /* Various sanity checks on crypto session parameters. */
 752 static bool
 753 check_csp(const struct crypto_session_params *csp)
 754 {
 755         struct auth_hash *axf;
 756
 757         /* Mode-independent checks. */
 758         if ((csp->csp_flags & ~CSP_F_SEPARATE_OUTPUT) != 0)
 759                 return (false);
 760         if (csp->csp_ivlen < 0 || csp->csp_cipher_klen < 0 ||
 761             csp->csp_auth_klen < 0 || csp->csp_auth_mlen < 0)
 762                 return (false);
 763         if (csp->csp_auth_key != NULL && csp->csp_auth_klen == 0)
 764                 return (false);
 765         if (csp->csp_cipher_key != NULL && csp->csp_cipher_klen == 0)
 766                 return (false);
 767
 768         switch (csp->csp_mode) {
 769         case CSP_MODE_COMPRESS:
 770                 if (!alg_is_compression(csp->csp_cipher_alg))
 771                         return (false);
 772                 if (csp->csp_flags & CSP_F_SEPARATE_OUTPUT)
 773                         return (false);
 774                 if (csp->csp_cipher_klen != 0 || csp->csp_ivlen != 0 ||
 775                     csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0 ||
 776                     csp->csp_auth_mlen != 0)
 777                         return (false);
 778                 break;
 779         case CSP_MODE_CIPHER:
 780                 if (!alg_is_cipher(csp->csp_cipher_alg))
 781                         return (false);
 782                 if (csp->csp_cipher_alg != CRYPTO_NULL_CBC) {
 783                         if (csp->csp_cipher_klen == 0)
 784                                 return (false);
 785                         if (csp->csp_ivlen == 0)
 786                                 return (false);
 787                 }
 788                 if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
 789                         return (false);
 790                 if (csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0 ||
 791                     csp->csp_auth_mlen != 0)
 792                         return (false);
 793                 break;
 794         case CSP_MODE_DIGEST:
 795                 if (csp->csp_cipher_alg != 0 || csp->csp_cipher_klen != 0)
 796                         return (false);
 797
 798                 /* IV is optional for digests (e.g. GMAC). */
 799                 if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
 800                         return (false);
 801                 if (!alg_is_digest(csp->csp_auth_alg))
 802                         return (false);
 803
 804                 /* Key is optional for BLAKE2 digests. */
 805                 if (csp->csp_auth_alg == CRYPTO_BLAKE2B ||
 806                     csp->csp_auth_alg == CRYPTO_BLAKE2S)
 807                         ;
 808                 else if (alg_is_keyed_digest(csp->csp_auth_alg)) {
 809                         if (csp->csp_auth_klen == 0)
 810                                 return (false);
 811                 } else {
 812                         if (csp->csp_auth_klen != 0)
 813                                 return (false);
 814                 }
 815                 if (csp->csp_auth_mlen != 0) {
 816                         axf = crypto_auth_hash(csp);
 817                         if (axf == NULL || csp->csp_auth_mlen > axf->hashsize)
 818                                 return (false);
 819                 }
 820                 break;
 821         case CSP_MODE_AEAD:
 822                 if (!alg_is_aead(csp->csp_cipher_alg))
 823                         return (false);
 824                 if (csp->csp_cipher_klen == 0)
 825                         return (false);
 826                 if (csp->csp_ivlen == 0 ||
 827                     csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
 828                         return (false);
 829                 if (csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0)
 830                         return (false);
 831
 832                 /*
 833                  * XXX: Would be nice to have a better way to get this
 834                  * value.
 835                  */
 836                 switch (csp->csp_cipher_alg) {
 837                 case CRYPTO_AES_NIST_GCM_16:
 838                 case CRYPTO_AES_CCM_16:
 839                         if (csp->csp_auth_mlen > 16)
 840                                 return (false);
 841                         break;
 842                 }
 843                 break;
 844         case CSP_MODE_ETA:
 845                 if (!alg_is_cipher(csp->csp_cipher_alg))
 846                         return (false);
 847                 if (csp->csp_cipher_alg != CRYPTO_NULL_CBC) {
 848                         if (csp->csp_cipher_klen == 0)
 849                                 return (false);
 850                         if (csp->csp_ivlen == 0)
 851                                 return (false);
 852                 }
 853                 if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
 854                         return (false);
 855                 if (!alg_is_digest(csp->csp_auth_alg))
 856                         return (false);
 857
 858                 /* Key is optional for BLAKE2 digests. */
 859                 if (csp->csp_auth_alg == CRYPTO_BLAKE2B ||
 860                     csp->csp_auth_alg == CRYPTO_BLAKE2S)
 861                         ;
 862                 else if (alg_is_keyed_digest(csp->csp_auth_alg)) {
 863                         if (csp->csp_auth_klen == 0)
 864                                 return (false);
 865                 } else {
 866                         if (csp->csp_auth_klen != 0)
 867                                 return (false);
 868                 }
 869                 if (csp->csp_auth_mlen != 0) {
 870                         axf = crypto_auth_hash(csp);
 871                         if (axf == NULL || csp->csp_auth_mlen > axf->hashsize)
 872                                 return (false);
 873                 }
 874                 break;
 875         default:
 876                 return (false);
 877         }
 878
 879         return (true);
 880 }
 881
 882 /*
 883  * Delete a session after it has been detached from its driver.
 884  */
 885 static void
 886 crypto_deletesession(crypto_session_t cses)
 887 {
 888         struct cryptocap *cap;
 889
 890         cap = cses->cap;
 891
 892         explicit_bzero(cses->softc, cap->cc_session_size);
 893         free(cses->softc, M_CRYPTO_DATA);
 894         uma_zfree(cryptoses_zone, cses);
 895
 896         CRYPTO_DRIVER_LOCK();
 897         cap->cc_sessions--;
 898         if (cap->cc_sessions == 0 && cap->cc_flags & CRYPTOCAP_F_CLEANUP)
 899                 wakeup(cap);
 900         CRYPTO_DRIVER_UNLOCK();
 901         cap_rele(cap);
 902 }
 903
 904 /*
 905  * Create a new session.  The crid argument specifies a crypto
 906  * driver to use or constraints on a driver to select (hardware
 907  * only, software only, either).  Whatever driver is selected
 908  * must be capable of the requested crypto algorithms.
 909  */
 910 int
 911 crypto_newsession(crypto_session_t *cses,
 912     const struct crypto_session_params *csp, int crid)
 913 {
 914         crypto_session_t res;
 915         struct cryptocap *cap;
 916         int err;
 917
 918         if (!check_csp(csp))
 919                 return (EINVAL);
 920
 921         res = NULL;
 922
 923         CRYPTO_DRIVER_LOCK();
 924         if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
 925                 /*
 926                  * Use specified driver; verify it is capable.
 927                  */
 928                 cap = crypto_checkdriver(crid);
 929                 if (cap != NULL && CRYPTODEV_PROBESESSION(cap->cc_dev, csp) > 0)
 930                         cap = NULL;
 931         } else {
 932                 /*
 933                  * No requested driver; select based on crid flags.
 934                  */
 935                 cap = crypto_select_driver(csp, crid);
 936         }
 937         if (cap == NULL) {
 938                 CRYPTO_DRIVER_UNLOCK();
 939                 CRYPTDEB("no driver");
 940                 return (EOPNOTSUPP);
 941         }
 942         cap_ref(cap);
 943         cap->cc_sessions++;
 944         CRYPTO_DRIVER_UNLOCK();
 945
 946         res = uma_zalloc(cryptoses_zone, M_WAITOK | M_ZERO);
 947         res->cap = cap;
 948         res->softc = malloc(cap->cc_session_size, M_CRYPTO_DATA, M_WAITOK |
 949             M_ZERO);
 950         res->csp = *csp;
 951
 952         /* Call the driver initialization routine. */
 953         err = CRYPTODEV_NEWSESSION(cap->cc_dev, res, csp);
 954         if (err != 0) {
 955                 CRYPTDEB("dev newsession failed: %d", err);
 956                 crypto_deletesession(res);
 957                 return (err);
 958         }
 959
 960         *cses = res;
 961         return (0);
 962 }
 963
 964 /*
 965  * Delete an existing session (or a reserved session on an unregistered
 966  * driver).
 967  */
 968 void
 969 crypto_freesession(crypto_session_t cses)
 970 {
 971         struct cryptocap *cap;
 972
 973         if (cses == NULL)
 974                 return;
 975
 976         cap = cses->cap;
 977
 978         /* Call the driver cleanup routine, if available. */
 979         CRYPTODEV_FREESESSION(cap->cc_dev, cses);
 980
 981         crypto_deletesession(cses);
 982 }
 983
 984 /*
 985  * Return a new driver id.  Registers a driver with the system so that
 986  * it can be probed by subsequent sessions.
 987  */
 988 int32_t
 989 crypto_get_driverid(device_t dev, size_t sessionsize, int flags)
 990 {
 991         struct cryptocap *cap, **newdrv;
 992         int i;
 993
 994         if ((flags & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
 995                 device_printf(dev,
 996                     "no flags specified when registering driver\n");
 997                 return -1;
 998         }
 999
1000         cap = malloc(sizeof(*cap), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
1001         cap->cc_dev = dev;
1002         cap->cc_session_size = sessionsize;
1003         cap->cc_flags = flags;
1004         refcount_init(&cap->cc_refs, 1);
1005
1006         CRYPTO_DRIVER_LOCK();
1007         for (;;) {
1008                 for (i = 0; i < crypto_drivers_size; i++) {
1009                         if (crypto_drivers[i] == NULL)
1010                                 break;
1011                 }
1012
1013                 if (i < crypto_drivers_size)
1014                         break;
1015
1016                 /* Out of entries, allocate some more. */
1017
1018                 if (2 * crypto_drivers_size <= crypto_drivers_size) {
1019                         CRYPTO_DRIVER_UNLOCK();
1020                         printf("crypto: driver count wraparound!\n");
1021                         cap_rele(cap);
1022                         return (-1);
1023                 }
1024                 CRYPTO_DRIVER_UNLOCK();
1025
1026                 newdrv = malloc(2 * crypto_drivers_size *
1027                     sizeof(*crypto_drivers), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
1028
1029                 CRYPTO_DRIVER_LOCK();
1030                 memcpy(newdrv, crypto_drivers,
1031                     crypto_drivers_size * sizeof(*crypto_drivers));
1032
1033                 crypto_drivers_size *= 2;
1034
1035                 free(crypto_drivers, M_CRYPTO_DATA);
1036                 crypto_drivers = newdrv;
1037         }
1038
1039         cap->cc_hid = i;
1040         crypto_drivers[i] = cap;
1041         CRYPTO_DRIVER_UNLOCK();
1042
1043         if (bootverbose)
1044                 printf("crypto: assign %s driver id %u, flags 0x%x\n",
1045                     device_get_nameunit(dev), i, flags);
1046
1047         return i;
1048 }
1049
1050 /*
1051  * Lookup a driver by name.  We match against the full device
1052  * name and unit, and against just the name.  The latter gives
1053  * us a simple widlcarding by device name.  On success return the
1054  * driver/hardware identifier; otherwise return -1.
1055  */
1056 int
1057 crypto_find_driver(const char *match)
1058 {
1059         struct cryptocap *cap;
1060         int i, len = strlen(match);
1061
1062         CRYPTO_DRIVER_LOCK();
1063         for (i = 0; i < crypto_drivers_size; i++) {
1064                 if (crypto_drivers[i] == NULL)
1065                         continue;
1066                 cap = crypto_drivers[i];
1067                 if (strncmp(match, device_get_nameunit(cap->cc_dev), len) == 0 ||
1068                     strncmp(match, device_get_name(cap->cc_dev), len) == 0) {
1069                         CRYPTO_DRIVER_UNLOCK();
1070                         return (i);
1071                 }
1072         }
1073         CRYPTO_DRIVER_UNLOCK();
1074         return (-1);
1075 }
1076
1077 /*
1078  * Return the device_t for the specified driver or NULL
1079  * if the driver identifier is invalid.
1080  */
1081 device_t
1082 crypto_find_device_byhid(int hid)
1083 {
1084         struct cryptocap *cap;
1085         device_t dev;
1086
1087         dev = NULL;
1088         CRYPTO_DRIVER_LOCK();
1089         cap = crypto_checkdriver(hid);
1090         if (cap != NULL)
1091                 dev = cap->cc_dev;
1092         CRYPTO_DRIVER_UNLOCK();
1093         return (dev);
1094 }
1095
1096 /*
1097  * Return the device/driver capabilities.
1098  */
1099 int
1100 crypto_getcaps(int hid)
1101 {
1102         struct cryptocap *cap;
1103         int flags;
1104
1105         flags = 0;
1106         CRYPTO_DRIVER_LOCK();
1107         cap = crypto_checkdriver(hid);
1108         if (cap != NULL)
1109                 flags = cap->cc_flags;
1110         CRYPTO_DRIVER_UNLOCK();
1111         return (flags);
1112 }
1113
1114 /*
1115  * Register support for a key-related algorithm.  This routine
1116  * is called once for each algorithm supported a driver.
1117  */
1118 int
1119 crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags)
1120 {
1121         struct cryptocap *cap;
1122         int err;
1123
1124         CRYPTO_DRIVER_LOCK();
1125
1126         cap = crypto_checkdriver(driverid);
1127         if (cap != NULL &&
1128             (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) {
1129                 /*
1130                  * XXX Do some performance testing to determine placing.
1131                  * XXX We probably need an auxiliary data structure that
1132                  * XXX describes relative performances.
1133                  */
1134
1135                 cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
1136                 if (bootverbose)
1137                         printf("crypto: %s registers key alg %u flags %u\n"
1138                                 , device_get_nameunit(cap->cc_dev)
1139                                 , kalg
1140                                 , flags
1141                         );
1142                 err = 0;
1143         } else
1144                 err = EINVAL;
1145
1146         CRYPTO_DRIVER_UNLOCK();
1147         return err;
1148 }
1149
1150 /*
1151  * Unregister all algorithms associated with a crypto driver.
1152  * If there are pending sessions using it, leave enough information
1153  * around so that subsequent calls using those sessions will
1154  * correctly detect the driver has been unregistered and reroute
1155  * requests.
1156  */
1157 int
1158 crypto_unregister_all(u_int32_t driverid)
1159 {
1160         struct cryptocap *cap;
1161
1162         CRYPTO_DRIVER_LOCK();
1163         cap = crypto_checkdriver(driverid);
1164         if (cap == NULL) {
1165                 CRYPTO_DRIVER_UNLOCK();
1166                 return (EINVAL);
1167         }
1168
1169         cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
1170         crypto_drivers[driverid] = NULL;
1171
1172         /*
1173          * XXX: This doesn't do anything to kick sessions that
1174          * have no pending operations.
1175          */
1176         while (cap->cc_sessions != 0 || cap->cc_koperations != 0)
1177                 mtx_sleep(cap, &crypto_drivers_mtx, 0, "cryunreg", 0);
1178         CRYPTO_DRIVER_UNLOCK();
1179         cap_rele(cap);
1180
1181         return (0);
1182 }
1183
1184 /*
1185  * Clear blockage on a driver.  The what parameter indicates whether
1186  * the driver is now ready for cryptop's and/or cryptokop's.
1187  */
1188 int
1189 crypto_unblock(u_int32_t driverid, int what)
1190 {
1191         struct cryptocap *cap;
1192         int err;
1193
1194         CRYPTO_Q_LOCK();
1195         cap = crypto_checkdriver(driverid);
1196         if (cap != NULL) {
1197                 if (what & CRYPTO_SYMQ)
1198                         cap->cc_qblocked = 0;
1199                 if (what & CRYPTO_ASYMQ)
1200                         cap->cc_kqblocked = 0;
1201                 if (crp_sleep)
1202                         wakeup_one(&crp_q);
1203                 err = 0;
1204         } else
1205                 err = EINVAL;
1206         CRYPTO_Q_UNLOCK();
1207
1208         return err;
1209 }
1210
1211 size_t
1212 crypto_buffer_len(struct crypto_buffer *cb)
1213 {
1214         switch (cb->cb_type) {
1215         case CRYPTO_BUF_CONTIG:
1216                 return (cb->cb_buf_len);
1217         case CRYPTO_BUF_MBUF:
1218                 if (cb->cb_mbuf->m_flags & M_PKTHDR)
1219                         return (cb->cb_mbuf->m_pkthdr.len);
1220                 return (m_length(cb->cb_mbuf, NULL));
1221         case CRYPTO_BUF_UIO:
1222                 return (cb->cb_uio->uio_resid);
1223         default:
1224                 return (0);
1225         }
1226 }
1227
1228 #ifdef INVARIANTS
1229 /* Various sanity checks on crypto requests. */
1230 static void
1231 cb_sanity(struct crypto_buffer *cb, const char *name)
1232 {
1233         KASSERT(cb->cb_type > CRYPTO_BUF_NONE && cb->cb_type <= CRYPTO_BUF_LAST,
1234             ("incoming crp with invalid %s buffer type", name));
1235         if (cb->cb_type == CRYPTO_BUF_CONTIG)
1236                 KASSERT(cb->cb_buf_len >= 0,
1237                     ("incoming crp with -ve %s buffer length", name));
1238 }
1239
1240 static void
1241 crp_sanity(struct cryptop *crp)
1242 {
1243         struct crypto_session_params *csp;
1244         struct crypto_buffer *out;
1245         size_t ilen, len, olen;
1246
1247         KASSERT(crp->crp_session != NULL, ("incoming crp without a session"));
1248         KASSERT(crp->crp_obuf.cb_type >= CRYPTO_BUF_NONE &&
1249             crp->crp_obuf.cb_type <= CRYPTO_BUF_LAST,
1250             ("incoming crp with invalid output buffer type"));
1251         KASSERT(crp->crp_etype == 0, ("incoming crp with error"));
1252         KASSERT(!(crp->crp_flags & CRYPTO_F_DONE),
1253             ("incoming crp already done"));
1254
1255         csp = &crp->crp_session->csp;
1256         cb_sanity(&crp->crp_buf, "input");
1257         ilen = crypto_buffer_len(&crp->crp_buf);
1258         olen = ilen;
1259         out = NULL;
1260         if (csp->csp_flags & CSP_F_SEPARATE_OUTPUT) {
1261                 if (crp->crp_obuf.cb_type != CRYPTO_BUF_NONE) {
1262                         cb_sanity(&crp->crp_obuf, "output");
1263                         out = &crp->crp_obuf;
1264                         olen = crypto_buffer_len(out);
1265                 }
1266         } else
1267                 KASSERT(crp->crp_obuf.cb_type == CRYPTO_BUF_NONE,
1268                     ("incoming crp with separate output buffer "
1269                     "but no session support"));
1270
1271         switch (csp->csp_mode) {
1272         case CSP_MODE_COMPRESS:
1273                 KASSERT(crp->crp_op == CRYPTO_OP_COMPRESS ||
1274                     crp->crp_op == CRYPTO_OP_DECOMPRESS,
1275                     ("invalid compression op %x", crp->crp_op));
1276                 break;
1277         case CSP_MODE_CIPHER:
1278                 KASSERT(crp->crp_op == CRYPTO_OP_ENCRYPT ||
1279                     crp->crp_op == CRYPTO_OP_DECRYPT,
1280                     ("invalid cipher op %x", crp->crp_op));
1281                 break;
1282         case CSP_MODE_DIGEST:
1283                 KASSERT(crp->crp_op == CRYPTO_OP_COMPUTE_DIGEST ||
1284                     crp->crp_op == CRYPTO_OP_VERIFY_DIGEST,
1285                     ("invalid digest op %x", crp->crp_op));
1286                 break;
1287         case CSP_MODE_AEAD:
1288                 KASSERT(crp->crp_op ==
1289                     (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST) ||
1290                     crp->crp_op ==
1291                     (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST),
1292                     ("invalid AEAD op %x", crp->crp_op));
1293                 if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16)
1294                         KASSERT(crp->crp_flags & CRYPTO_F_IV_SEPARATE,
1295                             ("GCM without a separate IV"));
1296                 if (csp->csp_cipher_alg == CRYPTO_AES_CCM_16)
1297                         KASSERT(crp->crp_flags & CRYPTO_F_IV_SEPARATE,
1298                             ("CCM without a separate IV"));
1299                 break;
1300         case CSP_MODE_ETA:
1301                 KASSERT(crp->crp_op ==
1302                     (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST) ||
1303                     crp->crp_op ==
1304                     (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST),
1305                     ("invalid ETA op %x", crp->crp_op));
1306                 break;
1307         }
1308         if (csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) {
1309                 KASSERT(crp->crp_aad_start == 0 ||
1310                     crp->crp_aad_start < ilen,
1311                     ("invalid AAD start"));
1312                 KASSERT(crp->crp_aad_length != 0 || crp->crp_aad_start == 0,
1313                     ("AAD with zero length and non-zero start"));
1314                 KASSERT(crp->crp_aad_length == 0 ||
1315                     crp->crp_aad_start + crp->crp_aad_length <= ilen,
1316                     ("AAD outside input length"));
1317         } else {
1318                 KASSERT(crp->crp_aad_start == 0 && crp->crp_aad_length == 0,
1319                     ("AAD region in request not supporting AAD"));
1320         }
1321         if (csp->csp_ivlen == 0) {
1322                 KASSERT((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0,
1323                     ("IV_SEPARATE set when IV isn't used"));
1324                 KASSERT(crp->crp_iv_start == 0,
1325                     ("crp_iv_start set when IV isn't used"));
1326         } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) {
1327                 KASSERT(crp->crp_iv_start == 0,
1328                     ("IV_SEPARATE used with non-zero IV start"));
1329         } else {
1330                 KASSERT(crp->crp_iv_start < ilen,
1331                     ("invalid IV start"));
1332                 KASSERT(crp->crp_iv_start + csp->csp_ivlen <= ilen,
1333                     ("IV outside buffer length"));
1334         }
1335         /* XXX: payload_start of 0 should always be < ilen? */
1336         KASSERT(crp->crp_payload_start == 0 ||
1337             crp->crp_payload_start < ilen,
1338             ("invalid payload start"));
1339         KASSERT(crp->crp_payload_start + crp->crp_payload_length <=
1340             ilen, ("payload outside input buffer"));
1341         if (out == NULL) {
1342                 KASSERT(crp->crp_payload_output_start == 0,
1343                     ("payload output start non-zero without output buffer"));
1344         } else {
1345                 KASSERT(crp->crp_payload_output_start < olen,
1346                     ("invalid payload output start"));
1347                 KASSERT(crp->crp_payload_output_start +
1348                     crp->crp_payload_length <= olen,
1349                     ("payload outside output buffer"));
1350         }
1351         if (csp->csp_mode == CSP_MODE_DIGEST ||
1352             csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) {
1353                 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST)
1354                         len = ilen;
1355                 else
1356                         len = olen;
1357                 KASSERT(crp->crp_digest_start == 0 ||
1358                     crp->crp_digest_start < len,
1359                     ("invalid digest start"));
1360                 /* XXX: For the mlen == 0 case this check isn't perfect. */
1361                 KASSERT(crp->crp_digest_start + csp->csp_auth_mlen <= len,
1362                     ("digest outside buffer"));
1363         } else {
1364                 KASSERT(crp->crp_digest_start == 0,
1365                     ("non-zero digest start for request without a digest"));
1366         }
1367         if (csp->csp_cipher_klen != 0)
1368                 KASSERT(csp->csp_cipher_key != NULL ||
1369                     crp->crp_cipher_key != NULL,
1370                     ("cipher request without a key"));
1371         if (csp->csp_auth_klen != 0)
1372                 KASSERT(csp->csp_auth_key != NULL || crp->crp_auth_key != NULL,
1373                     ("auth request without a key"));
1374         KASSERT(crp->crp_callback != NULL, ("incoming crp without callback"));
1375 }
1376 #endif
1377
1378 /*
1379  * Add a crypto request to a queue, to be processed by the kernel thread.
1380  */
1381 int
1382 crypto_dispatch(struct cryptop *crp)
1383 {
1384         struct cryptocap *cap;
1385         int result;
1386
1387 #ifdef INVARIANTS
1388         crp_sanity(crp);
1389 #endif
1390
1391         cryptostats.cs_ops++;
1392
1393 #ifdef CRYPTO_TIMING
1394         if (crypto_timing)
1395                 binuptime(&crp->crp_tstamp);
1396 #endif
1397
1398         crp->crp_retw_id = ((uintptr_t)crp->crp_session) % crypto_workers_num;
1399
1400         if (CRYPTOP_ASYNC(crp)) {
1401                 if (crp->crp_flags & CRYPTO_F_ASYNC_KEEPORDER) {
1402                         struct crypto_ret_worker *ret_worker;
1403
1404                         ret_worker = CRYPTO_RETW(crp->crp_retw_id);
1405
1406                         CRYPTO_RETW_LOCK(ret_worker);
1407                         crp->crp_seq = ret_worker->reorder_ops++;
1408                         CRYPTO_RETW_UNLOCK(ret_worker);
1409                 }
1410
1411                 TASK_INIT(&crp->crp_task, 0, crypto_task_invoke, crp);
1412                 taskqueue_enqueue(crypto_tq, &crp->crp_task);
1413                 return (0);
1414         }
1415
1416         if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) {
1417                 /*
1418                  * Caller marked the request to be processed
1419                  * immediately; dispatch it directly to the
1420                  * driver unless the driver is currently blocked.
1421                  */
1422                 cap = crp->crp_session->cap;
1423                 if (!cap->cc_qblocked) {
1424                         result = crypto_invoke(cap, crp, 0);
1425                         if (result != ERESTART)
1426                                 return (result);
1427                         /*
1428                          * The driver ran out of resources, put the request on
1429                          * the queue.
1430                          */
1431                 }
1432         }
1433         crypto_batch_enqueue(crp);
1434         return 0;
1435 }
1436
1437 void
1438 crypto_batch_enqueue(struct cryptop *crp)
1439 {
1440
1441         CRYPTO_Q_LOCK();
1442         TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
1443         if (crp_sleep)
1444                 wakeup_one(&crp_q);
1445         CRYPTO_Q_UNLOCK();
1446 }
1447
1448 /*
1449  * Add an asymetric crypto request to a queue,
1450  * to be processed by the kernel thread.
1451  */
1452 int
1453 crypto_kdispatch(struct cryptkop *krp)
1454 {
1455         int error;
1456
1457         cryptostats.cs_kops++;
1458
1459         krp->krp_cap = NULL;
1460         error = crypto_kinvoke(krp);
1461         if (error == ERESTART) {
1462                 CRYPTO_Q_LOCK();
1463                 TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next);
1464                 if (crp_sleep)
1465                         wakeup_one(&crp_q);
1466                 CRYPTO_Q_UNLOCK();
1467                 error = 0;
1468         }
1469         return error;
1470 }
1471
1472 /*
1473  * Verify a driver is suitable for the specified operation.
1474  */
1475 static __inline int
1476 kdriver_suitable(const struct cryptocap *cap, const struct cryptkop *krp)
1477 {
1478         return (cap->cc_kalg[krp->krp_op] & CRYPTO_ALG_FLAG_SUPPORTED) != 0;
1479 }
1480
1481 /*
1482  * Select a driver for an asym operation.  The driver must
1483  * support the necessary algorithm.  The caller can constrain
1484  * which device is selected with the flags parameter.  The
1485  * algorithm we use here is pretty stupid; just use the first
1486  * driver that supports the algorithms we need. If there are
1487  * multiple suitable drivers we choose the driver with the
1488  * fewest active operations.  We prefer hardware-backed
1489  * drivers to software ones when either may be used.
1490  */
1491 static struct cryptocap *
1492 crypto_select_kdriver(const struct cryptkop *krp, int flags)
1493 {
1494         struct cryptocap *cap, *best;
1495         int match, hid;
1496
1497         CRYPTO_DRIVER_ASSERT();
1498
1499         /*
1500          * Look first for hardware crypto devices if permitted.
1501          */
1502         if (flags & CRYPTOCAP_F_HARDWARE)
1503                 match = CRYPTOCAP_F_HARDWARE;
1504         else
1505                 match = CRYPTOCAP_F_SOFTWARE;
1506         best = NULL;
1507 again:
1508         for (hid = 0; hid < crypto_drivers_size; hid++) {
1509                 /*
1510                  * If there is no driver for this slot, or the driver
1511                  * is not appropriate (hardware or software based on
1512                  * match), then skip.
1513                  */
1514                 cap = crypto_drivers[hid];
1515                 if (cap->cc_dev == NULL ||
1516                     (cap->cc_flags & match) == 0)
1517                         continue;
1518
1519                 /* verify all the algorithms are supported. */
1520                 if (kdriver_suitable(cap, krp)) {
1521                         if (best == NULL ||
1522                             cap->cc_koperations < best->cc_koperations)
1523                                 best = cap;
1524                 }
1525         }
1526         if (best != NULL)
1527                 return best;
1528         if (match == CRYPTOCAP_F_HARDWARE && (flags & CRYPTOCAP_F_SOFTWARE)) {
1529                 /* sort of an Algol 68-style for loop */
1530                 match = CRYPTOCAP_F_SOFTWARE;
1531                 goto again;
1532         }
1533         return best;
1534 }
1535
1536 /*
1537  * Choose a driver for an asymmetric crypto request.
1538  */
1539 static struct cryptocap *
1540 crypto_lookup_kdriver(struct cryptkop *krp)
1541 {
1542         struct cryptocap *cap;
1543         uint32_t crid;
1544
1545         /* If this request is requeued, it might already have a driver. */
1546         cap = krp->krp_cap;
1547         if (cap != NULL)
1548                 return (cap);
1549
1550         /* Use krp_crid to choose a driver. */
1551         crid = krp->krp_crid;
1552         if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
1553                 cap = crypto_checkdriver(crid);
1554                 if (cap != NULL) {
1555                         /*
1556                          * Driver present, it must support the
1557                          * necessary algorithm and, if s/w drivers are
1558                          * excluded, it must be registered as
1559                          * hardware-backed.
1560                          */
1561                         if (!kdriver_suitable(cap, krp) ||
1562                             (!crypto_devallowsoft &&
1563                             (cap->cc_flags & CRYPTOCAP_F_HARDWARE) == 0))
1564                                 cap = NULL;
1565                 }
1566         } else {
1567                 /*
1568                  * No requested driver; select based on crid flags.
1569                  */
1570                 if (!crypto_devallowsoft)       /* NB: disallow s/w drivers */
1571                         crid &= ~CRYPTOCAP_F_SOFTWARE;
1572                 cap = crypto_select_kdriver(krp, crid);
1573         }
1574
1575         if (cap != NULL) {
1576                 krp->krp_cap = cap_ref(cap);
1577                 krp->krp_hid = cap->cc_hid;
1578         }
1579         return (cap);
1580 }
1581
1582 /*
1583  * Dispatch an asymmetric crypto request.
1584  */
1585 static int
1586 crypto_kinvoke(struct cryptkop *krp)
1587 {
1588         struct cryptocap *cap = NULL;
1589         int error;
1590
1591         KASSERT(krp != NULL, ("%s: krp == NULL", __func__));
1592         KASSERT(krp->krp_callback != NULL,
1593             ("%s: krp->crp_callback == NULL", __func__));
1594
1595         CRYPTO_DRIVER_LOCK();
1596         cap = crypto_lookup_kdriver(krp);
1597         if (cap == NULL) {
1598                 CRYPTO_DRIVER_UNLOCK();
1599                 krp->krp_status = ENODEV;
1600                 crypto_kdone(krp);
1601                 return (0);
1602         }
1603
1604         /*
1605          * If the device is blocked, return ERESTART to requeue it.
1606          */
1607         if (cap->cc_kqblocked) {
1608                 /*
1609                  * XXX: Previously this set krp_status to ERESTART and
1610                  * invoked crypto_kdone but the caller would still
1611                  * requeue it.
1612                  */
1613                 CRYPTO_DRIVER_UNLOCK();
1614                 return (ERESTART);
1615         }
1616
1617         cap->cc_koperations++;
1618         CRYPTO_DRIVER_UNLOCK();
1619         error = CRYPTODEV_KPROCESS(cap->cc_dev, krp, 0);
1620         if (error == ERESTART) {
1621                 CRYPTO_DRIVER_LOCK();
1622                 cap->cc_koperations--;
1623                 CRYPTO_DRIVER_UNLOCK();
1624                 return (error);
1625         }
1626
1627         KASSERT(error == 0, ("error %d returned from crypto_kprocess", error));
1628         return (0);
1629 }
1630
1631 #ifdef CRYPTO_TIMING
1632 static void
1633 crypto_tstat(struct cryptotstat *ts, struct bintime *bt)
1634 {
1635         struct bintime now, delta;
1636         struct timespec t;
1637         uint64_t u;
1638
1639         binuptime(&now);
1640         u = now.frac;
1641         delta.frac = now.frac - bt->frac;
1642         delta.sec = now.sec - bt->sec;
1643         if (u < delta.frac)
1644                 delta.sec--;
1645         bintime2timespec(&delta, &t);
1646         timespecadd(&ts->acc, &t, &ts->acc);
1647         if (timespeccmp(&t, &ts->min, <))
1648                 ts->min = t;
1649         if (timespeccmp(&t, &ts->max, >))
1650                 ts->max = t;
1651         ts->count++;
1652
1653         *bt = now;
1654 }
1655 #endif
1656
1657 static void
1658 crypto_task_invoke(void *ctx, int pending)
1659 {
1660         struct cryptocap *cap;
1661         struct cryptop *crp;
1662         int result;
1663
1664         crp = (struct cryptop *)ctx;
1665         cap = crp->crp_session->cap;
1666         result = crypto_invoke(cap, crp, 0);
1667         if (result == ERESTART)
1668                 crypto_batch_enqueue(crp);
1669 }
1670
1671 /*
1672  * Dispatch a crypto request to the appropriate crypto devices.
1673  */
1674 static int
1675 crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint)
1676 {
1677
1678         KASSERT(crp != NULL, ("%s: crp == NULL", __func__));
1679         KASSERT(crp->crp_callback != NULL,
1680             ("%s: crp->crp_callback == NULL", __func__));
1681         KASSERT(crp->crp_session != NULL,
1682             ("%s: crp->crp_session == NULL", __func__));
1683
1684 #ifdef CRYPTO_TIMING
1685         if (crypto_timing)
1686                 crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp);
1687 #endif
1688         if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
1689                 struct crypto_session_params csp;
1690                 crypto_session_t nses;
1691
1692                 /*
1693                  * Driver has unregistered; migrate the session and return
1694                  * an error to the caller so they'll resubmit the op.
1695                  *
1696                  * XXX: What if there are more already queued requests for this
1697                  *      session?
1698                  *
1699                  * XXX: Real solution is to make sessions refcounted
1700                  * and force callers to hold a reference when
1701                  * assigning to crp_session.  Could maybe change
1702                  * crypto_getreq to accept a session pointer to make
1703                  * that work.  Alternatively, we could abandon the
1704                  * notion of rewriting crp_session in requests forcing
1705                  * the caller to deal with allocating a new session.
1706                  * Perhaps provide a method to allow a crp's session to
1707                  * be swapped that callers could use.
1708                  */
1709                 csp = crp->crp_session->csp;
1710                 crypto_freesession(crp->crp_session);
1711
1712                 /*
1713                  * XXX: Key pointers may no longer be valid.  If we
1714                  * really want to support this we need to define the
1715                  * KPI such that 'csp' is required to be valid for the
1716                  * duration of a session by the caller perhaps.
1717                  *
1718                  * XXX: If the keys have been changed this will reuse
1719                  * the old keys.  This probably suggests making
1720                  * rekeying more explicit and updating the key
1721                  * pointers in 'csp' when the keys change.
1722                  */
1723                 if (crypto_newsession(&nses, &csp,
1724                     CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE) == 0)
1725                         crp->crp_session = nses;
1726
1727                 crp->crp_etype = EAGAIN;
1728                 crypto_done(crp);
1729                 return 0;
1730         } else {
1731                 /*
1732                  * Invoke the driver to process the request.
1733                  */
1734                 return CRYPTODEV_PROCESS(cap->cc_dev, crp, hint);
1735         }
1736 }
1737
1738 void
1739 crypto_freereq(struct cryptop *crp)
1740 {
1741
1742         if (crp == NULL)
1743                 return;
1744
1745 #ifdef DIAGNOSTIC
1746         {
1747                 struct cryptop *crp2;
1748                 struct crypto_ret_worker *ret_worker;
1749
1750                 CRYPTO_Q_LOCK();
1751                 TAILQ_FOREACH(crp2, &crp_q, crp_next) {
1752                         KASSERT(crp2 != crp,
1753                             ("Freeing cryptop from the crypto queue (%p).",
1754                             crp));
1755                 }
1756                 CRYPTO_Q_UNLOCK();
1757
1758                 FOREACH_CRYPTO_RETW(ret_worker) {
1759                         CRYPTO_RETW_LOCK(ret_worker);
1760                         TAILQ_FOREACH(crp2, &ret_worker->crp_ret_q, crp_next) {
1761                                 KASSERT(crp2 != crp,
1762                                     ("Freeing cryptop from the return queue (%p).",
1763                                     crp));
1764                         }
1765                         CRYPTO_RETW_UNLOCK(ret_worker);
1766                 }
1767         }
1768 #endif
1769
1770         uma_zfree(cryptop_zone, crp);
1771 }
1772
1773 struct cryptop *
1774 crypto_getreq(crypto_session_t cses, int how)
1775 {
1776         struct cryptop *crp;
1777
1778         MPASS(how == M_WAITOK || how == M_NOWAIT);
1779         crp = uma_zalloc(cryptop_zone, how | M_ZERO);
1780         crp->crp_session = cses;
1781         return (crp);
1782 }
1783
1784 /*
1785  * Invoke the callback on behalf of the driver.
1786  */
1787 void
1788 crypto_done(struct cryptop *crp)
1789 {
1790         KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0,
1791                 ("crypto_done: op already done, flags 0x%x", crp->crp_flags));
1792         crp->crp_flags |= CRYPTO_F_DONE;
1793         if (crp->crp_etype != 0)
1794                 cryptostats.cs_errs++;
1795 #ifdef CRYPTO_TIMING
1796         if (crypto_timing)
1797                 crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp);
1798 #endif
1799         /*
1800          * CBIMM means unconditionally do the callback immediately;
1801          * CBIFSYNC means do the callback immediately only if the
1802          * operation was done synchronously.  Both are used to avoid
1803          * doing extraneous context switches; the latter is mostly
1804          * used with the software crypto driver.
1805          */
1806         if (!CRYPTOP_ASYNC_KEEPORDER(crp) &&
1807             ((crp->crp_flags & CRYPTO_F_CBIMM) ||
1808             ((crp->crp_flags & CRYPTO_F_CBIFSYNC) &&
1809              (crypto_ses2caps(crp->crp_session) & CRYPTOCAP_F_SYNC)))) {
1810                 /*
1811                  * Do the callback directly.  This is ok when the
1812                  * callback routine does very little (e.g. the
1813                  * /dev/crypto callback method just does a wakeup).
1814                  */
1815 #ifdef CRYPTO_TIMING
1816                 if (crypto_timing) {
1817                         /*
1818                          * NB: We must copy the timestamp before
1819                          * doing the callback as the cryptop is
1820                          * likely to be reclaimed.
1821                          */
1822                         struct bintime t = crp->crp_tstamp;
1823                         crypto_tstat(&cryptostats.cs_cb, &t);
1824                         crp->crp_callback(crp);
1825                         crypto_tstat(&cryptostats.cs_finis, &t);
1826                 } else
1827 #endif
1828                         crp->crp_callback(crp);
1829         } else {
1830                 struct crypto_ret_worker *ret_worker;
1831                 bool wake;
1832
1833                 ret_worker = CRYPTO_RETW(crp->crp_retw_id);
1834                 wake = false;
1835
1836                 /*
1837                  * Normal case; queue the callback for the thread.
1838                  */
1839                 CRYPTO_RETW_LOCK(ret_worker);
1840                 if (CRYPTOP_ASYNC_KEEPORDER(crp)) {
1841                         struct cryptop *tmp;
1842
1843                         TAILQ_FOREACH_REVERSE(tmp, &ret_worker->crp_ordered_ret_q,
1844                                         cryptop_q, crp_next) {
1845                                 if (CRYPTO_SEQ_GT(crp->crp_seq, tmp->crp_seq)) {
1846                                         TAILQ_INSERT_AFTER(&ret_worker->crp_ordered_ret_q,
1847                                                         tmp, crp, crp_next);
1848                                         break;
1849                                 }
1850                         }
1851                         if (tmp == NULL) {
1852                                 TAILQ_INSERT_HEAD(&ret_worker->crp_ordered_ret_q,
1853                                                 crp, crp_next);
1854                         }
1855
1856                         if (crp->crp_seq == ret_worker->reorder_cur_seq)
1857                                 wake = true;
1858                 }
1859                 else {
1860                         if (CRYPTO_RETW_EMPTY(ret_worker))
1861                                 wake = true;
1862
1863                         TAILQ_INSERT_TAIL(&ret_worker->crp_ret_q, crp, crp_next);
1864                 }
1865
1866                 if (wake)
1867                         wakeup_one(&ret_worker->crp_ret_q);     /* shared wait channel */
1868                 CRYPTO_RETW_UNLOCK(ret_worker);
1869         }
1870 }
1871
1872 /*
1873  * Invoke the callback on behalf of the driver.
1874  */
1875 void
1876 crypto_kdone(struct cryptkop *krp)
1877 {
1878         struct crypto_ret_worker *ret_worker;
1879         struct cryptocap *cap;
1880
1881         if (krp->krp_status != 0)
1882                 cryptostats.cs_kerrs++;
1883         CRYPTO_DRIVER_LOCK();
1884         cap = krp->krp_cap;
1885         KASSERT(cap->cc_koperations > 0, ("cc_koperations == 0"));
1886         cap->cc_koperations--;
1887         if (cap->cc_koperations == 0 && cap->cc_flags & CRYPTOCAP_F_CLEANUP)
1888                 wakeup(cap);
1889         CRYPTO_DRIVER_UNLOCK();
1890         krp->krp_cap = NULL;
1891         cap_rele(cap);
1892
1893         ret_worker = CRYPTO_RETW(0);
1894
1895         CRYPTO_RETW_LOCK(ret_worker);
1896         if (CRYPTO_RETW_EMPTY(ret_worker))
1897                 wakeup_one(&ret_worker->crp_ret_q);             /* shared wait channel */
1898         TAILQ_INSERT_TAIL(&ret_worker->crp_ret_kq, krp, krp_next);
1899         CRYPTO_RETW_UNLOCK(ret_worker);
1900 }
1901
1902 int
1903 crypto_getfeat(int *featp)
1904 {
1905         int hid, kalg, feat = 0;
1906
1907         CRYPTO_DRIVER_LOCK();
1908         for (hid = 0; hid < crypto_drivers_size; hid++) {
1909                 const struct cryptocap *cap = crypto_drivers[hid];
1910
1911                 if (cap == NULL ||
1912                     ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) &&
1913                     !crypto_devallowsoft)) {
1914                         continue;
1915                 }
1916                 for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++)
1917                         if (cap->cc_kalg[kalg] & CRYPTO_ALG_FLAG_SUPPORTED)
1918                                 feat |=  1 << kalg;
1919         }
1920         CRYPTO_DRIVER_UNLOCK();
1921         *featp = feat;
1922         return (0);
1923 }
1924
1925 /*
1926  * Terminate a thread at module unload.  The process that
1927  * initiated this is waiting for us to signal that we're gone;
1928  * wake it up and exit.  We use the driver table lock to insure
1929  * we don't do the wakeup before they're waiting.  There is no
1930  * race here because the waiter sleeps on the proc lock for the
1931  * thread so it gets notified at the right time because of an
1932  * extra wakeup that's done in exit1().
1933  */
1934 static void
1935 crypto_finis(void *chan)
1936 {
1937         CRYPTO_DRIVER_LOCK();
1938         wakeup_one(chan);
1939         CRYPTO_DRIVER_UNLOCK();
1940         kproc_exit(0);
1941 }
1942
1943 /*
1944  * Crypto thread, dispatches crypto requests.
1945  */
1946 static void
1947 crypto_proc(void)
1948 {
1949         struct cryptop *crp, *submit;
1950         struct cryptkop *krp;
1951         struct cryptocap *cap;
1952         int result, hint;
1953
1954 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
1955         fpu_kern_thread(FPU_KERN_NORMAL);
1956 #endif
1957
1958         CRYPTO_Q_LOCK();
1959         for (;;) {
1960                 /*
1961                  * Find the first element in the queue that can be
1962                  * processed and look-ahead to see if multiple ops
1963                  * are ready for the same driver.
1964                  */
1965                 submit = NULL;
1966                 hint = 0;
1967                 TAILQ_FOREACH(crp, &crp_q, crp_next) {
1968                         cap = crp->crp_session->cap;
1969                         /*
1970                          * Driver cannot disappeared when there is an active
1971                          * session.
1972                          */
1973                         KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
1974                             __func__, __LINE__));
1975                         if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
1976                                 /* Op needs to be migrated, process it. */
1977                                 if (submit == NULL)
1978                                         submit = crp;
1979                                 break;
1980                         }
1981                         if (!cap->cc_qblocked) {
1982                                 if (submit != NULL) {
1983                                         /*
1984                                          * We stop on finding another op,
1985                                          * regardless whether its for the same
1986                                          * driver or not.  We could keep
1987                                          * searching the queue but it might be
1988                                          * better to just use a per-driver
1989                                          * queue instead.
1990                                          */
1991                                         if (submit->crp_session->cap == cap)
1992                                                 hint = CRYPTO_HINT_MORE;
1993                                         break;
1994                                 } else {
1995                                         submit = crp;
1996                                         if ((submit->crp_flags & CRYPTO_F_BATCH) == 0)
1997                                                 break;
1998                                         /* keep scanning for more are q'd */
1999                                 }
2000                         }
2001                 }
2002                 if (submit != NULL) {
2003                         TAILQ_REMOVE(&crp_q, submit, crp_next);
2004                         cap = submit->crp_session->cap;
2005                         KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
2006                             __func__, __LINE__));
2007                         CRYPTO_Q_UNLOCK();
2008                         result = crypto_invoke(cap, submit, hint);
2009                         CRYPTO_Q_LOCK();
2010                         if (result == ERESTART) {
2011                                 /*
2012                                  * The driver ran out of resources, mark the
2013                                  * driver ``blocked'' for cryptop's and put
2014                                  * the request back in the queue.  It would
2015                                  * best to put the request back where we got
2016                                  * it but that's hard so for now we put it
2017                                  * at the front.  This should be ok; putting
2018                                  * it at the end does not work.
2019                                  */
2020                                 cap->cc_qblocked = 1;
2021                                 TAILQ_INSERT_HEAD(&crp_q, submit, crp_next);
2022                                 cryptostats.cs_blocks++;
2023                         }
2024                 }
2025
2026                 /* As above, but for key ops */
2027                 TAILQ_FOREACH(krp, &crp_kq, krp_next) {
2028                         cap = krp->krp_cap;
2029                         if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
2030                                 /*
2031                                  * Operation needs to be migrated,
2032                                  * clear krp_cap so a new driver is
2033                                  * selected.
2034                                  */
2035                                 krp->krp_cap = NULL;
2036                                 cap_rele(cap);
2037                                 break;
2038                         }
2039                         if (!cap->cc_kqblocked)
2040                                 break;
2041                 }
2042                 if (krp != NULL) {
2043                         TAILQ_REMOVE(&crp_kq, krp, krp_next);
2044                         CRYPTO_Q_UNLOCK();
2045                         result = crypto_kinvoke(krp);
2046                         CRYPTO_Q_LOCK();
2047                         if (result == ERESTART) {
2048                                 /*
2049                                  * The driver ran out of resources, mark the
2050                                  * driver ``blocked'' for cryptkop's and put
2051                                  * the request back in the queue.  It would
2052                                  * best to put the request back where we got
2053                                  * it but that's hard so for now we put it
2054                                  * at the front.  This should be ok; putting
2055                                  * it at the end does not work.
2056                                  */
2057                                 krp->krp_cap->cc_kqblocked = 1;
2058                                 TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);
2059                                 cryptostats.cs_kblocks++;
2060                         }
2061                 }
2062
2063                 if (submit == NULL && krp == NULL) {
2064                         /*
2065                          * Nothing more to be processed.  Sleep until we're
2066                          * woken because there are more ops to process.
2067                          * This happens either by submission or by a driver
2068                          * becoming unblocked and notifying us through
2069                          * crypto_unblock.  Note that when we wakeup we
2070                          * start processing each queue again from the
2071                          * front. It's not clear that it's important to
2072                          * preserve this ordering since ops may finish
2073                          * out of order if dispatched to different devices
2074                          * and some become blocked while others do not.
2075                          */
2076                         crp_sleep = 1;
2077                         msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0);
2078                         crp_sleep = 0;
2079                         if (cryptoproc == NULL)
2080                                 break;
2081                         cryptostats.cs_intrs++;
2082                 }
2083         }
2084         CRYPTO_Q_UNLOCK();
2085
2086         crypto_finis(&crp_q);
2087 }
2088
2089 /*
2090  * Crypto returns thread, does callbacks for processed crypto requests.
2091  * Callbacks are done here, rather than in the crypto drivers, because
2092  * callbacks typically are expensive and would slow interrupt handling.
2093  */
2094 static void
2095 crypto_ret_proc(struct crypto_ret_worker *ret_worker)
2096 {
2097         struct cryptop *crpt;
2098         struct cryptkop *krpt;
2099
2100         CRYPTO_RETW_LOCK(ret_worker);
2101         for (;;) {
2102                 /* Harvest return q's for completed ops */
2103                 crpt = TAILQ_FIRST(&ret_worker->crp_ordered_ret_q);
2104                 if (crpt != NULL) {
2105                         if (crpt->crp_seq == ret_worker->reorder_cur_seq) {
2106                                 TAILQ_REMOVE(&ret_worker->crp_ordered_ret_q, crpt, crp_next);
2107                                 ret_worker->reorder_cur_seq++;
2108                         } else {
2109                                 crpt = NULL;
2110                         }
2111                 }
2112
2113                 if (crpt == NULL) {
2114                         crpt = TAILQ_FIRST(&ret_worker->crp_ret_q);
2115                         if (crpt != NULL)
2116                                 TAILQ_REMOVE(&ret_worker->crp_ret_q, crpt, crp_next);
2117                 }
2118
2119                 krpt = TAILQ_FIRST(&ret_worker->crp_ret_kq);
2120                 if (krpt != NULL)
2121                         TAILQ_REMOVE(&ret_worker->crp_ret_kq, krpt, krp_next);
2122
2123                 if (crpt != NULL || krpt != NULL) {
2124                         CRYPTO_RETW_UNLOCK(ret_worker);
2125                         /*
2126                          * Run callbacks unlocked.
2127                          */
2128                         if (crpt != NULL) {
2129 #ifdef CRYPTO_TIMING
2130                                 if (crypto_timing) {
2131                                         /*
2132                                          * NB: We must copy the timestamp before
2133                                          * doing the callback as the cryptop is
2134                                          * likely to be reclaimed.
2135                                          */
2136                                         struct bintime t = crpt->crp_tstamp;
2137                                         crypto_tstat(&cryptostats.cs_cb, &t);
2138                                         crpt->crp_callback(crpt);
2139                                         crypto_tstat(&cryptostats.cs_finis, &t);
2140                                 } else
2141 #endif
2142                                         crpt->crp_callback(crpt);
2143                         }
2144                         if (krpt != NULL)
2145                                 krpt->krp_callback(krpt);
2146                         CRYPTO_RETW_LOCK(ret_worker);
2147                 } else {
2148                         /*
2149                          * Nothing more to be processed.  Sleep until we're
2150                          * woken because there are more returns to process.
2151                          */
2152                         msleep(&ret_worker->crp_ret_q, &ret_worker->crypto_ret_mtx, PWAIT,
2153                                 "crypto_ret_wait", 0);
2154                         if (ret_worker->cryptoretproc == NULL)
2155                                 break;
2156                         cryptostats.cs_rets++;
2157                 }
2158         }
2159         CRYPTO_RETW_UNLOCK(ret_worker);
2160
2161         crypto_finis(&ret_worker->crp_ret_q);
2162 }
2163
2164 #ifdef DDB
2165 static void
2166 db_show_drivers(void)
2167 {
2168         int hid;
2169
2170         db_printf("%12s %4s %4s %8s %2s %2s\n"
2171                 , "Device"
2172                 , "Ses"
2173                 , "Kops"
2174                 , "Flags"
2175                 , "QB"
2176                 , "KB"
2177         );
2178         for (hid = 0; hid < crypto_drivers_size; hid++) {
2179                 const struct cryptocap *cap = crypto_drivers[hid];
2180                 if (cap == NULL)
2181                         continue;
2182                 db_printf("%-12s %4u %4u %08x %2u %2u\n"
2183                     , device_get_nameunit(cap->cc_dev)
2184                     , cap->cc_sessions
2185                     , cap->cc_koperations
2186                     , cap->cc_flags
2187                     , cap->cc_qblocked
2188                     , cap->cc_kqblocked
2189                 );
2190         }
2191 }
2192
2193 DB_SHOW_COMMAND(crypto, db_show_crypto)
2194 {
2195         struct cryptop *crp;
2196         struct crypto_ret_worker *ret_worker;
2197
2198         db_show_drivers();
2199         db_printf("\n");
2200
2201         db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n",
2202             "HID", "Caps", "Ilen", "Olen", "Etype", "Flags",
2203             "Device", "Callback");
2204         TAILQ_FOREACH(crp, &crp_q, crp_next) {
2205                 db_printf("%4u %08x %4u %4u %04x %8p %8p\n"
2206                     , crp->crp_session->cap->cc_hid
2207                     , (int) crypto_ses2caps(crp->crp_session)
2208                     , crp->crp_olen
2209                     , crp->crp_etype
2210                     , crp->crp_flags
2211                     , device_get_nameunit(crp->crp_session->cap->cc_dev)
2212                     , crp->crp_callback
2213                 );
2214         }
2215         FOREACH_CRYPTO_RETW(ret_worker) {
2216                 db_printf("\n%8s %4s %4s %4s %8s\n",
2217                     "ret_worker", "HID", "Etype", "Flags", "Callback");
2218                 if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) {
2219                         TAILQ_FOREACH(crp, &ret_worker->crp_ret_q, crp_next) {
2220                                 db_printf("%8td %4u %4u %04x %8p\n"
2221                                     , CRYPTO_RETW_ID(ret_worker)
2222                                     , crp->crp_session->cap->cc_hid
2223                                     , crp->crp_etype
2224                                     , crp->crp_flags
2225                                     , crp->crp_callback
2226                                 );
2227                         }
2228                 }
2229         }
2230 }
2231
2232 DB_SHOW_COMMAND(kcrypto, db_show_kcrypto)
2233 {
2234         struct cryptkop *krp;
2235         struct crypto_ret_worker *ret_worker;
2236
2237         db_show_drivers();
2238         db_printf("\n");
2239
2240         db_printf("%4s %5s %4s %4s %8s %4s %8s\n",
2241             "Op", "Status", "#IP", "#OP", "CRID", "HID", "Callback");
2242         TAILQ_FOREACH(krp, &crp_kq, krp_next) {
2243                 db_printf("%4u %5u %4u %4u %08x %4u %8p\n"
2244                     , krp->krp_op
2245                     , krp->krp_status
2246                     , krp->krp_iparams, krp->krp_oparams
2247                     , krp->krp_crid, krp->krp_hid
2248                     , krp->krp_callback
2249                 );
2250         }
2251
2252         ret_worker = CRYPTO_RETW(0);
2253         if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) {
2254                 db_printf("%4s %5s %8s %4s %8s\n",
2255                     "Op", "Status", "CRID", "HID", "Callback");
2256                 TAILQ_FOREACH(krp, &ret_worker->crp_ret_kq, krp_next) {
2257                         db_printf("%4u %5u %08x %4u %8p\n"
2258                             , krp->krp_op
2259                             , krp->krp_status
2260                             , krp->krp_crid, krp->krp_hid
2261                             , krp->krp_callback
2262                         );
2263                 }
2264         }
2265 }
2266 #endif
2267
2268 int crypto_modevent(module_t mod, int type, void *unused);
2269
2270 /*
2271  * Initialization code, both for static and dynamic loading.
2272  * Note this is not invoked with the usual MODULE_DECLARE
2273  * mechanism but instead is listed as a dependency by the
2274  * cryptosoft driver.  This guarantees proper ordering of
2275  * calls on module load/unload.
2276  */
2277 int
2278 crypto_modevent(module_t mod, int type, void *unused)
2279 {
2280         int error = EINVAL;
2281
2282         switch (type) {
2283         case MOD_LOAD:
2284                 error = crypto_init();
2285                 if (error == 0 && bootverbose)
2286                         printf("crypto: <crypto core>\n");
2287                 break;
2288         case MOD_UNLOAD:
2289                 /*XXX disallow if active sessions */
2290                 error = 0;
2291                 crypto_destroy();
2292                 return 0;
2293         }
2294         return error;
2295 }
2296 MODULE_VERSION(crypto, 1);
2297 MODULE_DEPEND(crypto, zlib, 1, 1, 1);