sys/opencrypto/crypto.c

   1 /*-
   2  * Copyright (c) 2002-2006 Sam Leffler.  All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  14  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  15  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  16  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  17  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  18  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  19  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  20  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  21  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  22  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  23  */
  24
  25 #include <sys/cdefs.h>
  26 __FBSDID("$FreeBSD$");
  27
  28 /*
  29  * Cryptographic Subsystem.
  30  *
  31  * This code is derived from the Openbsd Cryptographic Framework (OCF)
  32  * that has the copyright shown below.  Very little of the original
  33  * code remains.
  34  */
  35
  36 /*-
  37  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
  38  *
  39  * This code was written by Angelos D. Keromytis in Athens, Greece, in
  40  * February 2000. Network Security Technologies Inc. (NSTI) kindly
  41  * supported the development of this code.
  42  *
  43  * Copyright (c) 2000, 2001 Angelos D. Keromytis
  44  *
  45  * Permission to use, copy, and modify this software with or without fee
  46  * is hereby granted, provided that this entire notice is included in
  47  * all source code copies of any software which is or includes a copy or
  48  * modification of this software.
  49  *
  50  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
  51  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
  52  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
  53  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
  54  * PURPOSE.
  55  */
  56
  57 #define CRYPTO_TIMING                           /* enable timing support */
  58
  59 #include "opt_ddb.h"
  60
  61 #include <sys/param.h>
  62 #include <sys/systm.h>
  63 #include <sys/eventhandler.h>
  64 #include <sys/kernel.h>
  65 #include <sys/kthread.h>
  66 #include <sys/linker.h>
  67 #include <sys/lock.h>
  68 #include <sys/module.h>
  69 #include <sys/mutex.h>
  70 #include <sys/malloc.h>
  71 #include <sys/proc.h>
  72 #include <sys/sdt.h>
  73 #include <sys/smp.h>
  74 #include <sys/sysctl.h>
  75 #include <sys/taskqueue.h>
  76
  77 #include <ddb/ddb.h>
  78
  79 #include <vm/uma.h>
  80 #include <crypto/intake.h>
  81 #include <opencrypto/cryptodev.h>
  82 #include <opencrypto/xform.h>                   /* XXX for M_XDATA */
  83
  84 #include <sys/kobj.h>
  85 #include <sys/bus.h>
  86 #include "cryptodev_if.h"
  87
  88 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
  89 #include <machine/pcb.h>
  90 #endif
  91
  92 struct crypto_session {
  93         device_t parent;
  94         void *softc;
  95         uint32_t hid;
  96         uint32_t capabilities;
  97 };
  98
  99 SDT_PROVIDER_DEFINE(opencrypto);
 100
 101 /*
 102  * Crypto drivers register themselves by allocating a slot in the
 103  * crypto_drivers table with crypto_get_driverid() and then registering
 104  * each algorithm they support with crypto_register() and crypto_kregister().
 105  */
 106 static  struct mtx crypto_drivers_mtx;          /* lock on driver table */
 107 #define CRYPTO_DRIVER_LOCK()    mtx_lock(&crypto_drivers_mtx)
 108 #define CRYPTO_DRIVER_UNLOCK()  mtx_unlock(&crypto_drivers_mtx)
 109 #define CRYPTO_DRIVER_ASSERT()  mtx_assert(&crypto_drivers_mtx, MA_OWNED)
 110
 111 /*
 112  * Crypto device/driver capabilities structure.
 113  *
 114  * Synchronization:
 115  * (d) - protected by CRYPTO_DRIVER_LOCK()
 116  * (q) - protected by CRYPTO_Q_LOCK()
 117  * Not tagged fields are read-only.
 118  */
 119 struct cryptocap {
 120         device_t        cc_dev;                 /* (d) device/driver */
 121         u_int32_t       cc_sessions;            /* (d) # of sessions */
 122         u_int32_t       cc_koperations;         /* (d) # os asym operations */
 123         /*
 124          * Largest possible operator length (in bits) for each type of
 125          * encryption algorithm. XXX not used
 126          */
 127         u_int16_t       cc_max_op_len[CRYPTO_ALGORITHM_MAX + 1];
 128         u_int8_t        cc_alg[CRYPTO_ALGORITHM_MAX + 1];
 129         u_int8_t        cc_kalg[CRK_ALGORITHM_MAX + 1];
 130
 131         int             cc_flags;               /* (d) flags */
 132 #define CRYPTOCAP_F_CLEANUP     0x80000000      /* needs resource cleanup */
 133         int             cc_qblocked;            /* (q) symmetric q blocked */
 134         int             cc_kqblocked;           /* (q) asymmetric q blocked */
 135         size_t          cc_session_size;
 136 };
 137 static  struct cryptocap *crypto_drivers = NULL;
 138 static  int crypto_drivers_num = 0;
 139
 140 /*
 141  * There are two queues for crypto requests; one for symmetric (e.g.
 142  * cipher) operations and one for asymmetric (e.g. MOD)operations.
 143  * A single mutex is used to lock access to both queues.  We could
 144  * have one per-queue but having one simplifies handling of block/unblock
 145  * operations.
 146  */
 147 static  int crp_sleep = 0;
 148 static  TAILQ_HEAD(cryptop_q ,cryptop) crp_q;           /* request queues */
 149 static  TAILQ_HEAD(,cryptkop) crp_kq;
 150 static  struct mtx crypto_q_mtx;
 151 #define CRYPTO_Q_LOCK()         mtx_lock(&crypto_q_mtx)
 152 #define CRYPTO_Q_UNLOCK()       mtx_unlock(&crypto_q_mtx)
 153
 154 /*
 155  * Taskqueue used to dispatch the crypto requests
 156  * that have the CRYPTO_F_ASYNC flag
 157  */
 158 static struct taskqueue *crypto_tq;
 159
 160 /*
 161  * Crypto seq numbers are operated on with modular arithmetic
 162  */
 163 #define CRYPTO_SEQ_GT(a,b)      ((int)((a)-(b)) > 0)
 164
 165 struct crypto_ret_worker {
 166         struct mtx crypto_ret_mtx;
 167
 168         TAILQ_HEAD(,cryptop) crp_ordered_ret_q; /* ordered callback queue for symetric jobs */
 169         TAILQ_HEAD(,cryptop) crp_ret_q;         /* callback queue for symetric jobs */
 170         TAILQ_HEAD(,cryptkop) crp_ret_kq;       /* callback queue for asym jobs */
 171
 172         u_int32_t reorder_ops;          /* total ordered sym jobs received */
 173         u_int32_t reorder_cur_seq;      /* current sym job dispatched */
 174
 175         struct proc *cryptoretproc;
 176 };
 177 static struct crypto_ret_worker *crypto_ret_workers = NULL;
 178
 179 #define CRYPTO_RETW(i)          (&crypto_ret_workers[i])
 180 #define CRYPTO_RETW_ID(w)       ((w) - crypto_ret_workers)
 181 #define FOREACH_CRYPTO_RETW(w) \
 182         for (w = crypto_ret_workers; w < crypto_ret_workers + crypto_workers_num; ++w)
 183
 184 #define CRYPTO_RETW_LOCK(w)     mtx_lock(&w->crypto_ret_mtx)
 185 #define CRYPTO_RETW_UNLOCK(w)   mtx_unlock(&w->crypto_ret_mtx)
 186 #define CRYPTO_RETW_EMPTY(w) \
 187         (TAILQ_EMPTY(&w->crp_ret_q) && TAILQ_EMPTY(&w->crp_ret_kq) && TAILQ_EMPTY(&w->crp_ordered_ret_q))
 188
 189 static int crypto_workers_num = 0;
 190 SYSCTL_INT(_kern, OID_AUTO, crypto_workers_num, CTLFLAG_RDTUN,
 191            &crypto_workers_num, 0,
 192            "Number of crypto workers used to dispatch crypto jobs");
 193
 194 static  uma_zone_t cryptop_zone;
 195 static  uma_zone_t cryptodesc_zone;
 196 static  uma_zone_t cryptoses_zone;
 197
 198 int     crypto_userasymcrypto = 1;      /* userland may do asym crypto reqs */
 199 SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW,
 200            &crypto_userasymcrypto, 0,
 201            "Enable/disable user-mode access to asymmetric crypto support");
 202 int     crypto_devallowsoft = 0;        /* only use hardware crypto */
 203 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW,
 204            &crypto_devallowsoft, 0,
 205            "Enable/disable use of software crypto by /dev/crypto");
 206
 207 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
 208
 209 static  void crypto_proc(void);
 210 static  struct proc *cryptoproc;
 211 static  void crypto_ret_proc(struct crypto_ret_worker *ret_worker);
 212 static  void crypto_destroy(void);
 213 static  int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint);
 214 static  int crypto_kinvoke(struct cryptkop *krp, int flags);
 215 static  void crypto_remove(struct cryptocap *cap);
 216 static  void crypto_task_invoke(void *ctx, int pending);
 217 static void crypto_batch_enqueue(struct cryptop *crp);
 218
 219 static  struct cryptostats cryptostats;
 220 SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats,
 221             cryptostats, "Crypto system statistics");
 222
 223 #ifdef CRYPTO_TIMING
 224 static  int crypto_timing = 0;
 225 SYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW,
 226            &crypto_timing, 0, "Enable/disable crypto timing support");
 227 #endif
 228
 229 /* Try to avoid directly exposing the key buffer as a symbol */
 230 static struct keybuf *keybuf;
 231
 232 static struct keybuf empty_keybuf = {
 233         .kb_nents = 0
 234 };
 235
 236 /* Obtain the key buffer from boot metadata */
 237 static void
 238 keybuf_init(void)
 239 {
 240         caddr_t kmdp;
 241
 242         kmdp = preload_search_by_type("elf kernel");
 243
 244         if (kmdp == NULL)
 245                 kmdp = preload_search_by_type("elf64 kernel");
 246
 247         keybuf = (struct keybuf *)preload_search_info(kmdp,
 248             MODINFO_METADATA | MODINFOMD_KEYBUF);
 249
 250         if (keybuf == NULL)
 251                 keybuf = &empty_keybuf;
 252 }
 253
 254 /* It'd be nice if we could store these in some kind of secure memory... */
 255 struct keybuf * get_keybuf(void) {
 256
 257         return (keybuf);
 258 }
 259
 260 static int
 261 crypto_init(void)
 262 {
 263         struct crypto_ret_worker *ret_worker;
 264         int error;
 265
 266         mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table",
 267                 MTX_DEF|MTX_QUIET);
 268
 269         TAILQ_INIT(&crp_q);
 270         TAILQ_INIT(&crp_kq);
 271         mtx_init(&crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF);
 272
 273         cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop),
 274                                     0, 0, 0, 0,
 275                                     UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
 276         cryptodesc_zone = uma_zcreate("cryptodesc", sizeof (struct cryptodesc),
 277                                     0, 0, 0, 0,
 278                                     UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
 279         cryptoses_zone = uma_zcreate("crypto_session",
 280             sizeof(struct crypto_session), NULL, NULL, NULL, NULL,
 281             UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
 282
 283         if (cryptodesc_zone == NULL || cryptop_zone == NULL ||
 284             cryptoses_zone == NULL) {
 285                 printf("crypto_init: cannot setup crypto zones\n");
 286                 error = ENOMEM;
 287                 goto bad;
 288         }
 289
 290         crypto_drivers_num = CRYPTO_DRIVERS_INITIAL;
 291         crypto_drivers = malloc(crypto_drivers_num *
 292             sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO);
 293         if (crypto_drivers == NULL) {
 294                 printf("crypto_init: cannot setup crypto drivers\n");
 295                 error = ENOMEM;
 296                 goto bad;
 297         }
 298
 299         if (crypto_workers_num < 1 || crypto_workers_num > mp_ncpus)
 300                 crypto_workers_num = mp_ncpus;
 301
 302         crypto_tq = taskqueue_create("crypto", M_WAITOK|M_ZERO,
 303                                 taskqueue_thread_enqueue, &crypto_tq);
 304         if (crypto_tq == NULL) {
 305                 printf("crypto init: cannot setup crypto taskqueue\n");
 306                 error = ENOMEM;
 307                 goto bad;
 308         }
 309
 310         taskqueue_start_threads(&crypto_tq, crypto_workers_num, PRI_MIN_KERN,
 311                 "crypto");
 312
 313         error = kproc_create((void (*)(void *)) crypto_proc, NULL,
 314                     &cryptoproc, 0, 0, "crypto");
 315         if (error) {
 316                 printf("crypto_init: cannot start crypto thread; error %d",
 317                         error);
 318                 goto bad;
 319         }
 320
 321         crypto_ret_workers = malloc(crypto_workers_num * sizeof(struct crypto_ret_worker),
 322                         M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
 323         if (crypto_ret_workers == NULL) {
 324                 error = ENOMEM;
 325                 printf("crypto_init: cannot allocate ret workers\n");
 326                 goto bad;
 327         }
 328
 329
 330         FOREACH_CRYPTO_RETW(ret_worker) {
 331                 TAILQ_INIT(&ret_worker->crp_ordered_ret_q);
 332                 TAILQ_INIT(&ret_worker->crp_ret_q);
 333                 TAILQ_INIT(&ret_worker->crp_ret_kq);
 334
 335                 ret_worker->reorder_ops = 0;
 336                 ret_worker->reorder_cur_seq = 0;
 337
 338                 mtx_init(&ret_worker->crypto_ret_mtx, "crypto", "crypto return queues", MTX_DEF);
 339
 340                 error = kproc_create((void (*)(void *)) crypto_ret_proc, ret_worker,
 341                                 &ret_worker->cryptoretproc, 0, 0, "crypto returns %td", CRYPTO_RETW_ID(ret_worker));
 342                 if (error) {
 343                         printf("crypto_init: cannot start cryptoret thread; error %d",
 344                                 error);
 345                         goto bad;
 346                 }
 347         }
 348
 349         keybuf_init();
 350
 351         return 0;
 352 bad:
 353         crypto_destroy();
 354         return error;
 355 }
 356
 357 /*
 358  * Signal a crypto thread to terminate.  We use the driver
 359  * table lock to synchronize the sleep/wakeups so that we
 360  * are sure the threads have terminated before we release
 361  * the data structures they use.  See crypto_finis below
 362  * for the other half of this song-and-dance.
 363  */
 364 static void
 365 crypto_terminate(struct proc **pp, void *q)
 366 {
 367         struct proc *p;
 368
 369         mtx_assert(&crypto_drivers_mtx, MA_OWNED);
 370         p = *pp;
 371         *pp = NULL;
 372         if (p) {
 373                 wakeup_one(q);
 374                 PROC_LOCK(p);           /* NB: insure we don't miss wakeup */
 375                 CRYPTO_DRIVER_UNLOCK(); /* let crypto_finis progress */
 376                 msleep(p, &p->p_mtx, PWAIT, "crypto_destroy", 0);
 377                 PROC_UNLOCK(p);
 378                 CRYPTO_DRIVER_LOCK();
 379         }
 380 }
 381
 382 static void
 383 crypto_destroy(void)
 384 {
 385         struct crypto_ret_worker *ret_worker;
 386
 387         /*
 388          * Terminate any crypto threads.
 389          */
 390         if (crypto_tq != NULL)
 391                 taskqueue_drain_all(crypto_tq);
 392         CRYPTO_DRIVER_LOCK();
 393         crypto_terminate(&cryptoproc, &crp_q);
 394         FOREACH_CRYPTO_RETW(ret_worker)
 395                 crypto_terminate(&ret_worker->cryptoretproc, &ret_worker->crp_ret_q);
 396         CRYPTO_DRIVER_UNLOCK();
 397
 398         /* XXX flush queues??? */
 399
 400         /*
 401          * Reclaim dynamically allocated resources.
 402          */
 403         if (crypto_drivers != NULL)
 404                 free(crypto_drivers, M_CRYPTO_DATA);
 405
 406         if (cryptoses_zone != NULL)
 407                 uma_zdestroy(cryptoses_zone);
 408         if (cryptodesc_zone != NULL)
 409                 uma_zdestroy(cryptodesc_zone);
 410         if (cryptop_zone != NULL)
 411                 uma_zdestroy(cryptop_zone);
 412         mtx_destroy(&crypto_q_mtx);
 413         FOREACH_CRYPTO_RETW(ret_worker)
 414                 mtx_destroy(&ret_worker->crypto_ret_mtx);
 415         free(crypto_ret_workers, M_CRYPTO_DATA);
 416         if (crypto_tq != NULL)
 417                 taskqueue_free(crypto_tq);
 418         mtx_destroy(&crypto_drivers_mtx);
 419 }
 420
 421 uint32_t
 422 crypto_ses2hid(crypto_session_t crypto_session)
 423 {
 424         return (crypto_session->hid);
 425 }
 426
 427 uint32_t
 428 crypto_ses2caps(crypto_session_t crypto_session)
 429 {
 430         return (crypto_session->capabilities);
 431 }
 432
 433 void *
 434 crypto_get_driver_session(crypto_session_t crypto_session)
 435 {
 436         return (crypto_session->softc);
 437 }
 438
 439 static struct cryptocap *
 440 crypto_checkdriver(u_int32_t hid)
 441 {
 442         if (crypto_drivers == NULL)
 443                 return NULL;
 444         return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]);
 445 }
 446
 447 /*
 448  * Compare a driver's list of supported algorithms against another
 449  * list; return non-zero if all algorithms are supported.
 450  */
 451 static int
 452 driver_suitable(const struct cryptocap *cap, const struct cryptoini *cri)
 453 {
 454         const struct cryptoini *cr;
 455
 456         /* See if all the algorithms are supported. */
 457         for (cr = cri; cr; cr = cr->cri_next)
 458                 if (cap->cc_alg[cr->cri_alg] == 0)
 459                         return 0;
 460         return 1;
 461 }
 462
 463 /*
 464  * Select a driver for a new session that supports the specified
 465  * algorithms and, optionally, is constrained according to the flags.
 466  * The algorithm we use here is pretty stupid; just use the
 467  * first driver that supports all the algorithms we need. If there
 468  * are multiple drivers we choose the driver with the fewest active
 469  * sessions.  We prefer hardware-backed drivers to software ones.
 470  *
 471  * XXX We need more smarts here (in real life too, but that's
 472  * XXX another story altogether).
 473  */
 474 static struct cryptocap *
 475 crypto_select_driver(const struct cryptoini *cri, int flags)
 476 {
 477         struct cryptocap *cap, *best;
 478         int match, hid;
 479
 480         CRYPTO_DRIVER_ASSERT();
 481
 482         /*
 483          * Look first for hardware crypto devices if permitted.
 484          */
 485         if (flags & CRYPTOCAP_F_HARDWARE)
 486                 match = CRYPTOCAP_F_HARDWARE;
 487         else
 488                 match = CRYPTOCAP_F_SOFTWARE;
 489         best = NULL;
 490 again:
 491         for (hid = 0; hid < crypto_drivers_num; hid++) {
 492                 cap = &crypto_drivers[hid];
 493                 /*
 494                  * If it's not initialized, is in the process of
 495                  * going away, or is not appropriate (hardware
 496                  * or software based on match), then skip.
 497                  */
 498                 if (cap->cc_dev == NULL ||
 499                     (cap->cc_flags & CRYPTOCAP_F_CLEANUP) ||
 500                     (cap->cc_flags & match) == 0)
 501                         continue;
 502
 503                 /* verify all the algorithms are supported. */
 504                 if (driver_suitable(cap, cri)) {
 505                         if (best == NULL ||
 506                             cap->cc_sessions < best->cc_sessions)
 507                                 best = cap;
 508                 }
 509         }
 510         if (best == NULL && match == CRYPTOCAP_F_HARDWARE &&
 511             (flags & CRYPTOCAP_F_SOFTWARE)) {
 512                 /* sort of an Algol 68-style for loop */
 513                 match = CRYPTOCAP_F_SOFTWARE;
 514                 goto again;
 515         }
 516         return best;
 517 }
 518
 519 /*
 520  * Create a new session.  The crid argument specifies a crypto
 521  * driver to use or constraints on a driver to select (hardware
 522  * only, software only, either).  Whatever driver is selected
 523  * must be capable of the requested crypto algorithms.
 524  */
 525 int
 526 crypto_newsession(crypto_session_t *cses, struct cryptoini *cri, int crid)
 527 {
 528         crypto_session_t res;
 529         void *softc_mem;
 530         struct cryptocap *cap;
 531         u_int32_t hid;
 532         size_t softc_size;
 533         int err;
 534
 535 restart:
 536         res = NULL;
 537         softc_mem = NULL;
 538
 539         CRYPTO_DRIVER_LOCK();
 540         if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
 541                 /*
 542                  * Use specified driver; verify it is capable.
 543                  */
 544                 cap = crypto_checkdriver(crid);
 545                 if (cap != NULL && !driver_suitable(cap, cri))
 546                         cap = NULL;
 547         } else {
 548                 /*
 549                  * No requested driver; select based on crid flags.
 550                  */
 551                 cap = crypto_select_driver(cri, crid);
 552                 /*
 553                  * if NULL then can't do everything in one session.
 554                  * XXX Fix this. We need to inject a "virtual" session
 555                  * XXX layer right about here.
 556                  */
 557         }
 558         if (cap == NULL) {
 559                 CRYPTDEB("no driver");
 560                 err = EOPNOTSUPP;
 561                 goto out;
 562         }
 563         cap->cc_sessions++;
 564         softc_size = cap->cc_session_size;
 565         hid = cap - crypto_drivers;
 566         cap = NULL;
 567         CRYPTO_DRIVER_UNLOCK();
 568
 569         softc_mem = malloc(softc_size, M_CRYPTO_DATA, M_WAITOK | M_ZERO);
 570         res = uma_zalloc(cryptoses_zone, M_WAITOK | M_ZERO);
 571         res->softc = softc_mem;
 572
 573         CRYPTO_DRIVER_LOCK();
 574         cap = crypto_checkdriver(hid);
 575         if (cap != NULL && (cap->cc_flags & CRYPTOCAP_F_CLEANUP) != 0) {
 576                 cap->cc_sessions--;
 577                 crypto_remove(cap);
 578                 cap = NULL;
 579         }
 580         if (cap == NULL) {
 581                 free(softc_mem, M_CRYPTO_DATA);
 582                 uma_zfree(cryptoses_zone, res);
 583                 CRYPTO_DRIVER_UNLOCK();
 584                 goto restart;
 585         }
 586
 587         /* Call the driver initialization routine. */
 588         err = CRYPTODEV_NEWSESSION(cap->cc_dev, res, cri);
 589         if (err != 0) {
 590                 CRYPTDEB("dev newsession failed: %d", err);
 591                 goto out;
 592         }
 593
 594         res->capabilities = cap->cc_flags & 0xff000000;
 595         res->hid = hid;
 596         *cses = res;
 597
 598 out:
 599         CRYPTO_DRIVER_UNLOCK();
 600         if (err != 0) {
 601                 free(softc_mem, M_CRYPTO_DATA);
 602                 if (res != NULL)
 603                         uma_zfree(cryptoses_zone, res);
 604         }
 605         return err;
 606 }
 607
 608 static void
 609 crypto_remove(struct cryptocap *cap)
 610 {
 611
 612         mtx_assert(&crypto_drivers_mtx, MA_OWNED);
 613         if (cap->cc_sessions == 0 && cap->cc_koperations == 0)
 614                 bzero(cap, sizeof(*cap));
 615 }
 616
 617 /*
 618  * Delete an existing session (or a reserved session on an unregistered
 619  * driver).
 620  */
 621 void
 622 crypto_freesession(crypto_session_t cses)
 623 {
 624         struct cryptocap *cap;
 625         void *ses;
 626         size_t ses_size;
 627         u_int32_t hid;
 628
 629         if (cses == NULL)
 630                 return;
 631
 632         CRYPTO_DRIVER_LOCK();
 633
 634         hid = crypto_ses2hid(cses);
 635         KASSERT(hid < crypto_drivers_num,
 636             ("bogus crypto_session %p hid %u", cses, hid));
 637         cap = &crypto_drivers[hid];
 638
 639         ses = cses->softc;
 640         ses_size = cap->cc_session_size;
 641
 642         if (cap->cc_sessions)
 643                 cap->cc_sessions--;
 644
 645         /* Call the driver cleanup routine, if available. */
 646         CRYPTODEV_FREESESSION(cap->cc_dev, cses);
 647
 648         explicit_bzero(ses, ses_size);
 649         free(ses, M_CRYPTO_DATA);
 650         uma_zfree(cryptoses_zone, cses);
 651
 652         if (cap->cc_flags & CRYPTOCAP_F_CLEANUP)
 653                 crypto_remove(cap);
 654
 655         CRYPTO_DRIVER_UNLOCK();
 656 }
 657
 658 /*
 659  * Return an unused driver id.  Used by drivers prior to registering
 660  * support for the algorithms they handle.
 661  */
 662 int32_t
 663 crypto_get_driverid(device_t dev, size_t sessionsize, int flags)
 664 {
 665         struct cryptocap *newdrv;
 666         int i;
 667
 668         if ((flags & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
 669                 printf("%s: no flags specified when registering driver\n",
 670                     device_get_nameunit(dev));
 671                 return -1;
 672         }
 673
 674         CRYPTO_DRIVER_LOCK();
 675
 676         for (i = 0; i < crypto_drivers_num; i++) {
 677                 if (crypto_drivers[i].cc_dev == NULL &&
 678                     (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0) {
 679                         break;
 680                 }
 681         }
 682
 683         /* Out of entries, allocate some more. */
 684         if (i == crypto_drivers_num) {
 685                 /* Be careful about wrap-around. */
 686                 if (2 * crypto_drivers_num <= crypto_drivers_num) {
 687                         CRYPTO_DRIVER_UNLOCK();
 688                         printf("crypto: driver count wraparound!\n");
 689                         return -1;
 690                 }
 691
 692                 newdrv = malloc(2 * crypto_drivers_num *
 693                     sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
 694                 if (newdrv == NULL) {
 695                         CRYPTO_DRIVER_UNLOCK();
 696                         printf("crypto: no space to expand driver table!\n");
 697                         return -1;
 698                 }
 699
 700                 bcopy(crypto_drivers, newdrv,
 701                     crypto_drivers_num * sizeof(struct cryptocap));
 702
 703                 crypto_drivers_num *= 2;
 704
 705                 free(crypto_drivers, M_CRYPTO_DATA);
 706                 crypto_drivers = newdrv;
 707         }
 708
 709         /* NB: state is zero'd on free */
 710         crypto_drivers[i].cc_sessions = 1;      /* Mark */
 711         crypto_drivers[i].cc_dev = dev;
 712         crypto_drivers[i].cc_flags = flags;
 713         crypto_drivers[i].cc_session_size = sessionsize;
 714         if (bootverbose)
 715                 printf("crypto: assign %s driver id %u, flags 0x%x\n",
 716                     device_get_nameunit(dev), i, flags);
 717
 718         CRYPTO_DRIVER_UNLOCK();
 719
 720         return i;
 721 }
 722
 723 /*
 724  * Lookup a driver by name.  We match against the full device
 725  * name and unit, and against just the name.  The latter gives
 726  * us a simple widlcarding by device name.  On success return the
 727  * driver/hardware identifier; otherwise return -1.
 728  */
 729 int
 730 crypto_find_driver(const char *match)
 731 {
 732         int i, len = strlen(match);
 733
 734         CRYPTO_DRIVER_LOCK();
 735         for (i = 0; i < crypto_drivers_num; i++) {
 736                 device_t dev = crypto_drivers[i].cc_dev;
 737                 if (dev == NULL ||
 738                     (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP))
 739                         continue;
 740                 if (strncmp(match, device_get_nameunit(dev), len) == 0 ||
 741                     strncmp(match, device_get_name(dev), len) == 0)
 742                         break;
 743         }
 744         CRYPTO_DRIVER_UNLOCK();
 745         return i < crypto_drivers_num ? i : -1;
 746 }
 747
 748 /*
 749  * Return the device_t for the specified driver or NULL
 750  * if the driver identifier is invalid.
 751  */
 752 device_t
 753 crypto_find_device_byhid(int hid)
 754 {
 755         struct cryptocap *cap = crypto_checkdriver(hid);
 756         return cap != NULL ? cap->cc_dev : NULL;
 757 }
 758
 759 /*
 760  * Return the device/driver capabilities.
 761  */
 762 int
 763 crypto_getcaps(int hid)
 764 {
 765         struct cryptocap *cap = crypto_checkdriver(hid);
 766         return cap != NULL ? cap->cc_flags : 0;
 767 }
 768
 769 /*
 770  * Register support for a key-related algorithm.  This routine
 771  * is called once for each algorithm supported a driver.
 772  */
 773 int
 774 crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags)
 775 {
 776         struct cryptocap *cap;
 777         int err;
 778
 779         CRYPTO_DRIVER_LOCK();
 780
 781         cap = crypto_checkdriver(driverid);
 782         if (cap != NULL &&
 783             (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) {
 784                 /*
 785                  * XXX Do some performance testing to determine placing.
 786                  * XXX We probably need an auxiliary data structure that
 787                  * XXX describes relative performances.
 788                  */
 789
 790                 cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
 791                 if (bootverbose)
 792                         printf("crypto: %s registers key alg %u flags %u\n"
 793                                 , device_get_nameunit(cap->cc_dev)
 794                                 , kalg
 795                                 , flags
 796                         );
 797                 err = 0;
 798         } else
 799                 err = EINVAL;
 800
 801         CRYPTO_DRIVER_UNLOCK();
 802         return err;
 803 }
 804
 805 /*
 806  * Register support for a non-key-related algorithm.  This routine
 807  * is called once for each such algorithm supported by a driver.
 808  */
 809 int
 810 crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen,
 811     u_int32_t flags)
 812 {
 813         struct cryptocap *cap;
 814         int err;
 815
 816         CRYPTO_DRIVER_LOCK();
 817
 818         cap = crypto_checkdriver(driverid);
 819         /* NB: algorithms are in the range [1..max] */
 820         if (cap != NULL &&
 821             (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) {
 822                 /*
 823                  * XXX Do some performance testing to determine placing.
 824                  * XXX We probably need an auxiliary data structure that
 825                  * XXX describes relative performances.
 826                  */
 827
 828                 cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
 829                 cap->cc_max_op_len[alg] = maxoplen;
 830                 if (bootverbose)
 831                         printf("crypto: %s registers alg %u flags %u maxoplen %u\n"
 832                                 , device_get_nameunit(cap->cc_dev)
 833                                 , alg
 834                                 , flags
 835                                 , maxoplen
 836                         );
 837                 cap->cc_sessions = 0;           /* Unmark */
 838                 err = 0;
 839         } else
 840                 err = EINVAL;
 841
 842         CRYPTO_DRIVER_UNLOCK();
 843         return err;
 844 }
 845
 846 static void
 847 driver_finis(struct cryptocap *cap)
 848 {
 849         u_int32_t ses, kops;
 850
 851         CRYPTO_DRIVER_ASSERT();
 852
 853         ses = cap->cc_sessions;
 854         kops = cap->cc_koperations;
 855         bzero(cap, sizeof(*cap));
 856         if (ses != 0 || kops != 0) {
 857                 /*
 858                  * If there are pending sessions,
 859                  * just mark as invalid.
 860                  */
 861                 cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
 862                 cap->cc_sessions = ses;
 863                 cap->cc_koperations = kops;
 864         }
 865 }
 866
 867 /*
 868  * Unregister a crypto driver. If there are pending sessions using it,
 869  * leave enough information around so that subsequent calls using those
 870  * sessions will correctly detect the driver has been unregistered and
 871  * reroute requests.
 872  */
 873 int
 874 crypto_unregister(u_int32_t driverid, int alg)
 875 {
 876         struct cryptocap *cap;
 877         int i, err;
 878
 879         CRYPTO_DRIVER_LOCK();
 880         cap = crypto_checkdriver(driverid);
 881         if (cap != NULL &&
 882             (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) &&
 883             cap->cc_alg[alg] != 0) {
 884                 cap->cc_alg[alg] = 0;
 885                 cap->cc_max_op_len[alg] = 0;
 886
 887                 /* Was this the last algorithm ? */
 888                 for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++)
 889                         if (cap->cc_alg[i] != 0)
 890                                 break;
 891
 892                 if (i == CRYPTO_ALGORITHM_MAX + 1)
 893                         driver_finis(cap);
 894                 err = 0;
 895         } else
 896                 err = EINVAL;
 897         CRYPTO_DRIVER_UNLOCK();
 898
 899         return err;
 900 }
 901
 902 /*
 903  * Unregister all algorithms associated with a crypto driver.
 904  * If there are pending sessions using it, leave enough information
 905  * around so that subsequent calls using those sessions will
 906  * correctly detect the driver has been unregistered and reroute
 907  * requests.
 908  */
 909 int
 910 crypto_unregister_all(u_int32_t driverid)
 911 {
 912         struct cryptocap *cap;
 913         int err;
 914
 915         CRYPTO_DRIVER_LOCK();
 916         cap = crypto_checkdriver(driverid);
 917         if (cap != NULL) {
 918                 driver_finis(cap);
 919                 err = 0;
 920         } else
 921                 err = EINVAL;
 922         CRYPTO_DRIVER_UNLOCK();
 923
 924         return err;
 925 }
 926
 927 /*
 928  * Clear blockage on a driver.  The what parameter indicates whether
 929  * the driver is now ready for cryptop's and/or cryptokop's.
 930  */
 931 int
 932 crypto_unblock(u_int32_t driverid, int what)
 933 {
 934         struct cryptocap *cap;
 935         int err;
 936
 937         CRYPTO_Q_LOCK();
 938         cap = crypto_checkdriver(driverid);
 939         if (cap != NULL) {
 940                 if (what & CRYPTO_SYMQ)
 941                         cap->cc_qblocked = 0;
 942                 if (what & CRYPTO_ASYMQ)
 943                         cap->cc_kqblocked = 0;
 944                 if (crp_sleep)
 945                         wakeup_one(&crp_q);
 946                 err = 0;
 947         } else
 948                 err = EINVAL;
 949         CRYPTO_Q_UNLOCK();
 950
 951         return err;
 952 }
 953
 954 /*
 955  * Add a crypto request to a queue, to be processed by the kernel thread.
 956  */
 957 int
 958 crypto_dispatch(struct cryptop *crp)
 959 {
 960         struct cryptocap *cap;
 961         u_int32_t hid;
 962         int result;
 963
 964         cryptostats.cs_ops++;
 965
 966 #ifdef CRYPTO_TIMING
 967         if (crypto_timing)
 968                 binuptime(&crp->crp_tstamp);
 969 #endif
 970
 971         crp->crp_retw_id = ((uintptr_t)crp->crp_session) % crypto_workers_num;
 972
 973         if (CRYPTOP_ASYNC(crp)) {
 974                 if (crp->crp_flags & CRYPTO_F_ASYNC_KEEPORDER) {
 975                         struct crypto_ret_worker *ret_worker;
 976
 977                         ret_worker = CRYPTO_RETW(crp->crp_retw_id);
 978
 979                         CRYPTO_RETW_LOCK(ret_worker);
 980                         crp->crp_seq = ret_worker->reorder_ops++;
 981                         CRYPTO_RETW_UNLOCK(ret_worker);
 982                 }
 983
 984                 TASK_INIT(&crp->crp_task, 0, crypto_task_invoke, crp);
 985                 taskqueue_enqueue(crypto_tq, &crp->crp_task);
 986                 return (0);
 987         }
 988
 989         if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) {
 990                 hid = crypto_ses2hid(crp->crp_session);
 991
 992                 /*
 993                  * Caller marked the request to be processed
 994                  * immediately; dispatch it directly to the
 995                  * driver unless the driver is currently blocked.
 996                  */
 997                 cap = crypto_checkdriver(hid);
 998                 /* Driver cannot disappeared when there is an active session. */
 999                 KASSERT(cap != NULL, ("%s: Driver disappeared.", __func__));
1000                 if (!cap->cc_qblocked) {
1001                         result = crypto_invoke(cap, crp, 0);
1002                         if (result != ERESTART)
1003                                 return (result);
1004                         /*
1005                          * The driver ran out of resources, put the request on
1006                          * the queue.
1007                          */
1008                 }
1009         }
1010         crypto_batch_enqueue(crp);
1011         return 0;
1012 }
1013
1014 void
1015 crypto_batch_enqueue(struct cryptop *crp)
1016 {
1017
1018         CRYPTO_Q_LOCK();
1019         TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
1020         if (crp_sleep)
1021                 wakeup_one(&crp_q);
1022         CRYPTO_Q_UNLOCK();
1023 }
1024
1025 /*
1026  * Add an asymetric crypto request to a queue,
1027  * to be processed by the kernel thread.
1028  */
1029 int
1030 crypto_kdispatch(struct cryptkop *krp)
1031 {
1032         int error;
1033
1034         cryptostats.cs_kops++;
1035
1036         error = crypto_kinvoke(krp, krp->krp_crid);
1037         if (error == ERESTART) {
1038                 CRYPTO_Q_LOCK();
1039                 TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next);
1040                 if (crp_sleep)
1041                         wakeup_one(&crp_q);
1042                 CRYPTO_Q_UNLOCK();
1043                 error = 0;
1044         }
1045         return error;
1046 }
1047
1048 /*
1049  * Verify a driver is suitable for the specified operation.
1050  */
1051 static __inline int
1052 kdriver_suitable(const struct cryptocap *cap, const struct cryptkop *krp)
1053 {
1054         return (cap->cc_kalg[krp->krp_op] & CRYPTO_ALG_FLAG_SUPPORTED) != 0;
1055 }
1056
1057 /*
1058  * Select a driver for an asym operation.  The driver must
1059  * support the necessary algorithm.  The caller can constrain
1060  * which device is selected with the flags parameter.  The
1061  * algorithm we use here is pretty stupid; just use the first
1062  * driver that supports the algorithms we need. If there are
1063  * multiple suitable drivers we choose the driver with the
1064  * fewest active operations.  We prefer hardware-backed
1065  * drivers to software ones when either may be used.
1066  */
1067 static struct cryptocap *
1068 crypto_select_kdriver(const struct cryptkop *krp, int flags)
1069 {
1070         struct cryptocap *cap, *best;
1071         int match, hid;
1072
1073         CRYPTO_DRIVER_ASSERT();
1074
1075         /*
1076          * Look first for hardware crypto devices if permitted.
1077          */
1078         if (flags & CRYPTOCAP_F_HARDWARE)
1079                 match = CRYPTOCAP_F_HARDWARE;
1080         else
1081                 match = CRYPTOCAP_F_SOFTWARE;
1082         best = NULL;
1083 again:
1084         for (hid = 0; hid < crypto_drivers_num; hid++) {
1085                 cap = &crypto_drivers[hid];
1086                 /*
1087                  * If it's not initialized, is in the process of
1088                  * going away, or is not appropriate (hardware
1089                  * or software based on match), then skip.
1090                  */
1091                 if (cap->cc_dev == NULL ||
1092                     (cap->cc_flags & CRYPTOCAP_F_CLEANUP) ||
1093                     (cap->cc_flags & match) == 0)
1094                         continue;
1095
1096                 /* verify all the algorithms are supported. */
1097                 if (kdriver_suitable(cap, krp)) {
1098                         if (best == NULL ||
1099                             cap->cc_koperations < best->cc_koperations)
1100                                 best = cap;
1101                 }
1102         }
1103         if (best != NULL)
1104                 return best;
1105         if (match == CRYPTOCAP_F_HARDWARE && (flags & CRYPTOCAP_F_SOFTWARE)) {
1106                 /* sort of an Algol 68-style for loop */
1107                 match = CRYPTOCAP_F_SOFTWARE;
1108                 goto again;
1109         }
1110         return best;
1111 }
1112
1113 /*
1114  * Dispatch an asymmetric crypto request.
1115  */
1116 static int
1117 crypto_kinvoke(struct cryptkop *krp, int crid)
1118 {
1119         struct cryptocap *cap = NULL;
1120         int error;
1121
1122         KASSERT(krp != NULL, ("%s: krp == NULL", __func__));
1123         KASSERT(krp->krp_callback != NULL,
1124             ("%s: krp->crp_callback == NULL", __func__));
1125
1126         CRYPTO_DRIVER_LOCK();
1127         if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
1128                 cap = crypto_checkdriver(crid);
1129                 if (cap != NULL) {
1130                         /*
1131                          * Driver present, it must support the necessary
1132                          * algorithm and, if s/w drivers are excluded,
1133                          * it must be registered as hardware-backed.
1134                          */
1135                         if (!kdriver_suitable(cap, krp) ||
1136                             (!crypto_devallowsoft &&
1137                              (cap->cc_flags & CRYPTOCAP_F_HARDWARE) == 0))
1138                                 cap = NULL;
1139                 }
1140         } else {
1141                 /*
1142                  * No requested driver; select based on crid flags.
1143                  */
1144                 if (!crypto_devallowsoft)       /* NB: disallow s/w drivers */
1145                         crid &= ~CRYPTOCAP_F_SOFTWARE;
1146                 cap = crypto_select_kdriver(krp, crid);
1147         }
1148         if (cap != NULL && !cap->cc_kqblocked) {
1149                 krp->krp_hid = cap - crypto_drivers;
1150                 cap->cc_koperations++;
1151                 CRYPTO_DRIVER_UNLOCK();
1152                 error = CRYPTODEV_KPROCESS(cap->cc_dev, krp, 0);
1153                 CRYPTO_DRIVER_LOCK();
1154                 if (error == ERESTART) {
1155                         cap->cc_koperations--;
1156                         CRYPTO_DRIVER_UNLOCK();
1157                         return (error);
1158                 }
1159         } else {
1160                 /*
1161                  * NB: cap is !NULL if device is blocked; in
1162                  *     that case return ERESTART so the operation
1163                  *     is resubmitted if possible.
1164                  */
1165                 error = (cap == NULL) ? ENODEV : ERESTART;
1166         }
1167         CRYPTO_DRIVER_UNLOCK();
1168
1169         if (error) {
1170                 krp->krp_status = error;
1171                 crypto_kdone(krp);
1172         }
1173         return 0;
1174 }
1175
1176 #ifdef CRYPTO_TIMING
1177 static void
1178 crypto_tstat(struct cryptotstat *ts, struct bintime *bt)
1179 {
1180         struct bintime now, delta;
1181         struct timespec t;
1182         uint64_t u;
1183
1184         binuptime(&now);
1185         u = now.frac;
1186         delta.frac = now.frac - bt->frac;
1187         delta.sec = now.sec - bt->sec;
1188         if (u < delta.frac)
1189                 delta.sec--;
1190         bintime2timespec(&delta, &t);
1191         timespecadd(&ts->acc, &t, &ts->acc);
1192         if (timespeccmp(&t, &ts->min, <))
1193                 ts->min = t;
1194         if (timespeccmp(&t, &ts->max, >))
1195                 ts->max = t;
1196         ts->count++;
1197
1198         *bt = now;
1199 }
1200 #endif
1201
1202 static void
1203 crypto_task_invoke(void *ctx, int pending)
1204 {
1205         struct cryptocap *cap;
1206         struct cryptop *crp;
1207         int hid, result;
1208
1209         crp = (struct cryptop *)ctx;
1210
1211         hid = crypto_ses2hid(crp->crp_session);
1212         cap = crypto_checkdriver(hid);
1213
1214         result = crypto_invoke(cap, crp, 0);
1215         if (result == ERESTART)
1216                 crypto_batch_enqueue(crp);
1217 }
1218
1219 /*
1220  * Dispatch a crypto request to the appropriate crypto devices.
1221  */
1222 static int
1223 crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint)
1224 {
1225
1226         KASSERT(crp != NULL, ("%s: crp == NULL", __func__));
1227         KASSERT(crp->crp_callback != NULL,
1228             ("%s: crp->crp_callback == NULL", __func__));
1229         KASSERT(crp->crp_desc != NULL, ("%s: crp->crp_desc == NULL", __func__));
1230
1231 #ifdef CRYPTO_TIMING
1232         if (crypto_timing)
1233                 crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp);
1234 #endif
1235         if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
1236                 struct cryptodesc *crd;
1237                 crypto_session_t nses;
1238
1239                 /*
1240                  * Driver has unregistered; migrate the session and return
1241                  * an error to the caller so they'll resubmit the op.
1242                  *
1243                  * XXX: What if there are more already queued requests for this
1244                  *      session?
1245                  */
1246                 crypto_freesession(crp->crp_session);
1247
1248                 for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next)
1249                         crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI);
1250
1251                 /* XXX propagate flags from initial session? */
1252                 if (crypto_newsession(&nses, &(crp->crp_desc->CRD_INI),
1253                     CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE) == 0)
1254                         crp->crp_session = nses;
1255
1256                 crp->crp_etype = EAGAIN;
1257                 crypto_done(crp);
1258                 return 0;
1259         } else {
1260                 /*
1261                  * Invoke the driver to process the request.
1262                  */
1263                 return CRYPTODEV_PROCESS(cap->cc_dev, crp, hint);
1264         }
1265 }
1266
1267 /*
1268  * Release a set of crypto descriptors.
1269  */
1270 void
1271 crypto_freereq(struct cryptop *crp)
1272 {
1273         struct cryptodesc *crd;
1274
1275         if (crp == NULL)
1276                 return;
1277
1278 #ifdef DIAGNOSTIC
1279         {
1280                 struct cryptop *crp2;
1281                 struct crypto_ret_worker *ret_worker;
1282
1283                 CRYPTO_Q_LOCK();
1284                 TAILQ_FOREACH(crp2, &crp_q, crp_next) {
1285                         KASSERT(crp2 != crp,
1286                             ("Freeing cryptop from the crypto queue (%p).",
1287                             crp));
1288                 }
1289                 CRYPTO_Q_UNLOCK();
1290
1291                 FOREACH_CRYPTO_RETW(ret_worker) {
1292                         CRYPTO_RETW_LOCK(ret_worker);
1293                         TAILQ_FOREACH(crp2, &ret_worker->crp_ret_q, crp_next) {
1294                                 KASSERT(crp2 != crp,
1295                                     ("Freeing cryptop from the return queue (%p).",
1296                                     crp));
1297                         }
1298                         CRYPTO_RETW_UNLOCK(ret_worker);
1299                 }
1300         }
1301 #endif
1302
1303         while ((crd = crp->crp_desc) != NULL) {
1304                 crp->crp_desc = crd->crd_next;
1305                 uma_zfree(cryptodesc_zone, crd);
1306         }
1307         uma_zfree(cryptop_zone, crp);
1308 }
1309
1310 /*
1311  * Acquire a set of crypto descriptors.
1312  */
1313 struct cryptop *
1314 crypto_getreq(int num)
1315 {
1316         struct cryptodesc *crd;
1317         struct cryptop *crp;
1318
1319         crp = uma_zalloc(cryptop_zone, M_NOWAIT|M_ZERO);
1320         if (crp != NULL) {
1321                 while (num--) {
1322                         crd = uma_zalloc(cryptodesc_zone, M_NOWAIT|M_ZERO);
1323                         if (crd == NULL) {
1324                                 crypto_freereq(crp);
1325                                 return NULL;
1326                         }
1327
1328                         crd->crd_next = crp->crp_desc;
1329                         crp->crp_desc = crd;
1330                 }
1331         }
1332         return crp;
1333 }
1334
1335 /*
1336  * Invoke the callback on behalf of the driver.
1337  */
1338 void
1339 crypto_done(struct cryptop *crp)
1340 {
1341         KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0,
1342                 ("crypto_done: op already done, flags 0x%x", crp->crp_flags));
1343         crp->crp_flags |= CRYPTO_F_DONE;
1344         if (crp->crp_etype != 0)
1345                 cryptostats.cs_errs++;
1346 #ifdef CRYPTO_TIMING
1347         if (crypto_timing)
1348                 crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp);
1349 #endif
1350         /*
1351          * CBIMM means unconditionally do the callback immediately;
1352          * CBIFSYNC means do the callback immediately only if the
1353          * operation was done synchronously.  Both are used to avoid
1354          * doing extraneous context switches; the latter is mostly
1355          * used with the software crypto driver.
1356          */
1357         if (!CRYPTOP_ASYNC_KEEPORDER(crp) &&
1358             ((crp->crp_flags & CRYPTO_F_CBIMM) ||
1359             ((crp->crp_flags & CRYPTO_F_CBIFSYNC) &&
1360              (crypto_ses2caps(crp->crp_session) & CRYPTOCAP_F_SYNC)))) {
1361                 /*
1362                  * Do the callback directly.  This is ok when the
1363                  * callback routine does very little (e.g. the
1364                  * /dev/crypto callback method just does a wakeup).
1365                  */
1366 #ifdef CRYPTO_TIMING
1367                 if (crypto_timing) {
1368                         /*
1369                          * NB: We must copy the timestamp before
1370                          * doing the callback as the cryptop is
1371                          * likely to be reclaimed.
1372                          */
1373                         struct bintime t = crp->crp_tstamp;
1374                         crypto_tstat(&cryptostats.cs_cb, &t);
1375                         crp->crp_callback(crp);
1376                         crypto_tstat(&cryptostats.cs_finis, &t);
1377                 } else
1378 #endif
1379                         crp->crp_callback(crp);
1380         } else {
1381                 struct crypto_ret_worker *ret_worker;
1382                 bool wake;
1383
1384                 ret_worker = CRYPTO_RETW(crp->crp_retw_id);
1385                 wake = false;
1386
1387                 /*
1388                  * Normal case; queue the callback for the thread.
1389                  */
1390                 CRYPTO_RETW_LOCK(ret_worker);
1391                 if (CRYPTOP_ASYNC_KEEPORDER(crp)) {
1392                         struct cryptop *tmp;
1393
1394                         TAILQ_FOREACH_REVERSE(tmp, &ret_worker->crp_ordered_ret_q,
1395                                         cryptop_q, crp_next) {
1396                                 if (CRYPTO_SEQ_GT(crp->crp_seq, tmp->crp_seq)) {
1397                                         TAILQ_INSERT_AFTER(&ret_worker->crp_ordered_ret_q,
1398                                                         tmp, crp, crp_next);
1399                                         break;
1400                                 }
1401                         }
1402                         if (tmp == NULL) {
1403                                 TAILQ_INSERT_HEAD(&ret_worker->crp_ordered_ret_q,
1404                                                 crp, crp_next);
1405                         }
1406
1407                         if (crp->crp_seq == ret_worker->reorder_cur_seq)
1408                                 wake = true;
1409                 }
1410                 else {
1411                         if (CRYPTO_RETW_EMPTY(ret_worker))
1412                                 wake = true;
1413
1414                         TAILQ_INSERT_TAIL(&ret_worker->crp_ret_q, crp, crp_next);
1415                 }
1416
1417                 if (wake)
1418                         wakeup_one(&ret_worker->crp_ret_q);     /* shared wait channel */
1419                 CRYPTO_RETW_UNLOCK(ret_worker);
1420         }
1421 }
1422
1423 /*
1424  * Invoke the callback on behalf of the driver.
1425  */
1426 void
1427 crypto_kdone(struct cryptkop *krp)
1428 {
1429         struct crypto_ret_worker *ret_worker;
1430         struct cryptocap *cap;
1431
1432         if (krp->krp_status != 0)
1433                 cryptostats.cs_kerrs++;
1434         CRYPTO_DRIVER_LOCK();
1435         /* XXX: What if driver is loaded in the meantime? */
1436         if (krp->krp_hid < crypto_drivers_num) {
1437                 cap = &crypto_drivers[krp->krp_hid];
1438                 KASSERT(cap->cc_koperations > 0, ("cc_koperations == 0"));
1439                 cap->cc_koperations--;
1440                 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP)
1441                         crypto_remove(cap);
1442         }
1443         CRYPTO_DRIVER_UNLOCK();
1444
1445         ret_worker = CRYPTO_RETW(0);
1446
1447         CRYPTO_RETW_LOCK(ret_worker);
1448         if (CRYPTO_RETW_EMPTY(ret_worker))
1449                 wakeup_one(&ret_worker->crp_ret_q);             /* shared wait channel */
1450         TAILQ_INSERT_TAIL(&ret_worker->crp_ret_kq, krp, krp_next);
1451         CRYPTO_RETW_UNLOCK(ret_worker);
1452 }
1453
1454 int
1455 crypto_getfeat(int *featp)
1456 {
1457         int hid, kalg, feat = 0;
1458
1459         CRYPTO_DRIVER_LOCK();
1460         for (hid = 0; hid < crypto_drivers_num; hid++) {
1461                 const struct cryptocap *cap = &crypto_drivers[hid];
1462
1463                 if ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) &&
1464                     !crypto_devallowsoft) {
1465                         continue;
1466                 }
1467                 for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++)
1468                         if (cap->cc_kalg[kalg] & CRYPTO_ALG_FLAG_SUPPORTED)
1469                                 feat |=  1 << kalg;
1470         }
1471         CRYPTO_DRIVER_UNLOCK();
1472         *featp = feat;
1473         return (0);
1474 }
1475
1476 /*
1477  * Terminate a thread at module unload.  The process that
1478  * initiated this is waiting for us to signal that we're gone;
1479  * wake it up and exit.  We use the driver table lock to insure
1480  * we don't do the wakeup before they're waiting.  There is no
1481  * race here because the waiter sleeps on the proc lock for the
1482  * thread so it gets notified at the right time because of an
1483  * extra wakeup that's done in exit1().
1484  */
1485 static void
1486 crypto_finis(void *chan)
1487 {
1488         CRYPTO_DRIVER_LOCK();
1489         wakeup_one(chan);
1490         CRYPTO_DRIVER_UNLOCK();
1491         kproc_exit(0);
1492 }
1493
1494 /*
1495  * Crypto thread, dispatches crypto requests.
1496  */
1497 static void
1498 crypto_proc(void)
1499 {
1500         struct cryptop *crp, *submit;
1501         struct cryptkop *krp;
1502         struct cryptocap *cap;
1503         u_int32_t hid;
1504         int result, hint;
1505
1506 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
1507         fpu_kern_thread(FPU_KERN_NORMAL);
1508 #endif
1509
1510         CRYPTO_Q_LOCK();
1511         for (;;) {
1512                 /*
1513                  * Find the first element in the queue that can be
1514                  * processed and look-ahead to see if multiple ops
1515                  * are ready for the same driver.
1516                  */
1517                 submit = NULL;
1518                 hint = 0;
1519                 TAILQ_FOREACH(crp, &crp_q, crp_next) {
1520                         hid = crypto_ses2hid(crp->crp_session);
1521                         cap = crypto_checkdriver(hid);
1522                         /*
1523                          * Driver cannot disappeared when there is an active
1524                          * session.
1525                          */
1526                         KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
1527                             __func__, __LINE__));
1528                         if (cap == NULL || cap->cc_dev == NULL) {
1529                                 /* Op needs to be migrated, process it. */
1530                                 if (submit == NULL)
1531                                         submit = crp;
1532                                 break;
1533                         }
1534                         if (!cap->cc_qblocked) {
1535                                 if (submit != NULL) {
1536                                         /*
1537                                          * We stop on finding another op,
1538                                          * regardless whether its for the same
1539                                          * driver or not.  We could keep
1540                                          * searching the queue but it might be
1541                                          * better to just use a per-driver
1542                                          * queue instead.
1543                                          */
1544                                         if (crypto_ses2hid(submit->crp_session) == hid)
1545                                                 hint = CRYPTO_HINT_MORE;
1546                                         break;
1547                                 } else {
1548                                         submit = crp;
1549                                         if ((submit->crp_flags & CRYPTO_F_BATCH) == 0)
1550                                                 break;
1551                                         /* keep scanning for more are q'd */
1552                                 }
1553                         }
1554                 }
1555                 if (submit != NULL) {
1556                         TAILQ_REMOVE(&crp_q, submit, crp_next);
1557                         hid = crypto_ses2hid(submit->crp_session);
1558                         cap = crypto_checkdriver(hid);
1559                         KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
1560                             __func__, __LINE__));
1561                         result = crypto_invoke(cap, submit, hint);
1562                         if (result == ERESTART) {
1563                                 /*
1564                                  * The driver ran out of resources, mark the
1565                                  * driver ``blocked'' for cryptop's and put
1566                                  * the request back in the queue.  It would
1567                                  * best to put the request back where we got
1568                                  * it but that's hard so for now we put it
1569                                  * at the front.  This should be ok; putting
1570                                  * it at the end does not work.
1571                                  */
1572                                 /* XXX validate sid again? */
1573                                 crypto_drivers[crypto_ses2hid(submit->crp_session)].cc_qblocked = 1;
1574                                 TAILQ_INSERT_HEAD(&crp_q, submit, crp_next);
1575                                 cryptostats.cs_blocks++;
1576                         }
1577                 }
1578
1579                 /* As above, but for key ops */
1580                 TAILQ_FOREACH(krp, &crp_kq, krp_next) {
1581                         cap = crypto_checkdriver(krp->krp_hid);
1582                         if (cap == NULL || cap->cc_dev == NULL) {
1583                                 /*
1584                                  * Operation needs to be migrated, invalidate
1585                                  * the assigned device so it will reselect a
1586                                  * new one below.  Propagate the original
1587                                  * crid selection flags if supplied.
1588                                  */
1589                                 krp->krp_hid = krp->krp_crid &
1590                                     (CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE);
1591                                 if (krp->krp_hid == 0)
1592                                         krp->krp_hid =
1593                                     CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE;
1594                                 break;
1595                         }
1596                         if (!cap->cc_kqblocked)
1597                                 break;
1598                 }
1599                 if (krp != NULL) {
1600                         TAILQ_REMOVE(&crp_kq, krp, krp_next);
1601                         result = crypto_kinvoke(krp, krp->krp_hid);
1602                         if (result == ERESTART) {
1603                                 /*
1604                                  * The driver ran out of resources, mark the
1605                                  * driver ``blocked'' for cryptkop's and put
1606                                  * the request back in the queue.  It would
1607                                  * best to put the request back where we got
1608                                  * it but that's hard so for now we put it
1609                                  * at the front.  This should be ok; putting
1610                                  * it at the end does not work.
1611                                  */
1612                                 /* XXX validate sid again? */
1613                                 crypto_drivers[krp->krp_hid].cc_kqblocked = 1;
1614                                 TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);
1615                                 cryptostats.cs_kblocks++;
1616                         }
1617                 }
1618
1619                 if (submit == NULL && krp == NULL) {
1620                         /*
1621                          * Nothing more to be processed.  Sleep until we're
1622                          * woken because there are more ops to process.
1623                          * This happens either by submission or by a driver
1624                          * becoming unblocked and notifying us through
1625                          * crypto_unblock.  Note that when we wakeup we
1626                          * start processing each queue again from the
1627                          * front. It's not clear that it's important to
1628                          * preserve this ordering since ops may finish
1629                          * out of order if dispatched to different devices
1630                          * and some become blocked while others do not.
1631                          */
1632                         crp_sleep = 1;
1633                         msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0);
1634                         crp_sleep = 0;
1635                         if (cryptoproc == NULL)
1636                                 break;
1637                         cryptostats.cs_intrs++;
1638                 }
1639         }
1640         CRYPTO_Q_UNLOCK();
1641
1642         crypto_finis(&crp_q);
1643 }
1644
1645 /*
1646  * Crypto returns thread, does callbacks for processed crypto requests.
1647  * Callbacks are done here, rather than in the crypto drivers, because
1648  * callbacks typically are expensive and would slow interrupt handling.
1649  */
1650 static void
1651 crypto_ret_proc(struct crypto_ret_worker *ret_worker)
1652 {
1653         struct cryptop *crpt;
1654         struct cryptkop *krpt;
1655
1656         CRYPTO_RETW_LOCK(ret_worker);
1657         for (;;) {
1658                 /* Harvest return q's for completed ops */
1659                 crpt = TAILQ_FIRST(&ret_worker->crp_ordered_ret_q);
1660                 if (crpt != NULL) {
1661                         if (crpt->crp_seq == ret_worker->reorder_cur_seq) {
1662                                 TAILQ_REMOVE(&ret_worker->crp_ordered_ret_q, crpt, crp_next);
1663                                 ret_worker->reorder_cur_seq++;
1664                         } else {
1665                                 crpt = NULL;
1666                         }
1667                 }
1668
1669                 if (crpt == NULL) {
1670                         crpt = TAILQ_FIRST(&ret_worker->crp_ret_q);
1671                         if (crpt != NULL)
1672                                 TAILQ_REMOVE(&ret_worker->crp_ret_q, crpt, crp_next);
1673                 }
1674
1675                 krpt = TAILQ_FIRST(&ret_worker->crp_ret_kq);
1676                 if (krpt != NULL)
1677                         TAILQ_REMOVE(&ret_worker->crp_ret_kq, krpt, krp_next);
1678
1679                 if (crpt != NULL || krpt != NULL) {
1680                         CRYPTO_RETW_UNLOCK(ret_worker);
1681                         /*
1682                          * Run callbacks unlocked.
1683                          */
1684                         if (crpt != NULL) {
1685 #ifdef CRYPTO_TIMING
1686                                 if (crypto_timing) {
1687                                         /*
1688                                          * NB: We must copy the timestamp before
1689                                          * doing the callback as the cryptop is
1690                                          * likely to be reclaimed.
1691                                          */
1692                                         struct bintime t = crpt->crp_tstamp;
1693                                         crypto_tstat(&cryptostats.cs_cb, &t);
1694                                         crpt->crp_callback(crpt);
1695                                         crypto_tstat(&cryptostats.cs_finis, &t);
1696                                 } else
1697 #endif
1698                                         crpt->crp_callback(crpt);
1699                         }
1700                         if (krpt != NULL)
1701                                 krpt->krp_callback(krpt);
1702                         CRYPTO_RETW_LOCK(ret_worker);
1703                 } else {
1704                         /*
1705                          * Nothing more to be processed.  Sleep until we're
1706                          * woken because there are more returns to process.
1707                          */
1708                         msleep(&ret_worker->crp_ret_q, &ret_worker->crypto_ret_mtx, PWAIT,
1709                                 "crypto_ret_wait", 0);
1710                         if (ret_worker->cryptoretproc == NULL)
1711                                 break;
1712                         cryptostats.cs_rets++;
1713                 }
1714         }
1715         CRYPTO_RETW_UNLOCK(ret_worker);
1716
1717         crypto_finis(&ret_worker->crp_ret_q);
1718 }
1719
1720 #ifdef DDB
1721 static void
1722 db_show_drivers(void)
1723 {
1724         int hid;
1725
1726         db_printf("%12s %4s %4s %8s %2s %2s\n"
1727                 , "Device"
1728                 , "Ses"
1729                 , "Kops"
1730                 , "Flags"
1731                 , "QB"
1732                 , "KB"
1733         );
1734         for (hid = 0; hid < crypto_drivers_num; hid++) {
1735                 const struct cryptocap *cap = &crypto_drivers[hid];
1736                 if (cap->cc_dev == NULL)
1737                         continue;
1738                 db_printf("%-12s %4u %4u %08x %2u %2u\n"
1739                     , device_get_nameunit(cap->cc_dev)
1740                     , cap->cc_sessions
1741                     , cap->cc_koperations
1742                     , cap->cc_flags
1743                     , cap->cc_qblocked
1744                     , cap->cc_kqblocked
1745                 );
1746         }
1747 }
1748
1749 DB_SHOW_COMMAND(crypto, db_show_crypto)
1750 {
1751         struct cryptop *crp;
1752         struct crypto_ret_worker *ret_worker;
1753
1754         db_show_drivers();
1755         db_printf("\n");
1756
1757         db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n",
1758             "HID", "Caps", "Ilen", "Olen", "Etype", "Flags",
1759             "Desc", "Callback");
1760         TAILQ_FOREACH(crp, &crp_q, crp_next) {
1761                 db_printf("%4u %08x %4u %4u %4u %04x %8p %8p\n"
1762                     , (int) crypto_ses2hid(crp->crp_session)
1763                     , (int) crypto_ses2caps(crp->crp_session)
1764                     , crp->crp_ilen, crp->crp_olen
1765                     , crp->crp_etype
1766                     , crp->crp_flags
1767                     , crp->crp_desc
1768                     , crp->crp_callback
1769                 );
1770         }
1771         FOREACH_CRYPTO_RETW(ret_worker) {
1772                 db_printf("\n%8s %4s %4s %4s %8s\n",
1773                     "ret_worker", "HID", "Etype", "Flags", "Callback");
1774                 if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) {
1775                         TAILQ_FOREACH(crp, &ret_worker->crp_ret_q, crp_next) {
1776                                 db_printf("%8td %4u %4u %04x %8p\n"
1777                                     , CRYPTO_RETW_ID(ret_worker)
1778                                     , (int) crypto_ses2hid(crp->crp_session)
1779                                     , crp->crp_etype
1780                                     , crp->crp_flags
1781                                     , crp->crp_callback
1782                                 );
1783                         }
1784                 }
1785         }
1786 }
1787
1788 DB_SHOW_COMMAND(kcrypto, db_show_kcrypto)
1789 {
1790         struct cryptkop *krp;
1791         struct crypto_ret_worker *ret_worker;
1792
1793         db_show_drivers();
1794         db_printf("\n");
1795
1796         db_printf("%4s %5s %4s %4s %8s %4s %8s\n",
1797             "Op", "Status", "#IP", "#OP", "CRID", "HID", "Callback");
1798         TAILQ_FOREACH(krp, &crp_kq, krp_next) {
1799                 db_printf("%4u %5u %4u %4u %08x %4u %8p\n"
1800                     , krp->krp_op
1801                     , krp->krp_status
1802                     , krp->krp_iparams, krp->krp_oparams
1803                     , krp->krp_crid, krp->krp_hid
1804                     , krp->krp_callback
1805                 );
1806         }
1807
1808         ret_worker = CRYPTO_RETW(0);
1809         if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) {
1810                 db_printf("%4s %5s %8s %4s %8s\n",
1811                     "Op", "Status", "CRID", "HID", "Callback");
1812                 TAILQ_FOREACH(krp, &ret_worker->crp_ret_kq, krp_next) {
1813                         db_printf("%4u %5u %08x %4u %8p\n"
1814                             , krp->krp_op
1815                             , krp->krp_status
1816                             , krp->krp_crid, krp->krp_hid
1817                             , krp->krp_callback
1818                         );
1819                 }
1820         }
1821 }
1822 #endif
1823
1824 int crypto_modevent(module_t mod, int type, void *unused);
1825
1826 /*
1827  * Initialization code, both for static and dynamic loading.
1828  * Note this is not invoked with the usual MODULE_DECLARE
1829  * mechanism but instead is listed as a dependency by the
1830  * cryptosoft driver.  This guarantees proper ordering of
1831  * calls on module load/unload.
1832  */
1833 int
1834 crypto_modevent(module_t mod, int type, void *unused)
1835 {
1836         int error = EINVAL;
1837
1838         switch (type) {
1839         case MOD_LOAD:
1840                 error = crypto_init();
1841                 if (error == 0 && bootverbose)
1842                         printf("crypto: <crypto core>\n");
1843                 break;
1844         case MOD_UNLOAD:
1845                 /*XXX disallow if active sessions */
1846                 error = 0;
1847                 crypto_destroy();
1848                 return 0;
1849         }
1850         return error;
1851 }
1852 MODULE_VERSION(crypto, 1);
1853 MODULE_DEPEND(crypto, zlib, 1, 1, 1);