sys/opencrypto/crypto.c

   1 /*      $OpenBSD: crypto.c,v 1.38 2002/06/11 11:14:29 beck Exp $        */
   2 /*-
   3  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
   4  *
   5  * This code was written by Angelos D. Keromytis in Athens, Greece, in
   6  * February 2000. Network Security Technologies Inc. (NSTI) kindly
   7  * supported the development of this code.
   8  *
   9  * Copyright (c) 2000, 2001 Angelos D. Keromytis
  10  *
  11  * Permission to use, copy, and modify this software with or without fee
  12  * is hereby granted, provided that this entire notice is included in
  13  * all source code copies of any software which is or includes a copy or
  14  * modification of this software.
  15  *
  16  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
  17  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
  18  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
  19  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
  20  * PURPOSE.
  21  */
  22
  23 #include <sys/cdefs.h>
  24 __FBSDID("$FreeBSD$");
  25
  26 #define CRYPTO_TIMING                           /* enable timing support */
  27
  28 #include <sys/param.h>
  29 #include <sys/systm.h>
  30 #include <sys/eventhandler.h>
  31 #include <sys/kernel.h>
  32 #include <sys/kthread.h>
  33 #include <sys/lock.h>
  34 #include <sys/module.h>
  35 #include <sys/mutex.h>
  36 #include <sys/malloc.h>
  37 #include <sys/proc.h>
  38 #include <sys/sysctl.h>
  39
  40 #include <vm/uma.h>
  41 #include <opencrypto/cryptodev.h>
  42 #include <opencrypto/xform.h>                   /* XXX for M_XDATA */
  43
  44 /*
  45  * Crypto drivers register themselves by allocating a slot in the
  46  * crypto_drivers table with crypto_get_driverid() and then registering
  47  * each algorithm they support with crypto_register() and crypto_kregister().
  48  */
  49 static  struct mtx crypto_drivers_mtx;          /* lock on driver table */
  50 #define CRYPTO_DRIVER_LOCK()    mtx_lock(&crypto_drivers_mtx)
  51 #define CRYPTO_DRIVER_UNLOCK()  mtx_unlock(&crypto_drivers_mtx)
  52 static  struct cryptocap *crypto_drivers = NULL;
  53 static  int crypto_drivers_num = 0;
  54
  55 /*
  56  * There are two queues for crypto requests; one for symmetric (e.g.
  57  * cipher) operations and one for asymmetric (e.g. MOD)operations.
  58  * A single mutex is used to lock access to both queues.  We could
  59  * have one per-queue but having one simplifies handling of block/unblock
  60  * operations.
  61  */
  62 static  TAILQ_HEAD(,cryptop) crp_q;             /* request queues */
  63 static  TAILQ_HEAD(,cryptkop) crp_kq;
  64 static  struct mtx crypto_q_mtx;
  65 #define CRYPTO_Q_LOCK()         mtx_lock(&crypto_q_mtx)
  66 #define CRYPTO_Q_UNLOCK()       mtx_unlock(&crypto_q_mtx)
  67
  68 /*
  69  * There are two queues for processing completed crypto requests; one
  70  * for the symmetric and one for the asymmetric ops.  We only need one
  71  * but have two to avoid type futzing (cryptop vs. cryptkop).  A single
  72  * mutex is used to lock access to both queues.  Note that this lock
  73  * must be separate from the lock on request queues to insure driver
  74  * callbacks don't generate lock order reversals.
  75  */
  76 static  TAILQ_HEAD(,cryptop) crp_ret_q;         /* callback queues */
  77 static  TAILQ_HEAD(,cryptkop) crp_ret_kq;
  78 static  struct mtx crypto_ret_q_mtx;
  79 #define CRYPTO_RETQ_LOCK()      mtx_lock(&crypto_ret_q_mtx)
  80 #define CRYPTO_RETQ_UNLOCK()    mtx_unlock(&crypto_ret_q_mtx)
  81
  82 static  uma_zone_t cryptop_zone;
  83 static  uma_zone_t cryptodesc_zone;
  84
  85 int     crypto_userasymcrypto = 1;      /* userland may do asym crypto reqs */
  86 SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW,
  87            &crypto_userasymcrypto, 0,
  88            "Enable/disable user-mode access to asymmetric crypto support");
  89 int     crypto_devallowsoft = 0;        /* only use hardware crypto for asym */
  90 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW,
  91            &crypto_devallowsoft, 0,
  92            "Enable/disable use of software asym crypto support");
  93
  94 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
  95
  96 static  void crypto_proc(void);
  97 static  struct proc *cryptoproc;
  98 static  void crypto_ret_proc(void);
  99 static  struct proc *cryptoretproc;
 100 static  void crypto_destroy(void);
 101 static  int crypto_invoke(struct cryptop *crp, int hint);
 102 static  int crypto_kinvoke(struct cryptkop *krp, int hint);
 103
 104 static  struct cryptostats cryptostats;
 105 SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats,
 106             cryptostats, "Crypto system statistics");
 107
 108 #ifdef CRYPTO_TIMING
 109 static  int crypto_timing = 0;
 110 SYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW,
 111            &crypto_timing, 0, "Enable/disable crypto timing support");
 112 #endif
 113
 114 static int
 115 crypto_init(void)
 116 {
 117         int error;
 118
 119         mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table",
 120                 MTX_DEF|MTX_QUIET);
 121
 122         TAILQ_INIT(&crp_q);
 123         TAILQ_INIT(&crp_kq);
 124         mtx_init(&crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF);
 125
 126         TAILQ_INIT(&crp_ret_q);
 127         TAILQ_INIT(&crp_ret_kq);
 128         mtx_init(&crypto_ret_q_mtx, "crypto", "crypto return queues", MTX_DEF);
 129
 130         cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop),
 131                                     0, 0, 0, 0,
 132                                     UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
 133         cryptodesc_zone = uma_zcreate("cryptodesc", sizeof (struct cryptodesc),
 134                                     0, 0, 0, 0,
 135                                     UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
 136         if (cryptodesc_zone == NULL || cryptop_zone == NULL) {
 137                 printf("crypto_init: cannot setup crypto zones\n");
 138                 error = ENOMEM;
 139                 goto bad;
 140         }
 141
 142         crypto_drivers_num = CRYPTO_DRIVERS_INITIAL;
 143         crypto_drivers = malloc(crypto_drivers_num *
 144             sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO);
 145         if (crypto_drivers == NULL) {
 146                 printf("crypto_init: cannot setup crypto drivers\n");
 147                 error = ENOMEM;
 148                 goto bad;
 149         }
 150
 151         error = kthread_create((void (*)(void *)) crypto_proc, NULL,
 152                     &cryptoproc, 0, 0, "crypto");
 153         if (error) {
 154                 printf("crypto_init: cannot start crypto thread; error %d",
 155                         error);
 156                 goto bad;
 157         }
 158
 159         error = kthread_create((void (*)(void *)) crypto_ret_proc, NULL,
 160                     &cryptoretproc, 0, 0, "crypto returns");
 161         if (error) {
 162                 printf("crypto_init: cannot start cryptoret thread; error %d",
 163                         error);
 164                 goto bad;
 165         }
 166         return 0;
 167 bad:
 168         crypto_destroy();
 169         return error;
 170 }
 171
 172 /*
 173  * Signal a crypto thread to terminate.  We use the driver
 174  * table lock to synchronize the sleep/wakeups so that we
 175  * are sure the threads have terminated before we release
 176  * the data structures they use.  See crypto_finis below
 177  * for the other half of this song-and-dance.
 178  */
 179 static void
 180 crypto_terminate(struct proc **pp, void *q)
 181 {
 182         struct proc *p;
 183
 184         mtx_assert(&crypto_drivers_mtx, MA_OWNED);
 185         p = *pp;
 186         *pp = NULL;
 187         if (p) {
 188                 wakeup_one(q);
 189                 PROC_LOCK(p);           /* NB: insure we don't miss wakeup */
 190                 CRYPTO_DRIVER_UNLOCK(); /* let crypto_finis progress */
 191                 msleep(p, &p->p_mtx, PWAIT, "crypto_destroy", 0);
 192                 PROC_UNLOCK(p);
 193                 CRYPTO_DRIVER_LOCK();
 194         }
 195 }
 196
 197 static void
 198 crypto_destroy(void)
 199 {
 200         /*
 201          * Terminate any crypto threads.
 202          */
 203         CRYPTO_DRIVER_LOCK();
 204         crypto_terminate(&cryptoproc, &crp_q);
 205         crypto_terminate(&cryptoretproc, &crp_ret_q);
 206         CRYPTO_DRIVER_UNLOCK();
 207
 208         /* XXX flush queues??? */
 209
 210         /*
 211          * Reclaim dynamically allocated resources.
 212          */
 213         if (crypto_drivers != NULL)
 214                 free(crypto_drivers, M_CRYPTO_DATA);
 215
 216         if (cryptodesc_zone != NULL)
 217                 uma_zdestroy(cryptodesc_zone);
 218         if (cryptop_zone != NULL)
 219                 uma_zdestroy(cryptop_zone);
 220         mtx_destroy(&crypto_q_mtx);
 221         mtx_destroy(&crypto_ret_q_mtx);
 222         mtx_destroy(&crypto_drivers_mtx);
 223 }
 224
 225 /*
 226  * Initialization code, both for static and dynamic loading.
 227  */
 228 static int
 229 crypto_modevent(module_t mod, int type, void *unused)
 230 {
 231         int error = EINVAL;
 232
 233         switch (type) {
 234         case MOD_LOAD:
 235                 error = crypto_init();
 236                 if (error == 0 && bootverbose)
 237                         printf("crypto: <crypto core>\n");
 238                 break;
 239         case MOD_UNLOAD:
 240                 /*XXX disallow if active sessions */
 241                 error = 0;
 242                 crypto_destroy();
 243                 return 0;
 244         }
 245         return error;
 246 }
 247
 248 static moduledata_t crypto_mod = {
 249         "crypto",
 250         crypto_modevent,
 251         0
 252 };
 253 MODULE_VERSION(crypto, 1);
 254 DECLARE_MODULE(crypto, crypto_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST);
 255 MODULE_DEPEND(crypto, zlib, 1, 1, 1);
 256
 257 /*
 258  * Create a new session.
 259  */
 260 int
 261 crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int hard)
 262 {
 263         struct cryptoini *cr;
 264         u_int32_t hid, lid;
 265         int err = EINVAL;
 266
 267         CRYPTO_DRIVER_LOCK();
 268
 269         if (crypto_drivers == NULL)
 270                 goto done;
 271
 272         /*
 273          * The algorithm we use here is pretty stupid; just use the
 274          * first driver that supports all the algorithms we need.
 275          *
 276          * XXX We need more smarts here (in real life too, but that's
 277          * XXX another story altogether).
 278          */
 279
 280         for (hid = 0; hid < crypto_drivers_num; hid++) {
 281                 struct cryptocap *cap = &crypto_drivers[hid];
 282                 /*
 283                  * If it's not initialized or has remaining sessions
 284                  * referencing it, skip.
 285                  */
 286                 if (cap->cc_newsession == NULL ||
 287                     (cap->cc_flags & CRYPTOCAP_F_CLEANUP))
 288                         continue;
 289
 290                 /* Hardware required -- ignore software drivers. */
 291                 if (hard > 0 && (cap->cc_flags & CRYPTOCAP_F_SOFTWARE))
 292                         continue;
 293                 /* Software required -- ignore hardware drivers. */
 294                 if (hard < 0 && (cap->cc_flags & CRYPTOCAP_F_SOFTWARE) == 0)
 295                         continue;
 296
 297                 /* See if all the algorithms are supported. */
 298                 for (cr = cri; cr; cr = cr->cri_next)
 299                         if (cap->cc_alg[cr->cri_alg] == 0)
 300                                 break;
 301
 302                 if (cr == NULL) {
 303                         /* Ok, all algorithms are supported. */
 304
 305                         /*
 306                          * Can't do everything in one session.
 307                          *
 308                          * XXX Fix this. We need to inject a "virtual" session layer right
 309                          * XXX about here.
 310                          */
 311
 312                         /* Call the driver initialization routine. */
 313                         lid = hid;              /* Pass the driver ID. */
 314                         err = (*cap->cc_newsession)(cap->cc_arg, &lid, cri);
 315                         if (err == 0) {
 316                                 /* XXX assert (hid &~ 0xffffff) == 0 */
 317                                 /* XXX assert (cap->cc_flags &~ 0xff) == 0 */
 318                                 (*sid) = ((cap->cc_flags & 0xff) << 24) | hid;
 319                                 (*sid) <<= 32;
 320                                 (*sid) |= (lid & 0xffffffff);
 321                                 cap->cc_sessions++;
 322                         }
 323                         break;
 324                 }
 325         }
 326 done:
 327         CRYPTO_DRIVER_UNLOCK();
 328         return err;
 329 }
 330
 331 /*
 332  * Delete an existing session (or a reserved session on an unregistered
 333  * driver).
 334  */
 335 int
 336 crypto_freesession(u_int64_t sid)
 337 {
 338         u_int32_t hid;
 339         int err;
 340
 341         CRYPTO_DRIVER_LOCK();
 342
 343         if (crypto_drivers == NULL) {
 344                 err = EINVAL;
 345                 goto done;
 346         }
 347
 348         /* Determine two IDs. */
 349         hid = CRYPTO_SESID2HID(sid);
 350
 351         if (hid >= crypto_drivers_num) {
 352                 err = ENOENT;
 353                 goto done;
 354         }
 355
 356         if (crypto_drivers[hid].cc_sessions)
 357                 crypto_drivers[hid].cc_sessions--;
 358
 359         /* Call the driver cleanup routine, if available. */
 360         if (crypto_drivers[hid].cc_freesession)
 361                 err = crypto_drivers[hid].cc_freesession(
 362                                 crypto_drivers[hid].cc_arg, sid);
 363         else
 364                 err = 0;
 365
 366         /*
 367          * If this was the last session of a driver marked as invalid,
 368          * make the entry available for reuse.
 369          */
 370         if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP) &&
 371             crypto_drivers[hid].cc_sessions == 0)
 372                 bzero(&crypto_drivers[hid], sizeof(struct cryptocap));
 373
 374 done:
 375         CRYPTO_DRIVER_UNLOCK();
 376         return err;
 377 }
 378
 379 /*
 380  * Return an unused driver id.  Used by drivers prior to registering
 381  * support for the algorithms they handle.
 382  */
 383 int32_t
 384 crypto_get_driverid(u_int32_t flags)
 385 {
 386         struct cryptocap *newdrv;
 387         int i;
 388
 389         CRYPTO_DRIVER_LOCK();
 390
 391         for (i = 0; i < crypto_drivers_num; i++)
 392                 if (crypto_drivers[i].cc_process == NULL &&
 393                     (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0 &&
 394                     crypto_drivers[i].cc_sessions == 0)
 395                         break;
 396
 397         /* Out of entries, allocate some more. */
 398         if (i == crypto_drivers_num) {
 399                 /* Be careful about wrap-around. */
 400                 if (2 * crypto_drivers_num <= crypto_drivers_num) {
 401                         CRYPTO_DRIVER_UNLOCK();
 402                         printf("crypto: driver count wraparound!\n");
 403                         return -1;
 404                 }
 405
 406                 newdrv = malloc(2 * crypto_drivers_num *
 407                     sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
 408                 if (newdrv == NULL) {
 409                         CRYPTO_DRIVER_UNLOCK();
 410                         printf("crypto: no space to expand driver table!\n");
 411                         return -1;
 412                 }
 413
 414                 bcopy(crypto_drivers, newdrv,
 415                     crypto_drivers_num * sizeof(struct cryptocap));
 416
 417                 crypto_drivers_num *= 2;
 418
 419                 free(crypto_drivers, M_CRYPTO_DATA);
 420                 crypto_drivers = newdrv;
 421         }
 422
 423         /* NB: state is zero'd on free */
 424         crypto_drivers[i].cc_sessions = 1;      /* Mark */
 425         crypto_drivers[i].cc_flags = flags;
 426         if (bootverbose)
 427                 printf("crypto: assign driver %u, flags %u\n", i, flags);
 428
 429         CRYPTO_DRIVER_UNLOCK();
 430
 431         return i;
 432 }
 433
 434 static struct cryptocap *
 435 crypto_checkdriver(u_int32_t hid)
 436 {
 437         if (crypto_drivers == NULL)
 438                 return NULL;
 439         return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]);
 440 }
 441
 442 /*
 443  * Register support for a key-related algorithm.  This routine
 444  * is called once for each algorithm supported a driver.
 445  */
 446 int
 447 crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags,
 448     int (*kprocess)(void*, struct cryptkop *, int),
 449     void *karg)
 450 {
 451         struct cryptocap *cap;
 452         int err;
 453
 454         CRYPTO_DRIVER_LOCK();
 455
 456         cap = crypto_checkdriver(driverid);
 457         if (cap != NULL &&
 458             (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) {
 459                 /*
 460                  * XXX Do some performance testing to determine placing.
 461                  * XXX We probably need an auxiliary data structure that
 462                  * XXX describes relative performances.
 463                  */
 464
 465                 cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
 466                 if (bootverbose)
 467                         printf("crypto: driver %u registers key alg %u flags %u\n"
 468                                 , driverid
 469                                 , kalg
 470                                 , flags
 471                         );
 472
 473                 if (cap->cc_kprocess == NULL) {
 474                         cap->cc_karg = karg;
 475                         cap->cc_kprocess = kprocess;
 476                 }
 477                 err = 0;
 478         } else
 479                 err = EINVAL;
 480
 481         CRYPTO_DRIVER_UNLOCK();
 482         return err;
 483 }
 484
 485 /*
 486  * Register support for a non-key-related algorithm.  This routine
 487  * is called once for each such algorithm supported by a driver.
 488  */
 489 int
 490 crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen,
 491     u_int32_t flags,
 492     int (*newses)(void*, u_int32_t*, struct cryptoini*),
 493     int (*freeses)(void*, u_int64_t),
 494     int (*process)(void*, struct cryptop *, int),
 495     void *arg)
 496 {
 497         struct cryptocap *cap;
 498         int err;
 499
 500         CRYPTO_DRIVER_LOCK();
 501
 502         cap = crypto_checkdriver(driverid);
 503         /* NB: algorithms are in the range [1..max] */
 504         if (cap != NULL &&
 505             (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) {
 506                 /*
 507                  * XXX Do some performance testing to determine placing.
 508                  * XXX We probably need an auxiliary data structure that
 509                  * XXX describes relative performances.
 510                  */
 511
 512                 cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
 513                 cap->cc_max_op_len[alg] = maxoplen;
 514                 if (bootverbose)
 515                         printf("crypto: driver %u registers alg %u flags %u maxoplen %u\n"
 516                                 , driverid
 517                                 , alg
 518                                 , flags
 519                                 , maxoplen
 520                         );
 521
 522                 if (cap->cc_process == NULL) {
 523                         cap->cc_arg = arg;
 524                         cap->cc_newsession = newses;
 525                         cap->cc_process = process;
 526                         cap->cc_freesession = freeses;
 527                         cap->cc_sessions = 0;           /* Unmark */
 528                 }
 529                 err = 0;
 530         } else
 531                 err = EINVAL;
 532
 533         CRYPTO_DRIVER_UNLOCK();
 534         return err;
 535 }
 536
 537 /*
 538  * Unregister a crypto driver. If there are pending sessions using it,
 539  * leave enough information around so that subsequent calls using those
 540  * sessions will correctly detect the driver has been unregistered and
 541  * reroute requests.
 542  */
 543 int
 544 crypto_unregister(u_int32_t driverid, int alg)
 545 {
 546         int i, err;
 547         u_int32_t ses;
 548         struct cryptocap *cap;
 549
 550         CRYPTO_DRIVER_LOCK();
 551
 552         cap = crypto_checkdriver(driverid);
 553         if (cap != NULL &&
 554             (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) &&
 555             cap->cc_alg[alg] != 0) {
 556                 cap->cc_alg[alg] = 0;
 557                 cap->cc_max_op_len[alg] = 0;
 558
 559                 /* Was this the last algorithm ? */
 560                 for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++)
 561                         if (cap->cc_alg[i] != 0)
 562                                 break;
 563
 564                 if (i == CRYPTO_ALGORITHM_MAX + 1) {
 565                         ses = cap->cc_sessions;
 566                         bzero(cap, sizeof(struct cryptocap));
 567                         if (ses != 0) {
 568                                 /*
 569                                  * If there are pending sessions, just mark as invalid.
 570                                  */
 571                                 cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
 572                                 cap->cc_sessions = ses;
 573                         }
 574                 }
 575                 err = 0;
 576         } else
 577                 err = EINVAL;
 578
 579         CRYPTO_DRIVER_UNLOCK();
 580         return err;
 581 }
 582
 583 /*
 584  * Unregister all algorithms associated with a crypto driver.
 585  * If there are pending sessions using it, leave enough information
 586  * around so that subsequent calls using those sessions will
 587  * correctly detect the driver has been unregistered and reroute
 588  * requests.
 589  */
 590 int
 591 crypto_unregister_all(u_int32_t driverid)
 592 {
 593         int i, err;
 594         u_int32_t ses;
 595         struct cryptocap *cap;
 596
 597         CRYPTO_DRIVER_LOCK();
 598
 599         cap = crypto_checkdriver(driverid);
 600         if (cap != NULL) {
 601                 for (i = CRYPTO_ALGORITHM_MIN; i <= CRYPTO_ALGORITHM_MAX; i++) {
 602                         cap->cc_alg[i] = 0;
 603                         cap->cc_max_op_len[i] = 0;
 604                 }
 605                 ses = cap->cc_sessions;
 606                 bzero(cap, sizeof(struct cryptocap));
 607                 if (ses != 0) {
 608                         /*
 609                          * If there are pending sessions, just mark as invalid.
 610                          */
 611                         cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
 612                         cap->cc_sessions = ses;
 613                 }
 614                 err = 0;
 615         } else
 616                 err = EINVAL;
 617
 618         CRYPTO_DRIVER_UNLOCK();
 619         return err;
 620 }
 621
 622 /*
 623  * Clear blockage on a driver.  The what parameter indicates whether
 624  * the driver is now ready for cryptop's and/or cryptokop's.
 625  */
 626 int
 627 crypto_unblock(u_int32_t driverid, int what)
 628 {
 629         struct cryptocap *cap;
 630         int needwakeup, err;
 631
 632         CRYPTO_Q_LOCK();
 633         cap = crypto_checkdriver(driverid);
 634         if (cap != NULL) {
 635                 needwakeup = 0;
 636                 if (what & CRYPTO_SYMQ) {
 637                         needwakeup |= cap->cc_qblocked;
 638                         cap->cc_qblocked = 0;
 639                 }
 640                 if (what & CRYPTO_ASYMQ) {
 641                         needwakeup |= cap->cc_kqblocked;
 642                         cap->cc_kqblocked = 0;
 643                 }
 644                 if (needwakeup)
 645                         wakeup_one(&crp_q);
 646                 err = 0;
 647         } else
 648                 err = EINVAL;
 649         CRYPTO_Q_UNLOCK();
 650
 651         return err;
 652 }
 653
 654 /*
 655  * Add a crypto request to a queue, to be processed by the kernel thread.
 656  */
 657 int
 658 crypto_dispatch(struct cryptop *crp)
 659 {
 660         u_int32_t hid = CRYPTO_SESID2HID(crp->crp_sid);
 661         int result;
 662
 663         cryptostats.cs_ops++;
 664
 665 #ifdef CRYPTO_TIMING
 666         if (crypto_timing)
 667                 binuptime(&crp->crp_tstamp);
 668 #endif
 669
 670         CRYPTO_Q_LOCK();
 671         if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) {
 672                 struct cryptocap *cap;
 673                 /*
 674                  * Caller marked the request to be processed
 675                  * immediately; dispatch it directly to the
 676                  * driver unless the driver is currently blocked.
 677                  */
 678                 cap = crypto_checkdriver(hid);
 679                 if (cap && !cap->cc_qblocked) {
 680                         result = crypto_invoke(crp, 0);
 681                         if (result == ERESTART) {
 682                                 /*
 683                                  * The driver ran out of resources, mark the
 684                                  * driver ``blocked'' for cryptop's and put
 685                                  * the request on the queue.
 686                                  *
 687                                  * XXX ops are placed at the tail so their
 688                                  * order is preserved but this can place them
 689                                  * behind batch'd ops.
 690                                  */
 691                                 crypto_drivers[hid].cc_qblocked = 1;
 692                                 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
 693                                 cryptostats.cs_blocks++;
 694                                 result = 0;
 695                         }
 696                 } else {
 697                         /*
 698                          * The driver is blocked, just queue the op until
 699                          * it unblocks and the kernel thread gets kicked.
 700                          */
 701                         TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
 702                         result = 0;
 703                 }
 704         } else {
 705                 int wasempty;
 706                 /*
 707                  * Caller marked the request as ``ok to delay'';
 708                  * queue it for the dispatch thread.  This is desirable
 709                  * when the operation is low priority and/or suitable
 710                  * for batching.
 711                  */
 712                 wasempty = TAILQ_EMPTY(&crp_q);
 713                 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
 714                 if (wasempty)
 715                         wakeup_one(&crp_q);
 716                 result = 0;
 717         }
 718         CRYPTO_Q_UNLOCK();
 719
 720         return result;
 721 }
 722
 723 /*
 724  * Add an asymetric crypto request to a queue,
 725  * to be processed by the kernel thread.
 726  */
 727 int
 728 crypto_kdispatch(struct cryptkop *krp)
 729 {
 730         struct cryptocap *cap;
 731         int result;
 732
 733         cryptostats.cs_kops++;
 734
 735         CRYPTO_Q_LOCK();
 736         cap = crypto_checkdriver(krp->krp_hid);
 737         if (cap && !cap->cc_kqblocked) {
 738                 result = crypto_kinvoke(krp, 0);
 739                 if (result == ERESTART) {
 740                         /*
 741                          * The driver ran out of resources, mark the
 742                          * driver ``blocked'' for cryptkop's and put
 743                          * the request back in the queue.  It would
 744                          * best to put the request back where we got
 745                          * it but that's hard so for now we put it
 746                          * at the front.  This should be ok; putting
 747                          * it at the end does not work.
 748                          */
 749                         crypto_drivers[krp->krp_hid].cc_kqblocked = 1;
 750                         TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next);
 751                         cryptostats.cs_kblocks++;
 752                 }
 753         } else {
 754                 /*
 755                  * The driver is blocked, just queue the op until
 756                  * it unblocks and the kernel thread gets kicked.
 757                  */
 758                 TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next);
 759                 result = 0;
 760         }
 761         CRYPTO_Q_UNLOCK();
 762
 763         return result;
 764 }
 765
 766 /*
 767  * Dispatch an assymetric crypto request to the appropriate crypto devices.
 768  */
 769 static int
 770 crypto_kinvoke(struct cryptkop *krp, int hint)
 771 {
 772         u_int32_t hid;
 773         int error;
 774
 775         mtx_assert(&crypto_q_mtx, MA_OWNED);
 776
 777         /* Sanity checks. */
 778         if (krp == NULL)
 779                 return EINVAL;
 780         if (krp->krp_callback == NULL) {
 781                 free(krp, M_XDATA);             /* XXX allocated in cryptodev */
 782                 return EINVAL;
 783         }
 784
 785         for (hid = 0; hid < crypto_drivers_num; hid++) {
 786                 if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) &&
 787                     !crypto_devallowsoft)
 788                         continue;
 789                 if (crypto_drivers[hid].cc_kprocess == NULL)
 790                         continue;
 791                 if ((crypto_drivers[hid].cc_kalg[krp->krp_op] &
 792                     CRYPTO_ALG_FLAG_SUPPORTED) == 0)
 793                         continue;
 794                 break;
 795         }
 796         if (hid < crypto_drivers_num) {
 797                 krp->krp_hid = hid;
 798                 error = crypto_drivers[hid].cc_kprocess(
 799                                 crypto_drivers[hid].cc_karg, krp, hint);
 800         } else
 801                 error = ENODEV;
 802
 803         if (error) {
 804                 krp->krp_status = error;
 805                 crypto_kdone(krp);
 806         }
 807         return 0;
 808 }
 809
 810 #ifdef CRYPTO_TIMING
 811 static void
 812 crypto_tstat(struct cryptotstat *ts, struct bintime *bt)
 813 {
 814         struct bintime now, delta;
 815         struct timespec t;
 816         uint64_t u;
 817
 818         binuptime(&now);
 819         u = now.frac;
 820         delta.frac = now.frac - bt->frac;
 821         delta.sec = now.sec - bt->sec;
 822         if (u < delta.frac)
 823                 delta.sec--;
 824         bintime2timespec(&delta, &t);
 825         timespecadd(&ts->acc, &t);
 826         if (timespeccmp(&t, &ts->min, <))
 827                 ts->min = t;
 828         if (timespeccmp(&t, &ts->max, >))
 829                 ts->max = t;
 830         ts->count++;
 831
 832         *bt = now;
 833 }
 834 #endif
 835
 836 /*
 837  * Dispatch a crypto request to the appropriate crypto devices.
 838  */
 839 static int
 840 crypto_invoke(struct cryptop *crp, int hint)
 841 {
 842         u_int32_t hid;
 843         int (*process)(void*, struct cryptop *, int);
 844
 845 #ifdef CRYPTO_TIMING
 846         if (crypto_timing)
 847                 crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp);
 848 #endif
 849         /* Sanity checks. */
 850         if (crp == NULL)
 851                 return EINVAL;
 852         if (crp->crp_callback == NULL) {
 853                 crypto_freereq(crp);
 854                 return EINVAL;
 855         }
 856         if (crp->crp_desc == NULL) {
 857                 crp->crp_etype = EINVAL;
 858                 crypto_done(crp);
 859                 return 0;
 860         }
 861
 862         hid = CRYPTO_SESID2HID(crp->crp_sid);
 863         if (hid < crypto_drivers_num) {
 864                 if (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP)
 865                         crypto_freesession(crp->crp_sid);
 866                 process = crypto_drivers[hid].cc_process;
 867         } else {
 868                 process = NULL;
 869         }
 870
 871         if (process == NULL) {
 872                 struct cryptodesc *crd;
 873                 u_int64_t nid;
 874
 875                 /*
 876                  * Driver has unregistered; migrate the session and return
 877                  * an error to the caller so they'll resubmit the op.
 878                  */
 879                 for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next)
 880                         crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI);
 881
 882                 if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI), 0) == 0)
 883                         crp->crp_sid = nid;
 884
 885                 crp->crp_etype = EAGAIN;
 886                 crypto_done(crp);
 887                 return 0;
 888         } else {
 889                 /*
 890                  * Invoke the driver to process the request.
 891                  */
 892                 return (*process)(crypto_drivers[hid].cc_arg, crp, hint);
 893         }
 894 }
 895
 896 /*
 897  * Release a set of crypto descriptors.
 898  */
 899 void
 900 crypto_freereq(struct cryptop *crp)
 901 {
 902         struct cryptodesc *crd;
 903
 904         if (crp == NULL)
 905                 return;
 906
 907         while ((crd = crp->crp_desc) != NULL) {
 908                 crp->crp_desc = crd->crd_next;
 909                 uma_zfree(cryptodesc_zone, crd);
 910         }
 911
 912         uma_zfree(cryptop_zone, crp);
 913 }
 914
 915 /*
 916  * Acquire a set of crypto descriptors.
 917  */
 918 struct cryptop *
 919 crypto_getreq(int num)
 920 {
 921         struct cryptodesc *crd;
 922         struct cryptop *crp;
 923
 924         crp = uma_zalloc(cryptop_zone, M_NOWAIT|M_ZERO);
 925         if (crp != NULL) {
 926                 while (num--) {
 927                         crd = uma_zalloc(cryptodesc_zone, M_NOWAIT|M_ZERO);
 928                         if (crd == NULL) {
 929                                 crypto_freereq(crp);
 930                                 return NULL;
 931                         }
 932
 933                         crd->crd_next = crp->crp_desc;
 934                         crp->crp_desc = crd;
 935                 }
 936         }
 937         return crp;
 938 }
 939
 940 /*
 941  * Invoke the callback on behalf of the driver.
 942  */
 943 void
 944 crypto_done(struct cryptop *crp)
 945 {
 946         KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0,
 947                 ("crypto_done: op already done, flags 0x%x", crp->crp_flags));
 948         crp->crp_flags |= CRYPTO_F_DONE;
 949         if (crp->crp_etype != 0)
 950                 cryptostats.cs_errs++;
 951 #ifdef CRYPTO_TIMING
 952         if (crypto_timing)
 953                 crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp);
 954 #endif
 955         /*
 956          * CBIMM means unconditionally do the callback immediately;
 957          * CBIFSYNC means do the callback immediately only if the
 958          * operation was done synchronously.  Both are used to avoid
 959          * doing extraneous context switches; the latter is mostly
 960          * used with the software crypto driver.
 961          */
 962         if ((crp->crp_flags & CRYPTO_F_CBIMM) ||
 963             ((crp->crp_flags & CRYPTO_F_CBIFSYNC) &&
 964              (CRYPTO_SESID2CAPS(crp->crp_sid) & CRYPTOCAP_F_SYNC))) {
 965                 /*
 966                  * Do the callback directly.  This is ok when the
 967                  * callback routine does very little (e.g. the
 968                  * /dev/crypto callback method just does a wakeup).
 969                  */
 970 #ifdef CRYPTO_TIMING
 971                 if (crypto_timing) {
 972                         /*
 973                          * NB: We must copy the timestamp before
 974                          * doing the callback as the cryptop is
 975                          * likely to be reclaimed.
 976                          */
 977                         struct bintime t = crp->crp_tstamp;
 978                         crypto_tstat(&cryptostats.cs_cb, &t);
 979                         crp->crp_callback(crp);
 980                         crypto_tstat(&cryptostats.cs_finis, &t);
 981                 } else
 982 #endif
 983                         crp->crp_callback(crp);
 984         } else {
 985                 int wasempty;
 986                 /*
 987                  * Normal case; queue the callback for the thread.
 988                  */
 989                 CRYPTO_RETQ_LOCK();
 990                 wasempty = TAILQ_EMPTY(&crp_ret_q);
 991                 TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next);
 992
 993                 if (wasempty)
 994                         wakeup_one(&crp_ret_q); /* shared wait channel */
 995                 CRYPTO_RETQ_UNLOCK();
 996         }
 997 }
 998
 999 /*
1000  * Invoke the callback on behalf of the driver.
1001  */
1002 void
1003 crypto_kdone(struct cryptkop *krp)
1004 {
1005         int wasempty;
1006
1007         if (krp->krp_status != 0)
1008                 cryptostats.cs_kerrs++;
1009         CRYPTO_RETQ_LOCK();
1010         wasempty = TAILQ_EMPTY(&crp_ret_kq);
1011         TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next);
1012
1013         if (wasempty)
1014                 wakeup_one(&crp_ret_q);         /* shared wait channel */
1015         CRYPTO_RETQ_UNLOCK();
1016 }
1017
1018 int
1019 crypto_getfeat(int *featp)
1020 {
1021         int hid, kalg, feat = 0;
1022
1023         if (!crypto_userasymcrypto)
1024                 goto out;
1025
1026         CRYPTO_DRIVER_LOCK();
1027         for (hid = 0; hid < crypto_drivers_num; hid++) {
1028                 if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) &&
1029                     !crypto_devallowsoft) {
1030                         continue;
1031                 }
1032                 if (crypto_drivers[hid].cc_kprocess == NULL)
1033                         continue;
1034                 for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++)
1035                         if ((crypto_drivers[hid].cc_kalg[kalg] &
1036                             CRYPTO_ALG_FLAG_SUPPORTED) != 0)
1037                                 feat |=  1 << kalg;
1038         }
1039         CRYPTO_DRIVER_UNLOCK();
1040 out:
1041         *featp = feat;
1042         return (0);
1043 }
1044
1045 /*
1046  * Terminate a thread at module unload.  The process that
1047  * initiated this is waiting for us to signal that we're gone;
1048  * wake it up and exit.  We use the driver table lock to insure
1049  * we don't do the wakeup before they're waiting.  There is no
1050  * race here because the waiter sleeps on the proc lock for the
1051  * thread so it gets notified at the right time because of an
1052  * extra wakeup that's done in exit1().
1053  */
1054 static void
1055 crypto_finis(void *chan)
1056 {
1057         CRYPTO_DRIVER_LOCK();
1058         wakeup_one(chan);
1059         CRYPTO_DRIVER_UNLOCK();
1060         kthread_exit(0);
1061 }
1062
1063 /*
1064  * Crypto thread, dispatches crypto requests.
1065  */
1066 static void
1067 crypto_proc(void)
1068 {
1069         struct cryptop *crp, *submit;
1070         struct cryptkop *krp;
1071         struct cryptocap *cap;
1072         int result, hint;
1073
1074         CRYPTO_Q_LOCK();
1075         for (;;) {
1076                 /*
1077                  * Find the first element in the queue that can be
1078                  * processed and look-ahead to see if multiple ops
1079                  * are ready for the same driver.
1080                  */
1081                 submit = NULL;
1082                 hint = 0;
1083                 TAILQ_FOREACH(crp, &crp_q, crp_next) {
1084                         u_int32_t hid = CRYPTO_SESID2HID(crp->crp_sid);
1085                         cap = crypto_checkdriver(hid);
1086                         if (cap == NULL || cap->cc_process == NULL) {
1087                                 /* Op needs to be migrated, process it. */
1088                                 if (submit == NULL)
1089                                         submit = crp;
1090                                 break;
1091                         }
1092                         if (!cap->cc_qblocked) {
1093                                 if (submit != NULL) {
1094                                         /*
1095                                          * We stop on finding another op,
1096                                          * regardless whether its for the same
1097                                          * driver or not.  We could keep
1098                                          * searching the queue but it might be
1099                                          * better to just use a per-driver
1100                                          * queue instead.
1101                                          */
1102                                         if (CRYPTO_SESID2HID(submit->crp_sid) == hid)
1103                                                 hint = CRYPTO_HINT_MORE;
1104                                         break;
1105                                 } else {
1106                                         submit = crp;
1107                                         if ((submit->crp_flags & CRYPTO_F_BATCH) == 0)
1108                                                 break;
1109                                         /* keep scanning for more are q'd */
1110                                 }
1111                         }
1112                 }
1113                 if (submit != NULL) {
1114                         TAILQ_REMOVE(&crp_q, submit, crp_next);
1115                         result = crypto_invoke(submit, hint);
1116                         if (result == ERESTART) {
1117                                 /*
1118                                  * The driver ran out of resources, mark the
1119                                  * driver ``blocked'' for cryptop's and put
1120                                  * the request back in the queue.  It would
1121                                  * best to put the request back where we got
1122                                  * it but that's hard so for now we put it
1123                                  * at the front.  This should be ok; putting
1124                                  * it at the end does not work.
1125                                  */
1126                                 /* XXX validate sid again? */
1127                                 crypto_drivers[CRYPTO_SESID2HID(submit->crp_sid)].cc_qblocked = 1;
1128                                 TAILQ_INSERT_HEAD(&crp_q, submit, crp_next);
1129                                 cryptostats.cs_blocks++;
1130                         }
1131                 }
1132
1133                 /* As above, but for key ops */
1134                 TAILQ_FOREACH(krp, &crp_kq, krp_next) {
1135                         cap = crypto_checkdriver(krp->krp_hid);
1136                         if (cap == NULL || cap->cc_kprocess == NULL) {
1137                                 /* Op needs to be migrated, process it. */
1138                                 break;
1139                         }
1140                         if (!cap->cc_kqblocked)
1141                                 break;
1142                 }
1143                 if (krp != NULL) {
1144                         TAILQ_REMOVE(&crp_kq, krp, krp_next);
1145                         result = crypto_kinvoke(krp, 0);
1146                         if (result == ERESTART) {
1147                                 /*
1148                                  * The driver ran out of resources, mark the
1149                                  * driver ``blocked'' for cryptkop's and put
1150                                  * the request back in the queue.  It would
1151                                  * best to put the request back where we got
1152                                  * it but that's hard so for now we put it
1153                                  * at the front.  This should be ok; putting
1154                                  * it at the end does not work.
1155                                  */
1156                                 /* XXX validate sid again? */
1157                                 crypto_drivers[krp->krp_hid].cc_kqblocked = 1;
1158                                 TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);
1159                                 cryptostats.cs_kblocks++;
1160                         }
1161                 }
1162
1163                 if (submit == NULL && krp == NULL) {
1164                         /*
1165                          * Nothing more to be processed.  Sleep until we're
1166                          * woken because there are more ops to process.
1167                          * This happens either by submission or by a driver
1168                          * becoming unblocked and notifying us through
1169                          * crypto_unblock.  Note that when we wakeup we
1170                          * start processing each queue again from the
1171                          * front. It's not clear that it's important to
1172                          * preserve this ordering since ops may finish
1173                          * out of order if dispatched to different devices
1174                          * and some become blocked while others do not.
1175                          */
1176                         msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0);
1177                         if (cryptoproc == NULL)
1178                                 break;
1179                         cryptostats.cs_intrs++;
1180                 }
1181         }
1182         CRYPTO_Q_UNLOCK();
1183
1184         crypto_finis(&crp_q);
1185 }
1186
1187 /*
1188  * Crypto returns thread, does callbacks for processed crypto requests.
1189  * Callbacks are done here, rather than in the crypto drivers, because
1190  * callbacks typically are expensive and would slow interrupt handling.
1191  */
1192 static void
1193 crypto_ret_proc(void)
1194 {
1195         struct cryptop *crpt;
1196         struct cryptkop *krpt;
1197
1198         CRYPTO_RETQ_LOCK();
1199         for (;;) {
1200                 /* Harvest return q's for completed ops */
1201                 crpt = TAILQ_FIRST(&crp_ret_q);
1202                 if (crpt != NULL)
1203                         TAILQ_REMOVE(&crp_ret_q, crpt, crp_next);
1204
1205                 krpt = TAILQ_FIRST(&crp_ret_kq);
1206                 if (krpt != NULL)
1207                         TAILQ_REMOVE(&crp_ret_kq, krpt, krp_next);
1208
1209                 if (crpt != NULL || krpt != NULL) {
1210                         CRYPTO_RETQ_UNLOCK();
1211                         /*
1212                          * Run callbacks unlocked.
1213                          */
1214                         if (crpt != NULL) {
1215 #ifdef CRYPTO_TIMING
1216                                 if (crypto_timing) {
1217                                         /*
1218                                          * NB: We must copy the timestamp before
1219                                          * doing the callback as the cryptop is
1220                                          * likely to be reclaimed.
1221                                          */
1222                                         struct bintime t = crpt->crp_tstamp;
1223                                         crypto_tstat(&cryptostats.cs_cb, &t);
1224                                         crpt->crp_callback(crpt);
1225                                         crypto_tstat(&cryptostats.cs_finis, &t);
1226                                 } else
1227 #endif
1228                                         crpt->crp_callback(crpt);
1229                         }
1230                         if (krpt != NULL)
1231                                 krpt->krp_callback(krpt);
1232                         CRYPTO_RETQ_LOCK();
1233                 } else {
1234                         /*
1235                          * Nothing more to be processed.  Sleep until we're
1236                          * woken because there are more returns to process.
1237                          */
1238                         msleep(&crp_ret_q, &crypto_ret_q_mtx, PWAIT,
1239                                 "crypto_ret_wait", 0);
1240                         if (cryptoretproc == NULL)
1241                                 break;
1242                         cryptostats.cs_rets++;
1243                 }
1244         }
1245         CRYPTO_RETQ_UNLOCK();
1246
1247         crypto_finis(&crp_ret_q);
1248 }