module/os/linux/spl/spl-taskq.c

   1 /*
   2  *  Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
   3  *  Copyright (C) 2007 The Regents of the University of California.
   4  *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
   5  *  Written by Brian Behlendorf <behlendorf1@llnl.gov>.
   6  *  UCRL-CODE-235197
   7  *
   8  *  This file is part of the SPL, Solaris Porting Layer.
   9  *  For details, see <http://zfsonlinux.org/>.
  10  *
  11  *  The SPL is free software; you can redistribute it and/or modify it
  12  *  under the terms of the GNU General Public License as published by the
  13  *  Free Software Foundation; either version 2 of the License, or (at your
  14  *  option) any later version.
  15  *
  16  *  The SPL is distributed in the hope that it will be useful, but WITHOUT
  17  *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  18  *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  19  *  for more details.
  20  *
  21  *  You should have received a copy of the GNU General Public License along
  22  *  with the SPL.  If not, see <http://www.gnu.org/licenses/>.
  23  *
  24  *  Solaris Porting Layer (SPL) Task Queue Implementation.
  25  */
  26
  27 #include <sys/timer.h>
  28 #include <sys/taskq.h>
  29 #include <sys/kmem.h>
  30 #include <sys/tsd.h>
  31 #include <sys/trace_spl.h>
  32
  33 int spl_taskq_thread_bind = 0;
  34 module_param(spl_taskq_thread_bind, int, 0644);
  35 MODULE_PARM_DESC(spl_taskq_thread_bind, "Bind taskq thread to CPU by default");
  36
  37
  38 int spl_taskq_thread_dynamic = 1;
  39 module_param(spl_taskq_thread_dynamic, int, 0644);
  40 MODULE_PARM_DESC(spl_taskq_thread_dynamic, "Allow dynamic taskq threads");
  41
  42 int spl_taskq_thread_priority = 1;
  43 module_param(spl_taskq_thread_priority, int, 0644);
  44 MODULE_PARM_DESC(spl_taskq_thread_priority,
  45         "Allow non-default priority for taskq threads");
  46
  47 int spl_taskq_thread_sequential = 4;
  48 module_param(spl_taskq_thread_sequential, int, 0644);
  49 MODULE_PARM_DESC(spl_taskq_thread_sequential,
  50         "Create new taskq threads after N sequential tasks");
  51
  52 /* Global system-wide dynamic task queue available for all consumers */
  53 taskq_t *system_taskq;
  54 EXPORT_SYMBOL(system_taskq);
  55 /* Global dynamic task queue for long delay */
  56 taskq_t *system_delay_taskq;
  57 EXPORT_SYMBOL(system_delay_taskq);
  58
  59 /* Private dedicated taskq for creating new taskq threads on demand. */
  60 static taskq_t *dynamic_taskq;
  61 static taskq_thread_t *taskq_thread_create(taskq_t *);
  62
  63 /* List of all taskqs */
  64 LIST_HEAD(tq_list);
  65 struct rw_semaphore tq_list_sem;
  66 static uint_t taskq_tsd;
  67
  68 static int
  69 task_km_flags(uint_t flags)
  70 {
  71         if (flags & TQ_NOSLEEP)
  72                 return (KM_NOSLEEP);
  73
  74         if (flags & TQ_PUSHPAGE)
  75                 return (KM_PUSHPAGE);
  76
  77         return (KM_SLEEP);
  78 }
  79
  80 /*
  81  * taskq_find_by_name - Find the largest instance number of a named taskq.
  82  */
  83 static int
  84 taskq_find_by_name(const char *name)
  85 {
  86         struct list_head *tql = NULL;
  87         taskq_t *tq;
  88
  89         list_for_each_prev(tql, &tq_list) {
  90                 tq = list_entry(tql, taskq_t, tq_taskqs);
  91                 if (strcmp(name, tq->tq_name) == 0)
  92                         return (tq->tq_instance);
  93         }
  94         return (-1);
  95 }
  96
  97 /*
  98  * NOTE: Must be called with tq->tq_lock held, returns a list_t which
  99  * is not attached to the free, work, or pending taskq lists.
 100  */
 101 static taskq_ent_t *
 102 task_alloc(taskq_t *tq, uint_t flags, unsigned long *irqflags)
 103 {
 104         taskq_ent_t *t;
 105         int count = 0;
 106
 107         ASSERT(tq);
 108 retry:
 109         /* Acquire taskq_ent_t's from free list if available */
 110         if (!list_empty(&tq->tq_free_list) && !(flags & TQ_NEW)) {
 111                 t = list_entry(tq->tq_free_list.next, taskq_ent_t, tqent_list);
 112
 113                 ASSERT(!(t->tqent_flags & TQENT_FLAG_PREALLOC));
 114                 ASSERT(!(t->tqent_flags & TQENT_FLAG_CANCEL));
 115                 ASSERT(!timer_pending(&t->tqent_timer));
 116
 117                 list_del_init(&t->tqent_list);
 118                 return (t);
 119         }
 120
 121         /* Free list is empty and memory allocations are prohibited */
 122         if (flags & TQ_NOALLOC)
 123                 return (NULL);
 124
 125         /* Hit maximum taskq_ent_t pool size */
 126         if (tq->tq_nalloc >= tq->tq_maxalloc) {
 127                 if (flags & TQ_NOSLEEP)
 128                         return (NULL);
 129
 130                 /*
 131                  * Sleep periodically polling the free list for an available
 132                  * taskq_ent_t. Dispatching with TQ_SLEEP should always succeed
 133                  * but we cannot block forever waiting for an taskq_ent_t to
 134                  * show up in the free list, otherwise a deadlock can happen.
 135                  *
 136                  * Therefore, we need to allocate a new task even if the number
 137                  * of allocated tasks is above tq->tq_maxalloc, but we still
 138                  * end up delaying the task allocation by one second, thereby
 139                  * throttling the task dispatch rate.
 140                  */
 141                 spin_unlock_irqrestore(&tq->tq_lock, *irqflags);
 142                 schedule_timeout(HZ / 100);
 143                 spin_lock_irqsave_nested(&tq->tq_lock, *irqflags,
 144                     tq->tq_lock_class);
 145                 if (count < 100) {
 146                         count++;
 147                         goto retry;
 148                 }
 149         }
 150
 151         spin_unlock_irqrestore(&tq->tq_lock, *irqflags);
 152         t = kmem_alloc(sizeof (taskq_ent_t), task_km_flags(flags));
 153         spin_lock_irqsave_nested(&tq->tq_lock, *irqflags, tq->tq_lock_class);
 154
 155         if (t) {
 156                 taskq_init_ent(t);
 157                 tq->tq_nalloc++;
 158         }
 159
 160         return (t);
 161 }
 162
 163 /*
 164  * NOTE: Must be called with tq->tq_lock held, expects the taskq_ent_t
 165  * to already be removed from the free, work, or pending taskq lists.
 166  */
 167 static void
 168 task_free(taskq_t *tq, taskq_ent_t *t)
 169 {
 170         ASSERT(tq);
 171         ASSERT(t);
 172         ASSERT(list_empty(&t->tqent_list));
 173         ASSERT(!timer_pending(&t->tqent_timer));
 174
 175         kmem_free(t, sizeof (taskq_ent_t));
 176         tq->tq_nalloc--;
 177 }
 178
 179 /*
 180  * NOTE: Must be called with tq->tq_lock held, either destroys the
 181  * taskq_ent_t if too many exist or moves it to the free list for later use.
 182  */
 183 static void
 184 task_done(taskq_t *tq, taskq_ent_t *t)
 185 {
 186         ASSERT(tq);
 187         ASSERT(t);
 188
 189         /* Wake tasks blocked in taskq_wait_id() */
 190         wake_up_all(&t->tqent_waitq);
 191
 192         list_del_init(&t->tqent_list);
 193
 194         if (tq->tq_nalloc <= tq->tq_minalloc) {
 195                 t->tqent_id = TASKQID_INVALID;
 196                 t->tqent_func = NULL;
 197                 t->tqent_arg = NULL;
 198                 t->tqent_flags = 0;
 199
 200                 list_add_tail(&t->tqent_list, &tq->tq_free_list);
 201         } else {
 202                 task_free(tq, t);
 203         }
 204 }
 205
 206 /*
 207  * When a delayed task timer expires remove it from the delay list and
 208  * add it to the priority list in order for immediate processing.
 209  */
 210 static void
 211 task_expire_impl(taskq_ent_t *t)
 212 {
 213         taskq_ent_t *w;
 214         taskq_t *tq = t->tqent_taskq;
 215         struct list_head *l = NULL;
 216         unsigned long flags;
 217
 218         spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
 219
 220         if (t->tqent_flags & TQENT_FLAG_CANCEL) {
 221                 ASSERT(list_empty(&t->tqent_list));
 222                 spin_unlock_irqrestore(&tq->tq_lock, flags);
 223                 return;
 224         }
 225
 226         t->tqent_birth = jiffies;
 227         DTRACE_PROBE1(taskq_ent__birth, taskq_ent_t *, t);
 228
 229         /*
 230          * The priority list must be maintained in strict task id order
 231          * from lowest to highest for lowest_id to be easily calculable.
 232          */
 233         list_del(&t->tqent_list);
 234         list_for_each_prev(l, &tq->tq_prio_list) {
 235                 w = list_entry(l, taskq_ent_t, tqent_list);
 236                 if (w->tqent_id < t->tqent_id) {
 237                         list_add(&t->tqent_list, l);
 238                         break;
 239                 }
 240         }
 241         if (l == &tq->tq_prio_list)
 242                 list_add(&t->tqent_list, &tq->tq_prio_list);
 243
 244         spin_unlock_irqrestore(&tq->tq_lock, flags);
 245
 246         wake_up(&tq->tq_work_waitq);
 247 }
 248
 249 static void
 250 task_expire(spl_timer_list_t tl)
 251 {
 252         struct timer_list *tmr = (struct timer_list *)tl;
 253         taskq_ent_t *t = from_timer(t, tmr, tqent_timer);
 254         task_expire_impl(t);
 255 }
 256
 257 /*
 258  * Returns the lowest incomplete taskqid_t.  The taskqid_t may
 259  * be queued on the pending list, on the priority list, on the
 260  * delay list, or on the work list currently being handled, but
 261  * it is not 100% complete yet.
 262  */
 263 static taskqid_t
 264 taskq_lowest_id(taskq_t *tq)
 265 {
 266         taskqid_t lowest_id = tq->tq_next_id;
 267         taskq_ent_t *t;
 268         taskq_thread_t *tqt;
 269
 270         ASSERT(tq);
 271
 272         if (!list_empty(&tq->tq_pend_list)) {
 273                 t = list_entry(tq->tq_pend_list.next, taskq_ent_t, tqent_list);
 274                 lowest_id = MIN(lowest_id, t->tqent_id);
 275         }
 276
 277         if (!list_empty(&tq->tq_prio_list)) {
 278                 t = list_entry(tq->tq_prio_list.next, taskq_ent_t, tqent_list);
 279                 lowest_id = MIN(lowest_id, t->tqent_id);
 280         }
 281
 282         if (!list_empty(&tq->tq_delay_list)) {
 283                 t = list_entry(tq->tq_delay_list.next, taskq_ent_t, tqent_list);
 284                 lowest_id = MIN(lowest_id, t->tqent_id);
 285         }
 286
 287         if (!list_empty(&tq->tq_active_list)) {
 288                 tqt = list_entry(tq->tq_active_list.next, taskq_thread_t,
 289                     tqt_active_list);
 290                 ASSERT(tqt->tqt_id != TASKQID_INVALID);
 291                 lowest_id = MIN(lowest_id, tqt->tqt_id);
 292         }
 293
 294         return (lowest_id);
 295 }
 296
 297 /*
 298  * Insert a task into a list keeping the list sorted by increasing taskqid.
 299  */
 300 static void
 301 taskq_insert_in_order(taskq_t *tq, taskq_thread_t *tqt)
 302 {
 303         taskq_thread_t *w;
 304         struct list_head *l = NULL;
 305
 306         ASSERT(tq);
 307         ASSERT(tqt);
 308
 309         list_for_each_prev(l, &tq->tq_active_list) {
 310                 w = list_entry(l, taskq_thread_t, tqt_active_list);
 311                 if (w->tqt_id < tqt->tqt_id) {
 312                         list_add(&tqt->tqt_active_list, l);
 313                         break;
 314                 }
 315         }
 316         if (l == &tq->tq_active_list)
 317                 list_add(&tqt->tqt_active_list, &tq->tq_active_list);
 318 }
 319
 320 /*
 321  * Find and return a task from the given list if it exists.  The list
 322  * must be in lowest to highest task id order.
 323  */
 324 static taskq_ent_t *
 325 taskq_find_list(taskq_t *tq, struct list_head *lh, taskqid_t id)
 326 {
 327         struct list_head *l = NULL;
 328         taskq_ent_t *t;
 329
 330         list_for_each(l, lh) {
 331                 t = list_entry(l, taskq_ent_t, tqent_list);
 332
 333                 if (t->tqent_id == id)
 334                         return (t);
 335
 336                 if (t->tqent_id > id)
 337                         break;
 338         }
 339
 340         return (NULL);
 341 }
 342
 343 /*
 344  * Find an already dispatched task given the task id regardless of what
 345  * state it is in.  If a task is still pending it will be returned.
 346  * If a task is executing, then -EBUSY will be returned instead.
 347  * If the task has already been run then NULL is returned.
 348  */
 349 static taskq_ent_t *
 350 taskq_find(taskq_t *tq, taskqid_t id)
 351 {
 352         taskq_thread_t *tqt;
 353         struct list_head *l = NULL;
 354         taskq_ent_t *t;
 355
 356         t = taskq_find_list(tq, &tq->tq_delay_list, id);
 357         if (t)
 358                 return (t);
 359
 360         t = taskq_find_list(tq, &tq->tq_prio_list, id);
 361         if (t)
 362                 return (t);
 363
 364         t = taskq_find_list(tq, &tq->tq_pend_list, id);
 365         if (t)
 366                 return (t);
 367
 368         list_for_each(l, &tq->tq_active_list) {
 369                 tqt = list_entry(l, taskq_thread_t, tqt_active_list);
 370                 if (tqt->tqt_id == id) {
 371                         /*
 372                          * Instead of returning tqt_task, we just return a non
 373                          * NULL value to prevent misuse, since tqt_task only
 374                          * has two valid fields.
 375                          */
 376                         return (ERR_PTR(-EBUSY));
 377                 }
 378         }
 379
 380         return (NULL);
 381 }
 382
 383 /*
 384  * Theory for the taskq_wait_id(), taskq_wait_outstanding(), and
 385  * taskq_wait() functions below.
 386  *
 387  * Taskq waiting is accomplished by tracking the lowest outstanding task
 388  * id and the next available task id.  As tasks are dispatched they are
 389  * added to the tail of the pending, priority, or delay lists.  As worker
 390  * threads become available the tasks are removed from the heads of these
 391  * lists and linked to the worker threads.  This ensures the lists are
 392  * kept sorted by lowest to highest task id.
 393  *
 394  * Therefore the lowest outstanding task id can be quickly determined by
 395  * checking the head item from all of these lists.  This value is stored
 396  * with the taskq as the lowest id.  It only needs to be recalculated when
 397  * either the task with the current lowest id completes or is canceled.
 398  *
 399  * By blocking until the lowest task id exceeds the passed task id the
 400  * taskq_wait_outstanding() function can be easily implemented.  Similarly,
 401  * by blocking until the lowest task id matches the next task id taskq_wait()
 402  * can be implemented.
 403  *
 404  * Callers should be aware that when there are multiple worked threads it
 405  * is possible for larger task ids to complete before smaller ones.  Also
 406  * when the taskq contains delay tasks with small task ids callers may
 407  * block for a considerable length of time waiting for them to expire and
 408  * execute.
 409  */
 410 static int
 411 taskq_wait_id_check(taskq_t *tq, taskqid_t id)
 412 {
 413         int rc;
 414         unsigned long flags;
 415
 416         spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
 417         rc = (taskq_find(tq, id) == NULL);
 418         spin_unlock_irqrestore(&tq->tq_lock, flags);
 419
 420         return (rc);
 421 }
 422
 423 /*
 424  * The taskq_wait_id() function blocks until the passed task id completes.
 425  * This does not guarantee that all lower task ids have completed.
 426  */
 427 void
 428 taskq_wait_id(taskq_t *tq, taskqid_t id)
 429 {
 430         wait_event(tq->tq_wait_waitq, taskq_wait_id_check(tq, id));
 431 }
 432 EXPORT_SYMBOL(taskq_wait_id);
 433
 434 static int
 435 taskq_wait_outstanding_check(taskq_t *tq, taskqid_t id)
 436 {
 437         int rc;
 438         unsigned long flags;
 439
 440         spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
 441         rc = (id < tq->tq_lowest_id);
 442         spin_unlock_irqrestore(&tq->tq_lock, flags);
 443
 444         return (rc);
 445 }
 446
 447 /*
 448  * The taskq_wait_outstanding() function will block until all tasks with a
 449  * lower taskqid than the passed 'id' have been completed.  Note that all
 450  * task id's are assigned monotonically at dispatch time.  Zero may be
 451  * passed for the id to indicate all tasks dispatch up to this point,
 452  * but not after, should be waited for.
 453  */
 454 void
 455 taskq_wait_outstanding(taskq_t *tq, taskqid_t id)
 456 {
 457         id = id ? id : tq->tq_next_id - 1;
 458         wait_event(tq->tq_wait_waitq, taskq_wait_outstanding_check(tq, id));
 459 }
 460 EXPORT_SYMBOL(taskq_wait_outstanding);
 461
 462 static int
 463 taskq_wait_check(taskq_t *tq)
 464 {
 465         int rc;
 466         unsigned long flags;
 467
 468         spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
 469         rc = (tq->tq_lowest_id == tq->tq_next_id);
 470         spin_unlock_irqrestore(&tq->tq_lock, flags);
 471
 472         return (rc);
 473 }
 474
 475 /*
 476  * The taskq_wait() function will block until the taskq is empty.
 477  * This means that if a taskq re-dispatches work to itself taskq_wait()
 478  * callers will block indefinitely.
 479  */
 480 void
 481 taskq_wait(taskq_t *tq)
 482 {
 483         wait_event(tq->tq_wait_waitq, taskq_wait_check(tq));
 484 }
 485 EXPORT_SYMBOL(taskq_wait);
 486
 487 int
 488 taskq_member(taskq_t *tq, kthread_t *t)
 489 {
 490         return (tq == (taskq_t *)tsd_get_by_thread(taskq_tsd, t));
 491 }
 492 EXPORT_SYMBOL(taskq_member);
 493
 494 taskq_t *
 495 taskq_of_curthread(void)
 496 {
 497         return (tsd_get(taskq_tsd));
 498 }
 499 EXPORT_SYMBOL(taskq_of_curthread);
 500
 501 /*
 502  * Cancel an already dispatched task given the task id.  Still pending tasks
 503  * will be immediately canceled, and if the task is active the function will
 504  * block until it completes.  Preallocated tasks which are canceled must be
 505  * freed by the caller.
 506  */
 507 int
 508 taskq_cancel_id(taskq_t *tq, taskqid_t id)
 509 {
 510         taskq_ent_t *t;
 511         int rc = ENOENT;
 512         unsigned long flags;
 513
 514         ASSERT(tq);
 515
 516         spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
 517         t = taskq_find(tq, id);
 518         if (t && t != ERR_PTR(-EBUSY)) {
 519                 list_del_init(&t->tqent_list);
 520                 t->tqent_flags |= TQENT_FLAG_CANCEL;
 521
 522                 /*
 523                  * When canceling the lowest outstanding task id we
 524                  * must recalculate the new lowest outstanding id.
 525                  */
 526                 if (tq->tq_lowest_id == t->tqent_id) {
 527                         tq->tq_lowest_id = taskq_lowest_id(tq);
 528                         ASSERT3S(tq->tq_lowest_id, >, t->tqent_id);
 529                 }
 530
 531                 /*
 532                  * The task_expire() function takes the tq->tq_lock so drop
 533                  * drop the lock before synchronously cancelling the timer.
 534                  */
 535                 if (timer_pending(&t->tqent_timer)) {
 536                         spin_unlock_irqrestore(&tq->tq_lock, flags);
 537                         del_timer_sync(&t->tqent_timer);
 538                         spin_lock_irqsave_nested(&tq->tq_lock, flags,
 539                             tq->tq_lock_class);
 540                 }
 541
 542                 if (!(t->tqent_flags & TQENT_FLAG_PREALLOC))
 543                         task_done(tq, t);
 544
 545                 rc = 0;
 546         }
 547         spin_unlock_irqrestore(&tq->tq_lock, flags);
 548
 549         if (t == ERR_PTR(-EBUSY)) {
 550                 taskq_wait_id(tq, id);
 551                 rc = EBUSY;
 552         }
 553
 554         return (rc);
 555 }
 556 EXPORT_SYMBOL(taskq_cancel_id);
 557
 558 static int taskq_thread_spawn(taskq_t *tq);
 559
 560 taskqid_t
 561 taskq_dispatch(taskq_t *tq, task_func_t func, void *arg, uint_t flags)
 562 {
 563         taskq_ent_t *t;
 564         taskqid_t rc = TASKQID_INVALID;
 565         unsigned long irqflags;
 566
 567         ASSERT(tq);
 568         ASSERT(func);
 569
 570         spin_lock_irqsave_nested(&tq->tq_lock, irqflags, tq->tq_lock_class);
 571
 572         /* Taskq being destroyed and all tasks drained */
 573         if (!(tq->tq_flags & TASKQ_ACTIVE))
 574                 goto out;
 575
 576         /* Do not queue the task unless there is idle thread for it */
 577         ASSERT(tq->tq_nactive <= tq->tq_nthreads);
 578         if ((flags & TQ_NOQUEUE) && (tq->tq_nactive == tq->tq_nthreads)) {
 579                 /* Dynamic taskq may be able to spawn another thread */
 580                 if (!(tq->tq_flags & TASKQ_DYNAMIC) ||
 581                     taskq_thread_spawn(tq) == 0)
 582                         goto out;
 583         }
 584
 585         if ((t = task_alloc(tq, flags, &irqflags)) == NULL)
 586                 goto out;
 587
 588         spin_lock(&t->tqent_lock);
 589
 590         /* Queue to the front of the list to enforce TQ_NOQUEUE semantics */
 591         if (flags & TQ_NOQUEUE)
 592                 list_add(&t->tqent_list, &tq->tq_prio_list);
 593         /* Queue to the priority list instead of the pending list */
 594         else if (flags & TQ_FRONT)
 595                 list_add_tail(&t->tqent_list, &tq->tq_prio_list);
 596         else
 597                 list_add_tail(&t->tqent_list, &tq->tq_pend_list);
 598
 599         t->tqent_id = rc = tq->tq_next_id;
 600         tq->tq_next_id++;
 601         t->tqent_func = func;
 602         t->tqent_arg = arg;
 603         t->tqent_taskq = tq;
 604         t->tqent_timer.function = NULL;
 605         t->tqent_timer.expires = 0;
 606
 607         t->tqent_birth = jiffies;
 608         DTRACE_PROBE1(taskq_ent__birth, taskq_ent_t *, t);
 609
 610         ASSERT(!(t->tqent_flags & TQENT_FLAG_PREALLOC));
 611
 612         spin_unlock(&t->tqent_lock);
 613
 614         wake_up(&tq->tq_work_waitq);
 615 out:
 616         /* Spawn additional taskq threads if required. */
 617         if (!(flags & TQ_NOQUEUE) && tq->tq_nactive == tq->tq_nthreads)
 618                 (void) taskq_thread_spawn(tq);
 619
 620         spin_unlock_irqrestore(&tq->tq_lock, irqflags);
 621         return (rc);
 622 }
 623 EXPORT_SYMBOL(taskq_dispatch);
 624
 625 taskqid_t
 626 taskq_dispatch_delay(taskq_t *tq, task_func_t func, void *arg,
 627     uint_t flags, clock_t expire_time)
 628 {
 629         taskqid_t rc = TASKQID_INVALID;
 630         taskq_ent_t *t;
 631         unsigned long irqflags;
 632
 633         ASSERT(tq);
 634         ASSERT(func);
 635
 636         spin_lock_irqsave_nested(&tq->tq_lock, irqflags, tq->tq_lock_class);
 637
 638         /* Taskq being destroyed and all tasks drained */
 639         if (!(tq->tq_flags & TASKQ_ACTIVE))
 640                 goto out;
 641
 642         if ((t = task_alloc(tq, flags, &irqflags)) == NULL)
 643                 goto out;
 644
 645         spin_lock(&t->tqent_lock);
 646
 647         /* Queue to the delay list for subsequent execution */
 648         list_add_tail(&t->tqent_list, &tq->tq_delay_list);
 649
 650         t->tqent_id = rc = tq->tq_next_id;
 651         tq->tq_next_id++;
 652         t->tqent_func = func;
 653         t->tqent_arg = arg;
 654         t->tqent_taskq = tq;
 655         t->tqent_timer.function = task_expire;
 656         t->tqent_timer.expires = (unsigned long)expire_time;
 657         add_timer(&t->tqent_timer);
 658
 659         ASSERT(!(t->tqent_flags & TQENT_FLAG_PREALLOC));
 660
 661         spin_unlock(&t->tqent_lock);
 662 out:
 663         /* Spawn additional taskq threads if required. */
 664         if (tq->tq_nactive == tq->tq_nthreads)
 665                 (void) taskq_thread_spawn(tq);
 666         spin_unlock_irqrestore(&tq->tq_lock, irqflags);
 667         return (rc);
 668 }
 669 EXPORT_SYMBOL(taskq_dispatch_delay);
 670
 671 void
 672 taskq_dispatch_ent(taskq_t *tq, task_func_t func, void *arg, uint_t flags,
 673     taskq_ent_t *t)
 674 {
 675         unsigned long irqflags;
 676         ASSERT(tq);
 677         ASSERT(func);
 678
 679         spin_lock_irqsave_nested(&tq->tq_lock, irqflags,
 680             tq->tq_lock_class);
 681
 682         /* Taskq being destroyed and all tasks drained */
 683         if (!(tq->tq_flags & TASKQ_ACTIVE)) {
 684                 t->tqent_id = TASKQID_INVALID;
 685                 goto out;
 686         }
 687
 688         if ((flags & TQ_NOQUEUE) && (tq->tq_nactive == tq->tq_nthreads)) {
 689                 /* Dynamic taskq may be able to spawn another thread */
 690                 if (!(tq->tq_flags & TASKQ_DYNAMIC) ||
 691                     taskq_thread_spawn(tq) == 0)
 692                         goto out2;
 693                 flags |= TQ_FRONT;
 694         }
 695
 696         spin_lock(&t->tqent_lock);
 697
 698         /*
 699          * Make sure the entry is not on some other taskq; it is important to
 700          * ASSERT() under lock
 701          */
 702         ASSERT(taskq_empty_ent(t));
 703
 704         /*
 705          * Mark it as a prealloc'd task.  This is important
 706          * to ensure that we don't free it later.
 707          */
 708         t->tqent_flags |= TQENT_FLAG_PREALLOC;
 709
 710         /* Queue to the priority list instead of the pending list */
 711         if (flags & TQ_FRONT)
 712                 list_add_tail(&t->tqent_list, &tq->tq_prio_list);
 713         else
 714                 list_add_tail(&t->tqent_list, &tq->tq_pend_list);
 715
 716         t->tqent_id = tq->tq_next_id;
 717         tq->tq_next_id++;
 718         t->tqent_func = func;
 719         t->tqent_arg = arg;
 720         t->tqent_taskq = tq;
 721
 722         t->tqent_birth = jiffies;
 723         DTRACE_PROBE1(taskq_ent__birth, taskq_ent_t *, t);
 724
 725         spin_unlock(&t->tqent_lock);
 726
 727         wake_up(&tq->tq_work_waitq);
 728 out:
 729         /* Spawn additional taskq threads if required. */
 730         if (tq->tq_nactive == tq->tq_nthreads)
 731                 (void) taskq_thread_spawn(tq);
 732 out2:
 733         spin_unlock_irqrestore(&tq->tq_lock, irqflags);
 734 }
 735 EXPORT_SYMBOL(taskq_dispatch_ent);
 736
 737 int
 738 taskq_empty_ent(taskq_ent_t *t)
 739 {
 740         return (list_empty(&t->tqent_list));
 741 }
 742 EXPORT_SYMBOL(taskq_empty_ent);
 743
 744 void
 745 taskq_init_ent(taskq_ent_t *t)
 746 {
 747         spin_lock_init(&t->tqent_lock);
 748         init_waitqueue_head(&t->tqent_waitq);
 749         timer_setup(&t->tqent_timer, NULL, 0);
 750         INIT_LIST_HEAD(&t->tqent_list);
 751         t->tqent_id = 0;
 752         t->tqent_func = NULL;
 753         t->tqent_arg = NULL;
 754         t->tqent_flags = 0;
 755         t->tqent_taskq = NULL;
 756 }
 757 EXPORT_SYMBOL(taskq_init_ent);
 758
 759 /*
 760  * Return the next pending task, preference is given to tasks on the
 761  * priority list which were dispatched with TQ_FRONT.
 762  */
 763 static taskq_ent_t *
 764 taskq_next_ent(taskq_t *tq)
 765 {
 766         struct list_head *list;
 767
 768         if (!list_empty(&tq->tq_prio_list))
 769                 list = &tq->tq_prio_list;
 770         else if (!list_empty(&tq->tq_pend_list))
 771                 list = &tq->tq_pend_list;
 772         else
 773                 return (NULL);
 774
 775         return (list_entry(list->next, taskq_ent_t, tqent_list));
 776 }
 777
 778 /*
 779  * Spawns a new thread for the specified taskq.
 780  */
 781 static void
 782 taskq_thread_spawn_task(void *arg)
 783 {
 784         taskq_t *tq = (taskq_t *)arg;
 785         unsigned long flags;
 786
 787         if (taskq_thread_create(tq) == NULL) {
 788                 /* restore spawning count if failed */
 789                 spin_lock_irqsave_nested(&tq->tq_lock, flags,
 790                     tq->tq_lock_class);
 791                 tq->tq_nspawn--;
 792                 spin_unlock_irqrestore(&tq->tq_lock, flags);
 793         }
 794 }
 795
 796 /*
 797  * Spawn addition threads for dynamic taskqs (TASKQ_DYNAMIC) the current
 798  * number of threads is insufficient to handle the pending tasks.  These
 799  * new threads must be created by the dedicated dynamic_taskq to avoid
 800  * deadlocks between thread creation and memory reclaim.  The system_taskq
 801  * which is also a dynamic taskq cannot be safely used for this.
 802  */
 803 static int
 804 taskq_thread_spawn(taskq_t *tq)
 805 {
 806         int spawning = 0;
 807
 808         if (!(tq->tq_flags & TASKQ_DYNAMIC))
 809                 return (0);
 810
 811         if ((tq->tq_nthreads + tq->tq_nspawn < tq->tq_maxthreads) &&
 812             (tq->tq_flags & TASKQ_ACTIVE)) {
 813                 spawning = (++tq->tq_nspawn);
 814                 taskq_dispatch(dynamic_taskq, taskq_thread_spawn_task,
 815                     tq, TQ_NOSLEEP);
 816         }
 817
 818         return (spawning);
 819 }
 820
 821 /*
 822  * Threads in a dynamic taskq should only exit once it has been completely
 823  * drained and no other threads are actively servicing tasks.  This prevents
 824  * threads from being created and destroyed more than is required.
 825  *
 826  * The first thread is the thread list is treated as the primary thread.
 827  * There is nothing special about the primary thread but in order to avoid
 828  * all the taskq pids from changing we opt to make it long running.
 829  */
 830 static int
 831 taskq_thread_should_stop(taskq_t *tq, taskq_thread_t *tqt)
 832 {
 833         if (!(tq->tq_flags & TASKQ_DYNAMIC))
 834                 return (0);
 835
 836         if (list_first_entry(&(tq->tq_thread_list), taskq_thread_t,
 837             tqt_thread_list) == tqt)
 838                 return (0);
 839
 840         return
 841             ((tq->tq_nspawn == 0) &&    /* No threads are being spawned */
 842             (tq->tq_nactive == 0) &&    /* No threads are handling tasks */
 843             (tq->tq_nthreads > 1) &&    /* More than 1 thread is running */
 844             (!taskq_next_ent(tq)) &&    /* There are no pending tasks */
 845             (spl_taskq_thread_dynamic)); /* Dynamic taskqs are allowed */
 846 }
 847
 848 static int
 849 taskq_thread(void *args)
 850 {
 851         DECLARE_WAITQUEUE(wait, current);
 852         sigset_t blocked;
 853         taskq_thread_t *tqt = args;
 854         taskq_t *tq;
 855         taskq_ent_t *t;
 856         int seq_tasks = 0;
 857         unsigned long flags;
 858         taskq_ent_t dup_task = {};
 859
 860         ASSERT(tqt);
 861         ASSERT(tqt->tqt_tq);
 862         tq = tqt->tqt_tq;
 863         current->flags |= PF_NOFREEZE;
 864
 865         (void) spl_fstrans_mark();
 866
 867         sigfillset(&blocked);
 868         sigprocmask(SIG_BLOCK, &blocked, NULL);
 869         flush_signals(current);
 870
 871         tsd_set(taskq_tsd, tq);
 872         spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
 873         /*
 874          * If we are dynamically spawned, decrease spawning count. Note that
 875          * we could be created during taskq_create, in which case we shouldn't
 876          * do the decrement. But it's fine because taskq_create will reset
 877          * tq_nspawn later.
 878          */
 879         if (tq->tq_flags & TASKQ_DYNAMIC)
 880                 tq->tq_nspawn--;
 881
 882         /* Immediately exit if more threads than allowed were created. */
 883         if (tq->tq_nthreads >= tq->tq_maxthreads)
 884                 goto error;
 885
 886         tq->tq_nthreads++;
 887         list_add_tail(&tqt->tqt_thread_list, &tq->tq_thread_list);
 888         wake_up(&tq->tq_wait_waitq);
 889         set_current_state(TASK_INTERRUPTIBLE);
 890
 891         while (!kthread_should_stop()) {
 892
 893                 if (list_empty(&tq->tq_pend_list) &&
 894                     list_empty(&tq->tq_prio_list)) {
 895
 896                         if (taskq_thread_should_stop(tq, tqt)) {
 897                                 wake_up_all(&tq->tq_wait_waitq);
 898                                 break;
 899                         }
 900
 901                         add_wait_queue_exclusive(&tq->tq_work_waitq, &wait);
 902                         spin_unlock_irqrestore(&tq->tq_lock, flags);
 903
 904                         schedule();
 905                         seq_tasks = 0;
 906
 907                         spin_lock_irqsave_nested(&tq->tq_lock, flags,
 908                             tq->tq_lock_class);
 909                         remove_wait_queue(&tq->tq_work_waitq, &wait);
 910                 } else {
 911                         __set_current_state(TASK_RUNNING);
 912                 }
 913
 914                 if ((t = taskq_next_ent(tq)) != NULL) {
 915                         list_del_init(&t->tqent_list);
 916
 917                         /*
 918                          * A TQENT_FLAG_PREALLOC task may be reused or freed
 919                          * during the task function call. Store tqent_id and
 920                          * tqent_flags here.
 921                          *
 922                          * Also use an on stack taskq_ent_t for tqt_task
 923                          * assignment in this case; we want to make sure
 924                          * to duplicate all fields, so the values are
 925                          * correct when it's accessed via DTRACE_PROBE*.
 926                          */
 927                         tqt->tqt_id = t->tqent_id;
 928                         tqt->tqt_flags = t->tqent_flags;
 929
 930                         if (t->tqent_flags & TQENT_FLAG_PREALLOC) {
 931                                 dup_task = *t;
 932                                 t = &dup_task;
 933                         }
 934                         tqt->tqt_task = t;
 935
 936                         taskq_insert_in_order(tq, tqt);
 937                         tq->tq_nactive++;
 938                         spin_unlock_irqrestore(&tq->tq_lock, flags);
 939
 940                         DTRACE_PROBE1(taskq_ent__start, taskq_ent_t *, t);
 941
 942                         /* Perform the requested task */
 943                         t->tqent_func(t->tqent_arg);
 944
 945                         DTRACE_PROBE1(taskq_ent__finish, taskq_ent_t *, t);
 946
 947                         spin_lock_irqsave_nested(&tq->tq_lock, flags,
 948                             tq->tq_lock_class);
 949                         tq->tq_nactive--;
 950                         list_del_init(&tqt->tqt_active_list);
 951                         tqt->tqt_task = NULL;
 952
 953                         /* For prealloc'd tasks, we don't free anything. */
 954                         if (!(tqt->tqt_flags & TQENT_FLAG_PREALLOC))
 955                                 task_done(tq, t);
 956
 957                         /*
 958                          * When the current lowest outstanding taskqid is
 959                          * done calculate the new lowest outstanding id
 960                          */
 961                         if (tq->tq_lowest_id == tqt->tqt_id) {
 962                                 tq->tq_lowest_id = taskq_lowest_id(tq);
 963                                 ASSERT3S(tq->tq_lowest_id, >, tqt->tqt_id);
 964                         }
 965
 966                         /* Spawn additional taskq threads if required. */
 967                         if ((++seq_tasks) > spl_taskq_thread_sequential &&
 968                             taskq_thread_spawn(tq))
 969                                 seq_tasks = 0;
 970
 971                         tqt->tqt_id = TASKQID_INVALID;
 972                         tqt->tqt_flags = 0;
 973                         wake_up_all(&tq->tq_wait_waitq);
 974                 } else {
 975                         if (taskq_thread_should_stop(tq, tqt))
 976                                 break;
 977                 }
 978
 979                 set_current_state(TASK_INTERRUPTIBLE);
 980
 981         }
 982
 983         __set_current_state(TASK_RUNNING);
 984         tq->tq_nthreads--;
 985         list_del_init(&tqt->tqt_thread_list);
 986 error:
 987         kmem_free(tqt, sizeof (taskq_thread_t));
 988         spin_unlock_irqrestore(&tq->tq_lock, flags);
 989
 990         tsd_set(taskq_tsd, NULL);
 991
 992         return (0);
 993 }
 994
 995 static taskq_thread_t *
 996 taskq_thread_create(taskq_t *tq)
 997 {
 998         static int last_used_cpu = 0;
 999         taskq_thread_t *tqt;
1000
1001         tqt = kmem_alloc(sizeof (*tqt), KM_PUSHPAGE);
1002         INIT_LIST_HEAD(&tqt->tqt_thread_list);
1003         INIT_LIST_HEAD(&tqt->tqt_active_list);
1004         tqt->tqt_tq = tq;
1005         tqt->tqt_id = TASKQID_INVALID;
1006
1007         tqt->tqt_thread = spl_kthread_create(taskq_thread, tqt,
1008             "%s", tq->tq_name);
1009         if (tqt->tqt_thread == NULL) {
1010                 kmem_free(tqt, sizeof (taskq_thread_t));
1011                 return (NULL);
1012         }
1013
1014         if (spl_taskq_thread_bind) {
1015                 last_used_cpu = (last_used_cpu + 1) % num_online_cpus();
1016                 kthread_bind(tqt->tqt_thread, last_used_cpu);
1017         }
1018
1019         if (spl_taskq_thread_priority)
1020                 set_user_nice(tqt->tqt_thread, PRIO_TO_NICE(tq->tq_pri));
1021
1022         wake_up_process(tqt->tqt_thread);
1023
1024         return (tqt);
1025 }
1026
1027 taskq_t *
1028 taskq_create(const char *name, int nthreads, pri_t pri,
1029     int minalloc, int maxalloc, uint_t flags)
1030 {
1031         taskq_t *tq;
1032         taskq_thread_t *tqt;
1033         int count = 0, rc = 0, i;
1034         unsigned long irqflags;
1035
1036         ASSERT(name != NULL);
1037         ASSERT(minalloc >= 0);
1038         ASSERT(maxalloc <= INT_MAX);
1039         ASSERT(!(flags & (TASKQ_CPR_SAFE))); /* Unsupported */
1040
1041         /* Scale the number of threads using nthreads as a percentage */
1042         if (flags & TASKQ_THREADS_CPU_PCT) {
1043                 ASSERT(nthreads <= 100);
1044                 ASSERT(nthreads >= 0);
1045                 nthreads = MIN(nthreads, 100);
1046                 nthreads = MAX(nthreads, 0);
1047                 nthreads = MAX((num_online_cpus() * nthreads) / 100, 1);
1048         }
1049
1050         tq = kmem_alloc(sizeof (*tq), KM_PUSHPAGE);
1051         if (tq == NULL)
1052                 return (NULL);
1053
1054         spin_lock_init(&tq->tq_lock);
1055         INIT_LIST_HEAD(&tq->tq_thread_list);
1056         INIT_LIST_HEAD(&tq->tq_active_list);
1057         tq->tq_name = kmem_strdup(name);
1058         tq->tq_nactive = 0;
1059         tq->tq_nthreads = 0;
1060         tq->tq_nspawn = 0;
1061         tq->tq_maxthreads = nthreads;
1062         tq->tq_pri = pri;
1063         tq->tq_minalloc = minalloc;
1064         tq->tq_maxalloc = maxalloc;
1065         tq->tq_nalloc = 0;
1066         tq->tq_flags = (flags | TASKQ_ACTIVE);
1067         tq->tq_next_id = TASKQID_INITIAL;
1068         tq->tq_lowest_id = TASKQID_INITIAL;
1069         INIT_LIST_HEAD(&tq->tq_free_list);
1070         INIT_LIST_HEAD(&tq->tq_pend_list);
1071         INIT_LIST_HEAD(&tq->tq_prio_list);
1072         INIT_LIST_HEAD(&tq->tq_delay_list);
1073         init_waitqueue_head(&tq->tq_work_waitq);
1074         init_waitqueue_head(&tq->tq_wait_waitq);
1075         tq->tq_lock_class = TQ_LOCK_GENERAL;
1076         INIT_LIST_HEAD(&tq->tq_taskqs);
1077
1078         if (flags & TASKQ_PREPOPULATE) {
1079                 spin_lock_irqsave_nested(&tq->tq_lock, irqflags,
1080                     tq->tq_lock_class);
1081
1082                 for (i = 0; i < minalloc; i++)
1083                         task_done(tq, task_alloc(tq, TQ_PUSHPAGE | TQ_NEW,
1084                             &irqflags));
1085
1086                 spin_unlock_irqrestore(&tq->tq_lock, irqflags);
1087         }
1088
1089         if ((flags & TASKQ_DYNAMIC) && spl_taskq_thread_dynamic)
1090                 nthreads = 1;
1091
1092         for (i = 0; i < nthreads; i++) {
1093                 tqt = taskq_thread_create(tq);
1094                 if (tqt == NULL)
1095                         rc = 1;
1096                 else
1097                         count++;
1098         }
1099
1100         /* Wait for all threads to be started before potential destroy */
1101         wait_event(tq->tq_wait_waitq, tq->tq_nthreads == count);
1102         /*
1103          * taskq_thread might have touched nspawn, but we don't want them to
1104          * because they're not dynamically spawned. So we reset it to 0
1105          */
1106         tq->tq_nspawn = 0;
1107
1108         if (rc) {
1109                 taskq_destroy(tq);
1110                 tq = NULL;
1111         } else {
1112                 down_write(&tq_list_sem);
1113                 tq->tq_instance = taskq_find_by_name(name) + 1;
1114                 list_add_tail(&tq->tq_taskqs, &tq_list);
1115                 up_write(&tq_list_sem);
1116         }
1117
1118         return (tq);
1119 }
1120 EXPORT_SYMBOL(taskq_create);
1121
1122 void
1123 taskq_destroy(taskq_t *tq)
1124 {
1125         struct task_struct *thread;
1126         taskq_thread_t *tqt;
1127         taskq_ent_t *t;
1128         unsigned long flags;
1129
1130         ASSERT(tq);
1131         spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
1132         tq->tq_flags &= ~TASKQ_ACTIVE;
1133         spin_unlock_irqrestore(&tq->tq_lock, flags);
1134
1135         /*
1136          * When TASKQ_ACTIVE is clear new tasks may not be added nor may
1137          * new worker threads be spawned for dynamic taskq.
1138          */
1139         if (dynamic_taskq != NULL)
1140                 taskq_wait_outstanding(dynamic_taskq, 0);
1141
1142         taskq_wait(tq);
1143
1144         /* remove taskq from global list used by the kstats */
1145         down_write(&tq_list_sem);
1146         list_del(&tq->tq_taskqs);
1147         up_write(&tq_list_sem);
1148
1149         spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
1150         /* wait for spawning threads to insert themselves to the list */
1151         while (tq->tq_nspawn) {
1152                 spin_unlock_irqrestore(&tq->tq_lock, flags);
1153                 schedule_timeout_interruptible(1);
1154                 spin_lock_irqsave_nested(&tq->tq_lock, flags,
1155                     tq->tq_lock_class);
1156         }
1157
1158         /*
1159          * Signal each thread to exit and block until it does.  Each thread
1160          * is responsible for removing itself from the list and freeing its
1161          * taskq_thread_t.  This allows for idle threads to opt to remove
1162          * themselves from the taskq.  They can be recreated as needed.
1163          */
1164         while (!list_empty(&tq->tq_thread_list)) {
1165                 tqt = list_entry(tq->tq_thread_list.next,
1166                     taskq_thread_t, tqt_thread_list);
1167                 thread = tqt->tqt_thread;
1168                 spin_unlock_irqrestore(&tq->tq_lock, flags);
1169
1170                 kthread_stop(thread);
1171
1172                 spin_lock_irqsave_nested(&tq->tq_lock, flags,
1173                     tq->tq_lock_class);
1174         }
1175
1176         while (!list_empty(&tq->tq_free_list)) {
1177                 t = list_entry(tq->tq_free_list.next, taskq_ent_t, tqent_list);
1178
1179                 ASSERT(!(t->tqent_flags & TQENT_FLAG_PREALLOC));
1180
1181                 list_del_init(&t->tqent_list);
1182                 task_free(tq, t);
1183         }
1184
1185         ASSERT0(tq->tq_nthreads);
1186         ASSERT0(tq->tq_nalloc);
1187         ASSERT0(tq->tq_nspawn);
1188         ASSERT(list_empty(&tq->tq_thread_list));
1189         ASSERT(list_empty(&tq->tq_active_list));
1190         ASSERT(list_empty(&tq->tq_free_list));
1191         ASSERT(list_empty(&tq->tq_pend_list));
1192         ASSERT(list_empty(&tq->tq_prio_list));
1193         ASSERT(list_empty(&tq->tq_delay_list));
1194
1195         spin_unlock_irqrestore(&tq->tq_lock, flags);
1196
1197         kmem_strfree(tq->tq_name);
1198         kmem_free(tq, sizeof (taskq_t));
1199 }
1200 EXPORT_SYMBOL(taskq_destroy);
1201
1202
1203 static unsigned int spl_taskq_kick = 0;
1204
1205 /*
1206  * 2.6.36 API Change
1207  * module_param_cb is introduced to take kernel_param_ops and
1208  * module_param_call is marked as obsolete. Also set and get operations
1209  * were changed to take a 'const struct kernel_param *'.
1210  */
1211 static int
1212 #ifdef module_param_cb
1213 param_set_taskq_kick(const char *val, const struct kernel_param *kp)
1214 #else
1215 param_set_taskq_kick(const char *val, struct kernel_param *kp)
1216 #endif
1217 {
1218         int ret;
1219         taskq_t *tq = NULL;
1220         taskq_ent_t *t;
1221         unsigned long flags;
1222
1223         ret = param_set_uint(val, kp);
1224         if (ret < 0 || !spl_taskq_kick)
1225                 return (ret);
1226         /* reset value */
1227         spl_taskq_kick = 0;
1228
1229         down_read(&tq_list_sem);
1230         list_for_each_entry(tq, &tq_list, tq_taskqs) {
1231                 spin_lock_irqsave_nested(&tq->tq_lock, flags,
1232                     tq->tq_lock_class);
1233                 /* Check if the first pending is older than 5 seconds */
1234                 t = taskq_next_ent(tq);
1235                 if (t && time_after(jiffies, t->tqent_birth + 5*HZ)) {
1236                         (void) taskq_thread_spawn(tq);
1237                         printk(KERN_INFO "spl: Kicked taskq %s/%d\n",
1238                             tq->tq_name, tq->tq_instance);
1239                 }
1240                 spin_unlock_irqrestore(&tq->tq_lock, flags);
1241         }
1242         up_read(&tq_list_sem);
1243         return (ret);
1244 }
1245
1246 #ifdef module_param_cb
1247 static const struct kernel_param_ops param_ops_taskq_kick = {
1248         .set = param_set_taskq_kick,
1249         .get = param_get_uint,
1250 };
1251 module_param_cb(spl_taskq_kick, &param_ops_taskq_kick, &spl_taskq_kick, 0644);
1252 #else
1253 module_param_call(spl_taskq_kick, param_set_taskq_kick, param_get_uint,
1254         &spl_taskq_kick, 0644);
1255 #endif
1256 MODULE_PARM_DESC(spl_taskq_kick,
1257         "Write nonzero to kick stuck taskqs to spawn more threads");
1258
1259 int
1260 spl_taskq_init(void)
1261 {
1262         init_rwsem(&tq_list_sem);
1263         tsd_create(&taskq_tsd, NULL);
1264
1265         system_taskq = taskq_create("spl_system_taskq", MAX(boot_ncpus, 64),
1266             maxclsyspri, boot_ncpus, INT_MAX, TASKQ_PREPOPULATE|TASKQ_DYNAMIC);
1267         if (system_taskq == NULL)
1268                 return (1);
1269
1270         system_delay_taskq = taskq_create("spl_delay_taskq", MAX(boot_ncpus, 4),
1271             maxclsyspri, boot_ncpus, INT_MAX, TASKQ_PREPOPULATE|TASKQ_DYNAMIC);
1272         if (system_delay_taskq == NULL) {
1273                 taskq_destroy(system_taskq);
1274                 return (1);
1275         }
1276
1277         dynamic_taskq = taskq_create("spl_dynamic_taskq", 1,
1278             maxclsyspri, boot_ncpus, INT_MAX, TASKQ_PREPOPULATE);
1279         if (dynamic_taskq == NULL) {
1280                 taskq_destroy(system_taskq);
1281                 taskq_destroy(system_delay_taskq);
1282                 return (1);
1283         }
1284
1285         /*
1286          * This is used to annotate tq_lock, so
1287          *   taskq_dispatch -> taskq_thread_spawn -> taskq_dispatch
1288          * does not trigger a lockdep warning re: possible recursive locking
1289          */
1290         dynamic_taskq->tq_lock_class = TQ_LOCK_DYNAMIC;
1291
1292         return (0);
1293 }
1294
1295 void
1296 spl_taskq_fini(void)
1297 {
1298         taskq_destroy(dynamic_taskq);
1299         dynamic_taskq = NULL;
1300
1301         taskq_destroy(system_delay_taskq);
1302         system_delay_taskq = NULL;
1303
1304         taskq_destroy(system_taskq);
1305         system_taskq = NULL;
1306
1307         tsd_destroy(&taskq_tsd);
1308 }