2 * Copyright (c) 2009 Pawel Jakub Dawidek <pjd@FreeBSD.org>
5 * Copyright (c) 2012 Spectra Logic Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
32 #include <sys/types.h>
33 #include <sys/param.h>
34 #include <sys/kernel.h>
37 #include <sys/mutex.h>
38 #include <sys/queue.h>
39 #include <sys/taskqueue.h>
40 #include <sys/taskq.h>
41 #include <sys/zfs_context.h>
43 #include <sys/epoch.h>
47 #if __FreeBSD_version < 1201522
48 #define taskqueue_start_threads_in_proc(tqp, count, pri, proc, name, ...) \
49 taskqueue_start_threads(tqp, count, pri, name, __VA_ARGS__)
52 static uint_t taskq_tsd;
53 static uma_zone_t taskq_zone;
55 taskq_t *system_taskq = NULL;
56 taskq_t *system_delay_taskq = NULL;
57 taskq_t *dynamic_taskq = NULL;
61 extern int uma_align_cache;
63 static MALLOC_DEFINE(M_TASKQ, "taskq", "taskq structures");
65 static CK_LIST_HEAD(tqenthashhead, taskq_ent) *tqenthashtbl;
66 static unsigned long tqenthash;
67 static unsigned long tqenthashlock;
68 static struct sx *tqenthashtbl_lock;
70 static taskqid_t tqidnext;
72 #define TQIDHASH(tqid) (&tqenthashtbl[(tqid) & tqenthash])
73 #define TQIDHASHLOCK(tqid) (&tqenthashtbl_lock[((tqid) & tqenthashlock)])
75 #define TIMEOUT_TASK 1
79 system_taskq_init(void *arg)
83 tsd_create(&taskq_tsd, NULL);
84 tqenthashtbl = hashinit(mp_ncpus * 8, M_TASKQ, &tqenthash);
85 tqenthashlock = (tqenthash + 1) / 8;
86 if (tqenthashlock > 0)
89 malloc(sizeof (*tqenthashtbl_lock) * (tqenthashlock + 1),
90 M_TASKQ, M_WAITOK | M_ZERO);
91 for (i = 0; i < tqenthashlock + 1; i++)
92 sx_init_flags(&tqenthashtbl_lock[i], "tqenthash", SX_DUPOK);
93 taskq_zone = uma_zcreate("taskq_zone", sizeof (taskq_ent_t),
94 NULL, NULL, NULL, NULL,
96 system_taskq = taskq_create("system_taskq", mp_ncpus, minclsyspri,
98 system_delay_taskq = taskq_create("system_delay_taskq", mp_ncpus,
99 minclsyspri, 0, 0, 0);
101 SYSINIT(system_taskq_init, SI_SUB_CONFIGURE, SI_ORDER_ANY, system_taskq_init,
105 system_taskq_fini(void *arg)
109 taskq_destroy(system_delay_taskq);
110 taskq_destroy(system_taskq);
111 uma_zdestroy(taskq_zone);
112 tsd_destroy(&taskq_tsd);
113 for (i = 0; i < tqenthashlock + 1; i++)
114 sx_destroy(&tqenthashtbl_lock[i]);
115 for (i = 0; i < tqenthash + 1; i++)
116 VERIFY(CK_LIST_EMPTY(&tqenthashtbl[i]));
117 free(tqenthashtbl_lock, M_TASKQ);
118 free(tqenthashtbl, M_TASKQ);
120 SYSUNINIT(system_taskq_fini, SI_SUB_CONFIGURE, SI_ORDER_ANY, system_taskq_fini,
128 return (atomic_fetchadd_long(&tqidnext, 1) + 1);
137 tqid = atomic_fetchadd_int(&tqidnext, 1) + 1;
138 if (__predict_true(tqid != 0))
146 taskq_lookup(taskqid_t tqid)
148 taskq_ent_t *ent = NULL;
150 sx_xlock(TQIDHASHLOCK(tqid));
151 CK_LIST_FOREACH(ent, TQIDHASH(tqid), tqent_hash) {
152 if (ent->tqent_id == tqid)
156 refcount_acquire(&ent->tqent_rc);
157 sx_xunlock(TQIDHASHLOCK(tqid));
162 taskq_insert(taskq_ent_t *ent)
166 tqid = __taskq_genid();
168 ent->tqent_id = tqid;
169 ent->tqent_registered = B_TRUE;
170 sx_xlock(TQIDHASHLOCK(tqid));
171 CK_LIST_INSERT_HEAD(TQIDHASH(tqid), ent, tqent_hash);
172 sx_xunlock(TQIDHASHLOCK(tqid));
177 taskq_remove(taskq_ent_t *ent)
179 taskqid_t tqid = ent->tqent_id;
181 if (!ent->tqent_registered)
184 sx_xlock(TQIDHASHLOCK(tqid));
185 CK_LIST_REMOVE(ent, tqent_hash);
186 sx_xunlock(TQIDHASHLOCK(tqid));
187 ent->tqent_registered = B_FALSE;
191 taskq_tsd_set(void *context)
193 taskq_t *tq = context;
195 #if defined(__amd64__) || defined(__aarch64__)
196 if (context != NULL && tsd_get(taskq_tsd) == NULL)
197 fpu_kern_thread(FPU_KERN_NORMAL);
199 tsd_set(taskq_tsd, tq);
203 taskq_create_impl(const char *name, int nthreads, pri_t pri,
204 proc_t *proc __maybe_unused, uint_t flags)
208 if ((flags & TASKQ_THREADS_CPU_PCT) != 0)
209 nthreads = MAX((mp_ncpus * nthreads) / 100, 1);
211 tq = kmem_alloc(sizeof (*tq), KM_SLEEP);
212 tq->tq_queue = taskqueue_create(name, M_WAITOK,
213 taskqueue_thread_enqueue, &tq->tq_queue);
214 taskqueue_set_callback(tq->tq_queue, TASKQUEUE_CALLBACK_TYPE_INIT,
216 taskqueue_set_callback(tq->tq_queue, TASKQUEUE_CALLBACK_TYPE_SHUTDOWN,
217 taskq_tsd_set, NULL);
218 (void) taskqueue_start_threads_in_proc(&tq->tq_queue, nthreads, pri,
221 return ((taskq_t *)tq);
225 taskq_create(const char *name, int nthreads, pri_t pri, int minalloc __unused,
226 int maxalloc __unused, uint_t flags)
228 return (taskq_create_impl(name, nthreads, pri, system_proc, flags));
232 taskq_create_proc(const char *name, int nthreads, pri_t pri,
233 int minalloc __unused, int maxalloc __unused, proc_t *proc, uint_t flags)
235 return (taskq_create_impl(name, nthreads, pri, proc, flags));
239 taskq_destroy(taskq_t *tq)
242 taskqueue_free(tq->tq_queue);
243 kmem_free(tq, sizeof (*tq));
247 taskq_member(taskq_t *tq, kthread_t *thread)
250 return (taskqueue_member(tq->tq_queue, thread));
254 taskq_of_curthread(void)
256 return (tsd_get(taskq_tsd));
260 taskq_free(taskq_ent_t *task)
263 if (refcount_release(&task->tqent_rc))
264 uma_zfree(taskq_zone, task);
268 taskq_cancel_id(taskq_t *tq, taskqid_t tid)
277 if ((ent = taskq_lookup(tid)) == NULL)
280 ent->tqent_cancelled = B_TRUE;
281 if (ent->tqent_type == TIMEOUT_TASK) {
282 rc = taskqueue_cancel_timeout(tq->tq_queue,
283 &ent->tqent_timeout_task, &pend);
285 rc = taskqueue_cancel(tq->tq_queue, &ent->tqent_task, &pend);
287 taskqueue_drain(tq->tq_queue, &ent->tqent_task);
290 * Tasks normally free themselves when run, but here the task
291 * was cancelled so it did not free itself.
295 /* Free the extra reference we added with taskq_lookup. */
301 taskq_run(void *arg, int pending __unused)
303 taskq_ent_t *task = arg;
305 if (!task->tqent_cancelled)
306 task->tqent_func(task->tqent_arg);
311 taskq_dispatch_delay(taskq_t *tq, task_func_t func, void *arg,
312 uint_t flags, clock_t expire_time)
319 timo = expire_time - ddi_get_lbolt();
321 return (taskq_dispatch(tq, func, arg, flags));
323 if ((flags & (TQ_SLEEP | TQ_NOQUEUE)) == TQ_SLEEP)
328 task = uma_zalloc(taskq_zone, mflag);
331 task->tqent_func = func;
332 task->tqent_arg = arg;
333 task->tqent_type = TIMEOUT_TASK;
334 task->tqent_cancelled = B_FALSE;
335 refcount_init(&task->tqent_rc, 1);
336 tqid = taskq_insert(task);
337 TIMEOUT_TASK_INIT(tq->tq_queue, &task->tqent_timeout_task, 0,
340 taskqueue_enqueue_timeout(tq->tq_queue, &task->tqent_timeout_task,
346 taskq_dispatch(taskq_t *tq, task_func_t func, void *arg, uint_t flags)
352 if ((flags & (TQ_SLEEP | TQ_NOQUEUE)) == TQ_SLEEP)
357 * If TQ_FRONT is given, we want higher priority for this task, so it
358 * can go at the front of the queue.
360 prio = !!(flags & TQ_FRONT);
362 task = uma_zalloc(taskq_zone, mflag);
365 refcount_init(&task->tqent_rc, 1);
366 task->tqent_func = func;
367 task->tqent_arg = arg;
368 task->tqent_cancelled = B_FALSE;
369 task->tqent_type = NORMAL_TASK;
370 tqid = taskq_insert(task);
371 TASK_INIT(&task->tqent_task, prio, taskq_run, task);
372 taskqueue_enqueue(tq->tq_queue, &task->tqent_task);
377 taskq_run_ent(void *arg, int pending __unused)
379 taskq_ent_t *task = arg;
381 task->tqent_func(task->tqent_arg);
385 taskq_dispatch_ent(taskq_t *tq, task_func_t func, void *arg, uint32_t flags,
391 * If TQ_FRONT is given, we want higher priority for this task, so it
392 * can go at the front of the queue.
394 prio = !!(flags & TQ_FRONT);
395 task->tqent_cancelled = B_FALSE;
396 task->tqent_registered = B_FALSE;
398 task->tqent_func = func;
399 task->tqent_arg = arg;
401 TASK_INIT(&task->tqent_task, prio, taskq_run_ent, task);
402 taskqueue_enqueue(tq->tq_queue, &task->tqent_task);
406 taskq_wait(taskq_t *tq)
408 taskqueue_quiesce(tq->tq_queue);
412 taskq_wait_id(taskq_t *tq, taskqid_t tid)
418 if ((ent = taskq_lookup(tid)) == NULL)
421 taskqueue_drain(tq->tq_queue, &ent->tqent_task);
426 taskq_wait_outstanding(taskq_t *tq, taskqid_t id __unused)
428 taskqueue_drain_all(tq->tq_queue);
432 taskq_empty_ent(taskq_ent_t *t)
434 return (t->tqent_task.ta_pending == 0);