2 * Copyright (c) 1995-1998 John Birrell <jb@cimlogic.com.au>.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. Neither the name of the author nor the names of any co-contributors
14 * may be used to endorse or promote products derived from this software
15 * without specific prior written permission.
17 * THIS SOFTWARE IS PROVIDED BY JOHN BIRRELL AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * Private thread definitions for the uthread kernel.
34 #ifndef _THR_PRIVATE_H
35 #define _THR_PRIVATE_H
43 #include <sys/queue.h>
44 #include <sys/types.h>
46 #include <sys/cdefs.h>
52 #include <pthread_np.h>
56 #include "pthread_md.h"
60 * Unfortunately, libpthread had symbol versioning before libc.
61 * But now libc has symbol versioning, we need to occupy the
62 * same version namespace in order to override some libc functions.
63 * So in order to avoid breaking binaries requiring symbols from
64 * LIBTHREAD_1_0, we need to provide a compatible interface for
68 #define SYM_LT10(sym) __CONCAT(sym, _lt10)
69 #define SYM_FB10(sym) __CONCAT(sym, _fb10)
70 #define SYM_FBP10(sym) __CONCAT(sym, _fbp10)
71 #define WEAK_REF(sym, alias) __weak_reference(sym, alias)
72 #define SYM_COMPAT(sym, impl, ver) __sym_compat(sym, impl, ver)
73 #define SYM_DEFAULT(sym, impl, ver) __sym_default(sym, impl, ver)
75 #define LT10_COMPAT(sym) \
76 WEAK_REF(sym, SYM_LT10(sym)); \
77 SYM_COMPAT(sym, SYM_LT10(sym), LIBTHREAD_1_0)
79 #define LT10_COMPAT_DEFAULT(sym) \
81 WEAK_REF(sym, SYM_FB10(sym)); \
82 SYM_DEFAULT(sym, SYM_FB10(sym), FBSD_1.0)
84 #define LT10_COMPAT_PRIVATE(sym) \
86 WEAK_REF(sym, SYM_FBP10(sym)); \
87 SYM_DEFAULT(sym, SYM_FBP10(sym), FBSDprivate_1.0)
89 #define LT10_COMPAT_DEFAULT(sym)
90 #define LT10_COMPAT_PRIVATE(sym)
94 * Evaluate the storage class specifier.
96 #ifdef GLOBAL_PTHREAD_PRIVATE
98 #define SCLASS_PRESET(x...) = x
100 #define SCLASS extern
101 #define SCLASS_PRESET(x...)
105 * Kernel fatal error handler macro.
107 #define PANIC(string) _thr_exit(__FILE__, __LINE__, string)
110 /* Output debug messages like this: */
112 #define stdout_debug(...) _thread_printf(STDOUT_FILENO, __VA_ARGS__)
115 #define stderr_debug(...) _thread_printf(STDERR_FILENO, __VA_ARGS__)
118 #define DBG_MUTEX 0x0001
119 #define DBG_SIG 0x0002
120 #define DBG_INFO_DUMP 0x0004
122 #ifdef _PTHREADS_INVARIANTS
123 #define THR_ASSERT(cond, msg) do { \
128 #define THR_ASSERT(cond, msg)
132 * State change macro without scheduling queue change:
134 #define THR_SET_STATE(thrd, newstate) do { \
135 (thrd)->state = newstate; \
136 (thrd)->fname = __FILE__; \
137 (thrd)->lineno = __LINE__; \
141 #define TIMESPEC_ADD(dst, src, val) \
143 (dst)->tv_sec = (src)->tv_sec + (val)->tv_sec; \
144 (dst)->tv_nsec = (src)->tv_nsec + (val)->tv_nsec; \
145 if ((dst)->tv_nsec >= 1000000000) { \
147 (dst)->tv_nsec -= 1000000000; \
151 #define TIMESPEC_SUB(dst, src, val) \
153 (dst)->tv_sec = (src)->tv_sec - (val)->tv_sec; \
154 (dst)->tv_nsec = (src)->tv_nsec - (val)->tv_nsec; \
155 if ((dst)->tv_nsec < 0) { \
157 (dst)->tv_nsec += 1000000000; \
164 * XXX It'd be nice if these were contained in uthread_priority_queue.[ch].
166 typedef struct pq_list {
167 TAILQ_HEAD(, pthread) pl_head; /* list of threads at this priority */
168 TAILQ_ENTRY(pq_list) pl_link; /* link for queue of priority lists */
169 int pl_prio; /* the priority of this list */
170 int pl_queued; /* is this in the priority queue */
173 typedef struct pq_queue {
174 TAILQ_HEAD(, pq_list) pq_queue; /* queue of priority lists */
175 pq_list_t *pq_lists; /* array of all priority lists */
176 int pq_size; /* number of priority lists */
177 #define PQF_ACTIVE 0x0001
183 * Each KSEG has a scheduling queue. For now, threads that exist in their
184 * own KSEG (system scope) will get a full priority queue. In the future
185 * this can be optimized for the single thread per KSEG case.
189 TAILQ_HEAD(, pthread) sq_waitq; /* waiting in userland */
192 typedef struct kse_thr_mailbox *kse_critical_t;
196 #define MAX_KSE_LOCKLEVEL 5
198 /* -- location and order specific items for gdb -- */
200 struct pthread *k_curthread; /* current thread */
201 struct kse_group *k_kseg; /* parent KSEG */
202 struct sched_queue *k_schedq; /* scheduling queue */
203 /* -- end of location and order specific items -- */
204 TAILQ_ENTRY(kse) k_qe; /* KSE list link entry */
205 TAILQ_ENTRY(kse) k_kgqe; /* KSEG's KSE list entry */
207 * Items that are only modified by the kse, or that otherwise
208 * don't need to be locked when accessed
211 struct lockuser k_lockusers[MAX_KSE_LOCKLEVEL];
215 #define KF_STARTED 0x0001 /* kernel kse created */
216 #define KF_INITIALIZED 0x0002 /* initialized on 1st upcall */
217 #define KF_TERMINATED 0x0004 /* kse is terminated */
218 #define KF_IDLE 0x0008 /* kse is idle */
219 #define KF_SWITCH 0x0010 /* thread switch in UTS */
220 int k_error; /* syscall errno in critical */
221 int k_cpu; /* CPU ID when bound */
222 int k_sigseqno; /* signal buffered count */
225 #define KSE_SET_IDLE(kse) ((kse)->k_flags |= KF_IDLE)
226 #define KSE_CLEAR_IDLE(kse) ((kse)->k_flags &= ~KF_IDLE)
227 #define KSE_IS_IDLE(kse) (((kse)->k_flags & KF_IDLE) != 0)
228 #define KSE_SET_SWITCH(kse) ((kse)->k_flags |= KF_SWITCH)
229 #define KSE_CLEAR_SWITCH(kse) ((kse)->k_flags &= ~KF_SWITCH)
230 #define KSE_IS_SWITCH(kse) (((kse)->k_flags & KF_SWITCH) != 0)
233 * Each KSE group contains one or more KSEs in which threads can run.
234 * At least for now, there is one scheduling queue per KSE group; KSEs
235 * within the same KSE group compete for threads from the same scheduling
236 * queue. A scope system thread has one KSE in one KSE group; the group
237 * does not use its scheduling queue.
240 TAILQ_HEAD(, kse) kg_kseq; /* list of KSEs in group */
241 TAILQ_HEAD(, pthread) kg_threadq; /* list of threads in group */
242 TAILQ_ENTRY(kse_group) kg_qe; /* link entry */
243 struct sched_queue kg_schedq; /* scheduling queue */
245 int kg_threadcount; /* # of assigned threads */
246 int kg_ksecount; /* # of assigned KSEs */
249 #define KGF_SINGLE_THREAD 0x0001 /* scope system kse group */
250 #define KGF_SCHEDQ_INITED 0x0002 /* has an initialized schedq */
254 * Add/remove threads from a KSE's scheduling queue.
255 * For now the scheduling queue is hung off the KSEG.
257 #define KSEG_THRQ_ADD(kseg, thr) \
259 TAILQ_INSERT_TAIL(&(kseg)->kg_threadq, thr, kle);\
260 (kseg)->kg_threadcount++; \
263 #define KSEG_THRQ_REMOVE(kseg, thr) \
265 TAILQ_REMOVE(&(kseg)->kg_threadq, thr, kle); \
266 (kseg)->kg_threadcount--; \
271 * Lock acquire and release for KSEs.
273 #define KSE_LOCK_ACQUIRE(kse, lck) \
275 if ((kse)->k_locklevel < MAX_KSE_LOCKLEVEL) { \
276 (kse)->k_locklevel++; \
277 _lock_acquire((lck), \
278 &(kse)->k_lockusers[(kse)->k_locklevel - 1], 0); \
281 PANIC("Exceeded maximum lock level"); \
284 #define KSE_LOCK_RELEASE(kse, lck) \
286 if ((kse)->k_locklevel > 0) { \
287 _lock_release((lck), \
288 &(kse)->k_lockusers[(kse)->k_locklevel - 1]); \
289 (kse)->k_locklevel--; \
296 #define KSE_LOCK(curkse) \
297 KSE_LOCK_ACQUIRE(curkse, &(curkse)->k_kseg->kg_lock)
298 #define KSE_UNLOCK(curkse) \
299 KSE_LOCK_RELEASE(curkse, &(curkse)->k_kseg->kg_lock)
302 * Lock a potentially different KSEG.
304 #define KSE_SCHED_LOCK(curkse, kseg) \
305 KSE_LOCK_ACQUIRE(curkse, &(kseg)->kg_lock)
306 #define KSE_SCHED_UNLOCK(curkse, kseg) \
307 KSE_LOCK_RELEASE(curkse, &(kseg)->kg_lock)
310 * Waiting queue manipulation macros (using pqe link):
312 #define KSE_WAITQ_REMOVE(kse, thrd) \
314 if (((thrd)->flags & THR_FLAGS_IN_WAITQ) != 0) { \
315 TAILQ_REMOVE(&(kse)->k_schedq->sq_waitq, thrd, pqe); \
316 (thrd)->flags &= ~THR_FLAGS_IN_WAITQ; \
319 #define KSE_WAITQ_INSERT(kse, thrd) kse_waitq_insert(thrd)
320 #define KSE_WAITQ_FIRST(kse) TAILQ_FIRST(&(kse)->k_schedq->sq_waitq)
322 #define KSE_WAKEUP(kse) kse_wakeup(&(kse)->k_kcb->kcb_kmbx)
325 * TailQ initialization values.
327 #define TAILQ_INITIALIZER { NULL, NULL }
330 * lock initialization values.
332 #define LCK_INITIALIZER { NULL, NULL, LCK_DEFAULT }
334 struct pthread_mutex {
336 * Lock for accesses to this structure.
339 enum pthread_mutextype m_type;
341 TAILQ_HEAD(mutex_head, pthread) m_queue;
342 struct pthread *m_owner;
348 * Used for priority inheritence and protection.
350 * m_prio - For priority inheritence, the highest active
351 * priority (threads locking the mutex inherit
352 * this priority). For priority protection, the
353 * ceiling priority of this mutex.
354 * m_saved_prio - mutex owners inherited priority before
355 * taking the mutex, restored when the owner
362 * Link for list of all mutexes a thread currently owns.
364 TAILQ_ENTRY(pthread_mutex) m_qe;
370 #define MUTEX_FLAGS_PRIVATE 0x01
371 #define MUTEX_FLAGS_INITED 0x02
372 #define MUTEX_FLAGS_BUSY 0x04
375 * Static mutex initialization values.
377 #define PTHREAD_MUTEX_STATIC_INITIALIZER \
378 { LCK_INITIALIZER, PTHREAD_MUTEX_DEFAULT, PTHREAD_PRIO_NONE, \
379 TAILQ_INITIALIZER, NULL, MUTEX_FLAGS_PRIVATE, 0, 0, 0, 0, \
382 struct pthread_mutex_attr {
383 enum pthread_mutextype m_type;
389 #define PTHREAD_MUTEXATTR_STATIC_INITIALIZER \
390 { PTHREAD_MUTEX_DEFAULT, PTHREAD_PRIO_NONE, 0, MUTEX_FLAGS_PRIVATE }
393 * Condition variable definitions.
395 enum pthread_cond_type {
400 struct pthread_cond {
402 * Lock for accesses to this structure.
405 enum pthread_cond_type c_type;
406 TAILQ_HEAD(cond_head, pthread) c_queue;
407 struct pthread_mutex *c_mutex;
412 struct pthread_cond_attr {
413 enum pthread_cond_type c_type;
417 struct pthread_barrier {
418 pthread_mutex_t b_lock;
419 pthread_cond_t b_cond;
425 struct pthread_barrierattr {
429 struct pthread_spinlock {
435 * Flags for condition variables.
437 #define COND_FLAGS_PRIVATE 0x01
438 #define COND_FLAGS_INITED 0x02
439 #define COND_FLAGS_BUSY 0x04
442 * Static cond initialization values.
444 #define PTHREAD_COND_STATIC_INITIALIZER \
445 { LCK_INITIALIZER, COND_TYPE_FAST, TAILQ_INITIALIZER, \
449 * Cleanup definitions.
451 struct pthread_cleanup {
452 struct pthread_cleanup *next;
453 void (*routine) (void *);
458 #define THR_CLEANUP_PUSH(td, func, arg) { \
459 struct pthread_cleanup __cup; \
461 __cup.routine = func; \
462 __cup.routine_arg = arg; \
464 __cup.next = (td)->cleanup; \
465 (td)->cleanup = &__cup;
467 #define THR_CLEANUP_POP(td, exec) \
468 (td)->cleanup = __cup.next; \
470 __cup.routine(__cup.routine_arg); \
473 struct pthread_atfork {
474 TAILQ_ENTRY(pthread_atfork) qe;
475 void (*prepare)(void);
476 void (*parent)(void);
480 struct pthread_attr {
486 #define THR_STACK_USER 0x100 /* 0xFF reserved for <pthread.h> */
487 #define THR_SIGNAL_THREAD 0x200 /* This is a signal thread */
490 void (*cleanup_attr) (void *);
491 void *stackaddr_attr;
492 size_t stacksize_attr;
493 size_t guardsize_attr;
497 * Thread creation state attributes.
499 #define THR_CREATE_RUNNING 0
500 #define THR_CREATE_SUSPENDED 1
503 * Miscellaneous definitions.
505 #define THR_STACK32_DEFAULT (1 * 1024 * 1024)
506 #define THR_STACK64_DEFAULT (2 * 1024 * 1024)
509 * Maximum size of initial thread's stack. This perhaps deserves to be larger
510 * than the stacks of other threads, since many applications are likely to run
511 * almost entirely on this stack.
513 #define THR_STACK32_INITIAL (2 * 1024 * 1024)
514 #define THR_STACK64_INITIAL (4 * 1024 * 1024)
517 * Define the different priority ranges. All applications have thread
518 * priorities constrained within 0-31. The threads library raises the
519 * priority when delivering signals in order to ensure that signal
520 * delivery happens (from the POSIX spec) "as soon as possible".
521 * In the future, the threads library will also be able to map specific
522 * threads into real-time (cooperating) processes or kernel threads.
523 * The RT and SIGNAL priorities will be used internally and added to
524 * thread base priorities so that the scheduling queue can handle both
525 * normal and RT priority threads with and without signal handling.
527 * The approach taken is that, within each class, signal delivery
528 * always has priority over thread execution.
530 #define THR_DEFAULT_PRIORITY 15
531 #define THR_MIN_PRIORITY 0
532 #define THR_MAX_PRIORITY 31 /* 0x1F */
533 #define THR_SIGNAL_PRIORITY 32 /* 0x20 */
534 #define THR_RT_PRIORITY 64 /* 0x40 */
535 #define THR_FIRST_PRIORITY THR_MIN_PRIORITY
536 #define THR_LAST_PRIORITY \
537 (THR_MAX_PRIORITY + THR_SIGNAL_PRIORITY + THR_RT_PRIORITY)
538 #define THR_BASE_PRIORITY(prio) ((prio) & THR_MAX_PRIORITY)
541 * Clock resolution in microseconds.
543 #define CLOCK_RES_USEC 10000
546 * Time slice period in microseconds.
548 #define TIMESLICE_USEC 20000
551 * XXX - Define a thread-safe macro to get the current time of day
552 * which is updated at regular intervals by something.
554 * For now, we just make the system call to get the time.
556 #define KSE_GET_TOD(curkse, tsp) \
558 *tsp = (curkse)->k_kcb->kcb_kmbx.km_timeofday; \
559 if ((tsp)->tv_sec == 0) \
560 clock_gettime(CLOCK_REALTIME, tsp); \
563 struct pthread_rwlockattr {
567 struct pthread_rwlock {
568 pthread_mutex_t lock; /* monitor lock */
569 pthread_cond_t read_signal;
570 pthread_cond_t write_signal;
571 int state; /* 0 = idle >0 = # of readers -1 = writer */
593 struct sigwait_data {
595 siginfo_t *siginfo; /* used to save siginfo for sigwaitinfo() */
598 union pthread_wait_data {
599 pthread_mutex_t mutex;
602 struct sigwait_data *sigwait;
606 * Define a continuation routine that can be used to perform a
607 * transfer of control:
609 typedef void (*thread_continuation_t) (void *);
612 * This stores a thread's state prior to running a signal handler.
613 * It is used when a signal is delivered to a thread blocked in
614 * userland. If the signal handler returns normally, the thread's
615 * state is restored from here.
617 struct pthread_sigframe {
624 enum pthread_state psf_state;
625 union pthread_wait_data psf_wait_data;
626 struct timespec psf_wakeup_time;
628 sigset_t psf_sigmask;
630 thread_continuation_t psf_continuation;
634 struct pthread *thread;
639 struct pthread_specific_elem {
645 volatile int allocated;
648 void (*destructor) (void *);
651 #define MAX_THR_LOCKLEVEL 5
656 /* Thread control block */
660 * Magic value to help recognize a valid thread structure
661 * from an invalid one:
663 #define THR_MAGIC ((u_int32_t) 0xd09ba115)
666 u_int64_t uniqueid; /* for gdb */
668 /* Queue entry for list of all threads: */
669 TAILQ_ENTRY(pthread) tle; /* link for all threads in process */
670 TAILQ_ENTRY(pthread) kle; /* link for all threads in KSE/KSEG */
672 /* Queue entry for GC lists: */
673 TAILQ_ENTRY(pthread) gcle;
675 /* Hash queue entry */
676 LIST_ENTRY(pthread) hle;
679 * Lock for accesses to this thread structure.
682 struct lockuser lockusers[MAX_THR_LOCKLEVEL];
684 kse_critical_t critical[MAX_KSE_LOCKLEVEL];
686 struct kse_group *kseg;
689 * Thread start routine, argument, stack pointer and thread
692 void *(*start_routine)(void *);
694 struct pthread_attr attr;
696 int active; /* thread running */
697 int blocked; /* thread blocked in kernel */
701 * Used for tracking delivery of signal handlers.
704 thread_continuation_t sigbackout;
707 * Cancelability flags - the lower 2 bits are used by cancel
708 * definitions in pthread.h
710 #define THR_AT_CANCEL_POINT 0x0004
711 #define THR_CANCELLING 0x0008
712 #define THR_CANCEL_NEEDED 0x0010
715 thread_continuation_t continuation;
718 * The thread's base and pending signal masks. The active
719 * signal mask is stored in the thread's context (in mailbox).
723 sigset_t *oldsigmask;
724 volatile int check_pending;
728 enum pthread_state state;
729 volatile int lock_switch;
732 * Number of microseconds accumulated by this thread when
733 * time slicing is active.
738 * Time to wake up thread. This is used for sleeping threads and
739 * for any operation which may time out (such as select).
741 struct timespec wakeup_time;
743 /* TRUE if operation has timed out. */
747 * Error variable used instead of errno. The function __error()
748 * returns a pointer to this.
753 * The joiner is the thread that is joining to this thread. The
754 * join status keeps track of a join operation to another thread.
756 struct pthread *joiner;
757 struct join_status join_status;
760 * The current thread can belong to only one scheduling queue at
761 * a time (ready or waiting queue). It can also belong to:
763 * o A queue of threads waiting for a mutex
764 * o A queue of threads waiting for a condition variable
766 * It is possible for a thread to belong to more than one of the
767 * above queues if it is handling a signal. A thread may only
768 * enter a mutex or condition variable queue when it is not
769 * being called from a signal handler. If a thread is a member
770 * of one of these queues when a signal handler is invoked, it
771 * must be removed from the queue before invoking the handler
772 * and then added back to the queue after return from the handler.
774 * Use pqe for the scheduling queue link (both ready and waiting),
775 * sqe for synchronization (mutex, condition variable, and join)
776 * queue links, and qe for all other links.
778 TAILQ_ENTRY(pthread) pqe; /* priority, wait queues link */
779 TAILQ_ENTRY(pthread) sqe; /* synchronization queue link */
782 union pthread_wait_data data;
785 * Set to TRUE if a blocking operation was
786 * interrupted by a signal:
791 * Set to non-zero when this thread has entered a critical
792 * region. We allow for recursive entries into critical regions.
797 * Set to TRUE if this thread should yield after leaving a
798 * critical region to check for signals, messages, etc.
803 #define THR_FLAGS_IN_SYNCQ 0x0001
805 /* Miscellaneous flags; only set with scheduling lock held. */
807 #define THR_FLAGS_PRIVATE 0x0001
808 #define THR_FLAGS_IN_WAITQ 0x0002 /* in waiting queue using pqe link */
809 #define THR_FLAGS_IN_RUNQ 0x0004 /* in run queue using pqe link */
810 #define THR_FLAGS_EXITING 0x0008 /* thread is exiting */
811 #define THR_FLAGS_SUSPENDED 0x0010 /* thread is suspended */
813 /* Thread list flags; only set with thread list lock held. */
814 #define TLFLAGS_GC_SAFE 0x0001 /* thread safe for cleaning */
815 #define TLFLAGS_IN_TDLIST 0x0002 /* thread in all thread list */
816 #define TLFLAGS_IN_GCLIST 0x0004 /* thread in gc list */
820 * Base priority is the user setable and retrievable priority
821 * of the thread. It is only affected by explicit calls to
822 * set thread priority and upon thread creation via a thread
823 * attribute or default priority.
828 * Inherited priority is the priority a thread inherits by
829 * taking a priority inheritence or protection mutex. It
830 * is not affected by base priority changes. Inherited
831 * priority defaults to and remains 0 until a mutex is taken
832 * that is being waited on by any other thread whose priority
835 char inherited_priority;
838 * Active priority is always the maximum of the threads base
839 * priority and inherited priority. When there is a change
840 * in either the base or inherited priority, the active
841 * priority must be recalculated.
843 char active_priority;
845 /* Number of priority ceiling or protection mutexes owned. */
846 int priority_mutex_count;
848 /* Number rwlocks rdlocks held. */
852 * Queue of currently owned mutexes.
854 TAILQ_HEAD(, pthread_mutex) mutexq;
857 struct pthread_specific_elem *specific;
858 int specific_data_count;
860 /* Alternative stack for sigaltstack() */
864 * Current locks bitmap for rtld.
868 /* Cleanup handlers Link List */
869 struct pthread_cleanup *cleanup;
870 char *fname; /* Ptr to source file name */
871 int lineno; /* Source line number. */
875 * Critical regions can also be detected by looking at the threads
876 * current lock level. Ensure these macros increment and decrement
877 * the lock levels such that locks can not be held with a lock level
880 #define THR_IN_CRITICAL(thrd) \
881 (((thrd)->locklevel > 0) || \
882 ((thrd)->critical_count > 0))
884 #define THR_YIELD_CHECK(thrd) \
886 if (!THR_IN_CRITICAL(thrd)) { \
887 if (__predict_false(_libkse_debug)) \
888 _thr_debug_check_yield(thrd); \
889 if ((thrd)->critical_yield != 0) \
890 _thr_sched_switch(thrd); \
891 if ((thrd)->check_pending != 0) \
892 _thr_sig_check_pending(thrd); \
896 #define THR_LOCK_ACQUIRE(thrd, lck) \
898 if ((thrd)->locklevel < MAX_THR_LOCKLEVEL) { \
899 THR_DEACTIVATE_LAST_LOCK(thrd); \
900 (thrd)->locklevel++; \
901 _lock_acquire((lck), \
902 &(thrd)->lockusers[(thrd)->locklevel - 1], \
903 (thrd)->active_priority); \
905 PANIC("Exceeded maximum lock level"); \
908 #define THR_LOCK_RELEASE(thrd, lck) \
910 if ((thrd)->locklevel > 0) { \
911 _lock_release((lck), \
912 &(thrd)->lockusers[(thrd)->locklevel - 1]); \
913 (thrd)->locklevel--; \
914 THR_ACTIVATE_LAST_LOCK(thrd); \
915 if ((thrd)->locklevel == 0) \
916 THR_YIELD_CHECK(thrd); \
920 #define THR_ACTIVATE_LAST_LOCK(thrd) \
922 if ((thrd)->locklevel > 0) \
923 _lockuser_setactive( \
924 &(thrd)->lockusers[(thrd)->locklevel - 1], 1); \
927 #define THR_DEACTIVATE_LAST_LOCK(thrd) \
929 if ((thrd)->locklevel > 0) \
930 _lockuser_setactive( \
931 &(thrd)->lockusers[(thrd)->locklevel - 1], 0); \
935 * For now, threads will have their own lock separate from their
936 * KSE scheduling lock.
938 #define THR_LOCK(thr) THR_LOCK_ACQUIRE(thr, &(thr)->lock)
939 #define THR_UNLOCK(thr) THR_LOCK_RELEASE(thr, &(thr)->lock)
940 #define THR_THREAD_LOCK(curthrd, thr) THR_LOCK_ACQUIRE(curthrd, &(thr)->lock)
941 #define THR_THREAD_UNLOCK(curthrd, thr) THR_LOCK_RELEASE(curthrd, &(thr)->lock)
944 * Priority queue manipulation macros (using pqe link). We use
945 * the thread's kseg link instead of the kse link because a thread
946 * does not (currently) have a statically assigned kse.
948 #define THR_RUNQ_INSERT_HEAD(thrd) \
949 _pq_insert_head(&(thrd)->kseg->kg_schedq.sq_runq, thrd)
950 #define THR_RUNQ_INSERT_TAIL(thrd) \
951 _pq_insert_tail(&(thrd)->kseg->kg_schedq.sq_runq, thrd)
952 #define THR_RUNQ_REMOVE(thrd) \
953 _pq_remove(&(thrd)->kseg->kg_schedq.sq_runq, thrd)
956 * Macros to insert/remove threads to the all thread list and
959 #define THR_LIST_ADD(thrd) do { \
960 if (((thrd)->tlflags & TLFLAGS_IN_TDLIST) == 0) { \
961 TAILQ_INSERT_HEAD(&_thread_list, thrd, tle); \
962 _thr_hash_add(thrd); \
963 (thrd)->tlflags |= TLFLAGS_IN_TDLIST; \
966 #define THR_LIST_REMOVE(thrd) do { \
967 if (((thrd)->tlflags & TLFLAGS_IN_TDLIST) != 0) { \
968 TAILQ_REMOVE(&_thread_list, thrd, tle); \
969 _thr_hash_remove(thrd); \
970 (thrd)->tlflags &= ~TLFLAGS_IN_TDLIST; \
973 #define THR_GCLIST_ADD(thrd) do { \
974 if (((thrd)->tlflags & TLFLAGS_IN_GCLIST) == 0) { \
975 TAILQ_INSERT_HEAD(&_thread_gc_list, thrd, gcle);\
976 (thrd)->tlflags |= TLFLAGS_IN_GCLIST; \
980 #define THR_GCLIST_REMOVE(thrd) do { \
981 if (((thrd)->tlflags & TLFLAGS_IN_GCLIST) != 0) { \
982 TAILQ_REMOVE(&_thread_gc_list, thrd, gcle); \
983 (thrd)->tlflags &= ~TLFLAGS_IN_GCLIST; \
988 #define GC_NEEDED() (atomic_load_acq_int(&_gc_count) >= 5)
991 * Locking the scheduling queue for another thread uses that thread's
994 #define THR_SCHED_LOCK(curthr, thr) do { \
995 (curthr)->critical[(curthr)->locklevel] = _kse_critical_enter(); \
996 (curthr)->locklevel++; \
997 KSE_SCHED_LOCK((curthr)->kse, (thr)->kseg); \
1000 #define THR_SCHED_UNLOCK(curthr, thr) do { \
1001 KSE_SCHED_UNLOCK((curthr)->kse, (thr)->kseg); \
1002 (curthr)->locklevel--; \
1003 _kse_critical_leave((curthr)->critical[(curthr)->locklevel]); \
1006 /* Take the scheduling lock with the intent to call the scheduler. */
1007 #define THR_LOCK_SWITCH(curthr) do { \
1008 (void)_kse_critical_enter(); \
1009 KSE_SCHED_LOCK((curthr)->kse, (curthr)->kseg); \
1011 #define THR_UNLOCK_SWITCH(curthr) do { \
1012 KSE_SCHED_UNLOCK((curthr)->kse, (curthr)->kseg);\
1015 #define THR_CRITICAL_ENTER(thr) (thr)->critical_count++
1016 #define THR_CRITICAL_LEAVE(thr) do { \
1017 (thr)->critical_count--; \
1018 if (((thr)->critical_yield != 0) && \
1019 ((thr)->critical_count == 0)) { \
1020 (thr)->critical_yield = 0; \
1021 _thr_sched_switch(thr); \
1025 #define THR_IS_ACTIVE(thrd) \
1026 ((thrd)->kse != NULL) && ((thrd)->kse->k_curthread == (thrd))
1028 #define THR_IN_SYNCQ(thrd) (((thrd)->sflags & THR_FLAGS_IN_SYNCQ) != 0)
1030 #define THR_IS_SUSPENDED(thrd) \
1031 (((thrd)->state == PS_SUSPENDED) || \
1032 (((thrd)->flags & THR_FLAGS_SUSPENDED) != 0))
1033 #define THR_IS_EXITING(thrd) (((thrd)->flags & THR_FLAGS_EXITING) != 0)
1034 #define DBG_CAN_RUN(thrd) (((thrd)->tcb->tcb_tmbx.tm_dflags & \
1037 extern int __isthreaded;
1040 _kse_isthreaded(void)
1042 return (__isthreaded != 0);
1046 * Global variables for the pthread kernel.
1049 SCLASS void *_usrstack SCLASS_PRESET(NULL);
1050 SCLASS struct kse *_kse_initial SCLASS_PRESET(NULL);
1051 SCLASS struct pthread *_thr_initial SCLASS_PRESET(NULL);
1053 SCLASS int _libkse_debug SCLASS_PRESET(0);
1054 SCLASS int _thread_activated SCLASS_PRESET(0);
1055 SCLASS int _thread_scope_system SCLASS_PRESET(0);
1057 /* List of all threads: */
1058 SCLASS TAILQ_HEAD(, pthread) _thread_list
1059 SCLASS_PRESET(TAILQ_HEAD_INITIALIZER(_thread_list));
1061 /* List of threads needing GC: */
1062 SCLASS TAILQ_HEAD(, pthread) _thread_gc_list
1063 SCLASS_PRESET(TAILQ_HEAD_INITIALIZER(_thread_gc_list));
1065 SCLASS int _thread_active_threads SCLASS_PRESET(1);
1067 SCLASS TAILQ_HEAD(atfork_head, pthread_atfork) _thr_atfork_list;
1068 SCLASS pthread_mutex_t _thr_atfork_mutex;
1070 /* Default thread attributes: */
1071 SCLASS struct pthread_attr _pthread_attr_default
1073 SCHED_RR, 0, TIMESLICE_USEC, THR_DEFAULT_PRIORITY,
1074 THR_CREATE_RUNNING, PTHREAD_CREATE_JOINABLE, NULL,
1075 NULL, NULL, /* stacksize */0, /* guardsize */0
1078 /* Default mutex attributes: */
1079 SCLASS struct pthread_mutex_attr _pthread_mutexattr_default
1080 SCLASS_PRESET({PTHREAD_MUTEX_DEFAULT, PTHREAD_PRIO_NONE, 0, 0 });
1082 /* Default condition variable attributes: */
1083 SCLASS struct pthread_cond_attr _pthread_condattr_default
1084 SCLASS_PRESET({COND_TYPE_FAST, 0});
1086 /* Clock resolution in usec. */
1087 SCLASS int _clock_res_usec SCLASS_PRESET(CLOCK_RES_USEC);
1089 /* Array of signal actions for this process: */
1090 SCLASS struct sigaction _thread_sigact[_SIG_MAXSIG];
1093 * Lock for above count of dummy handlers and for the process signal
1094 * mask and pending signal sets.
1096 SCLASS struct lock _thread_signal_lock;
1098 /* Pending signals and mask for this process: */
1099 SCLASS sigset_t _thr_proc_sigpending;
1100 SCLASS siginfo_t _thr_proc_siginfo[_SIG_MAXSIG];
1102 SCLASS pid_t _thr_pid SCLASS_PRESET(0);
1104 /* Garbage collector lock. */
1105 SCLASS struct lock _gc_lock;
1106 SCLASS int _gc_check SCLASS_PRESET(0);
1107 SCLASS int _gc_count SCLASS_PRESET(0);
1109 SCLASS struct lock _mutex_static_lock;
1110 SCLASS struct lock _rwlock_static_lock;
1111 SCLASS struct lock _keytable_lock;
1112 SCLASS struct lock _thread_list_lock;
1113 SCLASS int _thr_guard_default;
1114 SCLASS int _thr_stack_default;
1115 SCLASS int _thr_stack_initial;
1116 SCLASS int _thr_page_size;
1117 SCLASS pthread_t _thr_sig_daemon;
1118 SCLASS int _thr_debug_flags SCLASS_PRESET(0);
1120 /* Undefine the storage class and preset specifiers: */
1122 #undef SCLASS_PRESET
1126 * Function prototype definitions.
1129 int _cond_reinit(pthread_cond_t *);
1130 struct kse *_kse_alloc(struct pthread *, int sys_scope);
1131 kse_critical_t _kse_critical_enter(void);
1132 void _kse_critical_leave(kse_critical_t);
1133 int _kse_in_critical(void);
1134 void _kse_free(struct pthread *, struct kse *);
1135 void _kse_init(void);
1136 struct kse_group *_kseg_alloc(struct pthread *);
1137 void _kse_lock_wait(struct lock *, struct lockuser *lu);
1138 void _kse_lock_wakeup(struct lock *, struct lockuser *lu);
1139 void _kse_single_thread(struct pthread *);
1140 int _kse_setthreaded(int);
1141 void _kseg_free(struct kse_group *);
1142 int _mutex_cv_lock(pthread_mutex_t *);
1143 int _mutex_cv_unlock(pthread_mutex_t *);
1144 void _mutex_notify_priochange(struct pthread *, struct pthread *, int);
1145 int _mutex_reinit(struct pthread_mutex *);
1146 void _mutex_unlock_private(struct pthread *);
1147 void _libpthread_init(struct pthread *);
1148 int _pq_alloc(struct pq_queue *, int, int);
1149 void _pq_free(struct pq_queue *);
1150 int _pq_init(struct pq_queue *);
1151 void _pq_remove(struct pq_queue *pq, struct pthread *);
1152 void _pq_insert_head(struct pq_queue *pq, struct pthread *);
1153 void _pq_insert_tail(struct pq_queue *pq, struct pthread *);
1154 struct pthread *_pq_first(struct pq_queue *pq);
1155 struct pthread *_pq_first_debug(struct pq_queue *pq);
1156 void *_pthread_getspecific(pthread_key_t);
1157 int _pthread_key_create(pthread_key_t *, void (*) (void *));
1158 int _pthread_key_delete(pthread_key_t);
1159 int _pthread_mutex_destroy(pthread_mutex_t *);
1160 int _pthread_mutex_init(pthread_mutex_t *, const pthread_mutexattr_t *);
1161 int _pthread_mutex_lock(pthread_mutex_t *);
1162 int _pthread_mutex_trylock(pthread_mutex_t *);
1163 int _pthread_mutex_unlock(pthread_mutex_t *);
1164 int _pthread_mutexattr_init(pthread_mutexattr_t *);
1165 int _pthread_mutexattr_destroy(pthread_mutexattr_t *);
1166 int _pthread_mutexattr_settype(pthread_mutexattr_t *, int);
1167 int _pthread_once(pthread_once_t *, void (*) (void));
1168 int _pthread_rwlock_init(pthread_rwlock_t *, const pthread_rwlockattr_t *);
1169 int _pthread_rwlock_destroy (pthread_rwlock_t *);
1170 struct pthread *_pthread_self(void);
1171 int _pthread_setspecific(pthread_key_t, const void *);
1172 void _pthread_yield(void);
1173 void _pthread_cleanup_push(void (*routine) (void *), void *routine_arg);
1174 void _pthread_cleanup_pop(int execute);
1175 struct pthread *_thr_alloc(struct pthread *);
1176 void _thr_exit(char *, int, char *);
1177 void _thr_exit_cleanup(void);
1178 void _thr_lock_wait(struct lock *lock, struct lockuser *lu);
1179 void _thr_lock_wakeup(struct lock *lock, struct lockuser *lu);
1180 void _thr_mutex_reinit(pthread_mutex_t *);
1181 int _thr_ref_add(struct pthread *, struct pthread *, int);
1182 void _thr_ref_delete(struct pthread *, struct pthread *);
1183 void _thr_rtld_init(void);
1184 void _thr_rtld_fini(void);
1185 int _thr_schedule_add(struct pthread *, struct pthread *);
1186 void _thr_schedule_remove(struct pthread *, struct pthread *);
1187 void _thr_setrunnable(struct pthread *curthread, struct pthread *thread);
1188 struct kse_mailbox *_thr_setrunnable_unlocked(struct pthread *thread);
1189 struct kse_mailbox *_thr_sig_add(struct pthread *, int, siginfo_t *);
1190 void _thr_sig_dispatch(struct kse *, int, siginfo_t *);
1191 int _thr_stack_alloc(struct pthread_attr *);
1192 void _thr_stack_free(struct pthread_attr *);
1193 void _thr_exit_cleanup(void);
1194 void _thr_free(struct pthread *, struct pthread *);
1195 void _thr_gc(struct pthread *);
1196 void _thr_panic_exit(char *, int, char *);
1197 void _thread_cleanupspecific(void);
1198 void _thread_dump_info(void);
1199 void _thread_printf(int, const char *, ...);
1200 void _thr_sched_switch(struct pthread *);
1201 void _thr_sched_switch_unlocked(struct pthread *);
1202 void _thr_set_timeout(const struct timespec *);
1203 void _thr_seterrno(struct pthread *, int);
1204 void _thr_sig_handler(int, siginfo_t *, ucontext_t *);
1205 void _thr_sig_check_pending(struct pthread *);
1206 void _thr_sig_rundown(struct pthread *, ucontext_t *);
1207 void _thr_sig_send(struct pthread *pthread, int sig);
1208 void _thr_sigframe_restore(struct pthread *thread, struct pthread_sigframe *psf);
1209 void _thr_spinlock_init(void);
1210 void _thr_cancel_enter(struct pthread *);
1211 void _thr_cancel_leave(struct pthread *, int);
1212 int _thr_setconcurrency(int new_level);
1213 int _thr_setmaxconcurrency(void);
1214 void _thr_critical_enter(struct pthread *);
1215 void _thr_critical_leave(struct pthread *);
1216 int _thr_start_sig_daemon(void);
1217 int _thr_getprocsig(int sig, siginfo_t *siginfo);
1218 int _thr_getprocsig_unlocked(int sig, siginfo_t *siginfo);
1219 void _thr_signal_init(void);
1220 void _thr_signal_deinit(void);
1221 void _thr_hash_add(struct pthread *);
1222 void _thr_hash_remove(struct pthread *);
1223 struct pthread *_thr_hash_find(struct pthread *);
1224 void _thr_finish_cancellation(void *arg);
1225 int _thr_sigonstack(void *sp);
1226 void _thr_debug_check_yield(struct pthread *);
1229 * Aliases for _pthread functions. Should be called instead of
1230 * originals if PLT replocation is unwanted at runtme.
1232 int _thr_cond_broadcast(pthread_cond_t *);
1233 int _thr_cond_signal(pthread_cond_t *);
1234 int _thr_cond_wait(pthread_cond_t *, pthread_mutex_t *);
1235 int _thr_mutex_lock(pthread_mutex_t *);
1236 int _thr_mutex_unlock(pthread_mutex_t *);
1237 int _thr_rwlock_rdlock (pthread_rwlock_t *);
1238 int _thr_rwlock_wrlock (pthread_rwlock_t *);
1239 int _thr_rwlock_unlock (pthread_rwlock_t *);
1241 /* #include <sys/aio.h> */
1243 int __sys_aio_suspend(const struct aiocb * const[], int, const struct timespec *);
1246 /* #include <fcntl.h> */
1247 #ifdef _SYS_FCNTL_H_
1248 int __sys_fcntl(int, int, ...);
1249 int __sys_open(const char *, int, ...);
1252 /* #include <sys/ioctl.h> */
1253 #ifdef _SYS_IOCTL_H_
1254 int __sys_ioctl(int, unsigned long, ...);
1257 /* #inclde <sched.h> */
1259 int __sys_sched_yield(void);
1262 /* #include <signal.h> */
1264 int __sys_kill(pid_t, int);
1265 int __sys_sigaction(int, const struct sigaction *, struct sigaction *);
1266 int __sys_sigpending(sigset_t *);
1267 int __sys_sigprocmask(int, const sigset_t *, sigset_t *);
1268 int __sys_sigsuspend(const sigset_t *);
1269 int __sys_sigreturn(ucontext_t *);
1270 int __sys_sigaltstack(const struct sigaltstack *, struct sigaltstack *);
1273 /* #include <sys/socket.h> */
1274 #ifdef _SYS_SOCKET_H_
1275 int __sys_accept(int, struct sockaddr *, socklen_t *);
1276 int __sys_connect(int, const struct sockaddr *, socklen_t);
1277 int __sys_sendfile(int, int, off_t, size_t, struct sf_hdtr *,
1281 /* #include <sys/uio.h> */
1283 ssize_t __sys_readv(int, const struct iovec *, int);
1284 ssize_t __sys_writev(int, const struct iovec *, int);
1287 /* #include <time.h> */
1289 int __sys_nanosleep(const struct timespec *, struct timespec *);
1292 /* #include <unistd.h> */
1294 int __sys_close(int);
1295 int __sys_execve(const char *, char * const *, char * const *);
1296 int __sys_fork(void);
1297 int __sys_fsync(int);
1298 pid_t __sys_getpid(void);
1299 int __sys_select(int, fd_set *, fd_set *, fd_set *, struct timeval *);
1300 ssize_t __sys_read(int, void *, size_t);
1301 ssize_t __sys_write(int, const void *, size_t);
1302 void __sys_exit(int);
1303 int __sys_sigwait(const sigset_t *, int *);
1304 int __sys_sigtimedwait(sigset_t *, siginfo_t *, struct timespec *);
1307 /* #include <poll.h> */
1309 int __sys_poll(struct pollfd *, unsigned, int);
1312 /* #include <sys/mman.h> */
1314 int __sys_msync(void *, size_t, int);
1318 _thr_dump_enabled(void)
1321 return ((_thr_debug_flags & DBG_INFO_DUMP) != 0);
1324 #endif /* !_THR_PRIVATE_H */