4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
21 * Portions Copyright 2010 The FreeBSD Foundation
27 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
28 * Use is subject to license terms.
32 * Copyright (c) 2015, Joyent, Inc. All rights reserved.
35 #include <sys/atomic.h>
36 #include <sys/errno.h>
38 #include <sys/modctl.h>
40 #include <sys/systm.h>
44 #include <sys/sunddi.h>
45 #include <sys/cpuvar.h>
48 #include <sys/strsubr.h>
50 #include <sys/fasttrap.h>
51 #include <sys/fasttrap_impl.h>
52 #include <sys/fasttrap_isa.h>
53 #include <sys/dtrace.h>
54 #include <sys/dtrace_impl.h>
55 #include <sys/sysmacros.h>
57 #include <sys/policy.h>
59 #include <util/qsort.h>
61 #include <sys/mutex.h>
62 #include <sys/kernel.h>
64 #include <sys/dtrace_bsd.h>
65 #include <sys/eventhandler.h>
66 #include <sys/rmlock.h>
67 #include <sys/sysent.h>
68 #include <sys/sysctl.h>
69 #include <sys/u8_textprep.h>
74 #include <vm/vm_map.h>
75 #include <vm/vm_param.h>
77 #include <cddl/dev/dtrace/dtrace_cddl.h>
81 * User-Land Trap-Based Tracing
82 * ----------------------------
84 * The fasttrap provider allows DTrace consumers to instrument any user-level
85 * instruction to gather data; this includes probes with semantic
86 * signifigance like entry and return as well as simple offsets into the
87 * function. While the specific techniques used are very ISA specific, the
88 * methodology is generalizable to any architecture.
91 * The General Methodology
92 * -----------------------
94 * With the primary goal of tracing every user-land instruction and the
95 * limitation that we can't trust user space so don't want to rely on much
96 * information there, we begin by replacing the instructions we want to trace
97 * with trap instructions. Each instruction we overwrite is saved into a hash
98 * table keyed by process ID and pc address. When we enter the kernel due to
99 * this trap instruction, we need the effects of the replaced instruction to
100 * appear to have occurred before we proceed with the user thread's
103 * Each user level thread is represented by a ulwp_t structure which is
104 * always easily accessible through a register. The most basic way to produce
105 * the effects of the instruction we replaced is to copy that instruction out
106 * to a bit of scratch space reserved in the user thread's ulwp_t structure
107 * (a sort of kernel-private thread local storage), set the PC to that
108 * scratch space and single step. When we reenter the kernel after single
109 * stepping the instruction we must then adjust the PC to point to what would
110 * normally be the next instruction. Of course, special care must be taken
111 * for branches and jumps, but these represent such a small fraction of any
112 * instruction set that writing the code to emulate these in the kernel is
115 * Return probes may require several tracepoints to trace every return site,
116 * and, conversely, each tracepoint may activate several probes (the entry
117 * and offset 0 probes, for example). To solve this muliplexing problem,
118 * tracepoints contain lists of probes to activate and probes contain lists
119 * of tracepoints to enable. If a probe is activated, it adds its ID to
120 * existing tracepoints or creates new ones as necessary.
122 * Most probes are activated _before_ the instruction is executed, but return
123 * probes are activated _after_ the effects of the last instruction of the
124 * function are visible. Return probes must be fired _after_ we have
125 * single-stepped the instruction whereas all other probes are fired
132 * The lock ordering below -- both internally and with respect to the DTrace
133 * framework -- is a little tricky and bears some explanation. Each provider
134 * has a lock (ftp_mtx) that protects its members including reference counts
135 * for enabled probes (ftp_rcount), consumers actively creating probes
136 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider
137 * from being freed. A provider is looked up by taking the bucket lock for the
138 * provider hash table, and is returned with its lock held. The provider lock
139 * may be taken in functions invoked by the DTrace framework, but may not be
140 * held while calling functions in the DTrace framework.
142 * To ensure consistency over multiple calls to the DTrace framework, the
143 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may
144 * not be taken when holding the provider lock as that would create a cyclic
145 * lock ordering. In situations where one would naturally take the provider
146 * lock and then the creation lock, we instead up a reference count to prevent
147 * the provider from disappearing, drop the provider lock, and acquire the
151 * bucket lock before provider lock
152 * DTrace before provider lock
153 * creation lock before DTrace
154 * never hold the provider lock and creation lock simultaneously
157 static d_open_t fasttrap_open;
158 static d_ioctl_t fasttrap_ioctl;
160 static struct cdevsw fasttrap_cdevsw = {
161 .d_version = D_VERSION,
162 .d_open = fasttrap_open,
163 .d_ioctl = fasttrap_ioctl,
164 .d_name = "fasttrap",
166 static struct cdev *fasttrap_cdev;
167 static dtrace_meta_provider_id_t fasttrap_meta_id;
169 static struct proc *fasttrap_cleanup_proc;
170 static struct mtx fasttrap_cleanup_mtx;
171 static uint_t fasttrap_cleanup_work, fasttrap_cleanup_drain, fasttrap_cleanup_cv;
174 * Generation count on modifications to the global tracepoint lookup table.
176 static volatile uint64_t fasttrap_mod_gen;
179 * When the fasttrap provider is loaded, fasttrap_max is set to either
180 * FASTTRAP_MAX_DEFAULT, or the value for fasttrap-max-probes in the
181 * fasttrap.conf file (Illumos), or the value provied in the loader.conf (FreeBSD).
182 * Each time a probe is created, fasttrap_total is incremented by the number
183 * of tracepoints that may be associated with that probe; fasttrap_total is capped
186 #define FASTTRAP_MAX_DEFAULT 250000
187 static uint32_t fasttrap_max = FASTTRAP_MAX_DEFAULT;
188 static uint32_t fasttrap_total;
191 * Copyright (c) 2011, Joyent, Inc. All rights reserved.
194 #define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000
195 #define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100
196 #define FASTTRAP_PROCS_DEFAULT_SIZE 0x100
198 #define FASTTRAP_PID_NAME "pid"
200 fasttrap_hash_t fasttrap_tpoints;
201 static fasttrap_hash_t fasttrap_provs;
202 static fasttrap_hash_t fasttrap_procs;
204 static uint64_t fasttrap_pid_count; /* pid ref count */
205 static kmutex_t fasttrap_count_mtx; /* lock on ref count */
207 #define FASTTRAP_ENABLE_FAIL 1
208 #define FASTTRAP_ENABLE_PARTIAL 2
210 static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
211 static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
213 static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
214 const dtrace_pattr_t *);
215 static void fasttrap_provider_retire(pid_t, const char *, int);
216 static void fasttrap_provider_free(fasttrap_provider_t *);
218 static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
219 static void fasttrap_proc_release(fasttrap_proc_t *);
222 static void fasttrap_thread_dtor(void *, struct thread *);
225 #define FASTTRAP_PROVS_INDEX(pid, name) \
226 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
228 #define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
231 struct rmlock fasttrap_tp_lock;
232 static eventhandler_tag fasttrap_thread_dtor_tag;
235 static unsigned long tpoints_hash_size = FASTTRAP_TPOINTS_DEFAULT_SIZE;
238 SYSCTL_DECL(_kern_dtrace);
239 SYSCTL_NODE(_kern_dtrace, OID_AUTO, fasttrap, CTLFLAG_RD, 0, "DTrace fasttrap parameters");
240 SYSCTL_UINT(_kern_dtrace_fasttrap, OID_AUTO, max_probes, CTLFLAG_RWTUN, &fasttrap_max,
241 FASTTRAP_MAX_DEFAULT, "Maximum number of fasttrap probes");
242 SYSCTL_ULONG(_kern_dtrace_fasttrap, OID_AUTO, tpoints_hash_size, CTLFLAG_RDTUN, &tpoints_hash_size,
243 FASTTRAP_TPOINTS_DEFAULT_SIZE, "Size of the tracepoint hash table");
247 fasttrap_highbit(ulong_t i)
254 if (i & 0xffffffff00000000ul) {
258 if (i & 0xffff0000) {
277 fasttrap_hash_str(const char *p)
283 hval = (hval << 4) + *p++;
284 if ((g = (hval & 0xf0000000)) != 0)
292 fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
295 sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
297 sqp->sq_info.si_signo = SIGTRAP;
298 sqp->sq_info.si_code = TRAP_DTRACE;
299 sqp->sq_info.si_addr = (caddr_t)pc;
301 mutex_enter(&p->p_lock);
303 mutex_exit(&p->p_lock);
308 ksiginfo_t *ksi = kmem_zalloc(sizeof (ksiginfo_t), KM_SLEEP);
311 ksi->ksi_signo = SIGTRAP;
312 ksi->ksi_code = TRAP_DTRACE;
313 ksi->ksi_addr = (caddr_t)pc;
315 (void) tdsendsignal(p, t, SIGTRAP, ksi);
322 * Obtain a chunk of scratch space in the address space of the target process.
324 fasttrap_scrspace_t *
325 fasttrap_scraddr(struct thread *td, fasttrap_proc_t *fprc)
327 fasttrap_scrblock_t *scrblk;
328 fasttrap_scrspace_t *scrspc;
334 if (td->t_dtrace_sscr != NULL) {
335 /* If the thread already has scratch space, we're done. */
336 scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr;
342 mutex_enter(&fprc->ftpc_mtx);
343 if (LIST_EMPTY(&fprc->ftpc_fscr)) {
345 * No scratch space is available, so we'll map a new scratch
346 * space block into the traced process' address space.
349 error = vm_map_find(&p->p_vmspace->vm_map, NULL, 0, &addr,
350 FASTTRAP_SCRBLOCK_SIZE, 0, VMFS_ANY_SPACE, VM_PROT_ALL,
352 if (error != KERN_SUCCESS)
355 scrblk = malloc(sizeof(*scrblk), M_SOLARIS, M_WAITOK);
356 scrblk->ftsb_addr = addr;
357 LIST_INSERT_HEAD(&fprc->ftpc_scrblks, scrblk, ftsb_next);
360 * Carve the block up into chunks and put them on the free list.
363 i < FASTTRAP_SCRBLOCK_SIZE / FASTTRAP_SCRSPACE_SIZE; i++) {
364 scrspc = malloc(sizeof(*scrspc), M_SOLARIS, M_WAITOK);
365 scrspc->ftss_addr = addr +
366 i * FASTTRAP_SCRSPACE_SIZE;
367 LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc,
373 * Take the first scratch chunk off the free list, put it on the
374 * allocated list, and return its address.
376 scrspc = LIST_FIRST(&fprc->ftpc_fscr);
377 LIST_REMOVE(scrspc, ftss_next);
378 LIST_INSERT_HEAD(&fprc->ftpc_ascr, scrspc, ftss_next);
381 * This scratch space is reserved for use by td until the thread exits.
383 td->t_dtrace_sscr = scrspc;
386 mutex_exit(&fprc->ftpc_mtx);
392 * Return any allocated per-thread scratch space chunks back to the process'
396 fasttrap_thread_dtor(void *arg __unused, struct thread *td)
398 fasttrap_bucket_t *bucket;
399 fasttrap_proc_t *fprc;
400 fasttrap_scrspace_t *scrspc;
403 if (td->t_dtrace_sscr == NULL)
406 pid = td->td_proc->p_pid;
407 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
410 /* Look up the fasttrap process handle for this process. */
411 mutex_enter(&bucket->ftb_mtx);
412 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
413 if (fprc->ftpc_pid == pid) {
414 mutex_enter(&fprc->ftpc_mtx);
415 mutex_exit(&bucket->ftb_mtx);
420 mutex_exit(&bucket->ftb_mtx);
424 scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr;
425 LIST_REMOVE(scrspc, ftss_next);
426 LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc, ftss_next);
428 mutex_exit(&fprc->ftpc_mtx);
433 * This function ensures that no threads are actively using the memory
434 * associated with probes that were formerly live.
437 fasttrap_mod_barrier(uint64_t gen)
441 if (gen < fasttrap_mod_gen)
448 mutex_enter(&fasttrap_cpuc_pid_lock[i]);
449 mutex_exit(&fasttrap_cpuc_pid_lock[i]);
452 rm_wlock(&fasttrap_tp_lock);
453 rm_wunlock(&fasttrap_tp_lock);
458 * This function performs asynchronous cleanup of fasttrap providers. The
459 * Solaris implementation of this mechanism use a timeout that's activated in
460 * fasttrap_pid_cleanup(), but this doesn't work in FreeBSD: one may sleep while
461 * holding the DTrace mutexes, but it is unsafe to sleep in a callout handler.
462 * Thus we use a dedicated process to perform the cleanup when requested.
466 fasttrap_pid_cleanup_cb(void *data)
468 fasttrap_provider_t **fpp, *fp;
469 fasttrap_bucket_t *bucket;
470 dtrace_provider_id_t provid;
471 int i, later = 0, rval;
473 mtx_lock(&fasttrap_cleanup_mtx);
474 while (!fasttrap_cleanup_drain || later > 0) {
475 fasttrap_cleanup_work = 0;
476 mtx_unlock(&fasttrap_cleanup_mtx);
481 * Iterate over all the providers trying to remove the marked
482 * ones. If a provider is marked but not retired, we just
483 * have to take a crack at removing it -- it's no big deal if
486 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
487 bucket = &fasttrap_provs.fth_table[i];
488 mutex_enter(&bucket->ftb_mtx);
489 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
491 while ((fp = *fpp) != NULL) {
492 if (!fp->ftp_marked) {
497 mutex_enter(&fp->ftp_mtx);
500 * If this provider has consumers actively
501 * creating probes (ftp_ccount) or is a USDT
502 * provider (ftp_mcount), we can't unregister
505 if (fp->ftp_ccount != 0 ||
506 fp->ftp_mcount != 0) {
507 mutex_exit(&fp->ftp_mtx);
512 if (!fp->ftp_retired || fp->ftp_rcount != 0)
515 mutex_exit(&fp->ftp_mtx);
518 * If we successfully unregister this
519 * provider we can remove it from the hash
520 * chain and free the memory. If our attempt
521 * to unregister fails and this is a retired
522 * provider, increment our flag to try again
523 * pretty soon. If we've consumed more than
524 * half of our total permitted number of
525 * probes call dtrace_condense() to try to
526 * clean out the unenabled probes.
528 provid = fp->ftp_provid;
529 if ((rval = dtrace_unregister(provid)) != 0) {
530 if (fasttrap_total > fasttrap_max / 2)
531 (void) dtrace_condense(provid);
536 later += fp->ftp_marked;
540 fasttrap_provider_free(fp);
543 mutex_exit(&bucket->ftb_mtx);
545 mtx_lock(&fasttrap_cleanup_mtx);
548 * If we were unable to retire a provider, try again after a
549 * second. This situation can occur in certain circumstances
550 * where providers cannot be unregistered even though they have
551 * no probes enabled because of an execution of dtrace -l or
554 if (later > 0 || fasttrap_cleanup_work ||
555 fasttrap_cleanup_drain) {
556 mtx_unlock(&fasttrap_cleanup_mtx);
557 pause("ftclean", hz);
558 mtx_lock(&fasttrap_cleanup_mtx);
560 mtx_sleep(&fasttrap_cleanup_cv, &fasttrap_cleanup_mtx,
565 * Wake up the thread in fasttrap_unload() now that we're done.
567 wakeup(&fasttrap_cleanup_drain);
568 mtx_unlock(&fasttrap_cleanup_mtx);
574 * Activates the asynchronous cleanup mechanism.
577 fasttrap_pid_cleanup(void)
580 mtx_lock(&fasttrap_cleanup_mtx);
581 if (!fasttrap_cleanup_work) {
582 fasttrap_cleanup_work = 1;
583 wakeup(&fasttrap_cleanup_cv);
585 mtx_unlock(&fasttrap_cleanup_mtx);
589 * This is called from cfork() via dtrace_fasttrap_fork(). The child
590 * process's address space is (roughly) a copy of the parent process's so
591 * we have to remove all the instrumentation we had previously enabled in the
595 fasttrap_fork(proc_t *p, proc_t *cp)
598 fasttrap_scrblock_t *scrblk;
599 fasttrap_proc_t *fprc = NULL;
601 pid_t ppid = p->p_pid;
604 ASSERT(curproc == p);
606 ASSERT(p->p_proc_flag & P_PR_LOCK);
608 PROC_LOCK_ASSERT(p, MA_OWNED);
611 ASSERT(p->p_dtrace_count > 0);
614 * This check is purposely here instead of in kern_fork.c because,
615 * for legal resons, we cannot include the dtrace_cddl.h header
616 * inside kern_fork.c and insert if-clause there.
618 if (p->p_dtrace_count == 0 && p->p_dtrace_helpers == NULL)
622 ASSERT(cp->p_dtrace_count == 0);
625 * This would be simpler and faster if we maintained per-process
626 * hash tables of enabled tracepoints. It could, however, potentially
627 * slow down execution of a tracepoint since we'd need to go
628 * through two levels of indirection. In the future, we should
629 * consider either maintaining per-process ancillary lists of
630 * enabled tracepoints or hanging a pointer to a per-process hash
631 * table of enabled tracepoints off the proc structure.
635 * We don't have to worry about the child process disappearing
636 * because we're in fork().
639 mtx_lock_spin(&cp->p_slock);
641 mtx_unlock_spin(&cp->p_slock);
644 * fasttrap_tracepoint_remove() expects the child process to be
645 * unlocked and the VM then expects curproc to be unlocked.
650 if (p->p_dtrace_count == 0)
655 * Iterate over every tracepoint looking for ones that belong to the
656 * parent process, and remove each from the child process.
658 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
659 fasttrap_tracepoint_t *tp;
660 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
662 mutex_enter(&bucket->ftb_mtx);
663 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
664 if (tp->ftt_pid == ppid &&
665 tp->ftt_proc->ftpc_acount != 0) {
666 int ret = fasttrap_tracepoint_remove(cp, tp);
670 * The count of active providers can only be
671 * decremented (i.e. to zero) during exec,
672 * exit, and removal of a meta provider so it
673 * should be impossible to drop the count
676 ASSERT(tp->ftt_proc->ftpc_acount != 0);
682 mutex_exit(&bucket->ftb_mtx);
686 * Unmap any scratch space inherited from the parent's address
690 mutex_enter(&fprc->ftpc_mtx);
691 LIST_FOREACH(scrblk, &fprc->ftpc_scrblks, ftsb_next) {
692 vm_map_remove(&cp->p_vmspace->vm_map,
694 scrblk->ftsb_addr + FASTTRAP_SCRBLOCK_SIZE);
696 mutex_exit(&fprc->ftpc_mtx);
702 mutex_enter(&cp->p_lock);
706 if (p->p_dtrace_helpers != NULL)
707 dtrace_helpers_duplicate(p, cp);
715 * This is called from proc_exit() or from exec_common() if p_dtrace_probes
716 * is set on the proc structure to indicate that there is a pid provider
717 * associated with this process.
720 fasttrap_exec_exit(proc_t *p)
727 ASSERT(p == curproc);
729 PROC_LOCK_ASSERT(p, MA_OWNED);
732 * Since struct threads may be recycled, we cannot rely on t_dtrace_sscr
733 * fields to be zeroed by kdtrace_thread_ctor. Thus we must zero it
734 * ourselves when a process exits.
736 FOREACH_THREAD_IN_PROC(p, td)
737 td->t_dtrace_sscr = NULL;
742 * We clean up the pid provider for this process here; user-land
743 * static probes are handled by the meta-provider remove entry point.
745 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
747 if (p->p_dtrace_helpers)
748 dtrace_helpers_destroy(p);
757 fasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc)
760 * There are no "default" pid probes.
765 fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
767 fasttrap_tracepoint_t *tp, *new_tp = NULL;
768 fasttrap_bucket_t *bucket;
773 ASSERT(index < probe->ftp_ntps);
775 pid = probe->ftp_pid;
776 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
777 id = &probe->ftp_tps[index].fit_id;
779 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
782 ASSERT(!(p->p_flag & SVFORK));
786 * Before we make any modifications, make sure we've imposed a barrier
787 * on the generation in which this probe was last modified.
789 fasttrap_mod_barrier(probe->ftp_gen);
791 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
794 * If the tracepoint has already been enabled, just add our id to the
795 * list of interested probes. This may be our second time through
796 * this path in which case we'll have constructed the tracepoint we'd
797 * like to install. If we can't find a match, and have an allocated
798 * tracepoint ready to go, enable that one now.
800 * A tracepoint whose process is defunct is also considered defunct.
803 mutex_enter(&bucket->ftb_mtx);
804 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
806 * Note that it's safe to access the active count on the
807 * associated proc structure because we know that at least one
808 * provider (this one) will still be around throughout this
811 if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
812 tp->ftt_proc->ftpc_acount == 0)
816 * Now that we've found a matching tracepoint, it would be
817 * a decent idea to confirm that the tracepoint is still
818 * enabled and the trap instruction hasn't been overwritten.
819 * Since this is a little hairy, we'll punt for now.
823 * This can't be the first interested probe. We don't have
824 * to worry about another thread being in the midst of
825 * deleting this tracepoint (which would be the only valid
826 * reason for a tracepoint to have no interested probes)
827 * since we're holding P_PR_LOCK for this process.
829 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
831 switch (id->fti_ptype) {
834 case DTFTP_IS_ENABLED:
835 id->fti_next = tp->ftt_ids;
842 case DTFTP_POST_OFFSETS:
843 id->fti_next = tp->ftt_retids;
853 mutex_exit(&bucket->ftb_mtx);
855 if (new_tp != NULL) {
856 new_tp->ftt_ids = NULL;
857 new_tp->ftt_retids = NULL;
864 * If we have a good tracepoint ready to go, install it now while
865 * we have the lock held and no one can screw with us.
867 if (new_tp != NULL) {
870 new_tp->ftt_next = bucket->ftb_data;
872 bucket->ftb_data = new_tp;
874 mutex_exit(&bucket->ftb_mtx);
877 * Activate the tracepoint in the ISA-specific manner.
878 * If this fails, we need to report the failure, but
879 * indicate that this tracepoint must still be disabled
880 * by calling fasttrap_tracepoint_disable().
882 if (fasttrap_tracepoint_install(p, new_tp) != 0)
883 rc = FASTTRAP_ENABLE_PARTIAL;
886 * Increment the count of the number of tracepoints active in
887 * the victim process.
890 ASSERT(p->p_proc_flag & P_PR_LOCK);
897 mutex_exit(&bucket->ftb_mtx);
900 * Initialize the tracepoint that's been preallocated with the probe.
902 new_tp = probe->ftp_tps[index].fit_tp;
904 ASSERT(new_tp->ftt_pid == pid);
905 ASSERT(new_tp->ftt_pc == pc);
906 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
907 ASSERT(new_tp->ftt_ids == NULL);
908 ASSERT(new_tp->ftt_retids == NULL);
910 switch (id->fti_ptype) {
913 case DTFTP_IS_ENABLED:
915 new_tp->ftt_ids = id;
919 case DTFTP_POST_OFFSETS:
921 new_tp->ftt_retids = id;
929 if (SV_PROC_FLAG(p, SV_LP64))
930 p->p_model = DATAMODEL_LP64;
932 p->p_model = DATAMODEL_ILP32;
936 * If the ISA-dependent initialization goes to plan, go back to the
937 * beginning and try to install this freshly made tracepoint.
939 if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
942 new_tp->ftt_ids = NULL;
943 new_tp->ftt_retids = NULL;
945 return (FASTTRAP_ENABLE_FAIL);
949 fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
951 fasttrap_bucket_t *bucket;
952 fasttrap_provider_t *provider = probe->ftp_prov;
953 fasttrap_tracepoint_t **pp, *tp;
954 fasttrap_id_t *id, **idp = NULL;
958 ASSERT(index < probe->ftp_ntps);
960 pid = probe->ftp_pid;
961 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
962 id = &probe->ftp_tps[index].fit_id;
964 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
967 * Find the tracepoint and make sure that our id is one of the
968 * ones registered with it.
970 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
971 mutex_enter(&bucket->ftb_mtx);
972 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
973 if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
974 tp->ftt_proc == provider->ftp_proc)
979 * If we somehow lost this tracepoint, we're in a world of hurt.
983 switch (id->fti_ptype) {
986 case DTFTP_IS_ENABLED:
987 ASSERT(tp->ftt_ids != NULL);
992 case DTFTP_POST_OFFSETS:
993 ASSERT(tp->ftt_retids != NULL);
994 idp = &tp->ftt_retids;
1001 while ((*idp)->fti_probe != probe) {
1002 idp = &(*idp)->fti_next;
1003 ASSERT(*idp != NULL);
1007 *idp = id->fti_next;
1010 ASSERT(id->fti_probe == probe);
1013 * If there are other registered enablings of this tracepoint, we're
1014 * all done, but if this was the last probe assocated with this
1015 * this tracepoint, we need to remove and free it.
1017 if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
1020 * If the current probe's tracepoint is in use, swap it
1021 * for an unused tracepoint.
1023 if (tp == probe->ftp_tps[index].fit_tp) {
1024 fasttrap_probe_t *tmp_probe;
1025 fasttrap_tracepoint_t **tmp_tp;
1028 if (tp->ftt_ids != NULL) {
1029 tmp_probe = tp->ftt_ids->fti_probe;
1030 /* LINTED - alignment */
1031 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
1032 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
1034 tmp_probe = tp->ftt_retids->fti_probe;
1035 /* LINTED - alignment */
1036 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
1037 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
1040 ASSERT(*tmp_tp != NULL);
1041 ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
1042 ASSERT((*tmp_tp)->ftt_ids == NULL);
1043 ASSERT((*tmp_tp)->ftt_retids == NULL);
1045 probe->ftp_tps[index].fit_tp = *tmp_tp;
1049 mutex_exit(&bucket->ftb_mtx);
1052 * Tag the modified probe with the generation in which it was
1055 probe->ftp_gen = fasttrap_mod_gen;
1059 mutex_exit(&bucket->ftb_mtx);
1062 * We can't safely remove the tracepoint from the set of active
1063 * tracepoints until we've actually removed the fasttrap instruction
1064 * from the process's text. We can, however, operate on this
1065 * tracepoint secure in the knowledge that no other thread is going to
1066 * be looking at it since we hold P_PR_LOCK on the process if it's
1067 * live or we hold the provider lock on the process if it's dead and
1072 * We only need to remove the actual instruction if we're looking
1073 * at an existing process
1077 * If we fail to restore the instruction we need to kill
1078 * this process since it's in a completely unrecoverable
1081 if (fasttrap_tracepoint_remove(p, tp) != 0)
1082 fasttrap_sigtrap(p, NULL, pc);
1085 * Decrement the count of the number of tracepoints active
1086 * in the victim process.
1089 ASSERT(p->p_proc_flag & P_PR_LOCK);
1091 p->p_dtrace_count--;
1095 * Remove the probe from the hash table of active tracepoints.
1097 mutex_enter(&bucket->ftb_mtx);
1098 pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
1099 ASSERT(*pp != NULL);
1101 pp = &(*pp)->ftt_next;
1102 ASSERT(*pp != NULL);
1108 mutex_exit(&bucket->ftb_mtx);
1111 * Tag the modified probe with the generation in which it was changed.
1113 probe->ftp_gen = fasttrap_mod_gen;
1117 fasttrap_enable_callbacks(void)
1120 * We don't have to play the rw lock game here because we're
1121 * providing something rather than taking something away --
1122 * we can be sure that no threads have tried to follow this
1123 * function pointer yet.
1125 mutex_enter(&fasttrap_count_mtx);
1126 if (fasttrap_pid_count == 0) {
1127 ASSERT(dtrace_pid_probe_ptr == NULL);
1128 ASSERT(dtrace_return_probe_ptr == NULL);
1129 dtrace_pid_probe_ptr = &fasttrap_pid_probe;
1130 dtrace_return_probe_ptr = &fasttrap_return_probe;
1132 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe);
1133 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe);
1134 fasttrap_pid_count++;
1135 mutex_exit(&fasttrap_count_mtx);
1139 fasttrap_disable_callbacks(void)
1142 ASSERT(MUTEX_HELD(&cpu_lock));
1146 mutex_enter(&fasttrap_count_mtx);
1147 ASSERT(fasttrap_pid_count > 0);
1148 fasttrap_pid_count--;
1149 if (fasttrap_pid_count == 0) {
1151 cpu_t *cur, *cpu = CPU;
1153 for (cur = cpu->cpu_next_onln; cur != cpu;
1154 cur = cur->cpu_next_onln) {
1155 rw_enter(&cur->cpu_ft_lock, RW_WRITER);
1158 dtrace_pid_probe_ptr = NULL;
1159 dtrace_return_probe_ptr = NULL;
1161 for (cur = cpu->cpu_next_onln; cur != cpu;
1162 cur = cur->cpu_next_onln) {
1163 rw_exit(&cur->cpu_ft_lock);
1167 mutex_exit(&fasttrap_count_mtx);
1172 fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
1174 fasttrap_probe_t *probe = parg;
1178 ASSERT(probe != NULL);
1179 ASSERT(!probe->ftp_enabled);
1180 ASSERT(id == probe->ftp_id);
1182 ASSERT(MUTEX_HELD(&cpu_lock));
1186 * Increment the count of enabled probes on this probe's provider;
1187 * the provider can't go away while the probe still exists. We
1188 * must increment this even if we aren't able to properly enable
1191 mutex_enter(&probe->ftp_prov->ftp_mtx);
1192 probe->ftp_prov->ftp_rcount++;
1193 mutex_exit(&probe->ftp_prov->ftp_mtx);
1196 * If this probe's provider is retired (meaning it was valid in a
1197 * previously exec'ed incarnation of this address space), bail out. The
1198 * provider can't go away while we're in this code path.
1200 if (probe->ftp_prov->ftp_retired)
1204 * If we can't find the process, it may be that we're in the context of
1205 * a fork in which the traced process is being born and we're copying
1206 * USDT probes. Otherwise, the process is gone so bail.
1209 if ((p = sprlock(probe->ftp_pid)) == NULL) {
1210 if ((curproc->p_flag & SFORKING) == 0)
1213 mutex_enter(&pidlock);
1214 p = prfind(probe->ftp_pid);
1218 * So it's not that the target process is being born,
1219 * it's that it isn't there at all (and we simply
1220 * happen to be forking). Anyway, we know that the
1221 * target is definitely gone, so bail out.
1223 mutex_exit(&pidlock);
1228 * Confirm that curproc is indeed forking the process in which
1229 * we're trying to enable probes.
1231 ASSERT(p->p_parent == curproc);
1232 ASSERT(p->p_stat == SIDL);
1234 mutex_enter(&p->p_lock);
1235 mutex_exit(&pidlock);
1240 ASSERT(!(p->p_flag & SVFORK));
1241 mutex_exit(&p->p_lock);
1243 if (pget(probe->ftp_pid, PGET_HOLD | PGET_NOTWEXIT, &p) != 0)
1248 * We have to enable the trap entry point before any user threads have
1249 * the chance to execute the trap instruction we're about to place
1250 * in their process's text.
1252 fasttrap_enable_callbacks();
1255 * Enable all the tracepoints and add this probe's id to each
1256 * tracepoint's list of active probes.
1258 for (i = 0; i < probe->ftp_ntps; i++) {
1259 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
1261 * If enabling the tracepoint failed completely,
1262 * we don't have to disable it; if the failure
1263 * was only partial we must disable it.
1265 if (rc == FASTTRAP_ENABLE_FAIL)
1268 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
1271 * Back up and pull out all the tracepoints we've
1272 * created so far for this probe.
1275 fasttrap_tracepoint_disable(p, probe, i);
1280 mutex_enter(&p->p_lock);
1287 * Since we're not actually enabling this probe,
1288 * drop our reference on the trap table entry.
1290 fasttrap_disable_callbacks();
1295 mutex_enter(&p->p_lock);
1301 probe->ftp_enabled = 1;
1306 fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
1308 fasttrap_probe_t *probe = parg;
1309 fasttrap_provider_t *provider = probe->ftp_prov;
1313 ASSERT(id == probe->ftp_id);
1315 mutex_enter(&provider->ftp_mtx);
1318 * We won't be able to acquire a /proc-esque lock on the process
1319 * iff the process is dead and gone. In this case, we rely on the
1320 * provider lock as a point of mutual exclusion to prevent other
1321 * DTrace consumers from disabling this probe.
1323 if (pget(probe->ftp_pid, PGET_HOLD | PGET_NOTWEXIT, &p) != 0)
1327 * Disable all the associated tracepoints (for fully enabled probes).
1329 if (probe->ftp_enabled) {
1330 for (i = 0; i < probe->ftp_ntps; i++) {
1331 fasttrap_tracepoint_disable(p, probe, i);
1335 ASSERT(provider->ftp_rcount > 0);
1336 provider->ftp_rcount--;
1340 * Even though we may not be able to remove it entirely, we
1341 * mark this retired provider to get a chance to remove some
1342 * of the associated probes.
1344 if (provider->ftp_retired && !provider->ftp_marked)
1345 whack = provider->ftp_marked = 1;
1346 mutex_exit(&provider->ftp_mtx);
1349 * If the process is dead, we're just waiting for the
1350 * last probe to be disabled to be able to free it.
1352 if (provider->ftp_rcount == 0 && !provider->ftp_marked)
1353 whack = provider->ftp_marked = 1;
1354 mutex_exit(&provider->ftp_mtx);
1358 fasttrap_pid_cleanup();
1364 if (!probe->ftp_enabled)
1367 probe->ftp_enabled = 0;
1370 ASSERT(MUTEX_HELD(&cpu_lock));
1372 fasttrap_disable_callbacks();
1377 fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
1378 dtrace_argdesc_t *desc)
1380 fasttrap_probe_t *probe = parg;
1384 desc->dtargd_native[0] = '\0';
1385 desc->dtargd_xlate[0] = '\0';
1387 if (probe->ftp_prov->ftp_retired != 0 ||
1388 desc->dtargd_ndx >= probe->ftp_nargs) {
1389 desc->dtargd_ndx = DTRACE_ARGNONE;
1393 ndx = (probe->ftp_argmap != NULL) ?
1394 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx;
1396 str = probe->ftp_ntypes;
1397 for (i = 0; i < ndx; i++) {
1398 str += strlen(str) + 1;
1401 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native));
1402 (void) strcpy(desc->dtargd_native, str);
1404 if (probe->ftp_xtypes == NULL)
1407 str = probe->ftp_xtypes;
1408 for (i = 0; i < desc->dtargd_ndx; i++) {
1409 str += strlen(str) + 1;
1412 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate));
1413 (void) strcpy(desc->dtargd_xlate, str);
1418 fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
1420 fasttrap_probe_t *probe = parg;
1424 ASSERT(probe != NULL);
1425 ASSERT(!probe->ftp_enabled);
1426 ASSERT(fasttrap_total >= probe->ftp_ntps);
1428 atomic_add_32(&fasttrap_total, -probe->ftp_ntps);
1429 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
1431 if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
1432 fasttrap_mod_barrier(probe->ftp_gen);
1434 for (i = 0; i < probe->ftp_ntps; i++) {
1435 kmem_free(probe->ftp_tps[i].fit_tp,
1436 sizeof (fasttrap_tracepoint_t));
1439 kmem_free(probe, size);
1443 static const dtrace_pattr_t pid_attr = {
1444 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1445 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1446 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1447 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1448 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1451 static dtrace_pops_t pid_pops = {
1452 .dtps_provide = fasttrap_pid_provide,
1453 .dtps_provide_module = NULL,
1454 .dtps_enable = fasttrap_pid_enable,
1455 .dtps_disable = fasttrap_pid_disable,
1456 .dtps_suspend = NULL,
1457 .dtps_resume = NULL,
1458 .dtps_getargdesc = fasttrap_pid_getargdesc,
1459 .dtps_getargval = fasttrap_pid_getarg,
1460 .dtps_usermode = NULL,
1461 .dtps_destroy = fasttrap_pid_destroy
1464 static dtrace_pops_t usdt_pops = {
1465 .dtps_provide = fasttrap_pid_provide,
1466 .dtps_provide_module = NULL,
1467 .dtps_enable = fasttrap_pid_enable,
1468 .dtps_disable = fasttrap_pid_disable,
1469 .dtps_suspend = NULL,
1470 .dtps_resume = NULL,
1471 .dtps_getargdesc = fasttrap_pid_getargdesc,
1472 .dtps_getargval = fasttrap_usdt_getarg,
1473 .dtps_usermode = NULL,
1474 .dtps_destroy = fasttrap_pid_destroy
1477 static fasttrap_proc_t *
1478 fasttrap_proc_lookup(pid_t pid)
1480 fasttrap_bucket_t *bucket;
1481 fasttrap_proc_t *fprc, *new_fprc;
1484 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1485 mutex_enter(&bucket->ftb_mtx);
1487 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1488 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1489 mutex_enter(&fprc->ftpc_mtx);
1490 mutex_exit(&bucket->ftb_mtx);
1491 fprc->ftpc_rcount++;
1492 atomic_inc_64(&fprc->ftpc_acount);
1493 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1494 mutex_exit(&fprc->ftpc_mtx);
1501 * Drop the bucket lock so we don't try to perform a sleeping
1502 * allocation under it.
1504 mutex_exit(&bucket->ftb_mtx);
1506 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP);
1507 new_fprc->ftpc_pid = pid;
1508 new_fprc->ftpc_rcount = 1;
1509 new_fprc->ftpc_acount = 1;
1511 mutex_init(&new_fprc->ftpc_mtx, "fasttrap proc mtx", MUTEX_DEFAULT,
1515 mutex_enter(&bucket->ftb_mtx);
1518 * Take another lap through the list to make sure a proc hasn't
1519 * been created for this pid while we weren't under the bucket lock.
1521 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1522 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1523 mutex_enter(&fprc->ftpc_mtx);
1524 mutex_exit(&bucket->ftb_mtx);
1525 fprc->ftpc_rcount++;
1526 atomic_inc_64(&fprc->ftpc_acount);
1527 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1528 mutex_exit(&fprc->ftpc_mtx);
1530 kmem_free(new_fprc, sizeof (fasttrap_proc_t));
1536 new_fprc->ftpc_next = bucket->ftb_data;
1537 bucket->ftb_data = new_fprc;
1539 mutex_exit(&bucket->ftb_mtx);
1545 fasttrap_proc_release(fasttrap_proc_t *proc)
1547 fasttrap_bucket_t *bucket;
1548 fasttrap_proc_t *fprc, **fprcp;
1549 pid_t pid = proc->ftpc_pid;
1551 fasttrap_scrblock_t *scrblk, *scrblktmp;
1552 fasttrap_scrspace_t *scrspc, *scrspctmp;
1557 mutex_enter(&proc->ftpc_mtx);
1559 ASSERT(proc->ftpc_rcount != 0);
1560 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount);
1562 if (--proc->ftpc_rcount != 0) {
1563 mutex_exit(&proc->ftpc_mtx);
1569 * Free all structures used to manage per-thread scratch space.
1571 LIST_FOREACH_SAFE(scrblk, &proc->ftpc_scrblks, ftsb_next,
1573 LIST_REMOVE(scrblk, ftsb_next);
1574 free(scrblk, M_SOLARIS);
1576 LIST_FOREACH_SAFE(scrspc, &proc->ftpc_fscr, ftss_next, scrspctmp) {
1577 LIST_REMOVE(scrspc, ftss_next);
1578 free(scrspc, M_SOLARIS);
1580 LIST_FOREACH_SAFE(scrspc, &proc->ftpc_ascr, ftss_next, scrspctmp) {
1581 LIST_REMOVE(scrspc, ftss_next);
1582 free(scrspc, M_SOLARIS);
1585 if ((p = pfind(pid)) != NULL) {
1586 FOREACH_THREAD_IN_PROC(p, td)
1587 td->t_dtrace_sscr = NULL;
1592 mutex_exit(&proc->ftpc_mtx);
1595 * There should definitely be no live providers associated with this
1596 * process at this point.
1598 ASSERT(proc->ftpc_acount == 0);
1600 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1601 mutex_enter(&bucket->ftb_mtx);
1603 fprcp = (fasttrap_proc_t **)&bucket->ftb_data;
1604 while ((fprc = *fprcp) != NULL) {
1608 fprcp = &fprc->ftpc_next;
1612 * Something strange has happened if we can't find the proc.
1614 ASSERT(fprc != NULL);
1616 *fprcp = fprc->ftpc_next;
1618 mutex_exit(&bucket->ftb_mtx);
1620 kmem_free(fprc, sizeof (fasttrap_proc_t));
1624 * Lookup a fasttrap-managed provider based on its name and associated pid.
1625 * If the pattr argument is non-NULL, this function instantiates the provider
1626 * if it doesn't exist otherwise it returns NULL. The provider is returned
1627 * with its lock held.
1629 static fasttrap_provider_t *
1630 fasttrap_provider_lookup(pid_t pid, const char *name,
1631 const dtrace_pattr_t *pattr)
1633 fasttrap_provider_t *fp, *new_fp = NULL;
1634 fasttrap_bucket_t *bucket;
1635 char provname[DTRACE_PROVNAMELEN];
1639 ASSERT(strlen(name) < sizeof (fp->ftp_name));
1640 ASSERT(pattr != NULL);
1642 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1643 mutex_enter(&bucket->ftb_mtx);
1646 * Take a lap through the list and return the match if we find it.
1648 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1649 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1651 mutex_enter(&fp->ftp_mtx);
1652 mutex_exit(&bucket->ftb_mtx);
1658 * Drop the bucket lock so we don't try to perform a sleeping
1659 * allocation under it.
1661 mutex_exit(&bucket->ftb_mtx);
1664 * Make sure the process exists, isn't a child created as the result
1665 * of a vfork(2), and isn't a zombie (but may be in fork).
1667 if ((p = pfind(pid)) == NULL)
1671 * Increment p_dtrace_probes so that the process knows to inform us
1672 * when it exits or execs. fasttrap_provider_free() decrements this
1673 * when we're done with this provider.
1675 p->p_dtrace_probes++;
1678 * Grab the credentials for this process so we have
1679 * something to pass to dtrace_register().
1681 PROC_LOCK_ASSERT(p, MA_OWNED);
1686 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP);
1687 new_fp->ftp_pid = pid;
1688 new_fp->ftp_proc = fasttrap_proc_lookup(pid);
1690 mutex_init(&new_fp->ftp_mtx, "provider mtx", MUTEX_DEFAULT, NULL);
1691 mutex_init(&new_fp->ftp_cmtx, "lock on creating", MUTEX_DEFAULT, NULL);
1694 ASSERT(new_fp->ftp_proc != NULL);
1696 mutex_enter(&bucket->ftb_mtx);
1699 * Take another lap through the list to make sure a provider hasn't
1700 * been created for this pid while we weren't under the bucket lock.
1702 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1703 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1705 mutex_enter(&fp->ftp_mtx);
1706 mutex_exit(&bucket->ftb_mtx);
1707 fasttrap_provider_free(new_fp);
1713 (void) strcpy(new_fp->ftp_name, name);
1716 * Fail and return NULL if either the provider name is too long
1717 * or we fail to register this new provider with the DTrace
1718 * framework. Note that this is the only place we ever construct
1719 * the full provider name -- we keep it in pieces in the provider
1722 if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >=
1723 sizeof (provname) ||
1724 dtrace_register(provname, pattr,
1725 DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred,
1726 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp,
1727 &new_fp->ftp_provid) != 0) {
1728 mutex_exit(&bucket->ftb_mtx);
1729 fasttrap_provider_free(new_fp);
1734 new_fp->ftp_next = bucket->ftb_data;
1735 bucket->ftb_data = new_fp;
1737 mutex_enter(&new_fp->ftp_mtx);
1738 mutex_exit(&bucket->ftb_mtx);
1745 fasttrap_provider_free(fasttrap_provider_t *provider)
1747 pid_t pid = provider->ftp_pid;
1751 * There need to be no associated enabled probes, no consumers
1752 * creating probes, and no meta providers referencing this provider.
1754 ASSERT(provider->ftp_rcount == 0);
1755 ASSERT(provider->ftp_ccount == 0);
1756 ASSERT(provider->ftp_mcount == 0);
1759 * If this provider hasn't been retired, we need to explicitly drop the
1760 * count of active providers on the associated process structure.
1762 if (!provider->ftp_retired) {
1763 atomic_dec_64(&provider->ftp_proc->ftpc_acount);
1764 ASSERT(provider->ftp_proc->ftpc_acount <
1765 provider->ftp_proc->ftpc_rcount);
1768 fasttrap_proc_release(provider->ftp_proc);
1771 mutex_destroy(&provider->ftp_mtx);
1772 mutex_destroy(&provider->ftp_cmtx);
1774 kmem_free(provider, sizeof (fasttrap_provider_t));
1777 * Decrement p_dtrace_probes on the process whose provider we're
1778 * freeing. We don't have to worry about clobbering somone else's
1779 * modifications to it because we have locked the bucket that
1780 * corresponds to this process's hash chain in the provider hash
1781 * table. Don't sweat it if we can't find the process.
1783 if ((p = pfind(pid)) == NULL) {
1787 p->p_dtrace_probes--;
1794 fasttrap_provider_retire(pid_t pid, const char *name, int mprov)
1796 fasttrap_provider_t *fp;
1797 fasttrap_bucket_t *bucket;
1798 dtrace_provider_id_t provid;
1800 ASSERT(strlen(name) < sizeof (fp->ftp_name));
1802 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1803 mutex_enter(&bucket->ftb_mtx);
1805 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1806 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1812 mutex_exit(&bucket->ftb_mtx);
1816 mutex_enter(&fp->ftp_mtx);
1817 ASSERT(!mprov || fp->ftp_mcount > 0);
1818 if (mprov && --fp->ftp_mcount != 0) {
1819 mutex_exit(&fp->ftp_mtx);
1820 mutex_exit(&bucket->ftb_mtx);
1825 * Mark the provider to be removed in our post-processing step, mark it
1826 * retired, and drop the active count on its proc. Marking it indicates
1827 * that we should try to remove it; setting the retired flag indicates
1828 * that we're done with this provider; dropping the active the proc
1829 * releases our hold, and when this reaches zero (as it will during
1830 * exit or exec) the proc and associated providers become defunct.
1832 * We obviously need to take the bucket lock before the provider lock
1833 * to perform the lookup, but we need to drop the provider lock
1834 * before calling into the DTrace framework since we acquire the
1835 * provider lock in callbacks invoked from the DTrace framework. The
1836 * bucket lock therefore protects the integrity of the provider hash
1839 atomic_dec_64(&fp->ftp_proc->ftpc_acount);
1840 ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount);
1842 fp->ftp_retired = 1;
1844 provid = fp->ftp_provid;
1845 mutex_exit(&fp->ftp_mtx);
1848 * We don't have to worry about invalidating the same provider twice
1849 * since fasttrap_provider_lookup() will ignore provider that have
1850 * been marked as retired.
1852 dtrace_invalidate(provid);
1854 mutex_exit(&bucket->ftb_mtx);
1856 fasttrap_pid_cleanup();
1860 fasttrap_uint32_cmp(const void *ap, const void *bp)
1862 return (*(const uint32_t *)ap - *(const uint32_t *)bp);
1866 fasttrap_uint64_cmp(const void *ap, const void *bp)
1868 return (*(const uint64_t *)ap - *(const uint64_t *)bp);
1872 fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
1874 fasttrap_provider_t *provider;
1875 fasttrap_probe_t *pp;
1876 fasttrap_tracepoint_t *tp;
1878 int i, aframes = 0, whack;
1881 * There needs to be at least one desired trace point.
1883 if (pdata->ftps_noffs == 0)
1886 switch (pdata->ftps_type) {
1889 aframes = FASTTRAP_ENTRY_AFRAMES;
1893 aframes = FASTTRAP_RETURN_AFRAMES;
1902 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid,
1903 FASTTRAP_PID_NAME, &pid_attr)) == NULL)
1907 * Increment this reference count to indicate that a consumer is
1908 * actively adding a new probe associated with this provider. This
1909 * prevents the provider from being deleted -- we'll need to check
1910 * for pending deletions when we drop this reference count.
1912 provider->ftp_ccount++;
1913 mutex_exit(&provider->ftp_mtx);
1916 * Grab the creation lock to ensure consistency between calls to
1917 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1918 * other threads creating probes. We must drop the provider lock
1919 * before taking this lock to avoid a three-way deadlock with the
1922 mutex_enter(&provider->ftp_cmtx);
1925 for (i = 0; i < pdata->ftps_noffs; i++) {
1928 (void) sprintf(name_str, "%llx",
1929 (unsigned long long)pdata->ftps_offs[i]);
1931 if (dtrace_probe_lookup(provider->ftp_provid,
1932 pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
1935 atomic_inc_32(&fasttrap_total);
1937 if (fasttrap_total > fasttrap_max) {
1938 atomic_dec_32(&fasttrap_total);
1942 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP);
1944 pp->ftp_prov = provider;
1945 pp->ftp_faddr = pdata->ftps_pc;
1946 pp->ftp_fsize = pdata->ftps_size;
1947 pp->ftp_pid = pdata->ftps_pid;
1950 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1953 tp->ftt_proc = provider->ftp_proc;
1954 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1955 tp->ftt_pid = pdata->ftps_pid;
1957 pp->ftp_tps[0].fit_tp = tp;
1958 pp->ftp_tps[0].fit_id.fti_probe = pp;
1959 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type;
1961 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1962 pdata->ftps_mod, pdata->ftps_func, name_str,
1963 FASTTRAP_OFFSET_AFRAMES, pp);
1966 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod,
1967 pdata->ftps_func, name) == 0) {
1968 atomic_add_32(&fasttrap_total, pdata->ftps_noffs);
1970 if (fasttrap_total > fasttrap_max) {
1971 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1976 * Make sure all tracepoint program counter values are unique.
1977 * We later assume that each probe has exactly one tracepoint
1980 qsort(pdata->ftps_offs, pdata->ftps_noffs,
1981 sizeof (uint64_t), fasttrap_uint64_cmp);
1982 for (i = 1; i < pdata->ftps_noffs; i++) {
1983 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1])
1986 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1990 ASSERT(pdata->ftps_noffs > 0);
1991 pp = kmem_zalloc(offsetof(fasttrap_probe_t,
1992 ftp_tps[pdata->ftps_noffs]), KM_SLEEP);
1994 pp->ftp_prov = provider;
1995 pp->ftp_faddr = pdata->ftps_pc;
1996 pp->ftp_fsize = pdata->ftps_size;
1997 pp->ftp_pid = pdata->ftps_pid;
1998 pp->ftp_ntps = pdata->ftps_noffs;
2000 for (i = 0; i < pdata->ftps_noffs; i++) {
2001 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
2004 tp->ftt_proc = provider->ftp_proc;
2005 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
2006 tp->ftt_pid = pdata->ftps_pid;
2008 pp->ftp_tps[i].fit_tp = tp;
2009 pp->ftp_tps[i].fit_id.fti_probe = pp;
2010 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type;
2013 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
2014 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp);
2017 mutex_exit(&provider->ftp_cmtx);
2020 * We know that the provider is still valid since we incremented the
2021 * creation reference count. If someone tried to clean up this provider
2022 * while we were using it (e.g. because the process called exec(2) or
2023 * exit(2)), take note of that and try to clean it up now.
2025 mutex_enter(&provider->ftp_mtx);
2026 provider->ftp_ccount--;
2027 whack = provider->ftp_retired;
2028 mutex_exit(&provider->ftp_mtx);
2031 fasttrap_pid_cleanup();
2037 * If we've exhausted the allowable resources, we'll try to remove
2038 * this provider to free some up. This is to cover the case where
2039 * the user has accidentally created many more probes than was
2040 * intended (e.g. pid123:::).
2042 mutex_exit(&provider->ftp_cmtx);
2043 mutex_enter(&provider->ftp_mtx);
2044 provider->ftp_ccount--;
2045 provider->ftp_marked = 1;
2046 mutex_exit(&provider->ftp_mtx);
2048 fasttrap_pid_cleanup();
2055 fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
2057 fasttrap_provider_t *provider;
2060 * A 32-bit unsigned integer (like a pid for example) can be
2061 * expressed in 10 or fewer decimal digits. Make sure that we'll
2062 * have enough space for the provider name.
2064 if (strlen(dhpv->dthpv_provname) + 10 >=
2065 sizeof (provider->ftp_name)) {
2066 printf("failed to instantiate provider %s: "
2067 "name too long to accomodate pid", dhpv->dthpv_provname);
2072 * Don't let folks spoof the true pid provider.
2074 if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) {
2075 printf("failed to instantiate provider %s: "
2076 "%s is an invalid name", dhpv->dthpv_provname,
2082 * The highest stability class that fasttrap supports is ISA; cap
2083 * the stability of the new provider accordingly.
2085 if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA)
2086 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA;
2087 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA)
2088 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA;
2089 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA)
2090 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA;
2091 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA)
2092 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA;
2093 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA)
2094 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA;
2096 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname,
2097 &dhpv->dthpv_pattr)) == NULL) {
2098 printf("failed to instantiate provider %s for "
2099 "process %u", dhpv->dthpv_provname, (uint_t)pid);
2104 * Up the meta provider count so this provider isn't removed until
2105 * the meta provider has been told to remove it.
2107 provider->ftp_mcount++;
2109 mutex_exit(&provider->ftp_mtx);
2115 * We know a few things about our context here: we know that the probe being
2116 * created doesn't already exist (DTrace won't load DOF at the same address
2117 * twice, even if explicitly told to do so) and we know that we are
2118 * single-threaded with respect to the meta provider machinery. Knowing that
2119 * this is a new probe and that there is no way for us to race with another
2120 * operation on this provider allows us an important optimization: we need not
2121 * lookup a probe before adding it. Saving this lookup is important because
2122 * this code is in the fork path for processes with USDT probes, and lookups
2123 * here are potentially very expensive because of long hash conflicts on
2124 * module, function and name (DTrace doesn't hash on provider name).
2128 fasttrap_meta_create_probe(void *arg, void *parg,
2129 dtrace_helper_probedesc_t *dhpb)
2131 fasttrap_provider_t *provider = parg;
2132 fasttrap_probe_t *pp;
2133 fasttrap_tracepoint_t *tp;
2138 * Since the meta provider count is non-zero we don't have to worry
2139 * about this provider disappearing.
2141 ASSERT(provider->ftp_mcount > 0);
2144 * The offsets must be unique.
2146 qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t),
2147 fasttrap_uint32_cmp);
2148 for (i = 1; i < dhpb->dthpb_noffs; i++) {
2149 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <=
2150 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1])
2154 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t),
2155 fasttrap_uint32_cmp);
2156 for (i = 1; i < dhpb->dthpb_nenoffs; i++) {
2157 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <=
2158 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1])
2162 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs;
2165 atomic_add_32(&fasttrap_total, ntps);
2167 if (fasttrap_total > fasttrap_max) {
2168 atomic_add_32(&fasttrap_total, -ntps);
2172 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP);
2174 pp->ftp_prov = provider;
2175 pp->ftp_pid = provider->ftp_pid;
2176 pp->ftp_ntps = ntps;
2177 pp->ftp_nargs = dhpb->dthpb_xargc;
2178 pp->ftp_xtypes = dhpb->dthpb_xtypes;
2179 pp->ftp_ntypes = dhpb->dthpb_ntypes;
2182 * First create a tracepoint for each actual point of interest.
2184 for (i = 0; i < dhpb->dthpb_noffs; i++) {
2185 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
2187 tp->ftt_proc = provider->ftp_proc;
2188 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i];
2189 tp->ftt_pid = provider->ftp_pid;
2191 pp->ftp_tps[i].fit_tp = tp;
2192 pp->ftp_tps[i].fit_id.fti_probe = pp;
2194 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS;
2196 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS;
2201 * Then create a tracepoint for each is-enabled point.
2203 for (j = 0; i < ntps; i++, j++) {
2204 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
2206 tp->ftt_proc = provider->ftp_proc;
2207 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j];
2208 tp->ftt_pid = provider->ftp_pid;
2210 pp->ftp_tps[i].fit_tp = tp;
2211 pp->ftp_tps[i].fit_id.fti_probe = pp;
2212 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED;
2216 * If the arguments are shuffled around we set the argument remapping
2217 * table. Later, when the probe fires, we only remap the arguments
2218 * if the table is non-NULL.
2220 for (i = 0; i < dhpb->dthpb_xargc; i++) {
2221 if (dhpb->dthpb_args[i] != i) {
2222 pp->ftp_argmap = dhpb->dthpb_args;
2228 * The probe is fully constructed -- register it with DTrace.
2230 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
2231 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp);
2236 fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
2239 * Clean up the USDT provider. There may be active consumers of the
2240 * provider busy adding probes, no damage will actually befall the
2241 * provider until that count has dropped to zero. This just puts
2242 * the provider on death row.
2244 fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1);
2247 static dtrace_mops_t fasttrap_mops = {
2248 .dtms_create_probe = fasttrap_meta_create_probe,
2249 .dtms_provide_pid = fasttrap_meta_provide,
2250 .dtms_remove_pid = fasttrap_meta_remove
2255 fasttrap_open(struct cdev *dev __unused, int oflags __unused,
2256 int devtype __unused, struct thread *td __unused)
2263 fasttrap_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int fflag,
2266 if (!dtrace_attached())
2269 if (cmd == FASTTRAPIOC_MAKEPROBE) {
2270 fasttrap_probe_spec_t *uprobe = *(fasttrap_probe_spec_t **)arg;
2271 fasttrap_probe_spec_t *probe;
2276 if (copyin(&uprobe->ftps_noffs, &noffs,
2277 sizeof (uprobe->ftps_noffs)))
2281 * Probes must have at least one tracepoint.
2286 size = sizeof (fasttrap_probe_spec_t) +
2287 sizeof (probe->ftps_offs[0]) * (noffs - 1);
2289 if (size > 1024 * 1024)
2292 probe = kmem_alloc(size, KM_SLEEP);
2294 if (copyin(uprobe, probe, size) != 0 ||
2295 probe->ftps_noffs != noffs) {
2296 kmem_free(probe, size);
2301 * Verify that the function and module strings contain no
2304 if (u8_validate(probe->ftps_func, strlen(probe->ftps_func),
2305 NULL, U8_VALIDATE_ENTIRE, &err) < 0) {
2310 if (u8_validate(probe->ftps_mod, strlen(probe->ftps_mod),
2311 NULL, U8_VALIDATE_ENTIRE, &err) < 0) {
2317 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2319 pid_t pid = probe->ftps_pid;
2321 mutex_enter(&pidlock);
2323 * Report an error if the process doesn't exist
2324 * or is actively being birthed.
2326 if ((p = pfind(pid)) == NULL || p->p_stat == SIDL) {
2327 mutex_exit(&pidlock);
2330 mutex_enter(&p->p_lock);
2331 mutex_exit(&pidlock);
2333 if ((ret = priv_proc_cred_perm(cr, p, NULL,
2334 VREAD | VWRITE)) != 0) {
2335 mutex_exit(&p->p_lock);
2338 mutex_exit(&p->p_lock);
2342 ret = fasttrap_add_probe(probe);
2344 kmem_free(probe, size);
2348 } else if (cmd == FASTTRAPIOC_GETINSTR) {
2349 fasttrap_instr_query_t instr;
2350 fasttrap_tracepoint_t *tp;
2357 if (copyin((void *)arg, &instr, sizeof (instr)) != 0)
2362 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2364 pid_t pid = instr.ftiq_pid;
2366 mutex_enter(&pidlock);
2368 * Report an error if the process doesn't exist
2369 * or is actively being birthed.
2371 if ((p == pfind(pid)) == NULL || p->p_stat == SIDL) {
2372 mutex_exit(&pidlock);
2375 mutex_enter(&p->p_lock);
2376 mutex_exit(&pidlock);
2378 if ((ret = priv_proc_cred_perm(cr, p, NULL,
2380 mutex_exit(&p->p_lock);
2384 mutex_exit(&p->p_lock);
2388 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc);
2390 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2391 tp = fasttrap_tpoints.fth_table[index].ftb_data;
2392 while (tp != NULL) {
2393 if (instr.ftiq_pid == tp->ftt_pid &&
2394 instr.ftiq_pc == tp->ftt_pc &&
2395 tp->ftt_proc->ftpc_acount != 0)
2402 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2406 bcopy(&tp->ftt_instr, &instr.ftiq_instr,
2407 sizeof (instr.ftiq_instr));
2408 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2410 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0)
2425 /* Create the /dev/dtrace/fasttrap entry. */
2426 fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600,
2429 mtx_init(&fasttrap_cleanup_mtx, "fasttrap clean", "dtrace", MTX_DEF);
2430 mutex_init(&fasttrap_count_mtx, "fasttrap count mtx", MUTEX_DEFAULT,
2434 fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2435 "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT);
2440 * Conjure up the tracepoints hashtable...
2443 nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2444 "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE);
2446 nent = tpoints_hash_size;
2449 if (nent == 0 || nent > 0x1000000)
2450 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2452 tpoints_hash_size = nent;
2455 fasttrap_tpoints.fth_nent = nent;
2457 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
2458 ASSERT(fasttrap_tpoints.fth_nent > 0);
2459 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1;
2460 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *
2461 sizeof (fasttrap_bucket_t), KM_SLEEP);
2463 for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2464 mutex_init(&fasttrap_tpoints.fth_table[i].ftb_mtx,
2465 "tracepoints bucket mtx", MUTEX_DEFAULT, NULL);
2469 * ... and the providers hash table...
2471 nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE;
2473 fasttrap_provs.fth_nent = nent;
2475 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
2476 ASSERT(fasttrap_provs.fth_nent > 0);
2477 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1;
2478 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *
2479 sizeof (fasttrap_bucket_t), KM_SLEEP);
2481 for (i = 0; i < fasttrap_provs.fth_nent; i++)
2482 mutex_init(&fasttrap_provs.fth_table[i].ftb_mtx,
2483 "providers bucket mtx", MUTEX_DEFAULT, NULL);
2486 ret = kproc_create(fasttrap_pid_cleanup_cb, NULL,
2487 &fasttrap_cleanup_proc, 0, 0, "ftcleanup");
2489 destroy_dev(fasttrap_cdev);
2491 for (i = 0; i < fasttrap_provs.fth_nent; i++)
2492 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx);
2493 for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2494 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx);
2496 kmem_free(fasttrap_provs.fth_table, fasttrap_provs.fth_nent *
2497 sizeof (fasttrap_bucket_t));
2498 mtx_destroy(&fasttrap_cleanup_mtx);
2499 mutex_destroy(&fasttrap_count_mtx);
2505 * ... and the procs hash table.
2507 nent = FASTTRAP_PROCS_DEFAULT_SIZE;
2509 fasttrap_procs.fth_nent = nent;
2511 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
2512 ASSERT(fasttrap_procs.fth_nent > 0);
2513 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1;
2514 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *
2515 sizeof (fasttrap_bucket_t), KM_SLEEP);
2517 for (i = 0; i < fasttrap_procs.fth_nent; i++)
2518 mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx,
2519 "processes bucket mtx", MUTEX_DEFAULT, NULL);
2521 rm_init(&fasttrap_tp_lock, "fasttrap tracepoint");
2524 * This event handler must run before kdtrace_thread_dtor() since it
2525 * accesses the thread's struct kdtrace_thread.
2527 fasttrap_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor,
2528 fasttrap_thread_dtor, NULL, EVENTHANDLER_PRI_FIRST);
2532 * Install our hooks into fork(2), exec(2), and exit(2).
2534 dtrace_fasttrap_fork = &fasttrap_fork;
2535 dtrace_fasttrap_exit = &fasttrap_exec_exit;
2536 dtrace_fasttrap_exec = &fasttrap_exec_exit;
2538 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2545 fasttrap_unload(void)
2550 * Unregister the meta-provider to make sure no new fasttrap-
2551 * managed providers come along while we're trying to close up
2552 * shop. If we fail to detach, we'll need to re-register as a
2553 * meta-provider. We can fail to unregister as a meta-provider
2554 * if providers we manage still exist.
2556 if (fasttrap_meta_id != DTRACE_METAPROVNONE &&
2557 dtrace_meta_unregister(fasttrap_meta_id) != 0)
2561 * Iterate over all of our providers. If there's still a process
2562 * that corresponds to that pid, fail to detach.
2564 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
2565 fasttrap_provider_t **fpp, *fp;
2566 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i];
2568 mutex_enter(&bucket->ftb_mtx);
2569 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
2570 while ((fp = *fpp) != NULL) {
2572 * Acquire and release the lock as a simple way of
2573 * waiting for any other consumer to finish with
2574 * this provider. A thread must first acquire the
2575 * bucket lock so there's no chance of another thread
2576 * blocking on the provider's lock.
2578 mutex_enter(&fp->ftp_mtx);
2579 mutex_exit(&fp->ftp_mtx);
2581 if (dtrace_unregister(fp->ftp_provid) != 0) {
2583 fpp = &fp->ftp_next;
2585 *fpp = fp->ftp_next;
2586 fasttrap_provider_free(fp);
2590 mutex_exit(&bucket->ftb_mtx);
2594 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2601 * Stop new processes from entering these hooks now, before the
2602 * fasttrap_cleanup thread runs. That way all processes will hopefully
2603 * be out of these hooks before we free fasttrap_provs.fth_table
2605 ASSERT(dtrace_fasttrap_fork == &fasttrap_fork);
2606 dtrace_fasttrap_fork = NULL;
2608 ASSERT(dtrace_fasttrap_exec == &fasttrap_exec_exit);
2609 dtrace_fasttrap_exec = NULL;
2611 ASSERT(dtrace_fasttrap_exit == &fasttrap_exec_exit);
2612 dtrace_fasttrap_exit = NULL;
2614 mtx_lock(&fasttrap_cleanup_mtx);
2615 fasttrap_cleanup_drain = 1;
2616 /* Wait for the cleanup thread to finish up and signal us. */
2617 wakeup(&fasttrap_cleanup_cv);
2618 mtx_sleep(&fasttrap_cleanup_drain, &fasttrap_cleanup_mtx, 0, "ftcld",
2620 fasttrap_cleanup_proc = NULL;
2621 mtx_destroy(&fasttrap_cleanup_mtx);
2624 mutex_enter(&fasttrap_count_mtx);
2625 ASSERT(fasttrap_pid_count == 0);
2626 mutex_exit(&fasttrap_count_mtx);
2630 EVENTHANDLER_DEREGISTER(thread_dtor, fasttrap_thread_dtor_tag);
2632 for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2633 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx);
2634 for (i = 0; i < fasttrap_provs.fth_nent; i++)
2635 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx);
2636 for (i = 0; i < fasttrap_procs.fth_nent; i++)
2637 mutex_destroy(&fasttrap_procs.fth_table[i].ftb_mtx);
2639 kmem_free(fasttrap_tpoints.fth_table,
2640 fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t));
2641 fasttrap_tpoints.fth_nent = 0;
2643 kmem_free(fasttrap_provs.fth_table,
2644 fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t));
2645 fasttrap_provs.fth_nent = 0;
2647 kmem_free(fasttrap_procs.fth_table,
2648 fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t));
2649 fasttrap_procs.fth_nent = 0;
2652 destroy_dev(fasttrap_cdev);
2653 mutex_destroy(&fasttrap_count_mtx);
2654 rm_destroy(&fasttrap_tp_lock);
2662 fasttrap_modevent(module_t mod __unused, int type, void *data __unused)
2683 SYSINIT(fasttrap_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fasttrap_load,
2685 SYSUNINIT(fasttrap_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY,
2686 fasttrap_unload, NULL);
2688 DEV_MODULE(fasttrap, fasttrap_modevent, NULL);
2689 MODULE_VERSION(fasttrap, 1);
2690 MODULE_DEPEND(fasttrap, dtrace, 1, 1, 1);
2691 MODULE_DEPEND(fasttrap, opensolaris, 1, 1, 1);