4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
21 * Portions Copyright 2010 The FreeBSD Foundation
27 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
28 * Use is subject to license terms.
32 #pragma ident "%Z%%M% %I% %E% SMI"
35 #include <sys/atomic.h>
36 #include <sys/errno.h>
38 #include <sys/modctl.h>
40 #include <sys/systm.h>
44 #include <sys/sunddi.h>
45 #include <sys/cpuvar.h>
48 #include <sys/strsubr.h>
50 #include <sys/fasttrap.h>
51 #include <sys/fasttrap_impl.h>
52 #include <sys/fasttrap_isa.h>
53 #include <sys/dtrace.h>
54 #include <sys/dtrace_impl.h>
55 #include <sys/sysmacros.h>
57 #include <sys/policy.h>
59 #include <util/qsort.h>
61 #include <sys/mutex.h>
62 #include <sys/kernel.h>
65 #include <sys/dtrace_bsd.h>
66 #include <cddl/dev/dtrace/dtrace_cddl.h>
70 * User-Land Trap-Based Tracing
71 * ----------------------------
73 * The fasttrap provider allows DTrace consumers to instrument any user-level
74 * instruction to gather data; this includes probes with semantic
75 * signifigance like entry and return as well as simple offsets into the
76 * function. While the specific techniques used are very ISA specific, the
77 * methodology is generalizable to any architecture.
80 * The General Methodology
81 * -----------------------
83 * With the primary goal of tracing every user-land instruction and the
84 * limitation that we can't trust user space so don't want to rely on much
85 * information there, we begin by replacing the instructions we want to trace
86 * with trap instructions. Each instruction we overwrite is saved into a hash
87 * table keyed by process ID and pc address. When we enter the kernel due to
88 * this trap instruction, we need the effects of the replaced instruction to
89 * appear to have occurred before we proceed with the user thread's
92 * Each user level thread is represented by a ulwp_t structure which is
93 * always easily accessible through a register. The most basic way to produce
94 * the effects of the instruction we replaced is to copy that instruction out
95 * to a bit of scratch space reserved in the user thread's ulwp_t structure
96 * (a sort of kernel-private thread local storage), set the PC to that
97 * scratch space and single step. When we reenter the kernel after single
98 * stepping the instruction we must then adjust the PC to point to what would
99 * normally be the next instruction. Of course, special care must be taken
100 * for branches and jumps, but these represent such a small fraction of any
101 * instruction set that writing the code to emulate these in the kernel is
104 * Return probes may require several tracepoints to trace every return site,
105 * and, conversely, each tracepoint may activate several probes (the entry
106 * and offset 0 probes, for example). To solve this muliplexing problem,
107 * tracepoints contain lists of probes to activate and probes contain lists
108 * of tracepoints to enable. If a probe is activated, it adds its ID to
109 * existing tracepoints or creates new ones as necessary.
111 * Most probes are activated _before_ the instruction is executed, but return
112 * probes are activated _after_ the effects of the last instruction of the
113 * function are visible. Return probes must be fired _after_ we have
114 * single-stepped the instruction whereas all other probes are fired
121 * The lock ordering below -- both internally and with respect to the DTrace
122 * framework -- is a little tricky and bears some explanation. Each provider
123 * has a lock (ftp_mtx) that protects its members including reference counts
124 * for enabled probes (ftp_rcount), consumers actively creating probes
125 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider
126 * from being freed. A provider is looked up by taking the bucket lock for the
127 * provider hash table, and is returned with its lock held. The provider lock
128 * may be taken in functions invoked by the DTrace framework, but may not be
129 * held while calling functions in the DTrace framework.
131 * To ensure consistency over multiple calls to the DTrace framework, the
132 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may
133 * not be taken when holding the provider lock as that would create a cyclic
134 * lock ordering. In situations where one would naturally take the provider
135 * lock and then the creation lock, we instead up a reference count to prevent
136 * the provider from disappearing, drop the provider lock, and acquire the
140 * bucket lock before provider lock
141 * DTrace before provider lock
142 * creation lock before DTrace
143 * never hold the provider lock and creation lock simultaneously
146 static d_open_t fasttrap_open;
147 static d_ioctl_t fasttrap_ioctl;
149 static struct cdevsw fasttrap_cdevsw = {
150 .d_version = D_VERSION,
151 .d_open = fasttrap_open,
152 .d_ioctl = fasttrap_ioctl,
153 .d_name = "fasttrap",
155 static struct cdev *fasttrap_cdev;
156 static dtrace_meta_provider_id_t fasttrap_meta_id;
158 static struct callout fasttrap_timeout;
159 static struct mtx fasttrap_cleanup_mtx;
160 static uint_t fasttrap_cleanup_work;
163 * Generation count on modifications to the global tracepoint lookup table.
165 static volatile uint64_t fasttrap_mod_gen;
168 * When the fasttrap provider is loaded, fasttrap_max is set to either
169 * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the
170 * fasttrap.conf file. Each time a probe is created, fasttrap_total is
171 * incremented by the number of tracepoints that may be associated with that
172 * probe; fasttrap_total is capped at fasttrap_max.
174 #define FASTTRAP_MAX_DEFAULT 250000
175 static uint32_t fasttrap_max;
176 static uint32_t fasttrap_total;
179 * Copyright (c) 2011, Joyent, Inc. All rights reserved.
182 #define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000
183 #define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100
184 #define FASTTRAP_PROCS_DEFAULT_SIZE 0x100
186 #define FASTTRAP_PID_NAME "pid"
188 fasttrap_hash_t fasttrap_tpoints;
189 static fasttrap_hash_t fasttrap_provs;
190 static fasttrap_hash_t fasttrap_procs;
192 static uint64_t fasttrap_pid_count; /* pid ref count */
193 static kmutex_t fasttrap_count_mtx; /* lock on ref count */
195 #define FASTTRAP_ENABLE_FAIL 1
196 #define FASTTRAP_ENABLE_PARTIAL 2
198 static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
199 static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
201 static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
202 const dtrace_pattr_t *);
203 static void fasttrap_provider_retire(pid_t, const char *, int);
204 static void fasttrap_provider_free(fasttrap_provider_t *);
206 static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
207 static void fasttrap_proc_release(fasttrap_proc_t *);
209 #define FASTTRAP_PROVS_INDEX(pid, name) \
210 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
212 #define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
215 static kmutex_t fasttrap_cpuc_pid_lock[MAXCPU];
219 fasttrap_highbit(ulong_t i)
226 if (i & 0xffffffff00000000ul) {
230 if (i & 0xffff0000) {
249 fasttrap_hash_str(const char *p)
255 hval = (hval << 4) + *p++;
256 if ((g = (hval & 0xf0000000)) != 0)
264 fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
267 sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
269 sqp->sq_info.si_signo = SIGTRAP;
270 sqp->sq_info.si_code = TRAP_DTRACE;
271 sqp->sq_info.si_addr = (caddr_t)pc;
273 mutex_enter(&p->p_lock);
275 mutex_exit(&p->p_lock);
280 ksiginfo_t *ksi = kmem_zalloc(sizeof (ksiginfo_t), KM_SLEEP);
283 ksi->ksi_signo = SIGTRAP;
284 ksi->ksi_code = TRAP_DTRACE;
285 ksi->ksi_addr = (caddr_t)pc;
287 (void) tdksignal(t, SIGTRAP, ksi);
293 * This function ensures that no threads are actively using the memory
294 * associated with probes that were formerly live.
297 fasttrap_mod_barrier(uint64_t gen)
301 if (gen < fasttrap_mod_gen)
307 mutex_enter(&fasttrap_cpuc_pid_lock[i]);
308 mutex_exit(&fasttrap_cpuc_pid_lock[i]);
313 * This is the timeout's callback for cleaning up the providers and their
318 fasttrap_pid_cleanup_cb(void *data)
320 fasttrap_provider_t **fpp, *fp;
321 fasttrap_bucket_t *bucket;
322 dtrace_provider_id_t provid;
323 int i, later = 0, rval;
325 static volatile int in = 0;
329 while (fasttrap_cleanup_work) {
330 fasttrap_cleanup_work = 0;
331 mtx_unlock(&fasttrap_cleanup_mtx);
336 * Iterate over all the providers trying to remove the marked
337 * ones. If a provider is marked but not retired, we just
338 * have to take a crack at removing it -- it's no big deal if
341 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
342 bucket = &fasttrap_provs.fth_table[i];
343 mutex_enter(&bucket->ftb_mtx);
344 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
346 while ((fp = *fpp) != NULL) {
347 if (!fp->ftp_marked) {
352 mutex_enter(&fp->ftp_mtx);
355 * If this provider has consumers actively
356 * creating probes (ftp_ccount) or is a USDT
357 * provider (ftp_mcount), we can't unregister
360 if (fp->ftp_ccount != 0 ||
361 fp->ftp_mcount != 0) {
362 mutex_exit(&fp->ftp_mtx);
367 if (!fp->ftp_retired || fp->ftp_rcount != 0)
370 mutex_exit(&fp->ftp_mtx);
373 * If we successfully unregister this
374 * provider we can remove it from the hash
375 * chain and free the memory. If our attempt
376 * to unregister fails and this is a retired
377 * provider, increment our flag to try again
378 * pretty soon. If we've consumed more than
379 * half of our total permitted number of
380 * probes call dtrace_condense() to try to
381 * clean out the unenabled probes.
383 provid = fp->ftp_provid;
384 if ((rval = dtrace_unregister(provid)) != 0) {
385 if (fasttrap_total > fasttrap_max / 2)
386 (void) dtrace_condense(provid);
391 later += fp->ftp_marked;
395 fasttrap_provider_free(fp);
398 mutex_exit(&bucket->ftb_mtx);
401 mtx_lock(&fasttrap_cleanup_mtx);
405 ASSERT(fasttrap_timeout != 0);
409 * If we were unable to remove a retired provider, try again after
410 * a second. This situation can occur in certain circumstances where
411 * providers cannot be unregistered even though they have no probes
412 * enabled because of an execution of dtrace -l or something similar.
413 * If the timeout has been disabled (set to 1 because we're trying
414 * to detach), we set fasttrap_cleanup_work to ensure that we'll
415 * get a chance to do that work if and when the timeout is reenabled
419 if (callout_active(&fasttrap_timeout)) {
420 callout_reset(&fasttrap_timeout, hz,
421 &fasttrap_pid_cleanup_cb, NULL);
425 fasttrap_cleanup_work = 1;
428 /* Nothing to be done for FreeBSD */
436 * Activates the asynchronous cleanup mechanism.
439 fasttrap_pid_cleanup(void)
442 mtx_lock(&fasttrap_cleanup_mtx);
443 fasttrap_cleanup_work = 1;
444 callout_reset(&fasttrap_timeout, 1, &fasttrap_pid_cleanup_cb, NULL);
445 mtx_unlock(&fasttrap_cleanup_mtx);
449 * This is called from cfork() via dtrace_fasttrap_fork(). The child
450 * process's address space is (roughly) a copy of the parent process's so
451 * we have to remove all the instrumentation we had previously enabled in the
455 fasttrap_fork(proc_t *p, proc_t *cp)
457 pid_t ppid = p->p_pid;
461 ASSERT(curproc == p);
462 ASSERT(p->p_proc_flag & P_PR_LOCK);
464 PROC_LOCK_ASSERT(p, MA_OWNED);
467 ASSERT(p->p_dtrace_count > 0);
469 if (p->p_dtrace_helpers) {
471 * dtrace_helpers_duplicate() allocates memory.
476 dtrace_helpers_duplicate(p, cp);
482 * This check is purposely here instead of in kern_fork.c because,
483 * for legal resons, we cannot include the dtrace_cddl.h header
484 * inside kern_fork.c and insert if-clause there.
486 if (p->p_dtrace_count == 0)
489 ASSERT(cp->p_dtrace_count == 0);
492 * This would be simpler and faster if we maintained per-process
493 * hash tables of enabled tracepoints. It could, however, potentially
494 * slow down execution of a tracepoint since we'd need to go
495 * through two levels of indirection. In the future, we should
496 * consider either maintaining per-process ancillary lists of
497 * enabled tracepoints or hanging a pointer to a per-process hash
498 * table of enabled tracepoints off the proc structure.
502 * We don't have to worry about the child process disappearing
503 * because we're in fork().
506 mtx_lock_spin(&cp->p_slock);
508 mtx_unlock_spin(&cp->p_slock);
514 * Iterate over every tracepoint looking for ones that belong to the
515 * parent process, and remove each from the child process.
517 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
518 fasttrap_tracepoint_t *tp;
519 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
521 mutex_enter(&bucket->ftb_mtx);
522 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
523 if (tp->ftt_pid == ppid &&
524 tp->ftt_proc->ftpc_acount != 0) {
525 int ret = fasttrap_tracepoint_remove(cp, tp);
529 * The count of active providers can only be
530 * decremented (i.e. to zero) during exec,
531 * exit, and removal of a meta provider so it
532 * should be impossible to drop the count
535 ASSERT(tp->ftt_proc->ftpc_acount != 0);
538 mutex_exit(&bucket->ftb_mtx);
542 mutex_enter(&cp->p_lock);
550 * This is called from proc_exit() or from exec_common() if p_dtrace_probes
551 * is set on the proc structure to indicate that there is a pid provider
552 * associated with this process.
555 fasttrap_exec_exit(proc_t *p)
558 ASSERT(p == curproc);
560 PROC_LOCK_ASSERT(p, MA_OWNED);
565 * We clean up the pid provider for this process here; user-land
566 * static probes are handled by the meta-provider remove entry point.
568 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
570 if (p->p_dtrace_helpers)
571 dtrace_helpers_destroy(p);
580 fasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc)
583 * There are no "default" pid probes.
588 fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
590 fasttrap_tracepoint_t *tp, *new_tp = NULL;
591 fasttrap_bucket_t *bucket;
596 ASSERT(index < probe->ftp_ntps);
598 pid = probe->ftp_pid;
599 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
600 id = &probe->ftp_tps[index].fit_id;
602 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
605 ASSERT(!(p->p_flag & SVFORK));
609 * Before we make any modifications, make sure we've imposed a barrier
610 * on the generation in which this probe was last modified.
612 fasttrap_mod_barrier(probe->ftp_gen);
614 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
617 * If the tracepoint has already been enabled, just add our id to the
618 * list of interested probes. This may be our second time through
619 * this path in which case we'll have constructed the tracepoint we'd
620 * like to install. If we can't find a match, and have an allocated
621 * tracepoint ready to go, enable that one now.
623 * A tracepoint whose process is defunct is also considered defunct.
626 mutex_enter(&bucket->ftb_mtx);
627 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
629 * Note that it's safe to access the active count on the
630 * associated proc structure because we know that at least one
631 * provider (this one) will still be around throughout this
634 if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
635 tp->ftt_proc->ftpc_acount == 0)
639 * Now that we've found a matching tracepoint, it would be
640 * a decent idea to confirm that the tracepoint is still
641 * enabled and the trap instruction hasn't been overwritten.
642 * Since this is a little hairy, we'll punt for now.
646 * This can't be the first interested probe. We don't have
647 * to worry about another thread being in the midst of
648 * deleting this tracepoint (which would be the only valid
649 * reason for a tracepoint to have no interested probes)
650 * since we're holding P_PR_LOCK for this process.
652 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
654 switch (id->fti_ptype) {
657 case DTFTP_IS_ENABLED:
658 id->fti_next = tp->ftt_ids;
665 case DTFTP_POST_OFFSETS:
666 id->fti_next = tp->ftt_retids;
676 mutex_exit(&bucket->ftb_mtx);
678 if (new_tp != NULL) {
679 new_tp->ftt_ids = NULL;
680 new_tp->ftt_retids = NULL;
687 * If we have a good tracepoint ready to go, install it now while
688 * we have the lock held and no one can screw with us.
690 if (new_tp != NULL) {
693 new_tp->ftt_next = bucket->ftb_data;
695 bucket->ftb_data = new_tp;
697 mutex_exit(&bucket->ftb_mtx);
700 * Activate the tracepoint in the ISA-specific manner.
701 * If this fails, we need to report the failure, but
702 * indicate that this tracepoint must still be disabled
703 * by calling fasttrap_tracepoint_disable().
705 if (fasttrap_tracepoint_install(p, new_tp) != 0)
706 rc = FASTTRAP_ENABLE_PARTIAL;
709 * Increment the count of the number of tracepoints active in
710 * the victim process.
713 ASSERT(p->p_proc_flag & P_PR_LOCK);
720 mutex_exit(&bucket->ftb_mtx);
723 * Initialize the tracepoint that's been preallocated with the probe.
725 new_tp = probe->ftp_tps[index].fit_tp;
727 ASSERT(new_tp->ftt_pid == pid);
728 ASSERT(new_tp->ftt_pc == pc);
729 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
730 ASSERT(new_tp->ftt_ids == NULL);
731 ASSERT(new_tp->ftt_retids == NULL);
733 switch (id->fti_ptype) {
736 case DTFTP_IS_ENABLED:
738 new_tp->ftt_ids = id;
742 case DTFTP_POST_OFFSETS:
744 new_tp->ftt_retids = id;
752 * If the ISA-dependent initialization goes to plan, go back to the
753 * beginning and try to install this freshly made tracepoint.
755 if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
758 new_tp->ftt_ids = NULL;
759 new_tp->ftt_retids = NULL;
761 return (FASTTRAP_ENABLE_FAIL);
765 fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
767 fasttrap_bucket_t *bucket;
768 fasttrap_provider_t *provider = probe->ftp_prov;
769 fasttrap_tracepoint_t **pp, *tp;
770 fasttrap_id_t *id, **idp = NULL;
774 ASSERT(index < probe->ftp_ntps);
776 pid = probe->ftp_pid;
777 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
778 id = &probe->ftp_tps[index].fit_id;
780 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
783 * Find the tracepoint and make sure that our id is one of the
784 * ones registered with it.
786 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
787 mutex_enter(&bucket->ftb_mtx);
788 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
789 if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
790 tp->ftt_proc == provider->ftp_proc)
795 * If we somehow lost this tracepoint, we're in a world of hurt.
799 switch (id->fti_ptype) {
802 case DTFTP_IS_ENABLED:
803 ASSERT(tp->ftt_ids != NULL);
808 case DTFTP_POST_OFFSETS:
809 ASSERT(tp->ftt_retids != NULL);
810 idp = &tp->ftt_retids;
817 while ((*idp)->fti_probe != probe) {
818 idp = &(*idp)->fti_next;
819 ASSERT(*idp != NULL);
826 ASSERT(id->fti_probe == probe);
829 * If there are other registered enablings of this tracepoint, we're
830 * all done, but if this was the last probe assocated with this
831 * this tracepoint, we need to remove and free it.
833 if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
836 * If the current probe's tracepoint is in use, swap it
837 * for an unused tracepoint.
839 if (tp == probe->ftp_tps[index].fit_tp) {
840 fasttrap_probe_t *tmp_probe;
841 fasttrap_tracepoint_t **tmp_tp;
844 if (tp->ftt_ids != NULL) {
845 tmp_probe = tp->ftt_ids->fti_probe;
846 /* LINTED - alignment */
847 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
848 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
850 tmp_probe = tp->ftt_retids->fti_probe;
851 /* LINTED - alignment */
852 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
853 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
856 ASSERT(*tmp_tp != NULL);
857 ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
858 ASSERT((*tmp_tp)->ftt_ids == NULL);
859 ASSERT((*tmp_tp)->ftt_retids == NULL);
861 probe->ftp_tps[index].fit_tp = *tmp_tp;
865 mutex_exit(&bucket->ftb_mtx);
868 * Tag the modified probe with the generation in which it was
871 probe->ftp_gen = fasttrap_mod_gen;
875 mutex_exit(&bucket->ftb_mtx);
878 * We can't safely remove the tracepoint from the set of active
879 * tracepoints until we've actually removed the fasttrap instruction
880 * from the process's text. We can, however, operate on this
881 * tracepoint secure in the knowledge that no other thread is going to
882 * be looking at it since we hold P_PR_LOCK on the process if it's
883 * live or we hold the provider lock on the process if it's dead and
888 * We only need to remove the actual instruction if we're looking
889 * at an existing process
893 * If we fail to restore the instruction we need to kill
894 * this process since it's in a completely unrecoverable
897 if (fasttrap_tracepoint_remove(p, tp) != 0)
898 fasttrap_sigtrap(p, NULL, pc);
901 * Decrement the count of the number of tracepoints active
902 * in the victim process.
905 ASSERT(p->p_proc_flag & P_PR_LOCK);
911 * Remove the probe from the hash table of active tracepoints.
913 mutex_enter(&bucket->ftb_mtx);
914 pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
917 pp = &(*pp)->ftt_next;
924 mutex_exit(&bucket->ftb_mtx);
927 * Tag the modified probe with the generation in which it was changed.
929 probe->ftp_gen = fasttrap_mod_gen;
933 fasttrap_enable_callbacks(void)
936 * We don't have to play the rw lock game here because we're
937 * providing something rather than taking something away --
938 * we can be sure that no threads have tried to follow this
939 * function pointer yet.
941 mutex_enter(&fasttrap_count_mtx);
942 if (fasttrap_pid_count == 0) {
943 ASSERT(dtrace_pid_probe_ptr == NULL);
944 ASSERT(dtrace_return_probe_ptr == NULL);
945 dtrace_pid_probe_ptr = &fasttrap_pid_probe;
946 dtrace_return_probe_ptr = &fasttrap_return_probe;
948 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe);
949 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe);
950 fasttrap_pid_count++;
951 mutex_exit(&fasttrap_count_mtx);
955 fasttrap_disable_callbacks(void)
958 ASSERT(MUTEX_HELD(&cpu_lock));
962 mutex_enter(&fasttrap_count_mtx);
963 ASSERT(fasttrap_pid_count > 0);
964 fasttrap_pid_count--;
965 if (fasttrap_pid_count == 0) {
967 cpu_t *cur, *cpu = CPU;
969 for (cur = cpu->cpu_next_onln; cur != cpu;
970 cur = cur->cpu_next_onln) {
971 rw_enter(&cur->cpu_ft_lock, RW_WRITER);
974 dtrace_pid_probe_ptr = NULL;
975 dtrace_return_probe_ptr = NULL;
977 for (cur = cpu->cpu_next_onln; cur != cpu;
978 cur = cur->cpu_next_onln) {
979 rw_exit(&cur->cpu_ft_lock);
983 mutex_exit(&fasttrap_count_mtx);
988 fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
990 fasttrap_probe_t *probe = parg;
995 ASSERT(probe != NULL);
996 ASSERT(!probe->ftp_enabled);
997 ASSERT(id == probe->ftp_id);
999 ASSERT(MUTEX_HELD(&cpu_lock));
1003 * Increment the count of enabled probes on this probe's provider;
1004 * the provider can't go away while the probe still exists. We
1005 * must increment this even if we aren't able to properly enable
1008 mutex_enter(&probe->ftp_prov->ftp_mtx);
1009 probe->ftp_prov->ftp_rcount++;
1010 mutex_exit(&probe->ftp_prov->ftp_mtx);
1013 * If this probe's provider is retired (meaning it was valid in a
1014 * previously exec'ed incarnation of this address space), bail out. The
1015 * provider can't go away while we're in this code path.
1017 if (probe->ftp_prov->ftp_retired)
1021 * If we can't find the process, it may be that we're in the context of
1022 * a fork in which the traced process is being born and we're copying
1023 * USDT probes. Otherwise, the process is gone so bail.
1026 if ((p = sprlock(probe->ftp_pid)) == NULL) {
1027 if ((curproc->p_flag & SFORKING) == 0)
1030 mutex_enter(&pidlock);
1031 p = prfind(probe->ftp_pid);
1034 * Confirm that curproc is indeed forking the process in which
1035 * we're trying to enable probes.
1038 ASSERT(p->p_parent == curproc);
1039 ASSERT(p->p_stat == SIDL);
1041 mutex_enter(&p->p_lock);
1042 mutex_exit(&pidlock);
1047 ASSERT(!(p->p_flag & SVFORK));
1048 mutex_exit(&p->p_lock);
1050 if ((p = pfind(probe->ftp_pid)) == NULL)
1055 * We have to enable the trap entry point before any user threads have
1056 * the chance to execute the trap instruction we're about to place
1057 * in their process's text.
1061 * pfind() returns a locked process.
1066 fasttrap_enable_callbacks();
1069 * Enable all the tracepoints and add this probe's id to each
1070 * tracepoint's list of active probes.
1072 for (i = 0; i < probe->ftp_ntps; i++) {
1073 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
1075 * If enabling the tracepoint failed completely,
1076 * we don't have to disable it; if the failure
1077 * was only partial we must disable it.
1079 if (rc == FASTTRAP_ENABLE_FAIL)
1082 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
1085 * Back up and pull out all the tracepoints we've
1086 * created so far for this probe.
1089 fasttrap_tracepoint_disable(p, probe, i);
1094 mutex_enter(&p->p_lock);
1101 * Since we're not actually enabling this probe,
1102 * drop our reference on the trap table entry.
1104 fasttrap_disable_callbacks();
1109 mutex_enter(&p->p_lock);
1115 probe->ftp_enabled = 1;
1120 fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
1122 fasttrap_probe_t *probe = parg;
1123 fasttrap_provider_t *provider = probe->ftp_prov;
1127 ASSERT(id == probe->ftp_id);
1129 mutex_enter(&provider->ftp_mtx);
1132 * We won't be able to acquire a /proc-esque lock on the process
1133 * iff the process is dead and gone. In this case, we rely on the
1134 * provider lock as a point of mutual exclusion to prevent other
1135 * DTrace consumers from disabling this probe.
1137 if ((p = pfind(probe->ftp_pid)) != NULL) {
1145 * Disable all the associated tracepoints (for fully enabled probes).
1147 if (probe->ftp_enabled) {
1148 for (i = 0; i < probe->ftp_ntps; i++) {
1149 fasttrap_tracepoint_disable(p, probe, i);
1153 ASSERT(provider->ftp_rcount > 0);
1154 provider->ftp_rcount--;
1158 * Even though we may not be able to remove it entirely, we
1159 * mark this retired provider to get a chance to remove some
1160 * of the associated probes.
1162 if (provider->ftp_retired && !provider->ftp_marked)
1163 whack = provider->ftp_marked = 1;
1164 mutex_exit(&provider->ftp_mtx);
1167 * If the process is dead, we're just waiting for the
1168 * last probe to be disabled to be able to free it.
1170 if (provider->ftp_rcount == 0 && !provider->ftp_marked)
1171 whack = provider->ftp_marked = 1;
1172 mutex_exit(&provider->ftp_mtx);
1176 fasttrap_pid_cleanup();
1182 if (!probe->ftp_enabled)
1185 probe->ftp_enabled = 0;
1188 ASSERT(MUTEX_HELD(&cpu_lock));
1190 fasttrap_disable_callbacks();
1195 fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
1196 dtrace_argdesc_t *desc)
1198 fasttrap_probe_t *probe = parg;
1202 desc->dtargd_native[0] = '\0';
1203 desc->dtargd_xlate[0] = '\0';
1205 if (probe->ftp_prov->ftp_retired != 0 ||
1206 desc->dtargd_ndx >= probe->ftp_nargs) {
1207 desc->dtargd_ndx = DTRACE_ARGNONE;
1211 ndx = (probe->ftp_argmap != NULL) ?
1212 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx;
1214 str = probe->ftp_ntypes;
1215 for (i = 0; i < ndx; i++) {
1216 str += strlen(str) + 1;
1219 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native));
1220 (void) strcpy(desc->dtargd_native, str);
1222 if (probe->ftp_xtypes == NULL)
1225 str = probe->ftp_xtypes;
1226 for (i = 0; i < desc->dtargd_ndx; i++) {
1227 str += strlen(str) + 1;
1230 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate));
1231 (void) strcpy(desc->dtargd_xlate, str);
1236 fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
1238 fasttrap_probe_t *probe = parg;
1242 ASSERT(probe != NULL);
1243 ASSERT(!probe->ftp_enabled);
1244 ASSERT(fasttrap_total >= probe->ftp_ntps);
1246 atomic_add_32(&fasttrap_total, -probe->ftp_ntps);
1247 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
1249 if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
1250 fasttrap_mod_barrier(probe->ftp_gen);
1252 for (i = 0; i < probe->ftp_ntps; i++) {
1253 kmem_free(probe->ftp_tps[i].fit_tp,
1254 sizeof (fasttrap_tracepoint_t));
1257 kmem_free(probe, size);
1261 static const dtrace_pattr_t pid_attr = {
1262 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1263 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1264 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1265 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1266 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1269 static dtrace_pops_t pid_pops = {
1270 fasttrap_pid_provide,
1272 fasttrap_pid_enable,
1273 fasttrap_pid_disable,
1276 fasttrap_pid_getargdesc,
1277 fasttrap_pid_getarg,
1279 fasttrap_pid_destroy
1282 static dtrace_pops_t usdt_pops = {
1283 fasttrap_pid_provide,
1285 fasttrap_pid_enable,
1286 fasttrap_pid_disable,
1289 fasttrap_pid_getargdesc,
1290 fasttrap_usdt_getarg,
1292 fasttrap_pid_destroy
1295 static fasttrap_proc_t *
1296 fasttrap_proc_lookup(pid_t pid)
1298 fasttrap_bucket_t *bucket;
1299 fasttrap_proc_t *fprc, *new_fprc;
1302 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1303 mutex_enter(&bucket->ftb_mtx);
1305 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1306 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1307 mutex_enter(&fprc->ftpc_mtx);
1308 mutex_exit(&bucket->ftb_mtx);
1309 fprc->ftpc_rcount++;
1310 atomic_add_64(&fprc->ftpc_acount, 1);
1311 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1312 mutex_exit(&fprc->ftpc_mtx);
1319 * Drop the bucket lock so we don't try to perform a sleeping
1320 * allocation under it.
1322 mutex_exit(&bucket->ftb_mtx);
1324 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP);
1325 new_fprc->ftpc_pid = pid;
1326 new_fprc->ftpc_rcount = 1;
1327 new_fprc->ftpc_acount = 1;
1329 mutex_init(&new_fprc->ftpc_mtx, "fasttrap proc mtx", MUTEX_DEFAULT,
1333 mutex_enter(&bucket->ftb_mtx);
1336 * Take another lap through the list to make sure a proc hasn't
1337 * been created for this pid while we weren't under the bucket lock.
1339 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1340 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1341 mutex_enter(&fprc->ftpc_mtx);
1342 mutex_exit(&bucket->ftb_mtx);
1343 fprc->ftpc_rcount++;
1344 atomic_add_64(&fprc->ftpc_acount, 1);
1345 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1346 mutex_exit(&fprc->ftpc_mtx);
1348 kmem_free(new_fprc, sizeof (fasttrap_proc_t));
1354 new_fprc->ftpc_next = bucket->ftb_data;
1355 bucket->ftb_data = new_fprc;
1357 mutex_exit(&bucket->ftb_mtx);
1363 fasttrap_proc_release(fasttrap_proc_t *proc)
1365 fasttrap_bucket_t *bucket;
1366 fasttrap_proc_t *fprc, **fprcp;
1367 pid_t pid = proc->ftpc_pid;
1369 mutex_enter(&proc->ftpc_mtx);
1371 ASSERT(proc->ftpc_rcount != 0);
1372 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount);
1374 if (--proc->ftpc_rcount != 0) {
1375 mutex_exit(&proc->ftpc_mtx);
1379 mutex_exit(&proc->ftpc_mtx);
1382 * There should definitely be no live providers associated with this
1383 * process at this point.
1385 ASSERT(proc->ftpc_acount == 0);
1387 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1388 mutex_enter(&bucket->ftb_mtx);
1390 fprcp = (fasttrap_proc_t **)&bucket->ftb_data;
1391 while ((fprc = *fprcp) != NULL) {
1395 fprcp = &fprc->ftpc_next;
1399 * Something strange has happened if we can't find the proc.
1401 ASSERT(fprc != NULL);
1403 *fprcp = fprc->ftpc_next;
1405 mutex_exit(&bucket->ftb_mtx);
1407 kmem_free(fprc, sizeof (fasttrap_proc_t));
1411 * Lookup a fasttrap-managed provider based on its name and associated pid.
1412 * If the pattr argument is non-NULL, this function instantiates the provider
1413 * if it doesn't exist otherwise it returns NULL. The provider is returned
1414 * with its lock held.
1416 static fasttrap_provider_t *
1417 fasttrap_provider_lookup(pid_t pid, const char *name,
1418 const dtrace_pattr_t *pattr)
1420 fasttrap_provider_t *fp, *new_fp = NULL;
1421 fasttrap_bucket_t *bucket;
1422 char provname[DTRACE_PROVNAMELEN];
1426 ASSERT(strlen(name) < sizeof (fp->ftp_name));
1427 ASSERT(pattr != NULL);
1429 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1430 mutex_enter(&bucket->ftb_mtx);
1433 * Take a lap through the list and return the match if we find it.
1435 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1436 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1438 mutex_enter(&fp->ftp_mtx);
1439 mutex_exit(&bucket->ftb_mtx);
1445 * Drop the bucket lock so we don't try to perform a sleeping
1446 * allocation under it.
1448 mutex_exit(&bucket->ftb_mtx);
1451 * Make sure the process exists, isn't a child created as the result
1452 * of a vfork(2), and isn't a zombie (but may be in fork).
1454 if ((p = pfind(pid)) == NULL)
1458 * Increment p_dtrace_probes so that the process knows to inform us
1459 * when it exits or execs. fasttrap_provider_free() decrements this
1460 * when we're done with this provider.
1462 p->p_dtrace_probes++;
1465 * Grab the credentials for this process so we have
1466 * something to pass to dtrace_register().
1468 PROC_LOCK_ASSERT(p, MA_OWNED);
1473 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP);
1474 new_fp->ftp_pid = pid;
1475 new_fp->ftp_proc = fasttrap_proc_lookup(pid);
1477 mutex_init(&new_fp->ftp_mtx, "provider mtx", MUTEX_DEFAULT, NULL);
1478 mutex_init(&new_fp->ftp_cmtx, "lock on creating", MUTEX_DEFAULT, NULL);
1481 ASSERT(new_fp->ftp_proc != NULL);
1483 mutex_enter(&bucket->ftb_mtx);
1486 * Take another lap through the list to make sure a provider hasn't
1487 * been created for this pid while we weren't under the bucket lock.
1489 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1490 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1492 mutex_enter(&fp->ftp_mtx);
1493 mutex_exit(&bucket->ftb_mtx);
1494 fasttrap_provider_free(new_fp);
1500 (void) strcpy(new_fp->ftp_name, name);
1503 * Fail and return NULL if either the provider name is too long
1504 * or we fail to register this new provider with the DTrace
1505 * framework. Note that this is the only place we ever construct
1506 * the full provider name -- we keep it in pieces in the provider
1509 if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >=
1510 sizeof (provname) ||
1511 dtrace_register(provname, pattr,
1512 DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred,
1513 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp,
1514 &new_fp->ftp_provid) != 0) {
1515 mutex_exit(&bucket->ftb_mtx);
1516 fasttrap_provider_free(new_fp);
1521 new_fp->ftp_next = bucket->ftb_data;
1522 bucket->ftb_data = new_fp;
1524 mutex_enter(&new_fp->ftp_mtx);
1525 mutex_exit(&bucket->ftb_mtx);
1532 fasttrap_provider_free(fasttrap_provider_t *provider)
1534 pid_t pid = provider->ftp_pid;
1538 * There need to be no associated enabled probes, no consumers
1539 * creating probes, and no meta providers referencing this provider.
1541 ASSERT(provider->ftp_rcount == 0);
1542 ASSERT(provider->ftp_ccount == 0);
1543 ASSERT(provider->ftp_mcount == 0);
1546 * If this provider hasn't been retired, we need to explicitly drop the
1547 * count of active providers on the associated process structure.
1549 if (!provider->ftp_retired) {
1550 atomic_add_64(&provider->ftp_proc->ftpc_acount, -1);
1551 ASSERT(provider->ftp_proc->ftpc_acount <
1552 provider->ftp_proc->ftpc_rcount);
1555 fasttrap_proc_release(provider->ftp_proc);
1558 mutex_destroy(&provider->ftp_mtx);
1559 mutex_destroy(&provider->ftp_cmtx);
1561 kmem_free(provider, sizeof (fasttrap_provider_t));
1564 * Decrement p_dtrace_probes on the process whose provider we're
1565 * freeing. We don't have to worry about clobbering somone else's
1566 * modifications to it because we have locked the bucket that
1567 * corresponds to this process's hash chain in the provider hash
1568 * table. Don't sweat it if we can't find the process.
1570 if ((p = pfind(pid)) == NULL) {
1574 p->p_dtrace_probes--;
1581 fasttrap_provider_retire(pid_t pid, const char *name, int mprov)
1583 fasttrap_provider_t *fp;
1584 fasttrap_bucket_t *bucket;
1585 dtrace_provider_id_t provid;
1587 ASSERT(strlen(name) < sizeof (fp->ftp_name));
1589 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1590 mutex_enter(&bucket->ftb_mtx);
1592 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1593 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1599 mutex_exit(&bucket->ftb_mtx);
1603 mutex_enter(&fp->ftp_mtx);
1604 ASSERT(!mprov || fp->ftp_mcount > 0);
1605 if (mprov && --fp->ftp_mcount != 0) {
1606 mutex_exit(&fp->ftp_mtx);
1607 mutex_exit(&bucket->ftb_mtx);
1612 * Mark the provider to be removed in our post-processing step, mark it
1613 * retired, and drop the active count on its proc. Marking it indicates
1614 * that we should try to remove it; setting the retired flag indicates
1615 * that we're done with this provider; dropping the active the proc
1616 * releases our hold, and when this reaches zero (as it will during
1617 * exit or exec) the proc and associated providers become defunct.
1619 * We obviously need to take the bucket lock before the provider lock
1620 * to perform the lookup, but we need to drop the provider lock
1621 * before calling into the DTrace framework since we acquire the
1622 * provider lock in callbacks invoked from the DTrace framework. The
1623 * bucket lock therefore protects the integrity of the provider hash
1626 atomic_add_64(&fp->ftp_proc->ftpc_acount, -1);
1627 ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount);
1629 fp->ftp_retired = 1;
1631 provid = fp->ftp_provid;
1632 mutex_exit(&fp->ftp_mtx);
1635 * We don't have to worry about invalidating the same provider twice
1636 * since fasttrap_provider_lookup() will ignore provider that have
1637 * been marked as retired.
1639 dtrace_invalidate(provid);
1641 mutex_exit(&bucket->ftb_mtx);
1643 fasttrap_pid_cleanup();
1647 fasttrap_uint32_cmp(const void *ap, const void *bp)
1649 return (*(const uint32_t *)ap - *(const uint32_t *)bp);
1653 fasttrap_uint64_cmp(const void *ap, const void *bp)
1655 return (*(const uint64_t *)ap - *(const uint64_t *)bp);
1659 fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
1661 fasttrap_provider_t *provider;
1662 fasttrap_probe_t *pp;
1663 fasttrap_tracepoint_t *tp;
1665 int i, aframes = 0, whack;
1668 * There needs to be at least one desired trace point.
1670 if (pdata->ftps_noffs == 0)
1673 switch (pdata->ftps_type) {
1676 aframes = FASTTRAP_ENTRY_AFRAMES;
1680 aframes = FASTTRAP_RETURN_AFRAMES;
1689 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid,
1690 FASTTRAP_PID_NAME, &pid_attr)) == NULL)
1694 * Increment this reference count to indicate that a consumer is
1695 * actively adding a new probe associated with this provider. This
1696 * prevents the provider from being deleted -- we'll need to check
1697 * for pending deletions when we drop this reference count.
1699 provider->ftp_ccount++;
1700 mutex_exit(&provider->ftp_mtx);
1703 * Grab the creation lock to ensure consistency between calls to
1704 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1705 * other threads creating probes. We must drop the provider lock
1706 * before taking this lock to avoid a three-way deadlock with the
1709 mutex_enter(&provider->ftp_cmtx);
1712 for (i = 0; i < pdata->ftps_noffs; i++) {
1715 (void) sprintf(name_str, "%llx",
1716 (unsigned long long)pdata->ftps_offs[i]);
1718 if (dtrace_probe_lookup(provider->ftp_provid,
1719 pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
1722 atomic_add_32(&fasttrap_total, 1);
1724 if (fasttrap_total > fasttrap_max) {
1725 atomic_add_32(&fasttrap_total, -1);
1729 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP);
1731 pp->ftp_prov = provider;
1732 pp->ftp_faddr = pdata->ftps_pc;
1733 pp->ftp_fsize = pdata->ftps_size;
1734 pp->ftp_pid = pdata->ftps_pid;
1737 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1740 tp->ftt_proc = provider->ftp_proc;
1741 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1742 tp->ftt_pid = pdata->ftps_pid;
1744 pp->ftp_tps[0].fit_tp = tp;
1745 pp->ftp_tps[0].fit_id.fti_probe = pp;
1746 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type;
1748 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1749 pdata->ftps_mod, pdata->ftps_func, name_str,
1750 FASTTRAP_OFFSET_AFRAMES, pp);
1753 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod,
1754 pdata->ftps_func, name) == 0) {
1755 atomic_add_32(&fasttrap_total, pdata->ftps_noffs);
1757 if (fasttrap_total > fasttrap_max) {
1758 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1763 * Make sure all tracepoint program counter values are unique.
1764 * We later assume that each probe has exactly one tracepoint
1767 qsort(pdata->ftps_offs, pdata->ftps_noffs,
1768 sizeof (uint64_t), fasttrap_uint64_cmp);
1769 for (i = 1; i < pdata->ftps_noffs; i++) {
1770 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1])
1773 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1777 ASSERT(pdata->ftps_noffs > 0);
1778 pp = kmem_zalloc(offsetof(fasttrap_probe_t,
1779 ftp_tps[pdata->ftps_noffs]), KM_SLEEP);
1781 pp->ftp_prov = provider;
1782 pp->ftp_faddr = pdata->ftps_pc;
1783 pp->ftp_fsize = pdata->ftps_size;
1784 pp->ftp_pid = pdata->ftps_pid;
1785 pp->ftp_ntps = pdata->ftps_noffs;
1787 for (i = 0; i < pdata->ftps_noffs; i++) {
1788 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1791 tp->ftt_proc = provider->ftp_proc;
1792 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1793 tp->ftt_pid = pdata->ftps_pid;
1795 pp->ftp_tps[i].fit_tp = tp;
1796 pp->ftp_tps[i].fit_id.fti_probe = pp;
1797 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type;
1800 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1801 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp);
1804 mutex_exit(&provider->ftp_cmtx);
1807 * We know that the provider is still valid since we incremented the
1808 * creation reference count. If someone tried to clean up this provider
1809 * while we were using it (e.g. because the process called exec(2) or
1810 * exit(2)), take note of that and try to clean it up now.
1812 mutex_enter(&provider->ftp_mtx);
1813 provider->ftp_ccount--;
1814 whack = provider->ftp_retired;
1815 mutex_exit(&provider->ftp_mtx);
1818 fasttrap_pid_cleanup();
1824 * If we've exhausted the allowable resources, we'll try to remove
1825 * this provider to free some up. This is to cover the case where
1826 * the user has accidentally created many more probes than was
1827 * intended (e.g. pid123:::).
1829 mutex_exit(&provider->ftp_cmtx);
1830 mutex_enter(&provider->ftp_mtx);
1831 provider->ftp_ccount--;
1832 provider->ftp_marked = 1;
1833 mutex_exit(&provider->ftp_mtx);
1835 fasttrap_pid_cleanup();
1842 fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1844 fasttrap_provider_t *provider;
1847 * A 32-bit unsigned integer (like a pid for example) can be
1848 * expressed in 10 or fewer decimal digits. Make sure that we'll
1849 * have enough space for the provider name.
1851 if (strlen(dhpv->dthpv_provname) + 10 >=
1852 sizeof (provider->ftp_name)) {
1853 printf("failed to instantiate provider %s: "
1854 "name too long to accomodate pid", dhpv->dthpv_provname);
1859 * Don't let folks spoof the true pid provider.
1861 if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) {
1862 printf("failed to instantiate provider %s: "
1863 "%s is an invalid name", dhpv->dthpv_provname,
1869 * The highest stability class that fasttrap supports is ISA; cap
1870 * the stability of the new provider accordingly.
1872 if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA)
1873 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA;
1874 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA)
1875 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA;
1876 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA)
1877 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA;
1878 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA)
1879 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA;
1880 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA)
1881 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA;
1883 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname,
1884 &dhpv->dthpv_pattr)) == NULL) {
1885 printf("failed to instantiate provider %s for "
1886 "process %u", dhpv->dthpv_provname, (uint_t)pid);
1891 * Up the meta provider count so this provider isn't removed until
1892 * the meta provider has been told to remove it.
1894 provider->ftp_mcount++;
1896 mutex_exit(&provider->ftp_mtx);
1903 fasttrap_meta_create_probe(void *arg, void *parg,
1904 dtrace_helper_probedesc_t *dhpb)
1906 fasttrap_provider_t *provider = parg;
1907 fasttrap_probe_t *pp;
1908 fasttrap_tracepoint_t *tp;
1913 * Since the meta provider count is non-zero we don't have to worry
1914 * about this provider disappearing.
1916 ASSERT(provider->ftp_mcount > 0);
1919 * The offsets must be unique.
1921 qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t),
1922 fasttrap_uint32_cmp);
1923 for (i = 1; i < dhpb->dthpb_noffs; i++) {
1924 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <=
1925 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1])
1929 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t),
1930 fasttrap_uint32_cmp);
1931 for (i = 1; i < dhpb->dthpb_nenoffs; i++) {
1932 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <=
1933 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1])
1938 * Grab the creation lock to ensure consistency between calls to
1939 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1940 * other threads creating probes.
1942 mutex_enter(&provider->ftp_cmtx);
1944 if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod,
1945 dhpb->dthpb_func, dhpb->dthpb_name) != 0) {
1946 mutex_exit(&provider->ftp_cmtx);
1950 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs;
1953 atomic_add_32(&fasttrap_total, ntps);
1955 if (fasttrap_total > fasttrap_max) {
1956 atomic_add_32(&fasttrap_total, -ntps);
1957 mutex_exit(&provider->ftp_cmtx);
1961 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP);
1963 pp->ftp_prov = provider;
1964 pp->ftp_pid = provider->ftp_pid;
1965 pp->ftp_ntps = ntps;
1966 pp->ftp_nargs = dhpb->dthpb_xargc;
1967 pp->ftp_xtypes = dhpb->dthpb_xtypes;
1968 pp->ftp_ntypes = dhpb->dthpb_ntypes;
1971 * First create a tracepoint for each actual point of interest.
1973 for (i = 0; i < dhpb->dthpb_noffs; i++) {
1974 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1976 tp->ftt_proc = provider->ftp_proc;
1977 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i];
1978 tp->ftt_pid = provider->ftp_pid;
1980 pp->ftp_tps[i].fit_tp = tp;
1981 pp->ftp_tps[i].fit_id.fti_probe = pp;
1983 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS;
1985 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS;
1990 * Then create a tracepoint for each is-enabled point.
1992 for (j = 0; i < ntps; i++, j++) {
1993 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1995 tp->ftt_proc = provider->ftp_proc;
1996 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j];
1997 tp->ftt_pid = provider->ftp_pid;
1999 pp->ftp_tps[i].fit_tp = tp;
2000 pp->ftp_tps[i].fit_id.fti_probe = pp;
2001 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED;
2005 * If the arguments are shuffled around we set the argument remapping
2006 * table. Later, when the probe fires, we only remap the arguments
2007 * if the table is non-NULL.
2009 for (i = 0; i < dhpb->dthpb_xargc; i++) {
2010 if (dhpb->dthpb_args[i] != i) {
2011 pp->ftp_argmap = dhpb->dthpb_args;
2017 * The probe is fully constructed -- register it with DTrace.
2019 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
2020 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp);
2022 mutex_exit(&provider->ftp_cmtx);
2027 fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
2030 * Clean up the USDT provider. There may be active consumers of the
2031 * provider busy adding probes, no damage will actually befall the
2032 * provider until that count has dropped to zero. This just puts
2033 * the provider on death row.
2035 fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1);
2038 static dtrace_mops_t fasttrap_mops = {
2039 fasttrap_meta_create_probe,
2040 fasttrap_meta_provide,
2041 fasttrap_meta_remove
2046 fasttrap_open(struct cdev *dev __unused, int oflags __unused,
2047 int devtype __unused, struct thread *td __unused)
2054 fasttrap_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int fflag,
2058 struct kinfo_proc kp;
2059 const cred_t *cr = td->td_ucred;
2061 if (!dtrace_attached())
2064 if (cmd == FASTTRAPIOC_MAKEPROBE) {
2065 fasttrap_probe_spec_t *uprobe = (void *)arg;
2066 fasttrap_probe_spec_t *probe;
2073 if (copyin(&uprobe->ftps_noffs, &noffs,
2074 sizeof (uprobe->ftps_noffs)))
2077 noffs = uprobe->ftps_noffs;
2081 * Probes must have at least one tracepoint.
2086 size = sizeof (fasttrap_probe_spec_t) +
2087 sizeof (probe->ftps_offs[0]) * (noffs - 1);
2089 if (size > 1024 * 1024)
2092 probe = kmem_alloc(size, KM_SLEEP);
2095 if (copyin(uprobe, probe, size) != 0) {
2096 kmem_free(probe, size);
2100 memcpy(probe, uprobe, sizeof(*probe));
2101 if (noffs > 1 && copyin(uprobe + 1, probe + 1, size) != 0) {
2102 kmem_free(probe, size);
2109 * Verify that the function and module strings contain no
2112 for (c = &probe->ftps_func[0]; *c != '\0'; c++) {
2113 if (*c < 0x20 || 0x7f <= *c) {
2119 for (c = &probe->ftps_mod[0]; *c != '\0'; c++) {
2120 if (*c < 0x20 || 0x7f <= *c) {
2127 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2129 pid_t pid = probe->ftps_pid;
2132 mutex_enter(&pidlock);
2135 * Report an error if the process doesn't exist
2136 * or is actively being birthed.
2140 fill_kinfo_proc(p, &kp);
2141 if (p == NULL || kp.ki_stat == SIDL) {
2143 mutex_exit(&pidlock);
2148 mutex_enter(&p->p_lock);
2149 mutex_exit(&pidlock);
2151 PROC_LOCK_ASSERT(p, MA_OWNED);
2155 if ((ret = priv_proc_cred_perm(cr, p, NULL,
2156 VREAD | VWRITE)) != 0) {
2158 mutex_exit(&p->p_lock);
2166 mutex_exit(&p->p_lock);
2173 ret = fasttrap_add_probe(probe);
2175 kmem_free(probe, size);
2179 } else if (cmd == FASTTRAPIOC_GETINSTR) {
2180 fasttrap_instr_query_t instr;
2181 fasttrap_tracepoint_t *tp;
2188 if (copyin((void *)arg, &instr, sizeof (instr)) != 0)
2193 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2195 pid_t pid = instr.ftiq_pid;
2198 mutex_enter(&pidlock);
2201 * Report an error if the process doesn't exist
2202 * or is actively being birthed.
2206 fill_kinfo_proc(p, &kp);
2207 if (p == NULL || kp.ki_stat == SIDL) {
2209 mutex_exit(&pidlock);
2214 mutex_enter(&p->p_lock);
2215 mutex_exit(&pidlock);
2217 PROC_LOCK_ASSERT(p, MA_OWNED);
2221 if ((ret = priv_proc_cred_perm(cr, p, NULL,
2224 mutex_exit(&p->p_lock);
2233 mutex_exit(&p->p_lock);
2240 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc);
2242 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2243 tp = fasttrap_tpoints.fth_table[index].ftb_data;
2244 while (tp != NULL) {
2245 if (instr.ftiq_pid == tp->ftt_pid &&
2246 instr.ftiq_pc == tp->ftt_pc &&
2247 tp->ftt_proc->ftpc_acount != 0)
2254 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2258 bcopy(&tp->ftt_instr, &instr.ftiq_instr,
2259 sizeof (instr.ftiq_instr));
2260 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2262 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0)
2277 /* Create the /dev/dtrace/fasttrap entry. */
2278 fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600,
2281 mtx_init(&fasttrap_cleanup_mtx, "fasttrap clean", "dtrace", MTX_DEF);
2282 callout_init_mtx(&fasttrap_timeout, &fasttrap_cleanup_mtx, 0);
2283 mutex_init(&fasttrap_count_mtx, "fasttrap count mtx", MUTEX_DEFAULT,
2287 fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2288 "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT);
2290 fasttrap_max = FASTTRAP_MAX_DEFAULT;
2295 * Conjure up the tracepoints hashtable...
2298 nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2299 "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE);
2301 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2304 if (nent == 0 || nent > 0x1000000)
2305 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2307 if ((nent & (nent - 1)) == 0)
2308 fasttrap_tpoints.fth_nent = nent;
2310 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
2311 ASSERT(fasttrap_tpoints.fth_nent > 0);
2312 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1;
2313 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *
2314 sizeof (fasttrap_bucket_t), KM_SLEEP);
2316 for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2317 mutex_init(&fasttrap_tpoints.fth_table[i].ftb_mtx,
2318 "tracepoints bucket mtx", MUTEX_DEFAULT, NULL);
2322 * ... and the providers hash table...
2324 nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE;
2325 if ((nent & (nent - 1)) == 0)
2326 fasttrap_provs.fth_nent = nent;
2328 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
2329 ASSERT(fasttrap_provs.fth_nent > 0);
2330 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1;
2331 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *
2332 sizeof (fasttrap_bucket_t), KM_SLEEP);
2334 for (i = 0; i < fasttrap_provs.fth_nent; i++)
2335 mutex_init(&fasttrap_provs.fth_table[i].ftb_mtx,
2336 "providers bucket mtx", MUTEX_DEFAULT, NULL);
2340 * ... and the procs hash table.
2342 nent = FASTTRAP_PROCS_DEFAULT_SIZE;
2343 if ((nent & (nent - 1)) == 0)
2344 fasttrap_procs.fth_nent = nent;
2346 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
2347 ASSERT(fasttrap_procs.fth_nent > 0);
2348 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1;
2349 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *
2350 sizeof (fasttrap_bucket_t), KM_SLEEP);
2352 for (i = 0; i < fasttrap_procs.fth_nent; i++)
2353 mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx,
2354 "processes bucket mtx", MUTEX_DEFAULT, NULL);
2357 mutex_init(&fasttrap_cpuc_pid_lock[i], "fasttrap barrier",
2358 MUTEX_DEFAULT, NULL);
2363 * Install our hooks into fork(2), exec(2), and exit(2).
2365 dtrace_fasttrap_fork = &fasttrap_fork;
2366 dtrace_fasttrap_exit = &fasttrap_exec_exit;
2367 dtrace_fasttrap_exec = &fasttrap_exec_exit;
2369 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2376 fasttrap_unload(void)
2381 * Unregister the meta-provider to make sure no new fasttrap-
2382 * managed providers come along while we're trying to close up
2383 * shop. If we fail to detach, we'll need to re-register as a
2384 * meta-provider. We can fail to unregister as a meta-provider
2385 * if providers we manage still exist.
2387 if (fasttrap_meta_id != DTRACE_METAPROVNONE &&
2388 dtrace_meta_unregister(fasttrap_meta_id) != 0)
2392 * Prevent any new timeouts from running by setting fasttrap_timeout
2393 * to a non-zero value, and wait for the current timeout to complete.
2395 mtx_lock(&fasttrap_cleanup_mtx);
2396 fasttrap_cleanup_work = 0;
2397 callout_drain(&fasttrap_timeout);
2398 mtx_unlock(&fasttrap_cleanup_mtx);
2401 * Iterate over all of our providers. If there's still a process
2402 * that corresponds to that pid, fail to detach.
2404 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
2405 fasttrap_provider_t **fpp, *fp;
2406 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i];
2408 mutex_enter(&bucket->ftb_mtx);
2409 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
2410 while ((fp = *fpp) != NULL) {
2412 * Acquire and release the lock as a simple way of
2413 * waiting for any other consumer to finish with
2414 * this provider. A thread must first acquire the
2415 * bucket lock so there's no chance of another thread
2416 * blocking on the provider's lock.
2418 mutex_enter(&fp->ftp_mtx);
2419 mutex_exit(&fp->ftp_mtx);
2421 if (dtrace_unregister(fp->ftp_provid) != 0) {
2423 fpp = &fp->ftp_next;
2425 *fpp = fp->ftp_next;
2426 fasttrap_provider_free(fp);
2430 mutex_exit(&bucket->ftb_mtx);
2436 * If we're failing to detach, we need to unblock timeouts
2437 * and start a new timeout if any work has accumulated while
2438 * we've been unsuccessfully trying to detach.
2440 mtx_lock(&fasttrap_cleanup_mtx);
2441 work = fasttrap_cleanup_work;
2442 callout_drain(&fasttrap_timeout);
2443 mtx_unlock(&fasttrap_cleanup_mtx);
2446 fasttrap_pid_cleanup();
2448 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2455 mutex_enter(&fasttrap_count_mtx);
2456 ASSERT(fasttrap_pid_count == 0);
2457 mutex_exit(&fasttrap_count_mtx);
2460 kmem_free(fasttrap_tpoints.fth_table,
2461 fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t));
2462 fasttrap_tpoints.fth_nent = 0;
2464 kmem_free(fasttrap_provs.fth_table,
2465 fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t));
2466 fasttrap_provs.fth_nent = 0;
2468 kmem_free(fasttrap_procs.fth_table,
2469 fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t));
2470 fasttrap_procs.fth_nent = 0;
2473 * We know there are no tracepoints in any process anywhere in
2474 * the system so there is no process which has its p_dtrace_count
2475 * greater than zero, therefore we know that no thread can actively
2476 * be executing code in fasttrap_fork(). Similarly for p_dtrace_probes
2477 * and fasttrap_exec() and fasttrap_exit().
2479 ASSERT(dtrace_fasttrap_fork == &fasttrap_fork);
2480 dtrace_fasttrap_fork = NULL;
2482 ASSERT(dtrace_fasttrap_exec == &fasttrap_exec_exit);
2483 dtrace_fasttrap_exec = NULL;
2485 ASSERT(dtrace_fasttrap_exit == &fasttrap_exec_exit);
2486 dtrace_fasttrap_exit = NULL;
2489 destroy_dev(fasttrap_cdev);
2490 mutex_destroy(&fasttrap_count_mtx);
2492 mutex_destroy(&fasttrap_cpuc_pid_lock[i]);
2501 fasttrap_modevent(module_t mod __unused, int type, void *data __unused)
2522 SYSINIT(fasttrap_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fasttrap_load,
2524 SYSUNINIT(fasttrap_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY,
2525 fasttrap_unload, NULL);
2527 DEV_MODULE(fasttrap, fasttrap_modevent, NULL);
2528 MODULE_VERSION(fasttrap, 1);
2529 MODULE_DEPEND(fasttrap, dtrace, 1, 1, 1);
2530 MODULE_DEPEND(fasttrap, opensolaris, 1, 1, 1);