4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
27 #pragma ident "%Z%%M% %I% %E% SMI"
29 #include <sys/atomic.h>
30 #include <sys/errno.h>
32 #include <sys/modctl.h>
34 #include <sys/systm.h>
36 #include <sys/sunddi.h>
37 #include <sys/cpuvar.h>
39 #include <sys/strsubr.h>
40 #include <sys/fasttrap.h>
41 #include <sys/fasttrap_impl.h>
42 #include <sys/fasttrap_isa.h>
43 #include <sys/dtrace.h>
44 #include <sys/dtrace_impl.h>
45 #include <sys/sysmacros.h>
48 #include <sys/policy.h>
49 #include <util/qsort.h>
52 * User-Land Trap-Based Tracing
53 * ----------------------------
55 * The fasttrap provider allows DTrace consumers to instrument any user-level
56 * instruction to gather data; this includes probes with semantic
57 * signifigance like entry and return as well as simple offsets into the
58 * function. While the specific techniques used are very ISA specific, the
59 * methodology is generalizable to any architecture.
62 * The General Methodology
63 * -----------------------
65 * With the primary goal of tracing every user-land instruction and the
66 * limitation that we can't trust user space so don't want to rely on much
67 * information there, we begin by replacing the instructions we want to trace
68 * with trap instructions. Each instruction we overwrite is saved into a hash
69 * table keyed by process ID and pc address. When we enter the kernel due to
70 * this trap instruction, we need the effects of the replaced instruction to
71 * appear to have occurred before we proceed with the user thread's
74 * Each user level thread is represented by a ulwp_t structure which is
75 * always easily accessible through a register. The most basic way to produce
76 * the effects of the instruction we replaced is to copy that instruction out
77 * to a bit of scratch space reserved in the user thread's ulwp_t structure
78 * (a sort of kernel-private thread local storage), set the PC to that
79 * scratch space and single step. When we reenter the kernel after single
80 * stepping the instruction we must then adjust the PC to point to what would
81 * normally be the next instruction. Of course, special care must be taken
82 * for branches and jumps, but these represent such a small fraction of any
83 * instruction set that writing the code to emulate these in the kernel is
86 * Return probes may require several tracepoints to trace every return site,
87 * and, conversely, each tracepoint may activate several probes (the entry
88 * and offset 0 probes, for example). To solve this muliplexing problem,
89 * tracepoints contain lists of probes to activate and probes contain lists
90 * of tracepoints to enable. If a probe is activated, it adds its ID to
91 * existing tracepoints or creates new ones as necessary.
93 * Most probes are activated _before_ the instruction is executed, but return
94 * probes are activated _after_ the effects of the last instruction of the
95 * function are visible. Return probes must be fired _after_ we have
96 * single-stepped the instruction whereas all other probes are fired
103 * The lock ordering below -- both internally and with respect to the DTrace
104 * framework -- is a little tricky and bears some explanation. Each provider
105 * has a lock (ftp_mtx) that protects its members including reference counts
106 * for enabled probes (ftp_rcount), consumers actively creating probes
107 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider
108 * from being freed. A provider is looked up by taking the bucket lock for the
109 * provider hash table, and is returned with its lock held. The provider lock
110 * may be taken in functions invoked by the DTrace framework, but may not be
111 * held while calling functions in the DTrace framework.
113 * To ensure consistency over multiple calls to the DTrace framework, the
114 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may
115 * not be taken when holding the provider lock as that would create a cyclic
116 * lock ordering. In situations where one would naturally take the provider
117 * lock and then the creation lock, we instead up a reference count to prevent
118 * the provider from disappearing, drop the provider lock, and acquire the
122 * bucket lock before provider lock
123 * DTrace before provider lock
124 * creation lock before DTrace
125 * never hold the provider lock and creation lock simultaneously
128 static dev_info_t *fasttrap_devi;
129 static dtrace_meta_provider_id_t fasttrap_meta_id;
131 static timeout_id_t fasttrap_timeout;
132 static kmutex_t fasttrap_cleanup_mtx;
133 static uint_t fasttrap_cleanup_work;
136 * Generation count on modifications to the global tracepoint lookup table.
138 static volatile uint64_t fasttrap_mod_gen;
141 * When the fasttrap provider is loaded, fasttrap_max is set to either
142 * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the
143 * fasttrap.conf file. Each time a probe is created, fasttrap_total is
144 * incremented by the number of tracepoints that may be associated with that
145 * probe; fasttrap_total is capped at fasttrap_max.
147 #define FASTTRAP_MAX_DEFAULT 250000
148 static uint32_t fasttrap_max;
149 static uint32_t fasttrap_total;
152 #define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000
153 #define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100
154 #define FASTTRAP_PROCS_DEFAULT_SIZE 0x100
156 #define FASTTRAP_PID_NAME "pid"
158 fasttrap_hash_t fasttrap_tpoints;
159 static fasttrap_hash_t fasttrap_provs;
160 static fasttrap_hash_t fasttrap_procs;
162 static uint64_t fasttrap_pid_count; /* pid ref count */
163 static kmutex_t fasttrap_count_mtx; /* lock on ref count */
165 #define FASTTRAP_ENABLE_FAIL 1
166 #define FASTTRAP_ENABLE_PARTIAL 2
168 static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
169 static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
171 static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
172 const dtrace_pattr_t *);
173 static void fasttrap_provider_retire(pid_t, const char *, int);
174 static void fasttrap_provider_free(fasttrap_provider_t *);
176 static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
177 static void fasttrap_proc_release(fasttrap_proc_t *);
179 #define FASTTRAP_PROVS_INDEX(pid, name) \
180 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
182 #define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
185 fasttrap_highbit(ulong_t i)
192 if (i & 0xffffffff00000000ul) {
196 if (i & 0xffff0000) {
215 fasttrap_hash_str(const char *p)
221 hval = (hval << 4) + *p++;
222 if ((g = (hval & 0xf0000000)) != 0)
230 fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
232 sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
234 sqp->sq_info.si_signo = SIGTRAP;
235 sqp->sq_info.si_code = TRAP_DTRACE;
236 sqp->sq_info.si_addr = (caddr_t)pc;
238 mutex_enter(&p->p_lock);
240 mutex_exit(&p->p_lock);
247 * This function ensures that no threads are actively using the memory
248 * associated with probes that were formerly live.
251 fasttrap_mod_barrier(uint64_t gen)
255 if (gen < fasttrap_mod_gen)
260 for (i = 0; i < NCPU; i++) {
261 mutex_enter(&cpu_core[i].cpuc_pid_lock);
262 mutex_exit(&cpu_core[i].cpuc_pid_lock);
267 * This is the timeout's callback for cleaning up the providers and their
272 fasttrap_pid_cleanup_cb(void *data)
274 fasttrap_provider_t **fpp, *fp;
275 fasttrap_bucket_t *bucket;
276 dtrace_provider_id_t provid;
279 static volatile int in = 0;
283 mutex_enter(&fasttrap_cleanup_mtx);
284 while (fasttrap_cleanup_work) {
285 fasttrap_cleanup_work = 0;
286 mutex_exit(&fasttrap_cleanup_mtx);
291 * Iterate over all the providers trying to remove the marked
292 * ones. If a provider is marked but not retired, we just
293 * have to take a crack at removing it -- it's no big deal if
296 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
297 bucket = &fasttrap_provs.fth_table[i];
298 mutex_enter(&bucket->ftb_mtx);
299 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
301 while ((fp = *fpp) != NULL) {
302 if (!fp->ftp_marked) {
307 mutex_enter(&fp->ftp_mtx);
310 * If this provider has consumers actively
311 * creating probes (ftp_ccount) or is a USDT
312 * provider (ftp_mcount), we can't unregister
315 if (fp->ftp_ccount != 0 ||
316 fp->ftp_mcount != 0) {
317 mutex_exit(&fp->ftp_mtx);
322 if (!fp->ftp_retired || fp->ftp_rcount != 0)
325 mutex_exit(&fp->ftp_mtx);
328 * If we successfully unregister this
329 * provider we can remove it from the hash
330 * chain and free the memory. If our attempt
331 * to unregister fails and this is a retired
332 * provider, increment our flag to try again
333 * pretty soon. If we've consumed more than
334 * half of our total permitted number of
335 * probes call dtrace_condense() to try to
336 * clean out the unenabled probes.
338 provid = fp->ftp_provid;
339 if (dtrace_unregister(provid) != 0) {
340 if (fasttrap_total > fasttrap_max / 2)
341 (void) dtrace_condense(provid);
342 later += fp->ftp_marked;
346 fasttrap_provider_free(fp);
349 mutex_exit(&bucket->ftb_mtx);
352 mutex_enter(&fasttrap_cleanup_mtx);
355 ASSERT(fasttrap_timeout != 0);
358 * If we were unable to remove a retired provider, try again after
359 * a second. This situation can occur in certain circumstances where
360 * providers cannot be unregistered even though they have no probes
361 * enabled because of an execution of dtrace -l or something similar.
362 * If the timeout has been disabled (set to 1 because we're trying
363 * to detach), we set fasttrap_cleanup_work to ensure that we'll
364 * get a chance to do that work if and when the timeout is reenabled
367 if (later > 0 && fasttrap_timeout != (timeout_id_t)1)
368 fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, hz);
370 fasttrap_cleanup_work = 1;
372 fasttrap_timeout = 0;
374 mutex_exit(&fasttrap_cleanup_mtx);
379 * Activates the asynchronous cleanup mechanism.
382 fasttrap_pid_cleanup(void)
384 mutex_enter(&fasttrap_cleanup_mtx);
385 fasttrap_cleanup_work = 1;
386 if (fasttrap_timeout == 0)
387 fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, 1);
388 mutex_exit(&fasttrap_cleanup_mtx);
392 * This is called from cfork() via dtrace_fasttrap_fork(). The child
393 * process's address space is (roughly) a copy of the parent process's so
394 * we have to remove all the instrumentation we had previously enabled in the
398 fasttrap_fork(proc_t *p, proc_t *cp)
400 pid_t ppid = p->p_pid;
403 ASSERT(curproc == p);
404 ASSERT(p->p_proc_flag & P_PR_LOCK);
405 ASSERT(p->p_dtrace_count > 0);
406 ASSERT(cp->p_dtrace_count == 0);
409 * This would be simpler and faster if we maintained per-process
410 * hash tables of enabled tracepoints. It could, however, potentially
411 * slow down execution of a tracepoint since we'd need to go
412 * through two levels of indirection. In the future, we should
413 * consider either maintaining per-process ancillary lists of
414 * enabled tracepoints or hanging a pointer to a per-process hash
415 * table of enabled tracepoints off the proc structure.
419 * We don't have to worry about the child process disappearing
420 * because we're in fork().
422 mutex_enter(&cp->p_lock);
424 mutex_exit(&cp->p_lock);
427 * Iterate over every tracepoint looking for ones that belong to the
428 * parent process, and remove each from the child process.
430 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
431 fasttrap_tracepoint_t *tp;
432 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
434 mutex_enter(&bucket->ftb_mtx);
435 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
436 if (tp->ftt_pid == ppid &&
437 tp->ftt_proc->ftpc_acount != 0) {
438 int ret = fasttrap_tracepoint_remove(cp, tp);
442 * The count of active providers can only be
443 * decremented (i.e. to zero) during exec,
444 * exit, and removal of a meta provider so it
445 * should be impossible to drop the count
448 ASSERT(tp->ftt_proc->ftpc_acount != 0);
451 mutex_exit(&bucket->ftb_mtx);
454 mutex_enter(&cp->p_lock);
459 * This is called from proc_exit() or from exec_common() if p_dtrace_probes
460 * is set on the proc structure to indicate that there is a pid provider
461 * associated with this process.
464 fasttrap_exec_exit(proc_t *p)
466 ASSERT(p == curproc);
467 ASSERT(MUTEX_HELD(&p->p_lock));
469 mutex_exit(&p->p_lock);
472 * We clean up the pid provider for this process here; user-land
473 * static probes are handled by the meta-provider remove entry point.
475 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
477 mutex_enter(&p->p_lock);
483 fasttrap_pid_provide(void *arg, const dtrace_probedesc_t *desc)
486 * There are no "default" pid probes.
491 fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
493 fasttrap_tracepoint_t *tp, *new_tp = NULL;
494 fasttrap_bucket_t *bucket;
499 ASSERT(index < probe->ftp_ntps);
501 pid = probe->ftp_pid;
502 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
503 id = &probe->ftp_tps[index].fit_id;
505 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
507 ASSERT(!(p->p_flag & SVFORK));
510 * Before we make any modifications, make sure we've imposed a barrier
511 * on the generation in which this probe was last modified.
513 fasttrap_mod_barrier(probe->ftp_gen);
515 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
518 * If the tracepoint has already been enabled, just add our id to the
519 * list of interested probes. This may be our second time through
520 * this path in which case we'll have constructed the tracepoint we'd
521 * like to install. If we can't find a match, and have an allocated
522 * tracepoint ready to go, enable that one now.
524 * A tracepoint whose process is defunct is also considered defunct.
527 mutex_enter(&bucket->ftb_mtx);
528 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
530 * Note that it's safe to access the active count on the
531 * associated proc structure because we know that at least one
532 * provider (this one) will still be around throughout this
535 if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
536 tp->ftt_proc->ftpc_acount == 0)
540 * Now that we've found a matching tracepoint, it would be
541 * a decent idea to confirm that the tracepoint is still
542 * enabled and the trap instruction hasn't been overwritten.
543 * Since this is a little hairy, we'll punt for now.
547 * This can't be the first interested probe. We don't have
548 * to worry about another thread being in the midst of
549 * deleting this tracepoint (which would be the only valid
550 * reason for a tracepoint to have no interested probes)
551 * since we're holding P_PR_LOCK for this process.
553 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
555 switch (id->fti_ptype) {
558 case DTFTP_IS_ENABLED:
559 id->fti_next = tp->ftt_ids;
566 case DTFTP_POST_OFFSETS:
567 id->fti_next = tp->ftt_retids;
577 mutex_exit(&bucket->ftb_mtx);
579 if (new_tp != NULL) {
580 new_tp->ftt_ids = NULL;
581 new_tp->ftt_retids = NULL;
588 * If we have a good tracepoint ready to go, install it now while
589 * we have the lock held and no one can screw with us.
591 if (new_tp != NULL) {
594 new_tp->ftt_next = bucket->ftb_data;
596 bucket->ftb_data = new_tp;
598 mutex_exit(&bucket->ftb_mtx);
601 * Activate the tracepoint in the ISA-specific manner.
602 * If this fails, we need to report the failure, but
603 * indicate that this tracepoint must still be disabled
604 * by calling fasttrap_tracepoint_disable().
606 if (fasttrap_tracepoint_install(p, new_tp) != 0)
607 rc = FASTTRAP_ENABLE_PARTIAL;
610 * Increment the count of the number of tracepoints active in
611 * the victim process.
613 ASSERT(p->p_proc_flag & P_PR_LOCK);
619 mutex_exit(&bucket->ftb_mtx);
622 * Initialize the tracepoint that's been preallocated with the probe.
624 new_tp = probe->ftp_tps[index].fit_tp;
626 ASSERT(new_tp->ftt_pid == pid);
627 ASSERT(new_tp->ftt_pc == pc);
628 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
629 ASSERT(new_tp->ftt_ids == NULL);
630 ASSERT(new_tp->ftt_retids == NULL);
632 switch (id->fti_ptype) {
635 case DTFTP_IS_ENABLED:
637 new_tp->ftt_ids = id;
641 case DTFTP_POST_OFFSETS:
643 new_tp->ftt_retids = id;
651 * If the ISA-dependent initialization goes to plan, go back to the
652 * beginning and try to install this freshly made tracepoint.
654 if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
657 new_tp->ftt_ids = NULL;
658 new_tp->ftt_retids = NULL;
660 return (FASTTRAP_ENABLE_FAIL);
664 fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
666 fasttrap_bucket_t *bucket;
667 fasttrap_provider_t *provider = probe->ftp_prov;
668 fasttrap_tracepoint_t **pp, *tp;
669 fasttrap_id_t *id, **idp;
673 ASSERT(index < probe->ftp_ntps);
675 pid = probe->ftp_pid;
676 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
677 id = &probe->ftp_tps[index].fit_id;
679 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
682 * Find the tracepoint and make sure that our id is one of the
683 * ones registered with it.
685 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
686 mutex_enter(&bucket->ftb_mtx);
687 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
688 if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
689 tp->ftt_proc == provider->ftp_proc)
694 * If we somehow lost this tracepoint, we're in a world of hurt.
698 switch (id->fti_ptype) {
701 case DTFTP_IS_ENABLED:
702 ASSERT(tp->ftt_ids != NULL);
707 case DTFTP_POST_OFFSETS:
708 ASSERT(tp->ftt_retids != NULL);
709 idp = &tp->ftt_retids;
716 while ((*idp)->fti_probe != probe) {
717 idp = &(*idp)->fti_next;
718 ASSERT(*idp != NULL);
725 ASSERT(id->fti_probe == probe);
728 * If there are other registered enablings of this tracepoint, we're
729 * all done, but if this was the last probe assocated with this
730 * this tracepoint, we need to remove and free it.
732 if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
735 * If the current probe's tracepoint is in use, swap it
736 * for an unused tracepoint.
738 if (tp == probe->ftp_tps[index].fit_tp) {
739 fasttrap_probe_t *tmp_probe;
740 fasttrap_tracepoint_t **tmp_tp;
743 if (tp->ftt_ids != NULL) {
744 tmp_probe = tp->ftt_ids->fti_probe;
745 /* LINTED - alignment */
746 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
747 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
749 tmp_probe = tp->ftt_retids->fti_probe;
750 /* LINTED - alignment */
751 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
752 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
755 ASSERT(*tmp_tp != NULL);
756 ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
757 ASSERT((*tmp_tp)->ftt_ids == NULL);
758 ASSERT((*tmp_tp)->ftt_retids == NULL);
760 probe->ftp_tps[index].fit_tp = *tmp_tp;
764 mutex_exit(&bucket->ftb_mtx);
767 * Tag the modified probe with the generation in which it was
770 probe->ftp_gen = fasttrap_mod_gen;
774 mutex_exit(&bucket->ftb_mtx);
777 * We can't safely remove the tracepoint from the set of active
778 * tracepoints until we've actually removed the fasttrap instruction
779 * from the process's text. We can, however, operate on this
780 * tracepoint secure in the knowledge that no other thread is going to
781 * be looking at it since we hold P_PR_LOCK on the process if it's
782 * live or we hold the provider lock on the process if it's dead and
787 * We only need to remove the actual instruction if we're looking
788 * at an existing process
792 * If we fail to restore the instruction we need to kill
793 * this process since it's in a completely unrecoverable
796 if (fasttrap_tracepoint_remove(p, tp) != 0)
797 fasttrap_sigtrap(p, NULL, pc);
800 * Decrement the count of the number of tracepoints active
801 * in the victim process.
803 ASSERT(p->p_proc_flag & P_PR_LOCK);
808 * Remove the probe from the hash table of active tracepoints.
810 mutex_enter(&bucket->ftb_mtx);
811 pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
814 pp = &(*pp)->ftt_next;
821 mutex_exit(&bucket->ftb_mtx);
824 * Tag the modified probe with the generation in which it was changed.
826 probe->ftp_gen = fasttrap_mod_gen;
830 fasttrap_enable_callbacks(void)
833 * We don't have to play the rw lock game here because we're
834 * providing something rather than taking something away --
835 * we can be sure that no threads have tried to follow this
836 * function pointer yet.
838 mutex_enter(&fasttrap_count_mtx);
839 if (fasttrap_pid_count == 0) {
840 ASSERT(dtrace_pid_probe_ptr == NULL);
841 ASSERT(dtrace_return_probe_ptr == NULL);
842 dtrace_pid_probe_ptr = &fasttrap_pid_probe;
843 dtrace_return_probe_ptr = &fasttrap_return_probe;
845 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe);
846 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe);
847 fasttrap_pid_count++;
848 mutex_exit(&fasttrap_count_mtx);
852 fasttrap_disable_callbacks(void)
854 ASSERT(MUTEX_HELD(&cpu_lock));
856 mutex_enter(&fasttrap_count_mtx);
857 ASSERT(fasttrap_pid_count > 0);
858 fasttrap_pid_count--;
859 if (fasttrap_pid_count == 0) {
860 cpu_t *cur, *cpu = CPU;
862 for (cur = cpu->cpu_next_onln; cur != cpu;
863 cur = cur->cpu_next_onln) {
864 rw_enter(&cur->cpu_ft_lock, RW_WRITER);
867 dtrace_pid_probe_ptr = NULL;
868 dtrace_return_probe_ptr = NULL;
870 for (cur = cpu->cpu_next_onln; cur != cpu;
871 cur = cur->cpu_next_onln) {
872 rw_exit(&cur->cpu_ft_lock);
875 mutex_exit(&fasttrap_count_mtx);
880 fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
882 fasttrap_probe_t *probe = parg;
886 ASSERT(probe != NULL);
887 ASSERT(!probe->ftp_enabled);
888 ASSERT(id == probe->ftp_id);
889 ASSERT(MUTEX_HELD(&cpu_lock));
892 * Increment the count of enabled probes on this probe's provider;
893 * the provider can't go away while the probe still exists. We
894 * must increment this even if we aren't able to properly enable
897 mutex_enter(&probe->ftp_prov->ftp_mtx);
898 probe->ftp_prov->ftp_rcount++;
899 mutex_exit(&probe->ftp_prov->ftp_mtx);
902 * If this probe's provider is retired (meaning it was valid in a
903 * previously exec'ed incarnation of this address space), bail out. The
904 * provider can't go away while we're in this code path.
906 if (probe->ftp_prov->ftp_retired)
910 * If we can't find the process, it may be that we're in the context of
911 * a fork in which the traced process is being born and we're copying
912 * USDT probes. Otherwise, the process is gone so bail.
914 if ((p = sprlock(probe->ftp_pid)) == NULL) {
915 if ((curproc->p_flag & SFORKING) == 0)
918 mutex_enter(&pidlock);
919 p = prfind(probe->ftp_pid);
922 * Confirm that curproc is indeed forking the process in which
923 * we're trying to enable probes.
926 ASSERT(p->p_parent == curproc);
927 ASSERT(p->p_stat == SIDL);
929 mutex_enter(&p->p_lock);
930 mutex_exit(&pidlock);
935 ASSERT(!(p->p_flag & SVFORK));
936 mutex_exit(&p->p_lock);
939 * We have to enable the trap entry point before any user threads have
940 * the chance to execute the trap instruction we're about to place
941 * in their process's text.
943 fasttrap_enable_callbacks();
946 * Enable all the tracepoints and add this probe's id to each
947 * tracepoint's list of active probes.
949 for (i = 0; i < probe->ftp_ntps; i++) {
950 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
952 * If enabling the tracepoint failed completely,
953 * we don't have to disable it; if the failure
954 * was only partial we must disable it.
956 if (rc == FASTTRAP_ENABLE_FAIL)
959 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
962 * Back up and pull out all the tracepoints we've
963 * created so far for this probe.
966 fasttrap_tracepoint_disable(p, probe, i);
970 mutex_enter(&p->p_lock);
974 * Since we're not actually enabling this probe,
975 * drop our reference on the trap table entry.
977 fasttrap_disable_callbacks();
982 mutex_enter(&p->p_lock);
985 probe->ftp_enabled = 1;
990 fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
992 fasttrap_probe_t *probe = parg;
993 fasttrap_provider_t *provider = probe->ftp_prov;
997 ASSERT(id == probe->ftp_id);
1000 * We won't be able to acquire a /proc-esque lock on the process
1001 * iff the process is dead and gone. In this case, we rely on the
1002 * provider lock as a point of mutual exclusion to prevent other
1003 * DTrace consumers from disabling this probe.
1005 if ((p = sprlock(probe->ftp_pid)) != NULL) {
1006 ASSERT(!(p->p_flag & SVFORK));
1007 mutex_exit(&p->p_lock);
1010 mutex_enter(&provider->ftp_mtx);
1013 * Disable all the associated tracepoints (for fully enabled probes).
1015 if (probe->ftp_enabled) {
1016 for (i = 0; i < probe->ftp_ntps; i++) {
1017 fasttrap_tracepoint_disable(p, probe, i);
1021 ASSERT(provider->ftp_rcount > 0);
1022 provider->ftp_rcount--;
1026 * Even though we may not be able to remove it entirely, we
1027 * mark this retired provider to get a chance to remove some
1028 * of the associated probes.
1030 if (provider->ftp_retired && !provider->ftp_marked)
1031 whack = provider->ftp_marked = 1;
1032 mutex_exit(&provider->ftp_mtx);
1034 mutex_enter(&p->p_lock);
1038 * If the process is dead, we're just waiting for the
1039 * last probe to be disabled to be able to free it.
1041 if (provider->ftp_rcount == 0 && !provider->ftp_marked)
1042 whack = provider->ftp_marked = 1;
1043 mutex_exit(&provider->ftp_mtx);
1047 fasttrap_pid_cleanup();
1049 if (!probe->ftp_enabled)
1052 probe->ftp_enabled = 0;
1054 ASSERT(MUTEX_HELD(&cpu_lock));
1055 fasttrap_disable_callbacks();
1060 fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
1061 dtrace_argdesc_t *desc)
1063 fasttrap_probe_t *probe = parg;
1067 desc->dtargd_native[0] = '\0';
1068 desc->dtargd_xlate[0] = '\0';
1070 if (probe->ftp_prov->ftp_retired != 0 ||
1071 desc->dtargd_ndx >= probe->ftp_nargs) {
1072 desc->dtargd_ndx = DTRACE_ARGNONE;
1076 ndx = (probe->ftp_argmap != NULL) ?
1077 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx;
1079 str = probe->ftp_ntypes;
1080 for (i = 0; i < ndx; i++) {
1081 str += strlen(str) + 1;
1084 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native));
1085 (void) strcpy(desc->dtargd_native, str);
1087 if (probe->ftp_xtypes == NULL)
1090 str = probe->ftp_xtypes;
1091 for (i = 0; i < desc->dtargd_ndx; i++) {
1092 str += strlen(str) + 1;
1095 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate));
1096 (void) strcpy(desc->dtargd_xlate, str);
1101 fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
1103 fasttrap_probe_t *probe = parg;
1107 ASSERT(probe != NULL);
1108 ASSERT(!probe->ftp_enabled);
1109 ASSERT(fasttrap_total >= probe->ftp_ntps);
1111 atomic_add_32(&fasttrap_total, -probe->ftp_ntps);
1112 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
1114 if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
1115 fasttrap_mod_barrier(probe->ftp_gen);
1117 for (i = 0; i < probe->ftp_ntps; i++) {
1118 kmem_free(probe->ftp_tps[i].fit_tp,
1119 sizeof (fasttrap_tracepoint_t));
1122 kmem_free(probe, size);
1126 static const dtrace_pattr_t pid_attr = {
1127 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1128 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1129 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1130 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1131 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1134 static dtrace_pops_t pid_pops = {
1135 fasttrap_pid_provide,
1137 fasttrap_pid_enable,
1138 fasttrap_pid_disable,
1141 fasttrap_pid_getargdesc,
1142 fasttrap_pid_getarg,
1144 fasttrap_pid_destroy
1147 static dtrace_pops_t usdt_pops = {
1148 fasttrap_pid_provide,
1150 fasttrap_pid_enable,
1151 fasttrap_pid_disable,
1154 fasttrap_pid_getargdesc,
1155 fasttrap_usdt_getarg,
1157 fasttrap_pid_destroy
1160 static fasttrap_proc_t *
1161 fasttrap_proc_lookup(pid_t pid)
1163 fasttrap_bucket_t *bucket;
1164 fasttrap_proc_t *fprc, *new_fprc;
1166 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1167 mutex_enter(&bucket->ftb_mtx);
1169 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1170 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1171 mutex_enter(&fprc->ftpc_mtx);
1172 mutex_exit(&bucket->ftb_mtx);
1173 fprc->ftpc_rcount++;
1174 atomic_add_64(&fprc->ftpc_acount, 1);
1175 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1176 mutex_exit(&fprc->ftpc_mtx);
1183 * Drop the bucket lock so we don't try to perform a sleeping
1184 * allocation under it.
1186 mutex_exit(&bucket->ftb_mtx);
1188 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP);
1189 new_fprc->ftpc_pid = pid;
1190 new_fprc->ftpc_rcount = 1;
1191 new_fprc->ftpc_acount = 1;
1193 mutex_enter(&bucket->ftb_mtx);
1196 * Take another lap through the list to make sure a proc hasn't
1197 * been created for this pid while we weren't under the bucket lock.
1199 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1200 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1201 mutex_enter(&fprc->ftpc_mtx);
1202 mutex_exit(&bucket->ftb_mtx);
1203 fprc->ftpc_rcount++;
1204 atomic_add_64(&fprc->ftpc_acount, 1);
1205 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1206 mutex_exit(&fprc->ftpc_mtx);
1208 kmem_free(new_fprc, sizeof (fasttrap_proc_t));
1214 new_fprc->ftpc_next = bucket->ftb_data;
1215 bucket->ftb_data = new_fprc;
1217 mutex_exit(&bucket->ftb_mtx);
1223 fasttrap_proc_release(fasttrap_proc_t *proc)
1225 fasttrap_bucket_t *bucket;
1226 fasttrap_proc_t *fprc, **fprcp;
1227 pid_t pid = proc->ftpc_pid;
1229 mutex_enter(&proc->ftpc_mtx);
1231 ASSERT(proc->ftpc_rcount != 0);
1232 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount);
1234 if (--proc->ftpc_rcount != 0) {
1235 mutex_exit(&proc->ftpc_mtx);
1239 mutex_exit(&proc->ftpc_mtx);
1242 * There should definitely be no live providers associated with this
1243 * process at this point.
1245 ASSERT(proc->ftpc_acount == 0);
1247 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1248 mutex_enter(&bucket->ftb_mtx);
1250 fprcp = (fasttrap_proc_t **)&bucket->ftb_data;
1251 while ((fprc = *fprcp) != NULL) {
1255 fprcp = &fprc->ftpc_next;
1259 * Something strange has happened if we can't find the proc.
1261 ASSERT(fprc != NULL);
1263 *fprcp = fprc->ftpc_next;
1265 mutex_exit(&bucket->ftb_mtx);
1267 kmem_free(fprc, sizeof (fasttrap_proc_t));
1271 * Lookup a fasttrap-managed provider based on its name and associated pid.
1272 * If the pattr argument is non-NULL, this function instantiates the provider
1273 * if it doesn't exist otherwise it returns NULL. The provider is returned
1274 * with its lock held.
1276 static fasttrap_provider_t *
1277 fasttrap_provider_lookup(pid_t pid, const char *name,
1278 const dtrace_pattr_t *pattr)
1280 fasttrap_provider_t *fp, *new_fp = NULL;
1281 fasttrap_bucket_t *bucket;
1282 char provname[DTRACE_PROVNAMELEN];
1286 ASSERT(strlen(name) < sizeof (fp->ftp_name));
1287 ASSERT(pattr != NULL);
1289 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1290 mutex_enter(&bucket->ftb_mtx);
1293 * Take a lap through the list and return the match if we find it.
1295 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1296 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1298 mutex_enter(&fp->ftp_mtx);
1299 mutex_exit(&bucket->ftb_mtx);
1305 * Drop the bucket lock so we don't try to perform a sleeping
1306 * allocation under it.
1308 mutex_exit(&bucket->ftb_mtx);
1311 * Make sure the process exists, isn't a child created as the result
1312 * of a vfork(2), and isn't a zombie (but may be in fork).
1314 mutex_enter(&pidlock);
1315 if ((p = prfind(pid)) == NULL) {
1316 mutex_exit(&pidlock);
1319 mutex_enter(&p->p_lock);
1320 mutex_exit(&pidlock);
1321 if (p->p_flag & (SVFORK | SEXITING)) {
1322 mutex_exit(&p->p_lock);
1327 * Increment p_dtrace_probes so that the process knows to inform us
1328 * when it exits or execs. fasttrap_provider_free() decrements this
1329 * when we're done with this provider.
1331 p->p_dtrace_probes++;
1334 * Grab the credentials for this process so we have
1335 * something to pass to dtrace_register().
1337 mutex_enter(&p->p_crlock);
1340 mutex_exit(&p->p_crlock);
1341 mutex_exit(&p->p_lock);
1343 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP);
1344 new_fp->ftp_pid = pid;
1345 new_fp->ftp_proc = fasttrap_proc_lookup(pid);
1347 ASSERT(new_fp->ftp_proc != NULL);
1349 mutex_enter(&bucket->ftb_mtx);
1352 * Take another lap through the list to make sure a provider hasn't
1353 * been created for this pid while we weren't under the bucket lock.
1355 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1356 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1358 mutex_enter(&fp->ftp_mtx);
1359 mutex_exit(&bucket->ftb_mtx);
1360 fasttrap_provider_free(new_fp);
1366 (void) strcpy(new_fp->ftp_name, name);
1369 * Fail and return NULL if either the provider name is too long
1370 * or we fail to register this new provider with the DTrace
1371 * framework. Note that this is the only place we ever construct
1372 * the full provider name -- we keep it in pieces in the provider
1375 if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >=
1376 sizeof (provname) ||
1377 dtrace_register(provname, pattr,
1378 DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred,
1379 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp,
1380 &new_fp->ftp_provid) != 0) {
1381 mutex_exit(&bucket->ftb_mtx);
1382 fasttrap_provider_free(new_fp);
1387 new_fp->ftp_next = bucket->ftb_data;
1388 bucket->ftb_data = new_fp;
1390 mutex_enter(&new_fp->ftp_mtx);
1391 mutex_exit(&bucket->ftb_mtx);
1398 fasttrap_provider_free(fasttrap_provider_t *provider)
1400 pid_t pid = provider->ftp_pid;
1404 * There need to be no associated enabled probes, no consumers
1405 * creating probes, and no meta providers referencing this provider.
1407 ASSERT(provider->ftp_rcount == 0);
1408 ASSERT(provider->ftp_ccount == 0);
1409 ASSERT(provider->ftp_mcount == 0);
1412 * If this provider hasn't been retired, we need to explicitly drop the
1413 * count of active providers on the associated process structure.
1415 if (!provider->ftp_retired) {
1416 atomic_add_64(&provider->ftp_proc->ftpc_acount, -1);
1417 ASSERT(provider->ftp_proc->ftpc_acount <
1418 provider->ftp_proc->ftpc_rcount);
1421 fasttrap_proc_release(provider->ftp_proc);
1423 kmem_free(provider, sizeof (fasttrap_provider_t));
1426 * Decrement p_dtrace_probes on the process whose provider we're
1427 * freeing. We don't have to worry about clobbering somone else's
1428 * modifications to it because we have locked the bucket that
1429 * corresponds to this process's hash chain in the provider hash
1430 * table. Don't sweat it if we can't find the process.
1432 mutex_enter(&pidlock);
1433 if ((p = prfind(pid)) == NULL) {
1434 mutex_exit(&pidlock);
1438 mutex_enter(&p->p_lock);
1439 mutex_exit(&pidlock);
1441 p->p_dtrace_probes--;
1442 mutex_exit(&p->p_lock);
1446 fasttrap_provider_retire(pid_t pid, const char *name, int mprov)
1448 fasttrap_provider_t *fp;
1449 fasttrap_bucket_t *bucket;
1450 dtrace_provider_id_t provid;
1452 ASSERT(strlen(name) < sizeof (fp->ftp_name));
1454 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1455 mutex_enter(&bucket->ftb_mtx);
1457 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1458 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1464 mutex_exit(&bucket->ftb_mtx);
1468 mutex_enter(&fp->ftp_mtx);
1469 ASSERT(!mprov || fp->ftp_mcount > 0);
1470 if (mprov && --fp->ftp_mcount != 0) {
1471 mutex_exit(&fp->ftp_mtx);
1472 mutex_exit(&bucket->ftb_mtx);
1477 * Mark the provider to be removed in our post-processing step, mark it
1478 * retired, and drop the active count on its proc. Marking it indicates
1479 * that we should try to remove it; setting the retired flag indicates
1480 * that we're done with this provider; dropping the active the proc
1481 * releases our hold, and when this reaches zero (as it will during
1482 * exit or exec) the proc and associated providers become defunct.
1484 * We obviously need to take the bucket lock before the provider lock
1485 * to perform the lookup, but we need to drop the provider lock
1486 * before calling into the DTrace framework since we acquire the
1487 * provider lock in callbacks invoked from the DTrace framework. The
1488 * bucket lock therefore protects the integrity of the provider hash
1491 atomic_add_64(&fp->ftp_proc->ftpc_acount, -1);
1492 ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount);
1494 fp->ftp_retired = 1;
1496 provid = fp->ftp_provid;
1497 mutex_exit(&fp->ftp_mtx);
1500 * We don't have to worry about invalidating the same provider twice
1501 * since fasttrap_provider_lookup() will ignore provider that have
1502 * been marked as retired.
1504 dtrace_invalidate(provid);
1506 mutex_exit(&bucket->ftb_mtx);
1508 fasttrap_pid_cleanup();
1512 fasttrap_uint32_cmp(const void *ap, const void *bp)
1514 return (*(const uint32_t *)ap - *(const uint32_t *)bp);
1518 fasttrap_uint64_cmp(const void *ap, const void *bp)
1520 return (*(const uint64_t *)ap - *(const uint64_t *)bp);
1524 fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
1526 fasttrap_provider_t *provider;
1527 fasttrap_probe_t *pp;
1528 fasttrap_tracepoint_t *tp;
1530 int i, aframes, whack;
1533 * There needs to be at least one desired trace point.
1535 if (pdata->ftps_noffs == 0)
1538 switch (pdata->ftps_type) {
1541 aframes = FASTTRAP_ENTRY_AFRAMES;
1545 aframes = FASTTRAP_RETURN_AFRAMES;
1554 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid,
1555 FASTTRAP_PID_NAME, &pid_attr)) == NULL)
1559 * Increment this reference count to indicate that a consumer is
1560 * actively adding a new probe associated with this provider. This
1561 * prevents the provider from being deleted -- we'll need to check
1562 * for pending deletions when we drop this reference count.
1564 provider->ftp_ccount++;
1565 mutex_exit(&provider->ftp_mtx);
1568 * Grab the creation lock to ensure consistency between calls to
1569 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1570 * other threads creating probes. We must drop the provider lock
1571 * before taking this lock to avoid a three-way deadlock with the
1574 mutex_enter(&provider->ftp_cmtx);
1577 for (i = 0; i < pdata->ftps_noffs; i++) {
1580 (void) sprintf(name_str, "%llx",
1581 (unsigned long long)pdata->ftps_offs[i]);
1583 if (dtrace_probe_lookup(provider->ftp_provid,
1584 pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
1587 atomic_add_32(&fasttrap_total, 1);
1589 if (fasttrap_total > fasttrap_max) {
1590 atomic_add_32(&fasttrap_total, -1);
1594 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP);
1596 pp->ftp_prov = provider;
1597 pp->ftp_faddr = pdata->ftps_pc;
1598 pp->ftp_fsize = pdata->ftps_size;
1599 pp->ftp_pid = pdata->ftps_pid;
1602 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1605 tp->ftt_proc = provider->ftp_proc;
1606 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1607 tp->ftt_pid = pdata->ftps_pid;
1609 pp->ftp_tps[0].fit_tp = tp;
1610 pp->ftp_tps[0].fit_id.fti_probe = pp;
1611 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type;
1613 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1614 pdata->ftps_mod, pdata->ftps_func, name_str,
1615 FASTTRAP_OFFSET_AFRAMES, pp);
1618 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod,
1619 pdata->ftps_func, name) == 0) {
1620 atomic_add_32(&fasttrap_total, pdata->ftps_noffs);
1622 if (fasttrap_total > fasttrap_max) {
1623 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1628 * Make sure all tracepoint program counter values are unique.
1629 * We later assume that each probe has exactly one tracepoint
1632 qsort(pdata->ftps_offs, pdata->ftps_noffs,
1633 sizeof (uint64_t), fasttrap_uint64_cmp);
1634 for (i = 1; i < pdata->ftps_noffs; i++) {
1635 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1])
1638 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1642 ASSERT(pdata->ftps_noffs > 0);
1643 pp = kmem_zalloc(offsetof(fasttrap_probe_t,
1644 ftp_tps[pdata->ftps_noffs]), KM_SLEEP);
1646 pp->ftp_prov = provider;
1647 pp->ftp_faddr = pdata->ftps_pc;
1648 pp->ftp_fsize = pdata->ftps_size;
1649 pp->ftp_pid = pdata->ftps_pid;
1650 pp->ftp_ntps = pdata->ftps_noffs;
1652 for (i = 0; i < pdata->ftps_noffs; i++) {
1653 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1656 tp->ftt_proc = provider->ftp_proc;
1657 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1658 tp->ftt_pid = pdata->ftps_pid;
1660 pp->ftp_tps[i].fit_tp = tp;
1661 pp->ftp_tps[i].fit_id.fti_probe = pp;
1662 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type;
1665 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1666 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp);
1669 mutex_exit(&provider->ftp_cmtx);
1672 * We know that the provider is still valid since we incremented the
1673 * creation reference count. If someone tried to clean up this provider
1674 * while we were using it (e.g. because the process called exec(2) or
1675 * exit(2)), take note of that and try to clean it up now.
1677 mutex_enter(&provider->ftp_mtx);
1678 provider->ftp_ccount--;
1679 whack = provider->ftp_retired;
1680 mutex_exit(&provider->ftp_mtx);
1683 fasttrap_pid_cleanup();
1689 * If we've exhausted the allowable resources, we'll try to remove
1690 * this provider to free some up. This is to cover the case where
1691 * the user has accidentally created many more probes than was
1692 * intended (e.g. pid123:::).
1694 mutex_exit(&provider->ftp_cmtx);
1695 mutex_enter(&provider->ftp_mtx);
1696 provider->ftp_ccount--;
1697 provider->ftp_marked = 1;
1698 mutex_exit(&provider->ftp_mtx);
1700 fasttrap_pid_cleanup();
1707 fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1709 fasttrap_provider_t *provider;
1712 * A 32-bit unsigned integer (like a pid for example) can be
1713 * expressed in 10 or fewer decimal digits. Make sure that we'll
1714 * have enough space for the provider name.
1716 if (strlen(dhpv->dthpv_provname) + 10 >=
1717 sizeof (provider->ftp_name)) {
1718 cmn_err(CE_WARN, "failed to instantiate provider %s: "
1719 "name too long to accomodate pid", dhpv->dthpv_provname);
1724 * Don't let folks spoof the true pid provider.
1726 if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) {
1727 cmn_err(CE_WARN, "failed to instantiate provider %s: "
1728 "%s is an invalid name", dhpv->dthpv_provname,
1734 * The highest stability class that fasttrap supports is ISA; cap
1735 * the stability of the new provider accordingly.
1737 if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA)
1738 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA;
1739 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA)
1740 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA;
1741 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA)
1742 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA;
1743 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA)
1744 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA;
1745 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA)
1746 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA;
1748 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname,
1749 &dhpv->dthpv_pattr)) == NULL) {
1750 cmn_err(CE_WARN, "failed to instantiate provider %s for "
1751 "process %u", dhpv->dthpv_provname, (uint_t)pid);
1756 * Up the meta provider count so this provider isn't removed until
1757 * the meta provider has been told to remove it.
1759 provider->ftp_mcount++;
1761 mutex_exit(&provider->ftp_mtx);
1768 fasttrap_meta_create_probe(void *arg, void *parg,
1769 dtrace_helper_probedesc_t *dhpb)
1771 fasttrap_provider_t *provider = parg;
1772 fasttrap_probe_t *pp;
1773 fasttrap_tracepoint_t *tp;
1778 * Since the meta provider count is non-zero we don't have to worry
1779 * about this provider disappearing.
1781 ASSERT(provider->ftp_mcount > 0);
1784 * The offsets must be unique.
1786 qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t),
1787 fasttrap_uint32_cmp);
1788 for (i = 1; i < dhpb->dthpb_noffs; i++) {
1789 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <=
1790 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1])
1794 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t),
1795 fasttrap_uint32_cmp);
1796 for (i = 1; i < dhpb->dthpb_nenoffs; i++) {
1797 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <=
1798 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1])
1803 * Grab the creation lock to ensure consistency between calls to
1804 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1805 * other threads creating probes.
1807 mutex_enter(&provider->ftp_cmtx);
1809 if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod,
1810 dhpb->dthpb_func, dhpb->dthpb_name) != 0) {
1811 mutex_exit(&provider->ftp_cmtx);
1815 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs;
1818 atomic_add_32(&fasttrap_total, ntps);
1820 if (fasttrap_total > fasttrap_max) {
1821 atomic_add_32(&fasttrap_total, -ntps);
1822 mutex_exit(&provider->ftp_cmtx);
1826 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP);
1828 pp->ftp_prov = provider;
1829 pp->ftp_pid = provider->ftp_pid;
1830 pp->ftp_ntps = ntps;
1831 pp->ftp_nargs = dhpb->dthpb_xargc;
1832 pp->ftp_xtypes = dhpb->dthpb_xtypes;
1833 pp->ftp_ntypes = dhpb->dthpb_ntypes;
1836 * First create a tracepoint for each actual point of interest.
1838 for (i = 0; i < dhpb->dthpb_noffs; i++) {
1839 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1841 tp->ftt_proc = provider->ftp_proc;
1842 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i];
1843 tp->ftt_pid = provider->ftp_pid;
1845 pp->ftp_tps[i].fit_tp = tp;
1846 pp->ftp_tps[i].fit_id.fti_probe = pp;
1848 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS;
1850 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS;
1855 * Then create a tracepoint for each is-enabled point.
1857 for (j = 0; i < ntps; i++, j++) {
1858 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1860 tp->ftt_proc = provider->ftp_proc;
1861 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j];
1862 tp->ftt_pid = provider->ftp_pid;
1864 pp->ftp_tps[i].fit_tp = tp;
1865 pp->ftp_tps[i].fit_id.fti_probe = pp;
1866 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED;
1870 * If the arguments are shuffled around we set the argument remapping
1871 * table. Later, when the probe fires, we only remap the arguments
1872 * if the table is non-NULL.
1874 for (i = 0; i < dhpb->dthpb_xargc; i++) {
1875 if (dhpb->dthpb_args[i] != i) {
1876 pp->ftp_argmap = dhpb->dthpb_args;
1882 * The probe is fully constructed -- register it with DTrace.
1884 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
1885 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp);
1887 mutex_exit(&provider->ftp_cmtx);
1892 fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1895 * Clean up the USDT provider. There may be active consumers of the
1896 * provider busy adding probes, no damage will actually befall the
1897 * provider until that count has dropped to zero. This just puts
1898 * the provider on death row.
1900 fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1);
1903 static dtrace_mops_t fasttrap_mops = {
1904 fasttrap_meta_create_probe,
1905 fasttrap_meta_provide,
1906 fasttrap_meta_remove
1911 fasttrap_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
1918 fasttrap_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv)
1920 if (!dtrace_attached())
1923 if (cmd == FASTTRAPIOC_MAKEPROBE) {
1924 fasttrap_probe_spec_t *uprobe = (void *)arg;
1925 fasttrap_probe_spec_t *probe;
1931 if (copyin(&uprobe->ftps_noffs, &noffs,
1932 sizeof (uprobe->ftps_noffs)))
1936 * Probes must have at least one tracepoint.
1941 size = sizeof (fasttrap_probe_spec_t) +
1942 sizeof (probe->ftps_offs[0]) * (noffs - 1);
1944 if (size > 1024 * 1024)
1947 probe = kmem_alloc(size, KM_SLEEP);
1949 if (copyin(uprobe, probe, size) != 0) {
1950 kmem_free(probe, size);
1955 * Verify that the function and module strings contain no
1958 for (c = &probe->ftps_func[0]; *c != '\0'; c++) {
1959 if (*c < 0x20 || 0x7f <= *c) {
1965 for (c = &probe->ftps_mod[0]; *c != '\0'; c++) {
1966 if (*c < 0x20 || 0x7f <= *c) {
1972 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
1974 pid_t pid = probe->ftps_pid;
1976 mutex_enter(&pidlock);
1978 * Report an error if the process doesn't exist
1979 * or is actively being birthed.
1981 if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) {
1982 mutex_exit(&pidlock);
1985 mutex_enter(&p->p_lock);
1986 mutex_exit(&pidlock);
1988 if ((ret = priv_proc_cred_perm(cr, p, NULL,
1989 VREAD | VWRITE)) != 0) {
1990 mutex_exit(&p->p_lock);
1994 mutex_exit(&p->p_lock);
1997 ret = fasttrap_add_probe(probe);
1999 kmem_free(probe, size);
2003 } else if (cmd == FASTTRAPIOC_GETINSTR) {
2004 fasttrap_instr_query_t instr;
2005 fasttrap_tracepoint_t *tp;
2009 if (copyin((void *)arg, &instr, sizeof (instr)) != 0)
2012 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2014 pid_t pid = instr.ftiq_pid;
2016 mutex_enter(&pidlock);
2018 * Report an error if the process doesn't exist
2019 * or is actively being birthed.
2021 if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) {
2022 mutex_exit(&pidlock);
2025 mutex_enter(&p->p_lock);
2026 mutex_exit(&pidlock);
2028 if ((ret = priv_proc_cred_perm(cr, p, NULL,
2030 mutex_exit(&p->p_lock);
2034 mutex_exit(&p->p_lock);
2037 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc);
2039 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2040 tp = fasttrap_tpoints.fth_table[index].ftb_data;
2041 while (tp != NULL) {
2042 if (instr.ftiq_pid == tp->ftt_pid &&
2043 instr.ftiq_pc == tp->ftt_pc &&
2044 tp->ftt_proc->ftpc_acount != 0)
2048 * The count of active providers can only be
2049 * decremented (i.e. to zero) during exec, exit, and
2050 * removal of a meta provider so it should be
2051 * impossible to drop the count during this operation().
2053 ASSERT(tp->ftt_proc->ftpc_acount != 0);
2058 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2062 bcopy(&tp->ftt_instr, &instr.ftiq_instr,
2063 sizeof (instr.ftiq_instr));
2064 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2066 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0)
2075 static struct cb_ops fasttrap_cb_ops = {
2076 fasttrap_open, /* open */
2078 nulldev, /* strategy */
2079 nulldev, /* print */
2083 fasttrap_ioctl, /* ioctl */
2087 nochpoll, /* poll */
2088 ddi_prop_op, /* cb_prop_op */
2090 D_NEW | D_MP /* Driver compatibility flag */
2095 fasttrap_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
2100 case DDI_INFO_DEVT2DEVINFO:
2101 *result = (void *)fasttrap_devi;
2102 error = DDI_SUCCESS;
2104 case DDI_INFO_DEVT2INSTANCE:
2105 *result = (void *)0;
2106 error = DDI_SUCCESS;
2109 error = DDI_FAILURE;
2115 fasttrap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
2123 return (DDI_SUCCESS);
2125 return (DDI_FAILURE);
2128 if (ddi_create_minor_node(devi, "fasttrap", S_IFCHR, 0,
2129 DDI_PSEUDO, NULL) == DDI_FAILURE) {
2130 ddi_remove_minor_node(devi, NULL);
2131 return (DDI_FAILURE);
2134 ddi_report_dev(devi);
2135 fasttrap_devi = devi;
2138 * Install our hooks into fork(2), exec(2), and exit(2).
2140 dtrace_fasttrap_fork_ptr = &fasttrap_fork;
2141 dtrace_fasttrap_exit_ptr = &fasttrap_exec_exit;
2142 dtrace_fasttrap_exec_ptr = &fasttrap_exec_exit;
2144 fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2145 "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT);
2149 * Conjure up the tracepoints hashtable...
2151 nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2152 "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE);
2154 if (nent == 0 || nent > 0x1000000)
2155 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2157 if ((nent & (nent - 1)) == 0)
2158 fasttrap_tpoints.fth_nent = nent;
2160 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
2161 ASSERT(fasttrap_tpoints.fth_nent > 0);
2162 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1;
2163 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *
2164 sizeof (fasttrap_bucket_t), KM_SLEEP);
2167 * ... and the providers hash table...
2169 nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE;
2170 if ((nent & (nent - 1)) == 0)
2171 fasttrap_provs.fth_nent = nent;
2173 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
2174 ASSERT(fasttrap_provs.fth_nent > 0);
2175 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1;
2176 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *
2177 sizeof (fasttrap_bucket_t), KM_SLEEP);
2180 * ... and the procs hash table.
2182 nent = FASTTRAP_PROCS_DEFAULT_SIZE;
2183 if ((nent & (nent - 1)) == 0)
2184 fasttrap_procs.fth_nent = nent;
2186 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
2187 ASSERT(fasttrap_procs.fth_nent > 0);
2188 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1;
2189 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *
2190 sizeof (fasttrap_bucket_t), KM_SLEEP);
2192 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2195 return (DDI_SUCCESS);
2199 fasttrap_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
2208 return (DDI_SUCCESS);
2210 return (DDI_FAILURE);
2214 * Unregister the meta-provider to make sure no new fasttrap-
2215 * managed providers come along while we're trying to close up
2216 * shop. If we fail to detach, we'll need to re-register as a
2217 * meta-provider. We can fail to unregister as a meta-provider
2218 * if providers we manage still exist.
2220 if (fasttrap_meta_id != DTRACE_METAPROVNONE &&
2221 dtrace_meta_unregister(fasttrap_meta_id) != 0)
2222 return (DDI_FAILURE);
2225 * Prevent any new timeouts from running by setting fasttrap_timeout
2226 * to a non-zero value, and wait for the current timeout to complete.
2228 mutex_enter(&fasttrap_cleanup_mtx);
2229 fasttrap_cleanup_work = 0;
2231 while (fasttrap_timeout != (timeout_id_t)1) {
2232 tmp = fasttrap_timeout;
2233 fasttrap_timeout = (timeout_id_t)1;
2236 mutex_exit(&fasttrap_cleanup_mtx);
2237 (void) untimeout(tmp);
2238 mutex_enter(&fasttrap_cleanup_mtx);
2242 fasttrap_cleanup_work = 0;
2243 mutex_exit(&fasttrap_cleanup_mtx);
2246 * Iterate over all of our providers. If there's still a process
2247 * that corresponds to that pid, fail to detach.
2249 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
2250 fasttrap_provider_t **fpp, *fp;
2251 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i];
2253 mutex_enter(&bucket->ftb_mtx);
2254 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
2255 while ((fp = *fpp) != NULL) {
2257 * Acquire and release the lock as a simple way of
2258 * waiting for any other consumer to finish with
2259 * this provider. A thread must first acquire the
2260 * bucket lock so there's no chance of another thread
2261 * blocking on the provider's lock.
2263 mutex_enter(&fp->ftp_mtx);
2264 mutex_exit(&fp->ftp_mtx);
2266 if (dtrace_unregister(fp->ftp_provid) != 0) {
2268 fpp = &fp->ftp_next;
2270 *fpp = fp->ftp_next;
2271 fasttrap_provider_free(fp);
2275 mutex_exit(&bucket->ftb_mtx);
2281 * If we're failing to detach, we need to unblock timeouts
2282 * and start a new timeout if any work has accumulated while
2283 * we've been unsuccessfully trying to detach.
2285 mutex_enter(&fasttrap_cleanup_mtx);
2286 fasttrap_timeout = 0;
2287 work = fasttrap_cleanup_work;
2288 mutex_exit(&fasttrap_cleanup_mtx);
2291 fasttrap_pid_cleanup();
2293 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2296 return (DDI_FAILURE);
2300 mutex_enter(&fasttrap_count_mtx);
2301 ASSERT(fasttrap_pid_count == 0);
2302 mutex_exit(&fasttrap_count_mtx);
2305 kmem_free(fasttrap_tpoints.fth_table,
2306 fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t));
2307 fasttrap_tpoints.fth_nent = 0;
2309 kmem_free(fasttrap_provs.fth_table,
2310 fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t));
2311 fasttrap_provs.fth_nent = 0;
2313 kmem_free(fasttrap_procs.fth_table,
2314 fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t));
2315 fasttrap_procs.fth_nent = 0;
2318 * We know there are no tracepoints in any process anywhere in
2319 * the system so there is no process which has its p_dtrace_count
2320 * greater than zero, therefore we know that no thread can actively
2321 * be executing code in fasttrap_fork(). Similarly for p_dtrace_probes
2322 * and fasttrap_exec() and fasttrap_exit().
2324 ASSERT(dtrace_fasttrap_fork_ptr == &fasttrap_fork);
2325 dtrace_fasttrap_fork_ptr = NULL;
2327 ASSERT(dtrace_fasttrap_exec_ptr == &fasttrap_exec_exit);
2328 dtrace_fasttrap_exec_ptr = NULL;
2330 ASSERT(dtrace_fasttrap_exit_ptr == &fasttrap_exec_exit);
2331 dtrace_fasttrap_exit_ptr = NULL;
2333 ddi_remove_minor_node(devi, NULL);
2335 return (DDI_SUCCESS);
2338 static struct dev_ops fasttrap_ops = {
2339 DEVO_REV, /* devo_rev */
2341 fasttrap_info, /* get_dev_info */
2342 nulldev, /* identify */
2343 nulldev, /* probe */
2344 fasttrap_attach, /* attach */
2345 fasttrap_detach, /* detach */
2347 &fasttrap_cb_ops, /* driver operations */
2348 NULL, /* bus operations */
2349 nodev /* dev power */
2353 * Module linkage information for the kernel.
2355 static struct modldrv modldrv = {
2356 &mod_driverops, /* module type (this is a pseudo driver) */
2357 "Fasttrap Tracing", /* name of module */
2358 &fasttrap_ops, /* driver ops */
2361 static struct modlinkage modlinkage = {
2370 return (mod_install(&modlinkage));
2374 _info(struct modinfo *modinfop)
2376 return (mod_info(&modlinkage, modinfop));
2382 return (mod_remove(&modlinkage));