4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
21 * Portions Copyright 2008 John Birrell <jb@freebsd.org>
27 * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
28 * Use is subject to license terms.
31 #include <sys/cdefs.h>
32 #include <sys/param.h>
33 #include <sys/systm.h>
36 #include <sys/kernel.h>
37 #include <sys/limits.h>
38 #include <sys/linker.h>
40 #include <sys/module.h>
42 #include <sys/unistd.h>
43 #include <sys/dtrace.h>
44 #include <sys/dtrace_bsd.h>
45 #include <sys/fasttrap_impl.h>
48 * User-Land Trap-Based Tracing
49 * ----------------------------
51 * The fasttrap provider allows DTrace consumers to instrument any user-level
52 * instruction to gather data; this includes probes with semantic
53 * signifigance like entry and return as well as simple offsets into the
54 * function. While the specific techniques used are very ISA specific, the
55 * methodology is generalizable to any architecture.
58 * The General Methodology
59 * -----------------------
61 * With the primary goal of tracing every user-land instruction and the
62 * limitation that we can't trust user space so don't want to rely on much
63 * information there, we begin by replacing the instructions we want to trace
64 * with trap instructions. Each instruction we overwrite is saved into a hash
65 * table keyed by process ID and pc address. When we enter the kernel due to
66 * this trap instruction, we need the effects of the replaced instruction to
67 * appear to have occurred before we proceed with the user thread's
70 * Each user level thread is represented by a ulwp_t structure which is
71 * always easily accessible through a register. The most basic way to produce
72 * the effects of the instruction we replaced is to copy that instruction out
73 * to a bit of scratch space reserved in the user thread's ulwp_t structure
74 * (a sort of kernel-private thread local storage), set the PC to that
75 * scratch space and single step. When we reenter the kernel after single
76 * stepping the instruction we must then adjust the PC to point to what would
77 * normally be the next instruction. Of course, special care must be taken
78 * for branches and jumps, but these represent such a small fraction of any
79 * instruction set that writing the code to emulate these in the kernel is
82 * Return probes may require several tracepoints to trace every return site,
83 * and, conversely, each tracepoint may activate several probes (the entry
84 * and offset 0 probes, for example). To solve this muliplexing problem,
85 * tracepoints contain lists of probes to activate and probes contain lists
86 * of tracepoints to enable. If a probe is activated, it adds its ID to
87 * existing tracepoints or creates new ones as necessary.
89 * Most probes are activated _before_ the instruction is executed, but return
90 * probes are activated _after_ the effects of the last instruction of the
91 * function are visible. Return probes must be fired _after_ we have
92 * single-stepped the instruction whereas all other probes are fired
99 * The lock ordering below -- both internally and with respect to the DTrace
100 * framework -- is a little tricky and bears some explanation. Each provider
101 * has a lock (ftp_mtx) that protects its members including reference counts
102 * for enabled probes (ftp_rcount), consumers actively creating probes
103 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider
104 * from being freed. A provider is looked up by taking the bucket lock for the
105 * provider hash table, and is returned with its lock held. The provider lock
106 * may be taken in functions invoked by the DTrace framework, but may not be
107 * held while calling functions in the DTrace framework.
109 * To ensure consistency over multiple calls to the DTrace framework, the
110 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may
111 * not be taken when holding the provider lock as that would create a cyclic
112 * lock ordering. In situations where one would naturally take the provider
113 * lock and then the creation lock, we instead up a reference count to prevent
114 * the provider from disappearing, drop the provider lock, and acquire the
118 * bucket lock before provider lock
119 * DTrace before provider lock
120 * creation lock before DTrace
121 * never hold the provider lock and creation lock simultaneously
124 static d_open_t fasttrap_open;
125 static d_ioctl_t fasttrap_ioctl;
126 static int fasttrap_unload(void);
127 static void fasttrap_load(void *);
129 static struct cdevsw fasttrap_cdevsw = {
130 .d_version = D_VERSION,
131 .d_open = fasttrap_open,
132 .d_ioctl = fasttrap_ioctl,
133 .d_name = "fasttrap",
136 static struct cdev *fasttrap_cdev;
137 static dtrace_provider_id_t fasttrap_id __used;
138 static dtrace_meta_provider_id_t fasttrap_meta_id;
141 * When the fasttrap provider is loaded, fasttrap_max is set to either
142 * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the
143 * fasttrap.conf file. Each time a probe is created, fasttrap_total is
144 * incremented by the number of tracepoints that may be associated with that
145 * probe; fasttrap_total is capped at fasttrap_max.
147 #define FASTTRAP_MAX_DEFAULT 250000
148 static uint32_t fasttrap_max;
149 static uint32_t fasttrap_total;
151 fasttrap_hash_t fasttrap_tpoints;
152 static fasttrap_hash_t fasttrap_provs;
153 static fasttrap_hash_t fasttrap_procs;
155 #define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000
156 #define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100
157 #define FASTTRAP_PROCS_DEFAULT_SIZE 0x100
159 #define FASTTRAP_PID_NAME "pid"
161 static struct callout_handle fasttrap_timeout = CALLOUT_HANDLE_INITIALIZER(&fasttrap_timeout);
162 static struct mtx fasttrap_cleanup_mtx;
163 MTX_SYSINIT(fasttrap_cleanup_mtx, &fasttrap_cleanup_mtx, "Fasttrap cleanup", MTX_DEF);
164 static uint_t fasttrap_cleanup_work;
167 * Generation count on modifications to the global tracepoint lookup table.
169 static volatile uint64_t fasttrap_mod_gen;
171 static uint64_t fasttrap_pid_count; /* pid ref count */
172 static struct mtx fasttrap_count_mtx; /* lock on ref count */
173 MTX_SYSINIT(fasttrap_count_mtx, &fasttrap_count_mtx, "Fasttrap ref count", MTX_DEF);
175 #define FASTTRAP_ENABLE_FAIL 1
176 #define FASTTRAP_ENABLE_PARTIAL 2
178 static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
179 static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
181 static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
182 const dtrace_pattr_t *);
183 static void fasttrap_provider_free(fasttrap_provider_t *);
184 static void fasttrap_provider_retire(pid_t, const char *, int);
186 static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
187 static void fasttrap_proc_release(fasttrap_proc_t *);
189 #define FASTTRAP_PROVS_INDEX(pid, name) \
190 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
192 #define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
195 fasttrap_highbit(ulong_t i)
202 if (i & 0xffffffff00000000ul) {
206 if (i & 0xffff0000) {
225 fasttrap_hash_str(const char *p)
231 hval = (hval << 4) + *p++;
232 if ((g = (hval & 0xf0000000)) != 0)
240 fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
242 printf("%s(%d): DOODAD\n",__func__,__LINE__);
244 sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
246 sqp->sq_info.si_signo = SIGTRAP;
247 sqp->sq_info.si_code = TRAP_DTRACE;
248 sqp->sq_info.si_addr = (caddr_t)pc;
260 * This function ensures that no threads are actively using the memory
261 * associated with probes that were formerly live.
264 fasttrap_mod_barrier(uint64_t gen)
266 printf("%s(%d): DOODAD\n",__func__,__LINE__);
270 if (gen < fasttrap_mod_gen)
275 for (i = 0; i < MAXCPU; i++) {
276 mtx_lock(&cpu_core[i].cpuc_pid_lock);
277 mtx_unlock(&cpu_core[i].cpuc_pid_lock);
283 * This is the timeout's callback for cleaning up the providers and their
287 fasttrap_pid_cleanup_cb(void *data)
289 fasttrap_provider_t **fpp, *fp;
290 fasttrap_bucket_t *bucket;
291 dtrace_provider_id_t provid;
294 static volatile int in = 0;
298 mtx_lock(&fasttrap_cleanup_mtx);
299 while (fasttrap_cleanup_work) {
300 fasttrap_cleanup_work = 0;
301 mtx_unlock(&fasttrap_cleanup_mtx);
306 * Iterate over all the providers trying to remove the marked
307 * ones. If a provider is marked but not retired, we just
308 * have to take a crack at removing it -- it's no big deal if
311 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
312 bucket = &fasttrap_provs.fth_table[i];
313 mtx_lock(&bucket->ftb_mtx);
314 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
316 while ((fp = *fpp) != NULL) {
317 if (!fp->ftp_marked) {
322 mtx_lock(&fp->ftp_mtx);
325 * If this provider has consumers actively
326 * creating probes (ftp_ccount) or is a USDT
327 * provider (ftp_mcount), we can't unregister
330 if (fp->ftp_ccount != 0 ||
331 fp->ftp_mcount != 0) {
332 mtx_unlock(&fp->ftp_mtx);
337 if (!fp->ftp_retired || fp->ftp_rcount != 0)
340 mtx_unlock(&fp->ftp_mtx);
343 * If we successfully unregister this
344 * provider we can remove it from the hash
345 * chain and free the memory. If our attempt
346 * to unregister fails and this is a retired
347 * provider, increment our flag to try again
348 * pretty soon. If we've consumed more than
349 * half of our total permitted number of
350 * probes call dtrace_condense() to try to
351 * clean out the unenabled probes.
353 provid = fp->ftp_provid;
354 if (dtrace_unregister(provid) != 0) {
355 if (fasttrap_total > fasttrap_max / 2)
356 (void) dtrace_condense(provid);
357 later += fp->ftp_marked;
361 fasttrap_provider_free(fp);
364 mtx_unlock(&bucket->ftb_mtx);
367 mtx_lock(&fasttrap_cleanup_mtx);
371 * If we were unable to remove a retired provider, try again after
372 * a second. This situation can occur in certain circumstances where
373 * providers cannot be unregistered even though they have no probes
374 * enabled because of an execution of dtrace -l or something similar.
375 * If the timeout has been disabled (set to 1 because we're trying
376 * to detach), we set fasttrap_cleanup_work to ensure that we'll
377 * get a chance to do that work if and when the timeout is reenabled
380 if (later > 0 && fasttrap_timeout.callout != (struct callout *)(uintptr_t)1)
381 fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, hz);
383 fasttrap_cleanup_work = 1;
385 fasttrap_timeout.callout = NULL;
387 mtx_unlock(&fasttrap_cleanup_mtx);
392 * Activates the asynchronous cleanup mechanism.
395 fasttrap_pid_cleanup(void)
397 mtx_lock(&fasttrap_cleanup_mtx);
398 fasttrap_cleanup_work = 1;
399 if (fasttrap_timeout.callout == NULL)
400 fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, 1);
401 mtx_unlock(&fasttrap_cleanup_mtx);
404 /* This function assumes that the process is already locked. */
406 fasttrap_fork(proc_t *p, proc_t *cp)
408 pid_t ppid = p->p_pid;
412 ASSERT(curproc == p);
414 if (curproc != p) printf("%s(%d): Warning curproc != p\n",__func__,__LINE__);
418 * This would be simpler and faster if we maintained per-process
419 * hash tables of enabled tracepoints. It could, however, potentially
420 * slow down execution of a tracepoint since we'd need to go
421 * through two levels of indirection. In the future, we should
422 * consider either maintaining per-process ancillary lists of
423 * enabled tracepoints or hanging a pointer to a per-process hash
424 * table of enabled tracepoints off the proc structure.
428 * Iterate over every tracepoint looking for ones that belong to the
429 * parent process, and remove each from the child process.
431 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
432 fasttrap_tracepoint_t *tp;
433 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
435 mtx_lock(&bucket->ftb_mtx);
436 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
437 if (tp->ftt_pid == ppid &&
438 tp->ftt_proc->ftpc_acount != 0) {
439 printf("%s(%d): DOODAD\n",__func__,__LINE__);
441 int ret = fasttrap_tracepoint_remove(cp, tp);
446 mtx_unlock(&bucket->ftb_mtx);
450 /* This function assumes that the process is already locked. */
452 fasttrap_exec_exit(proc_t *p)
454 ASSERT(p == curproc);
457 * We clean up the pid provider for this process here; user-land
458 * static probes are handled by the meta-provider remove entry point.
460 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
465 fasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc)
468 * There are no "default" pid probes.
470 printf("%s(%d): There are no default pid probes\n",__func__,__LINE__);
474 fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
476 fasttrap_tracepoint_t *tp, *new_tp = NULL;
477 fasttrap_bucket_t *bucket;
482 ASSERT(index < probe->ftp_ntps);
484 pid = probe->ftp_pid;
485 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
486 id = &probe->ftp_tps[index].fit_id;
488 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
491 * Before we make any modifications, make sure we've imposed a barrier
492 * on the generation in which this probe was last modified.
494 fasttrap_mod_barrier(probe->ftp_gen);
496 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
499 * If the tracepoint has already been enabled, just add our id to the
500 * list of interested probes. This may be our second time through
501 * this path in which case we'll have constructed the tracepoint we'd
502 * like to install. If we can't find a match, and have an allocated
503 * tracepoint ready to go, enable that one now.
505 * A tracepoint whose process is defunct is also considered defunct.
510 mtx_lock(&bucket->ftb_mtx);
511 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
512 if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
513 tp->ftt_proc->ftpc_acount == 0)
517 * Now that we've found a matching tracepoint, it would be
518 * a decent idea to confirm that the tracepoint is still
519 * enabled and the trap instruction hasn't been overwritten.
520 * Since this is a little hairy, we'll punt for now.
524 * This can't be the first interested probe. We don't have
525 * to worry about another thread being in the midst of
526 * deleting this tracepoint (which would be the only valid
527 * reason for a tracepoint to have no interested probes)
528 * since we're holding P_PR_LOCK for this process.
530 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
532 switch (id->fti_ptype) {
535 case DTFTP_IS_ENABLED:
536 id->fti_next = tp->ftt_ids;
547 case DTFTP_POST_OFFSETS:
548 id->fti_next = tp->ftt_retids;
562 mtx_unlock(&bucket->ftb_mtx);
564 if (new_tp != NULL) {
565 new_tp->ftt_ids = NULL;
566 new_tp->ftt_retids = NULL;
573 * If we have a good tracepoint ready to go, install it now while
574 * we have the lock held and no one can screw with us.
576 if (new_tp != NULL) {
579 new_tp->ftt_next = bucket->ftb_data;
583 bucket->ftb_data = new_tp;
587 mtx_unlock(&bucket->ftb_mtx);
590 * Activate the tracepoint in the ISA-specific manner.
591 * If this fails, we need to report the failure, but
592 * indicate that this tracepoint must still be disabled
593 * by calling fasttrap_tracepoint_disable().
595 if (fasttrap_tracepoint_install(p, new_tp) != 0)
596 rc = FASTTRAP_ENABLE_PARTIAL;
599 * Increment the count of the number of tracepoints active in
600 * the victim process.
603 ASSERT(p->p_proc_flag & P_PR_LOCK);
610 mtx_unlock(&bucket->ftb_mtx);
613 * Initialize the tracepoint that's been preallocated with the probe.
615 new_tp = probe->ftp_tps[index].fit_tp;
617 ASSERT(new_tp->ftt_pid == pid);
618 ASSERT(new_tp->ftt_pc == pc);
619 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
620 ASSERT(new_tp->ftt_ids == NULL);
621 ASSERT(new_tp->ftt_retids == NULL);
623 switch (id->fti_ptype) {
626 case DTFTP_IS_ENABLED:
628 new_tp->ftt_ids = id;
632 case DTFTP_POST_OFFSETS:
634 new_tp->ftt_retids = id;
641 printf("%s(%d): DOODAD\n",__func__,__LINE__);
644 * If the ISA-dependent initialization goes to plan, go back to the
645 * beginning and try to install this freshly made tracepoint.
647 if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
651 new_tp->ftt_ids = NULL;
652 new_tp->ftt_retids = NULL;
654 return (FASTTRAP_ENABLE_FAIL);
658 fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
660 fasttrap_bucket_t *bucket;
661 fasttrap_provider_t *provider = probe->ftp_prov;
662 fasttrap_tracepoint_t **pp, *tp;
663 fasttrap_id_t *id, **idp;
667 ASSERT(index < probe->ftp_ntps);
669 pid = probe->ftp_pid;
670 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
671 id = &probe->ftp_tps[index].fit_id;
673 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
676 * Find the tracepoint and make sure that our id is one of the
677 * ones registered with it.
679 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
680 mtx_lock(&bucket->ftb_mtx);
681 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
682 if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
683 tp->ftt_proc == provider->ftp_proc)
688 * If we somehow lost this tracepoint, we're in a world of hurt.
692 switch (id->fti_ptype) {
695 case DTFTP_IS_ENABLED:
696 ASSERT(tp->ftt_ids != NULL);
701 case DTFTP_POST_OFFSETS:
702 ASSERT(tp->ftt_retids != NULL);
703 idp = &tp->ftt_retids;
710 while ((*idp)->fti_probe != probe) {
711 idp = &(*idp)->fti_next;
712 ASSERT(*idp != NULL);
721 ASSERT(id->fti_probe == probe);
724 * If there are other registered enablings of this tracepoint, we're
725 * all done, but if this was the last probe assocated with this
726 * this tracepoint, we need to remove and free it.
728 if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
731 * If the current probe's tracepoint is in use, swap it
732 * for an unused tracepoint.
734 if (tp == probe->ftp_tps[index].fit_tp) {
735 fasttrap_probe_t *tmp_probe;
736 fasttrap_tracepoint_t **tmp_tp;
739 if (tp->ftt_ids != NULL) {
740 tmp_probe = tp->ftt_ids->fti_probe;
741 /* LINTED - alignment */
742 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
743 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
745 tmp_probe = tp->ftt_retids->fti_probe;
746 /* LINTED - alignment */
747 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
748 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
751 ASSERT(*tmp_tp != NULL);
752 ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
753 ASSERT((*tmp_tp)->ftt_ids == NULL);
754 ASSERT((*tmp_tp)->ftt_retids == NULL);
756 probe->ftp_tps[index].fit_tp = *tmp_tp;
760 mtx_unlock(&bucket->ftb_mtx);
763 * Tag the modified probe with the generation in which it was
766 probe->ftp_gen = fasttrap_mod_gen;
770 mtx_unlock(&bucket->ftb_mtx);
773 * We can't safely remove the tracepoint from the set of active
774 * tracepoints until we've actually removed the fasttrap instruction
775 * from the process's text. We can, however, operate on this
776 * tracepoint secure in the knowledge that no other thread is going to
777 * be looking at it since we hold P_PR_LOCK on the process if it's
778 * live or we hold the provider lock on the process if it's dead and
783 * We only need to remove the actual instruction if we're looking
784 * at an existing process
788 * If we fail to restore the instruction we need to kill
789 * this process since it's in a completely unrecoverable
792 printf("%s(%d): DOODAD\n",__func__,__LINE__);
794 if (fasttrap_tracepoint_remove(p, tp) != 0)
795 fasttrap_sigtrap(p, NULL, pc);
799 * Decrement the count of the number of tracepoints active
800 * in the victim process.
803 ASSERT(p->p_proc_flag & P_PR_LOCK);
809 * Remove the probe from the hash table of active tracepoints.
811 mtx_lock(&bucket->ftb_mtx);
812 pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
815 pp = &(*pp)->ftt_next;
824 mtx_unlock(&bucket->ftb_mtx);
827 * Tag the modified probe with the generation in which it was changed.
829 probe->ftp_gen = fasttrap_mod_gen;
833 fasttrap_enable_callbacks(void)
836 * We don't have to play the rw lock game here because we're
837 * providing something rather than taking something away --
838 * we can be sure that no threads have tried to follow this
839 * function pointer yet.
841 mtx_lock(&fasttrap_count_mtx);
842 printf("%s(%d): DOODAD\n",__func__,__LINE__);
844 if (fasttrap_pid_count == 0) {
845 ASSERT(dtrace_pid_probe_ptr == NULL);
846 ASSERT(dtrace_return_probe_ptr == NULL);
847 dtrace_pid_probe_ptr = &fasttrap_pid_probe;
848 dtrace_return_probe_ptr = &fasttrap_return_probe;
850 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe);
851 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe);
853 fasttrap_pid_count++;
854 mtx_unlock(&fasttrap_count_mtx);
858 fasttrap_disable_callbacks(void)
860 ASSERT(MUTEX_HELD(&cpu_lock));
862 mtx_lock(&fasttrap_count_mtx);
863 printf("%s(%d): DOODAD\n",__func__,__LINE__);
865 ASSERT(fasttrap_pid_count > 0);
866 fasttrap_pid_count--;
867 if (fasttrap_pid_count == 0) {
868 cpu_t *cur, *cpu = CPU;
870 for (cur = cpu->cpu_next_onln; cur != cpu;
871 cur = cur->cpu_next_onln) {
872 rw_enter(&cur->cpu_ft_lock, RW_WRITER);
875 dtrace_pid_probe_ptr = NULL;
876 dtrace_return_probe_ptr = NULL;
878 for (cur = cpu->cpu_next_onln; cur != cpu;
879 cur = cur->cpu_next_onln) {
880 rw_exit(&cur->cpu_ft_lock);
884 mtx_unlock(&fasttrap_count_mtx);
888 fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
890 fasttrap_probe_t *probe = parg;
894 ASSERT(probe != NULL);
895 ASSERT(!probe->ftp_enabled);
896 ASSERT(id == probe->ftp_id);
897 ASSERT(MUTEX_HELD(&cpu_lock));
900 * Increment the count of enabled probes on this probe's provider;
901 * the provider can't go away while the probe still exists. We
902 * must increment this even if we aren't able to properly enable
905 mtx_lock(&probe->ftp_prov->ftp_mtx);
906 probe->ftp_prov->ftp_rcount++;
907 mtx_unlock(&probe->ftp_prov->ftp_mtx);
910 * If this probe's provider is retired (meaning it was valid in a
911 * previously exec'ed incarnation of this address space), bail out. The
912 * provider can't go away while we're in this code path.
914 if (probe->ftp_prov->ftp_retired)
917 printf("%s(%d): DOODAD\n",__func__,__LINE__);
920 * If we can't find the process, it may be that we're in the context of
921 * a fork in which the traced process is being born and we're copying
922 * USDT probes. Otherwise, the process is gone so bail.
924 if ((p = sprlock(probe->ftp_pid)) == NULL) {
925 if ((curproc->p_flag & SFORKING) == 0)
929 p = prfind(probe->ftp_pid);
932 * Confirm that curproc is indeed forking the process in which
933 * we're trying to enable probes.
936 ASSERT(p->p_parent == curproc);
937 ASSERT(p->p_stat == SIDL);
939 mtx_lock(&p->p_lock);
940 mtx_unlock(&pidlock);
945 ASSERT(!(p->p_flag & SVFORK));
946 mtx_unlock(&p->p_lock);
948 p = pfind(probe->ftp_pid);
952 * We have to enable the trap entry point before any user threads have
953 * the chance to execute the trap instruction we're about to place
954 * in their process's text.
956 fasttrap_enable_callbacks();
959 * Enable all the tracepoints and add this probe's id to each
960 * tracepoint's list of active probes.
962 for (i = 0; i < probe->ftp_ntps; i++) {
963 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
965 * If enabling the tracepoint failed completely,
966 * we don't have to disable it; if the failure
967 * was only partial we must disable it.
969 if (rc == FASTTRAP_ENABLE_FAIL)
972 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
975 * Back up and pull out all the tracepoints we've
976 * created so far for this probe.
979 fasttrap_tracepoint_disable(p, probe, i);
984 mtx_lock(&p->p_lock);
989 * Since we're not actually enabling this probe,
990 * drop our reference on the trap table entry.
992 fasttrap_disable_callbacks();
998 mtx_lock(&p->p_lock);
1002 probe->ftp_enabled = 1;
1006 fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
1008 printf("%s(%d): DOODAD\n",__func__,__LINE__);
1010 fasttrap_probe_t *probe = parg;
1011 fasttrap_provider_t *provider = probe->ftp_prov;
1015 ASSERT(id == probe->ftp_id);
1018 * We won't be able to acquire a /proc-esque lock on the process
1019 * iff the process is dead and gone. In this case, we rely on the
1020 * provider lock as a point of mutual exclusion to prevent other
1021 * DTrace consumers from disabling this probe.
1023 if ((p = sprlock(probe->ftp_pid)) != NULL) {
1024 ASSERT(!(p->p_flag & SVFORK));
1025 mtx_unlock(&p->p_lock);
1028 mtx_lock(&provider->ftp_mtx);
1031 * Disable all the associated tracepoints (for fully enabled probes).
1033 if (probe->ftp_enabled) {
1034 for (i = 0; i < probe->ftp_ntps; i++) {
1035 fasttrap_tracepoint_disable(p, probe, i);
1039 ASSERT(provider->ftp_rcount > 0);
1040 provider->ftp_rcount--;
1044 * Even though we may not be able to remove it entirely, we
1045 * mark this retired provider to get a chance to remove some
1046 * of the associated probes.
1048 if (provider->ftp_retired && !provider->ftp_marked)
1049 whack = provider->ftp_marked = 1;
1050 mtx_unlock(&provider->ftp_mtx);
1052 mtx_lock(&p->p_lock);
1056 * If the process is dead, we're just waiting for the
1057 * last probe to be disabled to be able to free it.
1059 if (provider->ftp_rcount == 0 && !provider->ftp_marked)
1060 whack = provider->ftp_marked = 1;
1061 mtx_unlock(&provider->ftp_mtx);
1065 fasttrap_pid_cleanup();
1067 if (!probe->ftp_enabled)
1070 probe->ftp_enabled = 0;
1072 ASSERT(MUTEX_HELD(&cpu_lock));
1073 fasttrap_disable_callbacks();
1078 fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
1079 dtrace_argdesc_t *desc)
1081 printf("%s(%d): DOODAD\n",__func__,__LINE__);
1083 fasttrap_probe_t *probe = parg;
1087 desc->dtargd_native[0] = '\0';
1088 desc->dtargd_xlate[0] = '\0';
1090 if (probe->ftp_prov->ftp_retired != 0 ||
1091 desc->dtargd_ndx >= probe->ftp_nargs) {
1092 desc->dtargd_ndx = DTRACE_ARGNONE;
1096 ndx = (probe->ftp_argmap != NULL) ?
1097 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx;
1099 str = probe->ftp_ntypes;
1100 for (i = 0; i < ndx; i++) {
1101 str += strlen(str) + 1;
1104 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native));
1105 (void) strcpy(desc->dtargd_native, str);
1107 if (probe->ftp_xtypes == NULL)
1110 str = probe->ftp_xtypes;
1111 for (i = 0; i < desc->dtargd_ndx; i++) {
1112 str += strlen(str) + 1;
1115 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate));
1116 (void) strcpy(desc->dtargd_xlate, str);
1121 fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
1123 printf("%s(%d): DOODAD\n",__func__,__LINE__);
1125 fasttrap_probe_t *probe = parg;
1129 ASSERT(probe != NULL);
1130 ASSERT(!probe->ftp_enabled);
1131 ASSERT(fasttrap_total >= probe->ftp_ntps);
1133 atomic_add_32(&fasttrap_total, -probe->ftp_ntps);
1134 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
1136 if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
1137 fasttrap_mod_barrier(probe->ftp_gen);
1139 for (i = 0; i < probe->ftp_ntps; i++) {
1140 kmem_free(probe->ftp_tps[i].fit_tp,
1141 sizeof (fasttrap_tracepoint_t));
1144 kmem_free(probe, size);
1148 static const dtrace_pattr_t pid_attr = {
1149 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1150 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1151 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1152 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1153 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1156 static dtrace_pops_t pid_pops = {
1157 fasttrap_pid_provide,
1159 fasttrap_pid_enable,
1160 fasttrap_pid_disable,
1163 fasttrap_pid_getargdesc,
1164 NULL /*fasttrap_pid_getarg*/,
1166 fasttrap_pid_destroy
1169 static dtrace_pops_t usdt_pops = {
1170 fasttrap_pid_provide,
1172 fasttrap_pid_enable,
1173 fasttrap_pid_disable,
1176 fasttrap_pid_getargdesc,
1177 NULL /*fasttrap_usdt_getarg*/,
1179 fasttrap_pid_destroy
1182 static fasttrap_proc_t *
1183 fasttrap_proc_lookup(pid_t pid)
1185 fasttrap_bucket_t *bucket;
1186 fasttrap_proc_t *fprc, *new_fprc;
1188 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1189 mtx_lock(&bucket->ftb_mtx);
1191 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1192 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1193 mtx_lock(&fprc->ftpc_mtx);
1194 mtx_unlock(&bucket->ftb_mtx);
1195 fprc->ftpc_rcount++;
1196 atomic_add_64(&fprc->ftpc_acount, 1);
1197 mtx_unlock(&fprc->ftpc_mtx);
1204 * Drop the bucket lock so we don't try to perform a sleeping
1205 * allocation under it.
1207 mtx_unlock(&bucket->ftb_mtx);
1209 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP);
1210 new_fprc->ftpc_pid = pid;
1211 new_fprc->ftpc_rcount = 1;
1212 new_fprc->ftpc_acount = 1;
1214 mtx_lock(&bucket->ftb_mtx);
1217 * Take another lap through the list to make sure a proc hasn't
1218 * been created for this pid while we weren't under the bucket lock.
1220 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1221 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1222 mtx_lock(&fprc->ftpc_mtx);
1223 mtx_unlock(&bucket->ftb_mtx);
1224 fprc->ftpc_rcount++;
1225 atomic_add_64(&fprc->ftpc_acount, 1);
1226 mtx_unlock(&fprc->ftpc_mtx);
1228 kmem_free(new_fprc, sizeof (fasttrap_proc_t));
1234 new_fprc->ftpc_next = bucket->ftb_data;
1235 bucket->ftb_data = new_fprc;
1237 mtx_unlock(&bucket->ftb_mtx);
1243 fasttrap_proc_release(fasttrap_proc_t *proc)
1245 fasttrap_bucket_t *bucket;
1246 fasttrap_proc_t *fprc, **fprcp;
1247 pid_t pid = proc->ftpc_pid;
1249 mtx_lock(&proc->ftpc_mtx);
1251 ASSERT(proc->ftpc_rcount != 0);
1253 if (--proc->ftpc_rcount != 0) {
1254 mtx_unlock(&proc->ftpc_mtx);
1258 mtx_unlock(&proc->ftpc_mtx);
1261 * There should definitely be no live providers associated with this
1262 * process at this point.
1264 ASSERT(proc->ftpc_acount == 0);
1266 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1267 mtx_lock(&bucket->ftb_mtx);
1269 fprcp = (fasttrap_proc_t **)&bucket->ftb_data;
1270 while ((fprc = *fprcp) != NULL) {
1274 fprcp = &fprc->ftpc_next;
1278 * Something strange has happened if we can't find the proc.
1280 ASSERT(fprc != NULL);
1282 *fprcp = fprc->ftpc_next;
1284 mtx_unlock(&bucket->ftb_mtx);
1286 kmem_free(fprc, sizeof (fasttrap_proc_t));
1290 * Lookup a fasttrap-managed provider based on its name and associated pid.
1291 * If the pattr argument is non-NULL, this function instantiates the provider
1292 * if it doesn't exist otherwise it returns NULL. The provider is returned
1293 * with its lock held.
1295 static fasttrap_provider_t *
1296 fasttrap_provider_lookup(pid_t pid, const char *name,
1297 const dtrace_pattr_t *pattr)
1299 fasttrap_provider_t *fp, *new_fp = NULL;
1300 fasttrap_bucket_t *bucket;
1301 char provname[DTRACE_PROVNAMELEN];
1307 ASSERT(strlen(name) < sizeof (fp->ftp_name));
1308 ASSERT(pattr != NULL);
1310 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1311 mtx_lock(&bucket->ftb_mtx);
1314 * Take a lap through the list and return the match if we find it.
1316 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1317 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1319 mtx_lock(&fp->ftp_mtx);
1320 mtx_unlock(&bucket->ftb_mtx);
1326 * Drop the bucket lock so we don't try to perform a sleeping
1327 * allocation under it.
1329 mtx_unlock(&bucket->ftb_mtx);
1333 * Make sure the process exists, isn't a child created as the result
1334 * of a vfork(2), and isn't a zombie (but may be in fork).
1337 if ((p = prfind(pid)) == NULL) {
1338 mtx_unlock(&pidlock);
1341 mtx_lock(&p->p_lock);
1342 mtx_unlock(&pidlock);
1343 if (p->p_flag & (SVFORK | SEXITING)) {
1344 mtx_unlock(&p->p_lock);
1349 * Increment p_dtrace_probes so that the process knows to inform us
1350 * when it exits or execs. fasttrap_provider_free() decrements this
1351 * when we're done with this provider.
1353 p->p_dtrace_probes++;
1356 * Grab the credentials for this process so we have
1357 * something to pass to dtrace_register().
1359 mtx_lock(&p->p_crlock);
1362 mtx_unlock(&p->p_crlock);
1363 mtx_unlock(&p->p_lock);
1368 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP);
1369 new_fp->ftp_pid = pid;
1370 new_fp->ftp_proc = fasttrap_proc_lookup(pid);
1372 ASSERT(new_fp->ftp_proc != NULL);
1374 mtx_lock(&bucket->ftb_mtx);
1377 * Take another lap through the list to make sure a provider hasn't
1378 * been created for this pid while we weren't under the bucket lock.
1380 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1381 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1383 mtx_lock(&fp->ftp_mtx);
1384 mtx_unlock(&bucket->ftb_mtx);
1385 fasttrap_provider_free(new_fp);
1393 (void) strcpy(new_fp->ftp_name, name);
1396 * Fail and return NULL if either the provider name is too long
1397 * or we fail to register this new provider with the DTrace
1398 * framework. Note that this is the only place we ever construct
1399 * the full provider name -- we keep it in pieces in the provider
1402 printf("%s(%d): provname '%s%u'\n", __func__, __LINE__, name, (uint_t)pid);
1403 if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >=
1404 sizeof (provname) ||
1405 dtrace_register(provname, pattr,
1406 DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, NULL /*cred*/,
1407 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp,
1408 &new_fp->ftp_provid) != 0) {
1409 printf("%s(%d): could not register!\n",__func__,__LINE__);
1410 mtx_unlock(&bucket->ftb_mtx);
1411 fasttrap_provider_free(new_fp);
1418 new_fp->ftp_next = bucket->ftb_data;
1419 bucket->ftb_data = new_fp;
1421 mtx_lock(&new_fp->ftp_mtx);
1422 mtx_unlock(&bucket->ftb_mtx);
1431 fasttrap_provider_free(fasttrap_provider_t *provider)
1433 printf("%s(%d): DOODAD\n",__func__,__LINE__);
1435 pid_t pid = provider->ftp_pid;
1440 * There need to be no associated enabled probes, no consumers
1441 * creating probes, and no meta providers referencing this provider.
1443 ASSERT(provider->ftp_rcount == 0);
1444 ASSERT(provider->ftp_ccount == 0);
1445 ASSERT(provider->ftp_mcount == 0);
1447 fasttrap_proc_release(provider->ftp_proc);
1449 kmem_free(provider, sizeof (fasttrap_provider_t));
1453 * Decrement p_dtrace_probes on the process whose provider we're
1454 * freeing. We don't have to worry about clobbering somone else's
1455 * modifications to it because we have locked the bucket that
1456 * corresponds to this process's hash chain in the provider hash
1457 * table. Don't sweat it if we can't find the process.
1460 if ((p = prfind(pid)) == NULL) {
1461 mtx_unlock(&pidlock);
1465 mtx_lock(&p->p_lock);
1466 mtx_unlock(&pidlock);
1468 p->p_dtrace_probes--;
1469 mtx_unlock(&p->p_lock);
1474 fasttrap_provider_retire(pid_t pid, const char *name, int mprov)
1476 fasttrap_provider_t *fp;
1477 fasttrap_bucket_t *bucket;
1478 dtrace_provider_id_t provid;
1480 ASSERT(strlen(name) < sizeof (fp->ftp_name));
1482 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1483 mtx_lock(&bucket->ftb_mtx);
1485 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1486 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1492 mtx_unlock(&bucket->ftb_mtx);
1496 mtx_lock(&fp->ftp_mtx);
1497 ASSERT(!mprov || fp->ftp_mcount > 0);
1498 if (mprov && --fp->ftp_mcount != 0) {
1499 mtx_unlock(&fp->ftp_mtx);
1500 mtx_unlock(&bucket->ftb_mtx);
1505 * Mark the provider to be removed in our post-processing step, mark it
1506 * retired, and drop the active count on its proc. Marking it indicates
1507 * that we should try to remove it; setting the retired flag indicates
1508 * that we're done with this provider; dropping the active the proc
1509 * releases our hold, and when this reaches zero (as it will during
1510 * exit or exec) the proc and associated providers become defunct.
1512 * We obviously need to take the bucket lock before the provider lock
1513 * to perform the lookup, but we need to drop the provider lock
1514 * before calling into the DTrace framework since we acquire the
1515 * provider lock in callbacks invoked from the DTrace framework. The
1516 * bucket lock therefore protects the integrity of the provider hash
1519 atomic_add_64(&fp->ftp_proc->ftpc_acount, -1);
1520 fp->ftp_retired = 1;
1522 provid = fp->ftp_provid;
1523 mtx_unlock(&fp->ftp_mtx);
1526 * We don't have to worry about invalidating the same provider twice
1527 * since fasttrap_provider_lookup() will ignore provider that have
1528 * been marked as retired.
1530 dtrace_invalidate(provid);
1532 mtx_unlock(&bucket->ftb_mtx);
1534 fasttrap_pid_cleanup();
1538 fasttrap_uint32_cmp(const void *ap, const void *bp)
1540 return (*(const uint32_t *)ap - *(const uint32_t *)bp);
1544 fasttrap_uint64_cmp(const void *ap, const void *bp)
1546 return (*(const uint64_t *)ap - *(const uint64_t *)bp);
1550 fasttrap_add_probe (fasttrap_probe_spec_t *pdata)
1552 fasttrap_provider_t *provider;
1553 fasttrap_probe_t *pp;
1554 fasttrap_tracepoint_t *tp;
1556 int i, aframes, whack;
1559 * There needs to be at least one desired trace point.
1561 if (pdata->ftps_noffs == 0)
1564 switch (pdata->ftps_type) {
1567 aframes = FASTTRAP_ENTRY_AFRAMES;
1571 aframes = FASTTRAP_RETURN_AFRAMES;
1580 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid,
1581 FASTTRAP_PID_NAME, &pid_attr)) == NULL)
1585 * Increment this reference count to indicate that a consumer is
1586 * actively adding a new probe associated with this provider. This
1587 * prevents the provider from being deleted -- we'll need to check
1588 * for pending deletions when we drop this reference count.
1590 provider->ftp_ccount++;
1591 mtx_unlock(&provider->ftp_mtx);
1594 * Grab the creation lock to ensure consistency between calls to
1595 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1596 * other threads creating probes. We must drop the provider lock
1597 * before taking this lock to avoid a three-way deadlock with the
1600 mtx_lock(&provider->ftp_cmtx);
1603 for (i = 0; i < pdata->ftps_noffs; i++) {
1606 (void) sprintf(name_str, "%llx",
1607 (unsigned long long)pdata->ftps_offs[i]);
1609 if (dtrace_probe_lookup(provider->ftp_provid,
1610 pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
1613 atomic_add_32(&fasttrap_total, 1);
1615 if (fasttrap_total > fasttrap_max) {
1616 atomic_add_32(&fasttrap_total, -1);
1620 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP);
1622 pp->ftp_prov = provider;
1623 pp->ftp_faddr = pdata->ftps_pc;
1624 pp->ftp_fsize = pdata->ftps_size;
1625 pp->ftp_pid = pdata->ftps_pid;
1628 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1631 tp->ftt_proc = provider->ftp_proc;
1632 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1633 tp->ftt_pid = pdata->ftps_pid;
1635 pp->ftp_tps[0].fit_tp = tp;
1636 pp->ftp_tps[0].fit_id.fti_probe = pp;
1637 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type;
1639 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1640 pdata->ftps_mod, pdata->ftps_func, name_str,
1641 FASTTRAP_OFFSET_AFRAMES, pp);
1644 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod,
1645 pdata->ftps_func, name) == 0) {
1646 atomic_add_32(&fasttrap_total, pdata->ftps_noffs);
1648 if (fasttrap_total > fasttrap_max) {
1649 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1654 * Make sure all tracepoint program counter values are unique.
1655 * We later assume that each probe has exactly one tracepoint
1658 qsort(pdata->ftps_offs, pdata->ftps_noffs,
1659 sizeof (uint64_t), fasttrap_uint64_cmp);
1660 for (i = 1; i < pdata->ftps_noffs; i++) {
1661 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1])
1664 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1668 ASSERT(pdata->ftps_noffs > 0);
1669 pp = kmem_zalloc(offsetof(fasttrap_probe_t,
1670 ftp_tps[pdata->ftps_noffs]), KM_SLEEP);
1672 pp->ftp_prov = provider;
1673 pp->ftp_faddr = pdata->ftps_pc;
1674 pp->ftp_fsize = pdata->ftps_size;
1675 pp->ftp_pid = pdata->ftps_pid;
1676 pp->ftp_ntps = pdata->ftps_noffs;
1678 for (i = 0; i < pdata->ftps_noffs; i++) {
1679 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1682 tp->ftt_proc = provider->ftp_proc;
1683 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1684 tp->ftt_pid = pdata->ftps_pid;
1686 pp->ftp_tps[i].fit_tp = tp;
1687 pp->ftp_tps[i].fit_id.fti_probe = pp;
1688 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type;
1691 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1692 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp);
1695 mtx_unlock(&provider->ftp_cmtx);
1698 * We know that the provider is still valid since we incremented the
1699 * creation reference count. If someone tried to clean up this provider
1700 * while we were using it (e.g. because the process called exec(2) or
1701 * exit(2)), take note of that and try to clean it up now.
1703 mtx_lock(&provider->ftp_mtx);
1704 provider->ftp_ccount--;
1705 whack = provider->ftp_retired;
1706 mtx_unlock(&provider->ftp_mtx);
1709 fasttrap_pid_cleanup();
1715 * If we've exhausted the allowable resources, we'll try to remove
1716 * this provider to free some up. This is to cover the case where
1717 * the user has accidentally created many more probes than was
1718 * intended (e.g. pid123:::).
1720 mtx_unlock(&provider->ftp_cmtx);
1721 mtx_lock(&provider->ftp_mtx);
1722 provider->ftp_ccount--;
1723 provider->ftp_marked = 1;
1724 mtx_unlock(&provider->ftp_mtx);
1726 fasttrap_pid_cleanup();
1732 fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1734 fasttrap_provider_t *provider;
1737 * A 32-bit unsigned integer (like a pid for example) can be
1738 * expressed in 10 or fewer decimal digits. Make sure that we'll
1739 * have enough space for the provider name.
1741 if (strlen(dhpv->dthpv_provname) + 10 >=
1742 sizeof (provider->ftp_name)) {
1743 printf("failed to instantiate provider %s: name too long to accomodate pid\n", dhpv->dthpv_provname);
1748 * Don't let folks spoof the true pid provider.
1750 if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) {
1751 printf("failed to instantiate provider %s: %s is an invalid name\n", dhpv->dthpv_provname,
1757 * The highest stability class that fasttrap supports is ISA; cap
1758 * the stability of the new provider accordingly.
1760 if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA)
1761 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA;
1762 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA)
1763 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA;
1764 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA)
1765 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA;
1766 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA)
1767 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA;
1768 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA)
1769 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA;
1771 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname,
1772 &dhpv->dthpv_pattr)) == NULL) {
1773 printf("failed to instantiate provider %s for process %u\n", dhpv->dthpv_provname, (uint_t)pid);
1778 * Up the meta provider count so this provider isn't removed until
1779 * the meta provider has been told to remove it.
1781 provider->ftp_mcount++;
1783 mtx_unlock(&provider->ftp_mtx);
1789 fasttrap_meta_create_probe(void *arg, void *parg,
1790 dtrace_helper_probedesc_t *dhpb)
1792 printf("%s(%d): DOODAD\n",__func__,__LINE__);
1794 fasttrap_provider_t *provider = parg;
1795 fasttrap_probe_t *pp;
1796 fasttrap_tracepoint_t *tp;
1801 * Since the meta provider count is non-zero we don't have to worry
1802 * about this provider disappearing.
1804 ASSERT(provider->ftp_mcount > 0);
1807 * The offsets must be unique.
1809 qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t),
1810 fasttrap_uint32_cmp);
1811 for (i = 1; i < dhpb->dthpb_noffs; i++) {
1812 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <=
1813 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1])
1817 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t),
1818 fasttrap_uint32_cmp);
1819 for (i = 1; i < dhpb->dthpb_nenoffs; i++) {
1820 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <=
1821 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1])
1826 * Grab the creation lock to ensure consistency between calls to
1827 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1828 * other threads creating probes.
1830 mtx_lock(&provider->ftp_cmtx);
1832 if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod,
1833 dhpb->dthpb_func, dhpb->dthpb_name) != 0) {
1834 mtx_unlock(&provider->ftp_cmtx);
1838 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs;
1841 atomic_add_32(&fasttrap_total, ntps);
1843 if (fasttrap_total > fasttrap_max) {
1844 atomic_add_32(&fasttrap_total, -ntps);
1845 mtx_unlock(&provider->ftp_cmtx);
1849 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP);
1851 pp->ftp_prov = provider;
1852 pp->ftp_pid = provider->ftp_pid;
1853 pp->ftp_ntps = ntps;
1854 pp->ftp_nargs = dhpb->dthpb_xargc;
1855 pp->ftp_xtypes = dhpb->dthpb_xtypes;
1856 pp->ftp_ntypes = dhpb->dthpb_ntypes;
1859 * First create a tracepoint for each actual point of interest.
1861 for (i = 0; i < dhpb->dthpb_noffs; i++) {
1862 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1864 tp->ftt_proc = provider->ftp_proc;
1865 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i];
1866 tp->ftt_pid = provider->ftp_pid;
1868 pp->ftp_tps[i].fit_tp = tp;
1869 pp->ftp_tps[i].fit_id.fti_probe = pp;
1871 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS;
1873 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS;
1878 * Then create a tracepoint for each is-enabled point.
1880 for (j = 0; i < ntps; i++, j++) {
1881 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1883 tp->ftt_proc = provider->ftp_proc;
1884 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j];
1885 tp->ftt_pid = provider->ftp_pid;
1887 pp->ftp_tps[i].fit_tp = tp;
1888 pp->ftp_tps[i].fit_id.fti_probe = pp;
1889 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED;
1893 * If the arguments are shuffled around we set the argument remapping
1894 * table. Later, when the probe fires, we only remap the arguments
1895 * if the table is non-NULL.
1897 for (i = 0; i < dhpb->dthpb_xargc; i++) {
1898 if (dhpb->dthpb_args[i] != i) {
1899 pp->ftp_argmap = dhpb->dthpb_args;
1905 * The probe is fully constructed -- register it with DTrace.
1907 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
1908 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp);
1910 mtx_unlock(&provider->ftp_cmtx);
1915 fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1917 printf("%s(%d): \n",__func__,__LINE__);
1919 * Clean up the USDT provider. There may be active consumers of the
1920 * provider busy adding probes, no damage will actually befall the
1921 * provider until that count has dropped to zero. This just puts
1922 * the provider on death row.
1924 fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1);
1927 static dtrace_mops_t fasttrap_mops = {
1928 fasttrap_meta_create_probe,
1929 fasttrap_meta_provide,
1930 fasttrap_meta_remove
1934 fasttrap_load(void *dummy)
1939 /* Create the /dev/dtrace/fasttrap entry. */
1940 fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600,
1944 * Install our hooks into fork(2), exec(2), and exit(2).
1946 dtrace_fasttrap_fork = fasttrap_fork;
1947 dtrace_fasttrap_exit = fasttrap_exec_exit;
1948 dtrace_fasttrap_exec = fasttrap_exec_exit;
1950 fasttrap_max = FASTTRAP_MAX_DEFAULT;
1954 * Conjure up the tracepoints hashtable...
1956 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; /* XXX sysctl? */
1958 if (nent == 0 || nent > 0x1000000)
1959 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
1961 if ((nent & (nent - 1)) == 0)
1962 fasttrap_tpoints.fth_nent = nent;
1964 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
1965 ASSERT(fasttrap_tpoints.fth_nent > 0);
1966 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1;
1967 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *
1968 sizeof (fasttrap_bucket_t), KM_SLEEP);
1970 for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
1971 mtx_init(&fasttrap_tpoints.fth_table[i].ftb_mtx,
1972 "Fasttrap tpoints", NULL, MTX_DEF);
1975 * ... and the providers hash table...
1977 nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE;
1978 if ((nent & (nent - 1)) == 0)
1979 fasttrap_provs.fth_nent = nent;
1981 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
1982 ASSERT(fasttrap_provs.fth_nent > 0);
1983 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1;
1984 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *
1985 sizeof (fasttrap_bucket_t), KM_SLEEP);
1987 for (i = 0; i < fasttrap_provs.fth_nent; i++)
1988 mtx_init(&fasttrap_provs.fth_table[i].ftb_mtx,
1989 "Fasttrap provs", NULL, MTX_DEF);
1992 * ... and the procs hash table.
1994 nent = FASTTRAP_PROCS_DEFAULT_SIZE;
1995 if ((nent & (nent - 1)) == 0)
1996 fasttrap_procs.fth_nent = nent;
1998 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
1999 ASSERT(fasttrap_procs.fth_nent > 0);
2000 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1;
2001 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *
2002 sizeof (fasttrap_bucket_t), KM_SLEEP);
2004 for (i = 0; i < fasttrap_procs.fth_nent; i++)
2005 mtx_init(&fasttrap_procs.fth_table[i].ftb_mtx,
2006 "Fasttrap procs", NULL, MTX_DEF);
2009 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2018 struct callout_handle tmp;
2020 /* Reset our calback hooks. */
2021 dtrace_fasttrap_fork = NULL;
2022 dtrace_fasttrap_exec = NULL;
2023 dtrace_fasttrap_exit = NULL;
2026 * Unregister the meta-provider to make sure no new fasttrap-
2027 * managed providers come along while we're trying to close up
2028 * shop. If we fail to detach, we'll need to re-register as a
2029 * meta-provider. We can fail to unregister as a meta-provider
2030 * if providers we manage still exist.
2032 if (fasttrap_meta_id != DTRACE_METAPROVNONE &&
2033 dtrace_meta_unregister(fasttrap_meta_id) != 0)
2037 * Prevent any new timeouts from running by setting fasttrap_timeout
2038 * to a non-zero value, and wait for the current timeout to complete.
2040 mtx_lock(&fasttrap_cleanup_mtx);
2041 fasttrap_cleanup_work = 0;
2043 while (fasttrap_timeout.callout != (struct callout *)(uintptr_t)1) {
2044 tmp = fasttrap_timeout;
2045 fasttrap_timeout.callout = (struct callout *)(uintptr_t)1;
2047 if (tmp.callout != NULL) {
2048 mtx_unlock(&fasttrap_cleanup_mtx);
2049 (void) untimeout(&fasttrap_pid_cleanup_cb, NULL, tmp);
2050 mtx_lock(&fasttrap_cleanup_mtx);
2054 fasttrap_cleanup_work = 0;
2055 mtx_unlock(&fasttrap_cleanup_mtx);
2058 * Iterate over all of our providers. If there's still a process
2059 * that corresponds to that pid, fail to detach.
2061 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
2062 fasttrap_provider_t **fpp, *fp;
2063 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i];
2065 mtx_lock(&bucket->ftb_mtx);
2066 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
2067 while ((fp = *fpp) != NULL) {
2069 * Acquire and release the lock as a simple way of
2070 * waiting for any other consumer to finish with
2071 * this provider. A thread must first acquire the
2072 * bucket lock so there's no chance of another thread
2073 * blocking on the provider's lock.
2075 mtx_lock(&fp->ftp_mtx);
2076 mtx_unlock(&fp->ftp_mtx);
2078 if (dtrace_unregister(fp->ftp_provid) != 0) {
2080 fpp = &fp->ftp_next;
2082 *fpp = fp->ftp_next;
2083 fasttrap_provider_free(fp);
2087 mtx_unlock(&bucket->ftb_mtx);
2093 * If we're failing to detach, we need to unblock timeouts
2094 * and start a new timeout if any work has accumulated while
2095 * we've been unsuccessfully trying to detach.
2097 mtx_lock(&fasttrap_cleanup_mtx);
2098 fasttrap_timeout.callout = NULL;
2099 work = fasttrap_cleanup_work;
2100 mtx_unlock(&fasttrap_cleanup_mtx);
2103 fasttrap_pid_cleanup();
2105 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2112 mtx_lock(&fasttrap_count_mtx);
2113 ASSERT(fasttrap_pid_count == 0);
2114 mtx_unlock(&fasttrap_count_mtx);
2117 for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2118 mtx_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx);
2120 kmem_free(fasttrap_tpoints.fth_table,
2121 fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t));
2122 fasttrap_tpoints.fth_nent = 0;
2124 for (i = 0; i < fasttrap_provs.fth_nent; i++)
2125 mtx_destroy(&fasttrap_provs.fth_table[i].ftb_mtx);
2127 kmem_free(fasttrap_provs.fth_table,
2128 fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t));
2129 fasttrap_provs.fth_nent = 0;
2131 for (i = 0; i < fasttrap_procs.fth_nent; i++)
2132 mtx_destroy(&fasttrap_procs.fth_table[i].ftb_mtx);
2134 kmem_free(fasttrap_procs.fth_table,
2135 fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t));
2136 fasttrap_procs.fth_nent = 0;
2138 destroy_dev(fasttrap_cdev);
2144 fasttrap_modevent(module_t mod __unused, int type, void *data __unused)
2168 fasttrap_open(struct cdev *dev __unused, int oflags __unused, int devtype __unused, struct thread *td __unused)
2174 fasttrap_ioctl(struct cdev *dev __unused, u_long cmd __unused, caddr_t addr __unused, int flags __unused, struct thread *td __unused)
2176 printf("%s(%d): DOODAD\n",__func__,__LINE__);
2178 if (!dtrace_attached())
2181 if (cmd == FASTTRAPIOC_MAKEPROBE) {
2182 fasttrap_probe_spec_t *uprobe = (void *)arg;
2183 fasttrap_probe_spec_t *probe;
2189 if (copyin(&uprobe->ftps_noffs, &noffs,
2190 sizeof (uprobe->ftps_noffs)))
2194 * Probes must have at least one tracepoint.
2199 size = sizeof (fasttrap_probe_spec_t) +
2200 sizeof (probe->ftps_offs[0]) * (noffs - 1);
2202 if (size > 1024 * 1024)
2205 probe = kmem_alloc(size, KM_SLEEP);
2207 if (copyin(uprobe, probe, size) != 0) {
2208 kmem_free(probe, size);
2213 * Verify that the function and module strings contain no
2216 for (c = &probe->ftps_func[0]; *c != '\0'; c++) {
2217 if (*c < 0x20 || 0x7f <= *c) {
2223 for (c = &probe->ftps_mod[0]; *c != '\0'; c++) {
2224 if (*c < 0x20 || 0x7f <= *c) {
2230 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2232 pid_t pid = probe->ftps_pid;
2236 * Report an error if the process doesn't exist
2237 * or is actively being birthed.
2239 if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) {
2240 mtx_unlock(&pidlock);
2243 mtx_lock(&p->p_lock);
2244 mtx_unlock(&pidlock);
2246 if ((ret = priv_proc_cred_perm(cr, p, NULL,
2247 VREAD | VWRITE)) != 0) {
2248 mtx_unlock(&p->p_lock);
2252 mtx_unlock(&p->p_lock);
2255 ret = fasttrap_add_probe(probe);
2257 kmem_free(probe, size);
2261 } else if (cmd == FASTTRAPIOC_GETINSTR) {
2262 fasttrap_instr_query_t instr;
2263 fasttrap_tracepoint_t *tp;
2267 if (copyin((void *)arg, &instr, sizeof (instr)) != 0)
2270 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2272 pid_t pid = instr.ftiq_pid;
2276 * Report an error if the process doesn't exist
2277 * or is actively being birthed.
2279 if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) {
2280 mtx_unlock(&pidlock);
2283 mtx_lock(&p->p_lock);
2284 mtx_unlock(&pidlock);
2286 if ((ret = priv_proc_cred_perm(cr, p, NULL,
2288 mtx_unlock(&p->p_lock);
2292 mtx_unlock(&p->p_lock);
2295 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc);
2297 mtx_lock(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2298 tp = fasttrap_tpoints.fth_table[index].ftb_data;
2299 while (tp != NULL) {
2300 if (instr.ftiq_pid == tp->ftt_pid &&
2301 instr.ftiq_pc == tp->ftt_pc &&
2302 tp->ftt_proc->ftpc_acount != 0)
2309 mtx_unlock(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2313 bcopy(&tp->ftt_instr, &instr.ftiq_instr,
2314 sizeof (instr.ftiq_instr));
2315 mtx_unlock(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2317 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0)
2327 SYSINIT(fasttrap_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fasttrap_load, NULL);
2328 SYSUNINIT(fasttrap_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fasttrap_unload, NULL);
2330 DEV_MODULE(fasttrap, fasttrap_modevent, NULL);
2331 MODULE_VERSION(fasttrap, 1);
2332 MODULE_DEPEND(fasttrap, dtrace, 1, 1, 1);
2333 MODULE_DEPEND(fasttrap, opensolaris, 1, 1, 1);