4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
21 * Portions Copyright 2006-2008 John Birrell jb@freebsd.org
28 * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
29 * Use is subject to license terms.
32 #include <sys/cdefs.h>
33 #include <sys/param.h>
34 #include <sys/systm.h>
36 #include <sys/cpuvar.h>
37 #include <sys/fcntl.h>
38 #include <sys/filio.h>
40 #include <sys/kernel.h>
42 #include <sys/kthread.h>
43 #include <sys/limits.h>
44 #include <sys/linker.h>
46 #include <sys/malloc.h>
47 #include <sys/module.h>
48 #include <sys/mutex.h>
52 #include <sys/selinfo.h>
54 #include <sys/syscall.h>
55 #include <sys/sysent.h>
56 #include <sys/sysproto.h>
58 #include <sys/unistd.h>
59 #include <machine/stdarg.h>
61 #include <sys/dtrace.h>
62 #include <sys/dtrace_bsd.h>
64 MALLOC_DEFINE(M_FBT, "fbt", "Function Boundary Tracing");
66 #define FBT_PUSHL_EBP 0x55
67 #define FBT_MOVL_ESP_EBP0_V0 0x8b
68 #define FBT_MOVL_ESP_EBP1_V0 0xec
69 #define FBT_MOVL_ESP_EBP0_V1 0x89
70 #define FBT_MOVL_ESP_EBP1_V1 0xe5
71 #define FBT_REX_RSP_RBP 0x48
73 #define FBT_POPL_EBP 0x5d
75 #define FBT_RET_IMM16 0xc2
76 #define FBT_LEAVE 0xc9
79 #define FBT_PATCHVAL 0xcc
81 #define FBT_PATCHVAL 0xf0
84 static d_open_t fbt_open;
85 static int fbt_unload(void);
86 static void fbt_getargdesc(void *, dtrace_id_t, void *, dtrace_argdesc_t *);
87 static void fbt_provide_module(void *, modctl_t *);
88 static void fbt_destroy(void *, dtrace_id_t, void *);
89 static void fbt_enable(void *, dtrace_id_t, void *);
90 static void fbt_disable(void *, dtrace_id_t, void *);
91 static void fbt_load(void *);
92 static void fbt_suspend(void *, dtrace_id_t, void *);
93 static void fbt_resume(void *, dtrace_id_t, void *);
95 #define FBT_ENTRY "entry"
96 #define FBT_RETURN "return"
97 #define FBT_ADDR2NDX(addr) ((((uintptr_t)(addr)) >> 4) & fbt_probetab_mask)
98 #define FBT_PROBETAB_SIZE 0x8000 /* 32k entries -- 128K total */
100 static struct cdevsw fbt_cdevsw = {
101 .d_version = D_VERSION,
106 static dtrace_pattr_t fbt_attr = {
107 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
108 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
109 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
110 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
111 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
114 static dtrace_pops_t fbt_pops = {
127 typedef struct fbt_probe {
128 struct fbt_probe *fbtp_hashnext;
129 uint8_t *fbtp_patchpoint;
131 uint8_t fbtp_patchval;
132 uint8_t fbtp_savedval;
133 uintptr_t fbtp_roffset;
135 const char *fbtp_name;
141 struct fbt_probe *fbtp_next;
144 static struct cdev *fbt_cdev;
145 static dtrace_provider_id_t fbt_id;
146 static fbt_probe_t **fbt_probetab;
147 static int fbt_probetab_size;
148 static int fbt_probetab_mask;
149 static int fbt_verbose = 0;
157 for (i = 0; i < fbt_probetab_size; i++) {
158 fbt = fbt_probetab[i];
160 for (; fbt != NULL; fbt = fbt->fbtp_next)
161 *fbt->fbtp_patchpoint = fbt->fbtp_savedval;
166 fbt_invop(uintptr_t addr, uintptr_t *stack, uintptr_t rval)
168 solaris_cpu_t *cpu = &solaris_cpu[curcpu];
169 uintptr_t stack0, stack1, stack2, stack3, stack4;
170 fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)];
172 for (; fbt != NULL; fbt = fbt->fbtp_hashnext) {
173 if ((uintptr_t)fbt->fbtp_patchpoint == addr) {
174 fbt->fbtp_invop_cnt++;
175 if (fbt->fbtp_roffset == 0) {
178 * When accessing the arguments on the stack,
179 * we must protect against accessing beyond
180 * the stack. We can safely set NOFAULT here
181 * -- we know that interrupts are already
184 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
185 cpu->cpu_dtrace_caller = stack[i++];
191 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT |
194 dtrace_probe(fbt->fbtp_id, stack0, stack1,
195 stack2, stack3, stack4);
197 cpu->cpu_dtrace_caller = 0;
201 * On amd64, we instrument the ret, not the
202 * leave. We therefore need to set the caller
203 * to assure that the top frame of a stack()
206 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
207 cpu->cpu_dtrace_caller = stack[0];
208 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT |
212 dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset,
214 cpu->cpu_dtrace_caller = 0;
217 return (fbt->fbtp_rval);
225 fbt_provide_module_function(linker_file_t lf, int symindx,
226 linker_symval_t *symval, void *opaque)
228 char *modname = opaque;
229 const char *name = symval->name;
230 fbt_probe_t *fbt, *retfbt;
233 u_int8_t *instr, *limit;
235 if (strncmp(name, "dtrace_", 7) == 0 &&
236 strncmp(name, "dtrace_safe_", 12) != 0) {
238 * Anything beginning with "dtrace_" may be called
239 * from probe context unless it explicitly indicates
240 * that it won't be called from probe context by
241 * using the prefix "dtrace_safe_".
246 if (name[0] == '_' && name[1] == '_')
251 instr = (u_int8_t *) symval->value;
252 limit = (u_int8_t *) symval->value + symval->size;
255 while (instr < limit) {
256 if (*instr == FBT_PUSHL_EBP)
259 if ((size = dtrace_instr_size(instr)) <= 0)
265 if (instr >= limit || *instr != FBT_PUSHL_EBP) {
267 * We either don't save the frame pointer in this
268 * function, or we ran into some disassembly
269 * screw-up. Either way, we bail.
274 if (instr[0] != FBT_PUSHL_EBP)
277 if (!(instr[1] == FBT_MOVL_ESP_EBP0_V0 &&
278 instr[2] == FBT_MOVL_ESP_EBP1_V0) &&
279 !(instr[1] == FBT_MOVL_ESP_EBP0_V1 &&
280 instr[2] == FBT_MOVL_ESP_EBP1_V1))
284 fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO);
285 fbt->fbtp_name = name;
286 fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
287 name, FBT_ENTRY, 3, fbt);
288 fbt->fbtp_patchpoint = instr;
290 fbt->fbtp_loadcnt = lf->loadcnt;
291 fbt->fbtp_rval = DTRACE_INVOP_PUSHL_EBP;
292 fbt->fbtp_savedval = *instr;
293 fbt->fbtp_patchval = FBT_PATCHVAL;
294 fbt->fbtp_symindx = symindx;
296 fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
297 fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
307 * If this disassembly fails, then we've likely walked off into
308 * a jump table or some other unsuitable area. Bail out of the
311 if ((size = dtrace_instr_size(instr)) <= 0)
316 * We only instrument "ret" on amd64 -- we don't yet instrument
317 * ret imm16, largely because the compiler doesn't seem to
318 * (yet) emit them in the kernel...
320 if (*instr != FBT_RET) {
326 (*instr == FBT_POPL_EBP || *instr == FBT_LEAVE) &&
327 (*(instr + 1) == FBT_RET ||
328 *(instr + 1) == FBT_RET_IMM16))) {
335 * We (desperately) want to avoid erroneously instrumenting a
336 * jump table, especially given that our markers are pretty
337 * short: two bytes on x86, and just one byte on amd64. To
338 * determine if we're looking at a true instruction sequence
339 * or an inline jump table that happens to contain the same
340 * byte sequences, we resort to some heuristic sleeze: we
341 * treat this instruction as being contained within a pointer,
342 * and see if that pointer points to within the body of the
343 * function. If it does, we refuse to instrument it.
345 for (j = 0; j < sizeof (uintptr_t); j++) {
346 caddr_t check = (caddr_t) instr - j;
349 if (check < symval->value)
352 if (check + sizeof (caddr_t) > (caddr_t)limit)
355 ptr = *(uint8_t **)check;
357 if (ptr >= (uint8_t *) symval->value && ptr < limit) {
366 fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO);
367 fbt->fbtp_name = name;
369 if (retfbt == NULL) {
370 fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
371 name, FBT_RETURN, 3, fbt);
373 retfbt->fbtp_next = fbt;
374 fbt->fbtp_id = retfbt->fbtp_id;
378 fbt->fbtp_patchpoint = instr;
380 fbt->fbtp_loadcnt = lf->loadcnt;
381 fbt->fbtp_symindx = symindx;
384 if (*instr == FBT_POPL_EBP) {
385 fbt->fbtp_rval = DTRACE_INVOP_POPL_EBP;
387 ASSERT(*instr == FBT_LEAVE);
388 fbt->fbtp_rval = DTRACE_INVOP_LEAVE;
391 (uintptr_t)(instr - (uint8_t *) symval->value) + 1;
394 ASSERT(*instr == FBT_RET);
395 fbt->fbtp_rval = DTRACE_INVOP_RET;
397 (uintptr_t)(instr - (uint8_t *) symval->value);
400 fbt->fbtp_savedval = *instr;
401 fbt->fbtp_patchval = FBT_PATCHVAL;
402 fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
403 fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
412 fbt_provide_module(void *arg, modctl_t *lf)
414 char modname[MAXPATHLEN];
418 strlcpy(modname, lf->filename, sizeof(modname));
419 len = strlen(modname);
420 if (len > 3 && strcmp(modname + len - 3, ".ko") == 0)
421 modname[len - 3] = '\0';
424 * Employees of dtrace and their families are ineligible. Void
427 if (strcmp(modname, "dtrace") == 0)
431 * The cyclic timer subsystem can be built as a module and DTrace
432 * depends on that, so it is ineligible too.
434 if (strcmp(modname, "cyclic") == 0)
438 * To register with DTrace, a module must list 'dtrace' as a
439 * dependency in order for the kernel linker to resolve
440 * symbols like dtrace_register(). All modules with such a
441 * dependency are ineligible for FBT tracing.
443 for (i = 0; i < lf->ndeps; i++)
444 if (strncmp(lf->deps[i]->filename, "dtrace", 6) == 0)
447 if (lf->fbt_nentries) {
449 * This module has some FBT entries allocated; we're afraid
456 * List the functions in the module and the symbol values.
458 (void) linker_file_function_listall(lf, fbt_provide_module_function, modname);
462 fbt_destroy(void *arg, dtrace_id_t id, void *parg)
464 fbt_probe_t *fbt = parg, *next, *hash, *last;
474 * Now we need to remove this probe from the fbt_probetab.
476 ndx = FBT_ADDR2NDX(fbt->fbtp_patchpoint);
478 hash = fbt_probetab[ndx];
480 while (hash != fbt) {
481 ASSERT(hash != NULL);
483 hash = hash->fbtp_hashnext;
487 last->fbtp_hashnext = fbt->fbtp_hashnext;
489 fbt_probetab[ndx] = fbt->fbtp_hashnext;
492 next = fbt->fbtp_next;
496 } while (fbt != NULL);
500 fbt_enable(void *arg, dtrace_id_t id, void *parg)
502 fbt_probe_t *fbt = parg;
503 modctl_t *ctl = fbt->fbtp_ctl;
508 * Now check that our modctl has the expected load count. If it
509 * doesn't, this module must have been unloaded and reloaded -- and
510 * we're not going to touch it.
512 if (ctl->loadcnt != fbt->fbtp_loadcnt) {
514 printf("fbt is failing for probe %s "
515 "(module %s reloaded)",
516 fbt->fbtp_name, ctl->filename);
522 for (; fbt != NULL; fbt = fbt->fbtp_next) {
523 *fbt->fbtp_patchpoint = fbt->fbtp_patchval;
528 fbt_disable(void *arg, dtrace_id_t id, void *parg)
530 fbt_probe_t *fbt = parg;
531 modctl_t *ctl = fbt->fbtp_ctl;
533 ASSERT(ctl->nenabled > 0);
536 if ((ctl->loadcnt != fbt->fbtp_loadcnt))
539 for (; fbt != NULL; fbt = fbt->fbtp_next)
540 *fbt->fbtp_patchpoint = fbt->fbtp_savedval;
544 fbt_suspend(void *arg, dtrace_id_t id, void *parg)
546 fbt_probe_t *fbt = parg;
547 modctl_t *ctl = fbt->fbtp_ctl;
549 ASSERT(ctl->nenabled > 0);
551 if ((ctl->loadcnt != fbt->fbtp_loadcnt))
554 for (; fbt != NULL; fbt = fbt->fbtp_next)
555 *fbt->fbtp_patchpoint = fbt->fbtp_savedval;
559 fbt_resume(void *arg, dtrace_id_t id, void *parg)
561 fbt_probe_t *fbt = parg;
562 modctl_t *ctl = fbt->fbtp_ctl;
564 ASSERT(ctl->nenabled > 0);
566 if ((ctl->loadcnt != fbt->fbtp_loadcnt))
569 for (; fbt != NULL; fbt = fbt->fbtp_next)
570 *fbt->fbtp_patchpoint = fbt->fbtp_patchval;
574 fbt_ctfoff_init(modctl_t *lf, linker_ctf_t *lc)
576 const Elf_Sym *symp = lc->symtab;;
578 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;
579 const uint8_t *ctfdata = lc->ctftab + sizeof(ctf_header_t);
582 uint32_t objtoff = hp->cth_objtoff;
583 uint32_t funcoff = hp->cth_funcoff;
588 if (hp->cth_magic != CTF_MAGIC) {
589 printf("Bad magic value in CTF data of '%s'\n",lf->pathname);
593 if (lc->symtab == NULL) {
594 printf("No symbol table in '%s'\n",lf->pathname);
598 if ((ctfoff = malloc(sizeof(uint32_t) * lc->nsym, M_LINKER, M_WAITOK)) == NULL)
601 *lc->ctfoffp = ctfoff;
603 for (i = 0; i < lc->nsym; i++, ctfoff++, symp++) {
604 if (symp->st_name == 0 || symp->st_shndx == SHN_UNDEF) {
605 *ctfoff = 0xffffffff;
609 if (symp->st_name < lc->strcnt)
610 name = lc->strtab + symp->st_name;
614 switch (ELF_ST_TYPE(symp->st_info)) {
616 if (objtoff >= hp->cth_funcoff ||
617 (symp->st_shndx == SHN_ABS && symp->st_value == 0)) {
618 *ctfoff = 0xffffffff;
623 objtoff += sizeof (ushort_t);
627 if (funcoff >= hp->cth_typeoff) {
628 *ctfoff = 0xffffffff;
634 info = *((const ushort_t *)(ctfdata + funcoff));
635 vlen = CTF_INFO_VLEN(info);
638 * If we encounter a zero pad at the end, just skip it.
639 * Otherwise skip over the function and its return type
640 * (+2) and the argument list (vlen).
642 if (CTF_INFO_KIND(info) == CTF_K_UNKNOWN && vlen == 0)
643 funcoff += sizeof (ushort_t); /* skip pad */
645 funcoff += sizeof (ushort_t) * (vlen + 2);
649 *ctfoff = 0xffffffff;
658 fbt_get_ctt_size(uint8_t version, const ctf_type_t *tp, ssize_t *sizep,
661 ssize_t size, increment;
663 if (version > CTF_VERSION_1 &&
664 tp->ctt_size == CTF_LSIZE_SENT) {
665 size = CTF_TYPE_LSIZE(tp);
666 increment = sizeof (ctf_type_t);
669 increment = sizeof (ctf_stype_t);
675 *incrementp = increment;
681 fbt_typoff_init(linker_ctf_t *lc)
683 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;
684 const ctf_type_t *tbuf;
685 const ctf_type_t *tend;
686 const ctf_type_t *tp;
687 const uint8_t *ctfdata = lc->ctftab + sizeof(ctf_header_t);
690 ulong_t pop[CTF_K_MAX + 1] = { 0 };
694 if (hp->cth_magic != CTF_MAGIC)
697 tbuf = (const ctf_type_t *) (ctfdata + hp->cth_typeoff);
698 tend = (const ctf_type_t *) (ctfdata + hp->cth_stroff);
700 int child = hp->cth_parname != 0;
703 * We make two passes through the entire type section. In this first
704 * pass, we count the number of each type and the total number of types.
706 for (tp = tbuf; tp < tend; ctf_typemax++) {
707 ushort_t kind = CTF_INFO_KIND(tp->ctt_info);
708 ulong_t vlen = CTF_INFO_VLEN(tp->ctt_info);
709 ssize_t size, increment;
714 (void) fbt_get_ctt_size(hp->cth_version, tp, &size, &increment);
719 vbytes = sizeof (uint_t);
722 vbytes = sizeof (ctf_array_t);
725 vbytes = sizeof (ushort_t) * (vlen + (vlen & 1));
729 if (size < CTF_LSTRUCT_THRESH) {
730 ctf_member_t *mp = (ctf_member_t *)
731 ((uintptr_t)tp + increment);
733 vbytes = sizeof (ctf_member_t) * vlen;
734 for (n = vlen; n != 0; n--, mp++)
735 child |= CTF_TYPE_ISCHILD(mp->ctm_type);
737 ctf_lmember_t *lmp = (ctf_lmember_t *)
738 ((uintptr_t)tp + increment);
740 vbytes = sizeof (ctf_lmember_t) * vlen;
741 for (n = vlen; n != 0; n--, lmp++)
743 CTF_TYPE_ISCHILD(lmp->ctlm_type);
747 vbytes = sizeof (ctf_enum_t) * vlen;
751 * For forward declarations, ctt_type is the CTF_K_*
752 * kind for the tag, so bump that population count too.
753 * If ctt_type is unknown, treat the tag as a struct.
755 if (tp->ctt_type == CTF_K_UNKNOWN ||
756 tp->ctt_type >= CTF_K_MAX)
769 child |= CTF_TYPE_ISCHILD(tp->ctt_type);
773 printf("%s(%d): detected invalid CTF kind -- %u\n", __func__, __LINE__, kind);
776 tp = (ctf_type_t *)((uintptr_t)tp + increment + vbytes);
780 *lc->typlenp = ctf_typemax;
782 if ((xp = malloc(sizeof(uint32_t) * ctf_typemax, M_LINKER, M_ZERO | M_WAITOK)) == NULL)
787 /* type id 0 is used as a sentinel value */
791 * In the second pass, fill in the type offset.
793 for (tp = tbuf; tp < tend; xp++) {
794 ushort_t kind = CTF_INFO_KIND(tp->ctt_info);
795 ulong_t vlen = CTF_INFO_VLEN(tp->ctt_info);
796 ssize_t size, increment;
801 (void) fbt_get_ctt_size(hp->cth_version, tp, &size, &increment);
806 vbytes = sizeof (uint_t);
809 vbytes = sizeof (ctf_array_t);
812 vbytes = sizeof (ushort_t) * (vlen + (vlen & 1));
816 if (size < CTF_LSTRUCT_THRESH) {
817 ctf_member_t *mp = (ctf_member_t *)
818 ((uintptr_t)tp + increment);
820 vbytes = sizeof (ctf_member_t) * vlen;
821 for (n = vlen; n != 0; n--, mp++)
822 child |= CTF_TYPE_ISCHILD(mp->ctm_type);
824 ctf_lmember_t *lmp = (ctf_lmember_t *)
825 ((uintptr_t)tp + increment);
827 vbytes = sizeof (ctf_lmember_t) * vlen;
828 for (n = vlen; n != 0; n--, lmp++)
830 CTF_TYPE_ISCHILD(lmp->ctlm_type);
834 vbytes = sizeof (ctf_enum_t) * vlen;
848 printf("%s(%d): detected invalid CTF kind -- %u\n", __func__, __LINE__, kind);
851 *xp = (uint32_t)((uintptr_t) tp - (uintptr_t) ctfdata);
852 tp = (ctf_type_t *)((uintptr_t)tp + increment + vbytes);
859 * CTF Declaration Stack
861 * In order to implement ctf_type_name(), we must convert a type graph back
862 * into a C type declaration. Unfortunately, a type graph represents a storage
863 * class ordering of the type whereas a type declaration must obey the C rules
864 * for operator precedence, and the two orderings are frequently in conflict.
865 * For example, consider these CTF type graphs and their C declarations:
867 * CTF_K_POINTER -> CTF_K_FUNCTION -> CTF_K_INTEGER : int (*)()
868 * CTF_K_POINTER -> CTF_K_ARRAY -> CTF_K_INTEGER : int (*)[]
870 * In each case, parentheses are used to raise operator * to higher lexical
871 * precedence, so the string form of the C declaration cannot be constructed by
872 * walking the type graph links and forming the string from left to right.
874 * The functions in this file build a set of stacks from the type graph nodes
875 * corresponding to the C operator precedence levels in the appropriate order.
876 * The code in ctf_type_name() can then iterate over the levels and nodes in
877 * lexical precedence order and construct the final C declaration string.
879 typedef struct ctf_list {
880 struct ctf_list *l_prev; /* previous pointer or tail pointer */
881 struct ctf_list *l_next; /* next pointer or head pointer */
884 #define ctf_list_prev(elem) ((void *)(((ctf_list_t *)(elem))->l_prev))
885 #define ctf_list_next(elem) ((void *)(((ctf_list_t *)(elem))->l_next))
895 typedef struct ctf_decl_node {
896 ctf_list_t cd_list; /* linked list pointers */
897 ctf_id_t cd_type; /* type identifier */
898 uint_t cd_kind; /* type kind */
899 uint_t cd_n; /* type dimension if array */
902 typedef struct ctf_decl {
903 ctf_list_t cd_nodes[CTF_PREC_MAX]; /* declaration node stacks */
904 int cd_order[CTF_PREC_MAX]; /* storage order of decls */
905 ctf_decl_prec_t cd_qualp; /* qualifier precision */
906 ctf_decl_prec_t cd_ordp; /* ordered precision */
907 char *cd_buf; /* buffer for output */
908 char *cd_ptr; /* buffer location */
909 char *cd_end; /* buffer limit */
910 size_t cd_len; /* buffer space required */
911 int cd_err; /* saved error value */
915 * Simple doubly-linked list append routine. This implementation assumes that
916 * each list element contains an embedded ctf_list_t as the first member.
917 * An additional ctf_list_t is used to store the head (l_next) and tail
918 * (l_prev) pointers. The current head and tail list elements have their
919 * previous and next pointers set to NULL, respectively.
922 ctf_list_append(ctf_list_t *lp, void *new)
924 ctf_list_t *p = lp->l_prev; /* p = tail list element */
925 ctf_list_t *q = new; /* q = new list element */
938 * Prepend the specified existing element to the given ctf_list_t. The
939 * existing pointer should be pointing at a struct with embedded ctf_list_t.
942 ctf_list_prepend(ctf_list_t *lp, void *new)
944 ctf_list_t *p = new; /* p = new list element */
945 ctf_list_t *q = lp->l_next; /* q = head list element */
958 ctf_decl_init(ctf_decl_t *cd, char *buf, size_t len)
962 bzero(cd, sizeof (ctf_decl_t));
964 for (i = CTF_PREC_BASE; i < CTF_PREC_MAX; i++)
965 cd->cd_order[i] = CTF_PREC_BASE - 1;
967 cd->cd_qualp = CTF_PREC_BASE;
968 cd->cd_ordp = CTF_PREC_BASE;
972 cd->cd_end = buf + len;
976 ctf_decl_fini(ctf_decl_t *cd)
978 ctf_decl_node_t *cdp, *ndp;
981 for (i = CTF_PREC_BASE; i < CTF_PREC_MAX; i++) {
982 for (cdp = ctf_list_next(&cd->cd_nodes[i]);
983 cdp != NULL; cdp = ndp) {
984 ndp = ctf_list_next(cdp);
990 static const ctf_type_t *
991 ctf_lookup_by_id(linker_ctf_t *lc, ctf_id_t type)
993 const ctf_type_t *tp;
995 uint32_t *typoff = *lc->typoffp;
997 if (type >= *lc->typlenp) {
998 printf("%s(%d): type %d exceeds max %ld\n",__func__,__LINE__,(int) type,*lc->typlenp);
1002 /* Check if the type isn't cross-referenced. */
1003 if ((offset = typoff[type]) == 0) {
1004 printf("%s(%d): type %d isn't cross referenced\n",__func__,__LINE__, (int) type);
1008 tp = (const ctf_type_t *)(lc->ctftab + offset + sizeof(ctf_header_t));
1014 fbt_array_info(linker_ctf_t *lc, ctf_id_t type, ctf_arinfo_t *arp)
1016 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;
1017 const ctf_type_t *tp;
1018 const ctf_array_t *ap;
1021 bzero(arp, sizeof(*arp));
1023 if ((tp = ctf_lookup_by_id(lc, type)) == NULL)
1026 if (CTF_INFO_KIND(tp->ctt_info) != CTF_K_ARRAY)
1029 (void) fbt_get_ctt_size(hp->cth_version, tp, NULL, &increment);
1031 ap = (const ctf_array_t *)((uintptr_t)tp + increment);
1032 arp->ctr_contents = ap->cta_contents;
1033 arp->ctr_index = ap->cta_index;
1034 arp->ctr_nelems = ap->cta_nelems;
1038 ctf_strptr(linker_ctf_t *lc, int name)
1040 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;;
1041 const char *strp = "";
1043 if (name < 0 || name >= hp->cth_strlen)
1046 strp = (const char *)(lc->ctftab + hp->cth_stroff + name + sizeof(ctf_header_t));
1052 ctf_decl_push(ctf_decl_t *cd, linker_ctf_t *lc, ctf_id_t type)
1054 ctf_decl_node_t *cdp;
1055 ctf_decl_prec_t prec;
1059 const ctf_type_t *tp;
1062 if ((tp = ctf_lookup_by_id(lc, type)) == NULL) {
1063 cd->cd_err = ENOENT;
1067 switch (kind = CTF_INFO_KIND(tp->ctt_info)) {
1069 fbt_array_info(lc, type, &ar);
1070 ctf_decl_push(cd, lc, ar.ctr_contents);
1072 prec = CTF_PREC_ARRAY;
1076 if (ctf_strptr(lc, tp->ctt_name)[0] == '\0') {
1077 ctf_decl_push(cd, lc, tp->ctt_type);
1080 prec = CTF_PREC_BASE;
1083 case CTF_K_FUNCTION:
1084 ctf_decl_push(cd, lc, tp->ctt_type);
1085 prec = CTF_PREC_FUNCTION;
1089 ctf_decl_push(cd, lc, tp->ctt_type);
1090 prec = CTF_PREC_POINTER;
1093 case CTF_K_VOLATILE:
1095 case CTF_K_RESTRICT:
1096 ctf_decl_push(cd, lc, tp->ctt_type);
1097 prec = cd->cd_qualp;
1102 prec = CTF_PREC_BASE;
1105 if ((cdp = malloc(sizeof (ctf_decl_node_t), M_FBT, M_WAITOK)) == NULL) {
1106 cd->cd_err = EAGAIN;
1110 cdp->cd_type = type;
1111 cdp->cd_kind = kind;
1114 if (ctf_list_next(&cd->cd_nodes[prec]) == NULL)
1115 cd->cd_order[prec] = cd->cd_ordp++;
1118 * Reset cd_qualp to the highest precedence level that we've seen so
1119 * far that can be qualified (CTF_PREC_BASE or CTF_PREC_POINTER).
1121 if (prec > cd->cd_qualp && prec < CTF_PREC_ARRAY)
1122 cd->cd_qualp = prec;
1125 * C array declarators are ordered inside out so prepend them. Also by
1126 * convention qualifiers of base types precede the type specifier (e.g.
1127 * const int vs. int const) even though the two forms are equivalent.
1129 if (kind == CTF_K_ARRAY || (is_qual && prec == CTF_PREC_BASE))
1130 ctf_list_prepend(&cd->cd_nodes[prec], cdp);
1132 ctf_list_append(&cd->cd_nodes[prec], cdp);
1136 ctf_decl_sprintf(ctf_decl_t *cd, const char *format, ...)
1138 size_t len = (size_t)(cd->cd_end - cd->cd_ptr);
1142 va_start(ap, format);
1143 n = vsnprintf(cd->cd_ptr, len, format, ap);
1146 cd->cd_ptr += MIN(n, len);
1151 fbt_type_name(linker_ctf_t *lc, ctf_id_t type, char *buf, size_t len)
1154 ctf_decl_node_t *cdp;
1155 ctf_decl_prec_t prec, lp, rp;
1159 if (lc == NULL && type == CTF_ERR)
1160 return (-1); /* simplify caller code by permitting CTF_ERR */
1162 ctf_decl_init(&cd, buf, len);
1163 ctf_decl_push(&cd, lc, type);
1165 if (cd.cd_err != 0) {
1171 * If the type graph's order conflicts with lexical precedence order
1172 * for pointers or arrays, then we need to surround the declarations at
1173 * the corresponding lexical precedence with parentheses. This can
1174 * result in either a parenthesized pointer (*) as in int (*)() or
1175 * int (*)[], or in a parenthesized pointer and array as in int (*[])().
1177 ptr = cd.cd_order[CTF_PREC_POINTER] > CTF_PREC_POINTER;
1178 arr = cd.cd_order[CTF_PREC_ARRAY] > CTF_PREC_ARRAY;
1180 rp = arr ? CTF_PREC_ARRAY : ptr ? CTF_PREC_POINTER : -1;
1181 lp = ptr ? CTF_PREC_POINTER : arr ? CTF_PREC_ARRAY : -1;
1183 k = CTF_K_POINTER; /* avoid leading whitespace (see below) */
1185 for (prec = CTF_PREC_BASE; prec < CTF_PREC_MAX; prec++) {
1186 for (cdp = ctf_list_next(&cd.cd_nodes[prec]);
1187 cdp != NULL; cdp = ctf_list_next(cdp)) {
1189 const ctf_type_t *tp =
1190 ctf_lookup_by_id(lc, cdp->cd_type);
1191 const char *name = ctf_strptr(lc, tp->ctt_name);
1193 if (k != CTF_K_POINTER && k != CTF_K_ARRAY)
1194 ctf_decl_sprintf(&cd, " ");
1197 ctf_decl_sprintf(&cd, "(");
1201 switch (cdp->cd_kind) {
1205 ctf_decl_sprintf(&cd, "%s", name);
1208 ctf_decl_sprintf(&cd, "*");
1211 ctf_decl_sprintf(&cd, "[%u]", cdp->cd_n);
1213 case CTF_K_FUNCTION:
1214 ctf_decl_sprintf(&cd, "()");
1218 ctf_decl_sprintf(&cd, "struct %s", name);
1221 ctf_decl_sprintf(&cd, "union %s", name);
1224 ctf_decl_sprintf(&cd, "enum %s", name);
1226 case CTF_K_VOLATILE:
1227 ctf_decl_sprintf(&cd, "volatile");
1230 ctf_decl_sprintf(&cd, "const");
1232 case CTF_K_RESTRICT:
1233 ctf_decl_sprintf(&cd, "restrict");
1241 ctf_decl_sprintf(&cd, ")");
1249 fbt_getargdesc(void *arg __unused, dtrace_id_t id __unused, void *parg, dtrace_argdesc_t *desc)
1252 fbt_probe_t *fbt = parg;
1254 modctl_t *ctl = fbt->fbtp_ctl;
1255 int ndx = desc->dtargd_ndx;
1256 int symindx = fbt->fbtp_symindx;
1259 ushort_t info, kind, n;
1261 desc->dtargd_ndx = DTRACE_ARGNONE;
1263 /* Get a pointer to the CTF data and it's length. */
1264 if (linker_ctf_get(ctl, &lc) != 0)
1265 /* No CTF data? Something wrong? *shrug* */
1268 /* Check if this module hasn't been initialised yet. */
1269 if (*lc.ctfoffp == NULL) {
1271 * Initialise the CTF object and function symindx to
1272 * byte offset array.
1274 if (fbt_ctfoff_init(ctl, &lc) != 0)
1277 /* Initialise the CTF type to byte offset array. */
1278 if (fbt_typoff_init(&lc) != 0)
1282 ctfoff = *lc.ctfoffp;
1284 if (ctfoff == NULL || *lc.typoffp == NULL)
1287 /* Check if the symbol index is out of range. */
1288 if (symindx >= lc.nsym)
1291 /* Check if the symbol isn't cross-referenced. */
1292 if ((offset = ctfoff[symindx]) == 0xffffffff)
1295 dp = (const ushort_t *)(lc.ctftab + offset + sizeof(ctf_header_t));
1298 kind = CTF_INFO_KIND(info);
1299 n = CTF_INFO_VLEN(info);
1301 if (kind == CTF_K_UNKNOWN && n == 0) {
1302 printf("%s(%d): Unknown function!\n",__func__,__LINE__);
1306 if (kind != CTF_K_FUNCTION) {
1307 printf("%s(%d): Expected a function!\n",__func__,__LINE__);
1311 /* Check if the requested argument doesn't exist. */
1315 /* Skip the return type and arguments up to the one requested. */
1318 if (fbt_type_name(&lc, *dp, desc->dtargd_native, sizeof(desc->dtargd_native)) > 0)
1319 desc->dtargd_ndx = ndx;
1325 fbt_load(void *dummy)
1327 /* Create the /dev/dtrace/fbt entry. */
1328 fbt_cdev = make_dev(&fbt_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600,
1331 /* Default the probe table size if not specified. */
1332 if (fbt_probetab_size == 0)
1333 fbt_probetab_size = FBT_PROBETAB_SIZE;
1335 /* Choose the hash mask for the probe table. */
1336 fbt_probetab_mask = fbt_probetab_size - 1;
1338 /* Allocate memory for the probe table. */
1340 malloc(fbt_probetab_size * sizeof (fbt_probe_t *), M_FBT, M_WAITOK | M_ZERO);
1342 dtrace_doubletrap_func = fbt_doubletrap;
1343 dtrace_invop_add(fbt_invop);
1345 if (dtrace_register("fbt", &fbt_attr, DTRACE_PRIV_USER,
1346 NULL, &fbt_pops, NULL, &fbt_id) != 0)
1356 /* De-register the invalid opcode handler. */
1357 dtrace_invop_remove(fbt_invop);
1359 dtrace_doubletrap_func = NULL;
1361 /* De-register this DTrace provider. */
1362 if ((error = dtrace_unregister(fbt_id)) != 0)
1365 /* Free the probe table. */
1366 free(fbt_probetab, M_FBT);
1367 fbt_probetab = NULL;
1368 fbt_probetab_mask = 0;
1370 destroy_dev(fbt_cdev);
1376 fbt_modevent(module_t mod __unused, int type, void *data __unused)
1400 fbt_open(struct cdev *dev __unused, int oflags __unused, int devtype __unused, struct thread *td __unused)
1405 SYSINIT(fbt_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fbt_load, NULL);
1406 SYSUNINIT(fbt_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fbt_unload, NULL);
1408 DEV_MODULE(fbt, fbt_modevent, NULL);
1409 MODULE_VERSION(fbt, 1);
1410 MODULE_DEPEND(fbt, dtrace, 1, 1, 1);
1411 MODULE_DEPEND(fbt, opensolaris, 1, 1, 1);