4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
28 * Copyright (c) 2011, Joyent, Inc. All rights reserved.
40 #include <sys/sysctl.h>
41 #include <libproc_compat.h>
45 #define DTRACE_AHASHSIZE 32779 /* big 'ol prime */
48 * Because qsort(3C) does not allow an argument to be passed to a comparison
49 * function, the variables that affect comparison must regrettably be global;
50 * they are protected by a global static lock, dt_qsort_lock.
52 static pthread_mutex_t dt_qsort_lock = PTHREAD_MUTEX_INITIALIZER;
54 static int dt_revsort;
55 static int dt_keysort;
58 #define DT_LESSTHAN (dt_revsort == 0 ? -1 : 1)
59 #define DT_GREATERTHAN (dt_revsort == 0 ? 1 : -1)
62 dt_aggregate_count(int64_t *existing, int64_t *new, size_t size)
66 for (i = 0; i < size / sizeof (int64_t); i++)
67 existing[i] = existing[i] + new[i];
71 dt_aggregate_countcmp(int64_t *lhs, int64_t *rhs)
80 return (DT_GREATERTHAN);
87 dt_aggregate_min(int64_t *existing, int64_t *new, size_t size)
95 dt_aggregate_max(int64_t *existing, int64_t *new, size_t size)
102 dt_aggregate_averagecmp(int64_t *lhs, int64_t *rhs)
104 int64_t lavg = lhs[0] ? (lhs[1] / lhs[0]) : 0;
105 int64_t ravg = rhs[0] ? (rhs[1] / rhs[0]) : 0;
108 return (DT_LESSTHAN);
111 return (DT_GREATERTHAN);
117 dt_aggregate_stddevcmp(int64_t *lhs, int64_t *rhs)
119 uint64_t lsd = dt_stddev((uint64_t *)lhs, 1);
120 uint64_t rsd = dt_stddev((uint64_t *)rhs, 1);
123 return (DT_LESSTHAN);
126 return (DT_GREATERTHAN);
133 dt_aggregate_lquantize(int64_t *existing, int64_t *new, size_t size)
135 int64_t arg = *existing++;
136 uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg);
139 for (i = 0; i <= levels + 1; i++)
140 existing[i] = existing[i] + new[i + 1];
144 dt_aggregate_lquantizedsum(int64_t *lquanta)
146 int64_t arg = *lquanta++;
147 int32_t base = DTRACE_LQUANTIZE_BASE(arg);
148 uint16_t step = DTRACE_LQUANTIZE_STEP(arg);
149 uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg), i;
150 long double total = (long double)lquanta[0] * (long double)(base - 1);
152 for (i = 0; i < levels; base += step, i++)
153 total += (long double)lquanta[i + 1] * (long double)base;
155 return (total + (long double)lquanta[levels + 1] *
156 (long double)(base + 1));
160 dt_aggregate_lquantizedzero(int64_t *lquanta)
162 int64_t arg = *lquanta++;
163 int32_t base = DTRACE_LQUANTIZE_BASE(arg);
164 uint16_t step = DTRACE_LQUANTIZE_STEP(arg);
165 uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg), i;
170 for (i = 0; i < levels; base += step, i++) {
174 return (lquanta[i + 1]);
178 return (lquanta[levels + 1]);
184 dt_aggregate_lquantizedcmp(int64_t *lhs, int64_t *rhs)
186 long double lsum = dt_aggregate_lquantizedsum(lhs);
187 long double rsum = dt_aggregate_lquantizedsum(rhs);
188 int64_t lzero, rzero;
191 return (DT_LESSTHAN);
194 return (DT_GREATERTHAN);
197 * If they're both equal, then we will compare based on the weights at
198 * zero. If the weights at zero are equal (or if zero is not within
199 * the range of the linear quantization), then this will be judged a
200 * tie and will be resolved based on the key comparison.
202 lzero = dt_aggregate_lquantizedzero(lhs);
203 rzero = dt_aggregate_lquantizedzero(rhs);
206 return (DT_LESSTHAN);
209 return (DT_GREATERTHAN);
215 dt_aggregate_llquantize(int64_t *existing, int64_t *new, size_t size)
219 for (i = 1; i < size / sizeof (int64_t); i++)
220 existing[i] = existing[i] + new[i];
224 dt_aggregate_llquantizedsum(int64_t *llquanta)
226 int64_t arg = *llquanta++;
227 uint16_t factor = DTRACE_LLQUANTIZE_FACTOR(arg);
228 uint16_t low = DTRACE_LLQUANTIZE_LOW(arg);
229 uint16_t high = DTRACE_LLQUANTIZE_HIGH(arg);
230 uint16_t nsteps = DTRACE_LLQUANTIZE_NSTEP(arg);
232 int64_t value = 1, next, step;
235 assert(nsteps >= factor);
236 assert(nsteps % factor == 0);
238 for (order = 0; order < low; order++)
241 total = (long double)llquanta[bin++] * (long double)(value - 1);
243 next = value * factor;
244 step = next > nsteps ? next / nsteps : 1;
246 while (order <= high) {
247 assert(value < next);
248 total += (long double)llquanta[bin++] * (long double)(value);
250 if ((value += step) != next)
253 next = value * factor;
254 step = next > nsteps ? next / nsteps : 1;
258 return (total + (long double)llquanta[bin] * (long double)value);
262 dt_aggregate_llquantizedcmp(int64_t *lhs, int64_t *rhs)
264 long double lsum = dt_aggregate_llquantizedsum(lhs);
265 long double rsum = dt_aggregate_llquantizedsum(rhs);
266 int64_t lzero, rzero;
269 return (DT_LESSTHAN);
272 return (DT_GREATERTHAN);
275 * If they're both equal, then we will compare based on the weights at
276 * zero. If the weights at zero are equal, then this will be judged a
277 * tie and will be resolved based on the key comparison.
283 return (DT_LESSTHAN);
286 return (DT_GREATERTHAN);
292 dt_aggregate_quantizedcmp(int64_t *lhs, int64_t *rhs)
294 int nbuckets = DTRACE_QUANTIZE_NBUCKETS;
295 long double ltotal = 0, rtotal = 0;
296 int64_t lzero, rzero;
299 for (i = 0; i < nbuckets; i++) {
300 int64_t bucketval = DTRACE_QUANTIZE_BUCKETVAL(i);
302 if (bucketval == 0) {
307 ltotal += (long double)bucketval * (long double)lhs[i];
308 rtotal += (long double)bucketval * (long double)rhs[i];
312 return (DT_LESSTHAN);
315 return (DT_GREATERTHAN);
318 * If they're both equal, then we will compare based on the weights at
319 * zero. If the weights at zero are equal, then this will be judged a
320 * tie and will be resolved based on the key comparison.
323 return (DT_LESSTHAN);
326 return (DT_GREATERTHAN);
332 dt_aggregate_usym(dtrace_hdl_t *dtp, uint64_t *data)
334 uint64_t pid = data[0];
335 uint64_t *pc = &data[1];
336 struct ps_prochandle *P;
339 if (dtp->dt_vector != NULL)
342 if ((P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0)) == NULL)
345 dt_proc_lock(dtp, P);
347 if (Plookup_by_addr(P, *pc, NULL, 0, &sym) == 0)
350 dt_proc_unlock(dtp, P);
351 dt_proc_release(dtp, P);
355 dt_aggregate_umod(dtrace_hdl_t *dtp, uint64_t *data)
357 uint64_t pid = data[0];
358 uint64_t *pc = &data[1];
359 struct ps_prochandle *P;
362 if (dtp->dt_vector != NULL)
365 if ((P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0)) == NULL)
368 dt_proc_lock(dtp, P);
370 if ((map = Paddr_to_map(P, *pc)) != NULL)
373 dt_proc_unlock(dtp, P);
374 dt_proc_release(dtp, P);
378 dt_aggregate_sym(dtrace_hdl_t *dtp, uint64_t *data)
383 if (dtrace_lookup_by_addr(dtp, *pc, &sym, NULL) == 0)
388 dt_aggregate_mod(dtrace_hdl_t *dtp, uint64_t *data)
393 if (dtp->dt_vector != NULL) {
395 * We don't have a way of just getting the module for a
396 * vectored open, and it doesn't seem to be worth defining
397 * one. This means that use of mod() won't get true
398 * aggregation in the postmortem case (some modules may
399 * appear more than once in aggregation output). It seems
400 * unlikely that anyone will ever notice or care...
405 for (dmp = dt_list_next(&dtp->dt_modlist); dmp != NULL;
406 dmp = dt_list_next(dmp)) {
407 if (*pc - dmp->dm_text_va < dmp->dm_text_size) {
408 *pc = dmp->dm_text_va;
414 static dtrace_aggvarid_t
415 dt_aggregate_aggvarid(dt_ahashent_t *ent)
417 dtrace_aggdesc_t *agg = ent->dtahe_data.dtada_desc;
418 caddr_t data = ent->dtahe_data.dtada_data;
419 dtrace_recdesc_t *rec = agg->dtagd_rec;
422 * First, we'll check the variable ID in the aggdesc. If it's valid,
423 * we'll return it. If not, we'll use the compiler-generated ID
424 * present as the first record.
426 if (agg->dtagd_varid != DTRACE_AGGVARIDNONE)
427 return (agg->dtagd_varid);
429 agg->dtagd_varid = *((dtrace_aggvarid_t *)(uintptr_t)(data +
432 return (agg->dtagd_varid);
437 dt_aggregate_snap_cpu(dtrace_hdl_t *dtp, processorid_t cpu)
441 size_t offs, roffs, size, ndx;
444 dtrace_recdesc_t *rec;
445 dt_aggregate_t *agp = &dtp->dt_aggregate;
446 dtrace_aggdesc_t *agg;
447 dt_ahash_t *hash = &agp->dtat_hash;
449 dtrace_bufdesc_t b = agp->dtat_buf, *buf = &b;
450 dtrace_aggdata_t *aggdata;
451 int flags = agp->dtat_flags;
456 if (dt_ioctl(dtp, DTRACEIOC_AGGSNAP, buf) == -1) {
458 if (dt_ioctl(dtp, DTRACEIOC_AGGSNAP, &buf) == -1) {
460 if (errno == ENOENT) {
462 * If that failed with ENOENT, it may be because the
463 * CPU was unconfigured. This is okay; we'll just
464 * do nothing but return success.
469 return (dt_set_errno(dtp, errno));
472 if (buf->dtbd_drops != 0) {
473 if (dt_handle_cpudrop(dtp, cpu,
474 DTRACEDROP_AGGREGATION, buf->dtbd_drops) == -1)
478 if (buf->dtbd_size == 0)
481 if (hash->dtah_hash == NULL) {
484 hash->dtah_size = DTRACE_AHASHSIZE;
485 size = hash->dtah_size * sizeof (dt_ahashent_t *);
487 if ((hash->dtah_hash = malloc(size)) == NULL)
488 return (dt_set_errno(dtp, EDT_NOMEM));
490 bzero(hash->dtah_hash, size);
493 for (offs = 0; offs < buf->dtbd_size; ) {
495 * We're guaranteed to have an ID.
497 id = *((dtrace_epid_t *)((uintptr_t)buf->dtbd_data +
500 if (id == DTRACE_AGGIDNONE) {
502 * This is filler to assure proper alignment of the
503 * next record; we simply ignore it.
509 if ((rval = dt_aggid_lookup(dtp, id, &agg)) != 0)
512 addr = buf->dtbd_data + offs;
513 size = agg->dtagd_size;
516 for (j = 0; j < agg->dtagd_nrecs - 1; j++) {
517 rec = &agg->dtagd_rec[j];
518 roffs = rec->dtrd_offset;
520 switch (rec->dtrd_action) {
522 dt_aggregate_usym(dtp,
523 /* LINTED - alignment */
524 (uint64_t *)&addr[roffs]);
528 dt_aggregate_umod(dtp,
529 /* LINTED - alignment */
530 (uint64_t *)&addr[roffs]);
534 /* LINTED - alignment */
535 dt_aggregate_sym(dtp, (uint64_t *)&addr[roffs]);
539 /* LINTED - alignment */
540 dt_aggregate_mod(dtp, (uint64_t *)&addr[roffs]);
547 for (i = 0; i < rec->dtrd_size; i++)
548 hashval += addr[roffs + i];
551 ndx = hashval % hash->dtah_size;
553 for (h = hash->dtah_hash[ndx]; h != NULL; h = h->dtahe_next) {
554 if (h->dtahe_hashval != hashval)
557 if (h->dtahe_size != size)
560 aggdata = &h->dtahe_data;
561 data = aggdata->dtada_data;
563 for (j = 0; j < agg->dtagd_nrecs - 1; j++) {
564 rec = &agg->dtagd_rec[j];
565 roffs = rec->dtrd_offset;
567 for (i = 0; i < rec->dtrd_size; i++)
568 if (addr[roffs + i] != data[roffs + i])
573 * We found it. Now we need to apply the aggregating
574 * action on the data here.
576 rec = &agg->dtagd_rec[agg->dtagd_nrecs - 1];
577 roffs = rec->dtrd_offset;
578 /* LINTED - alignment */
579 h->dtahe_aggregate((int64_t *)&data[roffs],
580 /* LINTED - alignment */
581 (int64_t *)&addr[roffs], rec->dtrd_size);
584 * If we're keeping per CPU data, apply the aggregating
585 * action there as well.
587 if (aggdata->dtada_percpu != NULL) {
588 data = aggdata->dtada_percpu[cpu];
590 /* LINTED - alignment */
591 h->dtahe_aggregate((int64_t *)data,
592 /* LINTED - alignment */
593 (int64_t *)&addr[roffs], rec->dtrd_size);
602 * If we're here, we couldn't find an entry for this record.
604 if ((h = malloc(sizeof (dt_ahashent_t))) == NULL)
605 return (dt_set_errno(dtp, EDT_NOMEM));
606 bzero(h, sizeof (dt_ahashent_t));
607 aggdata = &h->dtahe_data;
609 if ((aggdata->dtada_data = malloc(size)) == NULL) {
611 return (dt_set_errno(dtp, EDT_NOMEM));
614 bcopy(addr, aggdata->dtada_data, size);
615 aggdata->dtada_size = size;
616 aggdata->dtada_desc = agg;
617 aggdata->dtada_handle = dtp;
618 (void) dt_epid_lookup(dtp, agg->dtagd_epid,
619 &aggdata->dtada_edesc, &aggdata->dtada_pdesc);
620 aggdata->dtada_normal = 1;
622 h->dtahe_hashval = hashval;
623 h->dtahe_size = size;
624 (void) dt_aggregate_aggvarid(h);
626 rec = &agg->dtagd_rec[agg->dtagd_nrecs - 1];
628 if (flags & DTRACE_A_PERCPU) {
629 int max_cpus = agp->dtat_maxcpu;
630 caddr_t *percpu = malloc(max_cpus * sizeof (caddr_t));
632 if (percpu == NULL) {
633 free(aggdata->dtada_data);
635 return (dt_set_errno(dtp, EDT_NOMEM));
638 for (j = 0; j < max_cpus; j++) {
639 percpu[j] = malloc(rec->dtrd_size);
641 if (percpu[j] == NULL) {
645 free(aggdata->dtada_data);
647 return (dt_set_errno(dtp, EDT_NOMEM));
651 bcopy(&addr[rec->dtrd_offset],
652 percpu[j], rec->dtrd_size);
654 bzero(percpu[j], rec->dtrd_size);
658 aggdata->dtada_percpu = percpu;
661 switch (rec->dtrd_action) {
663 h->dtahe_aggregate = dt_aggregate_min;
667 h->dtahe_aggregate = dt_aggregate_max;
670 case DTRACEAGG_LQUANTIZE:
671 h->dtahe_aggregate = dt_aggregate_lquantize;
674 case DTRACEAGG_LLQUANTIZE:
675 h->dtahe_aggregate = dt_aggregate_llquantize;
678 case DTRACEAGG_COUNT:
681 case DTRACEAGG_STDDEV:
682 case DTRACEAGG_QUANTIZE:
683 h->dtahe_aggregate = dt_aggregate_count;
687 return (dt_set_errno(dtp, EDT_BADAGG));
690 if (hash->dtah_hash[ndx] != NULL)
691 hash->dtah_hash[ndx]->dtahe_prev = h;
693 h->dtahe_next = hash->dtah_hash[ndx];
694 hash->dtah_hash[ndx] = h;
696 if (hash->dtah_all != NULL)
697 hash->dtah_all->dtahe_prevall = h;
699 h->dtahe_nextall = hash->dtah_all;
702 offs += agg->dtagd_size;
709 dtrace_aggregate_snap(dtrace_hdl_t *dtp)
712 dt_aggregate_t *agp = &dtp->dt_aggregate;
713 hrtime_t now = gethrtime();
714 dtrace_optval_t interval = dtp->dt_options[DTRACEOPT_AGGRATE];
716 if (dtp->dt_lastagg != 0) {
717 if (now - dtp->dt_lastagg < interval)
720 dtp->dt_lastagg += interval;
722 dtp->dt_lastagg = now;
726 return (dt_set_errno(dtp, EINVAL));
728 if (agp->dtat_buf.dtbd_size == 0)
731 for (i = 0; i < agp->dtat_ncpus; i++) {
732 if ((rval = dt_aggregate_snap_cpu(dtp, agp->dtat_cpus[i])))
740 dt_aggregate_hashcmp(const void *lhs, const void *rhs)
742 dt_ahashent_t *lh = *((dt_ahashent_t **)lhs);
743 dt_ahashent_t *rh = *((dt_ahashent_t **)rhs);
744 dtrace_aggdesc_t *lagg = lh->dtahe_data.dtada_desc;
745 dtrace_aggdesc_t *ragg = rh->dtahe_data.dtada_desc;
747 if (lagg->dtagd_nrecs < ragg->dtagd_nrecs)
748 return (DT_LESSTHAN);
750 if (lagg->dtagd_nrecs > ragg->dtagd_nrecs)
751 return (DT_GREATERTHAN);
757 dt_aggregate_varcmp(const void *lhs, const void *rhs)
759 dt_ahashent_t *lh = *((dt_ahashent_t **)lhs);
760 dt_ahashent_t *rh = *((dt_ahashent_t **)rhs);
761 dtrace_aggvarid_t lid, rid;
763 lid = dt_aggregate_aggvarid(lh);
764 rid = dt_aggregate_aggvarid(rh);
767 return (DT_LESSTHAN);
770 return (DT_GREATERTHAN);
776 dt_aggregate_keycmp(const void *lhs, const void *rhs)
778 dt_ahashent_t *lh = *((dt_ahashent_t **)lhs);
779 dt_ahashent_t *rh = *((dt_ahashent_t **)rhs);
780 dtrace_aggdesc_t *lagg = lh->dtahe_data.dtada_desc;
781 dtrace_aggdesc_t *ragg = rh->dtahe_data.dtada_desc;
782 dtrace_recdesc_t *lrec, *rrec;
784 int rval, i, j, keypos, nrecs;
786 if ((rval = dt_aggregate_hashcmp(lhs, rhs)) != 0)
789 nrecs = lagg->dtagd_nrecs - 1;
790 assert(nrecs == ragg->dtagd_nrecs - 1);
792 keypos = dt_keypos + 1 >= nrecs ? 0 : dt_keypos;
794 for (i = 1; i < nrecs; i++) {
796 int ndx = i + keypos;
799 ndx = ndx - nrecs + 1;
801 lrec = &lagg->dtagd_rec[ndx];
802 rrec = &ragg->dtagd_rec[ndx];
804 ldata = lh->dtahe_data.dtada_data + lrec->dtrd_offset;
805 rdata = rh->dtahe_data.dtada_data + rrec->dtrd_offset;
807 if (lrec->dtrd_size < rrec->dtrd_size)
808 return (DT_LESSTHAN);
810 if (lrec->dtrd_size > rrec->dtrd_size)
811 return (DT_GREATERTHAN);
813 switch (lrec->dtrd_size) {
814 case sizeof (uint64_t):
815 /* LINTED - alignment */
816 lval = *((uint64_t *)ldata);
817 /* LINTED - alignment */
818 rval = *((uint64_t *)rdata);
821 case sizeof (uint32_t):
822 /* LINTED - alignment */
823 lval = *((uint32_t *)ldata);
824 /* LINTED - alignment */
825 rval = *((uint32_t *)rdata);
828 case sizeof (uint16_t):
829 /* LINTED - alignment */
830 lval = *((uint16_t *)ldata);
831 /* LINTED - alignment */
832 rval = *((uint16_t *)rdata);
835 case sizeof (uint8_t):
836 lval = *((uint8_t *)ldata);
837 rval = *((uint8_t *)rdata);
841 switch (lrec->dtrd_action) {
843 case DTRACEACT_UADDR:
845 for (j = 0; j < 2; j++) {
846 /* LINTED - alignment */
847 lval = ((uint64_t *)ldata)[j];
848 /* LINTED - alignment */
849 rval = ((uint64_t *)rdata)[j];
852 return (DT_LESSTHAN);
855 return (DT_GREATERTHAN);
861 for (j = 0; j < lrec->dtrd_size; j++) {
862 lval = ((uint8_t *)ldata)[j];
863 rval = ((uint8_t *)rdata)[j];
866 return (DT_LESSTHAN);
869 return (DT_GREATERTHAN);
877 return (DT_LESSTHAN);
880 return (DT_GREATERTHAN);
887 dt_aggregate_valcmp(const void *lhs, const void *rhs)
889 dt_ahashent_t *lh = *((dt_ahashent_t **)lhs);
890 dt_ahashent_t *rh = *((dt_ahashent_t **)rhs);
891 dtrace_aggdesc_t *lagg = lh->dtahe_data.dtada_desc;
892 dtrace_aggdesc_t *ragg = rh->dtahe_data.dtada_desc;
893 caddr_t ldata = lh->dtahe_data.dtada_data;
894 caddr_t rdata = rh->dtahe_data.dtada_data;
895 dtrace_recdesc_t *lrec, *rrec;
896 int64_t *laddr, *raddr;
899 if ((rval = dt_aggregate_hashcmp(lhs, rhs)) != 0)
902 if (lagg->dtagd_nrecs > ragg->dtagd_nrecs)
903 return (DT_GREATERTHAN);
905 if (lagg->dtagd_nrecs < ragg->dtagd_nrecs)
906 return (DT_LESSTHAN);
908 for (i = 0; i < lagg->dtagd_nrecs; i++) {
909 lrec = &lagg->dtagd_rec[i];
910 rrec = &ragg->dtagd_rec[i];
912 if (lrec->dtrd_offset < rrec->dtrd_offset)
913 return (DT_LESSTHAN);
915 if (lrec->dtrd_offset > rrec->dtrd_offset)
916 return (DT_GREATERTHAN);
918 if (lrec->dtrd_action < rrec->dtrd_action)
919 return (DT_LESSTHAN);
921 if (lrec->dtrd_action > rrec->dtrd_action)
922 return (DT_GREATERTHAN);
925 laddr = (int64_t *)(uintptr_t)(ldata + lrec->dtrd_offset);
926 raddr = (int64_t *)(uintptr_t)(rdata + rrec->dtrd_offset);
928 switch (lrec->dtrd_action) {
930 rval = dt_aggregate_averagecmp(laddr, raddr);
933 case DTRACEAGG_STDDEV:
934 rval = dt_aggregate_stddevcmp(laddr, raddr);
937 case DTRACEAGG_QUANTIZE:
938 rval = dt_aggregate_quantizedcmp(laddr, raddr);
941 case DTRACEAGG_LQUANTIZE:
942 rval = dt_aggregate_lquantizedcmp(laddr, raddr);
945 case DTRACEAGG_LLQUANTIZE:
946 rval = dt_aggregate_llquantizedcmp(laddr, raddr);
949 case DTRACEAGG_COUNT:
953 rval = dt_aggregate_countcmp(laddr, raddr);
964 dt_aggregate_valkeycmp(const void *lhs, const void *rhs)
968 if ((rval = dt_aggregate_valcmp(lhs, rhs)) != 0)
972 * If we're here, the values for the two aggregation elements are
973 * equal. We already know that the key layout is the same for the two
974 * elements; we must now compare the keys themselves as a tie-breaker.
976 return (dt_aggregate_keycmp(lhs, rhs));
980 dt_aggregate_keyvarcmp(const void *lhs, const void *rhs)
984 if ((rval = dt_aggregate_keycmp(lhs, rhs)) != 0)
987 return (dt_aggregate_varcmp(lhs, rhs));
991 dt_aggregate_varkeycmp(const void *lhs, const void *rhs)
995 if ((rval = dt_aggregate_varcmp(lhs, rhs)) != 0)
998 return (dt_aggregate_keycmp(lhs, rhs));
1002 dt_aggregate_valvarcmp(const void *lhs, const void *rhs)
1006 if ((rval = dt_aggregate_valkeycmp(lhs, rhs)) != 0)
1009 return (dt_aggregate_varcmp(lhs, rhs));
1013 dt_aggregate_varvalcmp(const void *lhs, const void *rhs)
1017 if ((rval = dt_aggregate_varcmp(lhs, rhs)) != 0)
1020 return (dt_aggregate_valkeycmp(lhs, rhs));
1024 dt_aggregate_keyvarrevcmp(const void *lhs, const void *rhs)
1026 return (dt_aggregate_keyvarcmp(rhs, lhs));
1030 dt_aggregate_varkeyrevcmp(const void *lhs, const void *rhs)
1032 return (dt_aggregate_varkeycmp(rhs, lhs));
1036 dt_aggregate_valvarrevcmp(const void *lhs, const void *rhs)
1038 return (dt_aggregate_valvarcmp(rhs, lhs));
1042 dt_aggregate_varvalrevcmp(const void *lhs, const void *rhs)
1044 return (dt_aggregate_varvalcmp(rhs, lhs));
1048 dt_aggregate_bundlecmp(const void *lhs, const void *rhs)
1050 dt_ahashent_t **lh = *((dt_ahashent_t ***)lhs);
1051 dt_ahashent_t **rh = *((dt_ahashent_t ***)rhs);
1056 * If we're sorting on keys, we need to scan until we find the
1057 * last entry -- that's the representative key. (The order of
1058 * the bundle is values followed by key to accommodate the
1059 * default behavior of sorting by value.) If the keys are
1060 * equal, we'll fall into the value comparison loop, below.
1062 for (i = 0; lh[i + 1] != NULL; i++)
1066 assert(rh[i + 1] == NULL);
1068 if ((rval = dt_aggregate_keycmp(&lh[i], &rh[i])) != 0)
1072 for (i = 0; ; i++) {
1073 if (lh[i + 1] == NULL) {
1075 * All of the values are equal; if we're sorting on
1076 * keys, then we're only here because the keys were
1077 * found to be equal and these records are therefore
1078 * equal. If we're not sorting on keys, we'll use the
1079 * key comparison from the representative key as the
1086 assert(rh[i + 1] == NULL);
1087 return (dt_aggregate_keycmp(&lh[i], &rh[i]));
1089 if ((rval = dt_aggregate_valcmp(&lh[i], &rh[i])) != 0)
1096 dt_aggregate_go(dtrace_hdl_t *dtp)
1098 dt_aggregate_t *agp = &dtp->dt_aggregate;
1099 dtrace_optval_t size, cpu;
1100 dtrace_bufdesc_t *buf = &agp->dtat_buf;
1103 assert(agp->dtat_maxcpu == 0);
1104 assert(agp->dtat_ncpu == 0);
1105 assert(agp->dtat_cpus == NULL);
1107 agp->dtat_maxcpu = dt_sysconf(dtp, _SC_CPUID_MAX) + 1;
1108 agp->dtat_ncpu = dt_sysconf(dtp, _SC_NPROCESSORS_MAX);
1109 agp->dtat_cpus = malloc(agp->dtat_ncpu * sizeof (processorid_t));
1111 if (agp->dtat_cpus == NULL)
1112 return (dt_set_errno(dtp, EDT_NOMEM));
1115 * Use the aggregation buffer size as reloaded from the kernel.
1117 size = dtp->dt_options[DTRACEOPT_AGGSIZE];
1119 rval = dtrace_getopt(dtp, "aggsize", &size);
1122 if (size == 0 || size == DTRACEOPT_UNSET)
1125 buf = &agp->dtat_buf;
1126 buf->dtbd_size = size;
1128 if ((buf->dtbd_data = malloc(buf->dtbd_size)) == NULL)
1129 return (dt_set_errno(dtp, EDT_NOMEM));
1132 * Now query for the CPUs enabled.
1134 rval = dtrace_getopt(dtp, "cpu", &cpu);
1135 assert(rval == 0 && cpu != DTRACEOPT_UNSET);
1137 if (cpu != DTRACE_CPUALL) {
1138 assert(cpu < agp->dtat_ncpu);
1139 agp->dtat_cpus[agp->dtat_ncpus++] = (processorid_t)cpu;
1144 agp->dtat_ncpus = 0;
1145 for (i = 0; i < agp->dtat_maxcpu; i++) {
1146 if (dt_status(dtp, i) == -1)
1149 agp->dtat_cpus[agp->dtat_ncpus++] = i;
1156 dt_aggwalk_rval(dtrace_hdl_t *dtp, dt_ahashent_t *h, int rval)
1158 dt_aggregate_t *agp = &dtp->dt_aggregate;
1159 dtrace_aggdata_t *data;
1160 dtrace_aggdesc_t *aggdesc;
1161 dtrace_recdesc_t *rec;
1165 case DTRACE_AGGWALK_NEXT:
1168 case DTRACE_AGGWALK_CLEAR: {
1169 uint32_t size, offs = 0;
1171 aggdesc = h->dtahe_data.dtada_desc;
1172 rec = &aggdesc->dtagd_rec[aggdesc->dtagd_nrecs - 1];
1173 size = rec->dtrd_size;
1174 data = &h->dtahe_data;
1176 if (rec->dtrd_action == DTRACEAGG_LQUANTIZE) {
1177 offs = sizeof (uint64_t);
1178 size -= sizeof (uint64_t);
1181 bzero(&data->dtada_data[rec->dtrd_offset] + offs, size);
1183 if (data->dtada_percpu == NULL)
1186 for (i = 0; i < dtp->dt_aggregate.dtat_maxcpu; i++)
1187 bzero(data->dtada_percpu[i] + offs, size);
1191 case DTRACE_AGGWALK_ERROR:
1193 * We assume that errno is already set in this case.
1195 return (dt_set_errno(dtp, errno));
1197 case DTRACE_AGGWALK_ABORT:
1198 return (dt_set_errno(dtp, EDT_DIRABORT));
1200 case DTRACE_AGGWALK_DENORMALIZE:
1201 h->dtahe_data.dtada_normal = 1;
1204 case DTRACE_AGGWALK_NORMALIZE:
1205 if (h->dtahe_data.dtada_normal == 0) {
1206 h->dtahe_data.dtada_normal = 1;
1207 return (dt_set_errno(dtp, EDT_BADRVAL));
1212 case DTRACE_AGGWALK_REMOVE: {
1213 dtrace_aggdata_t *aggdata = &h->dtahe_data;
1214 int max_cpus = agp->dtat_maxcpu;
1217 * First, remove this hash entry from its hash chain.
1219 if (h->dtahe_prev != NULL) {
1220 h->dtahe_prev->dtahe_next = h->dtahe_next;
1222 dt_ahash_t *hash = &agp->dtat_hash;
1223 size_t ndx = h->dtahe_hashval % hash->dtah_size;
1225 assert(hash->dtah_hash[ndx] == h);
1226 hash->dtah_hash[ndx] = h->dtahe_next;
1229 if (h->dtahe_next != NULL)
1230 h->dtahe_next->dtahe_prev = h->dtahe_prev;
1233 * Now remove it from the list of all hash entries.
1235 if (h->dtahe_prevall != NULL) {
1236 h->dtahe_prevall->dtahe_nextall = h->dtahe_nextall;
1238 dt_ahash_t *hash = &agp->dtat_hash;
1240 assert(hash->dtah_all == h);
1241 hash->dtah_all = h->dtahe_nextall;
1244 if (h->dtahe_nextall != NULL)
1245 h->dtahe_nextall->dtahe_prevall = h->dtahe_prevall;
1248 * We're unlinked. We can safely destroy the data.
1250 if (aggdata->dtada_percpu != NULL) {
1251 for (i = 0; i < max_cpus; i++)
1252 free(aggdata->dtada_percpu[i]);
1253 free(aggdata->dtada_percpu);
1256 free(aggdata->dtada_data);
1263 return (dt_set_errno(dtp, EDT_BADRVAL));
1270 dt_aggregate_qsort(dtrace_hdl_t *dtp, void *base, size_t nel, size_t width,
1271 int (*compar)(const void *, const void *))
1273 int rev = dt_revsort, key = dt_keysort, keypos = dt_keypos;
1274 dtrace_optval_t keyposopt = dtp->dt_options[DTRACEOPT_AGGSORTKEYPOS];
1276 dt_revsort = (dtp->dt_options[DTRACEOPT_AGGSORTREV] != DTRACEOPT_UNSET);
1277 dt_keysort = (dtp->dt_options[DTRACEOPT_AGGSORTKEY] != DTRACEOPT_UNSET);
1279 if (keyposopt != DTRACEOPT_UNSET && keyposopt <= INT_MAX) {
1280 dt_keypos = (int)keyposopt;
1285 if (compar == NULL) {
1287 compar = dt_aggregate_varvalcmp;
1289 compar = dt_aggregate_varkeycmp;
1293 qsort(base, nel, width, compar);
1301 dtrace_aggregate_walk(dtrace_hdl_t *dtp, dtrace_aggregate_f *func, void *arg)
1303 dt_ahashent_t *h, *next;
1304 dt_ahash_t *hash = &dtp->dt_aggregate.dtat_hash;
1306 for (h = hash->dtah_all; h != NULL; h = next) {
1308 * dt_aggwalk_rval() can potentially remove the current hash
1309 * entry; we need to load the next hash entry before calling
1312 next = h->dtahe_nextall;
1314 if (dt_aggwalk_rval(dtp, h, func(&h->dtahe_data, arg)) == -1)
1322 dt_aggregate_walk_sorted(dtrace_hdl_t *dtp,
1323 dtrace_aggregate_f *func, void *arg,
1324 int (*sfunc)(const void *, const void *))
1326 dt_aggregate_t *agp = &dtp->dt_aggregate;
1327 dt_ahashent_t *h, **sorted;
1328 dt_ahash_t *hash = &agp->dtat_hash;
1329 size_t i, nentries = 0;
1331 for (h = hash->dtah_all; h != NULL; h = h->dtahe_nextall)
1334 sorted = dt_alloc(dtp, nentries * sizeof (dt_ahashent_t *));
1339 for (h = hash->dtah_all, i = 0; h != NULL; h = h->dtahe_nextall)
1342 (void) pthread_mutex_lock(&dt_qsort_lock);
1344 if (sfunc == NULL) {
1345 dt_aggregate_qsort(dtp, sorted, nentries,
1346 sizeof (dt_ahashent_t *), NULL);
1349 * If we've been explicitly passed a sorting function,
1350 * we'll use that -- ignoring the values of the "aggsortrev",
1351 * "aggsortkey" and "aggsortkeypos" options.
1353 qsort(sorted, nentries, sizeof (dt_ahashent_t *), sfunc);
1356 (void) pthread_mutex_unlock(&dt_qsort_lock);
1358 for (i = 0; i < nentries; i++) {
1361 if (dt_aggwalk_rval(dtp, h, func(&h->dtahe_data, arg)) == -1) {
1362 dt_free(dtp, sorted);
1367 dt_free(dtp, sorted);
1372 dtrace_aggregate_walk_sorted(dtrace_hdl_t *dtp,
1373 dtrace_aggregate_f *func, void *arg)
1375 return (dt_aggregate_walk_sorted(dtp, func, arg, NULL));
1379 dtrace_aggregate_walk_keysorted(dtrace_hdl_t *dtp,
1380 dtrace_aggregate_f *func, void *arg)
1382 return (dt_aggregate_walk_sorted(dtp, func,
1383 arg, dt_aggregate_varkeycmp));
1387 dtrace_aggregate_walk_valsorted(dtrace_hdl_t *dtp,
1388 dtrace_aggregate_f *func, void *arg)
1390 return (dt_aggregate_walk_sorted(dtp, func,
1391 arg, dt_aggregate_varvalcmp));
1395 dtrace_aggregate_walk_keyvarsorted(dtrace_hdl_t *dtp,
1396 dtrace_aggregate_f *func, void *arg)
1398 return (dt_aggregate_walk_sorted(dtp, func,
1399 arg, dt_aggregate_keyvarcmp));
1403 dtrace_aggregate_walk_valvarsorted(dtrace_hdl_t *dtp,
1404 dtrace_aggregate_f *func, void *arg)
1406 return (dt_aggregate_walk_sorted(dtp, func,
1407 arg, dt_aggregate_valvarcmp));
1411 dtrace_aggregate_walk_keyrevsorted(dtrace_hdl_t *dtp,
1412 dtrace_aggregate_f *func, void *arg)
1414 return (dt_aggregate_walk_sorted(dtp, func,
1415 arg, dt_aggregate_varkeyrevcmp));
1419 dtrace_aggregate_walk_valrevsorted(dtrace_hdl_t *dtp,
1420 dtrace_aggregate_f *func, void *arg)
1422 return (dt_aggregate_walk_sorted(dtp, func,
1423 arg, dt_aggregate_varvalrevcmp));
1427 dtrace_aggregate_walk_keyvarrevsorted(dtrace_hdl_t *dtp,
1428 dtrace_aggregate_f *func, void *arg)
1430 return (dt_aggregate_walk_sorted(dtp, func,
1431 arg, dt_aggregate_keyvarrevcmp));
1435 dtrace_aggregate_walk_valvarrevsorted(dtrace_hdl_t *dtp,
1436 dtrace_aggregate_f *func, void *arg)
1438 return (dt_aggregate_walk_sorted(dtp, func,
1439 arg, dt_aggregate_valvarrevcmp));
1443 dtrace_aggregate_walk_joined(dtrace_hdl_t *dtp, dtrace_aggvarid_t *aggvars,
1444 int naggvars, dtrace_aggregate_walk_joined_f *func, void *arg)
1446 dt_aggregate_t *agp = &dtp->dt_aggregate;
1447 dt_ahashent_t *h, **sorted = NULL, ***bundle, **nbundle;
1448 const dtrace_aggdata_t **data;
1449 dt_ahashent_t *zaggdata = NULL;
1450 dt_ahash_t *hash = &agp->dtat_hash;
1451 size_t nentries = 0, nbundles = 0, start, zsize = 0, bundlesize;
1452 dtrace_aggvarid_t max = 0, aggvar;
1453 int rval = -1, *map, *remap = NULL;
1455 dtrace_optval_t sortpos = dtp->dt_options[DTRACEOPT_AGGSORTPOS];
1458 * If the sorting position is greater than the number of aggregation
1459 * variable IDs, we silently set it to 0.
1461 if (sortpos == DTRACEOPT_UNSET || sortpos >= naggvars)
1465 * First we need to translate the specified aggregation variable IDs
1466 * into a linear map that will allow us to translate an aggregation
1467 * variable ID into its position in the specified aggvars.
1469 for (i = 0; i < naggvars; i++) {
1470 if (aggvars[i] == DTRACE_AGGVARIDNONE || aggvars[i] < 0)
1471 return (dt_set_errno(dtp, EDT_BADAGGVAR));
1473 if (aggvars[i] > max)
1477 if ((map = dt_zalloc(dtp, (max + 1) * sizeof (int))) == NULL)
1480 zaggdata = dt_zalloc(dtp, naggvars * sizeof (dt_ahashent_t));
1482 if (zaggdata == NULL)
1485 for (i = 0; i < naggvars; i++) {
1486 int ndx = i + sortpos;
1488 if (ndx >= naggvars)
1491 aggvar = aggvars[ndx];
1492 assert(aggvar <= max);
1496 * We have an aggregation variable that is present
1497 * more than once in the array of aggregation
1498 * variables. While it's unclear why one might want
1499 * to do this, it's legal. To support this construct,
1500 * we will allocate a remap that will indicate the
1501 * position from which this aggregation variable
1502 * should be pulled. (That is, where the remap will
1503 * map from one position to another.)
1505 if (remap == NULL) {
1506 remap = dt_zalloc(dtp, naggvars * sizeof (int));
1513 * Given that the variable is already present, assert
1514 * that following through the mapping and adjusting
1515 * for the sort position yields the same aggregation
1518 assert(aggvars[(map[aggvar] - 1 + sortpos) %
1519 naggvars] == aggvars[ndx]);
1521 remap[i] = map[aggvar];
1525 map[aggvar] = i + 1;
1529 * We need to take two passes over the data to size our allocation, so
1530 * we'll use the first pass to also fill in the zero-filled data to be
1531 * used to properly format a zero-valued aggregation.
1533 for (h = hash->dtah_all; h != NULL; h = h->dtahe_nextall) {
1534 dtrace_aggvarid_t id;
1537 if ((id = dt_aggregate_aggvarid(h)) > max || !(ndx = map[id]))
1540 if (zaggdata[ndx - 1].dtahe_size == 0) {
1541 zaggdata[ndx - 1].dtahe_size = h->dtahe_size;
1542 zaggdata[ndx - 1].dtahe_data = h->dtahe_data;
1548 if (nentries == 0) {
1550 * We couldn't find any entries; there is nothing else to do.
1557 * Before we sort the data, we're going to look for any holes in our
1558 * zero-filled data. This will occur if an aggregation variable that
1559 * we are being asked to print has not yet been assigned the result of
1560 * any aggregating action for _any_ tuple. The issue becomes that we
1561 * would like a zero value to be printed for all columns for this
1562 * aggregation, but without any record description, we don't know the
1563 * aggregating action that corresponds to the aggregation variable. To
1564 * try to find a match, we're simply going to lookup aggregation IDs
1565 * (which are guaranteed to be contiguous and to start from 1), looking
1566 * for the specified aggregation variable ID. If we find a match,
1567 * we'll use that. If we iterate over all aggregation IDs and don't
1568 * find a match, then we must be an anonymous enabling. (Anonymous
1569 * enablings can't currently derive either aggregation variable IDs or
1570 * aggregation variable names given only an aggregation ID.) In this
1571 * obscure case (anonymous enabling, multiple aggregation printa() with
1572 * some aggregations not represented for any tuple), our defined
1573 * behavior is that the zero will be printed in the format of the first
1574 * aggregation variable that contains any non-zero value.
1576 for (i = 0; i < naggvars; i++) {
1577 if (zaggdata[i].dtahe_size == 0) {
1578 dtrace_aggvarid_t aggvar;
1580 aggvar = aggvars[(i - sortpos + naggvars) % naggvars];
1581 assert(zaggdata[i].dtahe_data.dtada_data == NULL);
1583 for (j = DTRACE_AGGIDNONE + 1; ; j++) {
1584 dtrace_aggdesc_t *agg;
1585 dtrace_aggdata_t *aggdata;
1587 if (dt_aggid_lookup(dtp, j, &agg) != 0)
1590 if (agg->dtagd_varid != aggvar)
1594 * We have our description -- now we need to
1595 * cons up the zaggdata entry for it.
1597 aggdata = &zaggdata[i].dtahe_data;
1598 aggdata->dtada_size = agg->dtagd_size;
1599 aggdata->dtada_desc = agg;
1600 aggdata->dtada_handle = dtp;
1601 (void) dt_epid_lookup(dtp, agg->dtagd_epid,
1602 &aggdata->dtada_edesc,
1603 &aggdata->dtada_pdesc);
1604 aggdata->dtada_normal = 1;
1605 zaggdata[i].dtahe_hashval = 0;
1606 zaggdata[i].dtahe_size = agg->dtagd_size;
1610 if (zaggdata[i].dtahe_size == 0) {
1614 * We couldn't find this aggregation, meaning
1615 * that we have never seen it before for any
1616 * tuple _and_ this is an anonymous enabling.
1617 * That is, we're in the obscure case outlined
1618 * above. In this case, our defined behavior
1619 * is to format the data in the format of the
1620 * first non-zero aggregation -- of which, of
1621 * course, we know there to be at least one
1622 * (or nentries would have been zero).
1624 for (j = 0; j < naggvars; j++) {
1625 if (zaggdata[j].dtahe_size != 0)
1629 assert(j < naggvars);
1630 zaggdata[i] = zaggdata[j];
1632 data = zaggdata[i].dtahe_data.dtada_data;
1633 assert(data != NULL);
1639 * Now we need to allocate our zero-filled data for use for
1640 * aggregations that don't have a value corresponding to a given key.
1642 for (i = 0; i < naggvars; i++) {
1643 dtrace_aggdata_t *aggdata = &zaggdata[i].dtahe_data;
1644 dtrace_aggdesc_t *aggdesc = aggdata->dtada_desc;
1645 dtrace_recdesc_t *rec;
1649 zsize = zaggdata[i].dtahe_size;
1652 if ((zdata = dt_zalloc(dtp, zsize)) == NULL) {
1654 * If we failed to allocated some zero-filled data, we
1655 * need to zero out the remaining dtada_data pointers
1656 * to prevent the wrong data from being freed below.
1658 for (j = i; j < naggvars; j++)
1659 zaggdata[j].dtahe_data.dtada_data = NULL;
1663 aggvar = aggvars[(i - sortpos + naggvars) % naggvars];
1666 * First, the easy bit. To maintain compatibility with
1667 * consumers that pull the compiler-generated ID out of the
1668 * data, we put that ID at the top of the zero-filled data.
1670 rec = &aggdesc->dtagd_rec[0];
1671 /* LINTED - alignment */
1672 *((dtrace_aggvarid_t *)(zdata + rec->dtrd_offset)) = aggvar;
1674 rec = &aggdesc->dtagd_rec[aggdesc->dtagd_nrecs - 1];
1677 * Now for the more complicated part. If (and only if) this
1678 * is an lquantize() aggregating action, zero-filled data is
1679 * not equivalent to an empty record: we must also get the
1680 * parameters for the lquantize().
1682 if (rec->dtrd_action == DTRACEAGG_LQUANTIZE) {
1683 if (aggdata->dtada_data != NULL) {
1685 * The easier case here is if we actually have
1686 * some prototype data -- in which case we
1687 * manually dig it out of the aggregation
1690 /* LINTED - alignment */
1691 larg = *((uint64_t *)(aggdata->dtada_data +
1695 * We don't have any prototype data. As a
1696 * result, we know that we _do_ have the
1697 * compiler-generated information. (If this
1698 * were an anonymous enabling, all of our
1699 * zero-filled data would have prototype data
1700 * -- either directly or indirectly.) So as
1701 * gross as it is, we'll grovel around in the
1702 * compiler-generated information to find the
1703 * lquantize() parameters.
1705 dtrace_stmtdesc_t *sdp;
1709 sdp = (dtrace_stmtdesc_t *)(uintptr_t)
1710 aggdesc->dtagd_rec[0].dtrd_uarg;
1711 aid = sdp->dtsd_aggdata;
1712 isp = (dt_idsig_t *)aid->di_data;
1713 assert(isp->dis_auxinfo != 0);
1714 larg = isp->dis_auxinfo;
1717 /* LINTED - alignment */
1718 *((uint64_t *)(zdata + rec->dtrd_offset)) = larg;
1721 aggdata->dtada_data = zdata;
1725 * Now that we've dealt with setting up our zero-filled data, we can
1726 * allocate our sorted array, and take another pass over the data to
1729 sorted = dt_alloc(dtp, nentries * sizeof (dt_ahashent_t *));
1734 for (h = hash->dtah_all, i = 0; h != NULL; h = h->dtahe_nextall) {
1735 dtrace_aggvarid_t id;
1737 if ((id = dt_aggregate_aggvarid(h)) > max || !map[id])
1743 assert(i == nentries);
1746 * We've loaded our array; now we need to sort by value to allow us
1747 * to create bundles of like value. We're going to acquire the
1748 * dt_qsort_lock here, and hold it across all of our subsequent
1749 * comparison and sorting.
1751 (void) pthread_mutex_lock(&dt_qsort_lock);
1753 qsort(sorted, nentries, sizeof (dt_ahashent_t *),
1754 dt_aggregate_keyvarcmp);
1757 * Now we need to go through and create bundles. Because the number
1758 * of bundles is bounded by the size of the sorted array, we're going
1759 * to reuse the underlying storage. And note that "bundle" is an
1760 * array of pointers to arrays of pointers to dt_ahashent_t -- making
1761 * its type (regrettably) "dt_ahashent_t ***". (Regrettable because
1762 * '*' -- like '_' and 'X' -- should never appear in triplicate in
1765 bundle = (dt_ahashent_t ***)sorted;
1767 for (i = 1, start = 0; i <= nentries; i++) {
1769 dt_aggregate_keycmp(&sorted[i], &sorted[i - 1]) == 0)
1773 * We have a bundle boundary. Everything from start to
1774 * (i - 1) belongs in one bundle.
1776 assert(i - start <= naggvars);
1777 bundlesize = (naggvars + 2) * sizeof (dt_ahashent_t *);
1779 if ((nbundle = dt_zalloc(dtp, bundlesize)) == NULL) {
1780 (void) pthread_mutex_unlock(&dt_qsort_lock);
1784 for (j = start; j < i; j++) {
1785 dtrace_aggvarid_t id = dt_aggregate_aggvarid(sorted[j]);
1788 assert(map[id] != 0);
1789 assert(map[id] - 1 < naggvars);
1790 assert(nbundle[map[id] - 1] == NULL);
1791 nbundle[map[id] - 1] = sorted[j];
1793 if (nbundle[naggvars] == NULL)
1794 nbundle[naggvars] = sorted[j];
1797 for (j = 0; j < naggvars; j++) {
1798 if (nbundle[j] != NULL)
1802 * Before we assume that this aggregation variable
1803 * isn't present (and fall back to using the
1804 * zero-filled data allocated earlier), check the
1805 * remap. If we have a remapping, we'll drop it in
1806 * here. Note that we might be remapping an
1807 * aggregation variable that isn't present for this
1808 * key; in this case, the aggregation data that we
1809 * copy will point to the zeroed data.
1811 if (remap != NULL && remap[j]) {
1812 assert(remap[j] - 1 < j);
1813 assert(nbundle[remap[j] - 1] != NULL);
1814 nbundle[j] = nbundle[remap[j] - 1];
1816 nbundle[j] = &zaggdata[j];
1820 bundle[nbundles++] = nbundle;
1825 * Now we need to re-sort based on the first value.
1827 dt_aggregate_qsort(dtp, bundle, nbundles, sizeof (dt_ahashent_t **),
1828 dt_aggregate_bundlecmp);
1830 (void) pthread_mutex_unlock(&dt_qsort_lock);
1833 * We're done! Now we just need to go back over the sorted bundles,
1834 * calling the function.
1836 data = alloca((naggvars + 1) * sizeof (dtrace_aggdata_t *));
1838 for (i = 0; i < nbundles; i++) {
1839 for (j = 0; j < naggvars; j++)
1842 for (j = 0; j < naggvars; j++) {
1843 int ndx = j - sortpos;
1848 assert(bundle[i][ndx] != NULL);
1849 data[j + 1] = &bundle[i][ndx]->dtahe_data;
1852 for (j = 0; j < naggvars; j++)
1853 assert(data[j + 1] != NULL);
1856 * The representative key is the last element in the bundle.
1857 * Assert that we have one, and then set it to be the first
1860 assert(bundle[i][j] != NULL);
1861 data[0] = &bundle[i][j]->dtahe_data;
1863 if ((rval = func(data, naggvars + 1, arg)) == -1)
1869 for (i = 0; i < nbundles; i++)
1870 dt_free(dtp, bundle[i]);
1872 if (zaggdata != NULL) {
1873 for (i = 0; i < naggvars; i++)
1874 dt_free(dtp, zaggdata[i].dtahe_data.dtada_data);
1877 dt_free(dtp, zaggdata);
1878 dt_free(dtp, sorted);
1879 dt_free(dtp, remap);
1886 dtrace_aggregate_print(dtrace_hdl_t *dtp, FILE *fp,
1887 dtrace_aggregate_walk_f *func)
1889 dt_print_aggdata_t pd;
1893 pd.dtpa_allunprint = 1;
1896 func = dtrace_aggregate_walk_sorted;
1898 if ((*func)(dtp, dt_print_agg, &pd) == -1)
1899 return (dt_set_errno(dtp, dtp->dt_errno));
1905 dtrace_aggregate_clear(dtrace_hdl_t *dtp)
1907 dt_aggregate_t *agp = &dtp->dt_aggregate;
1908 dt_ahash_t *hash = &agp->dtat_hash;
1910 dtrace_aggdata_t *data;
1911 dtrace_aggdesc_t *aggdesc;
1912 dtrace_recdesc_t *rec;
1913 int i, max_cpus = agp->dtat_maxcpu;
1915 for (h = hash->dtah_all; h != NULL; h = h->dtahe_nextall) {
1916 aggdesc = h->dtahe_data.dtada_desc;
1917 rec = &aggdesc->dtagd_rec[aggdesc->dtagd_nrecs - 1];
1918 data = &h->dtahe_data;
1920 bzero(&data->dtada_data[rec->dtrd_offset], rec->dtrd_size);
1922 if (data->dtada_percpu == NULL)
1925 for (i = 0; i < max_cpus; i++)
1926 bzero(data->dtada_percpu[i], rec->dtrd_size);
1931 dt_aggregate_destroy(dtrace_hdl_t *dtp)
1933 dt_aggregate_t *agp = &dtp->dt_aggregate;
1934 dt_ahash_t *hash = &agp->dtat_hash;
1935 dt_ahashent_t *h, *next;
1936 dtrace_aggdata_t *aggdata;
1937 int i, max_cpus = agp->dtat_maxcpu;
1939 if (hash->dtah_hash == NULL) {
1940 assert(hash->dtah_all == NULL);
1942 free(hash->dtah_hash);
1944 for (h = hash->dtah_all; h != NULL; h = next) {
1945 next = h->dtahe_nextall;
1947 aggdata = &h->dtahe_data;
1949 if (aggdata->dtada_percpu != NULL) {
1950 for (i = 0; i < max_cpus; i++)
1951 free(aggdata->dtada_percpu[i]);
1952 free(aggdata->dtada_percpu);
1955 free(aggdata->dtada_data);
1959 hash->dtah_hash = NULL;
1960 hash->dtah_all = NULL;
1961 hash->dtah_size = 0;
1964 free(agp->dtat_buf.dtbd_data);
1965 free(agp->dtat_cpus);