cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25
  26 /*
  27  * Copyright (c) 2011, Joyent, Inc. All rights reserved.
  28  * Copyright (c) 2012 by Delphix. All rights reserved.
  29  */
  30
  31 #include <stdlib.h>
  32 #include <strings.h>
  33 #include <errno.h>
  34 #include <unistd.h>
  35 #include <limits.h>
  36 #include <assert.h>
  37 #include <ctype.h>
  38 #if defined(sun)
  39 #include <alloca.h>
  40 #endif
  41 #include <dt_impl.h>
  42 #include <dt_pq.h>
  43 #if !defined(sun)
  44 #include <libproc_compat.h>
  45 #endif
  46
  47 #define DT_MASK_LO 0x00000000FFFFFFFFULL
  48
  49 /*
  50  * We declare this here because (1) we need it and (2) we want to avoid a
  51  * dependency on libm in libdtrace.
  52  */
  53 static long double
  54 dt_fabsl(long double x)
  55 {
  56         if (x < 0)
  57                 return (-x);
  58
  59         return (x);
  60 }
  61
  62 /*
  63  * 128-bit arithmetic functions needed to support the stddev() aggregating
  64  * action.
  65  */
  66 static int
  67 dt_gt_128(uint64_t *a, uint64_t *b)
  68 {
  69         return (a[1] > b[1] || (a[1] == b[1] && a[0] > b[0]));
  70 }
  71
  72 static int
  73 dt_ge_128(uint64_t *a, uint64_t *b)
  74 {
  75         return (a[1] > b[1] || (a[1] == b[1] && a[0] >= b[0]));
  76 }
  77
  78 static int
  79 dt_le_128(uint64_t *a, uint64_t *b)
  80 {
  81         return (a[1] < b[1] || (a[1] == b[1] && a[0] <= b[0]));
  82 }
  83
  84 /*
  85  * Shift the 128-bit value in a by b. If b is positive, shift left.
  86  * If b is negative, shift right.
  87  */
  88 static void
  89 dt_shift_128(uint64_t *a, int b)
  90 {
  91         uint64_t mask;
  92
  93         if (b == 0)
  94                 return;
  95
  96         if (b < 0) {
  97                 b = -b;
  98                 if (b >= 64) {
  99                         a[0] = a[1] >> (b - 64);
 100                         a[1] = 0;
 101                 } else {
 102                         a[0] >>= b;
 103                         mask = 1LL << (64 - b);
 104                         mask -= 1;
 105                         a[0] |= ((a[1] & mask) << (64 - b));
 106                         a[1] >>= b;
 107                 }
 108         } else {
 109                 if (b >= 64) {
 110                         a[1] = a[0] << (b - 64);
 111                         a[0] = 0;
 112                 } else {
 113                         a[1] <<= b;
 114                         mask = a[0] >> (64 - b);
 115                         a[1] |= mask;
 116                         a[0] <<= b;
 117                 }
 118         }
 119 }
 120
 121 static int
 122 dt_nbits_128(uint64_t *a)
 123 {
 124         int nbits = 0;
 125         uint64_t tmp[2];
 126         uint64_t zero[2] = { 0, 0 };
 127
 128         tmp[0] = a[0];
 129         tmp[1] = a[1];
 130
 131         dt_shift_128(tmp, -1);
 132         while (dt_gt_128(tmp, zero)) {
 133                 dt_shift_128(tmp, -1);
 134                 nbits++;
 135         }
 136
 137         return (nbits);
 138 }
 139
 140 static void
 141 dt_subtract_128(uint64_t *minuend, uint64_t *subtrahend, uint64_t *difference)
 142 {
 143         uint64_t result[2];
 144
 145         result[0] = minuend[0] - subtrahend[0];
 146         result[1] = minuend[1] - subtrahend[1] -
 147             (minuend[0] < subtrahend[0] ? 1 : 0);
 148
 149         difference[0] = result[0];
 150         difference[1] = result[1];
 151 }
 152
 153 static void
 154 dt_add_128(uint64_t *addend1, uint64_t *addend2, uint64_t *sum)
 155 {
 156         uint64_t result[2];
 157
 158         result[0] = addend1[0] + addend2[0];
 159         result[1] = addend1[1] + addend2[1] +
 160             (result[0] < addend1[0] || result[0] < addend2[0] ? 1 : 0);
 161
 162         sum[0] = result[0];
 163         sum[1] = result[1];
 164 }
 165
 166 /*
 167  * The basic idea is to break the 2 64-bit values into 4 32-bit values,
 168  * use native multiplication on those, and then re-combine into the
 169  * resulting 128-bit value.
 170  *
 171  * (hi1 << 32 + lo1) * (hi2 << 32 + lo2) =
 172  *     hi1 * hi2 << 64 +
 173  *     hi1 * lo2 << 32 +
 174  *     hi2 * lo1 << 32 +
 175  *     lo1 * lo2
 176  */
 177 static void
 178 dt_multiply_128(uint64_t factor1, uint64_t factor2, uint64_t *product)
 179 {
 180         uint64_t hi1, hi2, lo1, lo2;
 181         uint64_t tmp[2];
 182
 183         hi1 = factor1 >> 32;
 184         hi2 = factor2 >> 32;
 185
 186         lo1 = factor1 & DT_MASK_LO;
 187         lo2 = factor2 & DT_MASK_LO;
 188
 189         product[0] = lo1 * lo2;
 190         product[1] = hi1 * hi2;
 191
 192         tmp[0] = hi1 * lo2;
 193         tmp[1] = 0;
 194         dt_shift_128(tmp, 32);
 195         dt_add_128(product, tmp, product);
 196
 197         tmp[0] = hi2 * lo1;
 198         tmp[1] = 0;
 199         dt_shift_128(tmp, 32);
 200         dt_add_128(product, tmp, product);
 201 }
 202
 203 /*
 204  * This is long-hand division.
 205  *
 206  * We initialize subtrahend by shifting divisor left as far as possible. We
 207  * loop, comparing subtrahend to dividend:  if subtrahend is smaller, we
 208  * subtract and set the appropriate bit in the result.  We then shift
 209  * subtrahend right by one bit for the next comparison.
 210  */
 211 static void
 212 dt_divide_128(uint64_t *dividend, uint64_t divisor, uint64_t *quotient)
 213 {
 214         uint64_t result[2] = { 0, 0 };
 215         uint64_t remainder[2];
 216         uint64_t subtrahend[2];
 217         uint64_t divisor_128[2];
 218         uint64_t mask[2] = { 1, 0 };
 219         int log = 0;
 220
 221         assert(divisor != 0);
 222
 223         divisor_128[0] = divisor;
 224         divisor_128[1] = 0;
 225
 226         remainder[0] = dividend[0];
 227         remainder[1] = dividend[1];
 228
 229         subtrahend[0] = divisor;
 230         subtrahend[1] = 0;
 231
 232         while (divisor > 0) {
 233                 log++;
 234                 divisor >>= 1;
 235         }
 236
 237         dt_shift_128(subtrahend, 128 - log);
 238         dt_shift_128(mask, 128 - log);
 239
 240         while (dt_ge_128(remainder, divisor_128)) {
 241                 if (dt_ge_128(remainder, subtrahend)) {
 242                         dt_subtract_128(remainder, subtrahend, remainder);
 243                         result[0] |= mask[0];
 244                         result[1] |= mask[1];
 245                 }
 246
 247                 dt_shift_128(subtrahend, -1);
 248                 dt_shift_128(mask, -1);
 249         }
 250
 251         quotient[0] = result[0];
 252         quotient[1] = result[1];
 253 }
 254
 255 /*
 256  * This is the long-hand method of calculating a square root.
 257  * The algorithm is as follows:
 258  *
 259  * 1. Group the digits by 2 from the right.
 260  * 2. Over the leftmost group, find the largest single-digit number
 261  *    whose square is less than that group.
 262  * 3. Subtract the result of the previous step (2 or 4, depending) and
 263  *    bring down the next two-digit group.
 264  * 4. For the result R we have so far, find the largest single-digit number
 265  *    x such that 2 * R * 10 * x + x^2 is less than the result from step 3.
 266  *    (Note that this is doubling R and performing a decimal left-shift by 1
 267  *    and searching for the appropriate decimal to fill the one's place.)
 268  *    The value x is the next digit in the square root.
 269  * Repeat steps 3 and 4 until the desired precision is reached.  (We're
 270  * dealing with integers, so the above is sufficient.)
 271  *
 272  * In decimal, the square root of 582,734 would be calculated as so:
 273  *
 274  *     __7__6__3
 275  *    | 58 27 34
 276  *     -49       (7^2 == 49 => 7 is the first digit in the square root)
 277  *      --
 278  *       9 27    (Subtract and bring down the next group.)
 279  * 146   8 76    (2 * 7 * 10 * 6 + 6^2 == 876 => 6 is the next digit in
 280  *      -----     the square root)
 281  *         51 34 (Subtract and bring down the next group.)
 282  * 1523    45 69 (2 * 76 * 10 * 3 + 3^2 == 4569 => 3 is the next digit in
 283  *         -----  the square root)
 284  *          5 65 (remainder)
 285  *
 286  * The above algorithm applies similarly in binary, but note that the
 287  * only possible non-zero value for x in step 4 is 1, so step 4 becomes a
 288  * simple decision: is 2 * R * 2 * 1 + 1^2 (aka R << 2 + 1) less than the
 289  * preceding difference?
 290  *
 291  * In binary, the square root of 11011011 would be calculated as so:
 292  *
 293  *     __1__1__1__0
 294  *    | 11 01 10 11
 295  *      01          (0 << 2 + 1 == 1 < 11 => this bit is 1)
 296  *      --
 297  *      10 01 10 11
 298  * 101   1 01       (1 << 2 + 1 == 101 < 1001 => next bit is 1)
 299  *      -----
 300  *       1 00 10 11
 301  * 1101    11 01    (11 << 2 + 1 == 1101 < 10010 => next bit is 1)
 302  *       -------
 303  *          1 01 11
 304  * 11101    1 11 01 (111 << 2 + 1 == 11101 > 10111 => last bit is 0)
 305  *
 306  */
 307 static uint64_t
 308 dt_sqrt_128(uint64_t *square)
 309 {
 310         uint64_t result[2] = { 0, 0 };
 311         uint64_t diff[2] = { 0, 0 };
 312         uint64_t one[2] = { 1, 0 };
 313         uint64_t next_pair[2];
 314         uint64_t next_try[2];
 315         uint64_t bit_pairs, pair_shift;
 316         int i;
 317
 318         bit_pairs = dt_nbits_128(square) / 2;
 319         pair_shift = bit_pairs * 2;
 320
 321         for (i = 0; i <= bit_pairs; i++) {
 322                 /*
 323                  * Bring down the next pair of bits.
 324                  */
 325                 next_pair[0] = square[0];
 326                 next_pair[1] = square[1];
 327                 dt_shift_128(next_pair, -pair_shift);
 328                 next_pair[0] &= 0x3;
 329                 next_pair[1] = 0;
 330
 331                 dt_shift_128(diff, 2);
 332                 dt_add_128(diff, next_pair, diff);
 333
 334                 /*
 335                  * next_try = R << 2 + 1
 336                  */
 337                 next_try[0] = result[0];
 338                 next_try[1] = result[1];
 339                 dt_shift_128(next_try, 2);
 340                 dt_add_128(next_try, one, next_try);
 341
 342                 if (dt_le_128(next_try, diff)) {
 343                         dt_subtract_128(diff, next_try, diff);
 344                         dt_shift_128(result, 1);
 345                         dt_add_128(result, one, result);
 346                 } else {
 347                         dt_shift_128(result, 1);
 348                 }
 349
 350                 pair_shift -= 2;
 351         }
 352
 353         assert(result[1] == 0);
 354
 355         return (result[0]);
 356 }
 357
 358 uint64_t
 359 dt_stddev(uint64_t *data, uint64_t normal)
 360 {
 361         uint64_t avg_of_squares[2];
 362         uint64_t square_of_avg[2];
 363         int64_t norm_avg;
 364         uint64_t diff[2];
 365
 366         /*
 367          * The standard approximation for standard deviation is
 368          * sqrt(average(x**2) - average(x)**2), i.e. the square root
 369          * of the average of the squares minus the square of the average.
 370          */
 371         dt_divide_128(data + 2, normal, avg_of_squares);
 372         dt_divide_128(avg_of_squares, data[0], avg_of_squares);
 373
 374         norm_avg = (int64_t)data[1] / (int64_t)normal / (int64_t)data[0];
 375
 376         if (norm_avg < 0)
 377                 norm_avg = -norm_avg;
 378
 379         dt_multiply_128((uint64_t)norm_avg, (uint64_t)norm_avg, square_of_avg);
 380
 381         dt_subtract_128(avg_of_squares, square_of_avg, diff);
 382
 383         return (dt_sqrt_128(diff));
 384 }
 385
 386 static int
 387 dt_flowindent(dtrace_hdl_t *dtp, dtrace_probedata_t *data, dtrace_epid_t last,
 388     dtrace_bufdesc_t *buf, size_t offs)
 389 {
 390         dtrace_probedesc_t *pd = data->dtpda_pdesc, *npd;
 391         dtrace_eprobedesc_t *epd = data->dtpda_edesc, *nepd;
 392         char *p = pd->dtpd_provider, *n = pd->dtpd_name, *sub;
 393         dtrace_flowkind_t flow = DTRACEFLOW_NONE;
 394         const char *str = NULL;
 395         static const char *e_str[2] = { " -> ", " => " };
 396         static const char *r_str[2] = { " <- ", " <= " };
 397         static const char *ent = "entry", *ret = "return";
 398         static int entlen = 0, retlen = 0;
 399         dtrace_epid_t next, id = epd->dtepd_epid;
 400         int rval;
 401
 402         if (entlen == 0) {
 403                 assert(retlen == 0);
 404                 entlen = strlen(ent);
 405                 retlen = strlen(ret);
 406         }
 407
 408         /*
 409          * If the name of the probe is "entry" or ends with "-entry", we
 410          * treat it as an entry; if it is "return" or ends with "-return",
 411          * we treat it as a return.  (This allows application-provided probes
 412          * like "method-entry" or "function-entry" to participate in flow
 413          * indentation -- without accidentally misinterpreting popular probe
 414          * names like "carpentry", "gentry" or "Coventry".)
 415          */
 416         if ((sub = strstr(n, ent)) != NULL && sub[entlen] == '\0' &&
 417             (sub == n || sub[-1] == '-')) {
 418                 flow = DTRACEFLOW_ENTRY;
 419                 str = e_str[strcmp(p, "syscall") == 0];
 420         } else if ((sub = strstr(n, ret)) != NULL && sub[retlen] == '\0' &&
 421             (sub == n || sub[-1] == '-')) {
 422                 flow = DTRACEFLOW_RETURN;
 423                 str = r_str[strcmp(p, "syscall") == 0];
 424         }
 425
 426         /*
 427          * If we're going to indent this, we need to check the ID of our last
 428          * call.  If we're looking at the same probe ID but a different EPID,
 429          * we _don't_ want to indent.  (Yes, there are some minor holes in
 430          * this scheme -- it's a heuristic.)
 431          */
 432         if (flow == DTRACEFLOW_ENTRY) {
 433                 if ((last != DTRACE_EPIDNONE && id != last &&
 434                     pd->dtpd_id == dtp->dt_pdesc[last]->dtpd_id))
 435                         flow = DTRACEFLOW_NONE;
 436         }
 437
 438         /*
 439          * If we're going to unindent this, it's more difficult to see if
 440          * we don't actually want to unindent it -- we need to look at the
 441          * _next_ EPID.
 442          */
 443         if (flow == DTRACEFLOW_RETURN) {
 444                 offs += epd->dtepd_size;
 445
 446                 do {
 447                         if (offs >= buf->dtbd_size)
 448                                 goto out;
 449
 450                         next = *(uint32_t *)((uintptr_t)buf->dtbd_data + offs);
 451
 452                         if (next == DTRACE_EPIDNONE)
 453                                 offs += sizeof (id);
 454                 } while (next == DTRACE_EPIDNONE);
 455
 456                 if ((rval = dt_epid_lookup(dtp, next, &nepd, &npd)) != 0)
 457                         return (rval);
 458
 459                 if (next != id && npd->dtpd_id == pd->dtpd_id)
 460                         flow = DTRACEFLOW_NONE;
 461         }
 462
 463 out:
 464         if (flow == DTRACEFLOW_ENTRY || flow == DTRACEFLOW_RETURN) {
 465                 data->dtpda_prefix = str;
 466         } else {
 467                 data->dtpda_prefix = "| ";
 468         }
 469
 470         if (flow == DTRACEFLOW_RETURN && data->dtpda_indent > 0)
 471                 data->dtpda_indent -= 2;
 472
 473         data->dtpda_flow = flow;
 474
 475         return (0);
 476 }
 477
 478 static int
 479 dt_nullprobe()
 480 {
 481         return (DTRACE_CONSUME_THIS);
 482 }
 483
 484 static int
 485 dt_nullrec()
 486 {
 487         return (DTRACE_CONSUME_NEXT);
 488 }
 489
 490 int
 491 dt_print_quantline(dtrace_hdl_t *dtp, FILE *fp, int64_t val,
 492     uint64_t normal, long double total, char positives, char negatives)
 493 {
 494         long double f;
 495         uint_t depth, len = 40;
 496
 497         const char *ats = "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@";
 498         const char *spaces = "                                        ";
 499
 500         assert(strlen(ats) == len && strlen(spaces) == len);
 501         assert(!(total == 0 && (positives || negatives)));
 502         assert(!(val < 0 && !negatives));
 503         assert(!(val > 0 && !positives));
 504         assert(!(val != 0 && total == 0));
 505
 506         if (!negatives) {
 507                 if (positives) {
 508                         f = (dt_fabsl((long double)val) * len) / total;
 509                         depth = (uint_t)(f + 0.5);
 510                 } else {
 511                         depth = 0;
 512                 }
 513
 514                 return (dt_printf(dtp, fp, "|%s%s %-9lld\n", ats + len - depth,
 515                     spaces + depth, (long long)val / normal));
 516         }
 517
 518         if (!positives) {
 519                 f = (dt_fabsl((long double)val) * len) / total;
 520                 depth = (uint_t)(f + 0.5);
 521
 522                 return (dt_printf(dtp, fp, "%s%s| %-9lld\n", spaces + depth,
 523                     ats + len - depth, (long long)val / normal));
 524         }
 525
 526         /*
 527          * If we're here, we have both positive and negative bucket values.
 528          * To express this graphically, we're going to generate both positive
 529          * and negative bars separated by a centerline.  These bars are half
 530          * the size of normal quantize()/lquantize() bars, so we divide the
 531          * length in half before calculating the bar length.
 532          */
 533         len /= 2;
 534         ats = &ats[len];
 535         spaces = &spaces[len];
 536
 537         f = (dt_fabsl((long double)val) * len) / total;
 538         depth = (uint_t)(f + 0.5);
 539
 540         if (val <= 0) {
 541                 return (dt_printf(dtp, fp, "%s%s|%*s %-9lld\n", spaces + depth,
 542                     ats + len - depth, len, "", (long long)val / normal));
 543         } else {
 544                 return (dt_printf(dtp, fp, "%20s|%s%s %-9lld\n", "",
 545                     ats + len - depth, spaces + depth,
 546                     (long long)val / normal));
 547         }
 548 }
 549
 550 int
 551 dt_print_quantize(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
 552     size_t size, uint64_t normal)
 553 {
 554         const int64_t *data = addr;
 555         int i, first_bin = 0, last_bin = DTRACE_QUANTIZE_NBUCKETS - 1;
 556         long double total = 0;
 557         char positives = 0, negatives = 0;
 558
 559         if (size != DTRACE_QUANTIZE_NBUCKETS * sizeof (uint64_t))
 560                 return (dt_set_errno(dtp, EDT_DMISMATCH));
 561
 562         while (first_bin < DTRACE_QUANTIZE_NBUCKETS - 1 && data[first_bin] == 0)
 563                 first_bin++;
 564
 565         if (first_bin == DTRACE_QUANTIZE_NBUCKETS - 1) {
 566                 /*
 567                  * There isn't any data.  This is possible if (and only if)
 568                  * negative increment values have been used.  In this case,
 569                  * we'll print the buckets around 0.
 570                  */
 571                 first_bin = DTRACE_QUANTIZE_ZEROBUCKET - 1;
 572                 last_bin = DTRACE_QUANTIZE_ZEROBUCKET + 1;
 573         } else {
 574                 if (first_bin > 0)
 575                         first_bin--;
 576
 577                 while (last_bin > 0 && data[last_bin] == 0)
 578                         last_bin--;
 579
 580                 if (last_bin < DTRACE_QUANTIZE_NBUCKETS - 1)
 581                         last_bin++;
 582         }
 583
 584         for (i = first_bin; i <= last_bin; i++) {
 585                 positives |= (data[i] > 0);
 586                 negatives |= (data[i] < 0);
 587                 total += dt_fabsl((long double)data[i]);
 588         }
 589
 590         if (dt_printf(dtp, fp, "\n%16s %41s %-9s\n", "value",
 591             "------------- Distribution -------------", "count") < 0)
 592                 return (-1);
 593
 594         for (i = first_bin; i <= last_bin; i++) {
 595                 if (dt_printf(dtp, fp, "%16lld ",
 596                     (long long)DTRACE_QUANTIZE_BUCKETVAL(i)) < 0)
 597                         return (-1);
 598
 599                 if (dt_print_quantline(dtp, fp, data[i], normal, total,
 600                     positives, negatives) < 0)
 601                         return (-1);
 602         }
 603
 604         return (0);
 605 }
 606
 607 int
 608 dt_print_lquantize(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
 609     size_t size, uint64_t normal)
 610 {
 611         const int64_t *data = addr;
 612         int i, first_bin, last_bin, base;
 613         uint64_t arg;
 614         long double total = 0;
 615         uint16_t step, levels;
 616         char positives = 0, negatives = 0;
 617
 618         if (size < sizeof (uint64_t))
 619                 return (dt_set_errno(dtp, EDT_DMISMATCH));
 620
 621         arg = *data++;
 622         size -= sizeof (uint64_t);
 623
 624         base = DTRACE_LQUANTIZE_BASE(arg);
 625         step = DTRACE_LQUANTIZE_STEP(arg);
 626         levels = DTRACE_LQUANTIZE_LEVELS(arg);
 627
 628         first_bin = 0;
 629         last_bin = levels + 1;
 630
 631         if (size != sizeof (uint64_t) * (levels + 2))
 632                 return (dt_set_errno(dtp, EDT_DMISMATCH));
 633
 634         while (first_bin <= levels + 1 && data[first_bin] == 0)
 635                 first_bin++;
 636
 637         if (first_bin > levels + 1) {
 638                 first_bin = 0;
 639                 last_bin = 2;
 640         } else {
 641                 if (first_bin > 0)
 642                         first_bin--;
 643
 644                 while (last_bin > 0 && data[last_bin] == 0)
 645                         last_bin--;
 646
 647                 if (last_bin < levels + 1)
 648                         last_bin++;
 649         }
 650
 651         for (i = first_bin; i <= last_bin; i++) {
 652                 positives |= (data[i] > 0);
 653                 negatives |= (data[i] < 0);
 654                 total += dt_fabsl((long double)data[i]);
 655         }
 656
 657         if (dt_printf(dtp, fp, "\n%16s %41s %-9s\n", "value",
 658             "------------- Distribution -------------", "count") < 0)
 659                 return (-1);
 660
 661         for (i = first_bin; i <= last_bin; i++) {
 662                 char c[32];
 663                 int err;
 664
 665                 if (i == 0) {
 666                         (void) snprintf(c, sizeof (c), "< %d",
 667                             base / (uint32_t)normal);
 668                         err = dt_printf(dtp, fp, "%16s ", c);
 669                 } else if (i == levels + 1) {
 670                         (void) snprintf(c, sizeof (c), ">= %d",
 671                             base + (levels * step));
 672                         err = dt_printf(dtp, fp, "%16s ", c);
 673                 } else {
 674                         err = dt_printf(dtp, fp, "%16d ",
 675                             base + (i - 1) * step);
 676                 }
 677
 678                 if (err < 0 || dt_print_quantline(dtp, fp, data[i], normal,
 679                     total, positives, negatives) < 0)
 680                         return (-1);
 681         }
 682
 683         return (0);
 684 }
 685
 686 int
 687 dt_print_llquantize(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
 688     size_t size, uint64_t normal)
 689 {
 690         int i, first_bin, last_bin, bin = 1, order, levels;
 691         uint16_t factor, low, high, nsteps;
 692         const int64_t *data = addr;
 693         int64_t value = 1, next, step;
 694         char positives = 0, negatives = 0;
 695         long double total = 0;
 696         uint64_t arg;
 697         char c[32];
 698
 699         if (size < sizeof (uint64_t))
 700                 return (dt_set_errno(dtp, EDT_DMISMATCH));
 701
 702         arg = *data++;
 703         size -= sizeof (uint64_t);
 704
 705         factor = DTRACE_LLQUANTIZE_FACTOR(arg);
 706         low = DTRACE_LLQUANTIZE_LOW(arg);
 707         high = DTRACE_LLQUANTIZE_HIGH(arg);
 708         nsteps = DTRACE_LLQUANTIZE_NSTEP(arg);
 709
 710         /*
 711          * We don't expect to be handed invalid llquantize() parameters here,
 712          * but sanity check them (to a degree) nonetheless.
 713          */
 714         if (size > INT32_MAX || factor < 2 || low >= high ||
 715             nsteps == 0 || factor > nsteps)
 716                 return (dt_set_errno(dtp, EDT_DMISMATCH));
 717
 718         levels = (int)size / sizeof (uint64_t);
 719
 720         first_bin = 0;
 721         last_bin = levels - 1;
 722
 723         while (first_bin < levels && data[first_bin] == 0)
 724                 first_bin++;
 725
 726         if (first_bin == levels) {
 727                 first_bin = 0;
 728                 last_bin = 1;
 729         } else {
 730                 if (first_bin > 0)
 731                         first_bin--;
 732
 733                 while (last_bin > 0 && data[last_bin] == 0)
 734                         last_bin--;
 735
 736                 if (last_bin < levels - 1)
 737                         last_bin++;
 738         }
 739
 740         for (i = first_bin; i <= last_bin; i++) {
 741                 positives |= (data[i] > 0);
 742                 negatives |= (data[i] < 0);
 743                 total += dt_fabsl((long double)data[i]);
 744         }
 745
 746         if (dt_printf(dtp, fp, "\n%16s %41s %-9s\n", "value",
 747             "------------- Distribution -------------", "count") < 0)
 748                 return (-1);
 749
 750         for (order = 0; order < low; order++)
 751                 value *= factor;
 752
 753         next = value * factor;
 754         step = next > nsteps ? next / nsteps : 1;
 755
 756         if (first_bin == 0) {
 757                 (void) snprintf(c, sizeof (c), "< %lld", (long long)value);
 758
 759                 if (dt_printf(dtp, fp, "%16s ", c) < 0)
 760                         return (-1);
 761
 762                 if (dt_print_quantline(dtp, fp, data[0], normal,
 763                     total, positives, negatives) < 0)
 764                         return (-1);
 765         }
 766
 767         while (order <= high) {
 768                 if (bin >= first_bin && bin <= last_bin) {
 769                         if (dt_printf(dtp, fp, "%16lld ", (long long)value) < 0)
 770                                 return (-1);
 771
 772                         if (dt_print_quantline(dtp, fp, data[bin],
 773                             normal, total, positives, negatives) < 0)
 774                                 return (-1);
 775                 }
 776
 777                 assert(value < next);
 778                 bin++;
 779
 780                 if ((value += step) != next)
 781                         continue;
 782
 783                 next = value * factor;
 784                 step = next > nsteps ? next / nsteps : 1;
 785                 order++;
 786         }
 787
 788         if (last_bin < bin)
 789                 return (0);
 790
 791         assert(last_bin == bin);
 792         (void) snprintf(c, sizeof (c), ">= %lld", (long long)value);
 793
 794         if (dt_printf(dtp, fp, "%16s ", c) < 0)
 795                 return (-1);
 796
 797         return (dt_print_quantline(dtp, fp, data[bin], normal,
 798             total, positives, negatives));
 799 }
 800
 801 /*ARGSUSED*/
 802 static int
 803 dt_print_average(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr,
 804     size_t size, uint64_t normal)
 805 {
 806         /* LINTED - alignment */
 807         int64_t *data = (int64_t *)addr;
 808
 809         return (dt_printf(dtp, fp, " %16lld", data[0] ?
 810             (long long)(data[1] / (int64_t)normal / data[0]) : 0));
 811 }
 812
 813 /*ARGSUSED*/
 814 static int
 815 dt_print_stddev(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr,
 816     size_t size, uint64_t normal)
 817 {
 818         /* LINTED - alignment */
 819         uint64_t *data = (uint64_t *)addr;
 820
 821         return (dt_printf(dtp, fp, " %16llu", data[0] ?
 822             (unsigned long long) dt_stddev(data, normal) : 0));
 823 }
 824
 825 /*ARGSUSED*/
 826 int
 827 dt_print_bytes(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr,
 828     size_t nbytes, int width, int quiet, int forceraw)
 829 {
 830         /*
 831          * If the byte stream is a series of printable characters, followed by
 832          * a terminating byte, we print it out as a string.  Otherwise, we
 833          * assume that it's something else and just print the bytes.
 834          */
 835         int i, j, margin = 5;
 836         char *c = (char *)addr;
 837
 838         if (nbytes == 0)
 839                 return (0);
 840
 841         if (forceraw)
 842                 goto raw;
 843
 844         if (dtp->dt_options[DTRACEOPT_RAWBYTES] != DTRACEOPT_UNSET)
 845                 goto raw;
 846
 847         for (i = 0; i < nbytes; i++) {
 848                 /*
 849                  * We define a "printable character" to be one for which
 850                  * isprint(3C) returns non-zero, isspace(3C) returns non-zero,
 851                  * or a character which is either backspace or the bell.
 852                  * Backspace and the bell are regrettably special because
 853                  * they fail the first two tests -- and yet they are entirely
 854                  * printable.  These are the only two control characters that
 855                  * have meaning for the terminal and for which isprint(3C) and
 856                  * isspace(3C) return 0.
 857                  */
 858                 if (isprint(c[i]) || isspace(c[i]) ||
 859                     c[i] == '\b' || c[i] == '\a')
 860                         continue;
 861
 862                 if (c[i] == '\0' && i > 0) {
 863                         /*
 864                          * This looks like it might be a string.  Before we
 865                          * assume that it is indeed a string, check the
 866                          * remainder of the byte range; if it contains
 867                          * additional non-nul characters, we'll assume that
 868                          * it's a binary stream that just happens to look like
 869                          * a string, and we'll print out the individual bytes.
 870                          */
 871                         for (j = i + 1; j < nbytes; j++) {
 872                                 if (c[j] != '\0')
 873                                         break;
 874                         }
 875
 876                         if (j != nbytes)
 877                                 break;
 878
 879                         if (quiet)
 880                                 return (dt_printf(dtp, fp, "%s", c));
 881                         else
 882                                 return (dt_printf(dtp, fp, "  %-*s", width, c));
 883                 }
 884
 885                 break;
 886         }
 887
 888         if (i == nbytes) {
 889                 /*
 890                  * The byte range is all printable characters, but there is
 891                  * no trailing nul byte.  We'll assume that it's a string and
 892                  * print it as such.
 893                  */
 894                 char *s = alloca(nbytes + 1);
 895                 bcopy(c, s, nbytes);
 896                 s[nbytes] = '\0';
 897                 return (dt_printf(dtp, fp, "  %-*s", width, s));
 898         }
 899
 900 raw:
 901         if (dt_printf(dtp, fp, "\n%*s      ", margin, "") < 0)
 902                 return (-1);
 903
 904         for (i = 0; i < 16; i++)
 905                 if (dt_printf(dtp, fp, "  %c", "0123456789abcdef"[i]) < 0)
 906                         return (-1);
 907
 908         if (dt_printf(dtp, fp, "  0123456789abcdef\n") < 0)
 909                 return (-1);
 910
 911
 912         for (i = 0; i < nbytes; i += 16) {
 913                 if (dt_printf(dtp, fp, "%*s%5x:", margin, "", i) < 0)
 914                         return (-1);
 915
 916                 for (j = i; j < i + 16 && j < nbytes; j++) {
 917                         if (dt_printf(dtp, fp, " %02x", (uchar_t)c[j]) < 0)
 918                                 return (-1);
 919                 }
 920
 921                 while (j++ % 16) {
 922                         if (dt_printf(dtp, fp, "   ") < 0)
 923                                 return (-1);
 924                 }
 925
 926                 if (dt_printf(dtp, fp, "  ") < 0)
 927                         return (-1);
 928
 929                 for (j = i; j < i + 16 && j < nbytes; j++) {
 930                         if (dt_printf(dtp, fp, "%c",
 931                             c[j] < ' ' || c[j] > '~' ? '.' : c[j]) < 0)
 932                                 return (-1);
 933                 }
 934
 935                 if (dt_printf(dtp, fp, "\n") < 0)
 936                         return (-1);
 937         }
 938
 939         return (0);
 940 }
 941
 942 int
 943 dt_print_stack(dtrace_hdl_t *dtp, FILE *fp, const char *format,
 944     caddr_t addr, int depth, int size)
 945 {
 946         dtrace_syminfo_t dts;
 947         GElf_Sym sym;
 948         int i, indent;
 949         char c[PATH_MAX * 2];
 950         uint64_t pc;
 951
 952         if (dt_printf(dtp, fp, "\n") < 0)
 953                 return (-1);
 954
 955         if (format == NULL)
 956                 format = "%s";
 957
 958         if (dtp->dt_options[DTRACEOPT_STACKINDENT] != DTRACEOPT_UNSET)
 959                 indent = (int)dtp->dt_options[DTRACEOPT_STACKINDENT];
 960         else
 961                 indent = _dtrace_stkindent;
 962
 963         for (i = 0; i < depth; i++) {
 964                 switch (size) {
 965                 case sizeof (uint32_t):
 966                         /* LINTED - alignment */
 967                         pc = *((uint32_t *)addr);
 968                         break;
 969
 970                 case sizeof (uint64_t):
 971                         /* LINTED - alignment */
 972                         pc = *((uint64_t *)addr);
 973                         break;
 974
 975                 default:
 976                         return (dt_set_errno(dtp, EDT_BADSTACKPC));
 977                 }
 978
 979                 if (pc == 0)
 980                         break;
 981
 982                 addr += size;
 983
 984                 if (dt_printf(dtp, fp, "%*s", indent, "") < 0)
 985                         return (-1);
 986
 987                 if (dtrace_lookup_by_addr(dtp, pc, &sym, &dts) == 0) {
 988                         if (pc > sym.st_value) {
 989                                 (void) snprintf(c, sizeof (c), "%s`%s+0x%llx",
 990                                     dts.dts_object, dts.dts_name,
 991                                     (u_longlong_t)(pc - sym.st_value));
 992                         } else {
 993                                 (void) snprintf(c, sizeof (c), "%s`%s",
 994                                     dts.dts_object, dts.dts_name);
 995                         }
 996                 } else {
 997                         /*
 998                          * We'll repeat the lookup, but this time we'll specify
 999                          * a NULL GElf_Sym -- indicating that we're only
1000                          * interested in the containing module.
1001                          */
1002                         if (dtrace_lookup_by_addr(dtp, pc, NULL, &dts) == 0) {
1003                                 (void) snprintf(c, sizeof (c), "%s`0x%llx",
1004                                     dts.dts_object, (u_longlong_t)pc);
1005                         } else {
1006                                 (void) snprintf(c, sizeof (c), "0x%llx",
1007                                     (u_longlong_t)pc);
1008                         }
1009                 }
1010
1011                 if (dt_printf(dtp, fp, format, c) < 0)
1012                         return (-1);
1013
1014                 if (dt_printf(dtp, fp, "\n") < 0)
1015                         return (-1);
1016         }
1017
1018         return (0);
1019 }
1020
1021 int
1022 dt_print_ustack(dtrace_hdl_t *dtp, FILE *fp, const char *format,
1023     caddr_t addr, uint64_t arg)
1024 {
1025         /* LINTED - alignment */
1026         uint64_t *pc = (uint64_t *)addr;
1027         uint32_t depth = DTRACE_USTACK_NFRAMES(arg);
1028         uint32_t strsize = DTRACE_USTACK_STRSIZE(arg);
1029         const char *strbase = addr + (depth + 1) * sizeof (uint64_t);
1030         const char *str = strsize ? strbase : NULL;
1031         int err = 0;
1032
1033         char name[PATH_MAX], objname[PATH_MAX], c[PATH_MAX * 2];
1034         struct ps_prochandle *P;
1035         GElf_Sym sym;
1036         int i, indent;
1037         pid_t pid;
1038
1039         if (depth == 0)
1040                 return (0);
1041
1042         pid = (pid_t)*pc++;
1043
1044         if (dt_printf(dtp, fp, "\n") < 0)
1045                 return (-1);
1046
1047         if (format == NULL)
1048                 format = "%s";
1049
1050         if (dtp->dt_options[DTRACEOPT_STACKINDENT] != DTRACEOPT_UNSET)
1051                 indent = (int)dtp->dt_options[DTRACEOPT_STACKINDENT];
1052         else
1053                 indent = _dtrace_stkindent;
1054
1055         /*
1056          * Ultimately, we need to add an entry point in the library vector for
1057          * determining <symbol, offset> from <pid, address>.  For now, if
1058          * this is a vector open, we just print the raw address or string.
1059          */
1060         if (dtp->dt_vector == NULL)
1061                 P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0);
1062         else
1063                 P = NULL;
1064
1065         if (P != NULL)
1066                 dt_proc_lock(dtp, P); /* lock handle while we perform lookups */
1067
1068         for (i = 0; i < depth && pc[i] != 0; i++) {
1069                 const prmap_t *map;
1070
1071                 if ((err = dt_printf(dtp, fp, "%*s", indent, "")) < 0)
1072                         break;
1073
1074                 if (P != NULL && Plookup_by_addr(P, pc[i],
1075                     name, sizeof (name), &sym) == 0) {
1076                         (void) Pobjname(P, pc[i], objname, sizeof (objname));
1077
1078                         if (pc[i] > sym.st_value) {
1079                                 (void) snprintf(c, sizeof (c),
1080                                     "%s`%s+0x%llx", dt_basename(objname), name,
1081                                     (u_longlong_t)(pc[i] - sym.st_value));
1082                         } else {
1083                                 (void) snprintf(c, sizeof (c),
1084                                     "%s`%s", dt_basename(objname), name);
1085                         }
1086                 } else if (str != NULL && str[0] != '\0' && str[0] != '@' &&
1087                     (P != NULL && ((map = Paddr_to_map(P, pc[i])) == NULL ||
1088                     (map->pr_mflags & MA_WRITE)))) {
1089                         /*
1090                          * If the current string pointer in the string table
1091                          * does not point to an empty string _and_ the program
1092                          * counter falls in a writable region, we'll use the
1093                          * string from the string table instead of the raw
1094                          * address.  This last condition is necessary because
1095                          * some (broken) ustack helpers will return a string
1096                          * even for a program counter that they can't
1097                          * identify.  If we have a string for a program
1098                          * counter that falls in a segment that isn't
1099                          * writable, we assume that we have fallen into this
1100                          * case and we refuse to use the string.
1101                          */
1102                         (void) snprintf(c, sizeof (c), "%s", str);
1103                 } else {
1104                         if (P != NULL && Pobjname(P, pc[i], objname,
1105                             sizeof (objname)) != 0) {
1106                                 (void) snprintf(c, sizeof (c), "%s`0x%llx",
1107                                     dt_basename(objname), (u_longlong_t)pc[i]);
1108                         } else {
1109                                 (void) snprintf(c, sizeof (c), "0x%llx",
1110                                     (u_longlong_t)pc[i]);
1111                         }
1112                 }
1113
1114                 if ((err = dt_printf(dtp, fp, format, c)) < 0)
1115                         break;
1116
1117                 if ((err = dt_printf(dtp, fp, "\n")) < 0)
1118                         break;
1119
1120                 if (str != NULL && str[0] == '@') {
1121                         /*
1122                          * If the first character of the string is an "at" sign,
1123                          * then the string is inferred to be an annotation --
1124                          * and it is printed out beneath the frame and offset
1125                          * with brackets.
1126                          */
1127                         if ((err = dt_printf(dtp, fp, "%*s", indent, "")) < 0)
1128                                 break;
1129
1130                         (void) snprintf(c, sizeof (c), "  [ %s ]", &str[1]);
1131
1132                         if ((err = dt_printf(dtp, fp, format, c)) < 0)
1133                                 break;
1134
1135                         if ((err = dt_printf(dtp, fp, "\n")) < 0)
1136                                 break;
1137                 }
1138
1139                 if (str != NULL) {
1140                         str += strlen(str) + 1;
1141                         if (str - strbase >= strsize)
1142                                 str = NULL;
1143                 }
1144         }
1145
1146         if (P != NULL) {
1147                 dt_proc_unlock(dtp, P);
1148                 dt_proc_release(dtp, P);
1149         }
1150
1151         return (err);
1152 }
1153
1154 static int
1155 dt_print_usym(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr, dtrace_actkind_t act)
1156 {
1157         /* LINTED - alignment */
1158         uint64_t pid = ((uint64_t *)addr)[0];
1159         /* LINTED - alignment */
1160         uint64_t pc = ((uint64_t *)addr)[1];
1161         const char *format = "  %-50s";
1162         char *s;
1163         int n, len = 256;
1164
1165         if (act == DTRACEACT_USYM && dtp->dt_vector == NULL) {
1166                 struct ps_prochandle *P;
1167
1168                 if ((P = dt_proc_grab(dtp, pid,
1169                     PGRAB_RDONLY | PGRAB_FORCE, 0)) != NULL) {
1170                         GElf_Sym sym;
1171
1172                         dt_proc_lock(dtp, P);
1173
1174                         if (Plookup_by_addr(P, pc, NULL, 0, &sym) == 0)
1175                                 pc = sym.st_value;
1176
1177                         dt_proc_unlock(dtp, P);
1178                         dt_proc_release(dtp, P);
1179                 }
1180         }
1181
1182         do {
1183                 n = len;
1184                 s = alloca(n);
1185         } while ((len = dtrace_uaddr2str(dtp, pid, pc, s, n)) > n);
1186
1187         return (dt_printf(dtp, fp, format, s));
1188 }
1189
1190 int
1191 dt_print_umod(dtrace_hdl_t *dtp, FILE *fp, const char *format, caddr_t addr)
1192 {
1193         /* LINTED - alignment */
1194         uint64_t pid = ((uint64_t *)addr)[0];
1195         /* LINTED - alignment */
1196         uint64_t pc = ((uint64_t *)addr)[1];
1197         int err = 0;
1198
1199         char objname[PATH_MAX], c[PATH_MAX * 2];
1200         struct ps_prochandle *P;
1201
1202         if (format == NULL)
1203                 format = "  %-50s";
1204
1205         /*
1206          * See the comment in dt_print_ustack() for the rationale for
1207          * printing raw addresses in the vectored case.
1208          */
1209         if (dtp->dt_vector == NULL)
1210                 P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0);
1211         else
1212                 P = NULL;
1213
1214         if (P != NULL)
1215                 dt_proc_lock(dtp, P); /* lock handle while we perform lookups */
1216
1217         if (P != NULL && Pobjname(P, pc, objname, sizeof (objname)) != 0) {
1218                 (void) snprintf(c, sizeof (c), "%s", dt_basename(objname));
1219         } else {
1220                 (void) snprintf(c, sizeof (c), "0x%llx", (u_longlong_t)pc);
1221         }
1222
1223         err = dt_printf(dtp, fp, format, c);
1224
1225         if (P != NULL) {
1226                 dt_proc_unlock(dtp, P);
1227                 dt_proc_release(dtp, P);
1228         }
1229
1230         return (err);
1231 }
1232
1233 int
1234 dt_print_memory(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr)
1235 {
1236         int quiet = (dtp->dt_options[DTRACEOPT_QUIET] != DTRACEOPT_UNSET);
1237         size_t nbytes = *((uintptr_t *) addr);
1238
1239         return (dt_print_bytes(dtp, fp, addr + sizeof(uintptr_t),
1240             nbytes, 50, quiet, 1));
1241 }
1242
1243 typedef struct dt_type_cbdata {
1244         dtrace_hdl_t            *dtp;
1245         dtrace_typeinfo_t       dtt;
1246         caddr_t                 addr;
1247         caddr_t                 addrend;
1248         const char              *name;
1249         int                     f_type;
1250         int                     indent;
1251         int                     type_width;
1252         int                     name_width;
1253         FILE                    *fp;
1254 } dt_type_cbdata_t;
1255
1256 static int      dt_print_type_data(dt_type_cbdata_t *, ctf_id_t);
1257
1258 static int
1259 dt_print_type_member(const char *name, ctf_id_t type, ulong_t off, void *arg)
1260 {
1261         dt_type_cbdata_t cbdata;
1262         dt_type_cbdata_t *cbdatap = arg;
1263         ssize_t ssz;
1264
1265         if ((ssz = ctf_type_size(cbdatap->dtt.dtt_ctfp, type)) <= 0)
1266                 return (0);
1267
1268         off /= 8;
1269
1270         cbdata = *cbdatap;
1271         cbdata.name = name;
1272         cbdata.addr += off;
1273         cbdata.addrend = cbdata.addr + ssz;
1274
1275         return (dt_print_type_data(&cbdata, type));
1276 }
1277
1278 static int
1279 dt_print_type_width(const char *name, ctf_id_t type, ulong_t off, void *arg)
1280 {
1281         char buf[DT_TYPE_NAMELEN];
1282         char *p;
1283         dt_type_cbdata_t *cbdatap = arg;
1284         size_t sz = strlen(name);
1285
1286         ctf_type_name(cbdatap->dtt.dtt_ctfp, type, buf, sizeof (buf));
1287
1288         if ((p = strchr(buf, '[')) != NULL)
1289                 p[-1] = '\0';
1290         else
1291                 p = "";
1292
1293         sz += strlen(p);
1294
1295         if (sz > cbdatap->name_width)
1296                 cbdatap->name_width = sz;
1297
1298         sz = strlen(buf);
1299
1300         if (sz > cbdatap->type_width)
1301                 cbdatap->type_width = sz;
1302
1303         return (0);
1304 }
1305
1306 static int
1307 dt_print_type_data(dt_type_cbdata_t *cbdatap, ctf_id_t type)
1308 {
1309         caddr_t addr = cbdatap->addr;
1310         caddr_t addrend = cbdatap->addrend;
1311         char buf[DT_TYPE_NAMELEN];
1312         char *p;
1313         int cnt = 0;
1314         uint_t kind = ctf_type_kind(cbdatap->dtt.dtt_ctfp, type);
1315         ssize_t ssz = ctf_type_size(cbdatap->dtt.dtt_ctfp, type);
1316
1317         ctf_type_name(cbdatap->dtt.dtt_ctfp, type, buf, sizeof (buf));
1318
1319         if ((p = strchr(buf, '[')) != NULL)
1320                 p[-1] = '\0';
1321         else
1322                 p = "";
1323
1324         if (cbdatap->f_type) {
1325                 int type_width = roundup(cbdatap->type_width + 1, 4);
1326                 int name_width = roundup(cbdatap->name_width + 1, 4);
1327
1328                 name_width -= strlen(cbdatap->name);
1329
1330                 dt_printf(cbdatap->dtp, cbdatap->fp, "%*s%-*s%s%-*s     = ",cbdatap->indent * 4,"",type_width,buf,cbdatap->name,name_width,p);
1331         }
1332
1333         while (addr < addrend) {
1334                 dt_type_cbdata_t cbdata;
1335                 ctf_arinfo_t arinfo;
1336                 ctf_encoding_t cte;
1337                 uintptr_t *up;
1338                 void *vp = addr;
1339                 cbdata = *cbdatap;
1340                 cbdata.name = "";
1341                 cbdata.addr = addr;
1342                 cbdata.addrend = addr + ssz;
1343                 cbdata.f_type = 0;
1344                 cbdata.indent++;
1345                 cbdata.type_width = 0;
1346                 cbdata.name_width = 0;
1347
1348                 if (cnt > 0)
1349                         dt_printf(cbdatap->dtp, cbdatap->fp, "%*s", cbdatap->indent * 4,"");
1350
1351                 switch (kind) {
1352                 case CTF_K_INTEGER:
1353                         if (ctf_type_encoding(cbdatap->dtt.dtt_ctfp, type, &cte) != 0)
1354                                 return (-1);
1355                         if ((cte.cte_format & CTF_INT_SIGNED) != 0)
1356                                 switch (cte.cte_bits) {
1357                                 case 8:
1358                                         if (isprint(*((char *) vp)))
1359                                                 dt_printf(cbdatap->dtp, cbdatap->fp, "'%c', ", *((char *) vp));
1360                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%d (0x%x);\n", *((char *) vp), *((char *) vp));
1361                                         break;
1362                                 case 16:
1363                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%hd (0x%hx);\n", *((short *) vp), *((u_short *) vp));
1364                                         break;
1365                                 case 32:
1366                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%d (0x%x);\n", *((int *) vp), *((u_int *) vp));
1367                                         break;
1368                                 case 64:
1369                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%jd (0x%jx);\n", *((long long *) vp), *((unsigned long long *) vp));
1370                                         break;
1371                                 default:
1372                                         dt_printf(cbdatap->dtp, cbdatap->fp, "CTF_K_INTEGER: format %x offset %u bits %u\n",cte.cte_format,cte.cte_offset,cte.cte_bits);
1373                                         break;
1374                                 }
1375                         else
1376                                 switch (cte.cte_bits) {
1377                                 case 8:
1378                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%u (0x%x);\n", *((uint8_t *) vp) & 0xff, *((uint8_t *) vp) & 0xff);
1379                                         break;
1380                                 case 16:
1381                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%hu (0x%hx);\n", *((u_short *) vp), *((u_short *) vp));
1382                                         break;
1383                                 case 32:
1384                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%u (0x%x);\n", *((u_int *) vp), *((u_int *) vp));
1385                                         break;
1386                                 case 64:
1387                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%ju (0x%jx);\n", *((unsigned long long *) vp), *((unsigned long long *) vp));
1388                                         break;
1389                                 default:
1390                                         dt_printf(cbdatap->dtp, cbdatap->fp, "CTF_K_INTEGER: format %x offset %u bits %u\n",cte.cte_format,cte.cte_offset,cte.cte_bits);
1391                                         break;
1392                                 }
1393                         break;
1394                 case CTF_K_FLOAT:
1395                         dt_printf(cbdatap->dtp, cbdatap->fp, "CTF_K_FLOAT: format %x offset %u bits %u\n",cte.cte_format,cte.cte_offset,cte.cte_bits);
1396                         break;
1397                 case CTF_K_POINTER:
1398                         dt_printf(cbdatap->dtp, cbdatap->fp, "%p;\n", *((void **) addr));
1399                         break;
1400                 case CTF_K_ARRAY:
1401                         if (ctf_array_info(cbdatap->dtt.dtt_ctfp, type, &arinfo) != 0)
1402                                 return (-1);
1403                         dt_printf(cbdatap->dtp, cbdatap->fp, "{\n%*s",cbdata.indent * 4,"");
1404                         dt_print_type_data(&cbdata, arinfo.ctr_contents);
1405                         dt_printf(cbdatap->dtp, cbdatap->fp, "%*s};\n",cbdatap->indent * 4,"");
1406                         break;
1407                 case CTF_K_FUNCTION:
1408                         dt_printf(cbdatap->dtp, cbdatap->fp, "CTF_K_FUNCTION:\n");
1409                         break;
1410                 case CTF_K_STRUCT:
1411                         cbdata.f_type = 1;
1412                         if (ctf_member_iter(cbdatap->dtt.dtt_ctfp, type,
1413                             dt_print_type_width, &cbdata) != 0)
1414                                 return (-1);
1415                         dt_printf(cbdatap->dtp, cbdatap->fp, "{\n");
1416                         if (ctf_member_iter(cbdatap->dtt.dtt_ctfp, type,
1417                             dt_print_type_member, &cbdata) != 0)
1418                                 return (-1);
1419                         dt_printf(cbdatap->dtp, cbdatap->fp, "%*s};\n",cbdatap->indent * 4,"");
1420                         break;
1421                 case CTF_K_UNION:
1422                         cbdata.f_type = 1;
1423                         if (ctf_member_iter(cbdatap->dtt.dtt_ctfp, type,
1424                             dt_print_type_width, &cbdata) != 0)
1425                                 return (-1);
1426                         dt_printf(cbdatap->dtp, cbdatap->fp, "{\n");
1427                         if (ctf_member_iter(cbdatap->dtt.dtt_ctfp, type,
1428                             dt_print_type_member, &cbdata) != 0)
1429                                 return (-1);
1430                         dt_printf(cbdatap->dtp, cbdatap->fp, "%*s};\n",cbdatap->indent * 4,"");
1431                         break;
1432                 case CTF_K_ENUM:
1433                         dt_printf(cbdatap->dtp, cbdatap->fp, "%s;\n", ctf_enum_name(cbdatap->dtt.dtt_ctfp, type, *((int *) vp)));
1434                         break;
1435                 case CTF_K_TYPEDEF:
1436                         dt_print_type_data(&cbdata, ctf_type_reference(cbdatap->dtt.dtt_ctfp,type));
1437                         break;
1438                 case CTF_K_VOLATILE:
1439                         if (cbdatap->f_type)
1440                                 dt_printf(cbdatap->dtp, cbdatap->fp, "volatile ");
1441                         dt_print_type_data(&cbdata, ctf_type_reference(cbdatap->dtt.dtt_ctfp,type));
1442                         break;
1443                 case CTF_K_CONST:
1444                         if (cbdatap->f_type)
1445                                 dt_printf(cbdatap->dtp, cbdatap->fp, "const ");
1446                         dt_print_type_data(&cbdata, ctf_type_reference(cbdatap->dtt.dtt_ctfp,type));
1447                         break;
1448                 case CTF_K_RESTRICT:
1449                         if (cbdatap->f_type)
1450                                 dt_printf(cbdatap->dtp, cbdatap->fp, "restrict ");
1451                         dt_print_type_data(&cbdata, ctf_type_reference(cbdatap->dtt.dtt_ctfp,type));
1452                         break;
1453                 default:
1454                         break;
1455                 }
1456
1457                 addr += ssz;
1458                 cnt++;
1459         }
1460
1461         return (0);
1462 }
1463
1464 static int
1465 dt_print_type(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr)
1466 {
1467         caddr_t addrend;
1468         char *p;
1469         dtrace_typeinfo_t dtt;
1470         dt_type_cbdata_t cbdata;
1471         int num = 0;
1472         int quiet = (dtp->dt_options[DTRACEOPT_QUIET] != DTRACEOPT_UNSET);
1473         ssize_t ssz;
1474
1475         if (!quiet)
1476                 dt_printf(dtp, fp, "\n");
1477
1478         /* Get the total number of bytes of data buffered. */
1479         size_t nbytes = *((uintptr_t *) addr);
1480         addr += sizeof(uintptr_t);
1481
1482         /*
1483          * Get the size of the type so that we can check that it matches
1484          * the CTF data we look up and so that we can figure out how many
1485          * type elements are buffered.
1486          */
1487         size_t typs = *((uintptr_t *) addr);
1488         addr += sizeof(uintptr_t);
1489
1490         /*
1491          * Point to the type string in the buffer. Get it's string
1492          * length and round it up to become the offset to the start
1493          * of the buffered type data which we would like to be aligned
1494          * for easy access.
1495          */
1496         char *strp = (char *) addr;
1497         int offset = roundup(strlen(strp) + 1, sizeof(uintptr_t));
1498
1499         /*
1500          * The type string might have a format such as 'int [20]'.
1501          * Check if there is an array dimension present.
1502          */
1503         if ((p = strchr(strp, '[')) != NULL) {
1504                 /* Strip off the array dimension. */
1505                 *p++ = '\0';
1506
1507                 for (; *p != '\0' && *p != ']'; p++)
1508                         num = num * 10 + *p - '0';
1509         } else
1510                 /* No array dimension, so default. */
1511                 num = 1;
1512
1513         /* Lookup the CTF type from the type string. */
1514         if (dtrace_lookup_by_type(dtp,  DTRACE_OBJ_EVERY, strp, &dtt) < 0)
1515                 return (-1);
1516
1517         /* Offset the buffer address to the start of the data... */
1518         addr += offset;
1519
1520         ssz = ctf_type_size(dtt.dtt_ctfp, dtt.dtt_type);
1521
1522         if (typs != ssz) {
1523                 printf("Expected type size from buffer (%lu) to match type size looked up now (%ld)\n", (u_long) typs, (long) ssz);
1524                 return (-1);
1525         }
1526
1527         cbdata.dtp = dtp;
1528         cbdata.dtt = dtt;
1529         cbdata.name = "";
1530         cbdata.addr = addr;
1531         cbdata.addrend = addr + nbytes;
1532         cbdata.indent = 1;
1533         cbdata.f_type = 1;
1534         cbdata.type_width = 0;
1535         cbdata.name_width = 0;
1536         cbdata.fp = fp;
1537
1538         return (dt_print_type_data(&cbdata, dtt.dtt_type));
1539 }
1540
1541 static int
1542 dt_print_sym(dtrace_hdl_t *dtp, FILE *fp, const char *format, caddr_t addr)
1543 {
1544         /* LINTED - alignment */
1545         uint64_t pc = *((uint64_t *)addr);
1546         dtrace_syminfo_t dts;
1547         GElf_Sym sym;
1548         char c[PATH_MAX * 2];
1549
1550         if (format == NULL)
1551                 format = "  %-50s";
1552
1553         if (dtrace_lookup_by_addr(dtp, pc, &sym, &dts) == 0) {
1554                 (void) snprintf(c, sizeof (c), "%s`%s",
1555                     dts.dts_object, dts.dts_name);
1556         } else {
1557                 /*
1558                  * We'll repeat the lookup, but this time we'll specify a
1559                  * NULL GElf_Sym -- indicating that we're only interested in
1560                  * the containing module.
1561                  */
1562                 if (dtrace_lookup_by_addr(dtp, pc, NULL, &dts) == 0) {
1563                         (void) snprintf(c, sizeof (c), "%s`0x%llx",
1564                             dts.dts_object, (u_longlong_t)pc);
1565                 } else {
1566                         (void) snprintf(c, sizeof (c), "0x%llx",
1567                             (u_longlong_t)pc);
1568                 }
1569         }
1570
1571         if (dt_printf(dtp, fp, format, c) < 0)
1572                 return (-1);
1573
1574         return (0);
1575 }
1576
1577 int
1578 dt_print_mod(dtrace_hdl_t *dtp, FILE *fp, const char *format, caddr_t addr)
1579 {
1580         /* LINTED - alignment */
1581         uint64_t pc = *((uint64_t *)addr);
1582         dtrace_syminfo_t dts;
1583         char c[PATH_MAX * 2];
1584
1585         if (format == NULL)
1586                 format = "  %-50s";
1587
1588         if (dtrace_lookup_by_addr(dtp, pc, NULL, &dts) == 0) {
1589                 (void) snprintf(c, sizeof (c), "%s", dts.dts_object);
1590         } else {
1591                 (void) snprintf(c, sizeof (c), "0x%llx", (u_longlong_t)pc);
1592         }
1593
1594         if (dt_printf(dtp, fp, format, c) < 0)
1595                 return (-1);
1596
1597         return (0);
1598 }
1599
1600 typedef struct dt_normal {
1601         dtrace_aggvarid_t dtnd_id;
1602         uint64_t dtnd_normal;
1603 } dt_normal_t;
1604
1605 static int
1606 dt_normalize_agg(const dtrace_aggdata_t *aggdata, void *arg)
1607 {
1608         dt_normal_t *normal = arg;
1609         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1610         dtrace_aggvarid_t id = normal->dtnd_id;
1611
1612         if (agg->dtagd_nrecs == 0)
1613                 return (DTRACE_AGGWALK_NEXT);
1614
1615         if (agg->dtagd_varid != id)
1616                 return (DTRACE_AGGWALK_NEXT);
1617
1618         ((dtrace_aggdata_t *)aggdata)->dtada_normal = normal->dtnd_normal;
1619         return (DTRACE_AGGWALK_NORMALIZE);
1620 }
1621
1622 static int
1623 dt_normalize(dtrace_hdl_t *dtp, caddr_t base, dtrace_recdesc_t *rec)
1624 {
1625         dt_normal_t normal;
1626         caddr_t addr;
1627
1628         /*
1629          * We (should) have two records:  the aggregation ID followed by the
1630          * normalization value.
1631          */
1632         addr = base + rec->dtrd_offset;
1633
1634         if (rec->dtrd_size != sizeof (dtrace_aggvarid_t))
1635                 return (dt_set_errno(dtp, EDT_BADNORMAL));
1636
1637         /* LINTED - alignment */
1638         normal.dtnd_id = *((dtrace_aggvarid_t *)addr);
1639         rec++;
1640
1641         if (rec->dtrd_action != DTRACEACT_LIBACT)
1642                 return (dt_set_errno(dtp, EDT_BADNORMAL));
1643
1644         if (rec->dtrd_arg != DT_ACT_NORMALIZE)
1645                 return (dt_set_errno(dtp, EDT_BADNORMAL));
1646
1647         addr = base + rec->dtrd_offset;
1648
1649         switch (rec->dtrd_size) {
1650         case sizeof (uint64_t):
1651                 /* LINTED - alignment */
1652                 normal.dtnd_normal = *((uint64_t *)addr);
1653                 break;
1654         case sizeof (uint32_t):
1655                 /* LINTED - alignment */
1656                 normal.dtnd_normal = *((uint32_t *)addr);
1657                 break;
1658         case sizeof (uint16_t):
1659                 /* LINTED - alignment */
1660                 normal.dtnd_normal = *((uint16_t *)addr);
1661                 break;
1662         case sizeof (uint8_t):
1663                 normal.dtnd_normal = *((uint8_t *)addr);
1664                 break;
1665         default:
1666                 return (dt_set_errno(dtp, EDT_BADNORMAL));
1667         }
1668
1669         (void) dtrace_aggregate_walk(dtp, dt_normalize_agg, &normal);
1670
1671         return (0);
1672 }
1673
1674 static int
1675 dt_denormalize_agg(const dtrace_aggdata_t *aggdata, void *arg)
1676 {
1677         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1678         dtrace_aggvarid_t id = *((dtrace_aggvarid_t *)arg);
1679
1680         if (agg->dtagd_nrecs == 0)
1681                 return (DTRACE_AGGWALK_NEXT);
1682
1683         if (agg->dtagd_varid != id)
1684                 return (DTRACE_AGGWALK_NEXT);
1685
1686         return (DTRACE_AGGWALK_DENORMALIZE);
1687 }
1688
1689 static int
1690 dt_clear_agg(const dtrace_aggdata_t *aggdata, void *arg)
1691 {
1692         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1693         dtrace_aggvarid_t id = *((dtrace_aggvarid_t *)arg);
1694
1695         if (agg->dtagd_nrecs == 0)
1696                 return (DTRACE_AGGWALK_NEXT);
1697
1698         if (agg->dtagd_varid != id)
1699                 return (DTRACE_AGGWALK_NEXT);
1700
1701         return (DTRACE_AGGWALK_CLEAR);
1702 }
1703
1704 typedef struct dt_trunc {
1705         dtrace_aggvarid_t dttd_id;
1706         uint64_t dttd_remaining;
1707 } dt_trunc_t;
1708
1709 static int
1710 dt_trunc_agg(const dtrace_aggdata_t *aggdata, void *arg)
1711 {
1712         dt_trunc_t *trunc = arg;
1713         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1714         dtrace_aggvarid_t id = trunc->dttd_id;
1715
1716         if (agg->dtagd_nrecs == 0)
1717                 return (DTRACE_AGGWALK_NEXT);
1718
1719         if (agg->dtagd_varid != id)
1720                 return (DTRACE_AGGWALK_NEXT);
1721
1722         if (trunc->dttd_remaining == 0)
1723                 return (DTRACE_AGGWALK_REMOVE);
1724
1725         trunc->dttd_remaining--;
1726         return (DTRACE_AGGWALK_NEXT);
1727 }
1728
1729 static int
1730 dt_trunc(dtrace_hdl_t *dtp, caddr_t base, dtrace_recdesc_t *rec)
1731 {
1732         dt_trunc_t trunc;
1733         caddr_t addr;
1734         int64_t remaining;
1735         int (*func)(dtrace_hdl_t *, dtrace_aggregate_f *, void *);
1736
1737         /*
1738          * We (should) have two records:  the aggregation ID followed by the
1739          * number of aggregation entries after which the aggregation is to be
1740          * truncated.
1741          */
1742         addr = base + rec->dtrd_offset;
1743
1744         if (rec->dtrd_size != sizeof (dtrace_aggvarid_t))
1745                 return (dt_set_errno(dtp, EDT_BADTRUNC));
1746
1747         /* LINTED - alignment */
1748         trunc.dttd_id = *((dtrace_aggvarid_t *)addr);
1749         rec++;
1750
1751         if (rec->dtrd_action != DTRACEACT_LIBACT)
1752                 return (dt_set_errno(dtp, EDT_BADTRUNC));
1753
1754         if (rec->dtrd_arg != DT_ACT_TRUNC)
1755                 return (dt_set_errno(dtp, EDT_BADTRUNC));
1756
1757         addr = base + rec->dtrd_offset;
1758
1759         switch (rec->dtrd_size) {
1760         case sizeof (uint64_t):
1761                 /* LINTED - alignment */
1762                 remaining = *((int64_t *)addr);
1763                 break;
1764         case sizeof (uint32_t):
1765                 /* LINTED - alignment */
1766                 remaining = *((int32_t *)addr);
1767                 break;
1768         case sizeof (uint16_t):
1769                 /* LINTED - alignment */
1770                 remaining = *((int16_t *)addr);
1771                 break;
1772         case sizeof (uint8_t):
1773                 remaining = *((int8_t *)addr);
1774                 break;
1775         default:
1776                 return (dt_set_errno(dtp, EDT_BADNORMAL));
1777         }
1778
1779         if (remaining < 0) {
1780                 func = dtrace_aggregate_walk_valsorted;
1781                 remaining = -remaining;
1782         } else {
1783                 func = dtrace_aggregate_walk_valrevsorted;
1784         }
1785
1786         assert(remaining >= 0);
1787         trunc.dttd_remaining = remaining;
1788
1789         (void) func(dtp, dt_trunc_agg, &trunc);
1790
1791         return (0);
1792 }
1793
1794 static int
1795 dt_print_datum(dtrace_hdl_t *dtp, FILE *fp, dtrace_recdesc_t *rec,
1796     caddr_t addr, size_t size, uint64_t normal)
1797 {
1798         int err;
1799         dtrace_actkind_t act = rec->dtrd_action;
1800
1801         switch (act) {
1802         case DTRACEACT_STACK:
1803                 return (dt_print_stack(dtp, fp, NULL, addr,
1804                     rec->dtrd_arg, rec->dtrd_size / rec->dtrd_arg));
1805
1806         case DTRACEACT_USTACK:
1807         case DTRACEACT_JSTACK:
1808                 return (dt_print_ustack(dtp, fp, NULL, addr, rec->dtrd_arg));
1809
1810         case DTRACEACT_USYM:
1811         case DTRACEACT_UADDR:
1812                 return (dt_print_usym(dtp, fp, addr, act));
1813
1814         case DTRACEACT_UMOD:
1815                 return (dt_print_umod(dtp, fp, NULL, addr));
1816
1817         case DTRACEACT_SYM:
1818                 return (dt_print_sym(dtp, fp, NULL, addr));
1819
1820         case DTRACEACT_MOD:
1821                 return (dt_print_mod(dtp, fp, NULL, addr));
1822
1823         case DTRACEAGG_QUANTIZE:
1824                 return (dt_print_quantize(dtp, fp, addr, size, normal));
1825
1826         case DTRACEAGG_LQUANTIZE:
1827                 return (dt_print_lquantize(dtp, fp, addr, size, normal));
1828
1829         case DTRACEAGG_LLQUANTIZE:
1830                 return (dt_print_llquantize(dtp, fp, addr, size, normal));
1831
1832         case DTRACEAGG_AVG:
1833                 return (dt_print_average(dtp, fp, addr, size, normal));
1834
1835         case DTRACEAGG_STDDEV:
1836                 return (dt_print_stddev(dtp, fp, addr, size, normal));
1837
1838         default:
1839                 break;
1840         }
1841
1842         switch (size) {
1843         case sizeof (uint64_t):
1844                 err = dt_printf(dtp, fp, " %16lld",
1845                     /* LINTED - alignment */
1846                     (long long)*((uint64_t *)addr) / normal);
1847                 break;
1848         case sizeof (uint32_t):
1849                 /* LINTED - alignment */
1850                 err = dt_printf(dtp, fp, " %8d", *((uint32_t *)addr) /
1851                     (uint32_t)normal);
1852                 break;
1853         case sizeof (uint16_t):
1854                 /* LINTED - alignment */
1855                 err = dt_printf(dtp, fp, " %5d", *((uint16_t *)addr) /
1856                     (uint32_t)normal);
1857                 break;
1858         case sizeof (uint8_t):
1859                 err = dt_printf(dtp, fp, " %3d", *((uint8_t *)addr) /
1860                     (uint32_t)normal);
1861                 break;
1862         default:
1863                 err = dt_print_bytes(dtp, fp, addr, size, 50, 0, 0);
1864                 break;
1865         }
1866
1867         return (err);
1868 }
1869
1870 int
1871 dt_print_aggs(const dtrace_aggdata_t **aggsdata, int naggvars, void *arg)
1872 {
1873         int i, aggact = 0;
1874         dt_print_aggdata_t *pd = arg;
1875         const dtrace_aggdata_t *aggdata = aggsdata[0];
1876         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1877         FILE *fp = pd->dtpa_fp;
1878         dtrace_hdl_t *dtp = pd->dtpa_dtp;
1879         dtrace_recdesc_t *rec;
1880         dtrace_actkind_t act;
1881         caddr_t addr;
1882         size_t size;
1883
1884         /*
1885          * Iterate over each record description in the key, printing the traced
1886          * data, skipping the first datum (the tuple member created by the
1887          * compiler).
1888          */
1889         for (i = 1; i < agg->dtagd_nrecs; i++) {
1890                 rec = &agg->dtagd_rec[i];
1891                 act = rec->dtrd_action;
1892                 addr = aggdata->dtada_data + rec->dtrd_offset;
1893                 size = rec->dtrd_size;
1894
1895                 if (DTRACEACT_ISAGG(act)) {
1896                         aggact = i;
1897                         break;
1898                 }
1899
1900                 if (dt_print_datum(dtp, fp, rec, addr, size, 1) < 0)
1901                         return (-1);
1902
1903                 if (dt_buffered_flush(dtp, NULL, rec, aggdata,
1904                     DTRACE_BUFDATA_AGGKEY) < 0)
1905                         return (-1);
1906         }
1907
1908         assert(aggact != 0);
1909
1910         for (i = (naggvars == 1 ? 0 : 1); i < naggvars; i++) {
1911                 uint64_t normal;
1912
1913                 aggdata = aggsdata[i];
1914                 agg = aggdata->dtada_desc;
1915                 rec = &agg->dtagd_rec[aggact];
1916                 act = rec->dtrd_action;
1917                 addr = aggdata->dtada_data + rec->dtrd_offset;
1918                 size = rec->dtrd_size;
1919
1920                 assert(DTRACEACT_ISAGG(act));
1921                 normal = aggdata->dtada_normal;
1922
1923                 if (dt_print_datum(dtp, fp, rec, addr, size, normal) < 0)
1924                         return (-1);
1925
1926                 if (dt_buffered_flush(dtp, NULL, rec, aggdata,
1927                     DTRACE_BUFDATA_AGGVAL) < 0)
1928                         return (-1);
1929
1930                 if (!pd->dtpa_allunprint)
1931                         agg->dtagd_flags |= DTRACE_AGD_PRINTED;
1932         }
1933
1934         if (dt_printf(dtp, fp, "\n") < 0)
1935                 return (-1);
1936
1937         if (dt_buffered_flush(dtp, NULL, NULL, aggdata,
1938             DTRACE_BUFDATA_AGGFORMAT | DTRACE_BUFDATA_AGGLAST) < 0)
1939                 return (-1);
1940
1941         return (0);
1942 }
1943
1944 int
1945 dt_print_agg(const dtrace_aggdata_t *aggdata, void *arg)
1946 {
1947         dt_print_aggdata_t *pd = arg;
1948         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1949         dtrace_aggvarid_t aggvarid = pd->dtpa_id;
1950
1951         if (pd->dtpa_allunprint) {
1952                 if (agg->dtagd_flags & DTRACE_AGD_PRINTED)
1953                         return (0);
1954         } else {
1955                 /*
1956                  * If we're not printing all unprinted aggregations, then the
1957                  * aggregation variable ID denotes a specific aggregation
1958                  * variable that we should print -- skip any other aggregations
1959                  * that we encounter.
1960                  */
1961                 if (agg->dtagd_nrecs == 0)
1962                         return (0);
1963
1964                 if (aggvarid != agg->dtagd_varid)
1965                         return (0);
1966         }
1967
1968         return (dt_print_aggs(&aggdata, 1, arg));
1969 }
1970
1971 int
1972 dt_setopt(dtrace_hdl_t *dtp, const dtrace_probedata_t *data,
1973     const char *option, const char *value)
1974 {
1975         int len, rval;
1976         char *msg;
1977         const char *errstr;
1978         dtrace_setoptdata_t optdata;
1979
1980         bzero(&optdata, sizeof (optdata));
1981         (void) dtrace_getopt(dtp, option, &optdata.dtsda_oldval);
1982
1983         if (dtrace_setopt(dtp, option, value) == 0) {
1984                 (void) dtrace_getopt(dtp, option, &optdata.dtsda_newval);
1985                 optdata.dtsda_probe = data;
1986                 optdata.dtsda_option = option;
1987                 optdata.dtsda_handle = dtp;
1988
1989                 if ((rval = dt_handle_setopt(dtp, &optdata)) != 0)
1990                         return (rval);
1991
1992                 return (0);
1993         }
1994
1995         errstr = dtrace_errmsg(dtp, dtrace_errno(dtp));
1996         len = strlen(option) + strlen(value) + strlen(errstr) + 80;
1997         msg = alloca(len);
1998
1999         (void) snprintf(msg, len, "couldn't set option \"%s\" to \"%s\": %s\n",
2000             option, value, errstr);
2001
2002         if ((rval = dt_handle_liberr(dtp, data, msg)) == 0)
2003                 return (0);
2004
2005         return (rval);
2006 }
2007
2008 static int
2009 dt_consume_cpu(dtrace_hdl_t *dtp, FILE *fp, int cpu,
2010     dtrace_bufdesc_t *buf, boolean_t just_one,
2011     dtrace_consume_probe_f *efunc, dtrace_consume_rec_f *rfunc, void *arg)
2012 {
2013         dtrace_epid_t id;
2014         size_t offs;
2015         int flow = (dtp->dt_options[DTRACEOPT_FLOWINDENT] != DTRACEOPT_UNSET);
2016         int quiet = (dtp->dt_options[DTRACEOPT_QUIET] != DTRACEOPT_UNSET);
2017         int rval, i, n;
2018         uint64_t tracememsize = 0;
2019         dtrace_probedata_t data;
2020         uint64_t drops;
2021
2022         bzero(&data, sizeof (data));
2023         data.dtpda_handle = dtp;
2024         data.dtpda_cpu = cpu;
2025         data.dtpda_flow = dtp->dt_flow;
2026         data.dtpda_indent = dtp->dt_indent;
2027         data.dtpda_prefix = dtp->dt_prefix;
2028
2029         for (offs = buf->dtbd_oldest; offs < buf->dtbd_size; ) {
2030                 dtrace_eprobedesc_t *epd;
2031
2032                 /*
2033                  * We're guaranteed to have an ID.
2034                  */
2035                 id = *(uint32_t *)((uintptr_t)buf->dtbd_data + offs);
2036
2037                 if (id == DTRACE_EPIDNONE) {
2038                         /*
2039                          * This is filler to assure proper alignment of the
2040                          * next record; we simply ignore it.
2041                          */
2042                         offs += sizeof (id);
2043                         continue;
2044                 }
2045
2046                 if ((rval = dt_epid_lookup(dtp, id, &data.dtpda_edesc,
2047                     &data.dtpda_pdesc)) != 0)
2048                         return (rval);
2049
2050                 epd = data.dtpda_edesc;
2051                 data.dtpda_data = buf->dtbd_data + offs;
2052
2053                 if (data.dtpda_edesc->dtepd_uarg != DT_ECB_DEFAULT) {
2054                         rval = dt_handle(dtp, &data);
2055
2056                         if (rval == DTRACE_CONSUME_NEXT)
2057                                 goto nextepid;
2058
2059                         if (rval == DTRACE_CONSUME_ERROR)
2060                                 return (-1);
2061                 }
2062
2063                 if (flow)
2064                         (void) dt_flowindent(dtp, &data, dtp->dt_last_epid,
2065                             buf, offs);
2066
2067                 rval = (*efunc)(&data, arg);
2068
2069                 if (flow) {
2070                         if (data.dtpda_flow == DTRACEFLOW_ENTRY)
2071                                 data.dtpda_indent += 2;
2072                 }
2073
2074                 if (rval == DTRACE_CONSUME_NEXT)
2075                         goto nextepid;
2076
2077                 if (rval == DTRACE_CONSUME_ABORT)
2078                         return (dt_set_errno(dtp, EDT_DIRABORT));
2079
2080                 if (rval != DTRACE_CONSUME_THIS)
2081                         return (dt_set_errno(dtp, EDT_BADRVAL));
2082
2083                 for (i = 0; i < epd->dtepd_nrecs; i++) {
2084                         caddr_t addr;
2085                         dtrace_recdesc_t *rec = &epd->dtepd_rec[i];
2086                         dtrace_actkind_t act = rec->dtrd_action;
2087
2088                         data.dtpda_data = buf->dtbd_data + offs +
2089                             rec->dtrd_offset;
2090                         addr = data.dtpda_data;
2091
2092                         if (act == DTRACEACT_LIBACT) {
2093                                 uint64_t arg = rec->dtrd_arg;
2094                                 dtrace_aggvarid_t id;
2095
2096                                 switch (arg) {
2097                                 case DT_ACT_CLEAR:
2098                                         /* LINTED - alignment */
2099                                         id = *((dtrace_aggvarid_t *)addr);
2100                                         (void) dtrace_aggregate_walk(dtp,
2101                                             dt_clear_agg, &id);
2102                                         continue;
2103
2104                                 case DT_ACT_DENORMALIZE:
2105                                         /* LINTED - alignment */
2106                                         id = *((dtrace_aggvarid_t *)addr);
2107                                         (void) dtrace_aggregate_walk(dtp,
2108                                             dt_denormalize_agg, &id);
2109                                         continue;
2110
2111                                 case DT_ACT_FTRUNCATE:
2112                                         if (fp == NULL)
2113                                                 continue;
2114
2115                                         (void) fflush(fp);
2116                                         (void) ftruncate(fileno(fp), 0);
2117                                         (void) fseeko(fp, 0, SEEK_SET);
2118                                         continue;
2119
2120                                 case DT_ACT_NORMALIZE:
2121                                         if (i == epd->dtepd_nrecs - 1)
2122                                                 return (dt_set_errno(dtp,
2123                                                     EDT_BADNORMAL));
2124
2125                                         if (dt_normalize(dtp,
2126                                             buf->dtbd_data + offs, rec) != 0)
2127                                                 return (-1);
2128
2129                                         i++;
2130                                         continue;
2131
2132                                 case DT_ACT_SETOPT: {
2133                                         uint64_t *opts = dtp->dt_options;
2134                                         dtrace_recdesc_t *valrec;
2135                                         uint32_t valsize;
2136                                         caddr_t val;
2137                                         int rv;
2138
2139                                         if (i == epd->dtepd_nrecs - 1) {
2140                                                 return (dt_set_errno(dtp,
2141                                                     EDT_BADSETOPT));
2142                                         }
2143
2144                                         valrec = &epd->dtepd_rec[++i];
2145                                         valsize = valrec->dtrd_size;
2146
2147                                         if (valrec->dtrd_action != act ||
2148                                             valrec->dtrd_arg != arg) {
2149                                                 return (dt_set_errno(dtp,
2150                                                     EDT_BADSETOPT));
2151                                         }
2152
2153                                         if (valsize > sizeof (uint64_t)) {
2154                                                 val = buf->dtbd_data + offs +
2155                                                     valrec->dtrd_offset;
2156                                         } else {
2157                                                 val = "1";
2158                                         }
2159
2160                                         rv = dt_setopt(dtp, &data, addr, val);
2161
2162                                         if (rv != 0)
2163                                                 return (-1);
2164
2165                                         flow = (opts[DTRACEOPT_FLOWINDENT] !=
2166                                             DTRACEOPT_UNSET);
2167                                         quiet = (opts[DTRACEOPT_QUIET] !=
2168                                             DTRACEOPT_UNSET);
2169
2170                                         continue;
2171                                 }
2172
2173                                 case DT_ACT_TRUNC:
2174                                         if (i == epd->dtepd_nrecs - 1)
2175                                                 return (dt_set_errno(dtp,
2176                                                     EDT_BADTRUNC));
2177
2178                                         if (dt_trunc(dtp,
2179                                             buf->dtbd_data + offs, rec) != 0)
2180                                                 return (-1);
2181
2182                                         i++;
2183                                         continue;
2184
2185                                 default:
2186                                         continue;
2187                                 }
2188                         }
2189
2190                         if (act == DTRACEACT_TRACEMEM_DYNSIZE &&
2191                             rec->dtrd_size == sizeof (uint64_t)) {
2192                                 /* LINTED - alignment */
2193                                 tracememsize = *((unsigned long long *)addr);
2194                                 continue;
2195                         }
2196
2197                         rval = (*rfunc)(&data, rec, arg);
2198
2199                         if (rval == DTRACE_CONSUME_NEXT)
2200                                 continue;
2201
2202                         if (rval == DTRACE_CONSUME_ABORT)
2203                                 return (dt_set_errno(dtp, EDT_DIRABORT));
2204
2205                         if (rval != DTRACE_CONSUME_THIS)
2206                                 return (dt_set_errno(dtp, EDT_BADRVAL));
2207
2208                         if (act == DTRACEACT_STACK) {
2209                                 int depth = rec->dtrd_arg;
2210
2211                                 if (dt_print_stack(dtp, fp, NULL, addr, depth,
2212                                     rec->dtrd_size / depth) < 0)
2213                                         return (-1);
2214                                 goto nextrec;
2215                         }
2216
2217                         if (act == DTRACEACT_USTACK ||
2218                             act == DTRACEACT_JSTACK) {
2219                                 if (dt_print_ustack(dtp, fp, NULL,
2220                                     addr, rec->dtrd_arg) < 0)
2221                                         return (-1);
2222                                 goto nextrec;
2223                         }
2224
2225                         if (act == DTRACEACT_SYM) {
2226                                 if (dt_print_sym(dtp, fp, NULL, addr) < 0)
2227                                         return (-1);
2228                                 goto nextrec;
2229                         }
2230
2231                         if (act == DTRACEACT_MOD) {
2232                                 if (dt_print_mod(dtp, fp, NULL, addr) < 0)
2233                                         return (-1);
2234                                 goto nextrec;
2235                         }
2236
2237                         if (act == DTRACEACT_USYM || act == DTRACEACT_UADDR) {
2238                                 if (dt_print_usym(dtp, fp, addr, act) < 0)
2239                                         return (-1);
2240                                 goto nextrec;
2241                         }
2242
2243                         if (act == DTRACEACT_UMOD) {
2244                                 if (dt_print_umod(dtp, fp, NULL, addr) < 0)
2245                                         return (-1);
2246                                 goto nextrec;
2247                         }
2248
2249                         if (act == DTRACEACT_PRINTM) {
2250                                 if (dt_print_memory(dtp, fp, addr) < 0)
2251                                         return (-1);
2252                                 goto nextrec;
2253                         }
2254
2255                         if (act == DTRACEACT_PRINTT) {
2256                                 if (dt_print_type(dtp, fp, addr) < 0)
2257                                         return (-1);
2258                                 goto nextrec;
2259                         }
2260
2261                         if (DTRACEACT_ISPRINTFLIKE(act)) {
2262                                 void *fmtdata;
2263                                 int (*func)(dtrace_hdl_t *, FILE *, void *,
2264                                     const dtrace_probedata_t *,
2265                                     const dtrace_recdesc_t *, uint_t,
2266                                     const void *buf, size_t);
2267
2268                                 if ((fmtdata = dt_format_lookup(dtp,
2269                                     rec->dtrd_format)) == NULL)
2270                                         goto nofmt;
2271
2272                                 switch (act) {
2273                                 case DTRACEACT_PRINTF:
2274                                         func = dtrace_fprintf;
2275                                         break;
2276                                 case DTRACEACT_PRINTA:
2277                                         func = dtrace_fprinta;
2278                                         break;
2279                                 case DTRACEACT_SYSTEM:
2280                                         func = dtrace_system;
2281                                         break;
2282                                 case DTRACEACT_FREOPEN:
2283                                         func = dtrace_freopen;
2284                                         break;
2285                                 }
2286
2287                                 n = (*func)(dtp, fp, fmtdata, &data,
2288                                     rec, epd->dtepd_nrecs - i,
2289                                     (uchar_t *)buf->dtbd_data + offs,
2290                                     buf->dtbd_size - offs);
2291
2292                                 if (n < 0)
2293                                         return (-1); /* errno is set for us */
2294
2295                                 if (n > 0)
2296                                         i += n - 1;
2297                                 goto nextrec;
2298                         }
2299
2300                         /*
2301                          * If this is a DIF expression, and the record has a
2302                          * format set, this indicates we have a CTF type name
2303                          * associated with the data and we should try to print
2304                          * it out by type.
2305                          */
2306                         if (act == DTRACEACT_DIFEXPR) {
2307                                 const char *strdata = dt_strdata_lookup(dtp,
2308                                     rec->dtrd_format);
2309                                 if (strdata != NULL) {
2310                                         n = dtrace_print(dtp, fp, strdata,
2311                                             addr, rec->dtrd_size);
2312
2313                                         /*
2314                                          * dtrace_print() will return -1 on
2315                                          * error, or return the number of bytes
2316                                          * consumed.  It will return 0 if the
2317                                          * type couldn't be determined, and we
2318                                          * should fall through to the normal
2319                                          * trace method.
2320                                          */
2321                                         if (n < 0)
2322                                                 return (-1);
2323
2324                                         if (n > 0)
2325                                                 goto nextrec;
2326                                 }
2327                         }
2328
2329 nofmt:
2330                         if (act == DTRACEACT_PRINTA) {
2331                                 dt_print_aggdata_t pd;
2332                                 dtrace_aggvarid_t *aggvars;
2333                                 int j, naggvars = 0;
2334                                 size_t size = ((epd->dtepd_nrecs - i) *
2335                                     sizeof (dtrace_aggvarid_t));
2336
2337                                 if ((aggvars = dt_alloc(dtp, size)) == NULL)
2338                                         return (-1);
2339
2340                                 /*
2341                                  * This might be a printa() with multiple
2342                                  * aggregation variables.  We need to scan
2343                                  * forward through the records until we find
2344                                  * a record from a different statement.
2345                                  */
2346                                 for (j = i; j < epd->dtepd_nrecs; j++) {
2347                                         dtrace_recdesc_t *nrec;
2348                                         caddr_t naddr;
2349
2350                                         nrec = &epd->dtepd_rec[j];
2351
2352                                         if (nrec->dtrd_uarg != rec->dtrd_uarg)
2353                                                 break;
2354
2355                                         if (nrec->dtrd_action != act) {
2356                                                 return (dt_set_errno(dtp,
2357                                                     EDT_BADAGG));
2358                                         }
2359
2360                                         naddr = buf->dtbd_data + offs +
2361                                             nrec->dtrd_offset;
2362
2363                                         aggvars[naggvars++] =
2364                                             /* LINTED - alignment */
2365                                             *((dtrace_aggvarid_t *)naddr);
2366                                 }
2367
2368                                 i = j - 1;
2369                                 bzero(&pd, sizeof (pd));
2370                                 pd.dtpa_dtp = dtp;
2371                                 pd.dtpa_fp = fp;
2372
2373                                 assert(naggvars >= 1);
2374
2375                                 if (naggvars == 1) {
2376                                         pd.dtpa_id = aggvars[0];
2377                                         dt_free(dtp, aggvars);
2378
2379                                         if (dt_printf(dtp, fp, "\n") < 0 ||
2380                                             dtrace_aggregate_walk_sorted(dtp,
2381                                             dt_print_agg, &pd) < 0)
2382                                                 return (-1);
2383                                         goto nextrec;
2384                                 }
2385
2386                                 if (dt_printf(dtp, fp, "\n") < 0 ||
2387                                     dtrace_aggregate_walk_joined(dtp, aggvars,
2388                                     naggvars, dt_print_aggs, &pd) < 0) {
2389                                         dt_free(dtp, aggvars);
2390                                         return (-1);
2391                                 }
2392
2393                                 dt_free(dtp, aggvars);
2394                                 goto nextrec;
2395                         }
2396
2397                         if (act == DTRACEACT_TRACEMEM) {
2398                                 if (tracememsize == 0 ||
2399                                     tracememsize > rec->dtrd_size) {
2400                                         tracememsize = rec->dtrd_size;
2401                                 }
2402
2403                                 n = dt_print_bytes(dtp, fp, addr,
2404                                     tracememsize, 33, quiet, 1);
2405
2406                                 tracememsize = 0;
2407
2408                                 if (n < 0)
2409                                         return (-1);
2410
2411                                 goto nextrec;
2412                         }
2413
2414                         switch (rec->dtrd_size) {
2415                         case sizeof (uint64_t):
2416                                 n = dt_printf(dtp, fp,
2417                                     quiet ? "%lld" : " %16lld",
2418                                     /* LINTED - alignment */
2419                                     *((unsigned long long *)addr));
2420                                 break;
2421                         case sizeof (uint32_t):
2422                                 n = dt_printf(dtp, fp, quiet ? "%d" : " %8d",
2423                                     /* LINTED - alignment */
2424                                     *((uint32_t *)addr));
2425                                 break;
2426                         case sizeof (uint16_t):
2427                                 n = dt_printf(dtp, fp, quiet ? "%d" : " %5d",
2428                                     /* LINTED - alignment */
2429                                     *((uint16_t *)addr));
2430                                 break;
2431                         case sizeof (uint8_t):
2432                                 n = dt_printf(dtp, fp, quiet ? "%d" : " %3d",
2433                                     *((uint8_t *)addr));
2434                                 break;
2435                         default:
2436                                 n = dt_print_bytes(dtp, fp, addr,
2437                                     rec->dtrd_size, 33, quiet, 0);
2438                                 break;
2439                         }
2440
2441                         if (n < 0)
2442                                 return (-1); /* errno is set for us */
2443
2444 nextrec:
2445                         if (dt_buffered_flush(dtp, &data, rec, NULL, 0) < 0)
2446                                 return (-1); /* errno is set for us */
2447                 }
2448
2449                 /*
2450                  * Call the record callback with a NULL record to indicate
2451                  * that we're done processing this EPID.
2452                  */
2453                 rval = (*rfunc)(&data, NULL, arg);
2454 nextepid:
2455                 offs += epd->dtepd_size;
2456                 dtp->dt_last_epid = id;
2457                 if (just_one) {
2458                         buf->dtbd_oldest = offs;
2459                         break;
2460                 }
2461         }
2462
2463         dtp->dt_flow = data.dtpda_flow;
2464         dtp->dt_indent = data.dtpda_indent;
2465         dtp->dt_prefix = data.dtpda_prefix;
2466
2467         if ((drops = buf->dtbd_drops) == 0)
2468                 return (0);
2469
2470         /*
2471          * Explicitly zero the drops to prevent us from processing them again.
2472          */
2473         buf->dtbd_drops = 0;
2474
2475         return (dt_handle_cpudrop(dtp, cpu, DTRACEDROP_PRINCIPAL, drops));
2476 }
2477
2478 /*
2479  * Reduce memory usage by shrinking the buffer if it's no more than half full.
2480  * Note, we need to preserve the alignment of the data at dtbd_oldest, which is
2481  * only 4-byte aligned.
2482  */
2483 static void
2484 dt_realloc_buf(dtrace_hdl_t *dtp, dtrace_bufdesc_t *buf, int cursize)
2485 {
2486         uint64_t used = buf->dtbd_size - buf->dtbd_oldest;
2487         if (used < cursize / 2) {
2488                 int misalign = buf->dtbd_oldest & (sizeof (uint64_t) - 1);
2489                 char *newdata = dt_alloc(dtp, used + misalign);
2490                 if (newdata == NULL)
2491                         return;
2492                 bzero(newdata, misalign);
2493                 bcopy(buf->dtbd_data + buf->dtbd_oldest,
2494                     newdata + misalign, used);
2495                 dt_free(dtp, buf->dtbd_data);
2496                 buf->dtbd_oldest = misalign;
2497                 buf->dtbd_size = used + misalign;
2498                 buf->dtbd_data = newdata;
2499         }
2500 }
2501
2502 /*
2503  * If the ring buffer has wrapped, the data is not in order.  Rearrange it
2504  * so that it is.  Note, we need to preserve the alignment of the data at
2505  * dtbd_oldest, which is only 4-byte aligned.
2506  */
2507 static int
2508 dt_unring_buf(dtrace_hdl_t *dtp, dtrace_bufdesc_t *buf)
2509 {
2510         int misalign;
2511         char *newdata, *ndp;
2512
2513         if (buf->dtbd_oldest == 0)
2514                 return (0);
2515
2516         misalign = buf->dtbd_oldest & (sizeof (uint64_t) - 1);
2517         newdata = ndp = dt_alloc(dtp, buf->dtbd_size + misalign);
2518
2519         if (newdata == NULL)
2520                 return (-1);
2521
2522         assert(0 == (buf->dtbd_size & (sizeof (uint64_t) - 1)));
2523
2524         bzero(ndp, misalign);
2525         ndp += misalign;
2526
2527         bcopy(buf->dtbd_data + buf->dtbd_oldest, ndp,
2528             buf->dtbd_size - buf->dtbd_oldest);
2529         ndp += buf->dtbd_size - buf->dtbd_oldest;
2530
2531         bcopy(buf->dtbd_data, ndp, buf->dtbd_oldest);
2532
2533         dt_free(dtp, buf->dtbd_data);
2534         buf->dtbd_oldest = 0;
2535         buf->dtbd_data = newdata;
2536         buf->dtbd_size += misalign;
2537
2538         return (0);
2539 }
2540
2541 static void
2542 dt_put_buf(dtrace_hdl_t *dtp, dtrace_bufdesc_t *buf)
2543 {
2544         dt_free(dtp, buf->dtbd_data);
2545         dt_free(dtp, buf);
2546 }
2547
2548 /*
2549  * Returns 0 on success, in which case *cbp will be filled in if we retrieved
2550  * data, or NULL if there is no data for this CPU.
2551  * Returns -1 on failure and sets dt_errno.
2552  */
2553 static int
2554 dt_get_buf(dtrace_hdl_t *dtp, int cpu, dtrace_bufdesc_t **bufp)
2555 {
2556         dtrace_optval_t size;
2557         dtrace_bufdesc_t *buf = dt_zalloc(dtp, sizeof (*buf));
2558         int error;
2559
2560         if (buf == NULL)
2561                 return (-1);
2562
2563         (void) dtrace_getopt(dtp, "bufsize", &size);
2564         buf->dtbd_data = dt_alloc(dtp, size);
2565         if (buf->dtbd_data == NULL) {
2566                 dt_free(dtp, buf);
2567                 return (-1);
2568         }
2569         buf->dtbd_size = size;
2570         buf->dtbd_cpu = cpu;
2571
2572 #if defined(sun)
2573         if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, buf) == -1) {
2574 #else
2575         if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, &buf) == -1) {
2576 #endif
2577                 dt_put_buf(dtp, buf);
2578                 /*
2579                  * If we failed with ENOENT, it may be because the
2580                  * CPU was unconfigured -- this is okay.  Any other
2581                  * error, however, is unexpected.
2582                  */
2583                 if (errno == ENOENT) {
2584                         *bufp = NULL;
2585                         return (0);
2586                 }
2587
2588                 return (dt_set_errno(dtp, errno));
2589         }
2590
2591         error = dt_unring_buf(dtp, buf);
2592         if (error != 0) {
2593                 dt_put_buf(dtp, buf);
2594                 return (error);
2595         }
2596         dt_realloc_buf(dtp, buf, size);
2597
2598         *bufp = buf;
2599         return (0);
2600 }
2601
2602 typedef struct dt_begin {
2603         dtrace_consume_probe_f *dtbgn_probefunc;
2604         dtrace_consume_rec_f *dtbgn_recfunc;
2605         void *dtbgn_arg;
2606         dtrace_handle_err_f *dtbgn_errhdlr;
2607         void *dtbgn_errarg;
2608         int dtbgn_beginonly;
2609 } dt_begin_t;
2610
2611 static int
2612 dt_consume_begin_probe(const dtrace_probedata_t *data, void *arg)
2613 {
2614         dt_begin_t *begin = arg;
2615         dtrace_probedesc_t *pd = data->dtpda_pdesc;
2616
2617         int r1 = (strcmp(pd->dtpd_provider, "dtrace") == 0);
2618         int r2 = (strcmp(pd->dtpd_name, "BEGIN") == 0);
2619
2620         if (begin->dtbgn_beginonly) {
2621                 if (!(r1 && r2))
2622                         return (DTRACE_CONSUME_NEXT);
2623         } else {
2624                 if (r1 && r2)
2625                         return (DTRACE_CONSUME_NEXT);
2626         }
2627
2628         /*
2629          * We have a record that we're interested in.  Now call the underlying
2630          * probe function...
2631          */
2632         return (begin->dtbgn_probefunc(data, begin->dtbgn_arg));
2633 }
2634
2635 static int
2636 dt_consume_begin_record(const dtrace_probedata_t *data,
2637     const dtrace_recdesc_t *rec, void *arg)
2638 {
2639         dt_begin_t *begin = arg;
2640
2641         return (begin->dtbgn_recfunc(data, rec, begin->dtbgn_arg));
2642 }
2643
2644 static int
2645 dt_consume_begin_error(const dtrace_errdata_t *data, void *arg)
2646 {
2647         dt_begin_t *begin = (dt_begin_t *)arg;
2648         dtrace_probedesc_t *pd = data->dteda_pdesc;
2649
2650         int r1 = (strcmp(pd->dtpd_provider, "dtrace") == 0);
2651         int r2 = (strcmp(pd->dtpd_name, "BEGIN") == 0);
2652
2653         if (begin->dtbgn_beginonly) {
2654                 if (!(r1 && r2))
2655                         return (DTRACE_HANDLE_OK);
2656         } else {
2657                 if (r1 && r2)
2658                         return (DTRACE_HANDLE_OK);
2659         }
2660
2661         return (begin->dtbgn_errhdlr(data, begin->dtbgn_errarg));
2662 }
2663
2664 static int
2665 dt_consume_begin(dtrace_hdl_t *dtp, FILE *fp,
2666     dtrace_consume_probe_f *pf, dtrace_consume_rec_f *rf, void *arg)
2667 {
2668         /*
2669          * There's this idea that the BEGIN probe should be processed before
2670          * everything else, and that the END probe should be processed after
2671          * anything else.  In the common case, this is pretty easy to deal
2672          * with.  However, a situation may arise where the BEGIN enabling and
2673          * END enabling are on the same CPU, and some enabling in the middle
2674          * occurred on a different CPU.  To deal with this (blech!) we need to
2675          * consume the BEGIN buffer up until the end of the BEGIN probe, and
2676          * then set it aside.  We will then process every other CPU, and then
2677          * we'll return to the BEGIN CPU and process the rest of the data
2678          * (which will inevitably include the END probe, if any).  Making this
2679          * even more complicated (!) is the library's ERROR enabling.  Because
2680          * this enabling is processed before we even get into the consume call
2681          * back, any ERROR firing would result in the library's ERROR enabling
2682          * being processed twice -- once in our first pass (for BEGIN probes),
2683          * and again in our second pass (for everything but BEGIN probes).  To
2684          * deal with this, we interpose on the ERROR handler to assure that we
2685          * only process ERROR enablings induced by BEGIN enablings in the
2686          * first pass, and that we only process ERROR enablings _not_ induced
2687          * by BEGIN enablings in the second pass.
2688          */
2689
2690         dt_begin_t begin;
2691         processorid_t cpu = dtp->dt_beganon;
2692         int rval, i;
2693         static int max_ncpus;
2694         dtrace_bufdesc_t *buf;
2695
2696         dtp->dt_beganon = -1;
2697
2698         if (dt_get_buf(dtp, cpu, &buf) != 0)
2699                 return (-1);
2700         if (buf == NULL)
2701                 return (0);
2702
2703         if (!dtp->dt_stopped || buf->dtbd_cpu != dtp->dt_endedon) {
2704                 /*
2705                  * This is the simple case.  We're either not stopped, or if
2706                  * we are, we actually processed any END probes on another
2707                  * CPU.  We can simply consume this buffer and return.
2708                  */
2709                 rval = dt_consume_cpu(dtp, fp, cpu, buf, B_FALSE,
2710                     pf, rf, arg);
2711                 dt_put_buf(dtp, buf);
2712                 return (rval);
2713         }
2714
2715         begin.dtbgn_probefunc = pf;
2716         begin.dtbgn_recfunc = rf;
2717         begin.dtbgn_arg = arg;
2718         begin.dtbgn_beginonly = 1;
2719
2720         /*
2721          * We need to interpose on the ERROR handler to be sure that we
2722          * only process ERRORs induced by BEGIN.
2723          */
2724         begin.dtbgn_errhdlr = dtp->dt_errhdlr;
2725         begin.dtbgn_errarg = dtp->dt_errarg;
2726         dtp->dt_errhdlr = dt_consume_begin_error;
2727         dtp->dt_errarg = &begin;
2728
2729         rval = dt_consume_cpu(dtp, fp, cpu, buf, B_FALSE,
2730             dt_consume_begin_probe, dt_consume_begin_record, &begin);
2731
2732         dtp->dt_errhdlr = begin.dtbgn_errhdlr;
2733         dtp->dt_errarg = begin.dtbgn_errarg;
2734
2735         if (rval != 0) {
2736                 dt_put_buf(dtp, buf);
2737                 return (rval);
2738         }
2739
2740         if (max_ncpus == 0)
2741                 max_ncpus = dt_sysconf(dtp, _SC_CPUID_MAX) + 1;
2742
2743         for (i = 0; i < max_ncpus; i++) {
2744                 dtrace_bufdesc_t *nbuf;
2745                 if (i == cpu)
2746                         continue;
2747
2748                 if (dt_get_buf(dtp, i, &nbuf) != 0) {
2749                         dt_put_buf(dtp, buf);
2750                         return (-1);
2751                 }
2752                 if (nbuf == NULL)
2753                         continue;
2754
2755                 rval = dt_consume_cpu(dtp, fp, i, nbuf, B_FALSE,
2756                     pf, rf, arg);
2757                 dt_put_buf(dtp, nbuf);
2758                 if (rval != 0) {
2759                         dt_put_buf(dtp, buf);
2760                         return (rval);
2761                 }
2762         }
2763
2764         /*
2765          * Okay -- we're done with the other buffers.  Now we want to
2766          * reconsume the first buffer -- but this time we're looking for
2767          * everything _but_ BEGIN.  And of course, in order to only consume
2768          * those ERRORs _not_ associated with BEGIN, we need to reinstall our
2769          * ERROR interposition function...
2770          */
2771         begin.dtbgn_beginonly = 0;
2772
2773         assert(begin.dtbgn_errhdlr == dtp->dt_errhdlr);
2774         assert(begin.dtbgn_errarg == dtp->dt_errarg);
2775         dtp->dt_errhdlr = dt_consume_begin_error;
2776         dtp->dt_errarg = &begin;
2777
2778         rval = dt_consume_cpu(dtp, fp, cpu, buf, B_FALSE,
2779             dt_consume_begin_probe, dt_consume_begin_record, &begin);
2780
2781         dtp->dt_errhdlr = begin.dtbgn_errhdlr;
2782         dtp->dt_errarg = begin.dtbgn_errarg;
2783
2784         return (rval);
2785 }
2786
2787 /* ARGSUSED */
2788 static uint64_t
2789 dt_buf_oldest(void *elem, void *arg)
2790 {
2791         dtrace_bufdesc_t *buf = elem;
2792         size_t offs = buf->dtbd_oldest;
2793
2794         while (offs < buf->dtbd_size) {
2795                 dtrace_rechdr_t *dtrh =
2796                     /* LINTED - alignment */
2797                     (dtrace_rechdr_t *)(buf->dtbd_data + offs);
2798                 if (dtrh->dtrh_epid == DTRACE_EPIDNONE) {
2799                         offs += sizeof (dtrace_epid_t);
2800                 } else {
2801                         return (DTRACE_RECORD_LOAD_TIMESTAMP(dtrh));
2802                 }
2803         }
2804
2805         /* There are no records left; use the time the buffer was retrieved. */
2806         return (buf->dtbd_timestamp);
2807 }
2808
2809 int
2810 dtrace_consume(dtrace_hdl_t *dtp, FILE *fp,
2811     dtrace_consume_probe_f *pf, dtrace_consume_rec_f *rf, void *arg)
2812 {
2813         dtrace_optval_t size;
2814         static int max_ncpus;
2815         int i, rval;
2816         dtrace_optval_t interval = dtp->dt_options[DTRACEOPT_SWITCHRATE];
2817         hrtime_t now = gethrtime();
2818
2819         if (dtp->dt_lastswitch != 0) {
2820                 if (now - dtp->dt_lastswitch < interval)
2821                         return (0);
2822
2823                 dtp->dt_lastswitch += interval;
2824         } else {
2825                 dtp->dt_lastswitch = now;
2826         }
2827
2828         if (!dtp->dt_active)
2829                 return (dt_set_errno(dtp, EINVAL));
2830
2831         if (max_ncpus == 0)
2832                 max_ncpus = dt_sysconf(dtp, _SC_CPUID_MAX) + 1;
2833
2834         if (pf == NULL)
2835                 pf = (dtrace_consume_probe_f *)dt_nullprobe;
2836
2837         if (rf == NULL)
2838                 rf = (dtrace_consume_rec_f *)dt_nullrec;
2839
2840         if (dtp->dt_options[DTRACEOPT_TEMPORAL] == DTRACEOPT_UNSET) {
2841                 /*
2842                  * The output will not be in the order it was traced.  Rather,
2843                  * we will consume all of the data from each CPU's buffer in
2844                  * turn.  We apply special handling for the records from BEGIN
2845                  * and END probes so that they are consumed first and last,
2846                  * respectively.
2847                  *
2848                  * If we have just begun, we want to first process the CPU that
2849                  * executed the BEGIN probe (if any).
2850                  */
2851                 if (dtp->dt_active && dtp->dt_beganon != -1 &&
2852                     (rval = dt_consume_begin(dtp, fp, pf, rf, arg)) != 0)
2853                         return (rval);
2854
2855                 for (i = 0; i < max_ncpus; i++) {
2856                         dtrace_bufdesc_t *buf;
2857
2858                         /*
2859                          * If we have stopped, we want to process the CPU on
2860                          * which the END probe was processed only _after_ we
2861                          * have processed everything else.
2862                          */
2863                         if (dtp->dt_stopped && (i == dtp->dt_endedon))
2864                                 continue;
2865
2866                         if (dt_get_buf(dtp, i, &buf) != 0)
2867                                 return (-1);
2868                         if (buf == NULL)
2869                                 continue;
2870
2871                         dtp->dt_flow = 0;
2872                         dtp->dt_indent = 0;
2873                         dtp->dt_prefix = NULL;
2874                         rval = dt_consume_cpu(dtp, fp, i,
2875                             buf, B_FALSE, pf, rf, arg);
2876                         dt_put_buf(dtp, buf);
2877                         if (rval != 0)
2878                                 return (rval);
2879                 }
2880                 if (dtp->dt_stopped) {
2881                         dtrace_bufdesc_t *buf;
2882
2883                         if (dt_get_buf(dtp, dtp->dt_endedon, &buf) != 0)
2884                                 return (-1);
2885                         if (buf == NULL)
2886                                 return (0);
2887
2888                         rval = dt_consume_cpu(dtp, fp, dtp->dt_endedon,
2889                             buf, B_FALSE, pf, rf, arg);
2890                         dt_put_buf(dtp, buf);
2891                         return (rval);
2892                 }
2893         } else {
2894                 /*
2895                  * The output will be in the order it was traced (or for
2896                  * speculations, when it was committed).  We retrieve a buffer
2897                  * from each CPU and put it into a priority queue, which sorts
2898                  * based on the first entry in the buffer.  This is sufficient
2899                  * because entries within a buffer are already sorted.
2900                  *
2901                  * We then consume records one at a time, always consuming the
2902                  * oldest record, as determined by the priority queue.  When
2903                  * we reach the end of the time covered by these buffers,
2904                  * we need to stop and retrieve more records on the next pass.
2905                  * The kernel tells us the time covered by each buffer, in
2906                  * dtbd_timestamp.  The first buffer's timestamp tells us the
2907                  * time covered by all buffers, as subsequently retrieved
2908                  * buffers will cover to a more recent time.
2909                  */
2910
2911                 uint64_t *drops = alloca(max_ncpus * sizeof (uint64_t));
2912                 uint64_t first_timestamp = 0;
2913                 uint_t cookie = 0;
2914                 dtrace_bufdesc_t *buf;
2915
2916                 bzero(drops, max_ncpus * sizeof (uint64_t));
2917
2918                 if (dtp->dt_bufq == NULL) {
2919                         dtp->dt_bufq = dt_pq_init(dtp, max_ncpus * 2,
2920                             dt_buf_oldest, NULL);
2921                         if (dtp->dt_bufq == NULL) /* ENOMEM */
2922                                 return (-1);
2923                 }
2924
2925                 /* Retrieve data from each CPU. */
2926                 (void) dtrace_getopt(dtp, "bufsize", &size);
2927                 for (i = 0; i < max_ncpus; i++) {
2928                         dtrace_bufdesc_t *buf;
2929
2930                         if (dt_get_buf(dtp, i, &buf) != 0)
2931                                 return (-1);
2932                         if (buf != NULL) {
2933                                 if (first_timestamp == 0)
2934                                         first_timestamp = buf->dtbd_timestamp;
2935                                 assert(buf->dtbd_timestamp >= first_timestamp);
2936
2937                                 dt_pq_insert(dtp->dt_bufq, buf);
2938                                 drops[i] = buf->dtbd_drops;
2939                                 buf->dtbd_drops = 0;
2940                         }
2941                 }
2942
2943                 /* Consume records. */
2944                 for (;;) {
2945                         dtrace_bufdesc_t *buf = dt_pq_pop(dtp->dt_bufq);
2946                         uint64_t timestamp;
2947
2948                         if (buf == NULL)
2949                                 break;
2950
2951                         timestamp = dt_buf_oldest(buf, dtp);
2952                         assert(timestamp >= dtp->dt_last_timestamp);
2953                         dtp->dt_last_timestamp = timestamp;
2954
2955                         if (timestamp == buf->dtbd_timestamp) {
2956                                 /*
2957                                  * We've reached the end of the time covered
2958                                  * by this buffer.  If this is the oldest
2959                                  * buffer, we must do another pass
2960                                  * to retrieve more data.
2961                                  */
2962                                 dt_put_buf(dtp, buf);
2963                                 if (timestamp == first_timestamp &&
2964                                     !dtp->dt_stopped)
2965                                         break;
2966                                 continue;
2967                         }
2968
2969                         if ((rval = dt_consume_cpu(dtp, fp,
2970                             buf->dtbd_cpu, buf, B_TRUE, pf, rf, arg)) != 0)
2971                                 return (rval);
2972                         dt_pq_insert(dtp->dt_bufq, buf);
2973                 }
2974
2975                 /* Consume drops. */
2976                 for (i = 0; i < max_ncpus; i++) {
2977                         if (drops[i] != 0) {
2978                                 int error = dt_handle_cpudrop(dtp, i,
2979                                     DTRACEDROP_PRINCIPAL, drops[i]);
2980                                 if (error != 0)
2981                                         return (error);
2982                         }
2983                 }
2984
2985                 /*
2986                  * Reduce memory usage by re-allocating smaller buffers
2987                  * for the "remnants".
2988                  */
2989                 while (buf = dt_pq_walk(dtp->dt_bufq, &cookie))
2990                         dt_realloc_buf(dtp, buf, buf->dtbd_size);
2991         }
2992
2993         return (0);
2994 }