cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25
  26 #include <stdlib.h>
  27 #include <strings.h>
  28 #include <errno.h>
  29 #include <unistd.h>
  30 #include <limits.h>
  31 #include <assert.h>
  32 #include <ctype.h>
  33 #if defined(sun)
  34 #include <alloca.h>
  35 #endif
  36 #include <dt_impl.h>
  37 #if !defined(sun)
  38 #include <libproc_compat.h>
  39 #endif
  40
  41 #define DT_MASK_LO 0x00000000FFFFFFFFULL
  42
  43 /*
  44  * We declare this here because (1) we need it and (2) we want to avoid a
  45  * dependency on libm in libdtrace.
  46  */
  47 static long double
  48 dt_fabsl(long double x)
  49 {
  50         if (x < 0)
  51                 return (-x);
  52
  53         return (x);
  54 }
  55
  56 /*
  57  * 128-bit arithmetic functions needed to support the stddev() aggregating
  58  * action.
  59  */
  60 static int
  61 dt_gt_128(uint64_t *a, uint64_t *b)
  62 {
  63         return (a[1] > b[1] || (a[1] == b[1] && a[0] > b[0]));
  64 }
  65
  66 static int
  67 dt_ge_128(uint64_t *a, uint64_t *b)
  68 {
  69         return (a[1] > b[1] || (a[1] == b[1] && a[0] >= b[0]));
  70 }
  71
  72 static int
  73 dt_le_128(uint64_t *a, uint64_t *b)
  74 {
  75         return (a[1] < b[1] || (a[1] == b[1] && a[0] <= b[0]));
  76 }
  77
  78 /*
  79  * Shift the 128-bit value in a by b. If b is positive, shift left.
  80  * If b is negative, shift right.
  81  */
  82 static void
  83 dt_shift_128(uint64_t *a, int b)
  84 {
  85         uint64_t mask;
  86
  87         if (b == 0)
  88                 return;
  89
  90         if (b < 0) {
  91                 b = -b;
  92                 if (b >= 64) {
  93                         a[0] = a[1] >> (b - 64);
  94                         a[1] = 0;
  95                 } else {
  96                         a[0] >>= b;
  97                         mask = 1LL << (64 - b);
  98                         mask -= 1;
  99                         a[0] |= ((a[1] & mask) << (64 - b));
 100                         a[1] >>= b;
 101                 }
 102         } else {
 103                 if (b >= 64) {
 104                         a[1] = a[0] << (b - 64);
 105                         a[0] = 0;
 106                 } else {
 107                         a[1] <<= b;
 108                         mask = a[0] >> (64 - b);
 109                         a[1] |= mask;
 110                         a[0] <<= b;
 111                 }
 112         }
 113 }
 114
 115 static int
 116 dt_nbits_128(uint64_t *a)
 117 {
 118         int nbits = 0;
 119         uint64_t tmp[2];
 120         uint64_t zero[2] = { 0, 0 };
 121
 122         tmp[0] = a[0];
 123         tmp[1] = a[1];
 124
 125         dt_shift_128(tmp, -1);
 126         while (dt_gt_128(tmp, zero)) {
 127                 dt_shift_128(tmp, -1);
 128                 nbits++;
 129         }
 130
 131         return (nbits);
 132 }
 133
 134 static void
 135 dt_subtract_128(uint64_t *minuend, uint64_t *subtrahend, uint64_t *difference)
 136 {
 137         uint64_t result[2];
 138
 139         result[0] = minuend[0] - subtrahend[0];
 140         result[1] = minuend[1] - subtrahend[1] -
 141             (minuend[0] < subtrahend[0] ? 1 : 0);
 142
 143         difference[0] = result[0];
 144         difference[1] = result[1];
 145 }
 146
 147 static void
 148 dt_add_128(uint64_t *addend1, uint64_t *addend2, uint64_t *sum)
 149 {
 150         uint64_t result[2];
 151
 152         result[0] = addend1[0] + addend2[0];
 153         result[1] = addend1[1] + addend2[1] +
 154             (result[0] < addend1[0] || result[0] < addend2[0] ? 1 : 0);
 155
 156         sum[0] = result[0];
 157         sum[1] = result[1];
 158 }
 159
 160 /*
 161  * The basic idea is to break the 2 64-bit values into 4 32-bit values,
 162  * use native multiplication on those, and then re-combine into the
 163  * resulting 128-bit value.
 164  *
 165  * (hi1 << 32 + lo1) * (hi2 << 32 + lo2) =
 166  *     hi1 * hi2 << 64 +
 167  *     hi1 * lo2 << 32 +
 168  *     hi2 * lo1 << 32 +
 169  *     lo1 * lo2
 170  */
 171 static void
 172 dt_multiply_128(uint64_t factor1, uint64_t factor2, uint64_t *product)
 173 {
 174         uint64_t hi1, hi2, lo1, lo2;
 175         uint64_t tmp[2];
 176
 177         hi1 = factor1 >> 32;
 178         hi2 = factor2 >> 32;
 179
 180         lo1 = factor1 & DT_MASK_LO;
 181         lo2 = factor2 & DT_MASK_LO;
 182
 183         product[0] = lo1 * lo2;
 184         product[1] = hi1 * hi2;
 185
 186         tmp[0] = hi1 * lo2;
 187         tmp[1] = 0;
 188         dt_shift_128(tmp, 32);
 189         dt_add_128(product, tmp, product);
 190
 191         tmp[0] = hi2 * lo1;
 192         tmp[1] = 0;
 193         dt_shift_128(tmp, 32);
 194         dt_add_128(product, tmp, product);
 195 }
 196
 197 /*
 198  * This is long-hand division.
 199  *
 200  * We initialize subtrahend by shifting divisor left as far as possible. We
 201  * loop, comparing subtrahend to dividend:  if subtrahend is smaller, we
 202  * subtract and set the appropriate bit in the result.  We then shift
 203  * subtrahend right by one bit for the next comparison.
 204  */
 205 static void
 206 dt_divide_128(uint64_t *dividend, uint64_t divisor, uint64_t *quotient)
 207 {
 208         uint64_t result[2] = { 0, 0 };
 209         uint64_t remainder[2];
 210         uint64_t subtrahend[2];
 211         uint64_t divisor_128[2];
 212         uint64_t mask[2] = { 1, 0 };
 213         int log = 0;
 214
 215         assert(divisor != 0);
 216
 217         divisor_128[0] = divisor;
 218         divisor_128[1] = 0;
 219
 220         remainder[0] = dividend[0];
 221         remainder[1] = dividend[1];
 222
 223         subtrahend[0] = divisor;
 224         subtrahend[1] = 0;
 225
 226         while (divisor > 0) {
 227                 log++;
 228                 divisor >>= 1;
 229         }
 230
 231         dt_shift_128(subtrahend, 128 - log);
 232         dt_shift_128(mask, 128 - log);
 233
 234         while (dt_ge_128(remainder, divisor_128)) {
 235                 if (dt_ge_128(remainder, subtrahend)) {
 236                         dt_subtract_128(remainder, subtrahend, remainder);
 237                         result[0] |= mask[0];
 238                         result[1] |= mask[1];
 239                 }
 240
 241                 dt_shift_128(subtrahend, -1);
 242                 dt_shift_128(mask, -1);
 243         }
 244
 245         quotient[0] = result[0];
 246         quotient[1] = result[1];
 247 }
 248
 249 /*
 250  * This is the long-hand method of calculating a square root.
 251  * The algorithm is as follows:
 252  *
 253  * 1. Group the digits by 2 from the right.
 254  * 2. Over the leftmost group, find the largest single-digit number
 255  *    whose square is less than that group.
 256  * 3. Subtract the result of the previous step (2 or 4, depending) and
 257  *    bring down the next two-digit group.
 258  * 4. For the result R we have so far, find the largest single-digit number
 259  *    x such that 2 * R * 10 * x + x^2 is less than the result from step 3.
 260  *    (Note that this is doubling R and performing a decimal left-shift by 1
 261  *    and searching for the appropriate decimal to fill the one's place.)
 262  *    The value x is the next digit in the square root.
 263  * Repeat steps 3 and 4 until the desired precision is reached.  (We're
 264  * dealing with integers, so the above is sufficient.)
 265  *
 266  * In decimal, the square root of 582,734 would be calculated as so:
 267  *
 268  *     __7__6__3
 269  *    | 58 27 34
 270  *     -49       (7^2 == 49 => 7 is the first digit in the square root)
 271  *      --
 272  *       9 27    (Subtract and bring down the next group.)
 273  * 146   8 76    (2 * 7 * 10 * 6 + 6^2 == 876 => 6 is the next digit in
 274  *      -----     the square root)
 275  *         51 34 (Subtract and bring down the next group.)
 276  * 1523    45 69 (2 * 76 * 10 * 3 + 3^2 == 4569 => 3 is the next digit in
 277  *         -----  the square root)
 278  *          5 65 (remainder)
 279  *
 280  * The above algorithm applies similarly in binary, but note that the
 281  * only possible non-zero value for x in step 4 is 1, so step 4 becomes a
 282  * simple decision: is 2 * R * 2 * 1 + 1^2 (aka R << 2 + 1) less than the
 283  * preceding difference?
 284  *
 285  * In binary, the square root of 11011011 would be calculated as so:
 286  *
 287  *     __1__1__1__0
 288  *    | 11 01 10 11
 289  *      01          (0 << 2 + 1 == 1 < 11 => this bit is 1)
 290  *      --
 291  *      10 01 10 11
 292  * 101   1 01       (1 << 2 + 1 == 101 < 1001 => next bit is 1)
 293  *      -----
 294  *       1 00 10 11
 295  * 1101    11 01    (11 << 2 + 1 == 1101 < 10010 => next bit is 1)
 296  *       -------
 297  *          1 01 11
 298  * 11101    1 11 01 (111 << 2 + 1 == 11101 > 10111 => last bit is 0)
 299  *
 300  */
 301 static uint64_t
 302 dt_sqrt_128(uint64_t *square)
 303 {
 304         uint64_t result[2] = { 0, 0 };
 305         uint64_t diff[2] = { 0, 0 };
 306         uint64_t one[2] = { 1, 0 };
 307         uint64_t next_pair[2];
 308         uint64_t next_try[2];
 309         uint64_t bit_pairs, pair_shift;
 310         int i;
 311
 312         bit_pairs = dt_nbits_128(square) / 2;
 313         pair_shift = bit_pairs * 2;
 314
 315         for (i = 0; i <= bit_pairs; i++) {
 316                 /*
 317                  * Bring down the next pair of bits.
 318                  */
 319                 next_pair[0] = square[0];
 320                 next_pair[1] = square[1];
 321                 dt_shift_128(next_pair, -pair_shift);
 322                 next_pair[0] &= 0x3;
 323                 next_pair[1] = 0;
 324
 325                 dt_shift_128(diff, 2);
 326                 dt_add_128(diff, next_pair, diff);
 327
 328                 /*
 329                  * next_try = R << 2 + 1
 330                  */
 331                 next_try[0] = result[0];
 332                 next_try[1] = result[1];
 333                 dt_shift_128(next_try, 2);
 334                 dt_add_128(next_try, one, next_try);
 335
 336                 if (dt_le_128(next_try, diff)) {
 337                         dt_subtract_128(diff, next_try, diff);
 338                         dt_shift_128(result, 1);
 339                         dt_add_128(result, one, result);
 340                 } else {
 341                         dt_shift_128(result, 1);
 342                 }
 343
 344                 pair_shift -= 2;
 345         }
 346
 347         assert(result[1] == 0);
 348
 349         return (result[0]);
 350 }
 351
 352 uint64_t
 353 dt_stddev(uint64_t *data, uint64_t normal)
 354 {
 355         uint64_t avg_of_squares[2];
 356         uint64_t square_of_avg[2];
 357         int64_t norm_avg;
 358         uint64_t diff[2];
 359
 360         /*
 361          * The standard approximation for standard deviation is
 362          * sqrt(average(x**2) - average(x)**2), i.e. the square root
 363          * of the average of the squares minus the square of the average.
 364          */
 365         dt_divide_128(data + 2, normal, avg_of_squares);
 366         dt_divide_128(avg_of_squares, data[0], avg_of_squares);
 367
 368         norm_avg = (int64_t)data[1] / (int64_t)normal / (int64_t)data[0];
 369
 370         if (norm_avg < 0)
 371                 norm_avg = -norm_avg;
 372
 373         dt_multiply_128((uint64_t)norm_avg, (uint64_t)norm_avg, square_of_avg);
 374
 375         dt_subtract_128(avg_of_squares, square_of_avg, diff);
 376
 377         return (dt_sqrt_128(diff));
 378 }
 379
 380 static int
 381 dt_flowindent(dtrace_hdl_t *dtp, dtrace_probedata_t *data, dtrace_epid_t last,
 382     dtrace_bufdesc_t *buf, size_t offs)
 383 {
 384         dtrace_probedesc_t *pd = data->dtpda_pdesc, *npd;
 385         dtrace_eprobedesc_t *epd = data->dtpda_edesc, *nepd;
 386         char *p = pd->dtpd_provider, *n = pd->dtpd_name, *sub;
 387         dtrace_flowkind_t flow = DTRACEFLOW_NONE;
 388         const char *str = NULL;
 389         static const char *e_str[2] = { " -> ", " => " };
 390         static const char *r_str[2] = { " <- ", " <= " };
 391         static const char *ent = "entry", *ret = "return";
 392         static int entlen = 0, retlen = 0;
 393         dtrace_epid_t next, id = epd->dtepd_epid;
 394         int rval;
 395
 396         if (entlen == 0) {
 397                 assert(retlen == 0);
 398                 entlen = strlen(ent);
 399                 retlen = strlen(ret);
 400         }
 401
 402         /*
 403          * If the name of the probe is "entry" or ends with "-entry", we
 404          * treat it as an entry; if it is "return" or ends with "-return",
 405          * we treat it as a return.  (This allows application-provided probes
 406          * like "method-entry" or "function-entry" to participate in flow
 407          * indentation -- without accidentally misinterpreting popular probe
 408          * names like "carpentry", "gentry" or "Coventry".)
 409          */
 410         if ((sub = strstr(n, ent)) != NULL && sub[entlen] == '\0' &&
 411             (sub == n || sub[-1] == '-')) {
 412                 flow = DTRACEFLOW_ENTRY;
 413                 str = e_str[strcmp(p, "syscall") == 0];
 414         } else if ((sub = strstr(n, ret)) != NULL && sub[retlen] == '\0' &&
 415             (sub == n || sub[-1] == '-')) {
 416                 flow = DTRACEFLOW_RETURN;
 417                 str = r_str[strcmp(p, "syscall") == 0];
 418         }
 419
 420         /*
 421          * If we're going to indent this, we need to check the ID of our last
 422          * call.  If we're looking at the same probe ID but a different EPID,
 423          * we _don't_ want to indent.  (Yes, there are some minor holes in
 424          * this scheme -- it's a heuristic.)
 425          */
 426         if (flow == DTRACEFLOW_ENTRY) {
 427                 if ((last != DTRACE_EPIDNONE && id != last &&
 428                     pd->dtpd_id == dtp->dt_pdesc[last]->dtpd_id))
 429                         flow = DTRACEFLOW_NONE;
 430         }
 431
 432         /*
 433          * If we're going to unindent this, it's more difficult to see if
 434          * we don't actually want to unindent it -- we need to look at the
 435          * _next_ EPID.
 436          */
 437         if (flow == DTRACEFLOW_RETURN) {
 438                 offs += epd->dtepd_size;
 439
 440                 do {
 441                         if (offs >= buf->dtbd_size) {
 442                                 /*
 443                                  * We're at the end -- maybe.  If the oldest
 444                                  * record is non-zero, we need to wrap.
 445                                  */
 446                                 if (buf->dtbd_oldest != 0) {
 447                                         offs = 0;
 448                                 } else {
 449                                         goto out;
 450                                 }
 451                         }
 452
 453                         next = *(uint32_t *)((uintptr_t)buf->dtbd_data + offs);
 454
 455                         if (next == DTRACE_EPIDNONE)
 456                                 offs += sizeof (id);
 457                 } while (next == DTRACE_EPIDNONE);
 458
 459                 if ((rval = dt_epid_lookup(dtp, next, &nepd, &npd)) != 0)
 460                         return (rval);
 461
 462                 if (next != id && npd->dtpd_id == pd->dtpd_id)
 463                         flow = DTRACEFLOW_NONE;
 464         }
 465
 466 out:
 467         if (flow == DTRACEFLOW_ENTRY || flow == DTRACEFLOW_RETURN) {
 468                 data->dtpda_prefix = str;
 469         } else {
 470                 data->dtpda_prefix = "| ";
 471         }
 472
 473         if (flow == DTRACEFLOW_RETURN && data->dtpda_indent > 0)
 474                 data->dtpda_indent -= 2;
 475
 476         data->dtpda_flow = flow;
 477
 478         return (0);
 479 }
 480
 481 static int
 482 dt_nullprobe()
 483 {
 484         return (DTRACE_CONSUME_THIS);
 485 }
 486
 487 static int
 488 dt_nullrec()
 489 {
 490         return (DTRACE_CONSUME_NEXT);
 491 }
 492
 493 int
 494 dt_print_quantline(dtrace_hdl_t *dtp, FILE *fp, int64_t val,
 495     uint64_t normal, long double total, char positives, char negatives)
 496 {
 497         long double f;
 498         uint_t depth, len = 40;
 499
 500         const char *ats = "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@";
 501         const char *spaces = "                                        ";
 502
 503         assert(strlen(ats) == len && strlen(spaces) == len);
 504         assert(!(total == 0 && (positives || negatives)));
 505         assert(!(val < 0 && !negatives));
 506         assert(!(val > 0 && !positives));
 507         assert(!(val != 0 && total == 0));
 508
 509         if (!negatives) {
 510                 if (positives) {
 511                         f = (dt_fabsl((long double)val) * len) / total;
 512                         depth = (uint_t)(f + 0.5);
 513                 } else {
 514                         depth = 0;
 515                 }
 516
 517                 return (dt_printf(dtp, fp, "|%s%s %-9lld\n", ats + len - depth,
 518                     spaces + depth, (long long)val / normal));
 519         }
 520
 521         if (!positives) {
 522                 f = (dt_fabsl((long double)val) * len) / total;
 523                 depth = (uint_t)(f + 0.5);
 524
 525                 return (dt_printf(dtp, fp, "%s%s| %-9lld\n", spaces + depth,
 526                     ats + len - depth, (long long)val / normal));
 527         }
 528
 529         /*
 530          * If we're here, we have both positive and negative bucket values.
 531          * To express this graphically, we're going to generate both positive
 532          * and negative bars separated by a centerline.  These bars are half
 533          * the size of normal quantize()/lquantize() bars, so we divide the
 534          * length in half before calculating the bar length.
 535          */
 536         len /= 2;
 537         ats = &ats[len];
 538         spaces = &spaces[len];
 539
 540         f = (dt_fabsl((long double)val) * len) / total;
 541         depth = (uint_t)(f + 0.5);
 542
 543         if (val <= 0) {
 544                 return (dt_printf(dtp, fp, "%s%s|%*s %-9lld\n", spaces + depth,
 545                     ats + len - depth, len, "", (long long)val / normal));
 546         } else {
 547                 return (dt_printf(dtp, fp, "%20s|%s%s %-9lld\n", "",
 548                     ats + len - depth, spaces + depth,
 549                     (long long)val / normal));
 550         }
 551 }
 552
 553 int
 554 dt_print_quantize(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
 555     size_t size, uint64_t normal)
 556 {
 557         const int64_t *data = addr;
 558         int i, first_bin = 0, last_bin = DTRACE_QUANTIZE_NBUCKETS - 1;
 559         long double total = 0;
 560         char positives = 0, negatives = 0;
 561
 562         if (size != DTRACE_QUANTIZE_NBUCKETS * sizeof (uint64_t))
 563                 return (dt_set_errno(dtp, EDT_DMISMATCH));
 564
 565         while (first_bin < DTRACE_QUANTIZE_NBUCKETS - 1 && data[first_bin] == 0)
 566                 first_bin++;
 567
 568         if (first_bin == DTRACE_QUANTIZE_NBUCKETS - 1) {
 569                 /*
 570                  * There isn't any data.  This is possible if (and only if)
 571                  * negative increment values have been used.  In this case,
 572                  * we'll print the buckets around 0.
 573                  */
 574                 first_bin = DTRACE_QUANTIZE_ZEROBUCKET - 1;
 575                 last_bin = DTRACE_QUANTIZE_ZEROBUCKET + 1;
 576         } else {
 577                 if (first_bin > 0)
 578                         first_bin--;
 579
 580                 while (last_bin > 0 && data[last_bin] == 0)
 581                         last_bin--;
 582
 583                 if (last_bin < DTRACE_QUANTIZE_NBUCKETS - 1)
 584                         last_bin++;
 585         }
 586
 587         for (i = first_bin; i <= last_bin; i++) {
 588                 positives |= (data[i] > 0);
 589                 negatives |= (data[i] < 0);
 590                 total += dt_fabsl((long double)data[i]);
 591         }
 592
 593         if (dt_printf(dtp, fp, "\n%16s %41s %-9s\n", "value",
 594             "------------- Distribution -------------", "count") < 0)
 595                 return (-1);
 596
 597         for (i = first_bin; i <= last_bin; i++) {
 598                 if (dt_printf(dtp, fp, "%16lld ",
 599                     (long long)DTRACE_QUANTIZE_BUCKETVAL(i)) < 0)
 600                         return (-1);
 601
 602                 if (dt_print_quantline(dtp, fp, data[i], normal, total,
 603                     positives, negatives) < 0)
 604                         return (-1);
 605         }
 606
 607         return (0);
 608 }
 609
 610 int
 611 dt_print_lquantize(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
 612     size_t size, uint64_t normal)
 613 {
 614         const int64_t *data = addr;
 615         int i, first_bin, last_bin, base;
 616         uint64_t arg;
 617         long double total = 0;
 618         uint16_t step, levels;
 619         char positives = 0, negatives = 0;
 620
 621         if (size < sizeof (uint64_t))
 622                 return (dt_set_errno(dtp, EDT_DMISMATCH));
 623
 624         arg = *data++;
 625         size -= sizeof (uint64_t);
 626
 627         base = DTRACE_LQUANTIZE_BASE(arg);
 628         step = DTRACE_LQUANTIZE_STEP(arg);
 629         levels = DTRACE_LQUANTIZE_LEVELS(arg);
 630
 631         first_bin = 0;
 632         last_bin = levels + 1;
 633
 634         if (size != sizeof (uint64_t) * (levels + 2))
 635                 return (dt_set_errno(dtp, EDT_DMISMATCH));
 636
 637         while (first_bin <= levels + 1 && data[first_bin] == 0)
 638                 first_bin++;
 639
 640         if (first_bin > levels + 1) {
 641                 first_bin = 0;
 642                 last_bin = 2;
 643         } else {
 644                 if (first_bin > 0)
 645                         first_bin--;
 646
 647                 while (last_bin > 0 && data[last_bin] == 0)
 648                         last_bin--;
 649
 650                 if (last_bin < levels + 1)
 651                         last_bin++;
 652         }
 653
 654         for (i = first_bin; i <= last_bin; i++) {
 655                 positives |= (data[i] > 0);
 656                 negatives |= (data[i] < 0);
 657                 total += dt_fabsl((long double)data[i]);
 658         }
 659
 660         if (dt_printf(dtp, fp, "\n%16s %41s %-9s\n", "value",
 661             "------------- Distribution -------------", "count") < 0)
 662                 return (-1);
 663
 664         for (i = first_bin; i <= last_bin; i++) {
 665                 char c[32];
 666                 int err;
 667
 668                 if (i == 0) {
 669                         (void) snprintf(c, sizeof (c), "< %d",
 670                             base / (uint32_t)normal);
 671                         err = dt_printf(dtp, fp, "%16s ", c);
 672                 } else if (i == levels + 1) {
 673                         (void) snprintf(c, sizeof (c), ">= %d",
 674                             base + (levels * step));
 675                         err = dt_printf(dtp, fp, "%16s ", c);
 676                 } else {
 677                         err = dt_printf(dtp, fp, "%16d ",
 678                             base + (i - 1) * step);
 679                 }
 680
 681                 if (err < 0 || dt_print_quantline(dtp, fp, data[i], normal,
 682                     total, positives, negatives) < 0)
 683                         return (-1);
 684         }
 685
 686         return (0);
 687 }
 688
 689 /*ARGSUSED*/
 690 static int
 691 dt_print_average(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr,
 692     size_t size, uint64_t normal)
 693 {
 694         /* LINTED - alignment */
 695         int64_t *data = (int64_t *)addr;
 696
 697         return (dt_printf(dtp, fp, " %16lld", data[0] ?
 698             (long long)(data[1] / (int64_t)normal / data[0]) : 0));
 699 }
 700
 701 /*ARGSUSED*/
 702 static int
 703 dt_print_stddev(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr,
 704     size_t size, uint64_t normal)
 705 {
 706         /* LINTED - alignment */
 707         uint64_t *data = (uint64_t *)addr;
 708
 709         return (dt_printf(dtp, fp, " %16llu", data[0] ?
 710             (unsigned long long) dt_stddev(data, normal) : 0));
 711 }
 712
 713 /*ARGSUSED*/
 714 int
 715 dt_print_bytes(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr,
 716     size_t nbytes, int width, int quiet, int raw)
 717 {
 718         /*
 719          * If the byte stream is a series of printable characters, followed by
 720          * a terminating byte, we print it out as a string.  Otherwise, we
 721          * assume that it's something else and just print the bytes.
 722          */
 723         int i, j, margin = 5;
 724         char *c = (char *)addr;
 725
 726         if (nbytes == 0)
 727                 return (0);
 728
 729         if (raw || dtp->dt_options[DTRACEOPT_RAWBYTES] != DTRACEOPT_UNSET)
 730                 goto raw;
 731
 732         for (i = 0; i < nbytes; i++) {
 733                 /*
 734                  * We define a "printable character" to be one for which
 735                  * isprint(3C) returns non-zero, isspace(3C) returns non-zero,
 736                  * or a character which is either backspace or the bell.
 737                  * Backspace and the bell are regrettably special because
 738                  * they fail the first two tests -- and yet they are entirely
 739                  * printable.  These are the only two control characters that
 740                  * have meaning for the terminal and for which isprint(3C) and
 741                  * isspace(3C) return 0.
 742                  */
 743                 if (isprint(c[i]) || isspace(c[i]) ||
 744                     c[i] == '\b' || c[i] == '\a')
 745                         continue;
 746
 747                 if (c[i] == '\0' && i > 0) {
 748                         /*
 749                          * This looks like it might be a string.  Before we
 750                          * assume that it is indeed a string, check the
 751                          * remainder of the byte range; if it contains
 752                          * additional non-nul characters, we'll assume that
 753                          * it's a binary stream that just happens to look like
 754                          * a string, and we'll print out the individual bytes.
 755                          */
 756                         for (j = i + 1; j < nbytes; j++) {
 757                                 if (c[j] != '\0')
 758                                         break;
 759                         }
 760
 761                         if (j != nbytes)
 762                                 break;
 763
 764                         if (quiet)
 765                                 return (dt_printf(dtp, fp, "%s", c));
 766                         else
 767                                 return (dt_printf(dtp, fp, "  %-*s", width, c));
 768                 }
 769
 770                 break;
 771         }
 772
 773         if (i == nbytes) {
 774                 /*
 775                  * The byte range is all printable characters, but there is
 776                  * no trailing nul byte.  We'll assume that it's a string and
 777                  * print it as such.
 778                  */
 779                 char *s = alloca(nbytes + 1);
 780                 bcopy(c, s, nbytes);
 781                 s[nbytes] = '\0';
 782                 return (dt_printf(dtp, fp, "  %-*s", width, s));
 783         }
 784
 785 raw:
 786         if (dt_printf(dtp, fp, "\n%*s      ", margin, "") < 0)
 787                 return (-1);
 788
 789         for (i = 0; i < 16; i++)
 790                 if (dt_printf(dtp, fp, "  %c", "0123456789abcdef"[i]) < 0)
 791                         return (-1);
 792
 793         if (dt_printf(dtp, fp, "  0123456789abcdef\n") < 0)
 794                 return (-1);
 795
 796
 797         for (i = 0; i < nbytes; i += 16) {
 798                 if (dt_printf(dtp, fp, "%*s%5x:", margin, "", i) < 0)
 799                         return (-1);
 800
 801                 for (j = i; j < i + 16 && j < nbytes; j++) {
 802                         if (dt_printf(dtp, fp, " %02x", (uchar_t)c[j]) < 0)
 803                                 return (-1);
 804                 }
 805
 806                 while (j++ % 16) {
 807                         if (dt_printf(dtp, fp, "   ") < 0)
 808                                 return (-1);
 809                 }
 810
 811                 if (dt_printf(dtp, fp, "  ") < 0)
 812                         return (-1);
 813
 814                 for (j = i; j < i + 16 && j < nbytes; j++) {
 815                         if (dt_printf(dtp, fp, "%c",
 816                             c[j] < ' ' || c[j] > '~' ? '.' : c[j]) < 0)
 817                                 return (-1);
 818                 }
 819
 820                 if (dt_printf(dtp, fp, "\n") < 0)
 821                         return (-1);
 822         }
 823
 824         return (0);
 825 }
 826
 827 int
 828 dt_print_stack(dtrace_hdl_t *dtp, FILE *fp, const char *format,
 829     caddr_t addr, int depth, int size)
 830 {
 831         dtrace_syminfo_t dts;
 832         GElf_Sym sym;
 833         int i, indent;
 834         char c[PATH_MAX * 2];
 835         uint64_t pc;
 836
 837         if (dt_printf(dtp, fp, "\n") < 0)
 838                 return (-1);
 839
 840         if (format == NULL)
 841                 format = "%s";
 842
 843         if (dtp->dt_options[DTRACEOPT_STACKINDENT] != DTRACEOPT_UNSET)
 844                 indent = (int)dtp->dt_options[DTRACEOPT_STACKINDENT];
 845         else
 846                 indent = _dtrace_stkindent;
 847
 848         for (i = 0; i < depth; i++) {
 849                 switch (size) {
 850                 case sizeof (uint32_t):
 851                         /* LINTED - alignment */
 852                         pc = *((uint32_t *)addr);
 853                         break;
 854
 855                 case sizeof (uint64_t):
 856                         /* LINTED - alignment */
 857                         pc = *((uint64_t *)addr);
 858                         break;
 859
 860                 default:
 861                         return (dt_set_errno(dtp, EDT_BADSTACKPC));
 862                 }
 863
 864                 if (pc == 0)
 865                         break;
 866
 867                 addr += size;
 868
 869                 if (dt_printf(dtp, fp, "%*s", indent, "") < 0)
 870                         return (-1);
 871
 872                 if (dtrace_lookup_by_addr(dtp, pc, &sym, &dts) == 0) {
 873                         if (pc > sym.st_value) {
 874                                 (void) snprintf(c, sizeof (c), "%s`%s+0x%llx",
 875                                     dts.dts_object, dts.dts_name,
 876                                     pc - sym.st_value);
 877                         } else {
 878                                 (void) snprintf(c, sizeof (c), "%s`%s",
 879                                     dts.dts_object, dts.dts_name);
 880                         }
 881                 } else {
 882                         /*
 883                          * We'll repeat the lookup, but this time we'll specify
 884                          * a NULL GElf_Sym -- indicating that we're only
 885                          * interested in the containing module.
 886                          */
 887                         if (dtrace_lookup_by_addr(dtp, pc, NULL, &dts) == 0) {
 888                                 (void) snprintf(c, sizeof (c), "%s`0x%llx",
 889                                     dts.dts_object, pc);
 890                         } else {
 891                                 (void) snprintf(c, sizeof (c), "0x%llx", pc);
 892                         }
 893                 }
 894
 895                 if (dt_printf(dtp, fp, format, c) < 0)
 896                         return (-1);
 897
 898                 if (dt_printf(dtp, fp, "\n") < 0)
 899                         return (-1);
 900         }
 901
 902         return (0);
 903 }
 904
 905 int
 906 dt_print_ustack(dtrace_hdl_t *dtp, FILE *fp, const char *format,
 907     caddr_t addr, uint64_t arg)
 908 {
 909         /* LINTED - alignment */
 910         uint64_t *pc = (uint64_t *)addr;
 911         uint32_t depth = DTRACE_USTACK_NFRAMES(arg);
 912         uint32_t strsize = DTRACE_USTACK_STRSIZE(arg);
 913         const char *strbase = addr + (depth + 1) * sizeof (uint64_t);
 914         const char *str = strsize ? strbase : NULL;
 915         int err = 0;
 916
 917         char name[PATH_MAX], objname[PATH_MAX], c[PATH_MAX * 2];
 918         struct ps_prochandle *P;
 919         GElf_Sym sym;
 920         int i, indent;
 921         pid_t pid;
 922
 923         if (depth == 0)
 924                 return (0);
 925
 926         pid = (pid_t)*pc++;
 927
 928         if (dt_printf(dtp, fp, "\n") < 0)
 929                 return (-1);
 930
 931         if (format == NULL)
 932                 format = "%s";
 933
 934         if (dtp->dt_options[DTRACEOPT_STACKINDENT] != DTRACEOPT_UNSET)
 935                 indent = (int)dtp->dt_options[DTRACEOPT_STACKINDENT];
 936         else
 937                 indent = _dtrace_stkindent;
 938
 939         /*
 940          * Ultimately, we need to add an entry point in the library vector for
 941          * determining <symbol, offset> from <pid, address>.  For now, if
 942          * this is a vector open, we just print the raw address or string.
 943          */
 944         if (dtp->dt_vector == NULL)
 945                 P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0);
 946         else
 947                 P = NULL;
 948
 949         if (P != NULL)
 950                 dt_proc_lock(dtp, P); /* lock handle while we perform lookups */
 951
 952         for (i = 0; i < depth && pc[i] != 0; i++) {
 953                 const prmap_t *map;
 954
 955                 if ((err = dt_printf(dtp, fp, "%*s", indent, "")) < 0)
 956                         break;
 957
 958                 if (P != NULL && Plookup_by_addr(P, pc[i],
 959                     name, sizeof (name), &sym) == 0) {
 960                         (void) Pobjname(P, pc[i], objname, sizeof (objname));
 961
 962                         if (pc[i] > sym.st_value) {
 963                                 (void) snprintf(c, sizeof (c),
 964                                     "%s`%s+0x%llx", dt_basename(objname), name,
 965                                     (u_longlong_t)(pc[i] - sym.st_value));
 966                         } else {
 967                                 (void) snprintf(c, sizeof (c),
 968                                     "%s`%s", dt_basename(objname), name);
 969                         }
 970                 } else if (str != NULL && str[0] != '\0' && str[0] != '@' &&
 971                     (P != NULL && ((map = Paddr_to_map(P, pc[i])) == NULL ||
 972                     (map->pr_mflags & MA_WRITE)))) {
 973                         /*
 974                          * If the current string pointer in the string table
 975                          * does not point to an empty string _and_ the program
 976                          * counter falls in a writable region, we'll use the
 977                          * string from the string table instead of the raw
 978                          * address.  This last condition is necessary because
 979                          * some (broken) ustack helpers will return a string
 980                          * even for a program counter that they can't
 981                          * identify.  If we have a string for a program
 982                          * counter that falls in a segment that isn't
 983                          * writable, we assume that we have fallen into this
 984                          * case and we refuse to use the string.
 985                          */
 986                         (void) snprintf(c, sizeof (c), "%s", str);
 987                 } else {
 988                         if (P != NULL && Pobjname(P, pc[i], objname,
 989                             sizeof (objname)) != 0) {
 990                                 (void) snprintf(c, sizeof (c), "%s`0x%llx",
 991                                     dt_basename(objname), (u_longlong_t)pc[i]);
 992                         } else {
 993                                 (void) snprintf(c, sizeof (c), "0x%llx",
 994                                     (u_longlong_t)pc[i]);
 995                         }
 996                 }
 997
 998                 if ((err = dt_printf(dtp, fp, format, c)) < 0)
 999                         break;
1000
1001                 if ((err = dt_printf(dtp, fp, "\n")) < 0)
1002                         break;
1003
1004                 if (str != NULL && str[0] == '@') {
1005                         /*
1006                          * If the first character of the string is an "at" sign,
1007                          * then the string is inferred to be an annotation --
1008                          * and it is printed out beneath the frame and offset
1009                          * with brackets.
1010                          */
1011                         if ((err = dt_printf(dtp, fp, "%*s", indent, "")) < 0)
1012                                 break;
1013
1014                         (void) snprintf(c, sizeof (c), "  [ %s ]", &str[1]);
1015
1016                         if ((err = dt_printf(dtp, fp, format, c)) < 0)
1017                                 break;
1018
1019                         if ((err = dt_printf(dtp, fp, "\n")) < 0)
1020                                 break;
1021                 }
1022
1023                 if (str != NULL) {
1024                         str += strlen(str) + 1;
1025                         if (str - strbase >= strsize)
1026                                 str = NULL;
1027                 }
1028         }
1029
1030         if (P != NULL) {
1031                 dt_proc_unlock(dtp, P);
1032                 dt_proc_release(dtp, P);
1033         }
1034
1035         return (err);
1036 }
1037
1038 static int
1039 dt_print_usym(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr, dtrace_actkind_t act)
1040 {
1041         /* LINTED - alignment */
1042         uint64_t pid = ((uint64_t *)addr)[0];
1043         /* LINTED - alignment */
1044         uint64_t pc = ((uint64_t *)addr)[1];
1045         const char *format = "  %-50s";
1046         char *s;
1047         int n, len = 256;
1048
1049         if (act == DTRACEACT_USYM && dtp->dt_vector == NULL) {
1050                 struct ps_prochandle *P;
1051
1052                 if ((P = dt_proc_grab(dtp, pid,
1053                     PGRAB_RDONLY | PGRAB_FORCE, 0)) != NULL) {
1054                         GElf_Sym sym;
1055
1056                         dt_proc_lock(dtp, P);
1057
1058                         if (Plookup_by_addr(P, pc, NULL, 0, &sym) == 0)
1059                                 pc = sym.st_value;
1060
1061                         dt_proc_unlock(dtp, P);
1062                         dt_proc_release(dtp, P);
1063                 }
1064         }
1065
1066         do {
1067                 n = len;
1068                 s = alloca(n);
1069         } while ((len = dtrace_uaddr2str(dtp, pid, pc, s, n)) > n);
1070
1071         return (dt_printf(dtp, fp, format, s));
1072 }
1073
1074 int
1075 dt_print_umod(dtrace_hdl_t *dtp, FILE *fp, const char *format, caddr_t addr)
1076 {
1077         /* LINTED - alignment */
1078         uint64_t pid = ((uint64_t *)addr)[0];
1079         /* LINTED - alignment */
1080         uint64_t pc = ((uint64_t *)addr)[1];
1081         int err = 0;
1082
1083         char objname[PATH_MAX], c[PATH_MAX * 2];
1084         struct ps_prochandle *P;
1085
1086         if (format == NULL)
1087                 format = "  %-50s";
1088
1089         /*
1090          * See the comment in dt_print_ustack() for the rationale for
1091          * printing raw addresses in the vectored case.
1092          */
1093         if (dtp->dt_vector == NULL)
1094                 P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0);
1095         else
1096                 P = NULL;
1097
1098         if (P != NULL)
1099                 dt_proc_lock(dtp, P); /* lock handle while we perform lookups */
1100
1101         if (P != NULL && Pobjname(P, pc, objname, sizeof (objname)) != 0) {
1102                 (void) snprintf(c, sizeof (c), "%s", dt_basename(objname));
1103         } else {
1104                 (void) snprintf(c, sizeof (c), "0x%llx", (u_longlong_t)pc);
1105         }
1106
1107         err = dt_printf(dtp, fp, format, c);
1108
1109         if (P != NULL) {
1110                 dt_proc_unlock(dtp, P);
1111                 dt_proc_release(dtp, P);
1112         }
1113
1114         return (err);
1115 }
1116
1117 int
1118 dt_print_memory(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr)
1119 {
1120         int quiet = (dtp->dt_options[DTRACEOPT_QUIET] != DTRACEOPT_UNSET);
1121         size_t nbytes = *((uintptr_t *) addr);
1122
1123         return (dt_print_bytes(dtp, fp, addr + sizeof(uintptr_t),
1124             nbytes, 50, quiet, 1));
1125 }
1126
1127 typedef struct dt_type_cbdata {
1128         dtrace_hdl_t            *dtp;
1129         dtrace_typeinfo_t       dtt;
1130         caddr_t                 addr;
1131         caddr_t                 addrend;
1132         const char              *name;
1133         int                     f_type;
1134         int                     indent;
1135         int                     type_width;
1136         int                     name_width;
1137         FILE                    *fp;
1138 } dt_type_cbdata_t;
1139
1140 static int      dt_print_type_data(dt_type_cbdata_t *, ctf_id_t);
1141
1142 static int
1143 dt_print_type_member(const char *name, ctf_id_t type, ulong_t off, void *arg)
1144 {
1145         dt_type_cbdata_t cbdata;
1146         dt_type_cbdata_t *cbdatap = arg;
1147         ssize_t ssz;
1148
1149         if ((ssz = ctf_type_size(cbdatap->dtt.dtt_ctfp, type)) <= 0)
1150                 return (0);
1151
1152         off /= 8;
1153
1154         cbdata = *cbdatap;
1155         cbdata.name = name;
1156         cbdata.addr += off;
1157         cbdata.addrend = cbdata.addr + ssz;
1158
1159         return (dt_print_type_data(&cbdata, type));
1160 }
1161
1162 static int
1163 dt_print_type_width(const char *name, ctf_id_t type, ulong_t off, void *arg)
1164 {
1165         char buf[DT_TYPE_NAMELEN];
1166         char *p;
1167         dt_type_cbdata_t *cbdatap = arg;
1168         size_t sz = strlen(name);
1169
1170         ctf_type_name(cbdatap->dtt.dtt_ctfp, type, buf, sizeof (buf));
1171
1172         if ((p = strchr(buf, '[')) != NULL)
1173                 p[-1] = '\0';
1174         else
1175                 p = "";
1176
1177         sz += strlen(p);
1178
1179         if (sz > cbdatap->name_width)
1180                 cbdatap->name_width = sz;
1181
1182         sz = strlen(buf);
1183
1184         if (sz > cbdatap->type_width)
1185                 cbdatap->type_width = sz;
1186
1187         return (0);
1188 }
1189
1190 static int
1191 dt_print_type_data(dt_type_cbdata_t *cbdatap, ctf_id_t type)
1192 {
1193         caddr_t addr = cbdatap->addr;
1194         caddr_t addrend = cbdatap->addrend;
1195         char buf[DT_TYPE_NAMELEN];
1196         char *p;
1197         int cnt = 0;
1198         uint_t kind = ctf_type_kind(cbdatap->dtt.dtt_ctfp, type);
1199         ssize_t ssz = ctf_type_size(cbdatap->dtt.dtt_ctfp, type);
1200
1201         ctf_type_name(cbdatap->dtt.dtt_ctfp, type, buf, sizeof (buf));
1202
1203         if ((p = strchr(buf, '[')) != NULL)
1204                 p[-1] = '\0';
1205         else
1206                 p = "";
1207
1208         if (cbdatap->f_type) {
1209                 int type_width = roundup(cbdatap->type_width + 1, 4);
1210                 int name_width = roundup(cbdatap->name_width + 1, 4);
1211
1212                 name_width -= strlen(cbdatap->name);
1213
1214                 dt_printf(cbdatap->dtp, cbdatap->fp, "%*s%-*s%s%-*s     = ",cbdatap->indent * 4,"",type_width,buf,cbdatap->name,name_width,p);
1215         }
1216
1217         while (addr < addrend) {
1218                 dt_type_cbdata_t cbdata;
1219                 ctf_arinfo_t arinfo;
1220                 ctf_encoding_t cte;
1221                 uintptr_t *up;
1222                 void *vp = addr;
1223                 cbdata = *cbdatap;
1224                 cbdata.name = "";
1225                 cbdata.addr = addr;
1226                 cbdata.addrend = addr + ssz;
1227                 cbdata.f_type = 0;
1228                 cbdata.indent++;
1229                 cbdata.type_width = 0;
1230                 cbdata.name_width = 0;
1231
1232                 if (cnt > 0)
1233                         dt_printf(cbdatap->dtp, cbdatap->fp, "%*s", cbdatap->indent * 4,"");
1234
1235                 switch (kind) {
1236                 case CTF_K_INTEGER:
1237                         if (ctf_type_encoding(cbdatap->dtt.dtt_ctfp, type, &cte) != 0)
1238                                 return (-1);
1239                         if ((cte.cte_format & CTF_INT_SIGNED) != 0)
1240                                 switch (cte.cte_bits) {
1241                                 case 8:
1242                                         if (isprint(*((char *) vp)))
1243                                                 dt_printf(cbdatap->dtp, cbdatap->fp, "'%c', ", *((char *) vp));
1244                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%d (0x%x);\n", *((char *) vp), *((char *) vp));
1245                                         break;
1246                                 case 16:
1247                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%hd (0x%hx);\n", *((short *) vp), *((u_short *) vp));
1248                                         break;
1249                                 case 32:
1250                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%d (0x%x);\n", *((int *) vp), *((u_int *) vp));
1251                                         break;
1252                                 case 64:
1253                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%jd (0x%jx);\n", *((long long *) vp), *((unsigned long long *) vp));
1254                                         break;
1255                                 default:
1256                                         dt_printf(cbdatap->dtp, cbdatap->fp, "CTF_K_INTEGER: format %x offset %u bits %u\n",cte.cte_format,cte.cte_offset,cte.cte_bits);
1257                                         break;
1258                                 }
1259                         else
1260                                 switch (cte.cte_bits) {
1261                                 case 8:
1262                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%u (0x%x);\n", *((uint8_t *) vp) & 0xff, *((uint8_t *) vp) & 0xff);
1263                                         break;
1264                                 case 16:
1265                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%hu (0x%hx);\n", *((u_short *) vp), *((u_short *) vp));
1266                                         break;
1267                                 case 32:
1268                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%u (0x%x);\n", *((u_int *) vp), *((u_int *) vp));
1269                                         break;
1270                                 case 64:
1271                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%ju (0x%jx);\n", *((unsigned long long *) vp), *((unsigned long long *) vp));
1272                                         break;
1273                                 default:
1274                                         dt_printf(cbdatap->dtp, cbdatap->fp, "CTF_K_INTEGER: format %x offset %u bits %u\n",cte.cte_format,cte.cte_offset,cte.cte_bits);
1275                                         break;
1276                                 }
1277                         break;
1278                 case CTF_K_FLOAT:
1279                         dt_printf(cbdatap->dtp, cbdatap->fp, "CTF_K_FLOAT: format %x offset %u bits %u\n",cte.cte_format,cte.cte_offset,cte.cte_bits);
1280                         break;
1281                 case CTF_K_POINTER:
1282                         dt_printf(cbdatap->dtp, cbdatap->fp, "%p;\n", *((void **) addr));
1283                         break;
1284                 case CTF_K_ARRAY:
1285                         if (ctf_array_info(cbdatap->dtt.dtt_ctfp, type, &arinfo) != 0)
1286                                 return (-1);
1287                         dt_printf(cbdatap->dtp, cbdatap->fp, "{\n%*s",cbdata.indent * 4,"");
1288                         dt_print_type_data(&cbdata, arinfo.ctr_contents);
1289                         dt_printf(cbdatap->dtp, cbdatap->fp, "%*s};\n",cbdatap->indent * 4,"");
1290                         break;
1291                 case CTF_K_FUNCTION:
1292                         dt_printf(cbdatap->dtp, cbdatap->fp, "CTF_K_FUNCTION:\n");
1293                         break;
1294                 case CTF_K_STRUCT:
1295                         cbdata.f_type = 1;
1296                         if (ctf_member_iter(cbdatap->dtt.dtt_ctfp, type,
1297                             dt_print_type_width, &cbdata) != 0)
1298                                 return (-1);
1299                         dt_printf(cbdatap->dtp, cbdatap->fp, "{\n");
1300                         if (ctf_member_iter(cbdatap->dtt.dtt_ctfp, type,
1301                             dt_print_type_member, &cbdata) != 0)
1302                                 return (-1);
1303                         dt_printf(cbdatap->dtp, cbdatap->fp, "%*s};\n",cbdatap->indent * 4,"");
1304                         break;
1305                 case CTF_K_UNION:
1306                         cbdata.f_type = 1;
1307                         if (ctf_member_iter(cbdatap->dtt.dtt_ctfp, type,
1308                             dt_print_type_width, &cbdata) != 0)
1309                                 return (-1);
1310                         dt_printf(cbdatap->dtp, cbdatap->fp, "{\n");
1311                         if (ctf_member_iter(cbdatap->dtt.dtt_ctfp, type,
1312                             dt_print_type_member, &cbdata) != 0)
1313                                 return (-1);
1314                         dt_printf(cbdatap->dtp, cbdatap->fp, "%*s};\n",cbdatap->indent * 4,"");
1315                         break;
1316                 case CTF_K_ENUM:
1317                         dt_printf(cbdatap->dtp, cbdatap->fp, "%s;\n", ctf_enum_name(cbdatap->dtt.dtt_ctfp, type, *((int *) vp)));
1318                         break;
1319                 case CTF_K_TYPEDEF:
1320                         dt_print_type_data(&cbdata, ctf_type_reference(cbdatap->dtt.dtt_ctfp,type));
1321                         break;
1322                 case CTF_K_VOLATILE:
1323                         if (cbdatap->f_type)
1324                                 dt_printf(cbdatap->dtp, cbdatap->fp, "volatile ");
1325                         dt_print_type_data(&cbdata, ctf_type_reference(cbdatap->dtt.dtt_ctfp,type));
1326                         break;
1327                 case CTF_K_CONST:
1328                         if (cbdatap->f_type)
1329                                 dt_printf(cbdatap->dtp, cbdatap->fp, "const ");
1330                         dt_print_type_data(&cbdata, ctf_type_reference(cbdatap->dtt.dtt_ctfp,type));
1331                         break;
1332                 case CTF_K_RESTRICT:
1333                         if (cbdatap->f_type)
1334                                 dt_printf(cbdatap->dtp, cbdatap->fp, "restrict ");
1335                         dt_print_type_data(&cbdata, ctf_type_reference(cbdatap->dtt.dtt_ctfp,type));
1336                         break;
1337                 default:
1338                         break;
1339                 }
1340
1341                 addr += ssz;
1342                 cnt++;
1343         }
1344
1345         return (0);
1346 }
1347
1348 static int
1349 dt_print_type(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr)
1350 {
1351         caddr_t addrend;
1352         char *p;
1353         dtrace_typeinfo_t dtt;
1354         dt_type_cbdata_t cbdata;
1355         int num = 0;
1356         int quiet = (dtp->dt_options[DTRACEOPT_QUIET] != DTRACEOPT_UNSET);
1357         ssize_t ssz;
1358
1359         if (!quiet)
1360                 dt_printf(dtp, fp, "\n");
1361
1362         /* Get the total number of bytes of data buffered. */
1363         size_t nbytes = *((uintptr_t *) addr);
1364         addr += sizeof(uintptr_t);
1365
1366         /*
1367          * Get the size of the type so that we can check that it matches
1368          * the CTF data we look up and so that we can figure out how many
1369          * type elements are buffered.
1370          */
1371         size_t typs = *((uintptr_t *) addr);
1372         addr += sizeof(uintptr_t);
1373
1374         /*
1375          * Point to the type string in the buffer. Get it's string
1376          * length and round it up to become the offset to the start
1377          * of the buffered type data which we would like to be aligned
1378          * for easy access.
1379          */
1380         char *strp = (char *) addr;
1381         int offset = roundup(strlen(strp) + 1, sizeof(uintptr_t));
1382
1383         /*
1384          * The type string might have a format such as 'int [20]'.
1385          * Check if there is an array dimension present.
1386          */
1387         if ((p = strchr(strp, '[')) != NULL) {
1388                 /* Strip off the array dimension. */
1389                 *p++ = '\0';
1390
1391                 for (; *p != '\0' && *p != ']'; p++)
1392                         num = num * 10 + *p - '0';
1393         } else
1394                 /* No array dimension, so default. */
1395                 num = 1;
1396
1397         /* Lookup the CTF type from the type string. */
1398         if (dtrace_lookup_by_type(dtp,  DTRACE_OBJ_EVERY, strp, &dtt) < 0)
1399                 return (-1);
1400
1401         /* Offset the buffer address to the start of the data... */
1402         addr += offset;
1403
1404         ssz = ctf_type_size(dtt.dtt_ctfp, dtt.dtt_type);
1405
1406         if (typs != ssz) {
1407                 printf("Expected type size from buffer (%lu) to match type size looked up now (%ld)\n", (u_long) typs, (long) ssz);
1408                 return (-1);
1409         }
1410
1411         cbdata.dtp = dtp;
1412         cbdata.dtt = dtt;
1413         cbdata.name = "";
1414         cbdata.addr = addr;
1415         cbdata.addrend = addr + nbytes;
1416         cbdata.indent = 1;
1417         cbdata.f_type = 1;
1418         cbdata.type_width = 0;
1419         cbdata.name_width = 0;
1420         cbdata.fp = fp;
1421
1422         return (dt_print_type_data(&cbdata, dtt.dtt_type));
1423 }
1424
1425 static int
1426 dt_print_sym(dtrace_hdl_t *dtp, FILE *fp, const char *format, caddr_t addr)
1427 {
1428         /* LINTED - alignment */
1429         uint64_t pc = *((uint64_t *)addr);
1430         dtrace_syminfo_t dts;
1431         GElf_Sym sym;
1432         char c[PATH_MAX * 2];
1433
1434         if (format == NULL)
1435                 format = "  %-50s";
1436
1437         if (dtrace_lookup_by_addr(dtp, pc, &sym, &dts) == 0) {
1438                 (void) snprintf(c, sizeof (c), "%s`%s",
1439                     dts.dts_object, dts.dts_name);
1440         } else {
1441                 /*
1442                  * We'll repeat the lookup, but this time we'll specify a
1443                  * NULL GElf_Sym -- indicating that we're only interested in
1444                  * the containing module.
1445                  */
1446                 if (dtrace_lookup_by_addr(dtp, pc, NULL, &dts) == 0) {
1447                         (void) snprintf(c, sizeof (c), "%s`0x%llx",
1448                             dts.dts_object, (u_longlong_t)pc);
1449                 } else {
1450                         (void) snprintf(c, sizeof (c), "0x%llx",
1451                             (u_longlong_t)pc);
1452                 }
1453         }
1454
1455         if (dt_printf(dtp, fp, format, c) < 0)
1456                 return (-1);
1457
1458         return (0);
1459 }
1460
1461 int
1462 dt_print_mod(dtrace_hdl_t *dtp, FILE *fp, const char *format, caddr_t addr)
1463 {
1464         /* LINTED - alignment */
1465         uint64_t pc = *((uint64_t *)addr);
1466         dtrace_syminfo_t dts;
1467         char c[PATH_MAX * 2];
1468
1469         if (format == NULL)
1470                 format = "  %-50s";
1471
1472         if (dtrace_lookup_by_addr(dtp, pc, NULL, &dts) == 0) {
1473                 (void) snprintf(c, sizeof (c), "%s", dts.dts_object);
1474         } else {
1475                 (void) snprintf(c, sizeof (c), "0x%llx", (u_longlong_t)pc);
1476         }
1477
1478         if (dt_printf(dtp, fp, format, c) < 0)
1479                 return (-1);
1480
1481         return (0);
1482 }
1483
1484 typedef struct dt_normal {
1485         dtrace_aggvarid_t dtnd_id;
1486         uint64_t dtnd_normal;
1487 } dt_normal_t;
1488
1489 static int
1490 dt_normalize_agg(const dtrace_aggdata_t *aggdata, void *arg)
1491 {
1492         dt_normal_t *normal = arg;
1493         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1494         dtrace_aggvarid_t id = normal->dtnd_id;
1495
1496         if (agg->dtagd_nrecs == 0)
1497                 return (DTRACE_AGGWALK_NEXT);
1498
1499         if (agg->dtagd_varid != id)
1500                 return (DTRACE_AGGWALK_NEXT);
1501
1502         ((dtrace_aggdata_t *)aggdata)->dtada_normal = normal->dtnd_normal;
1503         return (DTRACE_AGGWALK_NORMALIZE);
1504 }
1505
1506 static int
1507 dt_normalize(dtrace_hdl_t *dtp, caddr_t base, dtrace_recdesc_t *rec)
1508 {
1509         dt_normal_t normal;
1510         caddr_t addr;
1511
1512         /*
1513          * We (should) have two records:  the aggregation ID followed by the
1514          * normalization value.
1515          */
1516         addr = base + rec->dtrd_offset;
1517
1518         if (rec->dtrd_size != sizeof (dtrace_aggvarid_t))
1519                 return (dt_set_errno(dtp, EDT_BADNORMAL));
1520
1521         /* LINTED - alignment */
1522         normal.dtnd_id = *((dtrace_aggvarid_t *)addr);
1523         rec++;
1524
1525         if (rec->dtrd_action != DTRACEACT_LIBACT)
1526                 return (dt_set_errno(dtp, EDT_BADNORMAL));
1527
1528         if (rec->dtrd_arg != DT_ACT_NORMALIZE)
1529                 return (dt_set_errno(dtp, EDT_BADNORMAL));
1530
1531         addr = base + rec->dtrd_offset;
1532
1533         switch (rec->dtrd_size) {
1534         case sizeof (uint64_t):
1535                 /* LINTED - alignment */
1536                 normal.dtnd_normal = *((uint64_t *)addr);
1537                 break;
1538         case sizeof (uint32_t):
1539                 /* LINTED - alignment */
1540                 normal.dtnd_normal = *((uint32_t *)addr);
1541                 break;
1542         case sizeof (uint16_t):
1543                 /* LINTED - alignment */
1544                 normal.dtnd_normal = *((uint16_t *)addr);
1545                 break;
1546         case sizeof (uint8_t):
1547                 normal.dtnd_normal = *((uint8_t *)addr);
1548                 break;
1549         default:
1550                 return (dt_set_errno(dtp, EDT_BADNORMAL));
1551         }
1552
1553         (void) dtrace_aggregate_walk(dtp, dt_normalize_agg, &normal);
1554
1555         return (0);
1556 }
1557
1558 static int
1559 dt_denormalize_agg(const dtrace_aggdata_t *aggdata, void *arg)
1560 {
1561         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1562         dtrace_aggvarid_t id = *((dtrace_aggvarid_t *)arg);
1563
1564         if (agg->dtagd_nrecs == 0)
1565                 return (DTRACE_AGGWALK_NEXT);
1566
1567         if (agg->dtagd_varid != id)
1568                 return (DTRACE_AGGWALK_NEXT);
1569
1570         return (DTRACE_AGGWALK_DENORMALIZE);
1571 }
1572
1573 static int
1574 dt_clear_agg(const dtrace_aggdata_t *aggdata, void *arg)
1575 {
1576         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1577         dtrace_aggvarid_t id = *((dtrace_aggvarid_t *)arg);
1578
1579         if (agg->dtagd_nrecs == 0)
1580                 return (DTRACE_AGGWALK_NEXT);
1581
1582         if (agg->dtagd_varid != id)
1583                 return (DTRACE_AGGWALK_NEXT);
1584
1585         return (DTRACE_AGGWALK_CLEAR);
1586 }
1587
1588 typedef struct dt_trunc {
1589         dtrace_aggvarid_t dttd_id;
1590         uint64_t dttd_remaining;
1591 } dt_trunc_t;
1592
1593 static int
1594 dt_trunc_agg(const dtrace_aggdata_t *aggdata, void *arg)
1595 {
1596         dt_trunc_t *trunc = arg;
1597         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1598         dtrace_aggvarid_t id = trunc->dttd_id;
1599
1600         if (agg->dtagd_nrecs == 0)
1601                 return (DTRACE_AGGWALK_NEXT);
1602
1603         if (agg->dtagd_varid != id)
1604                 return (DTRACE_AGGWALK_NEXT);
1605
1606         if (trunc->dttd_remaining == 0)
1607                 return (DTRACE_AGGWALK_REMOVE);
1608
1609         trunc->dttd_remaining--;
1610         return (DTRACE_AGGWALK_NEXT);
1611 }
1612
1613 static int
1614 dt_trunc(dtrace_hdl_t *dtp, caddr_t base, dtrace_recdesc_t *rec)
1615 {
1616         dt_trunc_t trunc;
1617         caddr_t addr;
1618         int64_t remaining;
1619         int (*func)(dtrace_hdl_t *, dtrace_aggregate_f *, void *);
1620
1621         /*
1622          * We (should) have two records:  the aggregation ID followed by the
1623          * number of aggregation entries after which the aggregation is to be
1624          * truncated.
1625          */
1626         addr = base + rec->dtrd_offset;
1627
1628         if (rec->dtrd_size != sizeof (dtrace_aggvarid_t))
1629                 return (dt_set_errno(dtp, EDT_BADTRUNC));
1630
1631         /* LINTED - alignment */
1632         trunc.dttd_id = *((dtrace_aggvarid_t *)addr);
1633         rec++;
1634
1635         if (rec->dtrd_action != DTRACEACT_LIBACT)
1636                 return (dt_set_errno(dtp, EDT_BADTRUNC));
1637
1638         if (rec->dtrd_arg != DT_ACT_TRUNC)
1639                 return (dt_set_errno(dtp, EDT_BADTRUNC));
1640
1641         addr = base + rec->dtrd_offset;
1642
1643         switch (rec->dtrd_size) {
1644         case sizeof (uint64_t):
1645                 /* LINTED - alignment */
1646                 remaining = *((int64_t *)addr);
1647                 break;
1648         case sizeof (uint32_t):
1649                 /* LINTED - alignment */
1650                 remaining = *((int32_t *)addr);
1651                 break;
1652         case sizeof (uint16_t):
1653                 /* LINTED - alignment */
1654                 remaining = *((int16_t *)addr);
1655                 break;
1656         case sizeof (uint8_t):
1657                 remaining = *((int8_t *)addr);
1658                 break;
1659         default:
1660                 return (dt_set_errno(dtp, EDT_BADNORMAL));
1661         }
1662
1663         if (remaining < 0) {
1664                 func = dtrace_aggregate_walk_valsorted;
1665                 remaining = -remaining;
1666         } else {
1667                 func = dtrace_aggregate_walk_valrevsorted;
1668         }
1669
1670         assert(remaining >= 0);
1671         trunc.dttd_remaining = remaining;
1672
1673         (void) func(dtp, dt_trunc_agg, &trunc);
1674
1675         return (0);
1676 }
1677
1678 static int
1679 dt_print_datum(dtrace_hdl_t *dtp, FILE *fp, dtrace_recdesc_t *rec,
1680     caddr_t addr, size_t size, uint64_t normal)
1681 {
1682         int err;
1683         dtrace_actkind_t act = rec->dtrd_action;
1684
1685         switch (act) {
1686         case DTRACEACT_STACK:
1687                 return (dt_print_stack(dtp, fp, NULL, addr,
1688                     rec->dtrd_arg, rec->dtrd_size / rec->dtrd_arg));
1689
1690         case DTRACEACT_USTACK:
1691         case DTRACEACT_JSTACK:
1692                 return (dt_print_ustack(dtp, fp, NULL, addr, rec->dtrd_arg));
1693
1694         case DTRACEACT_USYM:
1695         case DTRACEACT_UADDR:
1696                 return (dt_print_usym(dtp, fp, addr, act));
1697
1698         case DTRACEACT_UMOD:
1699                 return (dt_print_umod(dtp, fp, NULL, addr));
1700
1701         case DTRACEACT_SYM:
1702                 return (dt_print_sym(dtp, fp, NULL, addr));
1703
1704         case DTRACEACT_MOD:
1705                 return (dt_print_mod(dtp, fp, NULL, addr));
1706
1707         case DTRACEAGG_QUANTIZE:
1708                 return (dt_print_quantize(dtp, fp, addr, size, normal));
1709
1710         case DTRACEAGG_LQUANTIZE:
1711                 return (dt_print_lquantize(dtp, fp, addr, size, normal));
1712
1713         case DTRACEAGG_AVG:
1714                 return (dt_print_average(dtp, fp, addr, size, normal));
1715
1716         case DTRACEAGG_STDDEV:
1717                 return (dt_print_stddev(dtp, fp, addr, size, normal));
1718
1719         default:
1720                 break;
1721         }
1722
1723         switch (size) {
1724         case sizeof (uint64_t):
1725                 err = dt_printf(dtp, fp, " %16lld",
1726                     /* LINTED - alignment */
1727                     (long long)*((uint64_t *)addr) / normal);
1728                 break;
1729         case sizeof (uint32_t):
1730                 /* LINTED - alignment */
1731                 err = dt_printf(dtp, fp, " %8d", *((uint32_t *)addr) /
1732                     (uint32_t)normal);
1733                 break;
1734         case sizeof (uint16_t):
1735                 /* LINTED - alignment */
1736                 err = dt_printf(dtp, fp, " %5d", *((uint16_t *)addr) /
1737                     (uint32_t)normal);
1738                 break;
1739         case sizeof (uint8_t):
1740                 err = dt_printf(dtp, fp, " %3d", *((uint8_t *)addr) /
1741                     (uint32_t)normal);
1742                 break;
1743         default:
1744                 err = dt_print_bytes(dtp, fp, addr, size, 50, 0, 0);
1745                 break;
1746         }
1747
1748         return (err);
1749 }
1750
1751 int
1752 dt_print_aggs(const dtrace_aggdata_t **aggsdata, int naggvars, void *arg)
1753 {
1754         int i, aggact = 0;
1755         dt_print_aggdata_t *pd = arg;
1756         const dtrace_aggdata_t *aggdata = aggsdata[0];
1757         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1758         FILE *fp = pd->dtpa_fp;
1759         dtrace_hdl_t *dtp = pd->dtpa_dtp;
1760         dtrace_recdesc_t *rec;
1761         dtrace_actkind_t act;
1762         caddr_t addr;
1763         size_t size;
1764
1765         /*
1766          * Iterate over each record description in the key, printing the traced
1767          * data, skipping the first datum (the tuple member created by the
1768          * compiler).
1769          */
1770         for (i = 1; i < agg->dtagd_nrecs; i++) {
1771                 rec = &agg->dtagd_rec[i];
1772                 act = rec->dtrd_action;
1773                 addr = aggdata->dtada_data + rec->dtrd_offset;
1774                 size = rec->dtrd_size;
1775
1776                 if (DTRACEACT_ISAGG(act)) {
1777                         aggact = i;
1778                         break;
1779                 }
1780
1781                 if (dt_print_datum(dtp, fp, rec, addr, size, 1) < 0)
1782                         return (-1);
1783
1784                 if (dt_buffered_flush(dtp, NULL, rec, aggdata,
1785                     DTRACE_BUFDATA_AGGKEY) < 0)
1786                         return (-1);
1787         }
1788
1789         assert(aggact != 0);
1790
1791         for (i = (naggvars == 1 ? 0 : 1); i < naggvars; i++) {
1792                 uint64_t normal;
1793
1794                 aggdata = aggsdata[i];
1795                 agg = aggdata->dtada_desc;
1796                 rec = &agg->dtagd_rec[aggact];
1797                 act = rec->dtrd_action;
1798                 addr = aggdata->dtada_data + rec->dtrd_offset;
1799                 size = rec->dtrd_size;
1800
1801                 assert(DTRACEACT_ISAGG(act));
1802                 normal = aggdata->dtada_normal;
1803
1804                 if (dt_print_datum(dtp, fp, rec, addr, size, normal) < 0)
1805                         return (-1);
1806
1807                 if (dt_buffered_flush(dtp, NULL, rec, aggdata,
1808                     DTRACE_BUFDATA_AGGVAL) < 0)
1809                         return (-1);
1810
1811                 if (!pd->dtpa_allunprint)
1812                         agg->dtagd_flags |= DTRACE_AGD_PRINTED;
1813         }
1814
1815         if (dt_printf(dtp, fp, "\n") < 0)
1816                 return (-1);
1817
1818         if (dt_buffered_flush(dtp, NULL, NULL, aggdata,
1819             DTRACE_BUFDATA_AGGFORMAT | DTRACE_BUFDATA_AGGLAST) < 0)
1820                 return (-1);
1821
1822         return (0);
1823 }
1824
1825 int
1826 dt_print_agg(const dtrace_aggdata_t *aggdata, void *arg)
1827 {
1828         dt_print_aggdata_t *pd = arg;
1829         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1830         dtrace_aggvarid_t aggvarid = pd->dtpa_id;
1831
1832         if (pd->dtpa_allunprint) {
1833                 if (agg->dtagd_flags & DTRACE_AGD_PRINTED)
1834                         return (0);
1835         } else {
1836                 /*
1837                  * If we're not printing all unprinted aggregations, then the
1838                  * aggregation variable ID denotes a specific aggregation
1839                  * variable that we should print -- skip any other aggregations
1840                  * that we encounter.
1841                  */
1842                 if (agg->dtagd_nrecs == 0)
1843                         return (0);
1844
1845                 if (aggvarid != agg->dtagd_varid)
1846                         return (0);
1847         }
1848
1849         return (dt_print_aggs(&aggdata, 1, arg));
1850 }
1851
1852 int
1853 dt_setopt(dtrace_hdl_t *dtp, const dtrace_probedata_t *data,
1854     const char *option, const char *value)
1855 {
1856         int len, rval;
1857         char *msg;
1858         const char *errstr;
1859         dtrace_setoptdata_t optdata;
1860
1861         bzero(&optdata, sizeof (optdata));
1862         (void) dtrace_getopt(dtp, option, &optdata.dtsda_oldval);
1863
1864         if (dtrace_setopt(dtp, option, value) == 0) {
1865                 (void) dtrace_getopt(dtp, option, &optdata.dtsda_newval);
1866                 optdata.dtsda_probe = data;
1867                 optdata.dtsda_option = option;
1868                 optdata.dtsda_handle = dtp;
1869
1870                 if ((rval = dt_handle_setopt(dtp, &optdata)) != 0)
1871                         return (rval);
1872
1873                 return (0);
1874         }
1875
1876         errstr = dtrace_errmsg(dtp, dtrace_errno(dtp));
1877         len = strlen(option) + strlen(value) + strlen(errstr) + 80;
1878         msg = alloca(len);
1879
1880         (void) snprintf(msg, len, "couldn't set option \"%s\" to \"%s\": %s\n",
1881             option, value, errstr);
1882
1883         if ((rval = dt_handle_liberr(dtp, data, msg)) == 0)
1884                 return (0);
1885
1886         return (rval);
1887 }
1888
1889 static int
1890 dt_consume_cpu(dtrace_hdl_t *dtp, FILE *fp, int cpu, dtrace_bufdesc_t *buf,
1891     dtrace_consume_probe_f *efunc, dtrace_consume_rec_f *rfunc, void *arg)
1892 {
1893         dtrace_epid_t id;
1894         size_t offs, start = buf->dtbd_oldest, end = buf->dtbd_size;
1895         int flow = (dtp->dt_options[DTRACEOPT_FLOWINDENT] != DTRACEOPT_UNSET);
1896         int quiet = (dtp->dt_options[DTRACEOPT_QUIET] != DTRACEOPT_UNSET);
1897         int rval, i, n;
1898         dtrace_epid_t last = DTRACE_EPIDNONE;
1899         dtrace_probedata_t data;
1900         uint64_t drops;
1901         caddr_t addr;
1902
1903         bzero(&data, sizeof (data));
1904         data.dtpda_handle = dtp;
1905         data.dtpda_cpu = cpu;
1906
1907 again:
1908         for (offs = start; offs < end; ) {
1909                 dtrace_eprobedesc_t *epd;
1910
1911                 /*
1912                  * We're guaranteed to have an ID.
1913                  */
1914                 id = *(uint32_t *)((uintptr_t)buf->dtbd_data + offs);
1915
1916                 if (id == DTRACE_EPIDNONE) {
1917                         /*
1918                          * This is filler to assure proper alignment of the
1919                          * next record; we simply ignore it.
1920                          */
1921                         offs += sizeof (id);
1922                         continue;
1923                 }
1924
1925                 if ((rval = dt_epid_lookup(dtp, id, &data.dtpda_edesc,
1926                     &data.dtpda_pdesc)) != 0)
1927                         return (rval);
1928
1929                 epd = data.dtpda_edesc;
1930                 data.dtpda_data = buf->dtbd_data + offs;
1931
1932                 if (data.dtpda_edesc->dtepd_uarg != DT_ECB_DEFAULT) {
1933                         rval = dt_handle(dtp, &data);
1934
1935                         if (rval == DTRACE_CONSUME_NEXT)
1936                                 goto nextepid;
1937
1938                         if (rval == DTRACE_CONSUME_ERROR)
1939                                 return (-1);
1940                 }
1941
1942                 if (flow)
1943                         (void) dt_flowindent(dtp, &data, last, buf, offs);
1944
1945                 rval = (*efunc)(&data, arg);
1946
1947                 if (flow) {
1948                         if (data.dtpda_flow == DTRACEFLOW_ENTRY)
1949                                 data.dtpda_indent += 2;
1950                 }
1951
1952                 if (rval == DTRACE_CONSUME_NEXT)
1953                         goto nextepid;
1954
1955                 if (rval == DTRACE_CONSUME_ABORT)
1956                         return (dt_set_errno(dtp, EDT_DIRABORT));
1957
1958                 if (rval != DTRACE_CONSUME_THIS)
1959                         return (dt_set_errno(dtp, EDT_BADRVAL));
1960
1961                 for (i = 0; i < epd->dtepd_nrecs; i++) {
1962                         dtrace_recdesc_t *rec = &epd->dtepd_rec[i];
1963                         dtrace_actkind_t act = rec->dtrd_action;
1964
1965                         data.dtpda_data = buf->dtbd_data + offs +
1966                             rec->dtrd_offset;
1967                         addr = data.dtpda_data;
1968
1969                         if (act == DTRACEACT_LIBACT) {
1970                                 uint64_t arg = rec->dtrd_arg;
1971                                 dtrace_aggvarid_t id;
1972
1973                                 switch (arg) {
1974                                 case DT_ACT_CLEAR:
1975                                         /* LINTED - alignment */
1976                                         id = *((dtrace_aggvarid_t *)addr);
1977                                         (void) dtrace_aggregate_walk(dtp,
1978                                             dt_clear_agg, &id);
1979                                         continue;
1980
1981                                 case DT_ACT_DENORMALIZE:
1982                                         /* LINTED - alignment */
1983                                         id = *((dtrace_aggvarid_t *)addr);
1984                                         (void) dtrace_aggregate_walk(dtp,
1985                                             dt_denormalize_agg, &id);
1986                                         continue;
1987
1988                                 case DT_ACT_FTRUNCATE:
1989                                         if (fp == NULL)
1990                                                 continue;
1991
1992                                         (void) fflush(fp);
1993                                         (void) ftruncate(fileno(fp), 0);
1994                                         (void) fseeko(fp, 0, SEEK_SET);
1995                                         continue;
1996
1997                                 case DT_ACT_NORMALIZE:
1998                                         if (i == epd->dtepd_nrecs - 1)
1999                                                 return (dt_set_errno(dtp,
2000                                                     EDT_BADNORMAL));
2001
2002                                         if (dt_normalize(dtp,
2003                                             buf->dtbd_data + offs, rec) != 0)
2004                                                 return (-1);
2005
2006                                         i++;
2007                                         continue;
2008
2009                                 case DT_ACT_SETOPT: {
2010                                         uint64_t *opts = dtp->dt_options;
2011                                         dtrace_recdesc_t *valrec;
2012                                         uint32_t valsize;
2013                                         caddr_t val;
2014                                         int rv;
2015
2016                                         if (i == epd->dtepd_nrecs - 1) {
2017                                                 return (dt_set_errno(dtp,
2018                                                     EDT_BADSETOPT));
2019                                         }
2020
2021                                         valrec = &epd->dtepd_rec[++i];
2022                                         valsize = valrec->dtrd_size;
2023
2024                                         if (valrec->dtrd_action != act ||
2025                                             valrec->dtrd_arg != arg) {
2026                                                 return (dt_set_errno(dtp,
2027                                                     EDT_BADSETOPT));
2028                                         }
2029
2030                                         if (valsize > sizeof (uint64_t)) {
2031                                                 val = buf->dtbd_data + offs +
2032                                                     valrec->dtrd_offset;
2033                                         } else {
2034                                                 val = "1";
2035                                         }
2036
2037                                         rv = dt_setopt(dtp, &data, addr, val);
2038
2039                                         if (rv != 0)
2040                                                 return (-1);
2041
2042                                         flow = (opts[DTRACEOPT_FLOWINDENT] !=
2043                                             DTRACEOPT_UNSET);
2044                                         quiet = (opts[DTRACEOPT_QUIET] !=
2045                                             DTRACEOPT_UNSET);
2046
2047                                         continue;
2048                                 }
2049
2050                                 case DT_ACT_TRUNC:
2051                                         if (i == epd->dtepd_nrecs - 1)
2052                                                 return (dt_set_errno(dtp,
2053                                                     EDT_BADTRUNC));
2054
2055                                         if (dt_trunc(dtp,
2056                                             buf->dtbd_data + offs, rec) != 0)
2057                                                 return (-1);
2058
2059                                         i++;
2060                                         continue;
2061
2062                                 default:
2063                                         continue;
2064                                 }
2065                         }
2066
2067                         rval = (*rfunc)(&data, rec, arg);
2068
2069                         if (rval == DTRACE_CONSUME_NEXT)
2070                                 continue;
2071
2072                         if (rval == DTRACE_CONSUME_ABORT)
2073                                 return (dt_set_errno(dtp, EDT_DIRABORT));
2074
2075                         if (rval != DTRACE_CONSUME_THIS)
2076                                 return (dt_set_errno(dtp, EDT_BADRVAL));
2077
2078                         if (act == DTRACEACT_STACK) {
2079                                 int depth = rec->dtrd_arg;
2080
2081                                 if (dt_print_stack(dtp, fp, NULL, addr, depth,
2082                                     rec->dtrd_size / depth) < 0)
2083                                         return (-1);
2084                                 goto nextrec;
2085                         }
2086
2087                         if (act == DTRACEACT_USTACK ||
2088                             act == DTRACEACT_JSTACK) {
2089                                 if (dt_print_ustack(dtp, fp, NULL,
2090                                     addr, rec->dtrd_arg) < 0)
2091                                         return (-1);
2092                                 goto nextrec;
2093                         }
2094
2095                         if (act == DTRACEACT_SYM) {
2096                                 if (dt_print_sym(dtp, fp, NULL, addr) < 0)
2097                                         return (-1);
2098                                 goto nextrec;
2099                         }
2100
2101                         if (act == DTRACEACT_MOD) {
2102                                 if (dt_print_mod(dtp, fp, NULL, addr) < 0)
2103                                         return (-1);
2104                                 goto nextrec;
2105                         }
2106
2107                         if (act == DTRACEACT_USYM || act == DTRACEACT_UADDR) {
2108                                 if (dt_print_usym(dtp, fp, addr, act) < 0)
2109                                         return (-1);
2110                                 goto nextrec;
2111                         }
2112
2113                         if (act == DTRACEACT_UMOD) {
2114                                 if (dt_print_umod(dtp, fp, NULL, addr) < 0)
2115                                         return (-1);
2116                                 goto nextrec;
2117                         }
2118
2119                         if (act == DTRACEACT_PRINTM) {
2120                                 if (dt_print_memory(dtp, fp, addr) < 0)
2121                                         return (-1);
2122                                 goto nextrec;
2123                         }
2124
2125                         if (act == DTRACEACT_PRINTT) {
2126                                 if (dt_print_type(dtp, fp, addr) < 0)
2127                                         return (-1);
2128                                 goto nextrec;
2129                         }
2130
2131                         if (DTRACEACT_ISPRINTFLIKE(act)) {
2132                                 void *fmtdata;
2133                                 int (*func)(dtrace_hdl_t *, FILE *, void *,
2134                                     const dtrace_probedata_t *,
2135                                     const dtrace_recdesc_t *, uint_t,
2136                                     const void *buf, size_t);
2137
2138                                 if ((fmtdata = dt_format_lookup(dtp,
2139                                     rec->dtrd_format)) == NULL)
2140                                         goto nofmt;
2141
2142                                 switch (act) {
2143                                 case DTRACEACT_PRINTF:
2144                                         func = dtrace_fprintf;
2145                                         break;
2146                                 case DTRACEACT_PRINTA:
2147                                         func = dtrace_fprinta;
2148                                         break;
2149                                 case DTRACEACT_SYSTEM:
2150                                         func = dtrace_system;
2151                                         break;
2152                                 case DTRACEACT_FREOPEN:
2153                                         func = dtrace_freopen;
2154                                         break;
2155                                 }
2156
2157                                 n = (*func)(dtp, fp, fmtdata, &data,
2158                                     rec, epd->dtepd_nrecs - i,
2159                                     (uchar_t *)buf->dtbd_data + offs,
2160                                     buf->dtbd_size - offs);
2161
2162                                 if (n < 0)
2163                                         return (-1); /* errno is set for us */
2164
2165                                 if (n > 0)
2166                                         i += n - 1;
2167                                 goto nextrec;
2168                         }
2169
2170 nofmt:
2171                         if (act == DTRACEACT_PRINTA) {
2172                                 dt_print_aggdata_t pd;
2173                                 dtrace_aggvarid_t *aggvars;
2174                                 int j, naggvars = 0;
2175                                 size_t size = ((epd->dtepd_nrecs - i) *
2176                                     sizeof (dtrace_aggvarid_t));
2177
2178                                 if ((aggvars = dt_alloc(dtp, size)) == NULL)
2179                                         return (-1);
2180
2181                                 /*
2182                                  * This might be a printa() with multiple
2183                                  * aggregation variables.  We need to scan
2184                                  * forward through the records until we find
2185                                  * a record from a different statement.
2186                                  */
2187                                 for (j = i; j < epd->dtepd_nrecs; j++) {
2188                                         dtrace_recdesc_t *nrec;
2189                                         caddr_t naddr;
2190
2191                                         nrec = &epd->dtepd_rec[j];
2192
2193                                         if (nrec->dtrd_uarg != rec->dtrd_uarg)
2194                                                 break;
2195
2196                                         if (nrec->dtrd_action != act) {
2197                                                 return (dt_set_errno(dtp,
2198                                                     EDT_BADAGG));
2199                                         }
2200
2201                                         naddr = buf->dtbd_data + offs +
2202                                             nrec->dtrd_offset;
2203
2204                                         aggvars[naggvars++] =
2205                                             /* LINTED - alignment */
2206                                             *((dtrace_aggvarid_t *)naddr);
2207                                 }
2208
2209                                 i = j - 1;
2210                                 bzero(&pd, sizeof (pd));
2211                                 pd.dtpa_dtp = dtp;
2212                                 pd.dtpa_fp = fp;
2213
2214                                 assert(naggvars >= 1);
2215
2216                                 if (naggvars == 1) {
2217                                         pd.dtpa_id = aggvars[0];
2218                                         dt_free(dtp, aggvars);
2219
2220                                         if (dt_printf(dtp, fp, "\n") < 0 ||
2221                                             dtrace_aggregate_walk_sorted(dtp,
2222                                             dt_print_agg, &pd) < 0)
2223                                                 return (-1);
2224                                         goto nextrec;
2225                                 }
2226
2227                                 if (dt_printf(dtp, fp, "\n") < 0 ||
2228                                     dtrace_aggregate_walk_joined(dtp, aggvars,
2229                                     naggvars, dt_print_aggs, &pd) < 0) {
2230                                         dt_free(dtp, aggvars);
2231                                         return (-1);
2232                                 }
2233
2234                                 dt_free(dtp, aggvars);
2235                                 goto nextrec;
2236                         }
2237
2238                         switch (rec->dtrd_size) {
2239                         case sizeof (uint64_t):
2240                                 n = dt_printf(dtp, fp,
2241                                     quiet ? "%lld" : " %16lld",
2242                                     /* LINTED - alignment */
2243                                     *((unsigned long long *)addr));
2244                                 break;
2245                         case sizeof (uint32_t):
2246                                 n = dt_printf(dtp, fp, quiet ? "%d" : " %8d",
2247                                     /* LINTED - alignment */
2248                                     *((uint32_t *)addr));
2249                                 break;
2250                         case sizeof (uint16_t):
2251                                 n = dt_printf(dtp, fp, quiet ? "%d" : " %5d",
2252                                     /* LINTED - alignment */
2253                                     *((uint16_t *)addr));
2254                                 break;
2255                         case sizeof (uint8_t):
2256                                 n = dt_printf(dtp, fp, quiet ? "%d" : " %3d",
2257                                     *((uint8_t *)addr));
2258                                 break;
2259                         default:
2260                                 n = dt_print_bytes(dtp, fp, addr,
2261                                     rec->dtrd_size, 33, quiet, 0);
2262                                 break;
2263                         }
2264
2265                         if (n < 0)
2266                                 return (-1); /* errno is set for us */
2267
2268 nextrec:
2269                         if (dt_buffered_flush(dtp, &data, rec, NULL, 0) < 0)
2270                                 return (-1); /* errno is set for us */
2271                 }
2272
2273                 /*
2274                  * Call the record callback with a NULL record to indicate
2275                  * that we're done processing this EPID.
2276                  */
2277                 rval = (*rfunc)(&data, NULL, arg);
2278 nextepid:
2279                 offs += epd->dtepd_size;
2280                 last = id;
2281         }
2282
2283         if (buf->dtbd_oldest != 0 && start == buf->dtbd_oldest) {
2284                 end = buf->dtbd_oldest;
2285                 start = 0;
2286                 goto again;
2287         }
2288
2289         if ((drops = buf->dtbd_drops) == 0)
2290                 return (0);
2291
2292         /*
2293          * Explicitly zero the drops to prevent us from processing them again.
2294          */
2295         buf->dtbd_drops = 0;
2296
2297         return (dt_handle_cpudrop(dtp, cpu, DTRACEDROP_PRINCIPAL, drops));
2298 }
2299
2300 typedef struct dt_begin {
2301         dtrace_consume_probe_f *dtbgn_probefunc;
2302         dtrace_consume_rec_f *dtbgn_recfunc;
2303         void *dtbgn_arg;
2304         dtrace_handle_err_f *dtbgn_errhdlr;
2305         void *dtbgn_errarg;
2306         int dtbgn_beginonly;
2307 } dt_begin_t;
2308
2309 static int
2310 dt_consume_begin_probe(const dtrace_probedata_t *data, void *arg)
2311 {
2312         dt_begin_t *begin = (dt_begin_t *)arg;
2313         dtrace_probedesc_t *pd = data->dtpda_pdesc;
2314
2315         int r1 = (strcmp(pd->dtpd_provider, "dtrace") == 0);
2316         int r2 = (strcmp(pd->dtpd_name, "BEGIN") == 0);
2317
2318         if (begin->dtbgn_beginonly) {
2319                 if (!(r1 && r2))
2320                         return (DTRACE_CONSUME_NEXT);
2321         } else {
2322                 if (r1 && r2)
2323                         return (DTRACE_CONSUME_NEXT);
2324         }
2325
2326         /*
2327          * We have a record that we're interested in.  Now call the underlying
2328          * probe function...
2329          */
2330         return (begin->dtbgn_probefunc(data, begin->dtbgn_arg));
2331 }
2332
2333 static int
2334 dt_consume_begin_record(const dtrace_probedata_t *data,
2335     const dtrace_recdesc_t *rec, void *arg)
2336 {
2337         dt_begin_t *begin = (dt_begin_t *)arg;
2338
2339         return (begin->dtbgn_recfunc(data, rec, begin->dtbgn_arg));
2340 }
2341
2342 static int
2343 dt_consume_begin_error(const dtrace_errdata_t *data, void *arg)
2344 {
2345         dt_begin_t *begin = (dt_begin_t *)arg;
2346         dtrace_probedesc_t *pd = data->dteda_pdesc;
2347
2348         int r1 = (strcmp(pd->dtpd_provider, "dtrace") == 0);
2349         int r2 = (strcmp(pd->dtpd_name, "BEGIN") == 0);
2350
2351         if (begin->dtbgn_beginonly) {
2352                 if (!(r1 && r2))
2353                         return (DTRACE_HANDLE_OK);
2354         } else {
2355                 if (r1 && r2)
2356                         return (DTRACE_HANDLE_OK);
2357         }
2358
2359         return (begin->dtbgn_errhdlr(data, begin->dtbgn_errarg));
2360 }
2361
2362 static int
2363 dt_consume_begin(dtrace_hdl_t *dtp, FILE *fp, dtrace_bufdesc_t *buf,
2364     dtrace_consume_probe_f *pf, dtrace_consume_rec_f *rf, void *arg)
2365 {
2366         /*
2367          * There's this idea that the BEGIN probe should be processed before
2368          * everything else, and that the END probe should be processed after
2369          * anything else.  In the common case, this is pretty easy to deal
2370          * with.  However, a situation may arise where the BEGIN enabling and
2371          * END enabling are on the same CPU, and some enabling in the middle
2372          * occurred on a different CPU.  To deal with this (blech!) we need to
2373          * consume the BEGIN buffer up until the end of the BEGIN probe, and
2374          * then set it aside.  We will then process every other CPU, and then
2375          * we'll return to the BEGIN CPU and process the rest of the data
2376          * (which will inevitably include the END probe, if any).  Making this
2377          * even more complicated (!) is the library's ERROR enabling.  Because
2378          * this enabling is processed before we even get into the consume call
2379          * back, any ERROR firing would result in the library's ERROR enabling
2380          * being processed twice -- once in our first pass (for BEGIN probes),
2381          * and again in our second pass (for everything but BEGIN probes).  To
2382          * deal with this, we interpose on the ERROR handler to assure that we
2383          * only process ERROR enablings induced by BEGIN enablings in the
2384          * first pass, and that we only process ERROR enablings _not_ induced
2385          * by BEGIN enablings in the second pass.
2386          */
2387         dt_begin_t begin;
2388         processorid_t cpu = dtp->dt_beganon;
2389         dtrace_bufdesc_t nbuf;
2390 #if !defined(sun)
2391         dtrace_bufdesc_t *pbuf;
2392 #endif
2393         int rval, i;
2394         static int max_ncpus;
2395         dtrace_optval_t size;
2396
2397         dtp->dt_beganon = -1;
2398
2399 #if defined(sun)
2400         if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, buf) == -1) {
2401 #else
2402         if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, &buf) == -1) {
2403 #endif
2404                 /*
2405                  * We really don't expect this to fail, but it is at least
2406                  * technically possible for this to fail with ENOENT.  In this
2407                  * case, we just drive on...
2408                  */
2409                 if (errno == ENOENT)
2410                         return (0);
2411
2412                 return (dt_set_errno(dtp, errno));
2413         }
2414
2415         if (!dtp->dt_stopped || buf->dtbd_cpu != dtp->dt_endedon) {
2416                 /*
2417                  * This is the simple case.  We're either not stopped, or if
2418                  * we are, we actually processed any END probes on another
2419                  * CPU.  We can simply consume this buffer and return.
2420                  */
2421                 return (dt_consume_cpu(dtp, fp, cpu, buf, pf, rf, arg));
2422         }
2423
2424         begin.dtbgn_probefunc = pf;
2425         begin.dtbgn_recfunc = rf;
2426         begin.dtbgn_arg = arg;
2427         begin.dtbgn_beginonly = 1;
2428
2429         /*
2430          * We need to interpose on the ERROR handler to be sure that we
2431          * only process ERRORs induced by BEGIN.
2432          */
2433         begin.dtbgn_errhdlr = dtp->dt_errhdlr;
2434         begin.dtbgn_errarg = dtp->dt_errarg;
2435         dtp->dt_errhdlr = dt_consume_begin_error;
2436         dtp->dt_errarg = &begin;
2437
2438         rval = dt_consume_cpu(dtp, fp, cpu, buf, dt_consume_begin_probe,
2439             dt_consume_begin_record, &begin);
2440
2441         dtp->dt_errhdlr = begin.dtbgn_errhdlr;
2442         dtp->dt_errarg = begin.dtbgn_errarg;
2443
2444         if (rval != 0)
2445                 return (rval);
2446
2447         /*
2448          * Now allocate a new buffer.  We'll use this to deal with every other
2449          * CPU.
2450          */
2451         bzero(&nbuf, sizeof (dtrace_bufdesc_t));
2452         (void) dtrace_getopt(dtp, "bufsize", &size);
2453         if ((nbuf.dtbd_data = malloc(size)) == NULL)
2454                 return (dt_set_errno(dtp, EDT_NOMEM));
2455
2456         if (max_ncpus == 0)
2457                 max_ncpus = dt_sysconf(dtp, _SC_CPUID_MAX) + 1;
2458
2459         for (i = 0; i < max_ncpus; i++) {
2460                 nbuf.dtbd_cpu = i;
2461
2462                 if (i == cpu)
2463                         continue;
2464
2465 #if defined(sun)
2466                 if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, &nbuf) == -1) {
2467 #else
2468                 pbuf = &nbuf;
2469                 if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, &pbuf) == -1) {
2470 #endif
2471                         /*
2472                          * If we failed with ENOENT, it may be because the
2473                          * CPU was unconfigured -- this is okay.  Any other
2474                          * error, however, is unexpected.
2475                          */
2476                         if (errno == ENOENT)
2477                                 continue;
2478
2479                         free(nbuf.dtbd_data);
2480
2481                         return (dt_set_errno(dtp, errno));
2482                 }
2483
2484                 if ((rval = dt_consume_cpu(dtp, fp,
2485                     i, &nbuf, pf, rf, arg)) != 0) {
2486                         free(nbuf.dtbd_data);
2487                         return (rval);
2488                 }
2489         }
2490
2491         free(nbuf.dtbd_data);
2492
2493         /*
2494          * Okay -- we're done with the other buffers.  Now we want to
2495          * reconsume the first buffer -- but this time we're looking for
2496          * everything _but_ BEGIN.  And of course, in order to only consume
2497          * those ERRORs _not_ associated with BEGIN, we need to reinstall our
2498          * ERROR interposition function...
2499          */
2500         begin.dtbgn_beginonly = 0;
2501
2502         assert(begin.dtbgn_errhdlr == dtp->dt_errhdlr);
2503         assert(begin.dtbgn_errarg == dtp->dt_errarg);
2504         dtp->dt_errhdlr = dt_consume_begin_error;
2505         dtp->dt_errarg = &begin;
2506
2507         rval = dt_consume_cpu(dtp, fp, cpu, buf, dt_consume_begin_probe,
2508             dt_consume_begin_record, &begin);
2509
2510         dtp->dt_errhdlr = begin.dtbgn_errhdlr;
2511         dtp->dt_errarg = begin.dtbgn_errarg;
2512
2513         return (rval);
2514 }
2515
2516 int
2517 dtrace_consume(dtrace_hdl_t *dtp, FILE *fp,
2518     dtrace_consume_probe_f *pf, dtrace_consume_rec_f *rf, void *arg)
2519 {
2520         dtrace_bufdesc_t *buf = &dtp->dt_buf;
2521         dtrace_optval_t size;
2522         static int max_ncpus;
2523         int i, rval;
2524         dtrace_optval_t interval = dtp->dt_options[DTRACEOPT_SWITCHRATE];
2525         hrtime_t now = gethrtime();
2526
2527         if (dtp->dt_lastswitch != 0) {
2528                 if (now - dtp->dt_lastswitch < interval)
2529                         return (0);
2530
2531                 dtp->dt_lastswitch += interval;
2532         } else {
2533                 dtp->dt_lastswitch = now;
2534         }
2535
2536         if (!dtp->dt_active)
2537                 return (dt_set_errno(dtp, EINVAL));
2538
2539         if (max_ncpus == 0)
2540                 max_ncpus = dt_sysconf(dtp, _SC_CPUID_MAX) + 1;
2541
2542         if (pf == NULL)
2543                 pf = (dtrace_consume_probe_f *)dt_nullprobe;
2544
2545         if (rf == NULL)
2546                 rf = (dtrace_consume_rec_f *)dt_nullrec;
2547
2548         if (buf->dtbd_data == NULL) {
2549                 (void) dtrace_getopt(dtp, "bufsize", &size);
2550                 if ((buf->dtbd_data = malloc(size)) == NULL)
2551                         return (dt_set_errno(dtp, EDT_NOMEM));
2552
2553                 buf->dtbd_size = size;
2554         }
2555
2556         /*
2557          * If we have just begun, we want to first process the CPU that
2558          * executed the BEGIN probe (if any).
2559          */
2560         if (dtp->dt_active && dtp->dt_beganon != -1) {
2561                 buf->dtbd_cpu = dtp->dt_beganon;
2562                 if ((rval = dt_consume_begin(dtp, fp, buf, pf, rf, arg)) != 0)
2563                         return (rval);
2564         }
2565
2566         for (i = 0; i < max_ncpus; i++) {
2567                 buf->dtbd_cpu = i;
2568
2569                 /*
2570                  * If we have stopped, we want to process the CPU on which the
2571                  * END probe was processed only _after_ we have processed
2572                  * everything else.
2573                  */
2574                 if (dtp->dt_stopped && (i == dtp->dt_endedon))
2575                         continue;
2576
2577 #if defined(sun)
2578                 if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, buf) == -1) {
2579 #else
2580                 if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, &buf) == -1) {
2581 #endif
2582                         /*
2583                          * If we failed with ENOENT, it may be because the
2584                          * CPU was unconfigured -- this is okay.  Any other
2585                          * error, however, is unexpected.
2586                          */
2587                         if (errno == ENOENT)
2588                                 continue;
2589
2590                         return (dt_set_errno(dtp, errno));
2591                 }
2592
2593                 if ((rval = dt_consume_cpu(dtp, fp, i, buf, pf, rf, arg)) != 0)
2594                         return (rval);
2595         }
2596
2597         if (!dtp->dt_stopped)
2598                 return (0);
2599
2600         buf->dtbd_cpu = dtp->dt_endedon;
2601
2602 #if defined(sun)
2603         if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, buf) == -1) {
2604 #else
2605         if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, &buf) == -1) {
2606 #endif
2607                 /*
2608                  * This _really_ shouldn't fail, but it is strictly speaking
2609                  * possible for this to return ENOENT if the CPU that called
2610                  * the END enabling somehow managed to become unconfigured.
2611                  * It's unclear how the user can possibly expect anything
2612                  * rational to happen in this case -- the state has been thrown
2613                  * out along with the unconfigured CPU -- so we'll just drive
2614                  * on...
2615                  */
2616                 if (errno == ENOENT)
2617                         return (0);
2618
2619                 return (dt_set_errno(dtp, errno));
2620         }
2621
2622         return (dt_consume_cpu(dtp, fp, dtp->dt_endedon, buf, pf, rf, arg));
2623 }