cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25
  26 #pragma ident   "%Z%%M% %I%     %E% SMI"
  27
  28 #include <stdlib.h>
  29 #include <strings.h>
  30 #include <errno.h>
  31 #include <unistd.h>
  32 #include <limits.h>
  33 #include <assert.h>
  34 #include <ctype.h>
  35 #if defined(sun)
  36 #include <alloca.h>
  37 #endif
  38 #include <dt_impl.h>
  39
  40 #define DT_MASK_LO 0x00000000FFFFFFFFULL
  41
  42 /*
  43  * We declare this here because (1) we need it and (2) we want to avoid a
  44  * dependency on libm in libdtrace.
  45  */
  46 static long double
  47 dt_fabsl(long double x)
  48 {
  49         if (x < 0)
  50                 return (-x);
  51
  52         return (x);
  53 }
  54
  55 /*
  56  * 128-bit arithmetic functions needed to support the stddev() aggregating
  57  * action.
  58  */
  59 static int
  60 dt_gt_128(uint64_t *a, uint64_t *b)
  61 {
  62         return (a[1] > b[1] || (a[1] == b[1] && a[0] > b[0]));
  63 }
  64
  65 static int
  66 dt_ge_128(uint64_t *a, uint64_t *b)
  67 {
  68         return (a[1] > b[1] || (a[1] == b[1] && a[0] >= b[0]));
  69 }
  70
  71 static int
  72 dt_le_128(uint64_t *a, uint64_t *b)
  73 {
  74         return (a[1] < b[1] || (a[1] == b[1] && a[0] <= b[0]));
  75 }
  76
  77 /*
  78  * Shift the 128-bit value in a by b. If b is positive, shift left.
  79  * If b is negative, shift right.
  80  */
  81 static void
  82 dt_shift_128(uint64_t *a, int b)
  83 {
  84         uint64_t mask;
  85
  86         if (b == 0)
  87                 return;
  88
  89         if (b < 0) {
  90                 b = -b;
  91                 if (b >= 64) {
  92                         a[0] = a[1] >> (b - 64);
  93                         a[1] = 0;
  94                 } else {
  95                         a[0] >>= b;
  96                         mask = 1LL << (64 - b);
  97                         mask -= 1;
  98                         a[0] |= ((a[1] & mask) << (64 - b));
  99                         a[1] >>= b;
 100                 }
 101         } else {
 102                 if (b >= 64) {
 103                         a[1] = a[0] << (b - 64);
 104                         a[0] = 0;
 105                 } else {
 106                         a[1] <<= b;
 107                         mask = a[0] >> (64 - b);
 108                         a[1] |= mask;
 109                         a[0] <<= b;
 110                 }
 111         }
 112 }
 113
 114 static int
 115 dt_nbits_128(uint64_t *a)
 116 {
 117         int nbits = 0;
 118         uint64_t tmp[2];
 119         uint64_t zero[2] = { 0, 0 };
 120
 121         tmp[0] = a[0];
 122         tmp[1] = a[1];
 123
 124         dt_shift_128(tmp, -1);
 125         while (dt_gt_128(tmp, zero)) {
 126                 dt_shift_128(tmp, -1);
 127                 nbits++;
 128         }
 129
 130         return (nbits);
 131 }
 132
 133 static void
 134 dt_subtract_128(uint64_t *minuend, uint64_t *subtrahend, uint64_t *difference)
 135 {
 136         uint64_t result[2];
 137
 138         result[0] = minuend[0] - subtrahend[0];
 139         result[1] = minuend[1] - subtrahend[1] -
 140             (minuend[0] < subtrahend[0] ? 1 : 0);
 141
 142         difference[0] = result[0];
 143         difference[1] = result[1];
 144 }
 145
 146 static void
 147 dt_add_128(uint64_t *addend1, uint64_t *addend2, uint64_t *sum)
 148 {
 149         uint64_t result[2];
 150
 151         result[0] = addend1[0] + addend2[0];
 152         result[1] = addend1[1] + addend2[1] +
 153             (result[0] < addend1[0] || result[0] < addend2[0] ? 1 : 0);
 154
 155         sum[0] = result[0];
 156         sum[1] = result[1];
 157 }
 158
 159 /*
 160  * The basic idea is to break the 2 64-bit values into 4 32-bit values,
 161  * use native multiplication on those, and then re-combine into the
 162  * resulting 128-bit value.
 163  *
 164  * (hi1 << 32 + lo1) * (hi2 << 32 + lo2) =
 165  *     hi1 * hi2 << 64 +
 166  *     hi1 * lo2 << 32 +
 167  *     hi2 * lo1 << 32 +
 168  *     lo1 * lo2
 169  */
 170 static void
 171 dt_multiply_128(uint64_t factor1, uint64_t factor2, uint64_t *product)
 172 {
 173         uint64_t hi1, hi2, lo1, lo2;
 174         uint64_t tmp[2];
 175
 176         hi1 = factor1 >> 32;
 177         hi2 = factor2 >> 32;
 178
 179         lo1 = factor1 & DT_MASK_LO;
 180         lo2 = factor2 & DT_MASK_LO;
 181
 182         product[0] = lo1 * lo2;
 183         product[1] = hi1 * hi2;
 184
 185         tmp[0] = hi1 * lo2;
 186         tmp[1] = 0;
 187         dt_shift_128(tmp, 32);
 188         dt_add_128(product, tmp, product);
 189
 190         tmp[0] = hi2 * lo1;
 191         tmp[1] = 0;
 192         dt_shift_128(tmp, 32);
 193         dt_add_128(product, tmp, product);
 194 }
 195
 196 /*
 197  * This is long-hand division.
 198  *
 199  * We initialize subtrahend by shifting divisor left as far as possible. We
 200  * loop, comparing subtrahend to dividend:  if subtrahend is smaller, we
 201  * subtract and set the appropriate bit in the result.  We then shift
 202  * subtrahend right by one bit for the next comparison.
 203  */
 204 static void
 205 dt_divide_128(uint64_t *dividend, uint64_t divisor, uint64_t *quotient)
 206 {
 207         uint64_t result[2] = { 0, 0 };
 208         uint64_t remainder[2];
 209         uint64_t subtrahend[2];
 210         uint64_t divisor_128[2];
 211         uint64_t mask[2] = { 1, 0 };
 212         int log = 0;
 213
 214         assert(divisor != 0);
 215
 216         divisor_128[0] = divisor;
 217         divisor_128[1] = 0;
 218
 219         remainder[0] = dividend[0];
 220         remainder[1] = dividend[1];
 221
 222         subtrahend[0] = divisor;
 223         subtrahend[1] = 0;
 224
 225         while (divisor > 0) {
 226                 log++;
 227                 divisor >>= 1;
 228         }
 229
 230         dt_shift_128(subtrahend, 128 - log);
 231         dt_shift_128(mask, 128 - log);
 232
 233         while (dt_ge_128(remainder, divisor_128)) {
 234                 if (dt_ge_128(remainder, subtrahend)) {
 235                         dt_subtract_128(remainder, subtrahend, remainder);
 236                         result[0] |= mask[0];
 237                         result[1] |= mask[1];
 238                 }
 239
 240                 dt_shift_128(subtrahend, -1);
 241                 dt_shift_128(mask, -1);
 242         }
 243
 244         quotient[0] = result[0];
 245         quotient[1] = result[1];
 246 }
 247
 248 /*
 249  * This is the long-hand method of calculating a square root.
 250  * The algorithm is as follows:
 251  *
 252  * 1. Group the digits by 2 from the right.
 253  * 2. Over the leftmost group, find the largest single-digit number
 254  *    whose square is less than that group.
 255  * 3. Subtract the result of the previous step (2 or 4, depending) and
 256  *    bring down the next two-digit group.
 257  * 4. For the result R we have so far, find the largest single-digit number
 258  *    x such that 2 * R * 10 * x + x^2 is less than the result from step 3.
 259  *    (Note that this is doubling R and performing a decimal left-shift by 1
 260  *    and searching for the appropriate decimal to fill the one's place.)
 261  *    The value x is the next digit in the square root.
 262  * Repeat steps 3 and 4 until the desired precision is reached.  (We're
 263  * dealing with integers, so the above is sufficient.)
 264  *
 265  * In decimal, the square root of 582,734 would be calculated as so:
 266  *
 267  *     __7__6__3
 268  *    | 58 27 34
 269  *     -49       (7^2 == 49 => 7 is the first digit in the square root)
 270  *      --
 271  *       9 27    (Subtract and bring down the next group.)
 272  * 146   8 76    (2 * 7 * 10 * 6 + 6^2 == 876 => 6 is the next digit in
 273  *      -----     the square root)
 274  *         51 34 (Subtract and bring down the next group.)
 275  * 1523    45 69 (2 * 76 * 10 * 3 + 3^2 == 4569 => 3 is the next digit in
 276  *         -----  the square root)
 277  *          5 65 (remainder)
 278  *
 279  * The above algorithm applies similarly in binary, but note that the
 280  * only possible non-zero value for x in step 4 is 1, so step 4 becomes a
 281  * simple decision: is 2 * R * 2 * 1 + 1^2 (aka R << 2 + 1) less than the
 282  * preceding difference?
 283  *
 284  * In binary, the square root of 11011011 would be calculated as so:
 285  *
 286  *     __1__1__1__0
 287  *    | 11 01 10 11
 288  *      01          (0 << 2 + 1 == 1 < 11 => this bit is 1)
 289  *      --
 290  *      10 01 10 11
 291  * 101   1 01       (1 << 2 + 1 == 101 < 1001 => next bit is 1)
 292  *      -----
 293  *       1 00 10 11
 294  * 1101    11 01    (11 << 2 + 1 == 1101 < 10010 => next bit is 1)
 295  *       -------
 296  *          1 01 11
 297  * 11101    1 11 01 (111 << 2 + 1 == 11101 > 10111 => last bit is 0)
 298  *
 299  */
 300 static uint64_t
 301 dt_sqrt_128(uint64_t *square)
 302 {
 303         uint64_t result[2] = { 0, 0 };
 304         uint64_t diff[2] = { 0, 0 };
 305         uint64_t one[2] = { 1, 0 };
 306         uint64_t next_pair[2];
 307         uint64_t next_try[2];
 308         uint64_t bit_pairs, pair_shift;
 309         int i;
 310
 311         bit_pairs = dt_nbits_128(square) / 2;
 312         pair_shift = bit_pairs * 2;
 313
 314         for (i = 0; i <= bit_pairs; i++) {
 315                 /*
 316                  * Bring down the next pair of bits.
 317                  */
 318                 next_pair[0] = square[0];
 319                 next_pair[1] = square[1];
 320                 dt_shift_128(next_pair, -pair_shift);
 321                 next_pair[0] &= 0x3;
 322                 next_pair[1] = 0;
 323
 324                 dt_shift_128(diff, 2);
 325                 dt_add_128(diff, next_pair, diff);
 326
 327                 /*
 328                  * next_try = R << 2 + 1
 329                  */
 330                 next_try[0] = result[0];
 331                 next_try[1] = result[1];
 332                 dt_shift_128(next_try, 2);
 333                 dt_add_128(next_try, one, next_try);
 334
 335                 if (dt_le_128(next_try, diff)) {
 336                         dt_subtract_128(diff, next_try, diff);
 337                         dt_shift_128(result, 1);
 338                         dt_add_128(result, one, result);
 339                 } else {
 340                         dt_shift_128(result, 1);
 341                 }
 342
 343                 pair_shift -= 2;
 344         }
 345
 346         assert(result[1] == 0);
 347
 348         return (result[0]);
 349 }
 350
 351 uint64_t
 352 dt_stddev(uint64_t *data, uint64_t normal)
 353 {
 354         uint64_t avg_of_squares[2];
 355         uint64_t square_of_avg[2];
 356         int64_t norm_avg;
 357         uint64_t diff[2];
 358
 359         /*
 360          * The standard approximation for standard deviation is
 361          * sqrt(average(x**2) - average(x)**2), i.e. the square root
 362          * of the average of the squares minus the square of the average.
 363          */
 364         dt_divide_128(data + 2, normal, avg_of_squares);
 365         dt_divide_128(avg_of_squares, data[0], avg_of_squares);
 366
 367         norm_avg = (int64_t)data[1] / (int64_t)normal / (int64_t)data[0];
 368
 369         if (norm_avg < 0)
 370                 norm_avg = -norm_avg;
 371
 372         dt_multiply_128((uint64_t)norm_avg, (uint64_t)norm_avg, square_of_avg);
 373
 374         dt_subtract_128(avg_of_squares, square_of_avg, diff);
 375
 376         return (dt_sqrt_128(diff));
 377 }
 378
 379 static int
 380 dt_flowindent(dtrace_hdl_t *dtp, dtrace_probedata_t *data, dtrace_epid_t last,
 381     dtrace_bufdesc_t *buf, size_t offs)
 382 {
 383         dtrace_probedesc_t *pd = data->dtpda_pdesc, *npd;
 384         dtrace_eprobedesc_t *epd = data->dtpda_edesc, *nepd;
 385         char *p = pd->dtpd_provider, *n = pd->dtpd_name, *sub;
 386         dtrace_flowkind_t flow = DTRACEFLOW_NONE;
 387         const char *str = NULL;
 388         static const char *e_str[2] = { " -> ", " => " };
 389         static const char *r_str[2] = { " <- ", " <= " };
 390         static const char *ent = "entry", *ret = "return";
 391         static int entlen = 0, retlen = 0;
 392         dtrace_epid_t next, id = epd->dtepd_epid;
 393         int rval;
 394
 395         if (entlen == 0) {
 396                 assert(retlen == 0);
 397                 entlen = strlen(ent);
 398                 retlen = strlen(ret);
 399         }
 400
 401         /*
 402          * If the name of the probe is "entry" or ends with "-entry", we
 403          * treat it as an entry; if it is "return" or ends with "-return",
 404          * we treat it as a return.  (This allows application-provided probes
 405          * like "method-entry" or "function-entry" to participate in flow
 406          * indentation -- without accidentally misinterpreting popular probe
 407          * names like "carpentry", "gentry" or "Coventry".)
 408          */
 409         if ((sub = strstr(n, ent)) != NULL && sub[entlen] == '\0' &&
 410             (sub == n || sub[-1] == '-')) {
 411                 flow = DTRACEFLOW_ENTRY;
 412                 str = e_str[strcmp(p, "syscall") == 0];
 413         } else if ((sub = strstr(n, ret)) != NULL && sub[retlen] == '\0' &&
 414             (sub == n || sub[-1] == '-')) {
 415                 flow = DTRACEFLOW_RETURN;
 416                 str = r_str[strcmp(p, "syscall") == 0];
 417         }
 418
 419         /*
 420          * If we're going to indent this, we need to check the ID of our last
 421          * call.  If we're looking at the same probe ID but a different EPID,
 422          * we _don't_ want to indent.  (Yes, there are some minor holes in
 423          * this scheme -- it's a heuristic.)
 424          */
 425         if (flow == DTRACEFLOW_ENTRY) {
 426                 if ((last != DTRACE_EPIDNONE && id != last &&
 427                     pd->dtpd_id == dtp->dt_pdesc[last]->dtpd_id))
 428                         flow = DTRACEFLOW_NONE;
 429         }
 430
 431         /*
 432          * If we're going to unindent this, it's more difficult to see if
 433          * we don't actually want to unindent it -- we need to look at the
 434          * _next_ EPID.
 435          */
 436         if (flow == DTRACEFLOW_RETURN) {
 437                 offs += epd->dtepd_size;
 438
 439                 do {
 440                         if (offs >= buf->dtbd_size) {
 441                                 /*
 442                                  * We're at the end -- maybe.  If the oldest
 443                                  * record is non-zero, we need to wrap.
 444                                  */
 445                                 if (buf->dtbd_oldest != 0) {
 446                                         offs = 0;
 447                                 } else {
 448                                         goto out;
 449                                 }
 450                         }
 451
 452                         next = *(uint32_t *)((uintptr_t)buf->dtbd_data + offs);
 453
 454                         if (next == DTRACE_EPIDNONE)
 455                                 offs += sizeof (id);
 456                 } while (next == DTRACE_EPIDNONE);
 457
 458                 if ((rval = dt_epid_lookup(dtp, next, &nepd, &npd)) != 0)
 459                         return (rval);
 460
 461                 if (next != id && npd->dtpd_id == pd->dtpd_id)
 462                         flow = DTRACEFLOW_NONE;
 463         }
 464
 465 out:
 466         if (flow == DTRACEFLOW_ENTRY || flow == DTRACEFLOW_RETURN) {
 467                 data->dtpda_prefix = str;
 468         } else {
 469                 data->dtpda_prefix = "| ";
 470         }
 471
 472         if (flow == DTRACEFLOW_RETURN && data->dtpda_indent > 0)
 473                 data->dtpda_indent -= 2;
 474
 475         data->dtpda_flow = flow;
 476
 477         return (0);
 478 }
 479
 480 static int
 481 dt_nullprobe()
 482 {
 483         return (DTRACE_CONSUME_THIS);
 484 }
 485
 486 static int
 487 dt_nullrec()
 488 {
 489         return (DTRACE_CONSUME_NEXT);
 490 }
 491
 492 int
 493 dt_print_quantline(dtrace_hdl_t *dtp, FILE *fp, int64_t val,
 494     uint64_t normal, long double total, char positives, char negatives)
 495 {
 496         long double f;
 497         uint_t depth, len = 40;
 498
 499         const char *ats = "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@";
 500         const char *spaces = "                                        ";
 501
 502         assert(strlen(ats) == len && strlen(spaces) == len);
 503         assert(!(total == 0 && (positives || negatives)));
 504         assert(!(val < 0 && !negatives));
 505         assert(!(val > 0 && !positives));
 506         assert(!(val != 0 && total == 0));
 507
 508         if (!negatives) {
 509                 if (positives) {
 510                         f = (dt_fabsl((long double)val) * len) / total;
 511                         depth = (uint_t)(f + 0.5);
 512                 } else {
 513                         depth = 0;
 514                 }
 515
 516                 return (dt_printf(dtp, fp, "|%s%s %-9lld\n", ats + len - depth,
 517                     spaces + depth, (long long)val / normal));
 518         }
 519
 520         if (!positives) {
 521                 f = (dt_fabsl((long double)val) * len) / total;
 522                 depth = (uint_t)(f + 0.5);
 523
 524                 return (dt_printf(dtp, fp, "%s%s| %-9lld\n", spaces + depth,
 525                     ats + len - depth, (long long)val / normal));
 526         }
 527
 528         /*
 529          * If we're here, we have both positive and negative bucket values.
 530          * To express this graphically, we're going to generate both positive
 531          * and negative bars separated by a centerline.  These bars are half
 532          * the size of normal quantize()/lquantize() bars, so we divide the
 533          * length in half before calculating the bar length.
 534          */
 535         len /= 2;
 536         ats = &ats[len];
 537         spaces = &spaces[len];
 538
 539         f = (dt_fabsl((long double)val) * len) / total;
 540         depth = (uint_t)(f + 0.5);
 541
 542         if (val <= 0) {
 543                 return (dt_printf(dtp, fp, "%s%s|%*s %-9lld\n", spaces + depth,
 544                     ats + len - depth, len, "", (long long)val / normal));
 545         } else {
 546                 return (dt_printf(dtp, fp, "%20s|%s%s %-9lld\n", "",
 547                     ats + len - depth, spaces + depth,
 548                     (long long)val / normal));
 549         }
 550 }
 551
 552 int
 553 dt_print_quantize(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
 554     size_t size, uint64_t normal)
 555 {
 556         const int64_t *data = addr;
 557         int i, first_bin = 0, last_bin = DTRACE_QUANTIZE_NBUCKETS - 1;
 558         long double total = 0;
 559         char positives = 0, negatives = 0;
 560
 561         if (size != DTRACE_QUANTIZE_NBUCKETS * sizeof (uint64_t))
 562                 return (dt_set_errno(dtp, EDT_DMISMATCH));
 563
 564         while (first_bin < DTRACE_QUANTIZE_NBUCKETS - 1 && data[first_bin] == 0)
 565                 first_bin++;
 566
 567         if (first_bin == DTRACE_QUANTIZE_NBUCKETS - 1) {
 568                 /*
 569                  * There isn't any data.  This is possible if (and only if)
 570                  * negative increment values have been used.  In this case,
 571                  * we'll print the buckets around 0.
 572                  */
 573                 first_bin = DTRACE_QUANTIZE_ZEROBUCKET - 1;
 574                 last_bin = DTRACE_QUANTIZE_ZEROBUCKET + 1;
 575         } else {
 576                 if (first_bin > 0)
 577                         first_bin--;
 578
 579                 while (last_bin > 0 && data[last_bin] == 0)
 580                         last_bin--;
 581
 582                 if (last_bin < DTRACE_QUANTIZE_NBUCKETS - 1)
 583                         last_bin++;
 584         }
 585
 586         for (i = first_bin; i <= last_bin; i++) {
 587                 positives |= (data[i] > 0);
 588                 negatives |= (data[i] < 0);
 589                 total += dt_fabsl((long double)data[i]);
 590         }
 591
 592         if (dt_printf(dtp, fp, "\n%16s %41s %-9s\n", "value",
 593             "------------- Distribution -------------", "count") < 0)
 594                 return (-1);
 595
 596         for (i = first_bin; i <= last_bin; i++) {
 597                 if (dt_printf(dtp, fp, "%16lld ",
 598                     (long long)DTRACE_QUANTIZE_BUCKETVAL(i)) < 0)
 599                         return (-1);
 600
 601                 if (dt_print_quantline(dtp, fp, data[i], normal, total,
 602                     positives, negatives) < 0)
 603                         return (-1);
 604         }
 605
 606         return (0);
 607 }
 608
 609 int
 610 dt_print_lquantize(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
 611     size_t size, uint64_t normal)
 612 {
 613         const int64_t *data = addr;
 614         int i, first_bin, last_bin, base;
 615         uint64_t arg;
 616         long double total = 0;
 617         uint16_t step, levels;
 618         char positives = 0, negatives = 0;
 619
 620         if (size < sizeof (uint64_t))
 621                 return (dt_set_errno(dtp, EDT_DMISMATCH));
 622
 623         arg = *data++;
 624         size -= sizeof (uint64_t);
 625
 626         base = DTRACE_LQUANTIZE_BASE(arg);
 627         step = DTRACE_LQUANTIZE_STEP(arg);
 628         levels = DTRACE_LQUANTIZE_LEVELS(arg);
 629
 630         first_bin = 0;
 631         last_bin = levels + 1;
 632
 633         if (size != sizeof (uint64_t) * (levels + 2))
 634                 return (dt_set_errno(dtp, EDT_DMISMATCH));
 635
 636         while (first_bin <= levels + 1 && data[first_bin] == 0)
 637                 first_bin++;
 638
 639         if (first_bin > levels + 1) {
 640                 first_bin = 0;
 641                 last_bin = 2;
 642         } else {
 643                 if (first_bin > 0)
 644                         first_bin--;
 645
 646                 while (last_bin > 0 && data[last_bin] == 0)
 647                         last_bin--;
 648
 649                 if (last_bin < levels + 1)
 650                         last_bin++;
 651         }
 652
 653         for (i = first_bin; i <= last_bin; i++) {
 654                 positives |= (data[i] > 0);
 655                 negatives |= (data[i] < 0);
 656                 total += dt_fabsl((long double)data[i]);
 657         }
 658
 659         if (dt_printf(dtp, fp, "\n%16s %41s %-9s\n", "value",
 660             "------------- Distribution -------------", "count") < 0)
 661                 return (-1);
 662
 663         for (i = first_bin; i <= last_bin; i++) {
 664                 char c[32];
 665                 int err;
 666
 667                 if (i == 0) {
 668                         (void) snprintf(c, sizeof (c), "< %d",
 669                             base / (uint32_t)normal);
 670                         err = dt_printf(dtp, fp, "%16s ", c);
 671                 } else if (i == levels + 1) {
 672                         (void) snprintf(c, sizeof (c), ">= %d",
 673                             base + (levels * step));
 674                         err = dt_printf(dtp, fp, "%16s ", c);
 675                 } else {
 676                         err = dt_printf(dtp, fp, "%16d ",
 677                             base + (i - 1) * step);
 678                 }
 679
 680                 if (err < 0 || dt_print_quantline(dtp, fp, data[i], normal,
 681                     total, positives, negatives) < 0)
 682                         return (-1);
 683         }
 684
 685         return (0);
 686 }
 687
 688 /*ARGSUSED*/
 689 static int
 690 dt_print_average(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr,
 691     size_t size, uint64_t normal)
 692 {
 693         /* LINTED - alignment */
 694         int64_t *data = (int64_t *)addr;
 695
 696         return (dt_printf(dtp, fp, " %16lld", data[0] ?
 697             (long long)(data[1] / (int64_t)normal / data[0]) : 0));
 698 }
 699
 700 /*ARGSUSED*/
 701 static int
 702 dt_print_stddev(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr,
 703     size_t size, uint64_t normal)
 704 {
 705         /* LINTED - alignment */
 706         uint64_t *data = (uint64_t *)addr;
 707
 708         return (dt_printf(dtp, fp, " %16llu", data[0] ?
 709             (unsigned long long) dt_stddev(data, normal) : 0));
 710 }
 711
 712 /*ARGSUSED*/
 713 int
 714 dt_print_bytes(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr,
 715     size_t nbytes, int width, int quiet, int raw)
 716 {
 717         /*
 718          * If the byte stream is a series of printable characters, followed by
 719          * a terminating byte, we print it out as a string.  Otherwise, we
 720          * assume that it's something else and just print the bytes.
 721          */
 722         int i, j, margin = 5;
 723         char *c = (char *)addr;
 724
 725         if (nbytes == 0)
 726                 return (0);
 727
 728         if (raw || dtp->dt_options[DTRACEOPT_RAWBYTES] != DTRACEOPT_UNSET)
 729                 goto raw;
 730
 731         for (i = 0; i < nbytes; i++) {
 732                 /*
 733                  * We define a "printable character" to be one for which
 734                  * isprint(3C) returns non-zero, isspace(3C) returns non-zero,
 735                  * or a character which is either backspace or the bell.
 736                  * Backspace and the bell are regrettably special because
 737                  * they fail the first two tests -- and yet they are entirely
 738                  * printable.  These are the only two control characters that
 739                  * have meaning for the terminal and for which isprint(3C) and
 740                  * isspace(3C) return 0.
 741                  */
 742                 if (isprint(c[i]) || isspace(c[i]) ||
 743                     c[i] == '\b' || c[i] == '\a')
 744                         continue;
 745
 746                 if (c[i] == '\0' && i > 0) {
 747                         /*
 748                          * This looks like it might be a string.  Before we
 749                          * assume that it is indeed a string, check the
 750                          * remainder of the byte range; if it contains
 751                          * additional non-nul characters, we'll assume that
 752                          * it's a binary stream that just happens to look like
 753                          * a string, and we'll print out the individual bytes.
 754                          */
 755                         for (j = i + 1; j < nbytes; j++) {
 756                                 if (c[j] != '\0')
 757                                         break;
 758                         }
 759
 760                         if (j != nbytes)
 761                                 break;
 762
 763                         if (quiet)
 764                                 return (dt_printf(dtp, fp, "%s", c));
 765                         else
 766                                 return (dt_printf(dtp, fp, "  %-*s", width, c));
 767                 }
 768
 769                 break;
 770         }
 771
 772         if (i == nbytes) {
 773                 /*
 774                  * The byte range is all printable characters, but there is
 775                  * no trailing nul byte.  We'll assume that it's a string and
 776                  * print it as such.
 777                  */
 778                 char *s = alloca(nbytes + 1);
 779                 bcopy(c, s, nbytes);
 780                 s[nbytes] = '\0';
 781                 return (dt_printf(dtp, fp, "  %-*s", width, s));
 782         }
 783
 784 raw:
 785         if (dt_printf(dtp, fp, "\n%*s      ", margin, "") < 0)
 786                 return (-1);
 787
 788         for (i = 0; i < 16; i++)
 789                 if (dt_printf(dtp, fp, "  %c", "0123456789abcdef"[i]) < 0)
 790                         return (-1);
 791
 792         if (dt_printf(dtp, fp, "  0123456789abcdef\n") < 0)
 793                 return (-1);
 794
 795
 796         for (i = 0; i < nbytes; i += 16) {
 797                 if (dt_printf(dtp, fp, "%*s%5x:", margin, "", i) < 0)
 798                         return (-1);
 799
 800                 for (j = i; j < i + 16 && j < nbytes; j++) {
 801                         if (dt_printf(dtp, fp, " %02x", (uchar_t)c[j]) < 0)
 802                                 return (-1);
 803                 }
 804
 805                 while (j++ % 16) {
 806                         if (dt_printf(dtp, fp, "   ") < 0)
 807                                 return (-1);
 808                 }
 809
 810                 if (dt_printf(dtp, fp, "  ") < 0)
 811                         return (-1);
 812
 813                 for (j = i; j < i + 16 && j < nbytes; j++) {
 814                         if (dt_printf(dtp, fp, "%c",
 815                             c[j] < ' ' || c[j] > '~' ? '.' : c[j]) < 0)
 816                                 return (-1);
 817                 }
 818
 819                 if (dt_printf(dtp, fp, "\n") < 0)
 820                         return (-1);
 821         }
 822
 823         return (0);
 824 }
 825
 826 int
 827 dt_print_stack(dtrace_hdl_t *dtp, FILE *fp, const char *format,
 828     caddr_t addr, int depth, int size)
 829 {
 830         dtrace_syminfo_t dts;
 831         GElf_Sym sym;
 832         int i, indent;
 833         char c[PATH_MAX * 2];
 834         uint64_t pc;
 835
 836         if (dt_printf(dtp, fp, "\n") < 0)
 837                 return (-1);
 838
 839         if (format == NULL)
 840                 format = "%s";
 841
 842         if (dtp->dt_options[DTRACEOPT_STACKINDENT] != DTRACEOPT_UNSET)
 843                 indent = (int)dtp->dt_options[DTRACEOPT_STACKINDENT];
 844         else
 845                 indent = _dtrace_stkindent;
 846
 847         for (i = 0; i < depth; i++) {
 848                 switch (size) {
 849                 case sizeof (uint32_t):
 850                         /* LINTED - alignment */
 851                         pc = *((uint32_t *)addr);
 852                         break;
 853
 854                 case sizeof (uint64_t):
 855                         /* LINTED - alignment */
 856                         pc = *((uint64_t *)addr);
 857                         break;
 858
 859                 default:
 860                         return (dt_set_errno(dtp, EDT_BADSTACKPC));
 861                 }
 862
 863                 if (pc == 0)
 864                         break;
 865
 866                 addr += size;
 867
 868                 if (dt_printf(dtp, fp, "%*s", indent, "") < 0)
 869                         return (-1);
 870
 871                 if (dtrace_lookup_by_addr(dtp, pc, &sym, &dts) == 0) {
 872                         if (pc > sym.st_value) {
 873                                 (void) snprintf(c, sizeof (c), "%s`%s+0x%llx",
 874                                     dts.dts_object, dts.dts_name,
 875                                     pc - sym.st_value);
 876                         } else {
 877                                 (void) snprintf(c, sizeof (c), "%s`%s",
 878                                     dts.dts_object, dts.dts_name);
 879                         }
 880                 } else {
 881                         /*
 882                          * We'll repeat the lookup, but this time we'll specify
 883                          * a NULL GElf_Sym -- indicating that we're only
 884                          * interested in the containing module.
 885                          */
 886                         if (dtrace_lookup_by_addr(dtp, pc, NULL, &dts) == 0) {
 887                                 (void) snprintf(c, sizeof (c), "%s`0x%llx",
 888                                     dts.dts_object, pc);
 889                         } else {
 890                                 (void) snprintf(c, sizeof (c), "0x%llx", pc);
 891                         }
 892                 }
 893
 894                 if (dt_printf(dtp, fp, format, c) < 0)
 895                         return (-1);
 896
 897                 if (dt_printf(dtp, fp, "\n") < 0)
 898                         return (-1);
 899         }
 900
 901         return (0);
 902 }
 903
 904 int
 905 dt_print_ustack(dtrace_hdl_t *dtp, FILE *fp, const char *format,
 906     caddr_t addr, uint64_t arg)
 907 {
 908         /* LINTED - alignment */
 909         uint64_t *pc = (uint64_t *)addr;
 910         uint32_t depth = DTRACE_USTACK_NFRAMES(arg);
 911         uint32_t strsize = DTRACE_USTACK_STRSIZE(arg);
 912         const char *strbase = addr + (depth + 1) * sizeof (uint64_t);
 913         const char *str = strsize ? strbase : NULL;
 914         int err = 0;
 915
 916         char name[PATH_MAX], objname[PATH_MAX], c[PATH_MAX * 2];
 917         struct ps_prochandle *P;
 918         GElf_Sym sym;
 919         int i, indent;
 920         pid_t pid;
 921
 922         if (depth == 0)
 923                 return (0);
 924
 925         pid = (pid_t)*pc++;
 926
 927         if (dt_printf(dtp, fp, "\n") < 0)
 928                 return (-1);
 929
 930         if (format == NULL)
 931                 format = "%s";
 932
 933         if (dtp->dt_options[DTRACEOPT_STACKINDENT] != DTRACEOPT_UNSET)
 934                 indent = (int)dtp->dt_options[DTRACEOPT_STACKINDENT];
 935         else
 936                 indent = _dtrace_stkindent;
 937
 938         /*
 939          * Ultimately, we need to add an entry point in the library vector for
 940          * determining <symbol, offset> from <pid, address>.  For now, if
 941          * this is a vector open, we just print the raw address or string.
 942          */
 943         if (dtp->dt_vector == NULL)
 944                 P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0);
 945         else
 946                 P = NULL;
 947
 948         if (P != NULL)
 949                 dt_proc_lock(dtp, P); /* lock handle while we perform lookups */
 950
 951         for (i = 0; i < depth && pc[i] != 0; i++) {
 952                 const prmap_t *map;
 953
 954                 if ((err = dt_printf(dtp, fp, "%*s", indent, "")) < 0)
 955                         break;
 956
 957 #if defined(sun)
 958                 if (P != NULL && Plookup_by_addr(P, pc[i],
 959 #else
 960                 if (P != NULL && proc_addr2sym(P, pc[i],
 961 #endif
 962                     name, sizeof (name), &sym) == 0) {
 963 #if defined(sun)
 964                         (void) Pobjname(P, pc[i], objname, sizeof (objname));
 965 #else
 966                         (void) proc_objname(P, pc[i], objname, sizeof (objname));
 967 #endif
 968
 969                         if (pc[i] > sym.st_value) {
 970                                 (void) snprintf(c, sizeof (c),
 971                                     "%s`%s+0x%llx", dt_basename(objname), name,
 972                                     (u_longlong_t)(pc[i] - sym.st_value));
 973                         } else {
 974                                 (void) snprintf(c, sizeof (c),
 975                                     "%s`%s", dt_basename(objname), name);
 976                         }
 977                 } else if (str != NULL && str[0] != '\0' && str[0] != '@' &&
 978 #if defined(sun)
 979                     (P != NULL && ((map = Paddr_to_map(P, pc[i])) == NULL ||
 980                     (map->pr_mflags & MA_WRITE)))) {
 981 #else
 982                     (P != NULL && ((map = proc_addr2map(P, pc[i])) == NULL))) {
 983 #endif
 984                         /*
 985                          * If the current string pointer in the string table
 986                          * does not point to an empty string _and_ the program
 987                          * counter falls in a writable region, we'll use the
 988                          * string from the string table instead of the raw
 989                          * address.  This last condition is necessary because
 990                          * some (broken) ustack helpers will return a string
 991                          * even for a program counter that they can't
 992                          * identify.  If we have a string for a program
 993                          * counter that falls in a segment that isn't
 994                          * writable, we assume that we have fallen into this
 995                          * case and we refuse to use the string.
 996                          */
 997                         (void) snprintf(c, sizeof (c), "%s", str);
 998                 } else {
 999 #if defined(sun)
1000                         if (P != NULL && Pobjname(P, pc[i], objname,
1001 #else
1002                         if (P != NULL && proc_objname(P, pc[i], objname,
1003 #endif
1004                             sizeof (objname)) != 0) {
1005                                 (void) snprintf(c, sizeof (c), "%s`0x%llx",
1006                                     dt_basename(objname), (u_longlong_t)pc[i]);
1007                         } else {
1008                                 (void) snprintf(c, sizeof (c), "0x%llx",
1009                                     (u_longlong_t)pc[i]);
1010                         }
1011                 }
1012
1013                 if ((err = dt_printf(dtp, fp, format, c)) < 0)
1014                         break;
1015
1016                 if ((err = dt_printf(dtp, fp, "\n")) < 0)
1017                         break;
1018
1019                 if (str != NULL && str[0] == '@') {
1020                         /*
1021                          * If the first character of the string is an "at" sign,
1022                          * then the string is inferred to be an annotation --
1023                          * and it is printed out beneath the frame and offset
1024                          * with brackets.
1025                          */
1026                         if ((err = dt_printf(dtp, fp, "%*s", indent, "")) < 0)
1027                                 break;
1028
1029                         (void) snprintf(c, sizeof (c), "  [ %s ]", &str[1]);
1030
1031                         if ((err = dt_printf(dtp, fp, format, c)) < 0)
1032                                 break;
1033
1034                         if ((err = dt_printf(dtp, fp, "\n")) < 0)
1035                                 break;
1036                 }
1037
1038                 if (str != NULL) {
1039                         str += strlen(str) + 1;
1040                         if (str - strbase >= strsize)
1041                                 str = NULL;
1042                 }
1043         }
1044
1045         if (P != NULL) {
1046                 dt_proc_unlock(dtp, P);
1047                 dt_proc_release(dtp, P);
1048         }
1049
1050         return (err);
1051 }
1052
1053 static int
1054 dt_print_usym(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr, dtrace_actkind_t act)
1055 {
1056         /* LINTED - alignment */
1057         uint64_t pid = ((uint64_t *)addr)[0];
1058         /* LINTED - alignment */
1059         uint64_t pc = ((uint64_t *)addr)[1];
1060         const char *format = "  %-50s";
1061         char *s;
1062         int n, len = 256;
1063
1064         if (act == DTRACEACT_USYM && dtp->dt_vector == NULL) {
1065                 struct ps_prochandle *P;
1066
1067                 if ((P = dt_proc_grab(dtp, pid,
1068                     PGRAB_RDONLY | PGRAB_FORCE, 0)) != NULL) {
1069                         GElf_Sym sym;
1070
1071                         dt_proc_lock(dtp, P);
1072
1073 #if defined(sun)
1074                         if (Plookup_by_addr(P, pc, NULL, 0, &sym) == 0)
1075 #else
1076                         if (proc_addr2sym(P, pc, NULL, 0, &sym) == 0)
1077 #endif
1078                                 pc = sym.st_value;
1079
1080                         dt_proc_unlock(dtp, P);
1081                         dt_proc_release(dtp, P);
1082                 }
1083         }
1084
1085         do {
1086                 n = len;
1087                 s = alloca(n);
1088         } while ((len = dtrace_uaddr2str(dtp, pid, pc, s, n)) >= n);
1089
1090         return (dt_printf(dtp, fp, format, s));
1091 }
1092
1093 int
1094 dt_print_umod(dtrace_hdl_t *dtp, FILE *fp, const char *format, caddr_t addr)
1095 {
1096         /* LINTED - alignment */
1097         uint64_t pid = ((uint64_t *)addr)[0];
1098         /* LINTED - alignment */
1099         uint64_t pc = ((uint64_t *)addr)[1];
1100         int err = 0;
1101
1102         char objname[PATH_MAX], c[PATH_MAX * 2];
1103         struct ps_prochandle *P;
1104
1105         if (format == NULL)
1106                 format = "  %-50s";
1107
1108         /*
1109          * See the comment in dt_print_ustack() for the rationale for
1110          * printing raw addresses in the vectored case.
1111          */
1112         if (dtp->dt_vector == NULL)
1113                 P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0);
1114         else
1115                 P = NULL;
1116
1117         if (P != NULL)
1118                 dt_proc_lock(dtp, P); /* lock handle while we perform lookups */
1119
1120 #if defined(sun)
1121         if (P != NULL && Pobjname(P, pc, objname, sizeof (objname)) != 0) {
1122 #else
1123         if (P != NULL && proc_objname(P, pc, objname, sizeof (objname)) != 0) {
1124 #endif
1125                 (void) snprintf(c, sizeof (c), "%s", dt_basename(objname));
1126         } else {
1127                 (void) snprintf(c, sizeof (c), "0x%llx", (u_longlong_t)pc);
1128         }
1129
1130         err = dt_printf(dtp, fp, format, c);
1131
1132         if (P != NULL) {
1133                 dt_proc_unlock(dtp, P);
1134                 dt_proc_release(dtp, P);
1135         }
1136
1137         return (err);
1138 }
1139
1140 int
1141 dt_print_memory(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr)
1142 {
1143         int quiet = (dtp->dt_options[DTRACEOPT_QUIET] != DTRACEOPT_UNSET);
1144         size_t nbytes = *((uintptr_t *) addr);
1145
1146         return (dt_print_bytes(dtp, fp, addr + sizeof(uintptr_t),
1147             nbytes, 50, quiet, 1));
1148 }
1149
1150 typedef struct dt_type_cbdata {
1151         dtrace_hdl_t            *dtp;
1152         dtrace_typeinfo_t       dtt;
1153         caddr_t                 addr;
1154         caddr_t                 addrend;
1155         const char              *name;
1156         int                     f_type;
1157         int                     indent;
1158         int                     type_width;
1159         int                     name_width;
1160         FILE                    *fp;
1161 } dt_type_cbdata_t;
1162
1163 static int      dt_print_type_data(dt_type_cbdata_t *, ctf_id_t);
1164
1165 static int
1166 dt_print_type_member(const char *name, ctf_id_t type, ulong_t off, void *arg)
1167 {
1168         dt_type_cbdata_t cbdata;
1169         dt_type_cbdata_t *cbdatap = arg;
1170         ssize_t ssz;
1171
1172         if ((ssz = ctf_type_size(cbdatap->dtt.dtt_ctfp, type)) <= 0)
1173                 return (0);
1174
1175         off /= 8;
1176
1177         cbdata = *cbdatap;
1178         cbdata.name = name;
1179         cbdata.addr += off;
1180         cbdata.addrend = cbdata.addr + ssz;
1181
1182         return (dt_print_type_data(&cbdata, type));
1183 }
1184
1185 static int
1186 dt_print_type_width(const char *name, ctf_id_t type, ulong_t off, void *arg)
1187 {
1188         char buf[DT_TYPE_NAMELEN];
1189         char *p;
1190         dt_type_cbdata_t *cbdatap = arg;
1191         size_t sz = strlen(name);
1192
1193         ctf_type_name(cbdatap->dtt.dtt_ctfp, type, buf, sizeof (buf));
1194
1195         if ((p = strchr(buf, '[')) != NULL)
1196                 p[-1] = '\0';
1197         else
1198                 p = "";
1199
1200         sz += strlen(p);
1201
1202         if (sz > cbdatap->name_width)
1203                 cbdatap->name_width = sz;
1204
1205         sz = strlen(buf);
1206
1207         if (sz > cbdatap->type_width)
1208                 cbdatap->type_width = sz;
1209
1210         return (0);
1211 }
1212
1213 static int
1214 dt_print_type_data(dt_type_cbdata_t *cbdatap, ctf_id_t type)
1215 {
1216         caddr_t addr = cbdatap->addr;
1217         caddr_t addrend = cbdatap->addrend;
1218         char buf[DT_TYPE_NAMELEN];
1219         char *p;
1220         int cnt = 0;
1221         uint_t kind = ctf_type_kind(cbdatap->dtt.dtt_ctfp, type);
1222         ssize_t ssz = ctf_type_size(cbdatap->dtt.dtt_ctfp, type);
1223
1224         ctf_type_name(cbdatap->dtt.dtt_ctfp, type, buf, sizeof (buf));
1225
1226         if ((p = strchr(buf, '[')) != NULL)
1227                 p[-1] = '\0';
1228         else
1229                 p = "";
1230
1231         if (cbdatap->f_type) {
1232                 int type_width = roundup(cbdatap->type_width + 1, 4);
1233                 int name_width = roundup(cbdatap->name_width + 1, 4);
1234
1235                 name_width -= strlen(cbdatap->name);
1236
1237                 dt_printf(cbdatap->dtp, cbdatap->fp, "%*s%-*s%s%-*s     = ",cbdatap->indent * 4,"",type_width,buf,cbdatap->name,name_width,p);
1238         }
1239
1240         while (addr < addrend) {
1241                 dt_type_cbdata_t cbdata;
1242                 ctf_arinfo_t arinfo;
1243                 ctf_encoding_t cte;
1244                 uintptr_t *up;
1245                 void *vp = addr;
1246                 cbdata = *cbdatap;
1247                 cbdata.name = "";
1248                 cbdata.addr = addr;
1249                 cbdata.addrend = addr + ssz;
1250                 cbdata.f_type = 0;
1251                 cbdata.indent++;
1252                 cbdata.type_width = 0;
1253                 cbdata.name_width = 0;
1254
1255                 if (cnt > 0)
1256                         dt_printf(cbdatap->dtp, cbdatap->fp, "%*s", cbdatap->indent * 4,"");
1257
1258                 switch (kind) {
1259                 case CTF_K_INTEGER:
1260                         if (ctf_type_encoding(cbdatap->dtt.dtt_ctfp, type, &cte) != 0)
1261                                 return (-1);
1262                         if ((cte.cte_format & CTF_INT_SIGNED) != 0)
1263                                 switch (cte.cte_bits) {
1264                                 case 8:
1265                                         if (isprint(*((char *) vp)))
1266                                                 dt_printf(cbdatap->dtp, cbdatap->fp, "'%c', ", *((char *) vp));
1267                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%d (0x%x);\n", *((char *) vp), *((char *) vp));
1268                                         break;
1269                                 case 16:
1270                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%hd (0x%hx);\n", *((short *) vp), *((u_short *) vp));
1271                                         break;
1272                                 case 32:
1273                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%d (0x%x);\n", *((int *) vp), *((u_int *) vp));
1274                                         break;
1275                                 case 64:
1276                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%jd (0x%jx);\n", *((long long *) vp), *((unsigned long long *) vp));
1277                                         break;
1278                                 default:
1279                                         dt_printf(cbdatap->dtp, cbdatap->fp, "CTF_K_INTEGER: format %x offset %u bits %u\n",cte.cte_format,cte.cte_offset,cte.cte_bits);
1280                                         break;
1281                                 }
1282                         else
1283                                 switch (cte.cte_bits) {
1284                                 case 8:
1285                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%u (0x%x);\n", *((uint8_t *) vp) & 0xff, *((uint8_t *) vp) & 0xff);
1286                                         break;
1287                                 case 16:
1288                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%hu (0x%hx);\n", *((u_short *) vp), *((u_short *) vp));
1289                                         break;
1290                                 case 32:
1291                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%u (0x%x);\n", *((u_int *) vp), *((u_int *) vp));
1292                                         break;
1293                                 case 64:
1294                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%ju (0x%jx);\n", *((unsigned long long *) vp), *((unsigned long long *) vp));
1295                                         break;
1296                                 default:
1297                                         dt_printf(cbdatap->dtp, cbdatap->fp, "CTF_K_INTEGER: format %x offset %u bits %u\n",cte.cte_format,cte.cte_offset,cte.cte_bits);
1298                                         break;
1299                                 }
1300                         break;
1301                 case CTF_K_FLOAT:
1302                         dt_printf(cbdatap->dtp, cbdatap->fp, "CTF_K_FLOAT: format %x offset %u bits %u\n",cte.cte_format,cte.cte_offset,cte.cte_bits);
1303                         break;
1304                 case CTF_K_POINTER:
1305                         dt_printf(cbdatap->dtp, cbdatap->fp, "%p;\n", *((void **) addr));
1306                         break;
1307                 case CTF_K_ARRAY:
1308                         if (ctf_array_info(cbdatap->dtt.dtt_ctfp, type, &arinfo) != 0)
1309                                 return (-1);
1310                         dt_printf(cbdatap->dtp, cbdatap->fp, "{\n%*s",cbdata.indent * 4,"");
1311                         dt_print_type_data(&cbdata, arinfo.ctr_contents);
1312                         dt_printf(cbdatap->dtp, cbdatap->fp, "%*s};\n",cbdatap->indent * 4,"");
1313                         break;
1314                 case CTF_K_FUNCTION:
1315                         dt_printf(cbdatap->dtp, cbdatap->fp, "CTF_K_FUNCTION:\n");
1316                         break;
1317                 case CTF_K_STRUCT:
1318                         cbdata.f_type = 1;
1319                         if (ctf_member_iter(cbdatap->dtt.dtt_ctfp, type,
1320                             dt_print_type_width, &cbdata) != 0)
1321                                 return (-1);
1322                         dt_printf(cbdatap->dtp, cbdatap->fp, "{\n");
1323                         if (ctf_member_iter(cbdatap->dtt.dtt_ctfp, type,
1324                             dt_print_type_member, &cbdata) != 0)
1325                                 return (-1);
1326                         dt_printf(cbdatap->dtp, cbdatap->fp, "%*s};\n",cbdatap->indent * 4,"");
1327                         break;
1328                 case CTF_K_UNION:
1329                         cbdata.f_type = 1;
1330                         if (ctf_member_iter(cbdatap->dtt.dtt_ctfp, type,
1331                             dt_print_type_width, &cbdata) != 0)
1332                                 return (-1);
1333                         dt_printf(cbdatap->dtp, cbdatap->fp, "{\n");
1334                         if (ctf_member_iter(cbdatap->dtt.dtt_ctfp, type,
1335                             dt_print_type_member, &cbdata) != 0)
1336                                 return (-1);
1337                         dt_printf(cbdatap->dtp, cbdatap->fp, "%*s};\n",cbdatap->indent * 4,"");
1338                         break;
1339                 case CTF_K_ENUM:
1340                         dt_printf(cbdatap->dtp, cbdatap->fp, "%s;\n", ctf_enum_name(cbdatap->dtt.dtt_ctfp, type, *((int *) vp)));
1341                         break;
1342                 case CTF_K_TYPEDEF:
1343                         dt_print_type_data(&cbdata, ctf_type_reference(cbdatap->dtt.dtt_ctfp,type));
1344                         break;
1345                 case CTF_K_VOLATILE:
1346                         if (cbdatap->f_type)
1347                                 dt_printf(cbdatap->dtp, cbdatap->fp, "volatile ");
1348                         dt_print_type_data(&cbdata, ctf_type_reference(cbdatap->dtt.dtt_ctfp,type));
1349                         break;
1350                 case CTF_K_CONST:
1351                         if (cbdatap->f_type)
1352                                 dt_printf(cbdatap->dtp, cbdatap->fp, "const ");
1353                         dt_print_type_data(&cbdata, ctf_type_reference(cbdatap->dtt.dtt_ctfp,type));
1354                         break;
1355                 case CTF_K_RESTRICT:
1356                         if (cbdatap->f_type)
1357                                 dt_printf(cbdatap->dtp, cbdatap->fp, "restrict ");
1358                         dt_print_type_data(&cbdata, ctf_type_reference(cbdatap->dtt.dtt_ctfp,type));
1359                         break;
1360                 default:
1361                         break;
1362                 }
1363
1364                 addr += ssz;
1365                 cnt++;
1366         }
1367
1368         return (0);
1369 }
1370
1371 static int
1372 dt_print_type(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr)
1373 {
1374         caddr_t addrend;
1375         char *p;
1376         dtrace_typeinfo_t dtt;
1377         dt_type_cbdata_t cbdata;
1378         int num = 0;
1379         int quiet = (dtp->dt_options[DTRACEOPT_QUIET] != DTRACEOPT_UNSET);
1380         ssize_t ssz;
1381
1382         if (!quiet)
1383                 dt_printf(dtp, fp, "\n");
1384
1385         /* Get the total number of bytes of data buffered. */
1386         size_t nbytes = *((uintptr_t *) addr);
1387         addr += sizeof(uintptr_t);
1388
1389         /*
1390          * Get the size of the type so that we can check that it matches
1391          * the CTF data we look up and so that we can figure out how many
1392          * type elements are buffered.
1393          */
1394         size_t typs = *((uintptr_t *) addr);
1395         addr += sizeof(uintptr_t);
1396
1397         /*
1398          * Point to the type string in the buffer. Get it's string
1399          * length and round it up to become the offset to the start
1400          * of the buffered type data which we would like to be aligned
1401          * for easy access.
1402          */
1403         char *strp = (char *) addr;
1404         int offset = roundup(strlen(strp) + 1, sizeof(uintptr_t));
1405
1406         /*
1407          * The type string might have a format such as 'int [20]'.
1408          * Check if there is an array dimension present.
1409          */
1410         if ((p = strchr(strp, '[')) != NULL) {
1411                 /* Strip off the array dimension. */
1412                 *p++ = '\0';
1413
1414                 for (; *p != '\0' && *p != ']'; p++)
1415                         num = num * 10 + *p - '0';
1416         } else
1417                 /* No array dimension, so default. */
1418                 num = 1;
1419
1420         /* Lookup the CTF type from the type string. */
1421         if (dtrace_lookup_by_type(dtp,  DTRACE_OBJ_EVERY, strp, &dtt) < 0)
1422                 return (-1);
1423
1424         /* Offset the buffer address to the start of the data... */
1425         addr += offset;
1426
1427         ssz = ctf_type_size(dtt.dtt_ctfp, dtt.dtt_type);
1428
1429         if (typs != ssz) {
1430                 printf("Expected type size from buffer (%lu) to match type size looked up now (%ld)\n", (u_long) typs, (long) ssz);
1431                 return (-1);
1432         }
1433
1434         cbdata.dtp = dtp;
1435         cbdata.dtt = dtt;
1436         cbdata.name = "";
1437         cbdata.addr = addr;
1438         cbdata.addrend = addr + nbytes;
1439         cbdata.indent = 1;
1440         cbdata.f_type = 1;
1441         cbdata.type_width = 0;
1442         cbdata.name_width = 0;
1443         cbdata.fp = fp;
1444
1445         return (dt_print_type_data(&cbdata, dtt.dtt_type));
1446 }
1447
1448 static int
1449 dt_print_sym(dtrace_hdl_t *dtp, FILE *fp, const char *format, caddr_t addr)
1450 {
1451         /* LINTED - alignment */
1452         uint64_t pc = *((uint64_t *)addr);
1453         dtrace_syminfo_t dts;
1454         GElf_Sym sym;
1455         char c[PATH_MAX * 2];
1456
1457         if (format == NULL)
1458                 format = "  %-50s";
1459
1460         if (dtrace_lookup_by_addr(dtp, pc, &sym, &dts) == 0) {
1461                 (void) snprintf(c, sizeof (c), "%s`%s",
1462                     dts.dts_object, dts.dts_name);
1463         } else {
1464                 /*
1465                  * We'll repeat the lookup, but this time we'll specify a
1466                  * NULL GElf_Sym -- indicating that we're only interested in
1467                  * the containing module.
1468                  */
1469                 if (dtrace_lookup_by_addr(dtp, pc, NULL, &dts) == 0) {
1470                         (void) snprintf(c, sizeof (c), "%s`0x%llx",
1471                             dts.dts_object, (u_longlong_t)pc);
1472                 } else {
1473                         (void) snprintf(c, sizeof (c), "0x%llx",
1474                             (u_longlong_t)pc);
1475                 }
1476         }
1477
1478         if (dt_printf(dtp, fp, format, c) < 0)
1479                 return (-1);
1480
1481         return (0);
1482 }
1483
1484 int
1485 dt_print_mod(dtrace_hdl_t *dtp, FILE *fp, const char *format, caddr_t addr)
1486 {
1487         /* LINTED - alignment */
1488         uint64_t pc = *((uint64_t *)addr);
1489         dtrace_syminfo_t dts;
1490         char c[PATH_MAX * 2];
1491
1492         if (format == NULL)
1493                 format = "  %-50s";
1494
1495         if (dtrace_lookup_by_addr(dtp, pc, NULL, &dts) == 0) {
1496                 (void) snprintf(c, sizeof (c), "%s", dts.dts_object);
1497         } else {
1498                 (void) snprintf(c, sizeof (c), "0x%llx", (u_longlong_t)pc);
1499         }
1500
1501         if (dt_printf(dtp, fp, format, c) < 0)
1502                 return (-1);
1503
1504         return (0);
1505 }
1506
1507 typedef struct dt_normal {
1508         dtrace_aggvarid_t dtnd_id;
1509         uint64_t dtnd_normal;
1510 } dt_normal_t;
1511
1512 static int
1513 dt_normalize_agg(const dtrace_aggdata_t *aggdata, void *arg)
1514 {
1515         dt_normal_t *normal = arg;
1516         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1517         dtrace_aggvarid_t id = normal->dtnd_id;
1518
1519         if (agg->dtagd_nrecs == 0)
1520                 return (DTRACE_AGGWALK_NEXT);
1521
1522         if (agg->dtagd_varid != id)
1523                 return (DTRACE_AGGWALK_NEXT);
1524
1525         ((dtrace_aggdata_t *)aggdata)->dtada_normal = normal->dtnd_normal;
1526         return (DTRACE_AGGWALK_NORMALIZE);
1527 }
1528
1529 static int
1530 dt_normalize(dtrace_hdl_t *dtp, caddr_t base, dtrace_recdesc_t *rec)
1531 {
1532         dt_normal_t normal;
1533         caddr_t addr;
1534
1535         /*
1536          * We (should) have two records:  the aggregation ID followed by the
1537          * normalization value.
1538          */
1539         addr = base + rec->dtrd_offset;
1540
1541         if (rec->dtrd_size != sizeof (dtrace_aggvarid_t))
1542                 return (dt_set_errno(dtp, EDT_BADNORMAL));
1543
1544         /* LINTED - alignment */
1545         normal.dtnd_id = *((dtrace_aggvarid_t *)addr);
1546         rec++;
1547
1548         if (rec->dtrd_action != DTRACEACT_LIBACT)
1549                 return (dt_set_errno(dtp, EDT_BADNORMAL));
1550
1551         if (rec->dtrd_arg != DT_ACT_NORMALIZE)
1552                 return (dt_set_errno(dtp, EDT_BADNORMAL));
1553
1554         addr = base + rec->dtrd_offset;
1555
1556         switch (rec->dtrd_size) {
1557         case sizeof (uint64_t):
1558                 /* LINTED - alignment */
1559                 normal.dtnd_normal = *((uint64_t *)addr);
1560                 break;
1561         case sizeof (uint32_t):
1562                 /* LINTED - alignment */
1563                 normal.dtnd_normal = *((uint32_t *)addr);
1564                 break;
1565         case sizeof (uint16_t):
1566                 /* LINTED - alignment */
1567                 normal.dtnd_normal = *((uint16_t *)addr);
1568                 break;
1569         case sizeof (uint8_t):
1570                 normal.dtnd_normal = *((uint8_t *)addr);
1571                 break;
1572         default:
1573                 return (dt_set_errno(dtp, EDT_BADNORMAL));
1574         }
1575
1576         (void) dtrace_aggregate_walk(dtp, dt_normalize_agg, &normal);
1577
1578         return (0);
1579 }
1580
1581 static int
1582 dt_denormalize_agg(const dtrace_aggdata_t *aggdata, void *arg)
1583 {
1584         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1585         dtrace_aggvarid_t id = *((dtrace_aggvarid_t *)arg);
1586
1587         if (agg->dtagd_nrecs == 0)
1588                 return (DTRACE_AGGWALK_NEXT);
1589
1590         if (agg->dtagd_varid != id)
1591                 return (DTRACE_AGGWALK_NEXT);
1592
1593         return (DTRACE_AGGWALK_DENORMALIZE);
1594 }
1595
1596 static int
1597 dt_clear_agg(const dtrace_aggdata_t *aggdata, void *arg)
1598 {
1599         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1600         dtrace_aggvarid_t id = *((dtrace_aggvarid_t *)arg);
1601
1602         if (agg->dtagd_nrecs == 0)
1603                 return (DTRACE_AGGWALK_NEXT);
1604
1605         if (agg->dtagd_varid != id)
1606                 return (DTRACE_AGGWALK_NEXT);
1607
1608         return (DTRACE_AGGWALK_CLEAR);
1609 }
1610
1611 typedef struct dt_trunc {
1612         dtrace_aggvarid_t dttd_id;
1613         uint64_t dttd_remaining;
1614 } dt_trunc_t;
1615
1616 static int
1617 dt_trunc_agg(const dtrace_aggdata_t *aggdata, void *arg)
1618 {
1619         dt_trunc_t *trunc = arg;
1620         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1621         dtrace_aggvarid_t id = trunc->dttd_id;
1622
1623         if (agg->dtagd_nrecs == 0)
1624                 return (DTRACE_AGGWALK_NEXT);
1625
1626         if (agg->dtagd_varid != id)
1627                 return (DTRACE_AGGWALK_NEXT);
1628
1629         if (trunc->dttd_remaining == 0)
1630                 return (DTRACE_AGGWALK_REMOVE);
1631
1632         trunc->dttd_remaining--;
1633         return (DTRACE_AGGWALK_NEXT);
1634 }
1635
1636 static int
1637 dt_trunc(dtrace_hdl_t *dtp, caddr_t base, dtrace_recdesc_t *rec)
1638 {
1639         dt_trunc_t trunc;
1640         caddr_t addr;
1641         int64_t remaining;
1642         int (*func)(dtrace_hdl_t *, dtrace_aggregate_f *, void *);
1643
1644         /*
1645          * We (should) have two records:  the aggregation ID followed by the
1646          * number of aggregation entries after which the aggregation is to be
1647          * truncated.
1648          */
1649         addr = base + rec->dtrd_offset;
1650
1651         if (rec->dtrd_size != sizeof (dtrace_aggvarid_t))
1652                 return (dt_set_errno(dtp, EDT_BADTRUNC));
1653
1654         /* LINTED - alignment */
1655         trunc.dttd_id = *((dtrace_aggvarid_t *)addr);
1656         rec++;
1657
1658         if (rec->dtrd_action != DTRACEACT_LIBACT)
1659                 return (dt_set_errno(dtp, EDT_BADTRUNC));
1660
1661         if (rec->dtrd_arg != DT_ACT_TRUNC)
1662                 return (dt_set_errno(dtp, EDT_BADTRUNC));
1663
1664         addr = base + rec->dtrd_offset;
1665
1666         switch (rec->dtrd_size) {
1667         case sizeof (uint64_t):
1668                 /* LINTED - alignment */
1669                 remaining = *((int64_t *)addr);
1670                 break;
1671         case sizeof (uint32_t):
1672                 /* LINTED - alignment */
1673                 remaining = *((int32_t *)addr);
1674                 break;
1675         case sizeof (uint16_t):
1676                 /* LINTED - alignment */
1677                 remaining = *((int16_t *)addr);
1678                 break;
1679         case sizeof (uint8_t):
1680                 remaining = *((int8_t *)addr);
1681                 break;
1682         default:
1683                 return (dt_set_errno(dtp, EDT_BADNORMAL));
1684         }
1685
1686         if (remaining < 0) {
1687                 func = dtrace_aggregate_walk_valsorted;
1688                 remaining = -remaining;
1689         } else {
1690                 func = dtrace_aggregate_walk_valrevsorted;
1691         }
1692
1693         assert(remaining >= 0);
1694         trunc.dttd_remaining = remaining;
1695
1696         (void) func(dtp, dt_trunc_agg, &trunc);
1697
1698         return (0);
1699 }
1700
1701 static int
1702 dt_print_datum(dtrace_hdl_t *dtp, FILE *fp, dtrace_recdesc_t *rec,
1703     caddr_t addr, size_t size, uint64_t normal)
1704 {
1705         int err;
1706         dtrace_actkind_t act = rec->dtrd_action;
1707
1708         switch (act) {
1709         case DTRACEACT_STACK:
1710                 return (dt_print_stack(dtp, fp, NULL, addr,
1711                     rec->dtrd_arg, rec->dtrd_size / rec->dtrd_arg));
1712
1713         case DTRACEACT_USTACK:
1714         case DTRACEACT_JSTACK:
1715                 return (dt_print_ustack(dtp, fp, NULL, addr, rec->dtrd_arg));
1716
1717         case DTRACEACT_USYM:
1718         case DTRACEACT_UADDR:
1719                 return (dt_print_usym(dtp, fp, addr, act));
1720
1721         case DTRACEACT_UMOD:
1722                 return (dt_print_umod(dtp, fp, NULL, addr));
1723
1724         case DTRACEACT_SYM:
1725                 return (dt_print_sym(dtp, fp, NULL, addr));
1726
1727         case DTRACEACT_MOD:
1728                 return (dt_print_mod(dtp, fp, NULL, addr));
1729
1730         case DTRACEAGG_QUANTIZE:
1731                 return (dt_print_quantize(dtp, fp, addr, size, normal));
1732
1733         case DTRACEAGG_LQUANTIZE:
1734                 return (dt_print_lquantize(dtp, fp, addr, size, normal));
1735
1736         case DTRACEAGG_AVG:
1737                 return (dt_print_average(dtp, fp, addr, size, normal));
1738
1739         case DTRACEAGG_STDDEV:
1740                 return (dt_print_stddev(dtp, fp, addr, size, normal));
1741
1742         default:
1743                 break;
1744         }
1745
1746         switch (size) {
1747         case sizeof (uint64_t):
1748                 err = dt_printf(dtp, fp, " %16lld",
1749                     /* LINTED - alignment */
1750                     (long long)*((uint64_t *)addr) / normal);
1751                 break;
1752         case sizeof (uint32_t):
1753                 /* LINTED - alignment */
1754                 err = dt_printf(dtp, fp, " %8d", *((uint32_t *)addr) /
1755                     (uint32_t)normal);
1756                 break;
1757         case sizeof (uint16_t):
1758                 /* LINTED - alignment */
1759                 err = dt_printf(dtp, fp, " %5d", *((uint16_t *)addr) /
1760                     (uint32_t)normal);
1761                 break;
1762         case sizeof (uint8_t):
1763                 err = dt_printf(dtp, fp, " %3d", *((uint8_t *)addr) /
1764                     (uint32_t)normal);
1765                 break;
1766         default:
1767                 err = dt_print_bytes(dtp, fp, addr, size, 50, 0, 0);
1768                 break;
1769         }
1770
1771         return (err);
1772 }
1773
1774 int
1775 dt_print_aggs(const dtrace_aggdata_t **aggsdata, int naggvars, void *arg)
1776 {
1777         int i, aggact = 0;
1778         dt_print_aggdata_t *pd = arg;
1779         const dtrace_aggdata_t *aggdata = aggsdata[0];
1780         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1781         FILE *fp = pd->dtpa_fp;
1782         dtrace_hdl_t *dtp = pd->dtpa_dtp;
1783         dtrace_recdesc_t *rec;
1784         dtrace_actkind_t act;
1785         caddr_t addr;
1786         size_t size;
1787
1788         /*
1789          * Iterate over each record description in the key, printing the traced
1790          * data, skipping the first datum (the tuple member created by the
1791          * compiler).
1792          */
1793         for (i = 1; i < agg->dtagd_nrecs; i++) {
1794                 rec = &agg->dtagd_rec[i];
1795                 act = rec->dtrd_action;
1796                 addr = aggdata->dtada_data + rec->dtrd_offset;
1797                 size = rec->dtrd_size;
1798
1799                 if (DTRACEACT_ISAGG(act)) {
1800                         aggact = i;
1801                         break;
1802                 }
1803
1804                 if (dt_print_datum(dtp, fp, rec, addr, size, 1) < 0)
1805                         return (-1);
1806
1807                 if (dt_buffered_flush(dtp, NULL, rec, aggdata,
1808                     DTRACE_BUFDATA_AGGKEY) < 0)
1809                         return (-1);
1810         }
1811
1812         assert(aggact != 0);
1813
1814         for (i = (naggvars == 1 ? 0 : 1); i < naggvars; i++) {
1815                 uint64_t normal;
1816
1817                 aggdata = aggsdata[i];
1818                 agg = aggdata->dtada_desc;
1819                 rec = &agg->dtagd_rec[aggact];
1820                 act = rec->dtrd_action;
1821                 addr = aggdata->dtada_data + rec->dtrd_offset;
1822                 size = rec->dtrd_size;
1823
1824                 assert(DTRACEACT_ISAGG(act));
1825                 normal = aggdata->dtada_normal;
1826
1827                 if (dt_print_datum(dtp, fp, rec, addr, size, normal) < 0)
1828                         return (-1);
1829
1830                 if (dt_buffered_flush(dtp, NULL, rec, aggdata,
1831                     DTRACE_BUFDATA_AGGVAL) < 0)
1832                         return (-1);
1833
1834                 if (!pd->dtpa_allunprint)
1835                         agg->dtagd_flags |= DTRACE_AGD_PRINTED;
1836         }
1837
1838         if (dt_printf(dtp, fp, "\n") < 0)
1839                 return (-1);
1840
1841         if (dt_buffered_flush(dtp, NULL, NULL, aggdata,
1842             DTRACE_BUFDATA_AGGFORMAT | DTRACE_BUFDATA_AGGLAST) < 0)
1843                 return (-1);
1844
1845         return (0);
1846 }
1847
1848 int
1849 dt_print_agg(const dtrace_aggdata_t *aggdata, void *arg)
1850 {
1851         dt_print_aggdata_t *pd = arg;
1852         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1853         dtrace_aggvarid_t aggvarid = pd->dtpa_id;
1854
1855         if (pd->dtpa_allunprint) {
1856                 if (agg->dtagd_flags & DTRACE_AGD_PRINTED)
1857                         return (0);
1858         } else {
1859                 /*
1860                  * If we're not printing all unprinted aggregations, then the
1861                  * aggregation variable ID denotes a specific aggregation
1862                  * variable that we should print -- skip any other aggregations
1863                  * that we encounter.
1864                  */
1865                 if (agg->dtagd_nrecs == 0)
1866                         return (0);
1867
1868                 if (aggvarid != agg->dtagd_varid)
1869                         return (0);
1870         }
1871
1872         return (dt_print_aggs(&aggdata, 1, arg));
1873 }
1874
1875 int
1876 dt_setopt(dtrace_hdl_t *dtp, const dtrace_probedata_t *data,
1877     const char *option, const char *value)
1878 {
1879         int len, rval;
1880         char *msg;
1881         const char *errstr;
1882         dtrace_setoptdata_t optdata;
1883
1884         bzero(&optdata, sizeof (optdata));
1885         (void) dtrace_getopt(dtp, option, &optdata.dtsda_oldval);
1886
1887         if (dtrace_setopt(dtp, option, value) == 0) {
1888                 (void) dtrace_getopt(dtp, option, &optdata.dtsda_newval);
1889                 optdata.dtsda_probe = data;
1890                 optdata.dtsda_option = option;
1891                 optdata.dtsda_handle = dtp;
1892
1893                 if ((rval = dt_handle_setopt(dtp, &optdata)) != 0)
1894                         return (rval);
1895
1896                 return (0);
1897         }
1898
1899         errstr = dtrace_errmsg(dtp, dtrace_errno(dtp));
1900         len = strlen(option) + strlen(value) + strlen(errstr) + 80;
1901         msg = alloca(len);
1902
1903         (void) snprintf(msg, len, "couldn't set option \"%s\" to \"%s\": %s\n",
1904             option, value, errstr);
1905
1906         if ((rval = dt_handle_liberr(dtp, data, msg)) == 0)
1907                 return (0);
1908
1909         return (rval);
1910 }
1911
1912 static int
1913 dt_consume_cpu(dtrace_hdl_t *dtp, FILE *fp, int cpu, dtrace_bufdesc_t *buf,
1914     dtrace_consume_probe_f *efunc, dtrace_consume_rec_f *rfunc, void *arg)
1915 {
1916         dtrace_epid_t id;
1917         size_t offs, start = buf->dtbd_oldest, end = buf->dtbd_size;
1918         int flow = (dtp->dt_options[DTRACEOPT_FLOWINDENT] != DTRACEOPT_UNSET);
1919         int quiet = (dtp->dt_options[DTRACEOPT_QUIET] != DTRACEOPT_UNSET);
1920         int rval, i, n;
1921         dtrace_epid_t last = DTRACE_EPIDNONE;
1922         dtrace_probedata_t data;
1923         uint64_t drops;
1924         caddr_t addr;
1925
1926         bzero(&data, sizeof (data));
1927         data.dtpda_handle = dtp;
1928         data.dtpda_cpu = cpu;
1929
1930 again:
1931         for (offs = start; offs < end; ) {
1932                 dtrace_eprobedesc_t *epd;
1933
1934                 /*
1935                  * We're guaranteed to have an ID.
1936                  */
1937                 id = *(uint32_t *)((uintptr_t)buf->dtbd_data + offs);
1938
1939                 if (id == DTRACE_EPIDNONE) {
1940                         /*
1941                          * This is filler to assure proper alignment of the
1942                          * next record; we simply ignore it.
1943                          */
1944                         offs += sizeof (id);
1945                         continue;
1946                 }
1947
1948                 if ((rval = dt_epid_lookup(dtp, id, &data.dtpda_edesc,
1949                     &data.dtpda_pdesc)) != 0)
1950                         return (rval);
1951
1952                 epd = data.dtpda_edesc;
1953                 data.dtpda_data = buf->dtbd_data + offs;
1954
1955                 if (data.dtpda_edesc->dtepd_uarg != DT_ECB_DEFAULT) {
1956                         rval = dt_handle(dtp, &data);
1957
1958                         if (rval == DTRACE_CONSUME_NEXT)
1959                                 goto nextepid;
1960
1961                         if (rval == DTRACE_CONSUME_ERROR)
1962                                 return (-1);
1963                 }
1964
1965                 if (flow)
1966                         (void) dt_flowindent(dtp, &data, last, buf, offs);
1967
1968                 rval = (*efunc)(&data, arg);
1969
1970                 if (flow) {
1971                         if (data.dtpda_flow == DTRACEFLOW_ENTRY)
1972                                 data.dtpda_indent += 2;
1973                 }
1974
1975                 if (rval == DTRACE_CONSUME_NEXT)
1976                         goto nextepid;
1977
1978                 if (rval == DTRACE_CONSUME_ABORT)
1979                         return (dt_set_errno(dtp, EDT_DIRABORT));
1980
1981                 if (rval != DTRACE_CONSUME_THIS)
1982                         return (dt_set_errno(dtp, EDT_BADRVAL));
1983
1984                 for (i = 0; i < epd->dtepd_nrecs; i++) {
1985                         dtrace_recdesc_t *rec = &epd->dtepd_rec[i];
1986                         dtrace_actkind_t act = rec->dtrd_action;
1987
1988                         data.dtpda_data = buf->dtbd_data + offs +
1989                             rec->dtrd_offset;
1990                         addr = data.dtpda_data;
1991
1992                         if (act == DTRACEACT_LIBACT) {
1993                                 uint64_t arg = rec->dtrd_arg;
1994                                 dtrace_aggvarid_t id;
1995
1996                                 switch (arg) {
1997                                 case DT_ACT_CLEAR:
1998                                         /* LINTED - alignment */
1999                                         id = *((dtrace_aggvarid_t *)addr);
2000                                         (void) dtrace_aggregate_walk(dtp,
2001                                             dt_clear_agg, &id);
2002                                         continue;
2003
2004                                 case DT_ACT_DENORMALIZE:
2005                                         /* LINTED - alignment */
2006                                         id = *((dtrace_aggvarid_t *)addr);
2007                                         (void) dtrace_aggregate_walk(dtp,
2008                                             dt_denormalize_agg, &id);
2009                                         continue;
2010
2011                                 case DT_ACT_FTRUNCATE:
2012                                         if (fp == NULL)
2013                                                 continue;
2014
2015                                         (void) fflush(fp);
2016                                         (void) ftruncate(fileno(fp), 0);
2017                                         (void) fseeko(fp, 0, SEEK_SET);
2018                                         continue;
2019
2020                                 case DT_ACT_NORMALIZE:
2021                                         if (i == epd->dtepd_nrecs - 1)
2022                                                 return (dt_set_errno(dtp,
2023                                                     EDT_BADNORMAL));
2024
2025                                         if (dt_normalize(dtp,
2026                                             buf->dtbd_data + offs, rec) != 0)
2027                                                 return (-1);
2028
2029                                         i++;
2030                                         continue;
2031
2032                                 case DT_ACT_SETOPT: {
2033                                         uint64_t *opts = dtp->dt_options;
2034                                         dtrace_recdesc_t *valrec;
2035                                         uint32_t valsize;
2036                                         caddr_t val;
2037                                         int rv;
2038
2039                                         if (i == epd->dtepd_nrecs - 1) {
2040                                                 return (dt_set_errno(dtp,
2041                                                     EDT_BADSETOPT));
2042                                         }
2043
2044                                         valrec = &epd->dtepd_rec[++i];
2045                                         valsize = valrec->dtrd_size;
2046
2047                                         if (valrec->dtrd_action != act ||
2048                                             valrec->dtrd_arg != arg) {
2049                                                 return (dt_set_errno(dtp,
2050                                                     EDT_BADSETOPT));
2051                                         }
2052
2053                                         if (valsize > sizeof (uint64_t)) {
2054                                                 val = buf->dtbd_data + offs +
2055                                                     valrec->dtrd_offset;
2056                                         } else {
2057                                                 val = "1";
2058                                         }
2059
2060                                         rv = dt_setopt(dtp, &data, addr, val);
2061
2062                                         if (rv != 0)
2063                                                 return (-1);
2064
2065                                         flow = (opts[DTRACEOPT_FLOWINDENT] !=
2066                                             DTRACEOPT_UNSET);
2067                                         quiet = (opts[DTRACEOPT_QUIET] !=
2068                                             DTRACEOPT_UNSET);
2069
2070                                         continue;
2071                                 }
2072
2073                                 case DT_ACT_TRUNC:
2074                                         if (i == epd->dtepd_nrecs - 1)
2075                                                 return (dt_set_errno(dtp,
2076                                                     EDT_BADTRUNC));
2077
2078                                         if (dt_trunc(dtp,
2079                                             buf->dtbd_data + offs, rec) != 0)
2080                                                 return (-1);
2081
2082                                         i++;
2083                                         continue;
2084
2085                                 default:
2086                                         continue;
2087                                 }
2088                         }
2089
2090                         rval = (*rfunc)(&data, rec, arg);
2091
2092                         if (rval == DTRACE_CONSUME_NEXT)
2093                                 continue;
2094
2095                         if (rval == DTRACE_CONSUME_ABORT)
2096                                 return (dt_set_errno(dtp, EDT_DIRABORT));
2097
2098                         if (rval != DTRACE_CONSUME_THIS)
2099                                 return (dt_set_errno(dtp, EDT_BADRVAL));
2100
2101                         if (act == DTRACEACT_STACK) {
2102                                 int depth = rec->dtrd_arg;
2103
2104                                 if (dt_print_stack(dtp, fp, NULL, addr, depth,
2105                                     rec->dtrd_size / depth) < 0)
2106                                         return (-1);
2107                                 goto nextrec;
2108                         }
2109
2110                         if (act == DTRACEACT_USTACK ||
2111                             act == DTRACEACT_JSTACK) {
2112                                 if (dt_print_ustack(dtp, fp, NULL,
2113                                     addr, rec->dtrd_arg) < 0)
2114                                         return (-1);
2115                                 goto nextrec;
2116                         }
2117
2118                         if (act == DTRACEACT_SYM) {
2119                                 if (dt_print_sym(dtp, fp, NULL, addr) < 0)
2120                                         return (-1);
2121                                 goto nextrec;
2122                         }
2123
2124                         if (act == DTRACEACT_MOD) {
2125                                 if (dt_print_mod(dtp, fp, NULL, addr) < 0)
2126                                         return (-1);
2127                                 goto nextrec;
2128                         }
2129
2130                         if (act == DTRACEACT_USYM || act == DTRACEACT_UADDR) {
2131                                 if (dt_print_usym(dtp, fp, addr, act) < 0)
2132                                         return (-1);
2133                                 goto nextrec;
2134                         }
2135
2136                         if (act == DTRACEACT_UMOD) {
2137                                 if (dt_print_umod(dtp, fp, NULL, addr) < 0)
2138                                         return (-1);
2139                                 goto nextrec;
2140                         }
2141
2142                         if (act == DTRACEACT_PRINTM) {
2143                                 if (dt_print_memory(dtp, fp, addr) < 0)
2144                                         return (-1);
2145                                 goto nextrec;
2146                         }
2147
2148                         if (act == DTRACEACT_PRINTT) {
2149                                 if (dt_print_type(dtp, fp, addr) < 0)
2150                                         return (-1);
2151                                 goto nextrec;
2152                         }
2153
2154                         if (DTRACEACT_ISPRINTFLIKE(act)) {
2155                                 void *fmtdata;
2156                                 int (*func)(dtrace_hdl_t *, FILE *, void *,
2157                                     const dtrace_probedata_t *,
2158                                     const dtrace_recdesc_t *, uint_t,
2159                                     const void *buf, size_t);
2160
2161                                 if ((fmtdata = dt_format_lookup(dtp,
2162                                     rec->dtrd_format)) == NULL)
2163                                         goto nofmt;
2164
2165                                 switch (act) {
2166                                 case DTRACEACT_PRINTF:
2167                                         func = dtrace_fprintf;
2168                                         break;
2169                                 case DTRACEACT_PRINTA:
2170                                         func = dtrace_fprinta;
2171                                         break;
2172                                 case DTRACEACT_SYSTEM:
2173                                         func = dtrace_system;
2174                                         break;
2175                                 case DTRACEACT_FREOPEN:
2176                                         func = dtrace_freopen;
2177                                         break;
2178                                 }
2179
2180                                 n = (*func)(dtp, fp, fmtdata, &data,
2181                                     rec, epd->dtepd_nrecs - i,
2182                                     (uchar_t *)buf->dtbd_data + offs,
2183                                     buf->dtbd_size - offs);
2184
2185                                 if (n < 0)
2186                                         return (-1); /* errno is set for us */
2187
2188                                 if (n > 0)
2189                                         i += n - 1;
2190                                 goto nextrec;
2191                         }
2192
2193 nofmt:
2194                         if (act == DTRACEACT_PRINTA) {
2195                                 dt_print_aggdata_t pd;
2196                                 dtrace_aggvarid_t *aggvars;
2197                                 int j, naggvars = 0;
2198                                 size_t size = ((epd->dtepd_nrecs - i) *
2199                                     sizeof (dtrace_aggvarid_t));
2200
2201                                 if ((aggvars = dt_alloc(dtp, size)) == NULL)
2202                                         return (-1);
2203
2204                                 /*
2205                                  * This might be a printa() with multiple
2206                                  * aggregation variables.  We need to scan
2207                                  * forward through the records until we find
2208                                  * a record from a different statement.
2209                                  */
2210                                 for (j = i; j < epd->dtepd_nrecs; j++) {
2211                                         dtrace_recdesc_t *nrec;
2212                                         caddr_t naddr;
2213
2214                                         nrec = &epd->dtepd_rec[j];
2215
2216                                         if (nrec->dtrd_uarg != rec->dtrd_uarg)
2217                                                 break;
2218
2219                                         if (nrec->dtrd_action != act) {
2220                                                 return (dt_set_errno(dtp,
2221                                                     EDT_BADAGG));
2222                                         }
2223
2224                                         naddr = buf->dtbd_data + offs +
2225                                             nrec->dtrd_offset;
2226
2227                                         aggvars[naggvars++] =
2228                                             /* LINTED - alignment */
2229                                             *((dtrace_aggvarid_t *)naddr);
2230                                 }
2231
2232                                 i = j - 1;
2233                                 bzero(&pd, sizeof (pd));
2234                                 pd.dtpa_dtp = dtp;
2235                                 pd.dtpa_fp = fp;
2236
2237                                 assert(naggvars >= 1);
2238
2239                                 if (naggvars == 1) {
2240                                         pd.dtpa_id = aggvars[0];
2241                                         dt_free(dtp, aggvars);
2242
2243                                         if (dt_printf(dtp, fp, "\n") < 0 ||
2244                                             dtrace_aggregate_walk_sorted(dtp,
2245                                             dt_print_agg, &pd) < 0)
2246                                                 return (-1);
2247                                         goto nextrec;
2248                                 }
2249
2250                                 if (dt_printf(dtp, fp, "\n") < 0 ||
2251                                     dtrace_aggregate_walk_joined(dtp, aggvars,
2252                                     naggvars, dt_print_aggs, &pd) < 0) {
2253                                         dt_free(dtp, aggvars);
2254                                         return (-1);
2255                                 }
2256
2257                                 dt_free(dtp, aggvars);
2258                                 goto nextrec;
2259                         }
2260
2261                         switch (rec->dtrd_size) {
2262                         case sizeof (uint64_t):
2263                                 n = dt_printf(dtp, fp,
2264                                     quiet ? "%lld" : " %16lld",
2265                                     /* LINTED - alignment */
2266                                     *((unsigned long long *)addr));
2267                                 break;
2268                         case sizeof (uint32_t):
2269                                 n = dt_printf(dtp, fp, quiet ? "%d" : " %8d",
2270                                     /* LINTED - alignment */
2271                                     *((uint32_t *)addr));
2272                                 break;
2273                         case sizeof (uint16_t):
2274                                 n = dt_printf(dtp, fp, quiet ? "%d" : " %5d",
2275                                     /* LINTED - alignment */
2276                                     *((uint16_t *)addr));
2277                                 break;
2278                         case sizeof (uint8_t):
2279                                 n = dt_printf(dtp, fp, quiet ? "%d" : " %3d",
2280                                     *((uint8_t *)addr));
2281                                 break;
2282                         default:
2283                                 n = dt_print_bytes(dtp, fp, addr,
2284                                     rec->dtrd_size, 33, quiet, 0);
2285                                 break;
2286                         }
2287
2288                         if (n < 0)
2289                                 return (-1); /* errno is set for us */
2290
2291 nextrec:
2292                         if (dt_buffered_flush(dtp, &data, rec, NULL, 0) < 0)
2293                                 return (-1); /* errno is set for us */
2294                 }
2295
2296                 /*
2297                  * Call the record callback with a NULL record to indicate
2298                  * that we're done processing this EPID.
2299                  */
2300                 rval = (*rfunc)(&data, NULL, arg);
2301 nextepid:
2302                 offs += epd->dtepd_size;
2303                 last = id;
2304         }
2305
2306         if (buf->dtbd_oldest != 0 && start == buf->dtbd_oldest) {
2307                 end = buf->dtbd_oldest;
2308                 start = 0;
2309                 goto again;
2310         }
2311
2312         if ((drops = buf->dtbd_drops) == 0)
2313                 return (0);
2314
2315         /*
2316          * Explicitly zero the drops to prevent us from processing them again.
2317          */
2318         buf->dtbd_drops = 0;
2319
2320         return (dt_handle_cpudrop(dtp, cpu, DTRACEDROP_PRINCIPAL, drops));
2321 }
2322
2323 typedef struct dt_begin {
2324         dtrace_consume_probe_f *dtbgn_probefunc;
2325         dtrace_consume_rec_f *dtbgn_recfunc;
2326         void *dtbgn_arg;
2327         dtrace_handle_err_f *dtbgn_errhdlr;
2328         void *dtbgn_errarg;
2329         int dtbgn_beginonly;
2330 } dt_begin_t;
2331
2332 static int
2333 dt_consume_begin_probe(const dtrace_probedata_t *data, void *arg)
2334 {
2335         dt_begin_t *begin = (dt_begin_t *)arg;
2336         dtrace_probedesc_t *pd = data->dtpda_pdesc;
2337
2338         int r1 = (strcmp(pd->dtpd_provider, "dtrace") == 0);
2339         int r2 = (strcmp(pd->dtpd_name, "BEGIN") == 0);
2340
2341         if (begin->dtbgn_beginonly) {
2342                 if (!(r1 && r2))
2343                         return (DTRACE_CONSUME_NEXT);
2344         } else {
2345                 if (r1 && r2)
2346                         return (DTRACE_CONSUME_NEXT);
2347         }
2348
2349         /*
2350          * We have a record that we're interested in.  Now call the underlying
2351          * probe function...
2352          */
2353         return (begin->dtbgn_probefunc(data, begin->dtbgn_arg));
2354 }
2355
2356 static int
2357 dt_consume_begin_record(const dtrace_probedata_t *data,
2358     const dtrace_recdesc_t *rec, void *arg)
2359 {
2360         dt_begin_t *begin = (dt_begin_t *)arg;
2361
2362         return (begin->dtbgn_recfunc(data, rec, begin->dtbgn_arg));
2363 }
2364
2365 static int
2366 dt_consume_begin_error(const dtrace_errdata_t *data, void *arg)
2367 {
2368         dt_begin_t *begin = (dt_begin_t *)arg;
2369         dtrace_probedesc_t *pd = data->dteda_pdesc;
2370
2371         int r1 = (strcmp(pd->dtpd_provider, "dtrace") == 0);
2372         int r2 = (strcmp(pd->dtpd_name, "BEGIN") == 0);
2373
2374         if (begin->dtbgn_beginonly) {
2375                 if (!(r1 && r2))
2376                         return (DTRACE_HANDLE_OK);
2377         } else {
2378                 if (r1 && r2)
2379                         return (DTRACE_HANDLE_OK);
2380         }
2381
2382         return (begin->dtbgn_errhdlr(data, begin->dtbgn_errarg));
2383 }
2384
2385 static int
2386 dt_consume_begin(dtrace_hdl_t *dtp, FILE *fp, dtrace_bufdesc_t *buf,
2387     dtrace_consume_probe_f *pf, dtrace_consume_rec_f *rf, void *arg)
2388 {
2389         /*
2390          * There's this idea that the BEGIN probe should be processed before
2391          * everything else, and that the END probe should be processed after
2392          * anything else.  In the common case, this is pretty easy to deal
2393          * with.  However, a situation may arise where the BEGIN enabling and
2394          * END enabling are on the same CPU, and some enabling in the middle
2395          * occurred on a different CPU.  To deal with this (blech!) we need to
2396          * consume the BEGIN buffer up until the end of the BEGIN probe, and
2397          * then set it aside.  We will then process every other CPU, and then
2398          * we'll return to the BEGIN CPU and process the rest of the data
2399          * (which will inevitably include the END probe, if any).  Making this
2400          * even more complicated (!) is the library's ERROR enabling.  Because
2401          * this enabling is processed before we even get into the consume call
2402          * back, any ERROR firing would result in the library's ERROR enabling
2403          * being processed twice -- once in our first pass (for BEGIN probes),
2404          * and again in our second pass (for everything but BEGIN probes).  To
2405          * deal with this, we interpose on the ERROR handler to assure that we
2406          * only process ERROR enablings induced by BEGIN enablings in the
2407          * first pass, and that we only process ERROR enablings _not_ induced
2408          * by BEGIN enablings in the second pass.
2409          */
2410         dt_begin_t begin;
2411         processorid_t cpu = dtp->dt_beganon;
2412         dtrace_bufdesc_t nbuf;
2413 #if !defined(sun)
2414         dtrace_bufdesc_t *pbuf;
2415 #endif
2416         int rval, i;
2417         static int max_ncpus;
2418         dtrace_optval_t size;
2419
2420         dtp->dt_beganon = -1;
2421
2422 #if defined(sun)
2423         if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, buf) == -1) {
2424 #else
2425         if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, &buf) == -1) {
2426 #endif
2427                 /*
2428                  * We really don't expect this to fail, but it is at least
2429                  * technically possible for this to fail with ENOENT.  In this
2430                  * case, we just drive on...
2431                  */
2432                 if (errno == ENOENT)
2433                         return (0);
2434
2435                 return (dt_set_errno(dtp, errno));
2436         }
2437
2438         if (!dtp->dt_stopped || buf->dtbd_cpu != dtp->dt_endedon) {
2439                 /*
2440                  * This is the simple case.  We're either not stopped, or if
2441                  * we are, we actually processed any END probes on another
2442                  * CPU.  We can simply consume this buffer and return.
2443                  */
2444                 return (dt_consume_cpu(dtp, fp, cpu, buf, pf, rf, arg));
2445         }
2446
2447         begin.dtbgn_probefunc = pf;
2448         begin.dtbgn_recfunc = rf;
2449         begin.dtbgn_arg = arg;
2450         begin.dtbgn_beginonly = 1;
2451
2452         /*
2453          * We need to interpose on the ERROR handler to be sure that we
2454          * only process ERRORs induced by BEGIN.
2455          */
2456         begin.dtbgn_errhdlr = dtp->dt_errhdlr;
2457         begin.dtbgn_errarg = dtp->dt_errarg;
2458         dtp->dt_errhdlr = dt_consume_begin_error;
2459         dtp->dt_errarg = &begin;
2460
2461         rval = dt_consume_cpu(dtp, fp, cpu, buf, dt_consume_begin_probe,
2462             dt_consume_begin_record, &begin);
2463
2464         dtp->dt_errhdlr = begin.dtbgn_errhdlr;
2465         dtp->dt_errarg = begin.dtbgn_errarg;
2466
2467         if (rval != 0)
2468                 return (rval);
2469
2470         /*
2471          * Now allocate a new buffer.  We'll use this to deal with every other
2472          * CPU.
2473          */
2474         bzero(&nbuf, sizeof (dtrace_bufdesc_t));
2475         (void) dtrace_getopt(dtp, "bufsize", &size);
2476         if ((nbuf.dtbd_data = malloc(size)) == NULL)
2477                 return (dt_set_errno(dtp, EDT_NOMEM));
2478
2479         if (max_ncpus == 0)
2480                 max_ncpus = dt_sysconf(dtp, _SC_CPUID_MAX) + 1;
2481
2482         for (i = 0; i < max_ncpus; i++) {
2483                 nbuf.dtbd_cpu = i;
2484
2485                 if (i == cpu)
2486                         continue;
2487
2488 #if defined(sun)
2489                 if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, &nbuf) == -1) {
2490 #else
2491                 pbuf = &nbuf;
2492                 if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, &pbuf) == -1) {
2493 #endif
2494                         /*
2495                          * If we failed with ENOENT, it may be because the
2496                          * CPU was unconfigured -- this is okay.  Any other
2497                          * error, however, is unexpected.
2498                          */
2499                         if (errno == ENOENT)
2500                                 continue;
2501
2502                         free(nbuf.dtbd_data);
2503
2504                         return (dt_set_errno(dtp, errno));
2505                 }
2506
2507                 if ((rval = dt_consume_cpu(dtp, fp,
2508                     i, &nbuf, pf, rf, arg)) != 0) {
2509                         free(nbuf.dtbd_data);
2510                         return (rval);
2511                 }
2512         }
2513
2514         free(nbuf.dtbd_data);
2515
2516         /*
2517          * Okay -- we're done with the other buffers.  Now we want to
2518          * reconsume the first buffer -- but this time we're looking for
2519          * everything _but_ BEGIN.  And of course, in order to only consume
2520          * those ERRORs _not_ associated with BEGIN, we need to reinstall our
2521          * ERROR interposition function...
2522          */
2523         begin.dtbgn_beginonly = 0;
2524
2525         assert(begin.dtbgn_errhdlr == dtp->dt_errhdlr);
2526         assert(begin.dtbgn_errarg == dtp->dt_errarg);
2527         dtp->dt_errhdlr = dt_consume_begin_error;
2528         dtp->dt_errarg = &begin;
2529
2530         rval = dt_consume_cpu(dtp, fp, cpu, buf, dt_consume_begin_probe,
2531             dt_consume_begin_record, &begin);
2532
2533         dtp->dt_errhdlr = begin.dtbgn_errhdlr;
2534         dtp->dt_errarg = begin.dtbgn_errarg;
2535
2536         return (rval);
2537 }
2538
2539 int
2540 dtrace_consume(dtrace_hdl_t *dtp, FILE *fp,
2541     dtrace_consume_probe_f *pf, dtrace_consume_rec_f *rf, void *arg)
2542 {
2543         dtrace_bufdesc_t *buf = &dtp->dt_buf;
2544         dtrace_optval_t size;
2545         static int max_ncpus;
2546         int i, rval;
2547         dtrace_optval_t interval = dtp->dt_options[DTRACEOPT_SWITCHRATE];
2548         hrtime_t now = gethrtime();
2549
2550         if (dtp->dt_lastswitch != 0) {
2551                 if (now - dtp->dt_lastswitch < interval)
2552                         return (0);
2553
2554                 dtp->dt_lastswitch += interval;
2555         } else {
2556                 dtp->dt_lastswitch = now;
2557         }
2558
2559         if (!dtp->dt_active)
2560                 return (dt_set_errno(dtp, EINVAL));
2561
2562         if (max_ncpus == 0)
2563                 max_ncpus = dt_sysconf(dtp, _SC_CPUID_MAX) + 1;
2564
2565         if (pf == NULL)
2566                 pf = (dtrace_consume_probe_f *)dt_nullprobe;
2567
2568         if (rf == NULL)
2569                 rf = (dtrace_consume_rec_f *)dt_nullrec;
2570
2571         if (buf->dtbd_data == NULL) {
2572                 (void) dtrace_getopt(dtp, "bufsize", &size);
2573                 if ((buf->dtbd_data = malloc(size)) == NULL)
2574                         return (dt_set_errno(dtp, EDT_NOMEM));
2575
2576                 buf->dtbd_size = size;
2577         }
2578
2579         /*
2580          * If we have just begun, we want to first process the CPU that
2581          * executed the BEGIN probe (if any).
2582          */
2583         if (dtp->dt_active && dtp->dt_beganon != -1) {
2584                 buf->dtbd_cpu = dtp->dt_beganon;
2585                 if ((rval = dt_consume_begin(dtp, fp, buf, pf, rf, arg)) != 0)
2586                         return (rval);
2587         }
2588
2589         for (i = 0; i < max_ncpus; i++) {
2590                 buf->dtbd_cpu = i;
2591
2592                 /*
2593                  * If we have stopped, we want to process the CPU on which the
2594                  * END probe was processed only _after_ we have processed
2595                  * everything else.
2596                  */
2597                 if (dtp->dt_stopped && (i == dtp->dt_endedon))
2598                         continue;
2599
2600 #if defined(sun)
2601                 if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, buf) == -1) {
2602 #else
2603                 if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, &buf) == -1) {
2604 #endif
2605                         /*
2606                          * If we failed with ENOENT, it may be because the
2607                          * CPU was unconfigured -- this is okay.  Any other
2608                          * error, however, is unexpected.
2609                          */
2610                         if (errno == ENOENT)
2611                                 continue;
2612
2613                         return (dt_set_errno(dtp, errno));
2614                 }
2615
2616                 if ((rval = dt_consume_cpu(dtp, fp, i, buf, pf, rf, arg)) != 0)
2617                         return (rval);
2618         }
2619
2620         if (!dtp->dt_stopped)
2621                 return (0);
2622
2623         buf->dtbd_cpu = dtp->dt_endedon;
2624
2625 #if defined(sun)
2626         if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, buf) == -1) {
2627 #else
2628         if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, &buf) == -1) {
2629 #endif
2630                 /*
2631                  * This _really_ shouldn't fail, but it is strictly speaking
2632                  * possible for this to return ENOENT if the CPU that called
2633                  * the END enabling somehow managed to become unconfigured.
2634                  * It's unclear how the user can possibly expect anything
2635                  * rational to happen in this case -- the state has been thrown
2636                  * out along with the unconfigured CPU -- so we'll just drive
2637                  * on...
2638                  */
2639                 if (errno == ENOENT)
2640                         return (0);
2641
2642                 return (dt_set_errno(dtp, errno));
2643         }
2644
2645         return (dt_consume_cpu(dtp, fp, dtp->dt_endedon, buf, pf, rf, arg));
2646 }