cddl/contrib/opensolaris/lib/libdtrace/common/dt_consume.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25
  26 /*
  27  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  28  * Copyright (c) 2012 by Delphix. All rights reserved.
  29  */
  30
  31 #include <stdlib.h>
  32 #include <strings.h>
  33 #include <errno.h>
  34 #include <unistd.h>
  35 #include <limits.h>
  36 #include <assert.h>
  37 #include <ctype.h>
  38 #if defined(sun)
  39 #include <alloca.h>
  40 #endif
  41 #include <dt_impl.h>
  42 #include <dt_pq.h>
  43 #if !defined(sun)
  44 #include <libproc_compat.h>
  45 #endif
  46
  47 #define DT_MASK_LO 0x00000000FFFFFFFFULL
  48
  49 /*
  50  * We declare this here because (1) we need it and (2) we want to avoid a
  51  * dependency on libm in libdtrace.
  52  */
  53 static long double
  54 dt_fabsl(long double x)
  55 {
  56         if (x < 0)
  57                 return (-x);
  58
  59         return (x);
  60 }
  61
  62 static int
  63 dt_ndigits(long long val)
  64 {
  65         int rval = 1;
  66         long long cmp = 10;
  67
  68         if (val < 0) {
  69                 val = val == INT64_MIN ? INT64_MAX : -val;
  70                 rval++;
  71         }
  72
  73         while (val > cmp && cmp > 0) {
  74                 rval++;
  75                 cmp *= 10;
  76         }
  77
  78         return (rval < 4 ? 4 : rval);
  79 }
  80
  81 /*
  82  * 128-bit arithmetic functions needed to support the stddev() aggregating
  83  * action.
  84  */
  85 static int
  86 dt_gt_128(uint64_t *a, uint64_t *b)
  87 {
  88         return (a[1] > b[1] || (a[1] == b[1] && a[0] > b[0]));
  89 }
  90
  91 static int
  92 dt_ge_128(uint64_t *a, uint64_t *b)
  93 {
  94         return (a[1] > b[1] || (a[1] == b[1] && a[0] >= b[0]));
  95 }
  96
  97 static int
  98 dt_le_128(uint64_t *a, uint64_t *b)
  99 {
 100         return (a[1] < b[1] || (a[1] == b[1] && a[0] <= b[0]));
 101 }
 102
 103 /*
 104  * Shift the 128-bit value in a by b. If b is positive, shift left.
 105  * If b is negative, shift right.
 106  */
 107 static void
 108 dt_shift_128(uint64_t *a, int b)
 109 {
 110         uint64_t mask;
 111
 112         if (b == 0)
 113                 return;
 114
 115         if (b < 0) {
 116                 b = -b;
 117                 if (b >= 64) {
 118                         a[0] = a[1] >> (b - 64);
 119                         a[1] = 0;
 120                 } else {
 121                         a[0] >>= b;
 122                         mask = 1LL << (64 - b);
 123                         mask -= 1;
 124                         a[0] |= ((a[1] & mask) << (64 - b));
 125                         a[1] >>= b;
 126                 }
 127         } else {
 128                 if (b >= 64) {
 129                         a[1] = a[0] << (b - 64);
 130                         a[0] = 0;
 131                 } else {
 132                         a[1] <<= b;
 133                         mask = a[0] >> (64 - b);
 134                         a[1] |= mask;
 135                         a[0] <<= b;
 136                 }
 137         }
 138 }
 139
 140 static int
 141 dt_nbits_128(uint64_t *a)
 142 {
 143         int nbits = 0;
 144         uint64_t tmp[2];
 145         uint64_t zero[2] = { 0, 0 };
 146
 147         tmp[0] = a[0];
 148         tmp[1] = a[1];
 149
 150         dt_shift_128(tmp, -1);
 151         while (dt_gt_128(tmp, zero)) {
 152                 dt_shift_128(tmp, -1);
 153                 nbits++;
 154         }
 155
 156         return (nbits);
 157 }
 158
 159 static void
 160 dt_subtract_128(uint64_t *minuend, uint64_t *subtrahend, uint64_t *difference)
 161 {
 162         uint64_t result[2];
 163
 164         result[0] = minuend[0] - subtrahend[0];
 165         result[1] = minuend[1] - subtrahend[1] -
 166             (minuend[0] < subtrahend[0] ? 1 : 0);
 167
 168         difference[0] = result[0];
 169         difference[1] = result[1];
 170 }
 171
 172 static void
 173 dt_add_128(uint64_t *addend1, uint64_t *addend2, uint64_t *sum)
 174 {
 175         uint64_t result[2];
 176
 177         result[0] = addend1[0] + addend2[0];
 178         result[1] = addend1[1] + addend2[1] +
 179             (result[0] < addend1[0] || result[0] < addend2[0] ? 1 : 0);
 180
 181         sum[0] = result[0];
 182         sum[1] = result[1];
 183 }
 184
 185 /*
 186  * The basic idea is to break the 2 64-bit values into 4 32-bit values,
 187  * use native multiplication on those, and then re-combine into the
 188  * resulting 128-bit value.
 189  *
 190  * (hi1 << 32 + lo1) * (hi2 << 32 + lo2) =
 191  *     hi1 * hi2 << 64 +
 192  *     hi1 * lo2 << 32 +
 193  *     hi2 * lo1 << 32 +
 194  *     lo1 * lo2
 195  */
 196 static void
 197 dt_multiply_128(uint64_t factor1, uint64_t factor2, uint64_t *product)
 198 {
 199         uint64_t hi1, hi2, lo1, lo2;
 200         uint64_t tmp[2];
 201
 202         hi1 = factor1 >> 32;
 203         hi2 = factor2 >> 32;
 204
 205         lo1 = factor1 & DT_MASK_LO;
 206         lo2 = factor2 & DT_MASK_LO;
 207
 208         product[0] = lo1 * lo2;
 209         product[1] = hi1 * hi2;
 210
 211         tmp[0] = hi1 * lo2;
 212         tmp[1] = 0;
 213         dt_shift_128(tmp, 32);
 214         dt_add_128(product, tmp, product);
 215
 216         tmp[0] = hi2 * lo1;
 217         tmp[1] = 0;
 218         dt_shift_128(tmp, 32);
 219         dt_add_128(product, tmp, product);
 220 }
 221
 222 /*
 223  * This is long-hand division.
 224  *
 225  * We initialize subtrahend by shifting divisor left as far as possible. We
 226  * loop, comparing subtrahend to dividend:  if subtrahend is smaller, we
 227  * subtract and set the appropriate bit in the result.  We then shift
 228  * subtrahend right by one bit for the next comparison.
 229  */
 230 static void
 231 dt_divide_128(uint64_t *dividend, uint64_t divisor, uint64_t *quotient)
 232 {
 233         uint64_t result[2] = { 0, 0 };
 234         uint64_t remainder[2];
 235         uint64_t subtrahend[2];
 236         uint64_t divisor_128[2];
 237         uint64_t mask[2] = { 1, 0 };
 238         int log = 0;
 239
 240         assert(divisor != 0);
 241
 242         divisor_128[0] = divisor;
 243         divisor_128[1] = 0;
 244
 245         remainder[0] = dividend[0];
 246         remainder[1] = dividend[1];
 247
 248         subtrahend[0] = divisor;
 249         subtrahend[1] = 0;
 250
 251         while (divisor > 0) {
 252                 log++;
 253                 divisor >>= 1;
 254         }
 255
 256         dt_shift_128(subtrahend, 128 - log);
 257         dt_shift_128(mask, 128 - log);
 258
 259         while (dt_ge_128(remainder, divisor_128)) {
 260                 if (dt_ge_128(remainder, subtrahend)) {
 261                         dt_subtract_128(remainder, subtrahend, remainder);
 262                         result[0] |= mask[0];
 263                         result[1] |= mask[1];
 264                 }
 265
 266                 dt_shift_128(subtrahend, -1);
 267                 dt_shift_128(mask, -1);
 268         }
 269
 270         quotient[0] = result[0];
 271         quotient[1] = result[1];
 272 }
 273
 274 /*
 275  * This is the long-hand method of calculating a square root.
 276  * The algorithm is as follows:
 277  *
 278  * 1. Group the digits by 2 from the right.
 279  * 2. Over the leftmost group, find the largest single-digit number
 280  *    whose square is less than that group.
 281  * 3. Subtract the result of the previous step (2 or 4, depending) and
 282  *    bring down the next two-digit group.
 283  * 4. For the result R we have so far, find the largest single-digit number
 284  *    x such that 2 * R * 10 * x + x^2 is less than the result from step 3.
 285  *    (Note that this is doubling R and performing a decimal left-shift by 1
 286  *    and searching for the appropriate decimal to fill the one's place.)
 287  *    The value x is the next digit in the square root.
 288  * Repeat steps 3 and 4 until the desired precision is reached.  (We're
 289  * dealing with integers, so the above is sufficient.)
 290  *
 291  * In decimal, the square root of 582,734 would be calculated as so:
 292  *
 293  *     __7__6__3
 294  *    | 58 27 34
 295  *     -49       (7^2 == 49 => 7 is the first digit in the square root)
 296  *      --
 297  *       9 27    (Subtract and bring down the next group.)
 298  * 146   8 76    (2 * 7 * 10 * 6 + 6^2 == 876 => 6 is the next digit in
 299  *      -----     the square root)
 300  *         51 34 (Subtract and bring down the next group.)
 301  * 1523    45 69 (2 * 76 * 10 * 3 + 3^2 == 4569 => 3 is the next digit in
 302  *         -----  the square root)
 303  *          5 65 (remainder)
 304  *
 305  * The above algorithm applies similarly in binary, but note that the
 306  * only possible non-zero value for x in step 4 is 1, so step 4 becomes a
 307  * simple decision: is 2 * R * 2 * 1 + 1^2 (aka R << 2 + 1) less than the
 308  * preceding difference?
 309  *
 310  * In binary, the square root of 11011011 would be calculated as so:
 311  *
 312  *     __1__1__1__0
 313  *    | 11 01 10 11
 314  *      01          (0 << 2 + 1 == 1 < 11 => this bit is 1)
 315  *      --
 316  *      10 01 10 11
 317  * 101   1 01       (1 << 2 + 1 == 101 < 1001 => next bit is 1)
 318  *      -----
 319  *       1 00 10 11
 320  * 1101    11 01    (11 << 2 + 1 == 1101 < 10010 => next bit is 1)
 321  *       -------
 322  *          1 01 11
 323  * 11101    1 11 01 (111 << 2 + 1 == 11101 > 10111 => last bit is 0)
 324  *
 325  */
 326 static uint64_t
 327 dt_sqrt_128(uint64_t *square)
 328 {
 329         uint64_t result[2] = { 0, 0 };
 330         uint64_t diff[2] = { 0, 0 };
 331         uint64_t one[2] = { 1, 0 };
 332         uint64_t next_pair[2];
 333         uint64_t next_try[2];
 334         uint64_t bit_pairs, pair_shift;
 335         int i;
 336
 337         bit_pairs = dt_nbits_128(square) / 2;
 338         pair_shift = bit_pairs * 2;
 339
 340         for (i = 0; i <= bit_pairs; i++) {
 341                 /*
 342                  * Bring down the next pair of bits.
 343                  */
 344                 next_pair[0] = square[0];
 345                 next_pair[1] = square[1];
 346                 dt_shift_128(next_pair, -pair_shift);
 347                 next_pair[0] &= 0x3;
 348                 next_pair[1] = 0;
 349
 350                 dt_shift_128(diff, 2);
 351                 dt_add_128(diff, next_pair, diff);
 352
 353                 /*
 354                  * next_try = R << 2 + 1
 355                  */
 356                 next_try[0] = result[0];
 357                 next_try[1] = result[1];
 358                 dt_shift_128(next_try, 2);
 359                 dt_add_128(next_try, one, next_try);
 360
 361                 if (dt_le_128(next_try, diff)) {
 362                         dt_subtract_128(diff, next_try, diff);
 363                         dt_shift_128(result, 1);
 364                         dt_add_128(result, one, result);
 365                 } else {
 366                         dt_shift_128(result, 1);
 367                 }
 368
 369                 pair_shift -= 2;
 370         }
 371
 372         assert(result[1] == 0);
 373
 374         return (result[0]);
 375 }
 376
 377 uint64_t
 378 dt_stddev(uint64_t *data, uint64_t normal)
 379 {
 380         uint64_t avg_of_squares[2];
 381         uint64_t square_of_avg[2];
 382         int64_t norm_avg;
 383         uint64_t diff[2];
 384
 385         if (data[0] == 0)
 386                 return (0);
 387
 388         /*
 389          * The standard approximation for standard deviation is
 390          * sqrt(average(x**2) - average(x)**2), i.e. the square root
 391          * of the average of the squares minus the square of the average.
 392          */
 393         dt_divide_128(data + 2, normal, avg_of_squares);
 394         dt_divide_128(avg_of_squares, data[0], avg_of_squares);
 395
 396         norm_avg = (int64_t)data[1] / (int64_t)normal / (int64_t)data[0];
 397
 398         if (norm_avg < 0)
 399                 norm_avg = -norm_avg;
 400
 401         dt_multiply_128((uint64_t)norm_avg, (uint64_t)norm_avg, square_of_avg);
 402
 403         dt_subtract_128(avg_of_squares, square_of_avg, diff);
 404
 405         return (dt_sqrt_128(diff));
 406 }
 407
 408 static int
 409 dt_flowindent(dtrace_hdl_t *dtp, dtrace_probedata_t *data, dtrace_epid_t last,
 410     dtrace_bufdesc_t *buf, size_t offs)
 411 {
 412         dtrace_probedesc_t *pd = data->dtpda_pdesc, *npd;
 413         dtrace_eprobedesc_t *epd = data->dtpda_edesc, *nepd;
 414         char *p = pd->dtpd_provider, *n = pd->dtpd_name, *sub;
 415         dtrace_flowkind_t flow = DTRACEFLOW_NONE;
 416         const char *str = NULL;
 417         static const char *e_str[2] = { " -> ", " => " };
 418         static const char *r_str[2] = { " <- ", " <= " };
 419         static const char *ent = "entry", *ret = "return";
 420         static int entlen = 0, retlen = 0;
 421         dtrace_epid_t next, id = epd->dtepd_epid;
 422         int rval;
 423
 424         if (entlen == 0) {
 425                 assert(retlen == 0);
 426                 entlen = strlen(ent);
 427                 retlen = strlen(ret);
 428         }
 429
 430         /*
 431          * If the name of the probe is "entry" or ends with "-entry", we
 432          * treat it as an entry; if it is "return" or ends with "-return",
 433          * we treat it as a return.  (This allows application-provided probes
 434          * like "method-entry" or "function-entry" to participate in flow
 435          * indentation -- without accidentally misinterpreting popular probe
 436          * names like "carpentry", "gentry" or "Coventry".)
 437          */
 438         if ((sub = strstr(n, ent)) != NULL && sub[entlen] == '\0' &&
 439             (sub == n || sub[-1] == '-')) {
 440                 flow = DTRACEFLOW_ENTRY;
 441                 str = e_str[strcmp(p, "syscall") == 0];
 442         } else if ((sub = strstr(n, ret)) != NULL && sub[retlen] == '\0' &&
 443             (sub == n || sub[-1] == '-')) {
 444                 flow = DTRACEFLOW_RETURN;
 445                 str = r_str[strcmp(p, "syscall") == 0];
 446         }
 447
 448         /*
 449          * If we're going to indent this, we need to check the ID of our last
 450          * call.  If we're looking at the same probe ID but a different EPID,
 451          * we _don't_ want to indent.  (Yes, there are some minor holes in
 452          * this scheme -- it's a heuristic.)
 453          */
 454         if (flow == DTRACEFLOW_ENTRY) {
 455                 if ((last != DTRACE_EPIDNONE && id != last &&
 456                     pd->dtpd_id == dtp->dt_pdesc[last]->dtpd_id))
 457                         flow = DTRACEFLOW_NONE;
 458         }
 459
 460         /*
 461          * If we're going to unindent this, it's more difficult to see if
 462          * we don't actually want to unindent it -- we need to look at the
 463          * _next_ EPID.
 464          */
 465         if (flow == DTRACEFLOW_RETURN) {
 466                 offs += epd->dtepd_size;
 467
 468                 do {
 469                         if (offs >= buf->dtbd_size)
 470                                 goto out;
 471
 472                         next = *(uint32_t *)((uintptr_t)buf->dtbd_data + offs);
 473
 474                         if (next == DTRACE_EPIDNONE)
 475                                 offs += sizeof (id);
 476                 } while (next == DTRACE_EPIDNONE);
 477
 478                 if ((rval = dt_epid_lookup(dtp, next, &nepd, &npd)) != 0)
 479                         return (rval);
 480
 481                 if (next != id && npd->dtpd_id == pd->dtpd_id)
 482                         flow = DTRACEFLOW_NONE;
 483         }
 484
 485 out:
 486         if (flow == DTRACEFLOW_ENTRY || flow == DTRACEFLOW_RETURN) {
 487                 data->dtpda_prefix = str;
 488         } else {
 489                 data->dtpda_prefix = "| ";
 490         }
 491
 492         if (flow == DTRACEFLOW_RETURN && data->dtpda_indent > 0)
 493                 data->dtpda_indent -= 2;
 494
 495         data->dtpda_flow = flow;
 496
 497         return (0);
 498 }
 499
 500 static int
 501 dt_nullprobe()
 502 {
 503         return (DTRACE_CONSUME_THIS);
 504 }
 505
 506 static int
 507 dt_nullrec()
 508 {
 509         return (DTRACE_CONSUME_NEXT);
 510 }
 511
 512 static void
 513 dt_quantize_total(dtrace_hdl_t *dtp, int64_t datum, long double *total)
 514 {
 515         long double val = dt_fabsl((long double)datum);
 516
 517         if (dtp->dt_options[DTRACEOPT_AGGZOOM] == DTRACEOPT_UNSET) {
 518                 *total += val;
 519                 return;
 520         }
 521
 522         /*
 523          * If we're zooming in on an aggregation, we want the height of the
 524          * highest value to be approximately 95% of total bar height -- so we
 525          * adjust up by the reciprocal of DTRACE_AGGZOOM_MAX when comparing to
 526          * our highest value.
 527          */
 528         val *= 1 / DTRACE_AGGZOOM_MAX;
 529
 530         if (*total < val)
 531                 *total = val;
 532 }
 533
 534 static int
 535 dt_print_quanthdr(dtrace_hdl_t *dtp, FILE *fp, int width)
 536 {
 537         return (dt_printf(dtp, fp, "\n%*s %41s %-9s\n",
 538             width ? width : 16, width ? "key" : "value",
 539             "------------- Distribution -------------", "count"));
 540 }
 541
 542 static int
 543 dt_print_quanthdr_packed(dtrace_hdl_t *dtp, FILE *fp, int width,
 544     const dtrace_aggdata_t *aggdata, dtrace_actkind_t action)
 545 {
 546         int min = aggdata->dtada_minbin, max = aggdata->dtada_maxbin;
 547         int minwidth, maxwidth, i;
 548
 549         assert(action == DTRACEAGG_QUANTIZE || action == DTRACEAGG_LQUANTIZE);
 550
 551         if (action == DTRACEAGG_QUANTIZE) {
 552                 if (min != 0 && min != DTRACE_QUANTIZE_ZEROBUCKET)
 553                         min--;
 554
 555                 if (max < DTRACE_QUANTIZE_NBUCKETS - 1)
 556                         max++;
 557
 558                 minwidth = dt_ndigits(DTRACE_QUANTIZE_BUCKETVAL(min));
 559                 maxwidth = dt_ndigits(DTRACE_QUANTIZE_BUCKETVAL(max));
 560         } else {
 561                 maxwidth = 8;
 562                 minwidth = maxwidth - 1;
 563                 max++;
 564         }
 565
 566         if (dt_printf(dtp, fp, "\n%*s %*s .",
 567             width, width > 0 ? "key" : "", minwidth, "min") < 0)
 568                 return (-1);
 569
 570         for (i = min; i <= max; i++) {
 571                 if (dt_printf(dtp, fp, "-") < 0)
 572                         return (-1);
 573         }
 574
 575         return (dt_printf(dtp, fp, ". %*s | count\n", -maxwidth, "max"));
 576 }
 577
 578 /*
 579  * We use a subset of the Unicode Block Elements (U+2588 through U+258F,
 580  * inclusive) to represent aggregations via UTF-8 -- which are expressed via
 581  * 3-byte UTF-8 sequences.
 582  */
 583 #define DTRACE_AGGUTF8_FULL     0x2588
 584 #define DTRACE_AGGUTF8_BASE     0x258f
 585 #define DTRACE_AGGUTF8_LEVELS   8
 586
 587 #define DTRACE_AGGUTF8_BYTE0(val)       (0xe0 | ((val) >> 12))
 588 #define DTRACE_AGGUTF8_BYTE1(val)       (0x80 | (((val) >> 6) & 0x3f))
 589 #define DTRACE_AGGUTF8_BYTE2(val)       (0x80 | ((val) & 0x3f))
 590
 591 static int
 592 dt_print_quantline_utf8(dtrace_hdl_t *dtp, FILE *fp, int64_t val,
 593     uint64_t normal, long double total)
 594 {
 595         uint_t len = 40, i, whole, partial;
 596         long double f = (dt_fabsl((long double)val) * len) / total;
 597         const char *spaces = "                                        ";
 598
 599         whole = (uint_t)f;
 600         partial = (uint_t)((f - (long double)(uint_t)f) *
 601             (long double)DTRACE_AGGUTF8_LEVELS);
 602
 603         if (dt_printf(dtp, fp, "|") < 0)
 604                 return (-1);
 605
 606         for (i = 0; i < whole; i++) {
 607                 if (dt_printf(dtp, fp, "%c%c%c",
 608                     DTRACE_AGGUTF8_BYTE0(DTRACE_AGGUTF8_FULL),
 609                     DTRACE_AGGUTF8_BYTE1(DTRACE_AGGUTF8_FULL),
 610                     DTRACE_AGGUTF8_BYTE2(DTRACE_AGGUTF8_FULL)) < 0)
 611                         return (-1);
 612         }
 613
 614         if (partial != 0) {
 615                 partial = DTRACE_AGGUTF8_BASE - (partial - 1);
 616
 617                 if (dt_printf(dtp, fp, "%c%c%c",
 618                     DTRACE_AGGUTF8_BYTE0(partial),
 619                     DTRACE_AGGUTF8_BYTE1(partial),
 620                     DTRACE_AGGUTF8_BYTE2(partial)) < 0)
 621                         return (-1);
 622
 623                 i++;
 624         }
 625
 626         return (dt_printf(dtp, fp, "%s %-9lld\n", spaces + i,
 627             (long long)val / normal));
 628 }
 629
 630 static int
 631 dt_print_quantline(dtrace_hdl_t *dtp, FILE *fp, int64_t val,
 632     uint64_t normal, long double total, char positives, char negatives)
 633 {
 634         long double f;
 635         uint_t depth, len = 40;
 636
 637         const char *ats = "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@";
 638         const char *spaces = "                                        ";
 639
 640         assert(strlen(ats) == len && strlen(spaces) == len);
 641         assert(!(total == 0 && (positives || negatives)));
 642         assert(!(val < 0 && !negatives));
 643         assert(!(val > 0 && !positives));
 644         assert(!(val != 0 && total == 0));
 645
 646         if (!negatives) {
 647                 if (positives) {
 648                         if (dtp->dt_encoding == DT_ENCODING_UTF8) {
 649                                 return (dt_print_quantline_utf8(dtp, fp, val,
 650                                     normal, total));
 651                         }
 652
 653                         f = (dt_fabsl((long double)val) * len) / total;
 654                         depth = (uint_t)(f + 0.5);
 655                 } else {
 656                         depth = 0;
 657                 }
 658
 659                 return (dt_printf(dtp, fp, "|%s%s %-9lld\n", ats + len - depth,
 660                     spaces + depth, (long long)val / normal));
 661         }
 662
 663         if (!positives) {
 664                 f = (dt_fabsl((long double)val) * len) / total;
 665                 depth = (uint_t)(f + 0.5);
 666
 667                 return (dt_printf(dtp, fp, "%s%s| %-9lld\n", spaces + depth,
 668                     ats + len - depth, (long long)val / normal));
 669         }
 670
 671         /*
 672          * If we're here, we have both positive and negative bucket values.
 673          * To express this graphically, we're going to generate both positive
 674          * and negative bars separated by a centerline.  These bars are half
 675          * the size of normal quantize()/lquantize() bars, so we divide the
 676          * length in half before calculating the bar length.
 677          */
 678         len /= 2;
 679         ats = &ats[len];
 680         spaces = &spaces[len];
 681
 682         f = (dt_fabsl((long double)val) * len) / total;
 683         depth = (uint_t)(f + 0.5);
 684
 685         if (val <= 0) {
 686                 return (dt_printf(dtp, fp, "%s%s|%*s %-9lld\n", spaces + depth,
 687                     ats + len - depth, len, "", (long long)val / normal));
 688         } else {
 689                 return (dt_printf(dtp, fp, "%20s|%s%s %-9lld\n", "",
 690                     ats + len - depth, spaces + depth,
 691                     (long long)val / normal));
 692         }
 693 }
 694
 695 /*
 696  * As with UTF-8 printing of aggregations, we use a subset of the Unicode
 697  * Block Elements (U+2581 through U+2588, inclusive) to represent our packed
 698  * aggregation.
 699  */
 700 #define DTRACE_AGGPACK_BASE     0x2581
 701 #define DTRACE_AGGPACK_LEVELS   8
 702
 703 static int
 704 dt_print_packed(dtrace_hdl_t *dtp, FILE *fp,
 705     long double datum, long double total)
 706 {
 707         static boolean_t utf8_checked = B_FALSE;
 708         static boolean_t utf8;
 709         char *ascii = "__xxxxXX";
 710         char *neg = "vvvvVV";
 711         unsigned int len;
 712         long double val;
 713
 714         if (!utf8_checked) {
 715                 char *term;
 716
 717                 /*
 718                  * We want to determine if we can reasonably emit UTF-8 for our
 719                  * packed aggregation.  To do this, we will check for terminals
 720                  * that are known to be primitive to emit UTF-8 on these.
 721                  */
 722                 utf8_checked = B_TRUE;
 723
 724                 if (dtp->dt_encoding == DT_ENCODING_ASCII) {
 725                         utf8 = B_FALSE;
 726                 } else if (dtp->dt_encoding == DT_ENCODING_UTF8) {
 727                         utf8 = B_TRUE;
 728                 } else if ((term = getenv("TERM")) != NULL &&
 729                     (strcmp(term, "sun") == 0 ||
 730                     strcmp(term, "sun-color") == 0 ||
 731                     strcmp(term, "dumb") == 0)) {
 732                         utf8 = B_FALSE;
 733                 } else {
 734                         utf8 = B_TRUE;
 735                 }
 736         }
 737
 738         if (datum == 0)
 739                 return (dt_printf(dtp, fp, " "));
 740
 741         if (datum < 0) {
 742                 len = strlen(neg);
 743                 val = dt_fabsl(datum * (len - 1)) / total;
 744                 return (dt_printf(dtp, fp, "%c", neg[(uint_t)(val + 0.5)]));
 745         }
 746
 747         if (utf8) {
 748                 int block = DTRACE_AGGPACK_BASE + (unsigned int)(((datum *
 749                     (DTRACE_AGGPACK_LEVELS - 1)) / total) + 0.5);
 750
 751                 return (dt_printf(dtp, fp, "%c%c%c",
 752                     DTRACE_AGGUTF8_BYTE0(block),
 753                     DTRACE_AGGUTF8_BYTE1(block),
 754                     DTRACE_AGGUTF8_BYTE2(block)));
 755         }
 756
 757         len = strlen(ascii);
 758         val = (datum * (len - 1)) / total;
 759         return (dt_printf(dtp, fp, "%c", ascii[(uint_t)(val + 0.5)]));
 760 }
 761
 762 int
 763 dt_print_quantize(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
 764     size_t size, uint64_t normal)
 765 {
 766         const int64_t *data = addr;
 767         int i, first_bin = 0, last_bin = DTRACE_QUANTIZE_NBUCKETS - 1;
 768         long double total = 0;
 769         char positives = 0, negatives = 0;
 770
 771         if (size != DTRACE_QUANTIZE_NBUCKETS * sizeof (uint64_t))
 772                 return (dt_set_errno(dtp, EDT_DMISMATCH));
 773
 774         while (first_bin < DTRACE_QUANTIZE_NBUCKETS - 1 && data[first_bin] == 0)
 775                 first_bin++;
 776
 777         if (first_bin == DTRACE_QUANTIZE_NBUCKETS - 1) {
 778                 /*
 779                  * There isn't any data.  This is possible if the aggregation
 780                  * has been clear()'d or if negative increment values have been
 781                  * used.  Regardless, we'll print the buckets around 0.
 782                  */
 783                 first_bin = DTRACE_QUANTIZE_ZEROBUCKET - 1;
 784                 last_bin = DTRACE_QUANTIZE_ZEROBUCKET + 1;
 785         } else {
 786                 if (first_bin > 0)
 787                         first_bin--;
 788
 789                 while (last_bin > 0 && data[last_bin] == 0)
 790                         last_bin--;
 791
 792                 if (last_bin < DTRACE_QUANTIZE_NBUCKETS - 1)
 793                         last_bin++;
 794         }
 795
 796         for (i = first_bin; i <= last_bin; i++) {
 797                 positives |= (data[i] > 0);
 798                 negatives |= (data[i] < 0);
 799                 dt_quantize_total(dtp, data[i], &total);
 800         }
 801
 802         if (dt_print_quanthdr(dtp, fp, 0) < 0)
 803                 return (-1);
 804
 805         for (i = first_bin; i <= last_bin; i++) {
 806                 if (dt_printf(dtp, fp, "%16lld ",
 807                     (long long)DTRACE_QUANTIZE_BUCKETVAL(i)) < 0)
 808                         return (-1);
 809
 810                 if (dt_print_quantline(dtp, fp, data[i], normal, total,
 811                     positives, negatives) < 0)
 812                         return (-1);
 813         }
 814
 815         return (0);
 816 }
 817
 818 int
 819 dt_print_quantize_packed(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
 820     size_t size, const dtrace_aggdata_t *aggdata)
 821 {
 822         const int64_t *data = addr;
 823         long double total = 0, count = 0;
 824         int min = aggdata->dtada_minbin, max = aggdata->dtada_maxbin, i;
 825         int64_t minval, maxval;
 826
 827         if (size != DTRACE_QUANTIZE_NBUCKETS * sizeof (uint64_t))
 828                 return (dt_set_errno(dtp, EDT_DMISMATCH));
 829
 830         if (min != 0 && min != DTRACE_QUANTIZE_ZEROBUCKET)
 831                 min--;
 832
 833         if (max < DTRACE_QUANTIZE_NBUCKETS - 1)
 834                 max++;
 835
 836         minval = DTRACE_QUANTIZE_BUCKETVAL(min);
 837         maxval = DTRACE_QUANTIZE_BUCKETVAL(max);
 838
 839         if (dt_printf(dtp, fp, " %*lld :", dt_ndigits(minval),
 840             (long long)minval) < 0)
 841                 return (-1);
 842
 843         for (i = min; i <= max; i++) {
 844                 dt_quantize_total(dtp, data[i], &total);
 845                 count += data[i];
 846         }
 847
 848         for (i = min; i <= max; i++) {
 849                 if (dt_print_packed(dtp, fp, data[i], total) < 0)
 850                         return (-1);
 851         }
 852
 853         if (dt_printf(dtp, fp, ": %*lld | %lld\n",
 854             -dt_ndigits(maxval), (long long)maxval, (long long)count) < 0)
 855                 return (-1);
 856
 857         return (0);
 858 }
 859
 860 int
 861 dt_print_lquantize(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
 862     size_t size, uint64_t normal)
 863 {
 864         const int64_t *data = addr;
 865         int i, first_bin, last_bin, base;
 866         uint64_t arg;
 867         long double total = 0;
 868         uint16_t step, levels;
 869         char positives = 0, negatives = 0;
 870
 871         if (size < sizeof (uint64_t))
 872                 return (dt_set_errno(dtp, EDT_DMISMATCH));
 873
 874         arg = *data++;
 875         size -= sizeof (uint64_t);
 876
 877         base = DTRACE_LQUANTIZE_BASE(arg);
 878         step = DTRACE_LQUANTIZE_STEP(arg);
 879         levels = DTRACE_LQUANTIZE_LEVELS(arg);
 880
 881         first_bin = 0;
 882         last_bin = levels + 1;
 883
 884         if (size != sizeof (uint64_t) * (levels + 2))
 885                 return (dt_set_errno(dtp, EDT_DMISMATCH));
 886
 887         while (first_bin <= levels + 1 && data[first_bin] == 0)
 888                 first_bin++;
 889
 890         if (first_bin > levels + 1) {
 891                 first_bin = 0;
 892                 last_bin = 2;
 893         } else {
 894                 if (first_bin > 0)
 895                         first_bin--;
 896
 897                 while (last_bin > 0 && data[last_bin] == 0)
 898                         last_bin--;
 899
 900                 if (last_bin < levels + 1)
 901                         last_bin++;
 902         }
 903
 904         for (i = first_bin; i <= last_bin; i++) {
 905                 positives |= (data[i] > 0);
 906                 negatives |= (data[i] < 0);
 907                 dt_quantize_total(dtp, data[i], &total);
 908         }
 909
 910         if (dt_printf(dtp, fp, "\n%16s %41s %-9s\n", "value",
 911             "------------- Distribution -------------", "count") < 0)
 912                 return (-1);
 913
 914         for (i = first_bin; i <= last_bin; i++) {
 915                 char c[32];
 916                 int err;
 917
 918                 if (i == 0) {
 919                         (void) snprintf(c, sizeof (c), "< %d", base);
 920                         err = dt_printf(dtp, fp, "%16s ", c);
 921                 } else if (i == levels + 1) {
 922                         (void) snprintf(c, sizeof (c), ">= %d",
 923                             base + (levels * step));
 924                         err = dt_printf(dtp, fp, "%16s ", c);
 925                 } else {
 926                         err = dt_printf(dtp, fp, "%16d ",
 927                             base + (i - 1) * step);
 928                 }
 929
 930                 if (err < 0 || dt_print_quantline(dtp, fp, data[i], normal,
 931                     total, positives, negatives) < 0)
 932                         return (-1);
 933         }
 934
 935         return (0);
 936 }
 937
 938 /*ARGSUSED*/
 939 int
 940 dt_print_lquantize_packed(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
 941     size_t size, const dtrace_aggdata_t *aggdata)
 942 {
 943         const int64_t *data = addr;
 944         long double total = 0, count = 0;
 945         int min, max, base, err;
 946         uint64_t arg;
 947         uint16_t step, levels;
 948         char c[32];
 949         unsigned int i;
 950
 951         if (size < sizeof (uint64_t))
 952                 return (dt_set_errno(dtp, EDT_DMISMATCH));
 953
 954         arg = *data++;
 955         size -= sizeof (uint64_t);
 956
 957         base = DTRACE_LQUANTIZE_BASE(arg);
 958         step = DTRACE_LQUANTIZE_STEP(arg);
 959         levels = DTRACE_LQUANTIZE_LEVELS(arg);
 960
 961         if (size != sizeof (uint64_t) * (levels + 2))
 962                 return (dt_set_errno(dtp, EDT_DMISMATCH));
 963
 964         min = 0;
 965         max = levels + 1;
 966
 967         if (min == 0) {
 968                 (void) snprintf(c, sizeof (c), "< %d", base);
 969                 err = dt_printf(dtp, fp, "%8s :", c);
 970         } else {
 971                 err = dt_printf(dtp, fp, "%8d :", base + (min - 1) * step);
 972         }
 973
 974         if (err < 0)
 975                 return (-1);
 976
 977         for (i = min; i <= max; i++) {
 978                 dt_quantize_total(dtp, data[i], &total);
 979                 count += data[i];
 980         }
 981
 982         for (i = min; i <= max; i++) {
 983                 if (dt_print_packed(dtp, fp, data[i], total) < 0)
 984                         return (-1);
 985         }
 986
 987         (void) snprintf(c, sizeof (c), ">= %d", base + (levels * step));
 988         return (dt_printf(dtp, fp, ": %-8s | %lld\n", c, (long long)count));
 989 }
 990
 991 int
 992 dt_print_llquantize(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
 993     size_t size, uint64_t normal)
 994 {
 995         int i, first_bin, last_bin, bin = 1, order, levels;
 996         uint16_t factor, low, high, nsteps;
 997         const int64_t *data = addr;
 998         int64_t value = 1, next, step;
 999         char positives = 0, negatives = 0;
1000         long double total = 0;
1001         uint64_t arg;
1002         char c[32];
1003
1004         if (size < sizeof (uint64_t))
1005                 return (dt_set_errno(dtp, EDT_DMISMATCH));
1006
1007         arg = *data++;
1008         size -= sizeof (uint64_t);
1009
1010         factor = DTRACE_LLQUANTIZE_FACTOR(arg);
1011         low = DTRACE_LLQUANTIZE_LOW(arg);
1012         high = DTRACE_LLQUANTIZE_HIGH(arg);
1013         nsteps = DTRACE_LLQUANTIZE_NSTEP(arg);
1014
1015         /*
1016          * We don't expect to be handed invalid llquantize() parameters here,
1017          * but sanity check them (to a degree) nonetheless.
1018          */
1019         if (size > INT32_MAX || factor < 2 || low >= high ||
1020             nsteps == 0 || factor > nsteps)
1021                 return (dt_set_errno(dtp, EDT_DMISMATCH));
1022
1023         levels = (int)size / sizeof (uint64_t);
1024
1025         first_bin = 0;
1026         last_bin = levels - 1;
1027
1028         while (first_bin < levels && data[first_bin] == 0)
1029                 first_bin++;
1030
1031         if (first_bin == levels) {
1032                 first_bin = 0;
1033                 last_bin = 1;
1034         } else {
1035                 if (first_bin > 0)
1036                         first_bin--;
1037
1038                 while (last_bin > 0 && data[last_bin] == 0)
1039                         last_bin--;
1040
1041                 if (last_bin < levels - 1)
1042                         last_bin++;
1043         }
1044
1045         for (i = first_bin; i <= last_bin; i++) {
1046                 positives |= (data[i] > 0);
1047                 negatives |= (data[i] < 0);
1048                 dt_quantize_total(dtp, data[i], &total);
1049         }
1050
1051         if (dt_printf(dtp, fp, "\n%16s %41s %-9s\n", "value",
1052             "------------- Distribution -------------", "count") < 0)
1053                 return (-1);
1054
1055         for (order = 0; order < low; order++)
1056                 value *= factor;
1057
1058         next = value * factor;
1059         step = next > nsteps ? next / nsteps : 1;
1060
1061         if (first_bin == 0) {
1062                 (void) snprintf(c, sizeof (c), "< %lld", (long long)value);
1063
1064                 if (dt_printf(dtp, fp, "%16s ", c) < 0)
1065                         return (-1);
1066
1067                 if (dt_print_quantline(dtp, fp, data[0], normal,
1068                     total, positives, negatives) < 0)
1069                         return (-1);
1070         }
1071
1072         while (order <= high) {
1073                 if (bin >= first_bin && bin <= last_bin) {
1074                         if (dt_printf(dtp, fp, "%16lld ", (long long)value) < 0)
1075                                 return (-1);
1076
1077                         if (dt_print_quantline(dtp, fp, data[bin],
1078                             normal, total, positives, negatives) < 0)
1079                                 return (-1);
1080                 }
1081
1082                 assert(value < next);
1083                 bin++;
1084
1085                 if ((value += step) != next)
1086                         continue;
1087
1088                 next = value * factor;
1089                 step = next > nsteps ? next / nsteps : 1;
1090                 order++;
1091         }
1092
1093         if (last_bin < bin)
1094                 return (0);
1095
1096         assert(last_bin == bin);
1097         (void) snprintf(c, sizeof (c), ">= %lld", (long long)value);
1098
1099         if (dt_printf(dtp, fp, "%16s ", c) < 0)
1100                 return (-1);
1101
1102         return (dt_print_quantline(dtp, fp, data[bin], normal,
1103             total, positives, negatives));
1104 }
1105
1106 /*ARGSUSED*/
1107 static int
1108 dt_print_average(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr,
1109     size_t size, uint64_t normal)
1110 {
1111         /* LINTED - alignment */
1112         int64_t *data = (int64_t *)addr;
1113
1114         return (dt_printf(dtp, fp, " %16lld", data[0] ?
1115             (long long)(data[1] / (int64_t)normal / data[0]) : 0));
1116 }
1117
1118 /*ARGSUSED*/
1119 static int
1120 dt_print_stddev(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr,
1121     size_t size, uint64_t normal)
1122 {
1123         /* LINTED - alignment */
1124         uint64_t *data = (uint64_t *)addr;
1125
1126         return (dt_printf(dtp, fp, " %16llu", data[0] ?
1127             (unsigned long long) dt_stddev(data, normal) : 0));
1128 }
1129
1130 /*ARGSUSED*/
1131 static int
1132 dt_print_bytes(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr,
1133     size_t nbytes, int width, int quiet, int forceraw)
1134 {
1135         /*
1136          * If the byte stream is a series of printable characters, followed by
1137          * a terminating byte, we print it out as a string.  Otherwise, we
1138          * assume that it's something else and just print the bytes.
1139          */
1140         int i, j, margin = 5;
1141         char *c = (char *)addr;
1142
1143         if (nbytes == 0)
1144                 return (0);
1145
1146         if (forceraw)
1147                 goto raw;
1148
1149         if (dtp->dt_options[DTRACEOPT_RAWBYTES] != DTRACEOPT_UNSET)
1150                 goto raw;
1151
1152         for (i = 0; i < nbytes; i++) {
1153                 /*
1154                  * We define a "printable character" to be one for which
1155                  * isprint(3C) returns non-zero, isspace(3C) returns non-zero,
1156                  * or a character which is either backspace or the bell.
1157                  * Backspace and the bell are regrettably special because
1158                  * they fail the first two tests -- and yet they are entirely
1159                  * printable.  These are the only two control characters that
1160                  * have meaning for the terminal and for which isprint(3C) and
1161                  * isspace(3C) return 0.
1162                  */
1163                 if (isprint(c[i]) || isspace(c[i]) ||
1164                     c[i] == '\b' || c[i] == '\a')
1165                         continue;
1166
1167                 if (c[i] == '\0' && i > 0) {
1168                         /*
1169                          * This looks like it might be a string.  Before we
1170                          * assume that it is indeed a string, check the
1171                          * remainder of the byte range; if it contains
1172                          * additional non-nul characters, we'll assume that
1173                          * it's a binary stream that just happens to look like
1174                          * a string, and we'll print out the individual bytes.
1175                          */
1176                         for (j = i + 1; j < nbytes; j++) {
1177                                 if (c[j] != '\0')
1178                                         break;
1179                         }
1180
1181                         if (j != nbytes)
1182                                 break;
1183
1184                         if (quiet) {
1185                                 return (dt_printf(dtp, fp, "%s", c));
1186                         } else {
1187                                 return (dt_printf(dtp, fp, " %s%*s",
1188                                     width < 0 ? " " : "", width, c));
1189                         }
1190                 }
1191
1192                 break;
1193         }
1194
1195         if (i == nbytes) {
1196                 /*
1197                  * The byte range is all printable characters, but there is
1198                  * no trailing nul byte.  We'll assume that it's a string and
1199                  * print it as such.
1200                  */
1201                 char *s = alloca(nbytes + 1);
1202                 bcopy(c, s, nbytes);
1203                 s[nbytes] = '\0';
1204                 return (dt_printf(dtp, fp, "  %-*s", width, s));
1205         }
1206
1207 raw:
1208         if (dt_printf(dtp, fp, "\n%*s      ", margin, "") < 0)
1209                 return (-1);
1210
1211         for (i = 0; i < 16; i++)
1212                 if (dt_printf(dtp, fp, "  %c", "0123456789abcdef"[i]) < 0)
1213                         return (-1);
1214
1215         if (dt_printf(dtp, fp, "  0123456789abcdef\n") < 0)
1216                 return (-1);
1217
1218
1219         for (i = 0; i < nbytes; i += 16) {
1220                 if (dt_printf(dtp, fp, "%*s%5x:", margin, "", i) < 0)
1221                         return (-1);
1222
1223                 for (j = i; j < i + 16 && j < nbytes; j++) {
1224                         if (dt_printf(dtp, fp, " %02x", (uchar_t)c[j]) < 0)
1225                                 return (-1);
1226                 }
1227
1228                 while (j++ % 16) {
1229                         if (dt_printf(dtp, fp, "   ") < 0)
1230                                 return (-1);
1231                 }
1232
1233                 if (dt_printf(dtp, fp, "  ") < 0)
1234                         return (-1);
1235
1236                 for (j = i; j < i + 16 && j < nbytes; j++) {
1237                         if (dt_printf(dtp, fp, "%c",
1238                             c[j] < ' ' || c[j] > '~' ? '.' : c[j]) < 0)
1239                                 return (-1);
1240                 }
1241
1242                 if (dt_printf(dtp, fp, "\n") < 0)
1243                         return (-1);
1244         }
1245
1246         return (0);
1247 }
1248
1249 int
1250 dt_print_stack(dtrace_hdl_t *dtp, FILE *fp, const char *format,
1251     caddr_t addr, int depth, int size)
1252 {
1253         dtrace_syminfo_t dts;
1254         GElf_Sym sym;
1255         int i, indent;
1256         char c[PATH_MAX * 2];
1257         uint64_t pc;
1258
1259         if (dt_printf(dtp, fp, "\n") < 0)
1260                 return (-1);
1261
1262         if (format == NULL)
1263                 format = "%s";
1264
1265         if (dtp->dt_options[DTRACEOPT_STACKINDENT] != DTRACEOPT_UNSET)
1266                 indent = (int)dtp->dt_options[DTRACEOPT_STACKINDENT];
1267         else
1268                 indent = _dtrace_stkindent;
1269
1270         for (i = 0; i < depth; i++) {
1271                 switch (size) {
1272                 case sizeof (uint32_t):
1273                         /* LINTED - alignment */
1274                         pc = *((uint32_t *)addr);
1275                         break;
1276
1277                 case sizeof (uint64_t):
1278                         /* LINTED - alignment */
1279                         pc = *((uint64_t *)addr);
1280                         break;
1281
1282                 default:
1283                         return (dt_set_errno(dtp, EDT_BADSTACKPC));
1284                 }
1285
1286                 if (pc == 0)
1287                         break;
1288
1289                 addr += size;
1290
1291                 if (dt_printf(dtp, fp, "%*s", indent, "") < 0)
1292                         return (-1);
1293
1294                 if (dtrace_lookup_by_addr(dtp, pc, &sym, &dts) == 0) {
1295                         if (pc > sym.st_value) {
1296                                 (void) snprintf(c, sizeof (c), "%s`%s+0x%llx",
1297                                     dts.dts_object, dts.dts_name,
1298                                     (u_longlong_t)(pc - sym.st_value));
1299                         } else {
1300                                 (void) snprintf(c, sizeof (c), "%s`%s",
1301                                     dts.dts_object, dts.dts_name);
1302                         }
1303                 } else {
1304                         /*
1305                          * We'll repeat the lookup, but this time we'll specify
1306                          * a NULL GElf_Sym -- indicating that we're only
1307                          * interested in the containing module.
1308                          */
1309                         if (dtrace_lookup_by_addr(dtp, pc, NULL, &dts) == 0) {
1310                                 (void) snprintf(c, sizeof (c), "%s`0x%llx",
1311                                     dts.dts_object, (u_longlong_t)pc);
1312                         } else {
1313                                 (void) snprintf(c, sizeof (c), "0x%llx",
1314                                     (u_longlong_t)pc);
1315                         }
1316                 }
1317
1318                 if (dt_printf(dtp, fp, format, c) < 0)
1319                         return (-1);
1320
1321                 if (dt_printf(dtp, fp, "\n") < 0)
1322                         return (-1);
1323         }
1324
1325         return (0);
1326 }
1327
1328 int
1329 dt_print_ustack(dtrace_hdl_t *dtp, FILE *fp, const char *format,
1330     caddr_t addr, uint64_t arg)
1331 {
1332         /* LINTED - alignment */
1333         uint64_t *pc = (uint64_t *)addr;
1334         uint32_t depth = DTRACE_USTACK_NFRAMES(arg);
1335         uint32_t strsize = DTRACE_USTACK_STRSIZE(arg);
1336         const char *strbase = addr + (depth + 1) * sizeof (uint64_t);
1337         const char *str = strsize ? strbase : NULL;
1338         int err = 0;
1339
1340         char name[PATH_MAX], objname[PATH_MAX], c[PATH_MAX * 2];
1341         struct ps_prochandle *P;
1342         GElf_Sym sym;
1343         int i, indent;
1344         pid_t pid;
1345
1346         if (depth == 0)
1347                 return (0);
1348
1349         pid = (pid_t)*pc++;
1350
1351         if (dt_printf(dtp, fp, "\n") < 0)
1352                 return (-1);
1353
1354         if (format == NULL)
1355                 format = "%s";
1356
1357         if (dtp->dt_options[DTRACEOPT_STACKINDENT] != DTRACEOPT_UNSET)
1358                 indent = (int)dtp->dt_options[DTRACEOPT_STACKINDENT];
1359         else
1360                 indent = _dtrace_stkindent;
1361
1362         /*
1363          * Ultimately, we need to add an entry point in the library vector for
1364          * determining <symbol, offset> from <pid, address>.  For now, if
1365          * this is a vector open, we just print the raw address or string.
1366          */
1367         if (dtp->dt_vector == NULL)
1368                 P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0);
1369         else
1370                 P = NULL;
1371
1372         if (P != NULL)
1373                 dt_proc_lock(dtp, P); /* lock handle while we perform lookups */
1374
1375         for (i = 0; i < depth && pc[i] != 0; i++) {
1376                 const prmap_t *map;
1377
1378                 if ((err = dt_printf(dtp, fp, "%*s", indent, "")) < 0)
1379                         break;
1380
1381                 if (P != NULL && Plookup_by_addr(P, pc[i],
1382                     name, sizeof (name), &sym) == 0) {
1383                         (void) Pobjname(P, pc[i], objname, sizeof (objname));
1384
1385                         if (pc[i] > sym.st_value) {
1386                                 (void) snprintf(c, sizeof (c),
1387                                     "%s`%s+0x%llx", dt_basename(objname), name,
1388                                     (u_longlong_t)(pc[i] - sym.st_value));
1389                         } else {
1390                                 (void) snprintf(c, sizeof (c),
1391                                     "%s`%s", dt_basename(objname), name);
1392                         }
1393                 } else if (str != NULL && str[0] != '\0' && str[0] != '@' &&
1394                     (P != NULL && ((map = Paddr_to_map(P, pc[i])) == NULL ||
1395                     (map->pr_mflags & MA_WRITE)))) {
1396                         /*
1397                          * If the current string pointer in the string table
1398                          * does not point to an empty string _and_ the program
1399                          * counter falls in a writable region, we'll use the
1400                          * string from the string table instead of the raw
1401                          * address.  This last condition is necessary because
1402                          * some (broken) ustack helpers will return a string
1403                          * even for a program counter that they can't
1404                          * identify.  If we have a string for a program
1405                          * counter that falls in a segment that isn't
1406                          * writable, we assume that we have fallen into this
1407                          * case and we refuse to use the string.
1408                          */
1409                         (void) snprintf(c, sizeof (c), "%s", str);
1410                 } else {
1411                         if (P != NULL && Pobjname(P, pc[i], objname,
1412                             sizeof (objname)) != 0) {
1413                                 (void) snprintf(c, sizeof (c), "%s`0x%llx",
1414                                     dt_basename(objname), (u_longlong_t)pc[i]);
1415                         } else {
1416                                 (void) snprintf(c, sizeof (c), "0x%llx",
1417                                     (u_longlong_t)pc[i]);
1418                         }
1419                 }
1420
1421                 if ((err = dt_printf(dtp, fp, format, c)) < 0)
1422                         break;
1423
1424                 if ((err = dt_printf(dtp, fp, "\n")) < 0)
1425                         break;
1426
1427                 if (str != NULL && str[0] == '@') {
1428                         /*
1429                          * If the first character of the string is an "at" sign,
1430                          * then the string is inferred to be an annotation --
1431                          * and it is printed out beneath the frame and offset
1432                          * with brackets.
1433                          */
1434                         if ((err = dt_printf(dtp, fp, "%*s", indent, "")) < 0)
1435                                 break;
1436
1437                         (void) snprintf(c, sizeof (c), "  [ %s ]", &str[1]);
1438
1439                         if ((err = dt_printf(dtp, fp, format, c)) < 0)
1440                                 break;
1441
1442                         if ((err = dt_printf(dtp, fp, "\n")) < 0)
1443                                 break;
1444                 }
1445
1446                 if (str != NULL) {
1447                         str += strlen(str) + 1;
1448                         if (str - strbase >= strsize)
1449                                 str = NULL;
1450                 }
1451         }
1452
1453         if (P != NULL) {
1454                 dt_proc_unlock(dtp, P);
1455                 dt_proc_release(dtp, P);
1456         }
1457
1458         return (err);
1459 }
1460
1461 static int
1462 dt_print_usym(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr, dtrace_actkind_t act)
1463 {
1464         /* LINTED - alignment */
1465         uint64_t pid = ((uint64_t *)addr)[0];
1466         /* LINTED - alignment */
1467         uint64_t pc = ((uint64_t *)addr)[1];
1468         const char *format = "  %-50s";
1469         char *s;
1470         int n, len = 256;
1471
1472         if (act == DTRACEACT_USYM && dtp->dt_vector == NULL) {
1473                 struct ps_prochandle *P;
1474
1475                 if ((P = dt_proc_grab(dtp, pid,
1476                     PGRAB_RDONLY | PGRAB_FORCE, 0)) != NULL) {
1477                         GElf_Sym sym;
1478
1479                         dt_proc_lock(dtp, P);
1480
1481                         if (Plookup_by_addr(P, pc, NULL, 0, &sym) == 0)
1482                                 pc = sym.st_value;
1483
1484                         dt_proc_unlock(dtp, P);
1485                         dt_proc_release(dtp, P);
1486                 }
1487         }
1488
1489         do {
1490                 n = len;
1491                 s = alloca(n);
1492         } while ((len = dtrace_uaddr2str(dtp, pid, pc, s, n)) > n);
1493
1494         return (dt_printf(dtp, fp, format, s));
1495 }
1496
1497 int
1498 dt_print_umod(dtrace_hdl_t *dtp, FILE *fp, const char *format, caddr_t addr)
1499 {
1500         /* LINTED - alignment */
1501         uint64_t pid = ((uint64_t *)addr)[0];
1502         /* LINTED - alignment */
1503         uint64_t pc = ((uint64_t *)addr)[1];
1504         int err = 0;
1505
1506         char objname[PATH_MAX], c[PATH_MAX * 2];
1507         struct ps_prochandle *P;
1508
1509         if (format == NULL)
1510                 format = "  %-50s";
1511
1512         /*
1513          * See the comment in dt_print_ustack() for the rationale for
1514          * printing raw addresses in the vectored case.
1515          */
1516         if (dtp->dt_vector == NULL)
1517                 P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0);
1518         else
1519                 P = NULL;
1520
1521         if (P != NULL)
1522                 dt_proc_lock(dtp, P); /* lock handle while we perform lookups */
1523
1524         if (P != NULL && Pobjname(P, pc, objname, sizeof (objname)) != 0) {
1525                 (void) snprintf(c, sizeof (c), "%s", dt_basename(objname));
1526         } else {
1527                 (void) snprintf(c, sizeof (c), "0x%llx", (u_longlong_t)pc);
1528         }
1529
1530         err = dt_printf(dtp, fp, format, c);
1531
1532         if (P != NULL) {
1533                 dt_proc_unlock(dtp, P);
1534                 dt_proc_release(dtp, P);
1535         }
1536
1537         return (err);
1538 }
1539
1540 int
1541 dt_print_memory(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr)
1542 {
1543         int quiet = (dtp->dt_options[DTRACEOPT_QUIET] != DTRACEOPT_UNSET);
1544         size_t nbytes = *((uintptr_t *) addr);
1545
1546         return (dt_print_bytes(dtp, fp, addr + sizeof(uintptr_t),
1547             nbytes, 50, quiet, 1));
1548 }
1549
1550 typedef struct dt_type_cbdata {
1551         dtrace_hdl_t            *dtp;
1552         dtrace_typeinfo_t       dtt;
1553         caddr_t                 addr;
1554         caddr_t                 addrend;
1555         const char              *name;
1556         int                     f_type;
1557         int                     indent;
1558         int                     type_width;
1559         int                     name_width;
1560         FILE                    *fp;
1561 } dt_type_cbdata_t;
1562
1563 static int      dt_print_type_data(dt_type_cbdata_t *, ctf_id_t);
1564
1565 static int
1566 dt_print_type_member(const char *name, ctf_id_t type, ulong_t off, void *arg)
1567 {
1568         dt_type_cbdata_t cbdata;
1569         dt_type_cbdata_t *cbdatap = arg;
1570         ssize_t ssz;
1571
1572         if ((ssz = ctf_type_size(cbdatap->dtt.dtt_ctfp, type)) <= 0)
1573                 return (0);
1574
1575         off /= 8;
1576
1577         cbdata = *cbdatap;
1578         cbdata.name = name;
1579         cbdata.addr += off;
1580         cbdata.addrend = cbdata.addr + ssz;
1581
1582         return (dt_print_type_data(&cbdata, type));
1583 }
1584
1585 static int
1586 dt_print_type_width(const char *name, ctf_id_t type, ulong_t off, void *arg)
1587 {
1588         char buf[DT_TYPE_NAMELEN];
1589         char *p;
1590         dt_type_cbdata_t *cbdatap = arg;
1591         size_t sz = strlen(name);
1592
1593         ctf_type_name(cbdatap->dtt.dtt_ctfp, type, buf, sizeof (buf));
1594
1595         if ((p = strchr(buf, '[')) != NULL)
1596                 p[-1] = '\0';
1597         else
1598                 p = "";
1599
1600         sz += strlen(p);
1601
1602         if (sz > cbdatap->name_width)
1603                 cbdatap->name_width = sz;
1604
1605         sz = strlen(buf);
1606
1607         if (sz > cbdatap->type_width)
1608                 cbdatap->type_width = sz;
1609
1610         return (0);
1611 }
1612
1613 static int
1614 dt_print_type_data(dt_type_cbdata_t *cbdatap, ctf_id_t type)
1615 {
1616         caddr_t addr = cbdatap->addr;
1617         caddr_t addrend = cbdatap->addrend;
1618         char buf[DT_TYPE_NAMELEN];
1619         char *p;
1620         int cnt = 0;
1621         uint_t kind = ctf_type_kind(cbdatap->dtt.dtt_ctfp, type);
1622         ssize_t ssz = ctf_type_size(cbdatap->dtt.dtt_ctfp, type);
1623
1624         ctf_type_name(cbdatap->dtt.dtt_ctfp, type, buf, sizeof (buf));
1625
1626         if ((p = strchr(buf, '[')) != NULL)
1627                 p[-1] = '\0';
1628         else
1629                 p = "";
1630
1631         if (cbdatap->f_type) {
1632                 int type_width = roundup(cbdatap->type_width + 1, 4);
1633                 int name_width = roundup(cbdatap->name_width + 1, 4);
1634
1635                 name_width -= strlen(cbdatap->name);
1636
1637                 dt_printf(cbdatap->dtp, cbdatap->fp, "%*s%-*s%s%-*s     = ",cbdatap->indent * 4,"",type_width,buf,cbdatap->name,name_width,p);
1638         }
1639
1640         while (addr < addrend) {
1641                 dt_type_cbdata_t cbdata;
1642                 ctf_arinfo_t arinfo;
1643                 ctf_encoding_t cte;
1644                 uintptr_t *up;
1645                 void *vp = addr;
1646                 cbdata = *cbdatap;
1647                 cbdata.name = "";
1648                 cbdata.addr = addr;
1649                 cbdata.addrend = addr + ssz;
1650                 cbdata.f_type = 0;
1651                 cbdata.indent++;
1652                 cbdata.type_width = 0;
1653                 cbdata.name_width = 0;
1654
1655                 if (cnt > 0)
1656                         dt_printf(cbdatap->dtp, cbdatap->fp, "%*s", cbdatap->indent * 4,"");
1657
1658                 switch (kind) {
1659                 case CTF_K_INTEGER:
1660                         if (ctf_type_encoding(cbdatap->dtt.dtt_ctfp, type, &cte) != 0)
1661                                 return (-1);
1662                         if ((cte.cte_format & CTF_INT_SIGNED) != 0)
1663                                 switch (cte.cte_bits) {
1664                                 case 8:
1665                                         if (isprint(*((char *) vp)))
1666                                                 dt_printf(cbdatap->dtp, cbdatap->fp, "'%c', ", *((char *) vp));
1667                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%d (0x%x);\n", *((char *) vp), *((char *) vp));
1668                                         break;
1669                                 case 16:
1670                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%hd (0x%hx);\n", *((short *) vp), *((u_short *) vp));
1671                                         break;
1672                                 case 32:
1673                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%d (0x%x);\n", *((int *) vp), *((u_int *) vp));
1674                                         break;
1675                                 case 64:
1676                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%jd (0x%jx);\n", *((long long *) vp), *((unsigned long long *) vp));
1677                                         break;
1678                                 default:
1679                                         dt_printf(cbdatap->dtp, cbdatap->fp, "CTF_K_INTEGER: format %x offset %u bits %u\n",cte.cte_format,cte.cte_offset,cte.cte_bits);
1680                                         break;
1681                                 }
1682                         else
1683                                 switch (cte.cte_bits) {
1684                                 case 8:
1685                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%u (0x%x);\n", *((uint8_t *) vp) & 0xff, *((uint8_t *) vp) & 0xff);
1686                                         break;
1687                                 case 16:
1688                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%hu (0x%hx);\n", *((u_short *) vp), *((u_short *) vp));
1689                                         break;
1690                                 case 32:
1691                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%u (0x%x);\n", *((u_int *) vp), *((u_int *) vp));
1692                                         break;
1693                                 case 64:
1694                                         dt_printf(cbdatap->dtp, cbdatap->fp, "%ju (0x%jx);\n", *((unsigned long long *) vp), *((unsigned long long *) vp));
1695                                         break;
1696                                 default:
1697                                         dt_printf(cbdatap->dtp, cbdatap->fp, "CTF_K_INTEGER: format %x offset %u bits %u\n",cte.cte_format,cte.cte_offset,cte.cte_bits);
1698                                         break;
1699                                 }
1700                         break;
1701                 case CTF_K_FLOAT:
1702                         dt_printf(cbdatap->dtp, cbdatap->fp, "CTF_K_FLOAT: format %x offset %u bits %u\n",cte.cte_format,cte.cte_offset,cte.cte_bits);
1703                         break;
1704                 case CTF_K_POINTER:
1705                         dt_printf(cbdatap->dtp, cbdatap->fp, "%p;\n", *((void **) addr));
1706                         break;
1707                 case CTF_K_ARRAY:
1708                         if (ctf_array_info(cbdatap->dtt.dtt_ctfp, type, &arinfo) != 0)
1709                                 return (-1);
1710                         dt_printf(cbdatap->dtp, cbdatap->fp, "{\n%*s",cbdata.indent * 4,"");
1711                         dt_print_type_data(&cbdata, arinfo.ctr_contents);
1712                         dt_printf(cbdatap->dtp, cbdatap->fp, "%*s};\n",cbdatap->indent * 4,"");
1713                         break;
1714                 case CTF_K_FUNCTION:
1715                         dt_printf(cbdatap->dtp, cbdatap->fp, "CTF_K_FUNCTION:\n");
1716                         break;
1717                 case CTF_K_STRUCT:
1718                         cbdata.f_type = 1;
1719                         if (ctf_member_iter(cbdatap->dtt.dtt_ctfp, type,
1720                             dt_print_type_width, &cbdata) != 0)
1721                                 return (-1);
1722                         dt_printf(cbdatap->dtp, cbdatap->fp, "{\n");
1723                         if (ctf_member_iter(cbdatap->dtt.dtt_ctfp, type,
1724                             dt_print_type_member, &cbdata) != 0)
1725                                 return (-1);
1726                         dt_printf(cbdatap->dtp, cbdatap->fp, "%*s};\n",cbdatap->indent * 4,"");
1727                         break;
1728                 case CTF_K_UNION:
1729                         cbdata.f_type = 1;
1730                         if (ctf_member_iter(cbdatap->dtt.dtt_ctfp, type,
1731                             dt_print_type_width, &cbdata) != 0)
1732                                 return (-1);
1733                         dt_printf(cbdatap->dtp, cbdatap->fp, "{\n");
1734                         if (ctf_member_iter(cbdatap->dtt.dtt_ctfp, type,
1735                             dt_print_type_member, &cbdata) != 0)
1736                                 return (-1);
1737                         dt_printf(cbdatap->dtp, cbdatap->fp, "%*s};\n",cbdatap->indent * 4,"");
1738                         break;
1739                 case CTF_K_ENUM:
1740                         dt_printf(cbdatap->dtp, cbdatap->fp, "%s;\n", ctf_enum_name(cbdatap->dtt.dtt_ctfp, type, *((int *) vp)));
1741                         break;
1742                 case CTF_K_TYPEDEF:
1743                         dt_print_type_data(&cbdata, ctf_type_reference(cbdatap->dtt.dtt_ctfp,type));
1744                         break;
1745                 case CTF_K_VOLATILE:
1746                         if (cbdatap->f_type)
1747                                 dt_printf(cbdatap->dtp, cbdatap->fp, "volatile ");
1748                         dt_print_type_data(&cbdata, ctf_type_reference(cbdatap->dtt.dtt_ctfp,type));
1749                         break;
1750                 case CTF_K_CONST:
1751                         if (cbdatap->f_type)
1752                                 dt_printf(cbdatap->dtp, cbdatap->fp, "const ");
1753                         dt_print_type_data(&cbdata, ctf_type_reference(cbdatap->dtt.dtt_ctfp,type));
1754                         break;
1755                 case CTF_K_RESTRICT:
1756                         if (cbdatap->f_type)
1757                                 dt_printf(cbdatap->dtp, cbdatap->fp, "restrict ");
1758                         dt_print_type_data(&cbdata, ctf_type_reference(cbdatap->dtt.dtt_ctfp,type));
1759                         break;
1760                 default:
1761                         break;
1762                 }
1763
1764                 addr += ssz;
1765                 cnt++;
1766         }
1767
1768         return (0);
1769 }
1770
1771 static int
1772 dt_print_type(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr)
1773 {
1774         caddr_t addrend;
1775         char *p;
1776         dtrace_typeinfo_t dtt;
1777         dt_type_cbdata_t cbdata;
1778         int num = 0;
1779         int quiet = (dtp->dt_options[DTRACEOPT_QUIET] != DTRACEOPT_UNSET);
1780         ssize_t ssz;
1781
1782         if (!quiet)
1783                 dt_printf(dtp, fp, "\n");
1784
1785         /* Get the total number of bytes of data buffered. */
1786         size_t nbytes = *((uintptr_t *) addr);
1787         addr += sizeof(uintptr_t);
1788
1789         /*
1790          * Get the size of the type so that we can check that it matches
1791          * the CTF data we look up and so that we can figure out how many
1792          * type elements are buffered.
1793          */
1794         size_t typs = *((uintptr_t *) addr);
1795         addr += sizeof(uintptr_t);
1796
1797         /*
1798          * Point to the type string in the buffer. Get it's string
1799          * length and round it up to become the offset to the start
1800          * of the buffered type data which we would like to be aligned
1801          * for easy access.
1802          */
1803         char *strp = (char *) addr;
1804         int offset = roundup(strlen(strp) + 1, sizeof(uintptr_t));
1805
1806         /*
1807          * The type string might have a format such as 'int [20]'.
1808          * Check if there is an array dimension present.
1809          */
1810         if ((p = strchr(strp, '[')) != NULL) {
1811                 /* Strip off the array dimension. */
1812                 *p++ = '\0';
1813
1814                 for (; *p != '\0' && *p != ']'; p++)
1815                         num = num * 10 + *p - '0';
1816         } else
1817                 /* No array dimension, so default. */
1818                 num = 1;
1819
1820         /* Lookup the CTF type from the type string. */
1821         if (dtrace_lookup_by_type(dtp,  DTRACE_OBJ_EVERY, strp, &dtt) < 0)
1822                 return (-1);
1823
1824         /* Offset the buffer address to the start of the data... */
1825         addr += offset;
1826
1827         ssz = ctf_type_size(dtt.dtt_ctfp, dtt.dtt_type);
1828
1829         if (typs != ssz) {
1830                 printf("Expected type size from buffer (%lu) to match type size looked up now (%ld)\n", (u_long) typs, (long) ssz);
1831                 return (-1);
1832         }
1833
1834         cbdata.dtp = dtp;
1835         cbdata.dtt = dtt;
1836         cbdata.name = "";
1837         cbdata.addr = addr;
1838         cbdata.addrend = addr + nbytes;
1839         cbdata.indent = 1;
1840         cbdata.f_type = 1;
1841         cbdata.type_width = 0;
1842         cbdata.name_width = 0;
1843         cbdata.fp = fp;
1844
1845         return (dt_print_type_data(&cbdata, dtt.dtt_type));
1846 }
1847
1848 static int
1849 dt_print_sym(dtrace_hdl_t *dtp, FILE *fp, const char *format, caddr_t addr)
1850 {
1851         /* LINTED - alignment */
1852         uint64_t pc = *((uint64_t *)addr);
1853         dtrace_syminfo_t dts;
1854         GElf_Sym sym;
1855         char c[PATH_MAX * 2];
1856
1857         if (format == NULL)
1858                 format = "  %-50s";
1859
1860         if (dtrace_lookup_by_addr(dtp, pc, &sym, &dts) == 0) {
1861                 (void) snprintf(c, sizeof (c), "%s`%s",
1862                     dts.dts_object, dts.dts_name);
1863         } else {
1864                 /*
1865                  * We'll repeat the lookup, but this time we'll specify a
1866                  * NULL GElf_Sym -- indicating that we're only interested in
1867                  * the containing module.
1868                  */
1869                 if (dtrace_lookup_by_addr(dtp, pc, NULL, &dts) == 0) {
1870                         (void) snprintf(c, sizeof (c), "%s`0x%llx",
1871                             dts.dts_object, (u_longlong_t)pc);
1872                 } else {
1873                         (void) snprintf(c, sizeof (c), "0x%llx",
1874                             (u_longlong_t)pc);
1875                 }
1876         }
1877
1878         if (dt_printf(dtp, fp, format, c) < 0)
1879                 return (-1);
1880
1881         return (0);
1882 }
1883
1884 int
1885 dt_print_mod(dtrace_hdl_t *dtp, FILE *fp, const char *format, caddr_t addr)
1886 {
1887         /* LINTED - alignment */
1888         uint64_t pc = *((uint64_t *)addr);
1889         dtrace_syminfo_t dts;
1890         char c[PATH_MAX * 2];
1891
1892         if (format == NULL)
1893                 format = "  %-50s";
1894
1895         if (dtrace_lookup_by_addr(dtp, pc, NULL, &dts) == 0) {
1896                 (void) snprintf(c, sizeof (c), "%s", dts.dts_object);
1897         } else {
1898                 (void) snprintf(c, sizeof (c), "0x%llx", (u_longlong_t)pc);
1899         }
1900
1901         if (dt_printf(dtp, fp, format, c) < 0)
1902                 return (-1);
1903
1904         return (0);
1905 }
1906
1907 typedef struct dt_normal {
1908         dtrace_aggvarid_t dtnd_id;
1909         uint64_t dtnd_normal;
1910 } dt_normal_t;
1911
1912 static int
1913 dt_normalize_agg(const dtrace_aggdata_t *aggdata, void *arg)
1914 {
1915         dt_normal_t *normal = arg;
1916         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1917         dtrace_aggvarid_t id = normal->dtnd_id;
1918
1919         if (agg->dtagd_nrecs == 0)
1920                 return (DTRACE_AGGWALK_NEXT);
1921
1922         if (agg->dtagd_varid != id)
1923                 return (DTRACE_AGGWALK_NEXT);
1924
1925         ((dtrace_aggdata_t *)aggdata)->dtada_normal = normal->dtnd_normal;
1926         return (DTRACE_AGGWALK_NORMALIZE);
1927 }
1928
1929 static int
1930 dt_normalize(dtrace_hdl_t *dtp, caddr_t base, dtrace_recdesc_t *rec)
1931 {
1932         dt_normal_t normal;
1933         caddr_t addr;
1934
1935         /*
1936          * We (should) have two records:  the aggregation ID followed by the
1937          * normalization value.
1938          */
1939         addr = base + rec->dtrd_offset;
1940
1941         if (rec->dtrd_size != sizeof (dtrace_aggvarid_t))
1942                 return (dt_set_errno(dtp, EDT_BADNORMAL));
1943
1944         /* LINTED - alignment */
1945         normal.dtnd_id = *((dtrace_aggvarid_t *)addr);
1946         rec++;
1947
1948         if (rec->dtrd_action != DTRACEACT_LIBACT)
1949                 return (dt_set_errno(dtp, EDT_BADNORMAL));
1950
1951         if (rec->dtrd_arg != DT_ACT_NORMALIZE)
1952                 return (dt_set_errno(dtp, EDT_BADNORMAL));
1953
1954         addr = base + rec->dtrd_offset;
1955
1956         switch (rec->dtrd_size) {
1957         case sizeof (uint64_t):
1958                 /* LINTED - alignment */
1959                 normal.dtnd_normal = *((uint64_t *)addr);
1960                 break;
1961         case sizeof (uint32_t):
1962                 /* LINTED - alignment */
1963                 normal.dtnd_normal = *((uint32_t *)addr);
1964                 break;
1965         case sizeof (uint16_t):
1966                 /* LINTED - alignment */
1967                 normal.dtnd_normal = *((uint16_t *)addr);
1968                 break;
1969         case sizeof (uint8_t):
1970                 normal.dtnd_normal = *((uint8_t *)addr);
1971                 break;
1972         default:
1973                 return (dt_set_errno(dtp, EDT_BADNORMAL));
1974         }
1975
1976         (void) dtrace_aggregate_walk(dtp, dt_normalize_agg, &normal);
1977
1978         return (0);
1979 }
1980
1981 static int
1982 dt_denormalize_agg(const dtrace_aggdata_t *aggdata, void *arg)
1983 {
1984         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1985         dtrace_aggvarid_t id = *((dtrace_aggvarid_t *)arg);
1986
1987         if (agg->dtagd_nrecs == 0)
1988                 return (DTRACE_AGGWALK_NEXT);
1989
1990         if (agg->dtagd_varid != id)
1991                 return (DTRACE_AGGWALK_NEXT);
1992
1993         return (DTRACE_AGGWALK_DENORMALIZE);
1994 }
1995
1996 static int
1997 dt_clear_agg(const dtrace_aggdata_t *aggdata, void *arg)
1998 {
1999         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
2000         dtrace_aggvarid_t id = *((dtrace_aggvarid_t *)arg);
2001
2002         if (agg->dtagd_nrecs == 0)
2003                 return (DTRACE_AGGWALK_NEXT);
2004
2005         if (agg->dtagd_varid != id)
2006                 return (DTRACE_AGGWALK_NEXT);
2007
2008         return (DTRACE_AGGWALK_CLEAR);
2009 }
2010
2011 typedef struct dt_trunc {
2012         dtrace_aggvarid_t dttd_id;
2013         uint64_t dttd_remaining;
2014 } dt_trunc_t;
2015
2016 static int
2017 dt_trunc_agg(const dtrace_aggdata_t *aggdata, void *arg)
2018 {
2019         dt_trunc_t *trunc = arg;
2020         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
2021         dtrace_aggvarid_t id = trunc->dttd_id;
2022
2023         if (agg->dtagd_nrecs == 0)
2024                 return (DTRACE_AGGWALK_NEXT);
2025
2026         if (agg->dtagd_varid != id)
2027                 return (DTRACE_AGGWALK_NEXT);
2028
2029         if (trunc->dttd_remaining == 0)
2030                 return (DTRACE_AGGWALK_REMOVE);
2031
2032         trunc->dttd_remaining--;
2033         return (DTRACE_AGGWALK_NEXT);
2034 }
2035
2036 static int
2037 dt_trunc(dtrace_hdl_t *dtp, caddr_t base, dtrace_recdesc_t *rec)
2038 {
2039         dt_trunc_t trunc;
2040         caddr_t addr;
2041         int64_t remaining;
2042         int (*func)(dtrace_hdl_t *, dtrace_aggregate_f *, void *);
2043
2044         /*
2045          * We (should) have two records:  the aggregation ID followed by the
2046          * number of aggregation entries after which the aggregation is to be
2047          * truncated.
2048          */
2049         addr = base + rec->dtrd_offset;
2050
2051         if (rec->dtrd_size != sizeof (dtrace_aggvarid_t))
2052                 return (dt_set_errno(dtp, EDT_BADTRUNC));
2053
2054         /* LINTED - alignment */
2055         trunc.dttd_id = *((dtrace_aggvarid_t *)addr);
2056         rec++;
2057
2058         if (rec->dtrd_action != DTRACEACT_LIBACT)
2059                 return (dt_set_errno(dtp, EDT_BADTRUNC));
2060
2061         if (rec->dtrd_arg != DT_ACT_TRUNC)
2062                 return (dt_set_errno(dtp, EDT_BADTRUNC));
2063
2064         addr = base + rec->dtrd_offset;
2065
2066         switch (rec->dtrd_size) {
2067         case sizeof (uint64_t):
2068                 /* LINTED - alignment */
2069                 remaining = *((int64_t *)addr);
2070                 break;
2071         case sizeof (uint32_t):
2072                 /* LINTED - alignment */
2073                 remaining = *((int32_t *)addr);
2074                 break;
2075         case sizeof (uint16_t):
2076                 /* LINTED - alignment */
2077                 remaining = *((int16_t *)addr);
2078                 break;
2079         case sizeof (uint8_t):
2080                 remaining = *((int8_t *)addr);
2081                 break;
2082         default:
2083                 return (dt_set_errno(dtp, EDT_BADNORMAL));
2084         }
2085
2086         if (remaining < 0) {
2087                 func = dtrace_aggregate_walk_valsorted;
2088                 remaining = -remaining;
2089         } else {
2090                 func = dtrace_aggregate_walk_valrevsorted;
2091         }
2092
2093         assert(remaining >= 0);
2094         trunc.dttd_remaining = remaining;
2095
2096         (void) func(dtp, dt_trunc_agg, &trunc);
2097
2098         return (0);
2099 }
2100
2101 static int
2102 dt_print_datum(dtrace_hdl_t *dtp, FILE *fp, dtrace_recdesc_t *rec,
2103     caddr_t addr, size_t size, const dtrace_aggdata_t *aggdata,
2104     uint64_t normal, dt_print_aggdata_t *pd)
2105 {
2106         int err, width;
2107         dtrace_actkind_t act = rec->dtrd_action;
2108         boolean_t packed = pd->dtpa_agghist || pd->dtpa_aggpack;
2109         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
2110
2111         static struct {
2112                 size_t size;
2113                 int width;
2114                 int packedwidth;
2115         } *fmt, fmttab[] = {
2116                 { sizeof (uint8_t),     3,      3 },
2117                 { sizeof (uint16_t),    5,      5 },
2118                 { sizeof (uint32_t),    8,      8 },
2119                 { sizeof (uint64_t),    16,     16 },
2120                 { 0,                    -50,    16 }
2121         };
2122
2123         if (packed && pd->dtpa_agghisthdr != agg->dtagd_varid) {
2124                 dtrace_recdesc_t *r;
2125
2126                 width = 0;
2127
2128                 /*
2129                  * To print our quantization header for either an agghist or
2130                  * aggpack aggregation, we need to iterate through all of our
2131                  * of our records to determine their width.
2132                  */
2133                 for (r = rec; !DTRACEACT_ISAGG(r->dtrd_action); r++) {
2134                         for (fmt = fmttab; fmt->size &&
2135                             fmt->size != r->dtrd_size; fmt++)
2136                                 continue;
2137
2138                         width += fmt->packedwidth + 1;
2139                 }
2140
2141                 if (pd->dtpa_agghist) {
2142                         if (dt_print_quanthdr(dtp, fp, width) < 0)
2143                                 return (-1);
2144                 } else {
2145                         if (dt_print_quanthdr_packed(dtp, fp,
2146                             width, aggdata, r->dtrd_action) < 0)
2147                                 return (-1);
2148                 }
2149
2150                 pd->dtpa_agghisthdr = agg->dtagd_varid;
2151         }
2152
2153         if (pd->dtpa_agghist && DTRACEACT_ISAGG(act)) {
2154                 char positives = aggdata->dtada_flags & DTRACE_A_HASPOSITIVES;
2155                 char negatives = aggdata->dtada_flags & DTRACE_A_HASNEGATIVES;
2156                 int64_t val;
2157
2158                 assert(act == DTRACEAGG_SUM || act == DTRACEAGG_COUNT);
2159                 val = (long long)*((uint64_t *)addr);
2160
2161                 if (dt_printf(dtp, fp, " ") < 0)
2162                         return (-1);
2163
2164                 return (dt_print_quantline(dtp, fp, val, normal,
2165                     aggdata->dtada_total, positives, negatives));
2166         }
2167
2168         if (pd->dtpa_aggpack && DTRACEACT_ISAGG(act)) {
2169                 switch (act) {
2170                 case DTRACEAGG_QUANTIZE:
2171                         return (dt_print_quantize_packed(dtp,
2172                             fp, addr, size, aggdata));
2173                 case DTRACEAGG_LQUANTIZE:
2174                         return (dt_print_lquantize_packed(dtp,
2175                             fp, addr, size, aggdata));
2176                 default:
2177                         break;
2178                 }
2179         }
2180
2181         switch (act) {
2182         case DTRACEACT_STACK:
2183                 return (dt_print_stack(dtp, fp, NULL, addr,
2184                     rec->dtrd_arg, rec->dtrd_size / rec->dtrd_arg));
2185
2186         case DTRACEACT_USTACK:
2187         case DTRACEACT_JSTACK:
2188                 return (dt_print_ustack(dtp, fp, NULL, addr, rec->dtrd_arg));
2189
2190         case DTRACEACT_USYM:
2191         case DTRACEACT_UADDR:
2192                 return (dt_print_usym(dtp, fp, addr, act));
2193
2194         case DTRACEACT_UMOD:
2195                 return (dt_print_umod(dtp, fp, NULL, addr));
2196
2197         case DTRACEACT_SYM:
2198                 return (dt_print_sym(dtp, fp, NULL, addr));
2199
2200         case DTRACEACT_MOD:
2201                 return (dt_print_mod(dtp, fp, NULL, addr));
2202
2203         case DTRACEAGG_QUANTIZE:
2204                 return (dt_print_quantize(dtp, fp, addr, size, normal));
2205
2206         case DTRACEAGG_LQUANTIZE:
2207                 return (dt_print_lquantize(dtp, fp, addr, size, normal));
2208
2209         case DTRACEAGG_LLQUANTIZE:
2210                 return (dt_print_llquantize(dtp, fp, addr, size, normal));
2211
2212         case DTRACEAGG_AVG:
2213                 return (dt_print_average(dtp, fp, addr, size, normal));
2214
2215         case DTRACEAGG_STDDEV:
2216                 return (dt_print_stddev(dtp, fp, addr, size, normal));
2217
2218         default:
2219                 break;
2220         }
2221
2222         for (fmt = fmttab; fmt->size && fmt->size != size; fmt++)
2223                 continue;
2224
2225         width = packed ? fmt->packedwidth : fmt->width;
2226
2227         switch (size) {
2228         case sizeof (uint64_t):
2229                 err = dt_printf(dtp, fp, " %*lld", width,
2230                     /* LINTED - alignment */
2231                     (long long)*((uint64_t *)addr) / normal);
2232                 break;
2233         case sizeof (uint32_t):
2234                 /* LINTED - alignment */
2235                 err = dt_printf(dtp, fp, " %*d", width, *((uint32_t *)addr) /
2236                     (uint32_t)normal);
2237                 break;
2238         case sizeof (uint16_t):
2239                 /* LINTED - alignment */
2240                 err = dt_printf(dtp, fp, " %*d", width, *((uint16_t *)addr) /
2241                     (uint32_t)normal);
2242                 break;
2243         case sizeof (uint8_t):
2244                 err = dt_printf(dtp, fp, " %*d", width, *((uint8_t *)addr) /
2245                     (uint32_t)normal);
2246                 break;
2247         default:
2248                 err = dt_print_bytes(dtp, fp, addr, size, width, 0, 0);
2249                 break;
2250         }
2251
2252         return (err);
2253 }
2254
2255 int
2256 dt_print_aggs(const dtrace_aggdata_t **aggsdata, int naggvars, void *arg)
2257 {
2258         int i, aggact = 0;
2259         dt_print_aggdata_t *pd = arg;
2260         const dtrace_aggdata_t *aggdata = aggsdata[0];
2261         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
2262         FILE *fp = pd->dtpa_fp;
2263         dtrace_hdl_t *dtp = pd->dtpa_dtp;
2264         dtrace_recdesc_t *rec;
2265         dtrace_actkind_t act;
2266         caddr_t addr;
2267         size_t size;
2268
2269         pd->dtpa_agghist = (aggdata->dtada_flags & DTRACE_A_TOTAL);
2270         pd->dtpa_aggpack = (aggdata->dtada_flags & DTRACE_A_MINMAXBIN);
2271
2272         /*
2273          * Iterate over each record description in the key, printing the traced
2274          * data, skipping the first datum (the tuple member created by the
2275          * compiler).
2276          */
2277         for (i = 1; i < agg->dtagd_nrecs; i++) {
2278                 rec = &agg->dtagd_rec[i];
2279                 act = rec->dtrd_action;
2280                 addr = aggdata->dtada_data + rec->dtrd_offset;
2281                 size = rec->dtrd_size;
2282
2283                 if (DTRACEACT_ISAGG(act)) {
2284                         aggact = i;
2285                         break;
2286                 }
2287
2288                 if (dt_print_datum(dtp, fp, rec, addr,
2289                     size, aggdata, 1, pd) < 0)
2290                         return (-1);
2291
2292                 if (dt_buffered_flush(dtp, NULL, rec, aggdata,
2293                     DTRACE_BUFDATA_AGGKEY) < 0)
2294                         return (-1);
2295         }
2296
2297         assert(aggact != 0);
2298
2299         for (i = (naggvars == 1 ? 0 : 1); i < naggvars; i++) {
2300                 uint64_t normal;
2301
2302                 aggdata = aggsdata[i];
2303                 agg = aggdata->dtada_desc;
2304                 rec = &agg->dtagd_rec[aggact];
2305                 act = rec->dtrd_action;
2306                 addr = aggdata->dtada_data + rec->dtrd_offset;
2307                 size = rec->dtrd_size;
2308
2309                 assert(DTRACEACT_ISAGG(act));
2310                 normal = aggdata->dtada_normal;
2311
2312                 if (dt_print_datum(dtp, fp, rec, addr,
2313                     size, aggdata, normal, pd) < 0)
2314                         return (-1);
2315
2316                 if (dt_buffered_flush(dtp, NULL, rec, aggdata,
2317                     DTRACE_BUFDATA_AGGVAL) < 0)
2318                         return (-1);
2319
2320                 if (!pd->dtpa_allunprint)
2321                         agg->dtagd_flags |= DTRACE_AGD_PRINTED;
2322         }
2323
2324         if (!pd->dtpa_agghist && !pd->dtpa_aggpack) {
2325                 if (dt_printf(dtp, fp, "\n") < 0)
2326                         return (-1);
2327         }
2328
2329         if (dt_buffered_flush(dtp, NULL, NULL, aggdata,
2330             DTRACE_BUFDATA_AGGFORMAT | DTRACE_BUFDATA_AGGLAST) < 0)
2331                 return (-1);
2332
2333         return (0);
2334 }
2335
2336 int
2337 dt_print_agg(const dtrace_aggdata_t *aggdata, void *arg)
2338 {
2339         dt_print_aggdata_t *pd = arg;
2340         dtrace_aggdesc_t *agg = aggdata->dtada_desc;
2341         dtrace_aggvarid_t aggvarid = pd->dtpa_id;
2342
2343         if (pd->dtpa_allunprint) {
2344                 if (agg->dtagd_flags & DTRACE_AGD_PRINTED)
2345                         return (0);
2346         } else {
2347                 /*
2348                  * If we're not printing all unprinted aggregations, then the
2349                  * aggregation variable ID denotes a specific aggregation
2350                  * variable that we should print -- skip any other aggregations
2351                  * that we encounter.
2352                  */
2353                 if (agg->dtagd_nrecs == 0)
2354                         return (0);
2355
2356                 if (aggvarid != agg->dtagd_varid)
2357                         return (0);
2358         }
2359
2360         return (dt_print_aggs(&aggdata, 1, arg));
2361 }
2362
2363 int
2364 dt_setopt(dtrace_hdl_t *dtp, const dtrace_probedata_t *data,
2365     const char *option, const char *value)
2366 {
2367         int len, rval;
2368         char *msg;
2369         const char *errstr;
2370         dtrace_setoptdata_t optdata;
2371
2372         bzero(&optdata, sizeof (optdata));
2373         (void) dtrace_getopt(dtp, option, &optdata.dtsda_oldval);
2374
2375         if (dtrace_setopt(dtp, option, value) == 0) {
2376                 (void) dtrace_getopt(dtp, option, &optdata.dtsda_newval);
2377                 optdata.dtsda_probe = data;
2378                 optdata.dtsda_option = option;
2379                 optdata.dtsda_handle = dtp;
2380
2381                 if ((rval = dt_handle_setopt(dtp, &optdata)) != 0)
2382                         return (rval);
2383
2384                 return (0);
2385         }
2386
2387         errstr = dtrace_errmsg(dtp, dtrace_errno(dtp));
2388         len = strlen(option) + strlen(value) + strlen(errstr) + 80;
2389         msg = alloca(len);
2390
2391         (void) snprintf(msg, len, "couldn't set option \"%s\" to \"%s\": %s\n",
2392             option, value, errstr);
2393
2394         if ((rval = dt_handle_liberr(dtp, data, msg)) == 0)
2395                 return (0);
2396
2397         return (rval);
2398 }
2399
2400 static int
2401 dt_consume_cpu(dtrace_hdl_t *dtp, FILE *fp, int cpu,
2402     dtrace_bufdesc_t *buf, boolean_t just_one,
2403     dtrace_consume_probe_f *efunc, dtrace_consume_rec_f *rfunc, void *arg)
2404 {
2405         dtrace_epid_t id;
2406         size_t offs;
2407         int flow = (dtp->dt_options[DTRACEOPT_FLOWINDENT] != DTRACEOPT_UNSET);
2408         int quiet = (dtp->dt_options[DTRACEOPT_QUIET] != DTRACEOPT_UNSET);
2409         int rval, i, n;
2410         uint64_t tracememsize = 0;
2411         dtrace_probedata_t data;
2412         uint64_t drops;
2413
2414         bzero(&data, sizeof (data));
2415         data.dtpda_handle = dtp;
2416         data.dtpda_cpu = cpu;
2417         data.dtpda_flow = dtp->dt_flow;
2418         data.dtpda_indent = dtp->dt_indent;
2419         data.dtpda_prefix = dtp->dt_prefix;
2420
2421         for (offs = buf->dtbd_oldest; offs < buf->dtbd_size; ) {
2422                 dtrace_eprobedesc_t *epd;
2423
2424                 /*
2425                  * We're guaranteed to have an ID.
2426                  */
2427                 id = *(uint32_t *)((uintptr_t)buf->dtbd_data + offs);
2428
2429                 if (id == DTRACE_EPIDNONE) {
2430                         /*
2431                          * This is filler to assure proper alignment of the
2432                          * next record; we simply ignore it.
2433                          */
2434                         offs += sizeof (id);
2435                         continue;
2436                 }
2437
2438                 if ((rval = dt_epid_lookup(dtp, id, &data.dtpda_edesc,
2439                     &data.dtpda_pdesc)) != 0)
2440                         return (rval);
2441
2442                 epd = data.dtpda_edesc;
2443                 data.dtpda_data = buf->dtbd_data + offs;
2444
2445                 if (data.dtpda_edesc->dtepd_uarg != DT_ECB_DEFAULT) {
2446                         rval = dt_handle(dtp, &data);
2447
2448                         if (rval == DTRACE_CONSUME_NEXT)
2449                                 goto nextepid;
2450
2451                         if (rval == DTRACE_CONSUME_ERROR)
2452                                 return (-1);
2453                 }
2454
2455                 if (flow)
2456                         (void) dt_flowindent(dtp, &data, dtp->dt_last_epid,
2457                             buf, offs);
2458
2459                 rval = (*efunc)(&data, arg);
2460
2461                 if (flow) {
2462                         if (data.dtpda_flow == DTRACEFLOW_ENTRY)
2463                                 data.dtpda_indent += 2;
2464                 }
2465
2466                 if (rval == DTRACE_CONSUME_NEXT)
2467                         goto nextepid;
2468
2469                 if (rval == DTRACE_CONSUME_ABORT)
2470                         return (dt_set_errno(dtp, EDT_DIRABORT));
2471
2472                 if (rval != DTRACE_CONSUME_THIS)
2473                         return (dt_set_errno(dtp, EDT_BADRVAL));
2474
2475                 for (i = 0; i < epd->dtepd_nrecs; i++) {
2476                         caddr_t addr;
2477                         dtrace_recdesc_t *rec = &epd->dtepd_rec[i];
2478                         dtrace_actkind_t act = rec->dtrd_action;
2479
2480                         data.dtpda_data = buf->dtbd_data + offs +
2481                             rec->dtrd_offset;
2482                         addr = data.dtpda_data;
2483
2484                         if (act == DTRACEACT_LIBACT) {
2485                                 uint64_t arg = rec->dtrd_arg;
2486                                 dtrace_aggvarid_t id;
2487
2488                                 switch (arg) {
2489                                 case DT_ACT_CLEAR:
2490                                         /* LINTED - alignment */
2491                                         id = *((dtrace_aggvarid_t *)addr);
2492                                         (void) dtrace_aggregate_walk(dtp,
2493                                             dt_clear_agg, &id);
2494                                         continue;
2495
2496                                 case DT_ACT_DENORMALIZE:
2497                                         /* LINTED - alignment */
2498                                         id = *((dtrace_aggvarid_t *)addr);
2499                                         (void) dtrace_aggregate_walk(dtp,
2500                                             dt_denormalize_agg, &id);
2501                                         continue;
2502
2503                                 case DT_ACT_FTRUNCATE:
2504                                         if (fp == NULL)
2505                                                 continue;
2506
2507                                         (void) fflush(fp);
2508                                         (void) ftruncate(fileno(fp), 0);
2509                                         (void) fseeko(fp, 0, SEEK_SET);
2510                                         continue;
2511
2512                                 case DT_ACT_NORMALIZE:
2513                                         if (i == epd->dtepd_nrecs - 1)
2514                                                 return (dt_set_errno(dtp,
2515                                                     EDT_BADNORMAL));
2516
2517                                         if (dt_normalize(dtp,
2518                                             buf->dtbd_data + offs, rec) != 0)
2519                                                 return (-1);
2520
2521                                         i++;
2522                                         continue;
2523
2524                                 case DT_ACT_SETOPT: {
2525                                         uint64_t *opts = dtp->dt_options;
2526                                         dtrace_recdesc_t *valrec;
2527                                         uint32_t valsize;
2528                                         caddr_t val;
2529                                         int rv;
2530
2531                                         if (i == epd->dtepd_nrecs - 1) {
2532                                                 return (dt_set_errno(dtp,
2533                                                     EDT_BADSETOPT));
2534                                         }
2535
2536                                         valrec = &epd->dtepd_rec[++i];
2537                                         valsize = valrec->dtrd_size;
2538
2539                                         if (valrec->dtrd_action != act ||
2540                                             valrec->dtrd_arg != arg) {
2541                                                 return (dt_set_errno(dtp,
2542                                                     EDT_BADSETOPT));
2543                                         }
2544
2545                                         if (valsize > sizeof (uint64_t)) {
2546                                                 val = buf->dtbd_data + offs +
2547                                                     valrec->dtrd_offset;
2548                                         } else {
2549                                                 val = "1";
2550                                         }
2551
2552                                         rv = dt_setopt(dtp, &data, addr, val);
2553
2554                                         if (rv != 0)
2555                                                 return (-1);
2556
2557                                         flow = (opts[DTRACEOPT_FLOWINDENT] !=
2558                                             DTRACEOPT_UNSET);
2559                                         quiet = (opts[DTRACEOPT_QUIET] !=
2560                                             DTRACEOPT_UNSET);
2561
2562                                         continue;
2563                                 }
2564
2565                                 case DT_ACT_TRUNC:
2566                                         if (i == epd->dtepd_nrecs - 1)
2567                                                 return (dt_set_errno(dtp,
2568                                                     EDT_BADTRUNC));
2569
2570                                         if (dt_trunc(dtp,
2571                                             buf->dtbd_data + offs, rec) != 0)
2572                                                 return (-1);
2573
2574                                         i++;
2575                                         continue;
2576
2577                                 default:
2578                                         continue;
2579                                 }
2580                         }
2581
2582                         if (act == DTRACEACT_TRACEMEM_DYNSIZE &&
2583                             rec->dtrd_size == sizeof (uint64_t)) {
2584                                 /* LINTED - alignment */
2585                                 tracememsize = *((unsigned long long *)addr);
2586                                 continue;
2587                         }
2588
2589                         rval = (*rfunc)(&data, rec, arg);
2590
2591                         if (rval == DTRACE_CONSUME_NEXT)
2592                                 continue;
2593
2594                         if (rval == DTRACE_CONSUME_ABORT)
2595                                 return (dt_set_errno(dtp, EDT_DIRABORT));
2596
2597                         if (rval != DTRACE_CONSUME_THIS)
2598                                 return (dt_set_errno(dtp, EDT_BADRVAL));
2599
2600                         if (act == DTRACEACT_STACK) {
2601                                 int depth = rec->dtrd_arg;
2602
2603                                 if (dt_print_stack(dtp, fp, NULL, addr, depth,
2604                                     rec->dtrd_size / depth) < 0)
2605                                         return (-1);
2606                                 goto nextrec;
2607                         }
2608
2609                         if (act == DTRACEACT_USTACK ||
2610                             act == DTRACEACT_JSTACK) {
2611                                 if (dt_print_ustack(dtp, fp, NULL,
2612                                     addr, rec->dtrd_arg) < 0)
2613                                         return (-1);
2614                                 goto nextrec;
2615                         }
2616
2617                         if (act == DTRACEACT_SYM) {
2618                                 if (dt_print_sym(dtp, fp, NULL, addr) < 0)
2619                                         return (-1);
2620                                 goto nextrec;
2621                         }
2622
2623                         if (act == DTRACEACT_MOD) {
2624                                 if (dt_print_mod(dtp, fp, NULL, addr) < 0)
2625                                         return (-1);
2626                                 goto nextrec;
2627                         }
2628
2629                         if (act == DTRACEACT_USYM || act == DTRACEACT_UADDR) {
2630                                 if (dt_print_usym(dtp, fp, addr, act) < 0)
2631                                         return (-1);
2632                                 goto nextrec;
2633                         }
2634
2635                         if (act == DTRACEACT_UMOD) {
2636                                 if (dt_print_umod(dtp, fp, NULL, addr) < 0)
2637                                         return (-1);
2638                                 goto nextrec;
2639                         }
2640
2641                         if (act == DTRACEACT_PRINTM) {
2642                                 if (dt_print_memory(dtp, fp, addr) < 0)
2643                                         return (-1);
2644                                 goto nextrec;
2645                         }
2646
2647                         if (act == DTRACEACT_PRINTT) {
2648                                 if (dt_print_type(dtp, fp, addr) < 0)
2649                                         return (-1);
2650                                 goto nextrec;
2651                         }
2652
2653                         if (DTRACEACT_ISPRINTFLIKE(act)) {
2654                                 void *fmtdata;
2655                                 int (*func)(dtrace_hdl_t *, FILE *, void *,
2656                                     const dtrace_probedata_t *,
2657                                     const dtrace_recdesc_t *, uint_t,
2658                                     const void *buf, size_t);
2659
2660                                 if ((fmtdata = dt_format_lookup(dtp,
2661                                     rec->dtrd_format)) == NULL)
2662                                         goto nofmt;
2663
2664                                 switch (act) {
2665                                 case DTRACEACT_PRINTF:
2666                                         func = dtrace_fprintf;
2667                                         break;
2668                                 case DTRACEACT_PRINTA:
2669                                         func = dtrace_fprinta;
2670                                         break;
2671                                 case DTRACEACT_SYSTEM:
2672                                         func = dtrace_system;
2673                                         break;
2674                                 case DTRACEACT_FREOPEN:
2675                                         func = dtrace_freopen;
2676                                         break;
2677                                 }
2678
2679                                 n = (*func)(dtp, fp, fmtdata, &data,
2680                                     rec, epd->dtepd_nrecs - i,
2681                                     (uchar_t *)buf->dtbd_data + offs,
2682                                     buf->dtbd_size - offs);
2683
2684                                 if (n < 0)
2685                                         return (-1); /* errno is set for us */
2686
2687                                 if (n > 0)
2688                                         i += n - 1;
2689                                 goto nextrec;
2690                         }
2691
2692                         /*
2693                          * If this is a DIF expression, and the record has a
2694                          * format set, this indicates we have a CTF type name
2695                          * associated with the data and we should try to print
2696                          * it out by type.
2697                          */
2698                         if (act == DTRACEACT_DIFEXPR) {
2699                                 const char *strdata = dt_strdata_lookup(dtp,
2700                                     rec->dtrd_format);
2701                                 if (strdata != NULL) {
2702                                         n = dtrace_print(dtp, fp, strdata,
2703                                             addr, rec->dtrd_size);
2704
2705                                         /*
2706                                          * dtrace_print() will return -1 on
2707                                          * error, or return the number of bytes
2708                                          * consumed.  It will return 0 if the
2709                                          * type couldn't be determined, and we
2710                                          * should fall through to the normal
2711                                          * trace method.
2712                                          */
2713                                         if (n < 0)
2714                                                 return (-1);
2715
2716                                         if (n > 0)
2717                                                 goto nextrec;
2718                                 }
2719                         }
2720
2721 nofmt:
2722                         if (act == DTRACEACT_PRINTA) {
2723                                 dt_print_aggdata_t pd;
2724                                 dtrace_aggvarid_t *aggvars;
2725                                 int j, naggvars = 0;
2726                                 size_t size = ((epd->dtepd_nrecs - i) *
2727                                     sizeof (dtrace_aggvarid_t));
2728
2729                                 if ((aggvars = dt_alloc(dtp, size)) == NULL)
2730                                         return (-1);
2731
2732                                 /*
2733                                  * This might be a printa() with multiple
2734                                  * aggregation variables.  We need to scan
2735                                  * forward through the records until we find
2736                                  * a record from a different statement.
2737                                  */
2738                                 for (j = i; j < epd->dtepd_nrecs; j++) {
2739                                         dtrace_recdesc_t *nrec;
2740                                         caddr_t naddr;
2741
2742                                         nrec = &epd->dtepd_rec[j];
2743
2744                                         if (nrec->dtrd_uarg != rec->dtrd_uarg)
2745                                                 break;
2746
2747                                         if (nrec->dtrd_action != act) {
2748                                                 return (dt_set_errno(dtp,
2749                                                     EDT_BADAGG));
2750                                         }
2751
2752                                         naddr = buf->dtbd_data + offs +
2753                                             nrec->dtrd_offset;
2754
2755                                         aggvars[naggvars++] =
2756                                             /* LINTED - alignment */
2757                                             *((dtrace_aggvarid_t *)naddr);
2758                                 }
2759
2760                                 i = j - 1;
2761                                 bzero(&pd, sizeof (pd));
2762                                 pd.dtpa_dtp = dtp;
2763                                 pd.dtpa_fp = fp;
2764
2765                                 assert(naggvars >= 1);
2766
2767                                 if (naggvars == 1) {
2768                                         pd.dtpa_id = aggvars[0];
2769                                         dt_free(dtp, aggvars);
2770
2771                                         if (dt_printf(dtp, fp, "\n") < 0 ||
2772                                             dtrace_aggregate_walk_sorted(dtp,
2773                                             dt_print_agg, &pd) < 0)
2774                                                 return (-1);
2775                                         goto nextrec;
2776                                 }
2777
2778                                 if (dt_printf(dtp, fp, "\n") < 0 ||
2779                                     dtrace_aggregate_walk_joined(dtp, aggvars,
2780                                     naggvars, dt_print_aggs, &pd) < 0) {
2781                                         dt_free(dtp, aggvars);
2782                                         return (-1);
2783                                 }
2784
2785                                 dt_free(dtp, aggvars);
2786                                 goto nextrec;
2787                         }
2788
2789                         if (act == DTRACEACT_TRACEMEM) {
2790                                 if (tracememsize == 0 ||
2791                                     tracememsize > rec->dtrd_size) {
2792                                         tracememsize = rec->dtrd_size;
2793                                 }
2794
2795                                 n = dt_print_bytes(dtp, fp, addr,
2796                                     tracememsize, -33, quiet, 1);
2797
2798                                 tracememsize = 0;
2799
2800                                 if (n < 0)
2801                                         return (-1);
2802
2803                                 goto nextrec;
2804                         }
2805
2806                         switch (rec->dtrd_size) {
2807                         case sizeof (uint64_t):
2808                                 n = dt_printf(dtp, fp,
2809                                     quiet ? "%lld" : " %16lld",
2810                                     /* LINTED - alignment */
2811                                     *((unsigned long long *)addr));
2812                                 break;
2813                         case sizeof (uint32_t):
2814                                 n = dt_printf(dtp, fp, quiet ? "%d" : " %8d",
2815                                     /* LINTED - alignment */
2816                                     *((uint32_t *)addr));
2817                                 break;
2818                         case sizeof (uint16_t):
2819                                 n = dt_printf(dtp, fp, quiet ? "%d" : " %5d",
2820                                     /* LINTED - alignment */
2821                                     *((uint16_t *)addr));
2822                                 break;
2823                         case sizeof (uint8_t):
2824                                 n = dt_printf(dtp, fp, quiet ? "%d" : " %3d",
2825                                     *((uint8_t *)addr));
2826                                 break;
2827                         default:
2828                                 n = dt_print_bytes(dtp, fp, addr,
2829                                     rec->dtrd_size, -33, quiet, 0);
2830                                 break;
2831                         }
2832
2833                         if (n < 0)
2834                                 return (-1); /* errno is set for us */
2835
2836 nextrec:
2837                         if (dt_buffered_flush(dtp, &data, rec, NULL, 0) < 0)
2838                                 return (-1); /* errno is set for us */
2839                 }
2840
2841                 /*
2842                  * Call the record callback with a NULL record to indicate
2843                  * that we're done processing this EPID.
2844                  */
2845                 rval = (*rfunc)(&data, NULL, arg);
2846 nextepid:
2847                 offs += epd->dtepd_size;
2848                 dtp->dt_last_epid = id;
2849                 if (just_one) {
2850                         buf->dtbd_oldest = offs;
2851                         break;
2852                 }
2853         }
2854
2855         dtp->dt_flow = data.dtpda_flow;
2856         dtp->dt_indent = data.dtpda_indent;
2857         dtp->dt_prefix = data.dtpda_prefix;
2858
2859         if ((drops = buf->dtbd_drops) == 0)
2860                 return (0);
2861
2862         /*
2863          * Explicitly zero the drops to prevent us from processing them again.
2864          */
2865         buf->dtbd_drops = 0;
2866
2867         return (dt_handle_cpudrop(dtp, cpu, DTRACEDROP_PRINCIPAL, drops));
2868 }
2869
2870 /*
2871  * Reduce memory usage by shrinking the buffer if it's no more than half full.
2872  * Note, we need to preserve the alignment of the data at dtbd_oldest, which is
2873  * only 4-byte aligned.
2874  */
2875 static void
2876 dt_realloc_buf(dtrace_hdl_t *dtp, dtrace_bufdesc_t *buf, int cursize)
2877 {
2878         uint64_t used = buf->dtbd_size - buf->dtbd_oldest;
2879         if (used < cursize / 2) {
2880                 int misalign = buf->dtbd_oldest & (sizeof (uint64_t) - 1);
2881                 char *newdata = dt_alloc(dtp, used + misalign);
2882                 if (newdata == NULL)
2883                         return;
2884                 bzero(newdata, misalign);
2885                 bcopy(buf->dtbd_data + buf->dtbd_oldest,
2886                     newdata + misalign, used);
2887                 dt_free(dtp, buf->dtbd_data);
2888                 buf->dtbd_oldest = misalign;
2889                 buf->dtbd_size = used + misalign;
2890                 buf->dtbd_data = newdata;
2891         }
2892 }
2893
2894 /*
2895  * If the ring buffer has wrapped, the data is not in order.  Rearrange it
2896  * so that it is.  Note, we need to preserve the alignment of the data at
2897  * dtbd_oldest, which is only 4-byte aligned.
2898  */
2899 static int
2900 dt_unring_buf(dtrace_hdl_t *dtp, dtrace_bufdesc_t *buf)
2901 {
2902         int misalign;
2903         char *newdata, *ndp;
2904
2905         if (buf->dtbd_oldest == 0)
2906                 return (0);
2907
2908         misalign = buf->dtbd_oldest & (sizeof (uint64_t) - 1);
2909         newdata = ndp = dt_alloc(dtp, buf->dtbd_size + misalign);
2910
2911         if (newdata == NULL)
2912                 return (-1);
2913
2914         assert(0 == (buf->dtbd_size & (sizeof (uint64_t) - 1)));
2915
2916         bzero(ndp, misalign);
2917         ndp += misalign;
2918
2919         bcopy(buf->dtbd_data + buf->dtbd_oldest, ndp,
2920             buf->dtbd_size - buf->dtbd_oldest);
2921         ndp += buf->dtbd_size - buf->dtbd_oldest;
2922
2923         bcopy(buf->dtbd_data, ndp, buf->dtbd_oldest);
2924
2925         dt_free(dtp, buf->dtbd_data);
2926         buf->dtbd_oldest = 0;
2927         buf->dtbd_data = newdata;
2928         buf->dtbd_size += misalign;
2929
2930         return (0);
2931 }
2932
2933 static void
2934 dt_put_buf(dtrace_hdl_t *dtp, dtrace_bufdesc_t *buf)
2935 {
2936         dt_free(dtp, buf->dtbd_data);
2937         dt_free(dtp, buf);
2938 }
2939
2940 /*
2941  * Returns 0 on success, in which case *cbp will be filled in if we retrieved
2942  * data, or NULL if there is no data for this CPU.
2943  * Returns -1 on failure and sets dt_errno.
2944  */
2945 static int
2946 dt_get_buf(dtrace_hdl_t *dtp, int cpu, dtrace_bufdesc_t **bufp)
2947 {
2948         dtrace_optval_t size;
2949         dtrace_bufdesc_t *buf = dt_zalloc(dtp, sizeof (*buf));
2950         int error, rval;
2951
2952         if (buf == NULL)
2953                 return (-1);
2954
2955         (void) dtrace_getopt(dtp, "bufsize", &size);
2956         buf->dtbd_data = dt_alloc(dtp, size);
2957         if (buf->dtbd_data == NULL) {
2958                 dt_free(dtp, buf);
2959                 return (-1);
2960         }
2961         buf->dtbd_size = size;
2962         buf->dtbd_cpu = cpu;
2963
2964 #if defined(sun)
2965         if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, buf) == -1) {
2966 #else
2967         if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, &buf) == -1) {
2968 #endif
2969                 /*
2970                  * If we failed with ENOENT, it may be because the
2971                  * CPU was unconfigured -- this is okay.  Any other
2972                  * error, however, is unexpected.
2973                  */
2974                 if (errno == ENOENT) {
2975                         *bufp = NULL;
2976                         rval = 0;
2977                 } else
2978                         rval = dt_set_errno(dtp, errno);
2979
2980                 dt_put_buf(dtp, buf);
2981                 return (rval);
2982         }
2983
2984         error = dt_unring_buf(dtp, buf);
2985         if (error != 0) {
2986                 dt_put_buf(dtp, buf);
2987                 return (error);
2988         }
2989         dt_realloc_buf(dtp, buf, size);
2990
2991         *bufp = buf;
2992         return (0);
2993 }
2994
2995 typedef struct dt_begin {
2996         dtrace_consume_probe_f *dtbgn_probefunc;
2997         dtrace_consume_rec_f *dtbgn_recfunc;
2998         void *dtbgn_arg;
2999         dtrace_handle_err_f *dtbgn_errhdlr;
3000         void *dtbgn_errarg;
3001         int dtbgn_beginonly;
3002 } dt_begin_t;
3003
3004 static int
3005 dt_consume_begin_probe(const dtrace_probedata_t *data, void *arg)
3006 {
3007         dt_begin_t *begin = arg;
3008         dtrace_probedesc_t *pd = data->dtpda_pdesc;
3009
3010         int r1 = (strcmp(pd->dtpd_provider, "dtrace") == 0);
3011         int r2 = (strcmp(pd->dtpd_name, "BEGIN") == 0);
3012
3013         if (begin->dtbgn_beginonly) {
3014                 if (!(r1 && r2))
3015                         return (DTRACE_CONSUME_NEXT);
3016         } else {
3017                 if (r1 && r2)
3018                         return (DTRACE_CONSUME_NEXT);
3019         }
3020
3021         /*
3022          * We have a record that we're interested in.  Now call the underlying
3023          * probe function...
3024          */
3025         return (begin->dtbgn_probefunc(data, begin->dtbgn_arg));
3026 }
3027
3028 static int
3029 dt_consume_begin_record(const dtrace_probedata_t *data,
3030     const dtrace_recdesc_t *rec, void *arg)
3031 {
3032         dt_begin_t *begin = arg;
3033
3034         return (begin->dtbgn_recfunc(data, rec, begin->dtbgn_arg));
3035 }
3036
3037 static int
3038 dt_consume_begin_error(const dtrace_errdata_t *data, void *arg)
3039 {
3040         dt_begin_t *begin = (dt_begin_t *)arg;
3041         dtrace_probedesc_t *pd = data->dteda_pdesc;
3042
3043         int r1 = (strcmp(pd->dtpd_provider, "dtrace") == 0);
3044         int r2 = (strcmp(pd->dtpd_name, "BEGIN") == 0);
3045
3046         if (begin->dtbgn_beginonly) {
3047                 if (!(r1 && r2))
3048                         return (DTRACE_HANDLE_OK);
3049         } else {
3050                 if (r1 && r2)
3051                         return (DTRACE_HANDLE_OK);
3052         }
3053
3054         return (begin->dtbgn_errhdlr(data, begin->dtbgn_errarg));
3055 }
3056
3057 static int
3058 dt_consume_begin(dtrace_hdl_t *dtp, FILE *fp,
3059     dtrace_consume_probe_f *pf, dtrace_consume_rec_f *rf, void *arg)
3060 {
3061         /*
3062          * There's this idea that the BEGIN probe should be processed before
3063          * everything else, and that the END probe should be processed after
3064          * anything else.  In the common case, this is pretty easy to deal
3065          * with.  However, a situation may arise where the BEGIN enabling and
3066          * END enabling are on the same CPU, and some enabling in the middle
3067          * occurred on a different CPU.  To deal with this (blech!) we need to
3068          * consume the BEGIN buffer up until the end of the BEGIN probe, and
3069          * then set it aside.  We will then process every other CPU, and then
3070          * we'll return to the BEGIN CPU and process the rest of the data
3071          * (which will inevitably include the END probe, if any).  Making this
3072          * even more complicated (!) is the library's ERROR enabling.  Because
3073          * this enabling is processed before we even get into the consume call
3074          * back, any ERROR firing would result in the library's ERROR enabling
3075          * being processed twice -- once in our first pass (for BEGIN probes),
3076          * and again in our second pass (for everything but BEGIN probes).  To
3077          * deal with this, we interpose on the ERROR handler to assure that we
3078          * only process ERROR enablings induced by BEGIN enablings in the
3079          * first pass, and that we only process ERROR enablings _not_ induced
3080          * by BEGIN enablings in the second pass.
3081          */
3082
3083         dt_begin_t begin;
3084         processorid_t cpu = dtp->dt_beganon;
3085         int rval, i;
3086         static int max_ncpus;
3087         dtrace_bufdesc_t *buf;
3088
3089         dtp->dt_beganon = -1;
3090
3091         if (dt_get_buf(dtp, cpu, &buf) != 0)
3092                 return (-1);
3093         if (buf == NULL)
3094                 return (0);
3095
3096         if (!dtp->dt_stopped || buf->dtbd_cpu != dtp->dt_endedon) {
3097                 /*
3098                  * This is the simple case.  We're either not stopped, or if
3099                  * we are, we actually processed any END probes on another
3100                  * CPU.  We can simply consume this buffer and return.
3101                  */
3102                 rval = dt_consume_cpu(dtp, fp, cpu, buf, B_FALSE,
3103                     pf, rf, arg);
3104                 dt_put_buf(dtp, buf);
3105                 return (rval);
3106         }
3107
3108         begin.dtbgn_probefunc = pf;
3109         begin.dtbgn_recfunc = rf;
3110         begin.dtbgn_arg = arg;
3111         begin.dtbgn_beginonly = 1;
3112
3113         /*
3114          * We need to interpose on the ERROR handler to be sure that we
3115          * only process ERRORs induced by BEGIN.
3116          */
3117         begin.dtbgn_errhdlr = dtp->dt_errhdlr;
3118         begin.dtbgn_errarg = dtp->dt_errarg;
3119         dtp->dt_errhdlr = dt_consume_begin_error;
3120         dtp->dt_errarg = &begin;
3121
3122         rval = dt_consume_cpu(dtp, fp, cpu, buf, B_FALSE,
3123             dt_consume_begin_probe, dt_consume_begin_record, &begin);
3124
3125         dtp->dt_errhdlr = begin.dtbgn_errhdlr;
3126         dtp->dt_errarg = begin.dtbgn_errarg;
3127
3128         if (rval != 0) {
3129                 dt_put_buf(dtp, buf);
3130                 return (rval);
3131         }
3132
3133         if (max_ncpus == 0)
3134                 max_ncpus = dt_sysconf(dtp, _SC_CPUID_MAX) + 1;
3135
3136         for (i = 0; i < max_ncpus; i++) {
3137                 dtrace_bufdesc_t *nbuf;
3138                 if (i == cpu)
3139                         continue;
3140
3141                 if (dt_get_buf(dtp, i, &nbuf) != 0) {
3142                         dt_put_buf(dtp, buf);
3143                         return (-1);
3144                 }
3145                 if (nbuf == NULL)
3146                         continue;
3147
3148                 rval = dt_consume_cpu(dtp, fp, i, nbuf, B_FALSE,
3149                     pf, rf, arg);
3150                 dt_put_buf(dtp, nbuf);
3151                 if (rval != 0) {
3152                         dt_put_buf(dtp, buf);
3153                         return (rval);
3154                 }
3155         }
3156
3157         /*
3158          * Okay -- we're done with the other buffers.  Now we want to
3159          * reconsume the first buffer -- but this time we're looking for
3160          * everything _but_ BEGIN.  And of course, in order to only consume
3161          * those ERRORs _not_ associated with BEGIN, we need to reinstall our
3162          * ERROR interposition function...
3163          */
3164         begin.dtbgn_beginonly = 0;
3165
3166         assert(begin.dtbgn_errhdlr == dtp->dt_errhdlr);
3167         assert(begin.dtbgn_errarg == dtp->dt_errarg);
3168         dtp->dt_errhdlr = dt_consume_begin_error;
3169         dtp->dt_errarg = &begin;
3170
3171         rval = dt_consume_cpu(dtp, fp, cpu, buf, B_FALSE,
3172             dt_consume_begin_probe, dt_consume_begin_record, &begin);
3173
3174         dtp->dt_errhdlr = begin.dtbgn_errhdlr;
3175         dtp->dt_errarg = begin.dtbgn_errarg;
3176
3177         return (rval);
3178 }
3179
3180 /* ARGSUSED */
3181 static uint64_t
3182 dt_buf_oldest(void *elem, void *arg)
3183 {
3184         dtrace_bufdesc_t *buf = elem;
3185         size_t offs = buf->dtbd_oldest;
3186
3187         while (offs < buf->dtbd_size) {
3188                 dtrace_rechdr_t *dtrh =
3189                     /* LINTED - alignment */
3190                     (dtrace_rechdr_t *)(buf->dtbd_data + offs);
3191                 if (dtrh->dtrh_epid == DTRACE_EPIDNONE) {
3192                         offs += sizeof (dtrace_epid_t);
3193                 } else {
3194                         return (DTRACE_RECORD_LOAD_TIMESTAMP(dtrh));
3195                 }
3196         }
3197
3198         /* There are no records left; use the time the buffer was retrieved. */
3199         return (buf->dtbd_timestamp);
3200 }
3201
3202 int
3203 dtrace_consume(dtrace_hdl_t *dtp, FILE *fp,
3204     dtrace_consume_probe_f *pf, dtrace_consume_rec_f *rf, void *arg)
3205 {
3206         dtrace_optval_t size;
3207         static int max_ncpus;
3208         int i, rval;
3209         dtrace_optval_t interval = dtp->dt_options[DTRACEOPT_SWITCHRATE];
3210         hrtime_t now = gethrtime();
3211
3212         if (dtp->dt_lastswitch != 0) {
3213                 if (now - dtp->dt_lastswitch < interval)
3214                         return (0);
3215
3216                 dtp->dt_lastswitch += interval;
3217         } else {
3218                 dtp->dt_lastswitch = now;
3219         }
3220
3221         if (!dtp->dt_active)
3222                 return (dt_set_errno(dtp, EINVAL));
3223
3224         if (max_ncpus == 0)
3225                 max_ncpus = dt_sysconf(dtp, _SC_CPUID_MAX) + 1;
3226
3227         if (pf == NULL)
3228                 pf = (dtrace_consume_probe_f *)dt_nullprobe;
3229
3230         if (rf == NULL)
3231                 rf = (dtrace_consume_rec_f *)dt_nullrec;
3232
3233         if (dtp->dt_options[DTRACEOPT_TEMPORAL] == DTRACEOPT_UNSET) {
3234                 /*
3235                  * The output will not be in the order it was traced.  Rather,
3236                  * we will consume all of the data from each CPU's buffer in
3237                  * turn.  We apply special handling for the records from BEGIN
3238                  * and END probes so that they are consumed first and last,
3239                  * respectively.
3240                  *
3241                  * If we have just begun, we want to first process the CPU that
3242                  * executed the BEGIN probe (if any).
3243                  */
3244                 if (dtp->dt_active && dtp->dt_beganon != -1 &&
3245                     (rval = dt_consume_begin(dtp, fp, pf, rf, arg)) != 0)
3246                         return (rval);
3247
3248                 for (i = 0; i < max_ncpus; i++) {
3249                         dtrace_bufdesc_t *buf;
3250
3251                         /*
3252                          * If we have stopped, we want to process the CPU on
3253                          * which the END probe was processed only _after_ we
3254                          * have processed everything else.
3255                          */
3256                         if (dtp->dt_stopped && (i == dtp->dt_endedon))
3257                                 continue;
3258
3259                         if (dt_get_buf(dtp, i, &buf) != 0)
3260                                 return (-1);
3261                         if (buf == NULL)
3262                                 continue;
3263
3264                         dtp->dt_flow = 0;
3265                         dtp->dt_indent = 0;
3266                         dtp->dt_prefix = NULL;
3267                         rval = dt_consume_cpu(dtp, fp, i,
3268                             buf, B_FALSE, pf, rf, arg);
3269                         dt_put_buf(dtp, buf);
3270                         if (rval != 0)
3271                                 return (rval);
3272                 }
3273                 if (dtp->dt_stopped) {
3274                         dtrace_bufdesc_t *buf;
3275
3276                         if (dt_get_buf(dtp, dtp->dt_endedon, &buf) != 0)
3277                                 return (-1);
3278                         if (buf == NULL)
3279                                 return (0);
3280
3281                         rval = dt_consume_cpu(dtp, fp, dtp->dt_endedon,
3282                             buf, B_FALSE, pf, rf, arg);
3283                         dt_put_buf(dtp, buf);
3284                         return (rval);
3285                 }
3286         } else {
3287                 /*
3288                  * The output will be in the order it was traced (or for
3289                  * speculations, when it was committed).  We retrieve a buffer
3290                  * from each CPU and put it into a priority queue, which sorts
3291                  * based on the first entry in the buffer.  This is sufficient
3292                  * because entries within a buffer are already sorted.
3293                  *
3294                  * We then consume records one at a time, always consuming the
3295                  * oldest record, as determined by the priority queue.  When
3296                  * we reach the end of the time covered by these buffers,
3297                  * we need to stop and retrieve more records on the next pass.
3298                  * The kernel tells us the time covered by each buffer, in
3299                  * dtbd_timestamp.  The first buffer's timestamp tells us the
3300                  * time covered by all buffers, as subsequently retrieved
3301                  * buffers will cover to a more recent time.
3302                  */
3303
3304                 uint64_t *drops = alloca(max_ncpus * sizeof (uint64_t));
3305                 uint64_t first_timestamp = 0;
3306                 uint_t cookie = 0;
3307                 dtrace_bufdesc_t *buf;
3308
3309                 bzero(drops, max_ncpus * sizeof (uint64_t));
3310
3311                 if (dtp->dt_bufq == NULL) {
3312                         dtp->dt_bufq = dt_pq_init(dtp, max_ncpus * 2,
3313                             dt_buf_oldest, NULL);
3314                         if (dtp->dt_bufq == NULL) /* ENOMEM */
3315                                 return (-1);
3316                 }
3317
3318                 /* Retrieve data from each CPU. */
3319                 (void) dtrace_getopt(dtp, "bufsize", &size);
3320                 for (i = 0; i < max_ncpus; i++) {
3321                         dtrace_bufdesc_t *buf;
3322
3323                         if (dt_get_buf(dtp, i, &buf) != 0)
3324                                 return (-1);
3325                         if (buf != NULL) {
3326                                 if (first_timestamp == 0)
3327                                         first_timestamp = buf->dtbd_timestamp;
3328                                 assert(buf->dtbd_timestamp >= first_timestamp);
3329
3330                                 dt_pq_insert(dtp->dt_bufq, buf);
3331                                 drops[i] = buf->dtbd_drops;
3332                                 buf->dtbd_drops = 0;
3333                         }
3334                 }
3335
3336                 /* Consume records. */
3337                 for (;;) {
3338                         dtrace_bufdesc_t *buf = dt_pq_pop(dtp->dt_bufq);
3339                         uint64_t timestamp;
3340
3341                         if (buf == NULL)
3342                                 break;
3343
3344                         timestamp = dt_buf_oldest(buf, dtp);
3345                         assert(timestamp >= dtp->dt_last_timestamp);
3346                         dtp->dt_last_timestamp = timestamp;
3347
3348                         if (timestamp == buf->dtbd_timestamp) {
3349                                 /*
3350                                  * We've reached the end of the time covered
3351                                  * by this buffer.  If this is the oldest
3352                                  * buffer, we must do another pass
3353                                  * to retrieve more data.
3354                                  */
3355                                 dt_put_buf(dtp, buf);
3356                                 if (timestamp == first_timestamp &&
3357                                     !dtp->dt_stopped)
3358                                         break;
3359                                 continue;
3360                         }
3361
3362                         if ((rval = dt_consume_cpu(dtp, fp,
3363                             buf->dtbd_cpu, buf, B_TRUE, pf, rf, arg)) != 0)
3364                                 return (rval);
3365                         dt_pq_insert(dtp->dt_bufq, buf);
3366                 }
3367
3368                 /* Consume drops. */
3369                 for (i = 0; i < max_ncpus; i++) {
3370                         if (drops[i] != 0) {
3371                                 int error = dt_handle_cpudrop(dtp, i,
3372                                     DTRACEDROP_PRINCIPAL, drops[i]);
3373                                 if (error != 0)
3374                                         return (error);
3375                         }
3376                 }
3377
3378                 /*
3379                  * Reduce memory usage by re-allocating smaller buffers
3380                  * for the "remnants".
3381                  */
3382                 while (buf = dt_pq_walk(dtp->dt_bufq, &cookie))
3383                         dt_realloc_buf(dtp, buf, buf->dtbd_size);
3384         }
3385
3386         return (0);
3387 }