sys/kern/subr_stats.c

   1 /*-
   2  * Copyright (c) 2014-2018 Netflix, Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  * $FreeBSD$
  27  */
  28
  29 /*
  30  * Author: Lawrence Stewart <lstewart@netflix.com>
  31  */
  32
  33 #include <sys/cdefs.h>
  34 __FBSDID("$FreeBSD$");
  35
  36 #include <sys/param.h>
  37 #include <sys/arb.h>
  38 #include <sys/ctype.h>
  39 #include <sys/errno.h>
  40 #include <sys/hash.h>
  41 #include <sys/limits.h>
  42 #include <sys/malloc.h>
  43 #include <sys/qmath.h>
  44 #include <sys/sbuf.h>
  45 #if defined(DIAGNOSTIC)
  46 #include <sys/tree.h>
  47 #endif
  48 #include <sys/stats.h> /* Must come after qmath.h and arb.h */
  49 #include <sys/stddef.h>
  50 #include <sys/stdint.h>
  51 #include <sys/time.h>
  52
  53 #ifdef _KERNEL
  54 #include <sys/kernel.h>
  55 #include <sys/lock.h>
  56 #include <sys/rwlock.h>
  57 #include <sys/sysctl.h>
  58 #include <sys/systm.h>
  59 #else /* ! _KERNEL */
  60 #include <pthread.h>
  61 #include <stdbool.h>
  62 #include <stdio.h>
  63 #include <stdlib.h>
  64 #include <string.h>
  65 #endif /* _KERNEL */
  66
  67 struct voistatdata_voistate {
  68         /* Previous VOI value for diff calculation. */
  69         struct voistatdata_numeric prev;
  70 };
  71
  72 #define VS_VSDVALID     0x0001  /* Stat's voistatdata updated at least once. */
  73 struct voistat {
  74         int8_t          stype;          /* Type of stat e.g. VS_STYPE_SUM. */
  75         enum vsd_dtype  dtype : 8;      /* Data type of this stat's data. */
  76         uint16_t        data_off;       /* Blob offset for this stat's data. */
  77         uint16_t        dsz;            /* Size of stat's data. */
  78 #define VS_EBITS 8
  79         uint16_t        errs : VS_EBITS;/* Non-wrapping error count. */
  80         uint16_t        flags : 16 - VS_EBITS;
  81 };
  82 /* The voistat error count is capped to avoid wrapping. */
  83 #define VS_INCERRS(vs) do {                                             \
  84         if ((vs)->errs < (1U << VS_EBITS) - 1)                          \
  85                 (vs)->errs++;                                           \
  86 } while (0)
  87
  88 /*
  89  * Ideas for flags:
  90  *   - Global or entity specific (global would imply use of counter(9)?)
  91  *   - Whether to reset stats on read or not
  92  *   - Signal an overflow?
  93  *   - Compressed voistat array
  94  */
  95 #define VOI_REQSTATE    0x0001  /* VOI requires VS_STYPE_VOISTATE. */
  96 struct voi {
  97         int16_t         id;             /* VOI id. */
  98         enum vsd_dtype  dtype : 8;      /* Data type of the VOI itself. */
  99         int8_t          voistatmaxid;   /* Largest allocated voistat index. */
 100         uint16_t        stats_off;      /* Blob offset for this VOIs stats. */
 101         uint16_t        flags;
 102 };
 103
 104 /*
 105  * Memory for the entire blob is allocated as a slab and then offsets are
 106  * maintained to carve up the slab into sections holding different data types.
 107  *
 108  * Ideas for flags:
 109  * - Compressed voi array (trade off memory usage vs search time)
 110  * - Units of offsets (default bytes, flag for e.g. vm_page/KiB/Mib)
 111  */
 112 struct statsblobv1 {
 113         uint8_t         abi;
 114         uint8_t         endian;
 115         uint16_t        flags;
 116         uint16_t        maxsz;
 117         uint16_t        cursz;
 118         /* Fields from here down are opaque to consumers. */
 119         uint32_t        tplhash;        /* Base template hash ID. */
 120         uint16_t        stats_off;      /* voistat array blob offset. */
 121         uint16_t        statsdata_off;  /* voistatdata array blob offset. */
 122         sbintime_t      created;        /* Blob creation time. */
 123         sbintime_t      lastrst;        /* Time of last reset. */
 124         struct voi      vois[];         /* Array indexed by [voi_id]. */
 125 } __aligned(sizeof(void *));
 126 _Static_assert(offsetof(struct statsblobv1, cursz) +
 127     SIZEOF_MEMBER(struct statsblobv1, cursz) == sizeof(struct statsblob),
 128     "statsblobv1 ABI mismatch");
 129
 130 struct statsblobv1_tpl {
 131         struct metablob         *mb;
 132         struct statsblobv1      *sb;
 133 };
 134
 135 /* Context passed to iterator callbacks. */
 136 struct sb_iter_ctx {
 137         void            *usrctx;        /* Caller supplied context. */
 138         uint32_t        flags;          /* Flags for current iteration. */
 139         int16_t         vslot;          /* struct voi slot index. */
 140         int8_t          vsslot;         /* struct voistat slot index. */
 141 };
 142
 143 struct sb_tostrcb_ctx {
 144         struct sbuf             *buf;
 145         struct statsblob_tpl    *tpl;
 146         enum sb_str_fmt fmt;
 147         uint32_t                flags;
 148 };
 149
 150 struct sb_visitcb_ctx {
 151         stats_blob_visitcb_t    cb;
 152         void                    *usrctx;
 153 };
 154
 155 /* Stats blob iterator callback. */
 156 typedef int (*stats_v1_blob_itercb_t)(struct statsblobv1 *sb, struct voi *v,
 157     struct voistat *vs, struct sb_iter_ctx *ctx);
 158
 159 #ifdef _KERNEL
 160 static struct rwlock tpllistlock;
 161 RW_SYSINIT(stats_tpl_list, &tpllistlock, "Stat template list lock");
 162 #define TPL_LIST_RLOCK() rw_rlock(&tpllistlock)
 163 #define TPL_LIST_RUNLOCK() rw_runlock(&tpllistlock)
 164 #define TPL_LIST_WLOCK() rw_wlock(&tpllistlock)
 165 #define TPL_LIST_WUNLOCK() rw_wunlock(&tpllistlock)
 166 #define TPL_LIST_LOCK_ASSERT() rw_assert(&tpllistlock, RA_LOCKED)
 167 #define TPL_LIST_RLOCK_ASSERT() rw_assert(&tpllistlock, RA_RLOCKED)
 168 #define TPL_LIST_WLOCK_ASSERT() rw_assert(&tpllistlock, RA_WLOCKED)
 169 MALLOC_DEFINE(M_STATS, "stats(9) related memory", "stats(9) related memory");
 170 #define stats_free(ptr) free((ptr), M_STATS)
 171 #else /* ! _KERNEL */
 172 static void stats_constructor(void);
 173 static void stats_destructor(void);
 174 static pthread_rwlock_t tpllistlock;
 175 #define TPL_LIST_UNLOCK() pthread_rwlock_unlock(&tpllistlock)
 176 #define TPL_LIST_RLOCK() pthread_rwlock_rdlock(&tpllistlock)
 177 #define TPL_LIST_RUNLOCK() TPL_LIST_UNLOCK()
 178 #define TPL_LIST_WLOCK() pthread_rwlock_wrlock(&tpllistlock)
 179 #define TPL_LIST_WUNLOCK() TPL_LIST_UNLOCK()
 180 #define TPL_LIST_LOCK_ASSERT() do { } while (0)
 181 #define TPL_LIST_RLOCK_ASSERT() do { } while (0)
 182 #define TPL_LIST_WLOCK_ASSERT() do { } while (0)
 183 #ifdef NDEBUG
 184 #define KASSERT(cond, msg) do {} while (0)
 185 #define stats_abort() do {} while (0)
 186 #else /* ! NDEBUG */
 187 #define KASSERT(cond, msg) do { \
 188         if (!(cond)) { \
 189                 panic msg; \
 190         } \
 191 } while (0)
 192 #define stats_abort() abort()
 193 #endif /* NDEBUG */
 194 #define stats_free(ptr) free(ptr)
 195 #define panic(fmt, ...) do { \
 196         fprintf(stderr, (fmt), ##__VA_ARGS__); \
 197         stats_abort(); \
 198 } while (0)
 199 #endif /* _KERNEL */
 200
 201 #define SB_V1_MAXSZ 65535
 202
 203 /* Obtain a blob offset pointer. */
 204 #define BLOB_OFFSET(sb, off) ((void *)(((uint8_t *)(sb)) + (off)))
 205
 206 /*
 207  * Number of VOIs in the blob's vois[] array. By virtue of struct voi being a
 208  * power of 2 size, we can shift instead of divide. The shift amount must be
 209  * updated if sizeof(struct voi) ever changes, which the assert should catch.
 210  */
 211 #define NVOIS(sb) ((int32_t)((((struct statsblobv1 *)(sb))->stats_off - \
 212     sizeof(struct statsblobv1)) >> 3))
 213 _Static_assert(sizeof(struct voi) == 8, "statsblobv1 voi ABI mismatch");
 214
 215 /* Try restrict names to alphanumeric and underscore to simplify JSON compat. */
 216 const char *vs_stype2name[VS_NUM_STYPES] = {
 217         [VS_STYPE_VOISTATE] = "VOISTATE",
 218         [VS_STYPE_SUM] = "SUM",
 219         [VS_STYPE_MAX] = "MAX",
 220         [VS_STYPE_MIN] = "MIN",
 221         [VS_STYPE_HIST] = "HIST",
 222         [VS_STYPE_TDGST] = "TDGST",
 223 };
 224
 225 const char *vs_stype2desc[VS_NUM_STYPES] = {
 226         [VS_STYPE_VOISTATE] = "VOI related state data (not a real stat)",
 227         [VS_STYPE_SUM] = "Simple arithmetic accumulator",
 228         [VS_STYPE_MAX] = "Maximum observed VOI value",
 229         [VS_STYPE_MIN] = "Minimum observed VOI value",
 230         [VS_STYPE_HIST] = "Histogram of observed VOI values",
 231         [VS_STYPE_TDGST] = "t-digest of observed VOI values",
 232 };
 233
 234 const char *vsd_dtype2name[VSD_NUM_DTYPES] = {
 235         [VSD_DTYPE_VOISTATE] = "VOISTATE",
 236         [VSD_DTYPE_INT_S32] = "INT_S32",
 237         [VSD_DTYPE_INT_U32] = "INT_U32",
 238         [VSD_DTYPE_INT_S64] = "INT_S64",
 239         [VSD_DTYPE_INT_U64] = "INT_U64",
 240         [VSD_DTYPE_INT_SLONG] = "INT_SLONG",
 241         [VSD_DTYPE_INT_ULONG] = "INT_ULONG",
 242         [VSD_DTYPE_Q_S32] = "Q_S32",
 243         [VSD_DTYPE_Q_U32] = "Q_U32",
 244         [VSD_DTYPE_Q_S64] = "Q_S64",
 245         [VSD_DTYPE_Q_U64] = "Q_U64",
 246         [VSD_DTYPE_CRHIST32] = "CRHIST32",
 247         [VSD_DTYPE_DRHIST32] = "DRHIST32",
 248         [VSD_DTYPE_DVHIST32] = "DVHIST32",
 249         [VSD_DTYPE_CRHIST64] = "CRHIST64",
 250         [VSD_DTYPE_DRHIST64] = "DRHIST64",
 251         [VSD_DTYPE_DVHIST64] = "DVHIST64",
 252         [VSD_DTYPE_TDGSTCLUST32] = "TDGSTCLUST32",
 253         [VSD_DTYPE_TDGSTCLUST64] = "TDGSTCLUST64",
 254 };
 255
 256 const size_t vsd_dtype2size[VSD_NUM_DTYPES] = {
 257         [VSD_DTYPE_VOISTATE] = sizeof(struct voistatdata_voistate),
 258         [VSD_DTYPE_INT_S32] = sizeof(struct voistatdata_int32),
 259         [VSD_DTYPE_INT_U32] = sizeof(struct voistatdata_int32),
 260         [VSD_DTYPE_INT_S64] = sizeof(struct voistatdata_int64),
 261         [VSD_DTYPE_INT_U64] = sizeof(struct voistatdata_int64),
 262         [VSD_DTYPE_INT_SLONG] = sizeof(struct voistatdata_intlong),
 263         [VSD_DTYPE_INT_ULONG] = sizeof(struct voistatdata_intlong),
 264         [VSD_DTYPE_Q_S32] = sizeof(struct voistatdata_q32),
 265         [VSD_DTYPE_Q_U32] = sizeof(struct voistatdata_q32),
 266         [VSD_DTYPE_Q_S64] = sizeof(struct voistatdata_q64),
 267         [VSD_DTYPE_Q_U64] = sizeof(struct voistatdata_q64),
 268         [VSD_DTYPE_CRHIST32] = sizeof(struct voistatdata_crhist32),
 269         [VSD_DTYPE_DRHIST32] = sizeof(struct voistatdata_drhist32),
 270         [VSD_DTYPE_DVHIST32] = sizeof(struct voistatdata_dvhist32),
 271         [VSD_DTYPE_CRHIST64] = sizeof(struct voistatdata_crhist64),
 272         [VSD_DTYPE_DRHIST64] = sizeof(struct voistatdata_drhist64),
 273         [VSD_DTYPE_DVHIST64] = sizeof(struct voistatdata_dvhist64),
 274         [VSD_DTYPE_TDGSTCLUST32] = sizeof(struct voistatdata_tdgstclust32),
 275         [VSD_DTYPE_TDGSTCLUST64] = sizeof(struct voistatdata_tdgstclust64),
 276 };
 277
 278 static const bool vsd_compoundtype[VSD_NUM_DTYPES] = {
 279         [VSD_DTYPE_VOISTATE] = true,
 280         [VSD_DTYPE_INT_S32] = false,
 281         [VSD_DTYPE_INT_U32] = false,
 282         [VSD_DTYPE_INT_S64] = false,
 283         [VSD_DTYPE_INT_U64] = false,
 284         [VSD_DTYPE_INT_SLONG] = false,
 285         [VSD_DTYPE_INT_ULONG] = false,
 286         [VSD_DTYPE_Q_S32] = false,
 287         [VSD_DTYPE_Q_U32] = false,
 288         [VSD_DTYPE_Q_S64] = false,
 289         [VSD_DTYPE_Q_U64] = false,
 290         [VSD_DTYPE_CRHIST32] = true,
 291         [VSD_DTYPE_DRHIST32] = true,
 292         [VSD_DTYPE_DVHIST32] = true,
 293         [VSD_DTYPE_CRHIST64] = true,
 294         [VSD_DTYPE_DRHIST64] = true,
 295         [VSD_DTYPE_DVHIST64] = true,
 296         [VSD_DTYPE_TDGSTCLUST32] = true,
 297         [VSD_DTYPE_TDGSTCLUST64] = true,
 298 };
 299
 300 const struct voistatdata_numeric numeric_limits[2][VSD_DTYPE_Q_U64 + 1] = {
 301         [LIM_MIN] = {
 302                 [VSD_DTYPE_VOISTATE] = {0},
 303                 [VSD_DTYPE_INT_S32] = {.int32 = {.s32 = INT32_MIN}},
 304                 [VSD_DTYPE_INT_U32] = {.int32 = {.u32 = 0}},
 305                 [VSD_DTYPE_INT_S64] = {.int64 = {.s64 = INT64_MIN}},
 306                 [VSD_DTYPE_INT_U64] = {.int64 = {.u64 = 0}},
 307                 [VSD_DTYPE_INT_SLONG] = {.intlong = {.slong = LONG_MIN}},
 308                 [VSD_DTYPE_INT_ULONG] = {.intlong = {.ulong = 0}},
 309                 [VSD_DTYPE_Q_S32] = {.q32 = {.sq32 = Q_IFMINVAL(INT32_MIN)}},
 310                 [VSD_DTYPE_Q_U32] = {.q32 = {.uq32 = 0}},
 311                 [VSD_DTYPE_Q_S64] = {.q64 = {.sq64 = Q_IFMINVAL(INT64_MIN)}},
 312                 [VSD_DTYPE_Q_U64] = {.q64 = {.uq64 = 0}},
 313         },
 314         [LIM_MAX] = {
 315                 [VSD_DTYPE_VOISTATE] = {0},
 316                 [VSD_DTYPE_INT_S32] = {.int32 = {.s32 = INT32_MAX}},
 317                 [VSD_DTYPE_INT_U32] = {.int32 = {.u32 = UINT32_MAX}},
 318                 [VSD_DTYPE_INT_S64] = {.int64 = {.s64 = INT64_MAX}},
 319                 [VSD_DTYPE_INT_U64] = {.int64 = {.u64 = UINT64_MAX}},
 320                 [VSD_DTYPE_INT_SLONG] = {.intlong = {.slong = LONG_MAX}},
 321                 [VSD_DTYPE_INT_ULONG] = {.intlong = {.ulong = ULONG_MAX}},
 322                 [VSD_DTYPE_Q_S32] = {.q32 = {.sq32 = Q_IFMAXVAL(INT32_MAX)}},
 323                 [VSD_DTYPE_Q_U32] = {.q32 = {.uq32 = Q_IFMAXVAL(UINT32_MAX)}},
 324                 [VSD_DTYPE_Q_S64] = {.q64 = {.sq64 = Q_IFMAXVAL(INT64_MAX)}},
 325                 [VSD_DTYPE_Q_U64] = {.q64 = {.uq64 = Q_IFMAXVAL(UINT64_MAX)}},
 326         }
 327 };
 328
 329 /* tpllistlock protects tpllist and ntpl */
 330 static uint32_t ntpl;
 331 static struct statsblob_tpl **tpllist;
 332
 333 static inline void * stats_realloc(void *ptr, size_t oldsz, size_t newsz,
 334     int flags);
 335 //static void stats_v1_blob_finalise(struct statsblobv1 *sb);
 336 static int stats_v1_blob_init_locked(struct statsblobv1 *sb, uint32_t tpl_id,
 337     uint32_t flags);
 338 static int stats_v1_blob_expand(struct statsblobv1 **sbpp, int newvoibytes,
 339     int newvoistatbytes, int newvoistatdatabytes);
 340 static void stats_v1_blob_iter(struct statsblobv1 *sb,
 341     stats_v1_blob_itercb_t icb, void *usrctx, uint32_t flags);
 342 static inline int stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype,
 343     struct voistatdata_tdgst *tdgst, s64q_t x, uint64_t weight, int attempt);
 344
 345 static inline int
 346 ctd32cmp(const struct voistatdata_tdgstctd32 *c1, const struct voistatdata_tdgstctd32 *c2)
 347 {
 348
 349         KASSERT(Q_PRECEQ(c1->mu, c2->mu),
 350             ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__,
 351             Q_RELPREC(c1->mu, c2->mu)));
 352
 353        return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1);
 354 }
 355 ARB_GENERATE_STATIC(ctdth32, voistatdata_tdgstctd32, ctdlnk, ctd32cmp);
 356
 357 static inline int
 358 ctd64cmp(const struct voistatdata_tdgstctd64 *c1, const struct voistatdata_tdgstctd64 *c2)
 359 {
 360
 361         KASSERT(Q_PRECEQ(c1->mu, c2->mu),
 362             ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__,
 363             Q_RELPREC(c1->mu, c2->mu)));
 364
 365        return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1);
 366 }
 367 ARB_GENERATE_STATIC(ctdth64, voistatdata_tdgstctd64, ctdlnk, ctd64cmp);
 368
 369 #ifdef DIAGNOSTIC
 370 RB_GENERATE_STATIC(rbctdth32, voistatdata_tdgstctd32, rblnk, ctd32cmp);
 371 RB_GENERATE_STATIC(rbctdth64, voistatdata_tdgstctd64, rblnk, ctd64cmp);
 372 #endif
 373
 374 static inline sbintime_t
 375 stats_sbinuptime(void)
 376 {
 377         sbintime_t sbt;
 378 #ifdef _KERNEL
 379
 380         sbt = sbinuptime();
 381 #else /* ! _KERNEL */
 382         struct timespec tp;
 383
 384         clock_gettime(CLOCK_MONOTONIC_FAST, &tp);
 385         sbt = tstosbt(tp);
 386 #endif /* _KERNEL */
 387
 388         return (sbt);
 389 }
 390
 391 static inline void *
 392 stats_realloc(void *ptr, size_t oldsz, size_t newsz, int flags)
 393 {
 394
 395 #ifdef _KERNEL
 396         /* Default to M_NOWAIT if neither M_NOWAIT or M_WAITOK are set. */
 397         if (!(flags & (M_WAITOK | M_NOWAIT)))
 398                 flags |= M_NOWAIT;
 399         ptr = realloc(ptr, newsz, M_STATS, flags);
 400 #else /* ! _KERNEL */
 401         ptr = realloc(ptr, newsz);
 402         if ((flags & M_ZERO) && ptr != NULL) {
 403                 if (oldsz == 0)
 404                         memset(ptr, '\0', newsz);
 405                 else if (newsz > oldsz)
 406                         memset(BLOB_OFFSET(ptr, oldsz), '\0', newsz - oldsz);
 407         }
 408 #endif /* _KERNEL */
 409
 410         return (ptr);
 411 }
 412
 413 static inline char *
 414 stats_strdup(const char *s,
 415 #ifdef _KERNEL
 416     int flags)
 417 {
 418         char *copy;
 419         size_t len;
 420
 421         if (!(flags & (M_WAITOK | M_NOWAIT)))
 422                 flags |= M_NOWAIT;
 423
 424         len = strlen(s) + 1;
 425         if ((copy = malloc(len, M_STATS, flags)) != NULL)
 426                 bcopy(s, copy, len);
 427
 428         return (copy);
 429 #else
 430     int flags __unused)
 431 {
 432         return (strdup(s));
 433 #endif
 434 }
 435
 436 static inline void
 437 stats_tpl_update_hash(struct statsblob_tpl *tpl)
 438 {
 439
 440         TPL_LIST_WLOCK_ASSERT();
 441         tpl->mb->tplhash = hash32_str(tpl->mb->tplname, 0);
 442         for (int voi_id = 0; voi_id < NVOIS(tpl->sb); voi_id++) {
 443                 if (tpl->mb->voi_meta[voi_id].name != NULL)
 444                         tpl->mb->tplhash = hash32_str(
 445                             tpl->mb->voi_meta[voi_id].name, tpl->mb->tplhash);
 446         }
 447         tpl->mb->tplhash = hash32_buf(tpl->sb, tpl->sb->cursz,
 448             tpl->mb->tplhash);
 449 }
 450
 451 static inline uint64_t
 452 stats_pow_u64(uint64_t base, uint64_t exp)
 453 {
 454         uint64_t result = 1;
 455
 456         while (exp) {
 457                 if (exp & 1)
 458                         result *= base;
 459                 exp >>= 1;
 460                 base *= base;
 461         }
 462
 463         return (result);
 464 }
 465
 466 static inline int
 467 stats_vss_hist_bkt_hlpr(struct vss_hist_hlpr_info *info, uint32_t curbkt,
 468     struct voistatdata_numeric *bkt_lb, struct voistatdata_numeric *bkt_ub)
 469 {
 470         uint64_t step = 0;
 471         int error = 0;
 472
 473         switch (info->scheme) {
 474         case BKT_LIN:
 475                 step = info->lin.stepinc;
 476                 break;
 477         case BKT_EXP:
 478                 step = stats_pow_u64(info->exp.stepbase,
 479                     info->exp.stepexp + curbkt);
 480                 break;
 481         case BKT_LINEXP:
 482                 {
 483                 uint64_t curstepexp = 1;
 484
 485                 switch (info->voi_dtype) {
 486                 case VSD_DTYPE_INT_S32:
 487                         while ((int32_t)stats_pow_u64(info->linexp.stepbase,
 488                             curstepexp) <= bkt_lb->int32.s32)
 489                                 curstepexp++;
 490                         break;
 491                 case VSD_DTYPE_INT_U32:
 492                         while ((uint32_t)stats_pow_u64(info->linexp.stepbase,
 493                             curstepexp) <= bkt_lb->int32.u32)
 494                                 curstepexp++;
 495                         break;
 496                 case VSD_DTYPE_INT_S64:
 497                         while ((int64_t)stats_pow_u64(info->linexp.stepbase,
 498                             curstepexp) <= bkt_lb->int64.s64)
 499                                 curstepexp++;
 500                         break;
 501                 case VSD_DTYPE_INT_U64:
 502                         while ((uint64_t)stats_pow_u64(info->linexp.stepbase,
 503                             curstepexp) <= bkt_lb->int64.u64)
 504                                 curstepexp++;
 505                         break;
 506                 case VSD_DTYPE_INT_SLONG:
 507                         while ((long)stats_pow_u64(info->linexp.stepbase,
 508                             curstepexp) <= bkt_lb->intlong.slong)
 509                                 curstepexp++;
 510                         break;
 511                 case VSD_DTYPE_INT_ULONG:
 512                         while ((unsigned long)stats_pow_u64(info->linexp.stepbase,
 513                             curstepexp) <= bkt_lb->intlong.ulong)
 514                                 curstepexp++;
 515                         break;
 516                 case VSD_DTYPE_Q_S32:
 517                         while ((s32q_t)stats_pow_u64(info->linexp.stepbase,
 518                             curstepexp) <= Q_GIVAL(bkt_lb->q32.sq32))
 519                         break;
 520                 case VSD_DTYPE_Q_U32:
 521                         while ((u32q_t)stats_pow_u64(info->linexp.stepbase,
 522                             curstepexp) <= Q_GIVAL(bkt_lb->q32.uq32))
 523                         break;
 524                 case VSD_DTYPE_Q_S64:
 525                         while ((s64q_t)stats_pow_u64(info->linexp.stepbase,
 526                             curstepexp) <= Q_GIVAL(bkt_lb->q64.sq64))
 527                                 curstepexp++;
 528                         break;
 529                 case VSD_DTYPE_Q_U64:
 530                         while ((u64q_t)stats_pow_u64(info->linexp.stepbase,
 531                             curstepexp) <= Q_GIVAL(bkt_lb->q64.uq64))
 532                                 curstepexp++;
 533                         break;
 534                 default:
 535                         break;
 536                 }
 537
 538                 step = stats_pow_u64(info->linexp.stepbase, curstepexp) /
 539                     info->linexp.linstepdiv;
 540                 if (step == 0)
 541                         step = 1;
 542                 break;
 543                 }
 544         default:
 545                 break;
 546         }
 547
 548         if (info->scheme == BKT_USR) {
 549                 *bkt_lb = info->usr.bkts[curbkt].lb;
 550                 *bkt_ub = info->usr.bkts[curbkt].ub;
 551         } else if (step != 0) {
 552                 switch (info->voi_dtype) {
 553                 case VSD_DTYPE_INT_S32:
 554                         bkt_ub->int32.s32 += (int32_t)step;
 555                         break;
 556                 case VSD_DTYPE_INT_U32:
 557                         bkt_ub->int32.u32 += (uint32_t)step;
 558                         break;
 559                 case VSD_DTYPE_INT_S64:
 560                         bkt_ub->int64.s64 += (int64_t)step;
 561                         break;
 562                 case VSD_DTYPE_INT_U64:
 563                         bkt_ub->int64.u64 += (uint64_t)step;
 564                         break;
 565                 case VSD_DTYPE_INT_SLONG:
 566                         bkt_ub->intlong.slong += (long)step;
 567                         break;
 568                 case VSD_DTYPE_INT_ULONG:
 569                         bkt_ub->intlong.ulong += (unsigned long)step;
 570                         break;
 571                 case VSD_DTYPE_Q_S32:
 572                         error = Q_QADDI(&bkt_ub->q32.sq32, step);
 573                         break;
 574                 case VSD_DTYPE_Q_U32:
 575                         error = Q_QADDI(&bkt_ub->q32.uq32, step);
 576                         break;
 577                 case VSD_DTYPE_Q_S64:
 578                         error = Q_QADDI(&bkt_ub->q64.sq64, step);
 579                         break;
 580                 case VSD_DTYPE_Q_U64:
 581                         error = Q_QADDI(&bkt_ub->q64.uq64, step);
 582                         break;
 583                 default:
 584                         break;
 585                 }
 586         } else { /* info->scheme != BKT_USR && step == 0 */
 587                 return (EINVAL);
 588         }
 589
 590         return (error);
 591 }
 592
 593 static uint32_t
 594 stats_vss_hist_nbkts_hlpr(struct vss_hist_hlpr_info *info)
 595 {
 596         struct voistatdata_numeric bkt_lb, bkt_ub;
 597         uint32_t nbkts;
 598         int done;
 599
 600         if (info->scheme == BKT_USR) {
 601                 /* XXXLAS: Setting info->{lb,ub} from macro is tricky. */
 602                 info->lb = info->usr.bkts[0].lb;
 603                 info->ub = info->usr.bkts[info->usr.nbkts - 1].lb;
 604         }
 605
 606         nbkts = 0;
 607         done = 0;
 608         bkt_ub = info->lb;
 609
 610         do {
 611                 bkt_lb = bkt_ub;
 612                 if (stats_vss_hist_bkt_hlpr(info, nbkts++, &bkt_lb, &bkt_ub))
 613                         return (0);
 614
 615                 if (info->scheme == BKT_USR)
 616                         done = (nbkts == info->usr.nbkts);
 617                 else {
 618                         switch (info->voi_dtype) {
 619                         case VSD_DTYPE_INT_S32:
 620                                 done = (bkt_ub.int32.s32 > info->ub.int32.s32);
 621                                 break;
 622                         case VSD_DTYPE_INT_U32:
 623                                 done = (bkt_ub.int32.u32 > info->ub.int32.u32);
 624                                 break;
 625                         case VSD_DTYPE_INT_S64:
 626                                 done = (bkt_ub.int64.s64 > info->ub.int64.s64);
 627                                 break;
 628                         case VSD_DTYPE_INT_U64:
 629                                 done = (bkt_ub.int64.u64 > info->ub.int64.u64);
 630                                 break;
 631                         case VSD_DTYPE_INT_SLONG:
 632                                 done = (bkt_ub.intlong.slong >
 633                                     info->ub.intlong.slong);
 634                                 break;
 635                         case VSD_DTYPE_INT_ULONG:
 636                                 done = (bkt_ub.intlong.ulong >
 637                                     info->ub.intlong.ulong);
 638                                 break;
 639                         case VSD_DTYPE_Q_S32:
 640                                 done = Q_QGTQ(bkt_ub.q32.sq32,
 641                                     info->ub.q32.sq32);
 642                                 break;
 643                         case VSD_DTYPE_Q_U32:
 644                                 done = Q_QGTQ(bkt_ub.q32.uq32,
 645                                     info->ub.q32.uq32);
 646                                 break;
 647                         case VSD_DTYPE_Q_S64:
 648                                 done = Q_QGTQ(bkt_ub.q64.sq64,
 649                                     info->ub.q64.sq64);
 650                                 break;
 651                         case VSD_DTYPE_Q_U64:
 652                                 done = Q_QGTQ(bkt_ub.q64.uq64,
 653                                     info->ub.q64.uq64);
 654                                 break;
 655                         default:
 656                                 return (0);
 657                         }
 658                 }
 659         } while (!done);
 660
 661         if (info->flags & VSD_HIST_LBOUND_INF)
 662                 nbkts++;
 663         if (info->flags & VSD_HIST_UBOUND_INF)
 664                 nbkts++;
 665
 666         return (nbkts);
 667 }
 668
 669 int
 670 stats_vss_hist_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss,
 671     struct vss_hist_hlpr_info *info)
 672 {
 673         struct voistatdata_hist *hist;
 674         struct voistatdata_numeric bkt_lb, bkt_ub, *lbinfbktlb, *lbinfbktub,
 675             *ubinfbktlb, *ubinfbktub;
 676         uint32_t bkt, nbkts, nloop;
 677
 678         if (vss == NULL || info == NULL || (info->flags &
 679         (VSD_HIST_LBOUND_INF|VSD_HIST_UBOUND_INF) && (info->hist_dtype ==
 680         VSD_DTYPE_DVHIST32 || info->hist_dtype == VSD_DTYPE_DVHIST64)))
 681                 return (EINVAL);
 682
 683         info->voi_dtype = voi_dtype;
 684
 685         if ((nbkts = stats_vss_hist_nbkts_hlpr(info)) == 0)
 686                 return (EINVAL);
 687
 688         switch (info->hist_dtype) {
 689         case VSD_DTYPE_CRHIST32:
 690                 vss->vsdsz = HIST_NBKTS2VSDSZ(crhist32, nbkts);
 691                 break;
 692         case VSD_DTYPE_DRHIST32:
 693                 vss->vsdsz = HIST_NBKTS2VSDSZ(drhist32, nbkts);
 694                 break;
 695         case VSD_DTYPE_DVHIST32:
 696                 vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist32, nbkts);
 697                 break;
 698         case VSD_DTYPE_CRHIST64:
 699                 vss->vsdsz = HIST_NBKTS2VSDSZ(crhist64, nbkts);
 700                 break;
 701         case VSD_DTYPE_DRHIST64:
 702                 vss->vsdsz = HIST_NBKTS2VSDSZ(drhist64, nbkts);
 703                 break;
 704         case VSD_DTYPE_DVHIST64:
 705                 vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist64, nbkts);
 706                 break;
 707         default:
 708                 return (EINVAL);
 709         }
 710
 711         vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO);
 712         if (vss->iv == NULL)
 713                 return (ENOMEM);
 714
 715         hist = (struct voistatdata_hist *)vss->iv;
 716         bkt_ub = info->lb;
 717
 718         for (bkt = (info->flags & VSD_HIST_LBOUND_INF), nloop = 0;
 719             bkt < nbkts;
 720             bkt++, nloop++) {
 721                 bkt_lb = bkt_ub;
 722                 if (stats_vss_hist_bkt_hlpr(info, nloop, &bkt_lb, &bkt_ub))
 723                         return (EINVAL);
 724
 725                 switch (info->hist_dtype) {
 726                 case VSD_DTYPE_CRHIST32:
 727                         VSD(crhist32, hist)->bkts[bkt].lb = bkt_lb;
 728                         break;
 729                 case VSD_DTYPE_DRHIST32:
 730                         VSD(drhist32, hist)->bkts[bkt].lb = bkt_lb;
 731                         VSD(drhist32, hist)->bkts[bkt].ub = bkt_ub;
 732                         break;
 733                 case VSD_DTYPE_DVHIST32:
 734                         VSD(dvhist32, hist)->bkts[bkt].val = bkt_lb;
 735                         break;
 736                 case VSD_DTYPE_CRHIST64:
 737                         VSD(crhist64, hist)->bkts[bkt].lb = bkt_lb;
 738                         break;
 739                 case VSD_DTYPE_DRHIST64:
 740                         VSD(drhist64, hist)->bkts[bkt].lb = bkt_lb;
 741                         VSD(drhist64, hist)->bkts[bkt].ub = bkt_ub;
 742                         break;
 743                 case VSD_DTYPE_DVHIST64:
 744                         VSD(dvhist64, hist)->bkts[bkt].val = bkt_lb;
 745                         break;
 746                 default:
 747                         return (EINVAL);
 748                 }
 749         }
 750
 751         lbinfbktlb = lbinfbktub = ubinfbktlb = ubinfbktub = NULL;
 752
 753         switch (info->hist_dtype) {
 754         case VSD_DTYPE_CRHIST32:
 755                 lbinfbktlb = &VSD(crhist32, hist)->bkts[0].lb;
 756                 ubinfbktlb = &VSD(crhist32, hist)->bkts[nbkts - 1].lb;
 757                 break;
 758         case VSD_DTYPE_DRHIST32:
 759                 lbinfbktlb = &VSD(drhist32, hist)->bkts[0].lb;
 760                 lbinfbktub = &VSD(drhist32, hist)->bkts[0].ub;
 761                 ubinfbktlb = &VSD(drhist32, hist)->bkts[nbkts - 1].lb;
 762                 ubinfbktub = &VSD(drhist32, hist)->bkts[nbkts - 1].ub;
 763                 break;
 764         case VSD_DTYPE_CRHIST64:
 765                 lbinfbktlb = &VSD(crhist64, hist)->bkts[0].lb;
 766                 ubinfbktlb = &VSD(crhist64, hist)->bkts[nbkts - 1].lb;
 767                 break;
 768         case VSD_DTYPE_DRHIST64:
 769                 lbinfbktlb = &VSD(drhist64, hist)->bkts[0].lb;
 770                 lbinfbktub = &VSD(drhist64, hist)->bkts[0].ub;
 771                 ubinfbktlb = &VSD(drhist64, hist)->bkts[nbkts - 1].lb;
 772                 ubinfbktub = &VSD(drhist64, hist)->bkts[nbkts - 1].ub;
 773                 break;
 774         case VSD_DTYPE_DVHIST32:
 775         case VSD_DTYPE_DVHIST64:
 776                 break;
 777         default:
 778                 return (EINVAL);
 779         }
 780
 781         if ((info->flags & VSD_HIST_LBOUND_INF) && lbinfbktlb) {
 782                 *lbinfbktlb = numeric_limits[LIM_MIN][info->voi_dtype];
 783                 /*
 784                  * Assignment from numeric_limit array for Q types assigns max
 785                  * possible integral/fractional value for underlying data type,
 786                  * but we must set control bits for this specific histogram per
 787                  * the user's choice of fractional bits, which we extract from
 788                  * info->lb.
 789                  */
 790                 if (info->voi_dtype == VSD_DTYPE_Q_S32 ||
 791                     info->voi_dtype == VSD_DTYPE_Q_U32) {
 792                         /* Signedness doesn't matter for setting control bits. */
 793                         Q_SCVAL(lbinfbktlb->q32.sq32,
 794                             Q_GCVAL(info->lb.q32.sq32));
 795                 } else if (info->voi_dtype == VSD_DTYPE_Q_S64 ||
 796                     info->voi_dtype == VSD_DTYPE_Q_U64) {
 797                         /* Signedness doesn't matter for setting control bits. */
 798                         Q_SCVAL(lbinfbktlb->q64.sq64,
 799                             Q_GCVAL(info->lb.q64.sq64));
 800                 }
 801                 if (lbinfbktub)
 802                         *lbinfbktub = info->lb;
 803         }
 804         if ((info->flags & VSD_HIST_UBOUND_INF) && ubinfbktlb) {
 805                 *ubinfbktlb = bkt_lb;
 806                 if (ubinfbktub) {
 807                         *ubinfbktub = numeric_limits[LIM_MAX][info->voi_dtype];
 808                         if (info->voi_dtype == VSD_DTYPE_Q_S32 ||
 809                             info->voi_dtype == VSD_DTYPE_Q_U32) {
 810                                 Q_SCVAL(ubinfbktub->q32.sq32,
 811                                     Q_GCVAL(info->lb.q32.sq32));
 812                         } else if (info->voi_dtype == VSD_DTYPE_Q_S64 ||
 813                             info->voi_dtype == VSD_DTYPE_Q_U64) {
 814                                 Q_SCVAL(ubinfbktub->q64.sq64,
 815                                     Q_GCVAL(info->lb.q64.sq64));
 816                         }
 817                 }
 818         }
 819
 820         return (0);
 821 }
 822
 823 int
 824 stats_vss_tdgst_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss,
 825     struct vss_tdgst_hlpr_info *info)
 826 {
 827         struct voistatdata_tdgst *tdgst;
 828         struct ctdth32 *ctd32tree;
 829         struct ctdth64 *ctd64tree;
 830         struct voistatdata_tdgstctd32 *ctd32;
 831         struct voistatdata_tdgstctd64 *ctd64;
 832
 833         info->voi_dtype = voi_dtype;
 834
 835         switch (info->tdgst_dtype) {
 836         case VSD_DTYPE_TDGSTCLUST32:
 837                 vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust32, info->nctds);
 838                 break;
 839         case VSD_DTYPE_TDGSTCLUST64:
 840                 vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust64, info->nctds);
 841                 break;
 842         default:
 843                 return (EINVAL);
 844         }
 845
 846         vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO);
 847         if (vss->iv == NULL)
 848                 return (ENOMEM);
 849
 850         tdgst = (struct voistatdata_tdgst *)vss->iv;
 851
 852         switch (info->tdgst_dtype) {
 853         case VSD_DTYPE_TDGSTCLUST32:
 854                 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
 855                 ARB_INIT(ctd32, ctdlnk, ctd32tree, info->nctds) {
 856                         Q_INI(&ctd32->mu, 0, 0, info->prec);
 857                 }
 858                 break;
 859         case VSD_DTYPE_TDGSTCLUST64:
 860                 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
 861                 ARB_INIT(ctd64, ctdlnk, ctd64tree, info->nctds) {
 862                         Q_INI(&ctd64->mu, 0, 0, info->prec);
 863                 }
 864                 break;
 865         default:
 866                 return (EINVAL);
 867         }
 868
 869         return (0);
 870 }
 871
 872 int
 873 stats_vss_numeric_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss,
 874     struct vss_numeric_hlpr_info *info)
 875 {
 876         struct voistatdata_numeric iv;
 877
 878         switch (vss->stype) {
 879         case VS_STYPE_SUM:
 880                 iv = stats_ctor_vsd_numeric(0);
 881                 break;
 882         case VS_STYPE_MIN:
 883                 iv = numeric_limits[LIM_MAX][voi_dtype];
 884                 break;
 885         case VS_STYPE_MAX:
 886                 iv = numeric_limits[LIM_MIN][voi_dtype];
 887                 break;
 888         default:
 889                 return (EINVAL);
 890         }
 891
 892         vss->iv = stats_realloc(NULL, 0, vsd_dtype2size[voi_dtype], 0);
 893         if (vss->iv == NULL)
 894                 return (ENOMEM);
 895
 896         vss->vs_dtype = voi_dtype;
 897         vss->vsdsz = vsd_dtype2size[voi_dtype];
 898         switch (voi_dtype) {
 899         case VSD_DTYPE_INT_S32:
 900                 *((int32_t *)vss->iv) = iv.int32.s32;
 901                 break;
 902         case VSD_DTYPE_INT_U32:
 903                 *((uint32_t *)vss->iv) = iv.int32.u32;
 904                 break;
 905         case VSD_DTYPE_INT_S64:
 906                 *((int64_t *)vss->iv) = iv.int64.s64;
 907                 break;
 908         case VSD_DTYPE_INT_U64:
 909                 *((uint64_t *)vss->iv) = iv.int64.u64;
 910                 break;
 911         case VSD_DTYPE_INT_SLONG:
 912                 *((long *)vss->iv) = iv.intlong.slong;
 913                 break;
 914         case VSD_DTYPE_INT_ULONG:
 915                 *((unsigned long *)vss->iv) = iv.intlong.ulong;
 916                 break;
 917         case VSD_DTYPE_Q_S32:
 918                 *((s32q_t *)vss->iv) = Q_SCVAL(iv.q32.sq32,
 919                     Q_CTRLINI(info->prec));
 920                 break;
 921         case VSD_DTYPE_Q_U32:
 922                 *((u32q_t *)vss->iv) = Q_SCVAL(iv.q32.uq32,
 923                     Q_CTRLINI(info->prec));
 924                 break;
 925         case VSD_DTYPE_Q_S64:
 926                 *((s64q_t *)vss->iv) = Q_SCVAL(iv.q64.sq64,
 927                     Q_CTRLINI(info->prec));
 928                 break;
 929         case VSD_DTYPE_Q_U64:
 930                 *((u64q_t *)vss->iv) = Q_SCVAL(iv.q64.uq64,
 931                     Q_CTRLINI(info->prec));
 932                 break;
 933         default:
 934                 break;
 935         }
 936
 937         return (0);
 938 }
 939
 940 int
 941 stats_vss_hlpr_init(enum vsd_dtype voi_dtype, uint32_t nvss,
 942     struct voistatspec *vss)
 943 {
 944         int i, ret;
 945
 946         for (i = nvss - 1; i >= 0; i--) {
 947                 if (vss[i].hlpr && (ret = vss[i].hlpr(voi_dtype, &vss[i],
 948                     vss[i].hlprinfo)) != 0)
 949                         return (ret);
 950         }
 951
 952         return (0);
 953 }
 954
 955 void
 956 stats_vss_hlpr_cleanup(uint32_t nvss, struct voistatspec *vss)
 957 {
 958         int i;
 959
 960         for (i = nvss - 1; i >= 0; i--) {
 961                 if (vss[i].hlpr) {
 962                         stats_free((void *)vss[i].iv);
 963                         vss[i].iv = NULL;
 964                 }
 965         }
 966 }
 967
 968 int
 969 stats_tpl_fetch(int tpl_id, struct statsblob_tpl **tpl)
 970 {
 971         int error;
 972
 973         error = 0;
 974
 975         TPL_LIST_WLOCK();
 976         if (tpl_id < 0 || tpl_id >= (int)ntpl) {
 977                 error = ENOENT;
 978         } else {
 979                 *tpl = tpllist[tpl_id];
 980                 /* XXXLAS: Acquire refcount on tpl. */
 981         }
 982         TPL_LIST_WUNLOCK();
 983
 984         return (error);
 985 }
 986
 987 int
 988 stats_tpl_fetch_allocid(const char *name, uint32_t hash)
 989 {
 990         int i, tpl_id;
 991
 992         tpl_id = -ESRCH;
 993
 994         TPL_LIST_RLOCK();
 995         for (i = ntpl - 1; i >= 0; i--) {
 996                 if (name != NULL) {
 997                         if (strlen(name) == strlen(tpllist[i]->mb->tplname) &&
 998                             strncmp(name, tpllist[i]->mb->tplname,
 999                             TPL_MAX_NAME_LEN) == 0 && (!hash || hash ==
1000                             tpllist[i]->mb->tplhash)) {
1001                                 tpl_id = i;
1002                                 break;
1003                         }
1004                 } else if (hash == tpllist[i]->mb->tplhash) {
1005                         tpl_id = i;
1006                         break;
1007                 }
1008         }
1009         TPL_LIST_RUNLOCK();
1010
1011         return (tpl_id);
1012 }
1013
1014 int
1015 stats_tpl_id2name(uint32_t tpl_id, char *buf, size_t len)
1016 {
1017         int error;
1018
1019         error = 0;
1020
1021         TPL_LIST_RLOCK();
1022         if (tpl_id < ntpl) {
1023                 if (buf != NULL && len > strlen(tpllist[tpl_id]->mb->tplname))
1024                         strlcpy(buf, tpllist[tpl_id]->mb->tplname, len);
1025                 else
1026                         error = EOVERFLOW;
1027         } else
1028                 error = ENOENT;
1029         TPL_LIST_RUNLOCK();
1030
1031         return (error);
1032 }
1033
1034 int
1035 stats_tpl_sample_rollthedice(struct stats_tpl_sample_rate *rates, int nrates,
1036     void *seed_bytes, size_t seed_len)
1037 {
1038         uint32_t cum_pct, rnd_pct;
1039         int i;
1040
1041         cum_pct = 0;
1042
1043         /*
1044          * Choose a pseudorandom or seeded number in range [0,100] and use
1045          * it to make a sampling decision and template selection where required.
1046          * If no seed is supplied, a PRNG is used to generate a pseudorandom
1047          * number so that every selection is independent. If a seed is supplied,
1048          * the caller desires random selection across different seeds, but
1049          * deterministic selection given the same seed. This is achieved by
1050          * hashing the seed and using the hash as the random number source.
1051          *
1052          * XXXLAS: Characterise hash function output distribution.
1053          */
1054         if (seed_bytes == NULL)
1055                 rnd_pct = random() / (INT32_MAX / 100);
1056         else
1057                 rnd_pct = hash32_buf(seed_bytes, seed_len, 0) /
1058                     (UINT32_MAX / 100U);
1059
1060         /*
1061          * We map the randomly selected percentage on to the interval [0,100]
1062          * consisting of the cumulatively summed template sampling percentages.
1063          * The difference between the cumulative sum of all template sampling
1064          * percentages and 100 is treated as a NULL assignment i.e. no stats
1065          * template will be assigned, and -1 returned instead.
1066          */
1067         for (i = 0; i < nrates; i++) {
1068                 cum_pct += rates[i].tpl_sample_pct;
1069
1070                 KASSERT(cum_pct <= 100, ("%s cum_pct %u > 100", __func__,
1071                     cum_pct));
1072                 if (rnd_pct > cum_pct || rates[i].tpl_sample_pct == 0)
1073                         continue;
1074
1075                 return (rates[i].tpl_slot_id);
1076         }
1077
1078         return (-1);
1079 }
1080
1081 int
1082 stats_v1_blob_clone(struct statsblobv1 **dst, size_t dstmaxsz,
1083     struct statsblobv1 *src, uint32_t flags)
1084 {
1085         int error;
1086
1087         error = 0;
1088
1089         if (src == NULL || dst == NULL ||
1090             src->cursz < sizeof(struct statsblob) ||
1091             ((flags & SB_CLONE_ALLOCDST) &&
1092             (flags & (SB_CLONE_USRDSTNOFAULT | SB_CLONE_USRDST)))) {
1093                 error = EINVAL;
1094         } else if (flags & SB_CLONE_ALLOCDST) {
1095                 *dst = stats_realloc(NULL, 0, src->cursz, 0);
1096                 if (*dst)
1097                         (*dst)->maxsz = dstmaxsz = src->cursz;
1098                 else
1099                         error = ENOMEM;
1100         } else if (*dst == NULL || dstmaxsz < sizeof(struct statsblob)) {
1101                 error = EINVAL;
1102         }
1103
1104         if (!error) {
1105                 size_t postcurszlen;
1106
1107                 /*
1108                  * Clone src into dst except for the maxsz field. If dst is too
1109                  * small to hold all of src, only copy src's header and return
1110                  * EOVERFLOW.
1111                  */
1112 #ifdef _KERNEL
1113                 if (flags & SB_CLONE_USRDSTNOFAULT)
1114                         copyout_nofault(src, *dst,
1115                             offsetof(struct statsblob, maxsz));
1116                 else if (flags & SB_CLONE_USRDST)
1117                         copyout(src, *dst, offsetof(struct statsblob, maxsz));
1118                 else
1119 #endif
1120                         memcpy(*dst, src, offsetof(struct statsblob, maxsz));
1121
1122                 if (dstmaxsz >= src->cursz) {
1123                         postcurszlen = src->cursz -
1124                             offsetof(struct statsblob, cursz);
1125                 } else {
1126                         error = EOVERFLOW;
1127                         postcurszlen = sizeof(struct statsblob) -
1128                             offsetof(struct statsblob, cursz);
1129                 }
1130 #ifdef _KERNEL
1131                 if (flags & SB_CLONE_USRDSTNOFAULT)
1132                         copyout_nofault(&(src->cursz), &((*dst)->cursz),
1133                             postcurszlen);
1134                 else if (flags & SB_CLONE_USRDST)
1135                         copyout(&(src->cursz), &((*dst)->cursz), postcurszlen);
1136                 else
1137 #endif
1138                         memcpy(&((*dst)->cursz), &(src->cursz), postcurszlen);
1139         }
1140
1141         return (error);
1142 }
1143
1144 int
1145 stats_v1_tpl_alloc(const char *name, uint32_t flags __unused)
1146 {
1147         struct statsblobv1_tpl *tpl, **newtpllist;
1148         struct statsblobv1 *tpl_sb;
1149         struct metablob *tpl_mb;
1150         int tpl_id;
1151
1152         if (name != NULL && strlen(name) > TPL_MAX_NAME_LEN)
1153                 return (-EINVAL);
1154
1155         if (name != NULL && stats_tpl_fetch_allocid(name, 0) >= 0)
1156                 return (-EEXIST);
1157
1158         tpl = stats_realloc(NULL, 0, sizeof(struct statsblobv1_tpl), M_ZERO);
1159         tpl_mb = stats_realloc(NULL, 0, sizeof(struct metablob), M_ZERO);
1160         tpl_sb = stats_realloc(NULL, 0, sizeof(struct statsblobv1), M_ZERO);
1161
1162         if (tpl_mb != NULL && name != NULL)
1163                 tpl_mb->tplname = stats_strdup(name, 0);
1164
1165         if (tpl == NULL || tpl_sb == NULL || tpl_mb == NULL ||
1166             tpl_mb->tplname == NULL) {
1167                 stats_free(tpl);
1168                 stats_free(tpl_sb);
1169                 if (tpl_mb != NULL) {
1170                         stats_free(tpl_mb->tplname);
1171                         stats_free(tpl_mb);
1172                 }
1173                 return (-ENOMEM);
1174         }
1175
1176         tpl->mb = tpl_mb;
1177         tpl->sb = tpl_sb;
1178
1179         tpl_sb->abi = STATS_ABI_V1;
1180         tpl_sb->endian =
1181 #if BYTE_ORDER == LITTLE_ENDIAN
1182             SB_LE;
1183 #elif BYTE_ORDER == BIG_ENDIAN
1184             SB_BE;
1185 #else
1186             SB_UE;
1187 #endif
1188         tpl_sb->cursz = tpl_sb->maxsz = sizeof(struct statsblobv1);
1189         tpl_sb->stats_off = tpl_sb->statsdata_off = sizeof(struct statsblobv1);
1190
1191         TPL_LIST_WLOCK();
1192         newtpllist = stats_realloc(tpllist, ntpl * sizeof(void *),
1193             (ntpl + 1) * sizeof(void *), 0);
1194         if (newtpllist != NULL) {
1195                 tpl_id = ntpl++;
1196                 tpllist = (struct statsblob_tpl **)newtpllist;
1197                 tpllist[tpl_id] = (struct statsblob_tpl *)tpl;
1198                 stats_tpl_update_hash(tpllist[tpl_id]);
1199         } else {
1200                 stats_free(tpl);
1201                 stats_free(tpl_sb);
1202                 if (tpl_mb != NULL) {
1203                         stats_free(tpl_mb->tplname);
1204                         stats_free(tpl_mb);
1205                 }
1206                 tpl_id = -ENOMEM;
1207         }
1208         TPL_LIST_WUNLOCK();
1209
1210         return (tpl_id);
1211 }
1212
1213 int
1214 stats_v1_tpl_add_voistats(uint32_t tpl_id, int32_t voi_id, const char *voi_name,
1215     enum vsd_dtype voi_dtype, uint32_t nvss, struct voistatspec *vss,
1216     uint32_t flags)
1217 {
1218         struct voi *voi;
1219         struct voistat *tmpstat;
1220         struct statsblobv1 *tpl_sb;
1221         struct metablob *tpl_mb;
1222         int error, i, newstatdataidx, newvoibytes, newvoistatbytes,
1223             newvoistatdatabytes, newvoistatmaxid;
1224         uint32_t nbytes;
1225
1226         if (voi_id < 0 || voi_dtype == 0 || voi_dtype >= VSD_NUM_DTYPES ||
1227             nvss == 0 || vss == NULL)
1228                 return (EINVAL);
1229
1230         error = nbytes = newvoibytes = newvoistatbytes =
1231             newvoistatdatabytes = 0;
1232         newvoistatmaxid = -1;
1233
1234         /* Calculate the number of bytes required for the new voistats. */
1235         for (i = nvss - 1; i >= 0; i--) {
1236                 if (vss[i].stype == 0 || vss[i].stype >= VS_NUM_STYPES ||
1237                     vss[i].vs_dtype == 0 || vss[i].vs_dtype >= VSD_NUM_DTYPES ||
1238                     vss[i].iv == NULL || vss[i].vsdsz == 0)
1239                         return (EINVAL);
1240                 if ((int)vss[i].stype > newvoistatmaxid)
1241                         newvoistatmaxid = vss[i].stype;
1242                 newvoistatdatabytes += vss[i].vsdsz;
1243         }
1244
1245         if (flags & SB_VOI_RELUPDATE) {
1246                 /* XXXLAS: VOI state bytes may need to vary based on stat types. */
1247                 newvoistatdatabytes += sizeof(struct voistatdata_voistate);
1248         }
1249         nbytes += newvoistatdatabytes;
1250
1251         TPL_LIST_WLOCK();
1252         if (tpl_id < ntpl) {
1253                 tpl_sb = (struct statsblobv1 *)tpllist[tpl_id]->sb;
1254                 tpl_mb = tpllist[tpl_id]->mb;
1255
1256                 if (voi_id >= NVOIS(tpl_sb) || tpl_sb->vois[voi_id].id == -1) {
1257                         /* Adding a new VOI and associated stats. */
1258                         if (voi_id >= NVOIS(tpl_sb)) {
1259                                 /* We need to grow the tpl_sb->vois array. */
1260                                 newvoibytes = (voi_id - (NVOIS(tpl_sb) - 1)) *
1261                                     sizeof(struct voi);
1262                                 nbytes += newvoibytes;
1263                         }
1264                         newvoistatbytes =
1265                             (newvoistatmaxid + 1) * sizeof(struct voistat);
1266                 } else {
1267                         /* Adding stats to an existing VOI. */
1268                         if (newvoistatmaxid >
1269                             tpl_sb->vois[voi_id].voistatmaxid) {
1270                                 newvoistatbytes = (newvoistatmaxid -
1271                                     tpl_sb->vois[voi_id].voistatmaxid) *
1272                                     sizeof(struct voistat);
1273                         }
1274                         /* XXXLAS: KPI does not yet support expanding VOIs. */
1275                         error = EOPNOTSUPP;
1276                 }
1277                 nbytes += newvoistatbytes;
1278
1279                 if (!error && newvoibytes > 0) {
1280                         struct voi_meta *voi_meta = tpl_mb->voi_meta;
1281
1282                         voi_meta = stats_realloc(voi_meta, voi_meta == NULL ?
1283                             0 : NVOIS(tpl_sb) * sizeof(struct voi_meta),
1284                             (1 + voi_id) * sizeof(struct voi_meta),
1285                             M_ZERO);
1286
1287                         if (voi_meta == NULL)
1288                                 error = ENOMEM;
1289                         else
1290                                 tpl_mb->voi_meta = voi_meta;
1291                 }
1292
1293                 if (!error) {
1294                         /* NB: Resizing can change where tpl_sb points. */
1295                         error = stats_v1_blob_expand(&tpl_sb, newvoibytes,
1296                             newvoistatbytes, newvoistatdatabytes);
1297                 }
1298
1299                 if (!error) {
1300                         tpl_mb->voi_meta[voi_id].name = stats_strdup(voi_name,
1301                             0);
1302                         if (tpl_mb->voi_meta[voi_id].name == NULL)
1303                                 error = ENOMEM;
1304                 }
1305
1306                 if (!error) {
1307                         /* Update the template list with the resized pointer. */
1308                         tpllist[tpl_id]->sb = (struct statsblob *)tpl_sb;
1309
1310                         /* Update the template. */
1311                         voi = &tpl_sb->vois[voi_id];
1312
1313                         if (voi->id < 0) {
1314                                 /* VOI is new and needs to be initialised. */
1315                                 voi->id = voi_id;
1316                                 voi->dtype = voi_dtype;
1317                                 voi->stats_off = tpl_sb->stats_off;
1318                                 if (flags & SB_VOI_RELUPDATE)
1319                                         voi->flags |= VOI_REQSTATE;
1320                         } else {
1321                                 /*
1322                                  * XXXLAS: When this else block is written, the
1323                                  * "KPI does not yet support expanding VOIs"
1324                                  * error earlier in this function can be
1325                                  * removed. What is required here is to shuffle
1326                                  * the voistat array such that the new stats for
1327                                  * the voi are contiguous, which will displace
1328                                  * stats for other vois that reside after the
1329                                  * voi being updated. The other vois then need
1330                                  * to have their stats_off adjusted post
1331                                  * shuffle.
1332                                  */
1333                         }
1334
1335                         voi->voistatmaxid = newvoistatmaxid;
1336                         newstatdataidx = 0;
1337
1338                         if (voi->flags & VOI_REQSTATE) {
1339                                 /* Initialise the voistate stat in slot 0. */
1340                                 tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off);
1341                                 tmpstat->stype = VS_STYPE_VOISTATE;
1342                                 tmpstat->flags = 0;
1343                                 tmpstat->dtype = VSD_DTYPE_VOISTATE;
1344                                 newstatdataidx = tmpstat->dsz =
1345                                     sizeof(struct voistatdata_numeric);
1346                                 tmpstat->data_off = tpl_sb->statsdata_off;
1347                         }
1348
1349                         for (i = 0; (uint32_t)i < nvss; i++) {
1350                                 tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off +
1351                                     (vss[i].stype * sizeof(struct voistat)));
1352                                 KASSERT(tmpstat->stype < 0, ("voistat %p "
1353                                     "already initialised", tmpstat));
1354                                 tmpstat->stype = vss[i].stype;
1355                                 tmpstat->flags = vss[i].flags;
1356                                 tmpstat->dtype = vss[i].vs_dtype;
1357                                 tmpstat->dsz = vss[i].vsdsz;
1358                                 tmpstat->data_off = tpl_sb->statsdata_off +
1359                                     newstatdataidx;
1360                                 memcpy(BLOB_OFFSET(tpl_sb, tmpstat->data_off),
1361                                     vss[i].iv, vss[i].vsdsz);
1362                                 newstatdataidx += vss[i].vsdsz;
1363                         }
1364
1365                         /* Update the template version hash. */
1366                         stats_tpl_update_hash(tpllist[tpl_id]);
1367                         /* XXXLAS: Confirm tpl name/hash pair remains unique. */
1368                 }
1369         } else
1370                 error = EINVAL;
1371         TPL_LIST_WUNLOCK();
1372
1373         return (error);
1374 }
1375
1376 struct statsblobv1 *
1377 stats_v1_blob_alloc(uint32_t tpl_id, uint32_t flags __unused)
1378 {
1379         struct statsblobv1 *sb;
1380         int error;
1381
1382         sb = NULL;
1383
1384         TPL_LIST_RLOCK();
1385         if (tpl_id < ntpl) {
1386                 sb = stats_realloc(NULL, 0, tpllist[tpl_id]->sb->maxsz, 0);
1387                 if (sb != NULL) {
1388                         sb->maxsz = tpllist[tpl_id]->sb->maxsz;
1389                         error = stats_v1_blob_init_locked(sb, tpl_id, 0);
1390                 } else
1391                         error = ENOMEM;
1392
1393                 if (error) {
1394                         stats_free(sb);
1395                         sb = NULL;
1396                 }
1397         }
1398         TPL_LIST_RUNLOCK();
1399
1400         return (sb);
1401 }
1402
1403 void
1404 stats_v1_blob_destroy(struct statsblobv1 *sb)
1405 {
1406
1407         stats_free(sb);
1408 }
1409
1410 int
1411 stats_v1_voistat_fetch_dptr(struct statsblobv1 *sb, int32_t voi_id,
1412     enum voi_stype stype, enum vsd_dtype *retdtype, struct voistatdata **retvsd,
1413     size_t *retvsdsz)
1414 {
1415         struct voi *v;
1416         struct voistat *vs;
1417
1418         if (retvsd == NULL || sb == NULL || sb->abi != STATS_ABI_V1 ||
1419             voi_id >= NVOIS(sb))
1420                 return (EINVAL);
1421
1422         v = &sb->vois[voi_id];
1423         if ((__typeof(v->voistatmaxid))stype > v->voistatmaxid)
1424                 return (EINVAL);
1425
1426         vs = BLOB_OFFSET(sb, v->stats_off + (stype * sizeof(struct voistat)));
1427         *retvsd = BLOB_OFFSET(sb, vs->data_off);
1428         if (retdtype != NULL)
1429                 *retdtype = vs->dtype;
1430         if (retvsdsz != NULL)
1431                 *retvsdsz = vs->dsz;
1432
1433         return (0);
1434 }
1435
1436 int
1437 stats_v1_blob_init(struct statsblobv1 *sb, uint32_t tpl_id, uint32_t flags)
1438 {
1439         int error;
1440
1441         error = 0;
1442
1443         TPL_LIST_RLOCK();
1444         if (sb == NULL || tpl_id >= ntpl) {
1445                 error = EINVAL;
1446         } else {
1447                 error = stats_v1_blob_init_locked(sb, tpl_id, flags);
1448         }
1449         TPL_LIST_RUNLOCK();
1450
1451         return (error);
1452 }
1453
1454 static inline int
1455 stats_v1_blob_init_locked(struct statsblobv1 *sb, uint32_t tpl_id,
1456     uint32_t flags __unused)
1457 {
1458         int error;
1459
1460         TPL_LIST_RLOCK_ASSERT();
1461         error = (sb->maxsz >= tpllist[tpl_id]->sb->cursz) ? 0 : EOVERFLOW;
1462         KASSERT(!error,
1463             ("sb %d instead of %d bytes", sb->maxsz, tpllist[tpl_id]->sb->cursz));
1464
1465         if (!error) {
1466                 memcpy(sb, tpllist[tpl_id]->sb, tpllist[tpl_id]->sb->cursz);
1467                 sb->created = sb->lastrst = stats_sbinuptime();
1468                 sb->tplhash = tpllist[tpl_id]->mb->tplhash;
1469         }
1470
1471         return (error);
1472 }
1473
1474 static int
1475 stats_v1_blob_expand(struct statsblobv1 **sbpp, int newvoibytes,
1476     int newvoistatbytes, int newvoistatdatabytes)
1477 {
1478         struct statsblobv1 *sb;
1479         struct voi *tmpvoi;
1480         struct voistat *tmpvoistat, *voistat_array;
1481         int error, i, idxnewvois, idxnewvoistats, nbytes, nvoistats;
1482
1483         KASSERT(newvoibytes % sizeof(struct voi) == 0,
1484             ("Bad newvoibytes %d", newvoibytes));
1485         KASSERT(newvoistatbytes % sizeof(struct voistat) == 0,
1486             ("Bad newvoistatbytes %d", newvoistatbytes));
1487
1488         error = ((newvoibytes % sizeof(struct voi) == 0) &&
1489             (newvoistatbytes % sizeof(struct voistat) == 0)) ? 0 : EINVAL;
1490         sb = *sbpp;
1491         nbytes = newvoibytes + newvoistatbytes + newvoistatdatabytes;
1492
1493         /*
1494          * XXXLAS: Required until we gain support for flags which alter the
1495          * units of size/offset fields in key structs.
1496          */
1497         if (!error && ((((int)sb->cursz) + nbytes) > SB_V1_MAXSZ))
1498                 error = EFBIG;
1499
1500         if (!error && (sb->cursz + nbytes > sb->maxsz)) {
1501                 /* Need to expand our blob. */
1502                 sb = stats_realloc(sb, sb->maxsz, sb->cursz + nbytes, M_ZERO);
1503                 if (sb != NULL) {
1504                         sb->maxsz = sb->cursz + nbytes;
1505                         *sbpp = sb;
1506                 } else
1507                     error = ENOMEM;
1508         }
1509
1510         if (!error) {
1511                 /*
1512                  * Shuffle memory within the expanded blob working from the end
1513                  * backwards, leaving gaps for the new voistat and voistatdata
1514                  * structs at the beginning of their respective blob regions,
1515                  * and for the new voi structs at the end of their blob region.
1516                  */
1517                 memmove(BLOB_OFFSET(sb, sb->statsdata_off + nbytes),
1518                     BLOB_OFFSET(sb, sb->statsdata_off),
1519                     sb->cursz - sb->statsdata_off);
1520                 memmove(BLOB_OFFSET(sb, sb->stats_off + newvoibytes +
1521                     newvoistatbytes), BLOB_OFFSET(sb, sb->stats_off),
1522                     sb->statsdata_off - sb->stats_off);
1523
1524                 /* First index of new voi/voistat structs to be initialised. */
1525                 idxnewvois = NVOIS(sb);
1526                 idxnewvoistats = (newvoistatbytes / sizeof(struct voistat)) - 1;
1527
1528                 /* Update housekeeping variables and offsets. */
1529                 sb->cursz += nbytes;
1530                 sb->stats_off += newvoibytes;
1531                 sb->statsdata_off += newvoibytes + newvoistatbytes;
1532
1533                 /* XXXLAS: Zeroing not strictly needed but aids debugging. */
1534                 memset(&sb->vois[idxnewvois], '\0', newvoibytes);
1535                 memset(BLOB_OFFSET(sb, sb->stats_off), '\0',
1536                     newvoistatbytes);
1537                 memset(BLOB_OFFSET(sb, sb->statsdata_off), '\0',
1538                     newvoistatdatabytes);
1539
1540                 /* Initialise new voi array members and update offsets. */
1541                 for (i = 0; i < NVOIS(sb); i++) {
1542                         tmpvoi = &sb->vois[i];
1543                         if (i >= idxnewvois) {
1544                                 tmpvoi->id = tmpvoi->voistatmaxid = -1;
1545                         } else if (tmpvoi->id > -1) {
1546                                 tmpvoi->stats_off += newvoibytes +
1547                                     newvoistatbytes;
1548                         }
1549                 }
1550
1551                 /* Initialise new voistat array members and update offsets. */
1552                 nvoistats = (sb->statsdata_off - sb->stats_off) /
1553                     sizeof(struct voistat);
1554                 voistat_array = BLOB_OFFSET(sb, sb->stats_off);
1555                 for (i = 0; i < nvoistats; i++) {
1556                         tmpvoistat = &voistat_array[i];
1557                         if (i <= idxnewvoistats) {
1558                                 tmpvoistat->stype = -1;
1559                         } else if (tmpvoistat->stype > -1) {
1560                                 tmpvoistat->data_off += nbytes;
1561                         }
1562                 }
1563         }
1564
1565         return (error);
1566 }
1567
1568 static void
1569 stats_v1_blob_finalise(struct statsblobv1 *sb __unused)
1570 {
1571
1572         /* XXXLAS: Fill this in. */
1573 }
1574
1575 static void
1576 stats_v1_blob_iter(struct statsblobv1 *sb, stats_v1_blob_itercb_t icb,
1577     void *usrctx, uint32_t flags)
1578 {
1579         struct voi *v;
1580         struct voistat *vs;
1581         struct sb_iter_ctx ctx;
1582         int i, j, firstvoi;
1583
1584         ctx.usrctx = usrctx;
1585         ctx.flags |= SB_IT_FIRST_CB;
1586         ctx.flags &= ~(SB_IT_FIRST_VOI | SB_IT_LAST_VOI | SB_IT_FIRST_VOISTAT |
1587             SB_IT_LAST_VOISTAT);
1588         firstvoi = 1;
1589
1590         for (i = 0; i < NVOIS(sb); i++) {
1591                 v = &sb->vois[i];
1592                 ctx.vslot = i;
1593                 ctx.vsslot = -1;
1594                 ctx.flags |= SB_IT_FIRST_VOISTAT;
1595
1596                 if (firstvoi)
1597                         ctx.flags |= SB_IT_FIRST_VOI;
1598                 else if (i == (NVOIS(sb) - 1))
1599                         ctx.flags |= SB_IT_LAST_VOI | SB_IT_LAST_CB;
1600
1601                 if (v->id < 0 && (flags & SB_IT_NULLVOI)) {
1602                         if (icb(sb, v, NULL, &ctx))
1603                                 return;
1604                         firstvoi = 0;
1605                         ctx.flags &= ~SB_IT_FIRST_CB;
1606                 }
1607
1608                 /* If NULL voi, v->voistatmaxid == -1 */
1609                 for (j = 0; j <= v->voistatmaxid; j++) {
1610                         vs = &((struct voistat *)BLOB_OFFSET(sb,
1611                             v->stats_off))[j];
1612                         if (vs->stype < 0 &&
1613                             !(flags & SB_IT_NULLVOISTAT))
1614                                 continue;
1615
1616                         if (j == v->voistatmaxid) {
1617                                 ctx.flags |= SB_IT_LAST_VOISTAT;
1618                                 if (i == (NVOIS(sb) - 1))
1619                                         ctx.flags |=
1620                                             SB_IT_LAST_CB;
1621                         } else
1622                                 ctx.flags &= ~SB_IT_LAST_CB;
1623
1624                         ctx.vsslot = j;
1625                         if (icb(sb, v, vs, &ctx))
1626                                 return;
1627
1628                         ctx.flags &= ~(SB_IT_FIRST_CB | SB_IT_FIRST_VOISTAT |
1629                             SB_IT_LAST_VOISTAT);
1630                 }
1631                 ctx.flags &= ~(SB_IT_FIRST_VOI | SB_IT_LAST_VOI);
1632         }
1633 }
1634
1635 static inline void
1636 stats_voistatdata_tdgst_tostr(enum vsd_dtype voi_dtype __unused,
1637     const struct voistatdata_tdgst *tdgst, enum vsd_dtype tdgst_dtype,
1638     size_t tdgst_dsz __unused, enum sb_str_fmt fmt, struct sbuf *buf, int objdump)
1639 {
1640         const struct ctdth32 *ctd32tree;
1641         const struct ctdth64 *ctd64tree;
1642         const struct voistatdata_tdgstctd32 *ctd32;
1643         const struct voistatdata_tdgstctd64 *ctd64;
1644         const char *fmtstr;
1645         uint64_t smplcnt, compcnt;
1646         int is32bit, qmaxstrlen;
1647         uint16_t maxctds, curctds;
1648
1649         switch (tdgst_dtype) {
1650         case VSD_DTYPE_TDGSTCLUST32:
1651                 smplcnt = CONSTVSD(tdgstclust32, tdgst)->smplcnt;
1652                 compcnt = CONSTVSD(tdgstclust32, tdgst)->compcnt;
1653                 maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree);
1654                 curctds = ARB_CURNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree);
1655                 ctd32tree = &CONSTVSD(tdgstclust32, tdgst)->ctdtree;
1656                 ctd32 = (objdump ? ARB_CNODE(ctd32tree, 0) :
1657                     ARB_CMIN(ctdth32, ctd32tree));
1658                 qmaxstrlen = (ctd32 == NULL) ? 1 : Q_MAXSTRLEN(ctd32->mu, 10);
1659                 is32bit = 1;
1660                 ctd64tree = NULL;
1661                 ctd64 = NULL;
1662                 break;
1663         case VSD_DTYPE_TDGSTCLUST64:
1664                 smplcnt = CONSTVSD(tdgstclust64, tdgst)->smplcnt;
1665                 compcnt = CONSTVSD(tdgstclust64, tdgst)->compcnt;
1666                 maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree);
1667                 curctds = ARB_CURNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree);
1668                 ctd64tree = &CONSTVSD(tdgstclust64, tdgst)->ctdtree;
1669                 ctd64 = (objdump ? ARB_CNODE(ctd64tree, 0) :
1670                     ARB_CMIN(ctdth64, ctd64tree));
1671                 qmaxstrlen = (ctd64 == NULL) ? 1 : Q_MAXSTRLEN(ctd64->mu, 10);
1672                 is32bit = 0;
1673                 ctd32tree = NULL;
1674                 ctd32 = NULL;
1675                 break;
1676         default:
1677                 return;
1678         }
1679
1680         switch (fmt) {
1681         case SB_STRFMT_FREEFORM:
1682                 fmtstr = "smplcnt=%ju, compcnt=%ju, maxctds=%hu, nctds=%hu";
1683                 break;
1684         case SB_STRFMT_JSON:
1685         default:
1686                 fmtstr =
1687                     "\"smplcnt\":%ju,\"compcnt\":%ju,\"maxctds\":%hu,"
1688                     "\"nctds\":%hu,\"ctds\":[";
1689                 break;
1690         }
1691         sbuf_printf(buf, fmtstr, (uintmax_t)smplcnt, (uintmax_t)compcnt,
1692             maxctds, curctds);
1693
1694         while ((is32bit ? NULL != ctd32 : NULL != ctd64)) {
1695                 char qstr[qmaxstrlen];
1696
1697                 switch (fmt) {
1698                 case SB_STRFMT_FREEFORM:
1699                         fmtstr = "\n\t\t\t\t";
1700                         break;
1701                 case SB_STRFMT_JSON:
1702                 default:
1703                         fmtstr = "{";
1704                         break;
1705                 }
1706                 sbuf_cat(buf, fmtstr);
1707
1708                 if (objdump) {
1709                         switch (fmt) {
1710                         case SB_STRFMT_FREEFORM:
1711                                 fmtstr = "ctd[%hu].";
1712                                 break;
1713                         case SB_STRFMT_JSON:
1714                         default:
1715                                 fmtstr = "\"ctd\":%hu,";
1716                                 break;
1717                         }
1718                         sbuf_printf(buf, fmtstr, is32bit ?
1719                             ARB_SELFIDX(ctd32tree, ctd32) :
1720                             ARB_SELFIDX(ctd64tree, ctd64));
1721                 }
1722
1723                 switch (fmt) {
1724                 case SB_STRFMT_FREEFORM:
1725                         fmtstr = "{mu=";
1726                         break;
1727                 case SB_STRFMT_JSON:
1728                 default:
1729                         fmtstr = "\"mu\":";
1730                         break;
1731                 }
1732                 sbuf_cat(buf, fmtstr);
1733                 Q_TOSTR((is32bit ? ctd32->mu : ctd64->mu), -1, 10, qstr,
1734                     sizeof(qstr));
1735                 sbuf_cat(buf, qstr);
1736
1737
1738                 switch (fmt) {
1739                 case SB_STRFMT_FREEFORM:
1740                         fmtstr = is32bit ? ",cnt=%u}" : ",cnt=%ju}";
1741                         break;
1742                 case SB_STRFMT_JSON:
1743                 default:
1744                         fmtstr = is32bit ? ",\"cnt\":%u}" : ",\"cnt\":%ju}";
1745                         break;
1746                 }
1747                 sbuf_printf(buf, fmtstr,
1748                     is32bit ? ctd32->cnt : (uintmax_t)ctd64->cnt);
1749
1750                 if (is32bit)
1751                         ctd32 = (objdump ? ARB_CNODE(ctd32tree,
1752                             ARB_SELFIDX(ctd32tree, ctd32) + 1) :
1753                             ARB_CNEXT(ctdth32, ctd32tree, ctd32));
1754                 else
1755                         ctd64 = (objdump ? ARB_CNODE(ctd64tree,
1756                             ARB_SELFIDX(ctd64tree, ctd64) + 1) :
1757                             ARB_CNEXT(ctdth64, ctd64tree, ctd64));
1758
1759                 if (fmt == SB_STRFMT_JSON &&
1760                     (is32bit ? NULL != ctd32 : NULL != ctd64))
1761                         sbuf_putc(buf, ',');
1762         }
1763         if (fmt == SB_STRFMT_JSON)
1764                 sbuf_cat(buf, "]");
1765 }
1766
1767 static inline void
1768 stats_voistatdata_hist_tostr(enum vsd_dtype voi_dtype,
1769     const struct voistatdata_hist *hist, enum vsd_dtype hist_dtype,
1770     size_t hist_dsz, enum sb_str_fmt fmt, struct sbuf *buf, int objdump)
1771 {
1772         const struct voistatdata_numeric *bkt_lb, *bkt_ub;
1773         const char *fmtstr;
1774         int is32bit;
1775         uint16_t i, nbkts;
1776
1777         switch (hist_dtype) {
1778         case VSD_DTYPE_CRHIST32:
1779                 nbkts = HIST_VSDSZ2NBKTS(crhist32, hist_dsz);
1780                 is32bit = 1;
1781                 break;
1782         case VSD_DTYPE_DRHIST32:
1783                 nbkts = HIST_VSDSZ2NBKTS(drhist32, hist_dsz);
1784                 is32bit = 1;
1785                 break;
1786         case VSD_DTYPE_DVHIST32:
1787                 nbkts = HIST_VSDSZ2NBKTS(dvhist32, hist_dsz);
1788                 is32bit = 1;
1789                 break;
1790         case VSD_DTYPE_CRHIST64:
1791                 nbkts = HIST_VSDSZ2NBKTS(crhist64, hist_dsz);
1792                 is32bit = 0;
1793                 break;
1794         case VSD_DTYPE_DRHIST64:
1795                 nbkts = HIST_VSDSZ2NBKTS(drhist64, hist_dsz);
1796                 is32bit = 0;
1797                 break;
1798         case VSD_DTYPE_DVHIST64:
1799                 nbkts = HIST_VSDSZ2NBKTS(dvhist64, hist_dsz);
1800                 is32bit = 0;
1801                 break;
1802         default:
1803                 return;
1804         }
1805
1806         switch (fmt) {
1807         case SB_STRFMT_FREEFORM:
1808                 fmtstr = "nbkts=%hu, ";
1809                 break;
1810         case SB_STRFMT_JSON:
1811         default:
1812                 fmtstr = "\"nbkts\":%hu,";
1813                 break;
1814         }
1815         sbuf_printf(buf, fmtstr, nbkts);
1816
1817         switch (fmt) {
1818                 case SB_STRFMT_FREEFORM:
1819                         fmtstr = (is32bit ? "oob=%u" : "oob=%ju");
1820                         break;
1821                 case SB_STRFMT_JSON:
1822                 default:
1823                         fmtstr = (is32bit ? "\"oob\":%u,\"bkts\":[" :
1824                             "\"oob\":%ju,\"bkts\":[");
1825                         break;
1826         }
1827         sbuf_printf(buf, fmtstr, is32bit ? VSD_CONSTHIST_FIELDVAL(hist,
1828             hist_dtype, oob) : (uintmax_t)VSD_CONSTHIST_FIELDVAL(hist,
1829             hist_dtype, oob));
1830
1831         for (i = 0; i < nbkts; i++) {
1832                 switch (hist_dtype) {
1833                 case VSD_DTYPE_CRHIST32:
1834                 case VSD_DTYPE_CRHIST64:
1835                         bkt_lb = VSD_CONSTCRHIST_FIELDPTR(hist, hist_dtype,
1836                             bkts[i].lb);
1837                         if (i < nbkts - 1)
1838                                 bkt_ub = VSD_CONSTCRHIST_FIELDPTR(hist,
1839                                     hist_dtype, bkts[i + 1].lb);
1840                         else
1841                                 bkt_ub = &numeric_limits[LIM_MAX][voi_dtype];
1842                         break;
1843                 case VSD_DTYPE_DRHIST32:
1844                 case VSD_DTYPE_DRHIST64:
1845                         bkt_lb = VSD_CONSTDRHIST_FIELDPTR(hist, hist_dtype,
1846                             bkts[i].lb);
1847                         bkt_ub = VSD_CONSTDRHIST_FIELDPTR(hist, hist_dtype,
1848                             bkts[i].ub);
1849                         break;
1850                 case VSD_DTYPE_DVHIST32:
1851                 case VSD_DTYPE_DVHIST64:
1852                         bkt_lb = bkt_ub = VSD_CONSTDVHIST_FIELDPTR(hist,
1853                             hist_dtype, bkts[i].val);
1854                         break;
1855                 default:
1856                         break;
1857                 }
1858
1859                 switch (fmt) {
1860                 case SB_STRFMT_FREEFORM:
1861                         fmtstr = "\n\t\t\t\t";
1862                         break;
1863                 case SB_STRFMT_JSON:
1864                 default:
1865                         fmtstr = "{";
1866                         break;
1867                 }
1868                 sbuf_cat(buf, fmtstr);
1869
1870                 if (objdump) {
1871                         switch (fmt) {
1872                         case SB_STRFMT_FREEFORM:
1873                                 fmtstr = "bkt[%hu].";
1874                                 break;
1875                         case SB_STRFMT_JSON:
1876                         default:
1877                                 fmtstr = "\"bkt\":%hu,";
1878                                 break;
1879                         }
1880                         sbuf_printf(buf, fmtstr, i);
1881                 }
1882
1883                 switch (fmt) {
1884                 case SB_STRFMT_FREEFORM:
1885                         fmtstr = "{lb=";
1886                         break;
1887                 case SB_STRFMT_JSON:
1888                 default:
1889                         fmtstr = "\"lb\":";
1890                         break;
1891                 }
1892                 sbuf_cat(buf, fmtstr);
1893                 stats_voistatdata_tostr((const struct voistatdata *)bkt_lb,
1894                     voi_dtype, voi_dtype, sizeof(struct voistatdata_numeric),
1895                     fmt, buf, objdump);
1896
1897                 switch (fmt) {
1898                 case SB_STRFMT_FREEFORM:
1899                         fmtstr = ",ub=";
1900                         break;
1901                 case SB_STRFMT_JSON:
1902                 default:
1903                         fmtstr = ",\"ub\":";
1904                         break;
1905                 }
1906                 sbuf_cat(buf, fmtstr);
1907                 stats_voistatdata_tostr((const struct voistatdata *)bkt_ub,
1908                     voi_dtype, voi_dtype, sizeof(struct voistatdata_numeric),
1909                     fmt, buf, objdump);
1910
1911                 switch (fmt) {
1912                 case SB_STRFMT_FREEFORM:
1913                         fmtstr = is32bit ? ",cnt=%u}" : ",cnt=%ju}";
1914                         break;
1915                 case SB_STRFMT_JSON:
1916                 default:
1917                         fmtstr = is32bit ? ",\"cnt\":%u}" : ",\"cnt\":%ju}";
1918                         break;
1919                 }
1920                 sbuf_printf(buf, fmtstr, is32bit ?
1921                     VSD_CONSTHIST_FIELDVAL(hist, hist_dtype, bkts[i].cnt) :
1922                     (uintmax_t)VSD_CONSTHIST_FIELDVAL(hist, hist_dtype,
1923                     bkts[i].cnt));
1924
1925                 if (fmt == SB_STRFMT_JSON && i < nbkts - 1)
1926                         sbuf_putc(buf, ',');
1927         }
1928         if (fmt == SB_STRFMT_JSON)
1929                 sbuf_cat(buf, "]");
1930 }
1931
1932 int
1933 stats_voistatdata_tostr(const struct voistatdata *vsd, enum vsd_dtype voi_dtype,
1934     enum vsd_dtype vsd_dtype, size_t vsd_sz, enum sb_str_fmt fmt,
1935     struct sbuf *buf, int objdump)
1936 {
1937         const char *fmtstr;
1938
1939         if (vsd == NULL || buf == NULL || voi_dtype >= VSD_NUM_DTYPES ||
1940             vsd_dtype >= VSD_NUM_DTYPES || fmt >= SB_STRFMT_NUM_FMTS)
1941                 return (EINVAL);
1942
1943         switch (vsd_dtype) {
1944         case VSD_DTYPE_VOISTATE:
1945                 switch (fmt) {
1946                 case SB_STRFMT_FREEFORM:
1947                         fmtstr = "prev=";
1948                         break;
1949                 case SB_STRFMT_JSON:
1950                 default:
1951                         fmtstr = "\"prev\":";
1952                         break;
1953                 }
1954                 sbuf_cat(buf, fmtstr);
1955                 /*
1956                  * Render prev by passing it as *vsd and voi_dtype as vsd_dtype.
1957                  */
1958                 stats_voistatdata_tostr(
1959                     (const struct voistatdata *)&CONSTVSD(voistate, vsd)->prev,
1960                     voi_dtype, voi_dtype, vsd_sz, fmt, buf, objdump);
1961                 break;
1962         case VSD_DTYPE_INT_S32:
1963                 sbuf_printf(buf, "%d", vsd->int32.s32);
1964                 break;
1965         case VSD_DTYPE_INT_U32:
1966                 sbuf_printf(buf, "%u", vsd->int32.u32);
1967                 break;
1968         case VSD_DTYPE_INT_S64:
1969                 sbuf_printf(buf, "%jd", (intmax_t)vsd->int64.s64);
1970                 break;
1971         case VSD_DTYPE_INT_U64:
1972                 sbuf_printf(buf, "%ju", (uintmax_t)vsd->int64.u64);
1973                 break;
1974         case VSD_DTYPE_INT_SLONG:
1975                 sbuf_printf(buf, "%ld", vsd->intlong.slong);
1976                 break;
1977         case VSD_DTYPE_INT_ULONG:
1978                 sbuf_printf(buf, "%lu", vsd->intlong.ulong);
1979                 break;
1980         case VSD_DTYPE_Q_S32:
1981                 {
1982                 char qstr[Q_MAXSTRLEN(vsd->q32.sq32, 10)];
1983                 Q_TOSTR((s32q_t)vsd->q32.sq32, -1, 10, qstr, sizeof(qstr));
1984                 sbuf_cat(buf, qstr);
1985                 }
1986                 break;
1987         case VSD_DTYPE_Q_U32:
1988                 {
1989                 char qstr[Q_MAXSTRLEN(vsd->q32.uq32, 10)];
1990                 Q_TOSTR((u32q_t)vsd->q32.uq32, -1, 10, qstr, sizeof(qstr));
1991                 sbuf_cat(buf, qstr);
1992                 }
1993                 break;
1994         case VSD_DTYPE_Q_S64:
1995                 {
1996                 char qstr[Q_MAXSTRLEN(vsd->q64.sq64, 10)];
1997                 Q_TOSTR((s64q_t)vsd->q64.sq64, -1, 10, qstr, sizeof(qstr));
1998                 sbuf_cat(buf, qstr);
1999                 }
2000                 break;
2001         case VSD_DTYPE_Q_U64:
2002                 {
2003                 char qstr[Q_MAXSTRLEN(vsd->q64.uq64, 10)];
2004                 Q_TOSTR((u64q_t)vsd->q64.uq64, -1, 10, qstr, sizeof(qstr));
2005                 sbuf_cat(buf, qstr);
2006                 }
2007                 break;
2008         case VSD_DTYPE_CRHIST32:
2009         case VSD_DTYPE_DRHIST32:
2010         case VSD_DTYPE_DVHIST32:
2011         case VSD_DTYPE_CRHIST64:
2012         case VSD_DTYPE_DRHIST64:
2013         case VSD_DTYPE_DVHIST64:
2014                 stats_voistatdata_hist_tostr(voi_dtype, CONSTVSD(hist, vsd),
2015                     vsd_dtype, vsd_sz, fmt, buf, objdump);
2016                 break;
2017         case VSD_DTYPE_TDGSTCLUST32:
2018         case VSD_DTYPE_TDGSTCLUST64:
2019                 stats_voistatdata_tdgst_tostr(voi_dtype,
2020                     CONSTVSD(tdgst, vsd), vsd_dtype, vsd_sz, fmt, buf,
2021                     objdump);
2022                 break;
2023         default:
2024                 break;
2025         }
2026
2027         return (sbuf_error(buf));
2028 }
2029
2030 static void
2031 stats_v1_itercb_tostr_freeform(struct statsblobv1 *sb, struct voi *v,
2032     struct voistat *vs, struct sb_iter_ctx *ctx)
2033 {
2034         struct sb_tostrcb_ctx *sctx;
2035         struct metablob *tpl_mb;
2036         struct sbuf *buf;
2037         void *vsd;
2038         uint8_t dump;
2039
2040         sctx = ctx->usrctx;
2041         buf = sctx->buf;
2042         tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL;
2043         dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0);
2044
2045         if (ctx->flags & SB_IT_FIRST_CB) {
2046                 sbuf_printf(buf, "struct statsblobv1@%p", sb);
2047                 if (dump) {
2048                         sbuf_printf(buf, ", abi=%hhu, endian=%hhu, maxsz=%hu, "
2049                             "cursz=%hu, created=%jd, lastrst=%jd, flags=0x%04hx, "
2050                             "stats_off=%hu, statsdata_off=%hu",
2051                             sb->abi, sb->endian, sb->maxsz, sb->cursz,
2052                             sb->created, sb->lastrst, sb->flags, sb->stats_off,
2053                             sb->statsdata_off);
2054                 }
2055                 sbuf_printf(buf, ", tplhash=%u", sb->tplhash);
2056         }
2057
2058         if (ctx->flags & SB_IT_FIRST_VOISTAT) {
2059                 sbuf_printf(buf, "\n\tvois[%hd]: id=%hd", ctx->vslot, v->id);
2060                 if (v->id < 0)
2061                         return;
2062                 sbuf_printf(buf, ", name=\"%s\"", (tpl_mb == NULL) ? "" :
2063                     tpl_mb->voi_meta[v->id].name);
2064                 if (dump)
2065                     sbuf_printf(buf, ", flags=0x%04hx, dtype=%s, "
2066                     "voistatmaxid=%hhd, stats_off=%hu", v->flags,
2067                     vsd_dtype2name[v->dtype], v->voistatmaxid, v->stats_off);
2068         }
2069
2070         if (!dump && vs->stype <= 0)
2071                 return;
2072
2073         sbuf_printf(buf, "\n\t\tvois[%hd]stat[%hhd]: stype=", v->id, ctx->vsslot);
2074         if (vs->stype < 0) {
2075                 sbuf_printf(buf, "%hhd", vs->stype);
2076                 return;
2077         } else
2078                 sbuf_printf(buf, "%s, errs=%hu", vs_stype2name[vs->stype],
2079                     vs->errs);
2080         vsd = BLOB_OFFSET(sb, vs->data_off);
2081         if (dump)
2082                 sbuf_printf(buf, ", flags=0x%04x, dtype=%s, dsz=%hu, "
2083                     "data_off=%hu", vs->flags, vsd_dtype2name[vs->dtype],
2084                     vs->dsz, vs->data_off);
2085
2086         sbuf_printf(buf, "\n\t\t\tvoistatdata: ");
2087         stats_voistatdata_tostr(vsd, v->dtype, vs->dtype, vs->dsz,
2088             sctx->fmt, buf, dump);
2089 }
2090
2091 static void
2092 stats_v1_itercb_tostr_json(struct statsblobv1 *sb, struct voi *v, struct voistat *vs,
2093     struct sb_iter_ctx *ctx)
2094 {
2095         struct sb_tostrcb_ctx *sctx;
2096         struct metablob *tpl_mb;
2097         struct sbuf *buf;
2098         const char *fmtstr;
2099         void *vsd;
2100         uint8_t dump;
2101
2102         sctx = ctx->usrctx;
2103         buf = sctx->buf;
2104         tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL;
2105         dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0);
2106
2107         if (ctx->flags & SB_IT_FIRST_CB) {
2108                 sbuf_putc(buf, '{');
2109                 if (dump) {
2110                         sbuf_printf(buf, "\"abi\":%hhu,\"endian\":%hhu,"
2111                             "\"maxsz\":%hu,\"cursz\":%hu,\"created\":%jd,"
2112                             "\"lastrst\":%jd,\"flags\":%hu,\"stats_off\":%hu,"
2113                             "\"statsdata_off\":%hu,", sb->abi,
2114                             sb->endian, sb->maxsz, sb->cursz, sb->created,
2115                             sb->lastrst, sb->flags, sb->stats_off,
2116                             sb->statsdata_off);
2117                 }
2118
2119                 if (tpl_mb == NULL)
2120                         fmtstr = "\"tplname\":%s,\"tplhash\":%u,\"vois\":{";
2121                 else
2122                         fmtstr = "\"tplname\":\"%s\",\"tplhash\":%u,\"vois\":{";
2123
2124                 sbuf_printf(buf, fmtstr, tpl_mb ? tpl_mb->tplname : "null",
2125                     sb->tplhash);
2126         }
2127
2128         if (ctx->flags & SB_IT_FIRST_VOISTAT) {
2129                 if (dump) {
2130                         sbuf_printf(buf, "\"[%d]\":{\"id\":%d", ctx->vslot,
2131                             v->id);
2132                         if (v->id < 0) {
2133                                 sbuf_printf(buf, "},");
2134                                 return;
2135                         }
2136
2137                         if (tpl_mb == NULL)
2138                                 fmtstr = ",\"name\":%s,\"flags\":%hu,"
2139                                     "\"dtype\":\"%s\",\"voistatmaxid\":%hhd,"
2140                                     "\"stats_off\":%hu,";
2141                         else
2142                                 fmtstr = ",\"name\":\"%s\",\"flags\":%hu,"
2143                                     "\"dtype\":\"%s\",\"voistatmaxid\":%hhd,"
2144                                     "\"stats_off\":%hu,";
2145
2146                         sbuf_printf(buf, fmtstr, tpl_mb ?
2147                             tpl_mb->voi_meta[v->id].name : "null", v->flags,
2148                             vsd_dtype2name[v->dtype], v->voistatmaxid,
2149                             v->stats_off);
2150                 } else {
2151                         if (tpl_mb == NULL) {
2152                                 sbuf_printf(buf, "\"[%hd]\":{", v->id);
2153                         } else {
2154                                 sbuf_printf(buf, "\"%s\":{",
2155                                     tpl_mb->voi_meta[v->id].name);
2156                         }
2157                 }
2158                 sbuf_cat(buf, "\"stats\":{");
2159         }
2160
2161         vsd = BLOB_OFFSET(sb, vs->data_off);
2162         if (dump) {
2163                 sbuf_printf(buf, "\"[%hhd]\":", ctx->vsslot);
2164                 if (vs->stype < 0) {
2165                         sbuf_printf(buf, "{\"stype\":-1},");
2166                         return;
2167                 }
2168                 sbuf_printf(buf, "{\"stype\":\"%s\",\"errs\":%hu,\"flags\":%hu,"
2169                     "\"dtype\":\"%s\",\"data_off\":%hu,\"voistatdata\":{",
2170                     vs_stype2name[vs->stype], vs->errs, vs->flags,
2171                     vsd_dtype2name[vs->dtype], vs->data_off);
2172         } else if (vs->stype > 0) {
2173                 if (tpl_mb == NULL)
2174                         sbuf_printf(buf, "\"[%hhd]\":", vs->stype);
2175                 else
2176                         sbuf_printf(buf, "\"%s\":", vs_stype2name[vs->stype]);
2177         } else
2178                 return;
2179
2180         if ((vs->flags & VS_VSDVALID) || dump) {
2181                 if (!dump)
2182                         sbuf_printf(buf, "{\"errs\":%hu,", vs->errs);
2183                 /* Simple non-compound VSD types need a key. */
2184                 if (!vsd_compoundtype[vs->dtype])
2185                         sbuf_cat(buf, "\"val\":");
2186                 stats_voistatdata_tostr(vsd, v->dtype, vs->dtype, vs->dsz,
2187                     sctx->fmt, buf, dump);
2188                 sbuf_cat(buf, dump ? "}}" : "}");
2189         } else
2190                 sbuf_cat(buf, dump ? "null}" : "null");
2191
2192         if (ctx->flags & SB_IT_LAST_VOISTAT)
2193                 sbuf_cat(buf, "}}");
2194
2195         if (ctx->flags & SB_IT_LAST_CB)
2196                 sbuf_cat(buf, "}}");
2197         else
2198                 sbuf_putc(buf, ',');
2199 }
2200
2201 static int
2202 stats_v1_itercb_tostr(struct statsblobv1 *sb, struct voi *v, struct voistat *vs,
2203     struct sb_iter_ctx *ctx)
2204 {
2205         struct sb_tostrcb_ctx *sctx;
2206
2207         sctx = ctx->usrctx;
2208
2209         switch (sctx->fmt) {
2210         case SB_STRFMT_FREEFORM:
2211                 stats_v1_itercb_tostr_freeform(sb, v, vs, ctx);
2212                 break;
2213         case SB_STRFMT_JSON:
2214                 stats_v1_itercb_tostr_json(sb, v, vs, ctx);
2215                 break;
2216         default:
2217                 break;
2218         }
2219
2220         return (sbuf_error(sctx->buf));
2221 }
2222
2223 int
2224 stats_v1_blob_tostr(struct statsblobv1 *sb, struct sbuf *buf,
2225     enum sb_str_fmt fmt, uint32_t flags)
2226 {
2227         struct sb_tostrcb_ctx sctx;
2228         uint32_t iflags;
2229
2230         if (sb == NULL || sb->abi != STATS_ABI_V1 || buf == NULL ||
2231             fmt >= SB_STRFMT_NUM_FMTS)
2232                 return (EINVAL);
2233
2234         sctx.buf = buf;
2235         sctx.fmt = fmt;
2236         sctx.flags = flags;
2237
2238         if (flags & SB_TOSTR_META) {
2239                 if (stats_tpl_fetch(stats_tpl_fetch_allocid(NULL, sb->tplhash),
2240                     &sctx.tpl))
2241                         return (EINVAL);
2242         } else
2243                 sctx.tpl = NULL;
2244
2245         iflags = 0;
2246         if (flags & SB_TOSTR_OBJDUMP)
2247                 iflags |= (SB_IT_NULLVOI | SB_IT_NULLVOISTAT);
2248         stats_v1_blob_iter(sb, stats_v1_itercb_tostr, &sctx, iflags);
2249
2250         return (sbuf_error(buf));
2251 }
2252
2253 static int
2254 stats_v1_itercb_visit(struct statsblobv1 *sb, struct voi *v,
2255     struct voistat *vs, struct sb_iter_ctx *ctx)
2256 {
2257         struct sb_visitcb_ctx *vctx;
2258         struct sb_visit sbv;
2259
2260         vctx = ctx->usrctx;
2261
2262         sbv.tplhash = sb->tplhash;
2263         sbv.voi_id = v->id;
2264         sbv.voi_dtype = v->dtype;
2265         sbv.vs_stype = vs->stype;
2266         sbv.vs_dtype = vs->dtype;
2267         sbv.vs_dsz = vs->dsz;
2268         sbv.vs_data = BLOB_OFFSET(sb, vs->data_off);
2269         sbv.vs_errs = vs->errs;
2270         sbv.flags = ctx->flags & (SB_IT_FIRST_CB | SB_IT_LAST_CB |
2271             SB_IT_FIRST_VOI | SB_IT_LAST_VOI | SB_IT_FIRST_VOISTAT |
2272             SB_IT_LAST_VOISTAT);
2273
2274         return (vctx->cb(&sbv, vctx->usrctx));
2275 }
2276
2277 int
2278 stats_v1_blob_visit(struct statsblobv1 *sb, stats_blob_visitcb_t func,
2279     void *usrctx)
2280 {
2281         struct sb_visitcb_ctx vctx;
2282
2283         if (sb == NULL || sb->abi != STATS_ABI_V1 || func == NULL)
2284                 return (EINVAL);
2285
2286         vctx.cb = func;
2287         vctx.usrctx = usrctx;
2288
2289         stats_v1_blob_iter(sb, stats_v1_itercb_visit, &vctx, 0);
2290
2291         return (0);
2292 }
2293
2294 static int
2295 stats_v1_icb_reset_voistat(struct statsblobv1 *sb, struct voi *v __unused,
2296     struct voistat *vs, struct sb_iter_ctx *ctx __unused)
2297 {
2298         void *vsd;
2299
2300         if (vs->stype == VS_STYPE_VOISTATE)
2301                 return (0);
2302
2303         vsd = BLOB_OFFSET(sb, vs->data_off);
2304
2305         /* Perform the stat type's default reset action. */
2306         switch (vs->stype) {
2307         case VS_STYPE_SUM:
2308                 switch (vs->dtype) {
2309                 case VSD_DTYPE_Q_S32:
2310                         Q_SIFVAL(VSD(q32, vsd)->sq32, 0);
2311                         break;
2312                 case VSD_DTYPE_Q_U32:
2313                         Q_SIFVAL(VSD(q32, vsd)->uq32, 0);
2314                         break;
2315                 case VSD_DTYPE_Q_S64:
2316                         Q_SIFVAL(VSD(q64, vsd)->sq64, 0);
2317                         break;
2318                 case VSD_DTYPE_Q_U64:
2319                         Q_SIFVAL(VSD(q64, vsd)->uq64, 0);
2320                         break;
2321                 default:
2322                         bzero(vsd, vs->dsz);
2323                         break;
2324                 }
2325                 break;
2326         case VS_STYPE_MAX:
2327                 switch (vs->dtype) {
2328                 case VSD_DTYPE_Q_S32:
2329                         Q_SIFVAL(VSD(q32, vsd)->sq32,
2330                             Q_IFMINVAL(VSD(q32, vsd)->sq32));
2331                         break;
2332                 case VSD_DTYPE_Q_U32:
2333                         Q_SIFVAL(VSD(q32, vsd)->uq32,
2334                             Q_IFMINVAL(VSD(q32, vsd)->uq32));
2335                         break;
2336                 case VSD_DTYPE_Q_S64:
2337                         Q_SIFVAL(VSD(q64, vsd)->sq64,
2338                             Q_IFMINVAL(VSD(q64, vsd)->sq64));
2339                         break;
2340                 case VSD_DTYPE_Q_U64:
2341                         Q_SIFVAL(VSD(q64, vsd)->uq64,
2342                             Q_IFMINVAL(VSD(q64, vsd)->uq64));
2343                         break;
2344                 default:
2345                         memcpy(vsd, &numeric_limits[LIM_MIN][vs->dtype],
2346                             vs->dsz);
2347                         break;
2348                 }
2349                 break;
2350         case VS_STYPE_MIN:
2351                 switch (vs->dtype) {
2352                 case VSD_DTYPE_Q_S32:
2353                         Q_SIFVAL(VSD(q32, vsd)->sq32,
2354                             Q_IFMAXVAL(VSD(q32, vsd)->sq32));
2355                         break;
2356                 case VSD_DTYPE_Q_U32:
2357                         Q_SIFVAL(VSD(q32, vsd)->uq32,
2358                             Q_IFMAXVAL(VSD(q32, vsd)->uq32));
2359                         break;
2360                 case VSD_DTYPE_Q_S64:
2361                         Q_SIFVAL(VSD(q64, vsd)->sq64,
2362                             Q_IFMAXVAL(VSD(q64, vsd)->sq64));
2363                         break;
2364                 case VSD_DTYPE_Q_U64:
2365                         Q_SIFVAL(VSD(q64, vsd)->uq64,
2366                             Q_IFMAXVAL(VSD(q64, vsd)->uq64));
2367                         break;
2368                 default:
2369                         memcpy(vsd, &numeric_limits[LIM_MAX][vs->dtype],
2370                             vs->dsz);
2371                         break;
2372                 }
2373                 break;
2374         case VS_STYPE_HIST:
2375                 {
2376                 /* Reset bucket counts. */
2377                 struct voistatdata_hist *hist;
2378                 int i, is32bit;
2379                 uint16_t nbkts;
2380
2381                 hist = VSD(hist, vsd);
2382                 switch (vs->dtype) {
2383                 case VSD_DTYPE_CRHIST32:
2384                         nbkts = HIST_VSDSZ2NBKTS(crhist32, vs->dsz);
2385                         is32bit = 1;
2386                         break;
2387                 case VSD_DTYPE_DRHIST32:
2388                         nbkts = HIST_VSDSZ2NBKTS(drhist32, vs->dsz);
2389                         is32bit = 1;
2390                         break;
2391                 case VSD_DTYPE_DVHIST32:
2392                         nbkts = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz);
2393                         is32bit = 1;
2394                         break;
2395                 case VSD_DTYPE_CRHIST64:
2396                         nbkts = HIST_VSDSZ2NBKTS(crhist64, vs->dsz);
2397                         is32bit = 0;
2398                         break;
2399                 case VSD_DTYPE_DRHIST64:
2400                         nbkts = HIST_VSDSZ2NBKTS(drhist64, vs->dsz);
2401                         is32bit = 0;
2402                         break;
2403                 case VSD_DTYPE_DVHIST64:
2404                         nbkts = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz);
2405                         is32bit = 0;
2406                         break;
2407                 default:
2408                         return (0);
2409                 }
2410
2411                 bzero(VSD_HIST_FIELDPTR(hist, vs->dtype, oob),
2412                     is32bit ? sizeof(uint32_t) : sizeof(uint64_t));
2413                 for (i = nbkts - 1; i >= 0; i--) {
2414                         bzero(VSD_HIST_FIELDPTR(hist, vs->dtype,
2415                             bkts[i].cnt), is32bit ? sizeof(uint32_t) :
2416                             sizeof(uint64_t));
2417                 }
2418                 break;
2419                 }
2420         case VS_STYPE_TDGST:
2421                 {
2422                 /* Reset sample count centroids array/tree. */
2423                 struct voistatdata_tdgst *tdgst;
2424                 struct ctdth32 *ctd32tree;
2425                 struct ctdth64 *ctd64tree;
2426                 struct voistatdata_tdgstctd32 *ctd32;
2427                 struct voistatdata_tdgstctd64 *ctd64;
2428
2429                 tdgst = VSD(tdgst, vsd);
2430                 switch (vs->dtype) {
2431                 case VSD_DTYPE_TDGSTCLUST32:
2432                         VSD(tdgstclust32, tdgst)->smplcnt = 0;
2433                         VSD(tdgstclust32, tdgst)->compcnt = 0;
2434                         ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
2435                         ARB_INIT(ctd32, ctdlnk, ctd32tree,
2436                             ARB_MAXNODES(ctd32tree)) {
2437                                 ctd32->cnt = 0;
2438                                 Q_SIFVAL(ctd32->mu, 0);
2439                         }
2440 #ifdef DIAGNOSTIC
2441                         RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree);
2442 #endif
2443                 break;
2444                 case VSD_DTYPE_TDGSTCLUST64:
2445                         VSD(tdgstclust64, tdgst)->smplcnt = 0;
2446                         VSD(tdgstclust64, tdgst)->compcnt = 0;
2447                         ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
2448                         ARB_INIT(ctd64, ctdlnk, ctd64tree,
2449                             ARB_MAXNODES(ctd64tree)) {
2450                                 ctd64->cnt = 0;
2451                                 Q_SIFVAL(ctd64->mu, 0);
2452                         }
2453 #ifdef DIAGNOSTIC
2454                         RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree);
2455 #endif
2456                 break;
2457                 default:
2458                         return (0);
2459                 }
2460                 break;
2461                 }
2462         default:
2463                 KASSERT(0, ("Unknown VOI stat type %d", vs->stype));
2464                 break;
2465         }
2466
2467         vs->errs = 0;
2468         vs->flags &= ~VS_VSDVALID;
2469
2470         return (0);
2471 }
2472
2473 int
2474 stats_v1_blob_snapshot(struct statsblobv1 **dst, size_t dstmaxsz,
2475     struct statsblobv1 *src, uint32_t flags)
2476 {
2477         int error;
2478
2479         if (src != NULL && src->abi == STATS_ABI_V1) {
2480                 error = stats_v1_blob_clone(dst, dstmaxsz, src, flags);
2481                 if (!error) {
2482                         if (flags & SB_CLONE_RSTSRC) {
2483                                 stats_v1_blob_iter(src,
2484                                     stats_v1_icb_reset_voistat, NULL, 0);
2485                                 src->lastrst = stats_sbinuptime();
2486                         }
2487                         stats_v1_blob_finalise(*dst);
2488                 }
2489         } else
2490                 error = EINVAL;
2491
2492         return (error);
2493 }
2494
2495 static inline int
2496 stats_v1_voi_update_max(enum vsd_dtype voi_dtype __unused,
2497     struct voistatdata *voival, struct voistat *vs, void *vsd)
2498 {
2499         int error;
2500
2501         KASSERT(vs->dtype < VSD_NUM_DTYPES,
2502             ("Unknown VSD dtype %d", vs->dtype));
2503
2504         error = 0;
2505
2506         switch (vs->dtype) {
2507         case VSD_DTYPE_INT_S32:
2508                 if (VSD(int32, vsd)->s32 < voival->int32.s32) {
2509                         VSD(int32, vsd)->s32 = voival->int32.s32;
2510                         vs->flags |= VS_VSDVALID;
2511                 }
2512                 break;
2513         case VSD_DTYPE_INT_U32:
2514                 if (VSD(int32, vsd)->u32 < voival->int32.u32) {
2515                         VSD(int32, vsd)->u32 = voival->int32.u32;
2516                         vs->flags |= VS_VSDVALID;
2517                 }
2518                 break;
2519         case VSD_DTYPE_INT_S64:
2520                 if (VSD(int64, vsd)->s64 < voival->int64.s64) {
2521                         VSD(int64, vsd)->s64 = voival->int64.s64;
2522                         vs->flags |= VS_VSDVALID;
2523                 }
2524                 break;
2525         case VSD_DTYPE_INT_U64:
2526                 if (VSD(int64, vsd)->u64 < voival->int64.u64) {
2527                         VSD(int64, vsd)->u64 = voival->int64.u64;
2528                         vs->flags |= VS_VSDVALID;
2529                 }
2530                 break;
2531         case VSD_DTYPE_INT_SLONG:
2532                 if (VSD(intlong, vsd)->slong < voival->intlong.slong) {
2533                         VSD(intlong, vsd)->slong = voival->intlong.slong;
2534                         vs->flags |= VS_VSDVALID;
2535                 }
2536                 break;
2537         case VSD_DTYPE_INT_ULONG:
2538                 if (VSD(intlong, vsd)->ulong < voival->intlong.ulong) {
2539                         VSD(intlong, vsd)->ulong = voival->intlong.ulong;
2540                         vs->flags |= VS_VSDVALID;
2541                 }
2542                 break;
2543         case VSD_DTYPE_Q_S32:
2544                 if (Q_QLTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) &&
2545                     (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32,
2546                     voival->q32.sq32)))) {
2547                         vs->flags |= VS_VSDVALID;
2548                 }
2549                 break;
2550         case VSD_DTYPE_Q_U32:
2551                 if (Q_QLTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) &&
2552                     (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32,
2553                     voival->q32.uq32)))) {
2554                         vs->flags |= VS_VSDVALID;
2555                 }
2556                 break;
2557         case VSD_DTYPE_Q_S64:
2558                 if (Q_QLTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) &&
2559                     (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64,
2560                     voival->q64.sq64)))) {
2561                         vs->flags |= VS_VSDVALID;
2562                 }
2563                 break;
2564         case VSD_DTYPE_Q_U64:
2565                 if (Q_QLTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) &&
2566                     (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64,
2567                     voival->q64.uq64)))) {
2568                         vs->flags |= VS_VSDVALID;
2569                 }
2570                 break;
2571         default:
2572                 error = EINVAL;
2573                 break;
2574         }
2575
2576         return (error);
2577 }
2578
2579 static inline int
2580 stats_v1_voi_update_min(enum vsd_dtype voi_dtype __unused,
2581     struct voistatdata *voival, struct voistat *vs, void *vsd)
2582 {
2583         int error;
2584
2585         KASSERT(vs->dtype < VSD_NUM_DTYPES,
2586             ("Unknown VSD dtype %d", vs->dtype));
2587
2588         error = 0;
2589
2590         switch (vs->dtype) {
2591         case VSD_DTYPE_INT_S32:
2592                 if (VSD(int32, vsd)->s32 > voival->int32.s32) {
2593                         VSD(int32, vsd)->s32 = voival->int32.s32;
2594                         vs->flags |= VS_VSDVALID;
2595                 }
2596                 break;
2597         case VSD_DTYPE_INT_U32:
2598                 if (VSD(int32, vsd)->u32 > voival->int32.u32) {
2599                         VSD(int32, vsd)->u32 = voival->int32.u32;
2600                         vs->flags |= VS_VSDVALID;
2601                 }
2602                 break;
2603         case VSD_DTYPE_INT_S64:
2604                 if (VSD(int64, vsd)->s64 > voival->int64.s64) {
2605                         VSD(int64, vsd)->s64 = voival->int64.s64;
2606                         vs->flags |= VS_VSDVALID;
2607                 }
2608                 break;
2609         case VSD_DTYPE_INT_U64:
2610                 if (VSD(int64, vsd)->u64 > voival->int64.u64) {
2611                         VSD(int64, vsd)->u64 = voival->int64.u64;
2612                         vs->flags |= VS_VSDVALID;
2613                 }
2614                 break;
2615         case VSD_DTYPE_INT_SLONG:
2616                 if (VSD(intlong, vsd)->slong > voival->intlong.slong) {
2617                         VSD(intlong, vsd)->slong = voival->intlong.slong;
2618                         vs->flags |= VS_VSDVALID;
2619                 }
2620                 break;
2621         case VSD_DTYPE_INT_ULONG:
2622                 if (VSD(intlong, vsd)->ulong > voival->intlong.ulong) {
2623                         VSD(intlong, vsd)->ulong = voival->intlong.ulong;
2624                         vs->flags |= VS_VSDVALID;
2625                 }
2626                 break;
2627         case VSD_DTYPE_Q_S32:
2628                 if (Q_QGTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) &&
2629                     (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32,
2630                     voival->q32.sq32)))) {
2631                         vs->flags |= VS_VSDVALID;
2632                 }
2633                 break;
2634         case VSD_DTYPE_Q_U32:
2635                 if (Q_QGTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) &&
2636                     (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32,
2637                     voival->q32.uq32)))) {
2638                         vs->flags |= VS_VSDVALID;
2639                 }
2640                 break;
2641         case VSD_DTYPE_Q_S64:
2642                 if (Q_QGTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) &&
2643                     (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64,
2644                     voival->q64.sq64)))) {
2645                         vs->flags |= VS_VSDVALID;
2646                 }
2647                 break;
2648         case VSD_DTYPE_Q_U64:
2649                 if (Q_QGTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) &&
2650                     (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64,
2651                     voival->q64.uq64)))) {
2652                         vs->flags |= VS_VSDVALID;
2653                 }
2654                 break;
2655         default:
2656                 error = EINVAL;
2657                 break;
2658         }
2659
2660         return (error);
2661 }
2662
2663 static inline int
2664 stats_v1_voi_update_sum(enum vsd_dtype voi_dtype __unused,
2665     struct voistatdata *voival, struct voistat *vs, void *vsd)
2666 {
2667         int error;
2668
2669         KASSERT(vs->dtype < VSD_NUM_DTYPES,
2670             ("Unknown VSD dtype %d", vs->dtype));
2671
2672         error = 0;
2673
2674         switch (vs->dtype) {
2675         case VSD_DTYPE_INT_S32:
2676                 VSD(int32, vsd)->s32 += voival->int32.s32;
2677                 break;
2678         case VSD_DTYPE_INT_U32:
2679                 VSD(int32, vsd)->u32 += voival->int32.u32;
2680                 break;
2681         case VSD_DTYPE_INT_S64:
2682                 VSD(int64, vsd)->s64 += voival->int64.s64;
2683                 break;
2684         case VSD_DTYPE_INT_U64:
2685                 VSD(int64, vsd)->u64 += voival->int64.u64;
2686                 break;
2687         case VSD_DTYPE_INT_SLONG:
2688                 VSD(intlong, vsd)->slong += voival->intlong.slong;
2689                 break;
2690         case VSD_DTYPE_INT_ULONG:
2691                 VSD(intlong, vsd)->ulong += voival->intlong.ulong;
2692                 break;
2693         case VSD_DTYPE_Q_S32:
2694                 error = Q_QADDQ(&VSD(q32, vsd)->sq32, voival->q32.sq32);
2695                 break;
2696         case VSD_DTYPE_Q_U32:
2697                 error = Q_QADDQ(&VSD(q32, vsd)->uq32, voival->q32.uq32);
2698                 break;
2699         case VSD_DTYPE_Q_S64:
2700                 error = Q_QADDQ(&VSD(q64, vsd)->sq64, voival->q64.sq64);
2701                 break;
2702         case VSD_DTYPE_Q_U64:
2703                 error = Q_QADDQ(&VSD(q64, vsd)->uq64, voival->q64.uq64);
2704                 break;
2705         default:
2706                 error = EINVAL;
2707                 break;
2708         }
2709
2710         if (!error)
2711                 vs->flags |= VS_VSDVALID;
2712
2713         return (error);
2714 }
2715
2716 static inline int
2717 stats_v1_voi_update_hist(enum vsd_dtype voi_dtype, struct voistatdata *voival,
2718     struct voistat *vs, struct voistatdata_hist *hist)
2719 {
2720         struct voistatdata_numeric *bkt_lb, *bkt_ub;
2721         uint64_t *oob64, *cnt64;
2722         uint32_t *oob32, *cnt32;
2723         int error, i, found, is32bit, has_ub, eq_only;
2724
2725         error = 0;
2726
2727         switch (vs->dtype) {
2728         case VSD_DTYPE_CRHIST32:
2729                 i = HIST_VSDSZ2NBKTS(crhist32, vs->dsz);
2730                 is32bit = 1;
2731                 has_ub = eq_only = 0;
2732                 oob32 = &VSD(crhist32, hist)->oob;
2733                 break;
2734         case VSD_DTYPE_DRHIST32:
2735                 i = HIST_VSDSZ2NBKTS(drhist32, vs->dsz);
2736                 is32bit = has_ub = 1;
2737                 eq_only = 0;
2738                 oob32 = &VSD(drhist32, hist)->oob;
2739                 break;
2740         case VSD_DTYPE_DVHIST32:
2741                 i = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz);
2742                 is32bit = eq_only = 1;
2743                 has_ub = 0;
2744                 oob32 = &VSD(dvhist32, hist)->oob;
2745                 break;
2746         case VSD_DTYPE_CRHIST64:
2747                 i = HIST_VSDSZ2NBKTS(crhist64, vs->dsz);
2748                 is32bit = has_ub = eq_only = 0;
2749                 oob64 = &VSD(crhist64, hist)->oob;
2750                 break;
2751         case VSD_DTYPE_DRHIST64:
2752                 i = HIST_VSDSZ2NBKTS(drhist64, vs->dsz);
2753                 is32bit = eq_only = 0;
2754                 has_ub = 1;
2755                 oob64 = &VSD(drhist64, hist)->oob;
2756                 break;
2757         case VSD_DTYPE_DVHIST64:
2758                 i = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz);
2759                 is32bit = has_ub = 0;
2760                 eq_only = 1;
2761                 oob64 = &VSD(dvhist64, hist)->oob;
2762                 break;
2763         default:
2764                 return (EINVAL);
2765         }
2766         i--; /* Adjust for 0-based array index. */
2767
2768         /* XXXLAS: Should probably use a better bucket search algorithm. ARB? */
2769         for (found = 0; i >= 0 && !found; i--) {
2770                 switch (vs->dtype) {
2771                 case VSD_DTYPE_CRHIST32:
2772                         bkt_lb = &VSD(crhist32, hist)->bkts[i].lb;
2773                         cnt32 = &VSD(crhist32, hist)->bkts[i].cnt;
2774                         break;
2775                 case VSD_DTYPE_DRHIST32:
2776                         bkt_lb = &VSD(drhist32, hist)->bkts[i].lb;
2777                         bkt_ub = &VSD(drhist32, hist)->bkts[i].ub;
2778                         cnt32 = &VSD(drhist32, hist)->bkts[i].cnt;
2779                         break;
2780                 case VSD_DTYPE_DVHIST32:
2781                         bkt_lb = &VSD(dvhist32, hist)->bkts[i].val;
2782                         cnt32 = &VSD(dvhist32, hist)->bkts[i].cnt;
2783                         break;
2784                 case VSD_DTYPE_CRHIST64:
2785                         bkt_lb = &VSD(crhist64, hist)->bkts[i].lb;
2786                         cnt64 = &VSD(crhist64, hist)->bkts[i].cnt;
2787                         break;
2788                 case VSD_DTYPE_DRHIST64:
2789                         bkt_lb = &VSD(drhist64, hist)->bkts[i].lb;
2790                         bkt_ub = &VSD(drhist64, hist)->bkts[i].ub;
2791                         cnt64 = &VSD(drhist64, hist)->bkts[i].cnt;
2792                         break;
2793                 case VSD_DTYPE_DVHIST64:
2794                         bkt_lb = &VSD(dvhist64, hist)->bkts[i].val;
2795                         cnt64 = &VSD(dvhist64, hist)->bkts[i].cnt;
2796                         break;
2797                 default:
2798                         return (EINVAL);
2799                 }
2800
2801                 switch (voi_dtype) {
2802                 case VSD_DTYPE_INT_S32:
2803                         if (voival->int32.s32 >= bkt_lb->int32.s32) {
2804                                 if ((eq_only && voival->int32.s32 ==
2805                                     bkt_lb->int32.s32) ||
2806                                     (!eq_only && (!has_ub ||
2807                                     voival->int32.s32 < bkt_ub->int32.s32)))
2808                                         found = 1;
2809                         }
2810                         break;
2811                 case VSD_DTYPE_INT_U32:
2812                         if (voival->int32.u32 >= bkt_lb->int32.u32) {
2813                                 if ((eq_only && voival->int32.u32 ==
2814                                     bkt_lb->int32.u32) ||
2815                                     (!eq_only && (!has_ub ||
2816                                     voival->int32.u32 < bkt_ub->int32.u32)))
2817                                         found = 1;
2818                         }
2819                         break;
2820                 case VSD_DTYPE_INT_S64:
2821                         if (voival->int64.s64 >= bkt_lb->int64.s64)
2822                                 if ((eq_only && voival->int64.s64 ==
2823                                     bkt_lb->int64.s64) ||
2824                                     (!eq_only && (!has_ub ||
2825                                     voival->int64.s64 < bkt_ub->int64.s64)))
2826                                         found = 1;
2827                         break;
2828                 case VSD_DTYPE_INT_U64:
2829                         if (voival->int64.u64 >= bkt_lb->int64.u64)
2830                                 if ((eq_only && voival->int64.u64 ==
2831                                     bkt_lb->int64.u64) ||
2832                                     (!eq_only && (!has_ub ||
2833                                     voival->int64.u64 < bkt_ub->int64.u64)))
2834                                         found = 1;
2835                         break;
2836                 case VSD_DTYPE_INT_SLONG:
2837                         if (voival->intlong.slong >= bkt_lb->intlong.slong)
2838                                 if ((eq_only && voival->intlong.slong ==
2839                                     bkt_lb->intlong.slong) ||
2840                                     (!eq_only && (!has_ub ||
2841                                     voival->intlong.slong <
2842                                     bkt_ub->intlong.slong)))
2843                                         found = 1;
2844                         break;
2845                 case VSD_DTYPE_INT_ULONG:
2846                         if (voival->intlong.ulong >= bkt_lb->intlong.ulong)
2847                                 if ((eq_only && voival->intlong.ulong ==
2848                                     bkt_lb->intlong.ulong) ||
2849                                     (!eq_only && (!has_ub ||
2850                                     voival->intlong.ulong <
2851                                     bkt_ub->intlong.ulong)))
2852                                         found = 1;
2853                         break;
2854                 case VSD_DTYPE_Q_S32:
2855                         if (Q_QGEQ(voival->q32.sq32, bkt_lb->q32.sq32))
2856                                 if ((eq_only && Q_QEQ(voival->q32.sq32,
2857                                     bkt_lb->q32.sq32)) ||
2858                                     (!eq_only && (!has_ub ||
2859                                     Q_QLTQ(voival->q32.sq32,
2860                                     bkt_ub->q32.sq32))))
2861                                         found = 1;
2862                         break;
2863                 case VSD_DTYPE_Q_U32:
2864                         if (Q_QGEQ(voival->q32.uq32, bkt_lb->q32.uq32))
2865                                 if ((eq_only && Q_QEQ(voival->q32.uq32,
2866                                     bkt_lb->q32.uq32)) ||
2867                                     (!eq_only && (!has_ub ||
2868                                     Q_QLTQ(voival->q32.uq32,
2869                                     bkt_ub->q32.uq32))))
2870                                         found = 1;
2871                         break;
2872                 case VSD_DTYPE_Q_S64:
2873                         if (Q_QGEQ(voival->q64.sq64, bkt_lb->q64.sq64))
2874                                 if ((eq_only && Q_QEQ(voival->q64.sq64,
2875                                     bkt_lb->q64.sq64)) ||
2876                                     (!eq_only && (!has_ub ||
2877                                     Q_QLTQ(voival->q64.sq64,
2878                                     bkt_ub->q64.sq64))))
2879                                         found = 1;
2880                         break;
2881                 case VSD_DTYPE_Q_U64:
2882                         if (Q_QGEQ(voival->q64.uq64, bkt_lb->q64.uq64))
2883                                 if ((eq_only && Q_QEQ(voival->q64.uq64,
2884                                     bkt_lb->q64.uq64)) ||
2885                                     (!eq_only && (!has_ub ||
2886                                     Q_QLTQ(voival->q64.uq64,
2887                                     bkt_ub->q64.uq64))))
2888                                         found = 1;
2889                         break;
2890                 default:
2891                         break;
2892                 }
2893         }
2894
2895         if (found) {
2896                 if (is32bit)
2897                         *cnt32 += 1;
2898                 else
2899                         *cnt64 += 1;
2900         } else {
2901                 if (is32bit)
2902                         *oob32 += 1;
2903                 else
2904                         *oob64 += 1;
2905         }
2906
2907         vs->flags |= VS_VSDVALID;
2908         return (error);
2909 }
2910
2911 static inline int
2912 stats_v1_vsd_tdgst_compress(enum vsd_dtype vs_dtype,
2913     struct voistatdata_tdgst *tdgst, int attempt)
2914 {
2915         struct ctdth32 *ctd32tree;
2916         struct ctdth64 *ctd64tree;
2917         struct voistatdata_tdgstctd32 *ctd32;
2918         struct voistatdata_tdgstctd64 *ctd64;
2919         uint64_t ebits, idxmask;
2920         uint32_t bitsperidx, nebits;
2921         int error, idx, is32bit, maxctds, remctds, tmperr;
2922
2923         error = 0;
2924
2925         switch (vs_dtype) {
2926         case VSD_DTYPE_TDGSTCLUST32:
2927                 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
2928                 if (!ARB_FULL(ctd32tree))
2929                         return (0);
2930                 VSD(tdgstclust32, tdgst)->compcnt++;
2931                 maxctds = remctds = ARB_MAXNODES(ctd32tree);
2932                 ARB_RESET_TREE(ctd32tree, ctdth32, maxctds);
2933                 VSD(tdgstclust32, tdgst)->smplcnt = 0;
2934                 is32bit = 1;
2935                 ctd64tree = NULL;
2936                 ctd64 = NULL;
2937 #ifdef DIAGNOSTIC
2938                 RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree);
2939 #endif
2940                 break;
2941         case VSD_DTYPE_TDGSTCLUST64:
2942                 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
2943                 if (!ARB_FULL(ctd64tree))
2944                         return (0);
2945                 VSD(tdgstclust64, tdgst)->compcnt++;
2946                 maxctds = remctds = ARB_MAXNODES(ctd64tree);
2947                 ARB_RESET_TREE(ctd64tree, ctdth64, maxctds);
2948                 VSD(tdgstclust64, tdgst)->smplcnt = 0;
2949                 is32bit = 0;
2950                 ctd32tree = NULL;
2951                 ctd32 = NULL;
2952 #ifdef DIAGNOSTIC
2953                 RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree);
2954 #endif
2955                 break;
2956         default:
2957                 return (EINVAL);
2958         }
2959
2960         /*
2961          * Rebuild the t-digest ARB by pseudorandomly selecting centroids and
2962          * re-inserting the mu/cnt of each as a value and corresponding weight.
2963          */
2964
2965 #define bitsperrand 31 /* Per random(3). */
2966         ebits = 0;
2967         nebits = 0;
2968         bitsperidx = fls(maxctds);
2969         KASSERT(bitsperidx <= sizeof(ebits) << 3,
2970             ("%s: bitsperidx=%d, ebits=%d",
2971             __func__, bitsperidx, (int)(sizeof(ebits) << 3)));
2972         idxmask = (UINT64_C(1) << bitsperidx) - 1;
2973         srandom(stats_sbinuptime());
2974
2975         /* Initialise the free list with randomised centroid indices. */
2976         for (; remctds > 0; remctds--) {
2977                 while (nebits < bitsperidx) {
2978                         ebits |= ((uint64_t)random()) << nebits;
2979                         nebits += bitsperrand;
2980                         if (nebits > (sizeof(ebits) << 3))
2981                                 nebits = sizeof(ebits) << 3;
2982                 }
2983                 idx = ebits & idxmask;
2984                 nebits -= bitsperidx;
2985                 ebits >>= bitsperidx;
2986
2987                 /*
2988                  * Select the next centroid to put on the ARB free list. We
2989                  * start with the centroid at our randomly selected array index,
2990                  * and work our way forwards until finding one (the latter
2991                  * aspect reduces re-insertion randomness, but is good enough).
2992                  */
2993                 do {
2994                         if (idx >= maxctds)
2995                                 idx %= maxctds;
2996
2997                         if (is32bit)
2998                                 ctd32 = ARB_NODE(ctd32tree, idx);
2999                         else
3000                                 ctd64 = ARB_NODE(ctd64tree, idx);
3001                 } while ((is32bit ? ARB_ISFREE(ctd32, ctdlnk) :
3002                     ARB_ISFREE(ctd64, ctdlnk)) && ++idx);
3003
3004                 /* Put the centroid on the ARB free list. */
3005                 if (is32bit)
3006                         ARB_RETURNFREE(ctd32tree, ctd32, ctdlnk);
3007                 else
3008                         ARB_RETURNFREE(ctd64tree, ctd64, ctdlnk);
3009         }
3010
3011         /*
3012          * The free list now contains the randomised indices of every centroid.
3013          * Walk the free list from start to end, re-inserting each centroid's
3014          * mu/cnt. The tdgst_add() call may or may not consume the free centroid
3015          * we re-insert values from during each loop iteration, so we must latch
3016          * the index of the next free list centroid before the re-insertion
3017          * call. The previous loop above should have left the centroid pointer
3018          * pointing to the element at the head of the free list.
3019          */
3020         KASSERT((is32bit ?
3021             ARB_FREEIDX(ctd32tree) == ARB_SELFIDX(ctd32tree, ctd32) :
3022             ARB_FREEIDX(ctd64tree) == ARB_SELFIDX(ctd64tree, ctd64)),
3023             ("%s: t-digest ARB@%p free list bug", __func__,
3024             (is32bit ? (void *)ctd32tree : (void *)ctd64tree)));
3025         remctds = maxctds;
3026         while ((is32bit ? ctd32 != NULL : ctd64 != NULL)) {
3027                 tmperr = 0;
3028                 if (is32bit) {
3029                         s64q_t x;
3030
3031                         idx = ARB_NEXTFREEIDX(ctd32, ctdlnk);
3032                         /* Cloning a s32q_t into a s64q_t should never fail. */
3033                         tmperr = Q_QCLONEQ(&x, ctd32->mu);
3034                         tmperr = tmperr ? tmperr : stats_v1_vsd_tdgst_add(
3035                             vs_dtype, tdgst, x, ctd32->cnt, attempt);
3036                         ctd32 = ARB_NODE(ctd32tree, idx);
3037                         KASSERT(ctd32 == NULL || ARB_ISFREE(ctd32, ctdlnk),
3038                             ("%s: t-digest ARB@%p free list bug", __func__,
3039                             ctd32tree));
3040                 } else {
3041                         idx = ARB_NEXTFREEIDX(ctd64, ctdlnk);
3042                         tmperr = stats_v1_vsd_tdgst_add(vs_dtype, tdgst,
3043                             ctd64->mu, ctd64->cnt, attempt);
3044                         ctd64 = ARB_NODE(ctd64tree, idx);
3045                         KASSERT(ctd64 == NULL || ARB_ISFREE(ctd64, ctdlnk),
3046                             ("%s: t-digest ARB@%p free list bug", __func__,
3047                             ctd64tree));
3048                 }
3049                 /*
3050                  * This process should not produce errors, bugs notwithstanding.
3051                  * Just in case, latch any errors and attempt all re-insertions.
3052                  */
3053                 error = tmperr ? tmperr : error;
3054                 remctds--;
3055         }
3056
3057         KASSERT(remctds == 0, ("%s: t-digest ARB@%p free list bug", __func__,
3058             (is32bit ? (void *)ctd32tree : (void *)ctd64tree)));
3059
3060         return (error);
3061 }
3062
3063 static inline int
3064 stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype, struct voistatdata_tdgst *tdgst,
3065     s64q_t x, uint64_t weight, int attempt)
3066 {
3067 #ifdef DIAGNOSTIC
3068         char qstr[Q_MAXSTRLEN(x, 10)];
3069 #endif
3070         struct ctdth32 *ctd32tree;
3071         struct ctdth64 *ctd64tree;
3072         void *closest, *cur, *lb, *ub;
3073         struct voistatdata_tdgstctd32 *ctd32;
3074         struct voistatdata_tdgstctd64 *ctd64;
3075         uint64_t cnt, smplcnt, sum, tmpsum;
3076         s64q_t k, minz, q, z;
3077         int error, is32bit, n;
3078
3079         error = 0;
3080         minz = Q_INI(&z, 0, 0, Q_NFBITS(x));
3081
3082         switch (vs_dtype) {
3083         case VSD_DTYPE_TDGSTCLUST32:
3084                 if ((UINT32_MAX - weight) < VSD(tdgstclust32, tdgst)->smplcnt)
3085                         error = EOVERFLOW;
3086                 smplcnt = VSD(tdgstclust32, tdgst)->smplcnt;
3087                 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
3088                 is32bit = 1;
3089                 ctd64tree = NULL;
3090                 ctd64 = NULL;
3091                 break;
3092         case VSD_DTYPE_TDGSTCLUST64:
3093                 if ((UINT64_MAX - weight) < VSD(tdgstclust64, tdgst)->smplcnt)
3094                         error = EOVERFLOW;
3095                 smplcnt = VSD(tdgstclust64, tdgst)->smplcnt;
3096                 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
3097                 is32bit = 0;
3098                 ctd32tree = NULL;
3099                 ctd32 = NULL;
3100                 break;
3101         default:
3102                 error = EINVAL;
3103                 break;
3104         }
3105
3106         if (error)
3107                 return (error);
3108
3109         /*
3110          * Inspired by Ted Dunning's AVLTreeDigest.java
3111          */
3112         do {
3113 #if defined(DIAGNOSTIC)
3114                 KASSERT(attempt < 5,
3115                     ("%s: Too many attempts", __func__));
3116 #endif
3117                 if (attempt >= 5)
3118                         return (EAGAIN);
3119
3120                 Q_SIFVAL(minz, Q_IFMAXVAL(minz));
3121                 closest = ub = NULL;
3122                 sum = tmpsum = 0;
3123
3124                 if (is32bit)
3125                         lb = cur = (void *)(ctd32 = ARB_MIN(ctdth32, ctd32tree));
3126                 else
3127                         lb = cur = (void *)(ctd64 = ARB_MIN(ctdth64, ctd64tree));
3128
3129                 if (lb == NULL) /* Empty tree. */
3130                         lb = (is32bit ? (void *)ARB_ROOT(ctd32tree) :
3131                             (void *)ARB_ROOT(ctd64tree));
3132
3133                 /*
3134                  * Find the set of centroids with minimum distance to x and
3135                  * compute the sum of counts for all centroids with mean less
3136                  * than the first centroid in the set.
3137                  */
3138                 for (; cur != NULL;
3139                     cur = (is32bit ?
3140                     (void *)(ctd32 = ARB_NEXT(ctdth32, ctd32tree, ctd32)) :
3141                     (void *)(ctd64 = ARB_NEXT(ctdth64, ctd64tree, ctd64)))) {
3142                         if (is32bit) {
3143                                 cnt = ctd32->cnt;
3144                                 KASSERT(Q_PRECEQ(ctd32->mu, x),
3145                                     ("%s: Q_RELPREC(mu,x)=%d", __func__,
3146                                     Q_RELPREC(ctd32->mu, x)));
3147                                 /* Ok to assign as both have same precision. */
3148                                 z = ctd32->mu;
3149                         } else {
3150                                 cnt = ctd64->cnt;
3151                                 KASSERT(Q_PRECEQ(ctd64->mu, x),
3152                                     ("%s: Q_RELPREC(mu,x)=%d", __func__,
3153                                     Q_RELPREC(ctd64->mu, x)));
3154                                 /* Ok to assign as both have same precision. */
3155                                 z = ctd64->mu;
3156                         }
3157
3158                         error = Q_QSUBQ(&z, x);
3159 #if defined(DIAGNOSTIC)
3160                         KASSERT(!error, ("%s: unexpected error %d", __func__,
3161                             error));
3162 #endif
3163                         if (error)
3164                                 return (error);
3165
3166                         z = Q_QABS(z);
3167                         if (Q_QLTQ(z, minz)) {
3168                                 minz = z;
3169                                 lb = cur;
3170                                 sum = tmpsum;
3171                                 tmpsum += cnt;
3172                         } else if (Q_QGTQ(z, minz)) {
3173                                 ub = cur;
3174                                 break;
3175                         }
3176                 }
3177
3178                 cur = (is32bit ?
3179                     (void *)(ctd32 = (struct voistatdata_tdgstctd32 *)lb) :
3180                     (void *)(ctd64 = (struct voistatdata_tdgstctd64 *)lb));
3181
3182                 for (n = 0; cur != ub; cur = (is32bit ?
3183                     (void *)(ctd32 = ARB_NEXT(ctdth32, ctd32tree, ctd32)) :
3184                     (void *)(ctd64 = ARB_NEXT(ctdth64, ctd64tree, ctd64)))) {
3185                         if (is32bit)
3186                                 cnt = ctd32->cnt;
3187                         else
3188                                 cnt = ctd64->cnt;
3189
3190                         q = Q_CTRLINI(16);
3191                         if (smplcnt == 1)
3192                                 error = Q_QFRACI(&q, 1, 2);
3193                         else
3194                                 /* [ sum + ((cnt - 1) / 2) ] / (smplcnt - 1) */
3195                                 error = Q_QFRACI(&q, (sum << 1) + cnt - 1,
3196                                     (smplcnt - 1) << 1);
3197                         k = q;
3198                         /* k = q x 4 x samplcnt x attempt */
3199                         error |= Q_QMULI(&k, 4 * smplcnt * attempt);
3200                         /* k = k x (1 - q) */
3201                         error |= Q_QSUBI(&q, 1);
3202                         q = Q_QABS(q);
3203                         error |= Q_QMULQ(&k, q);
3204 #if defined(DIAGNOSTIC)
3205 #if !defined(_KERNEL)
3206                         double q_dbl, k_dbl, q2d, k2d;
3207                         q2d = Q_Q2D(q);
3208                         k2d = Q_Q2D(k);
3209                         q_dbl = smplcnt == 1 ? 0.5 :
3210                             (sum + ((cnt - 1)  / 2.0)) / (double)(smplcnt - 1);
3211                         k_dbl = 4 * smplcnt * q_dbl * (1.0 - q_dbl) * attempt;
3212                         /*
3213                          * If the difference between q and q_dbl is greater than
3214                          * the fractional precision of q, something is off.
3215                          * NB: q is holding the value of 1 - q
3216                          */
3217                         q_dbl = 1.0 - q_dbl;
3218                         KASSERT((q_dbl > q2d ? q_dbl - q2d : q2d - q_dbl) <
3219                             (1.05 * ((double)1 / (double)(1ULL << Q_NFBITS(q)))),
3220                             ("Q-type q bad precision"));
3221                         KASSERT((k_dbl > k2d ? k_dbl - k2d : k2d - k_dbl) <
3222                             1.0 + (0.01 * smplcnt),
3223                             ("Q-type k bad precision"));
3224 #endif /* !_KERNEL */
3225                         KASSERT(!error, ("%s: unexpected error %d", __func__,
3226                             error));
3227 #endif /* DIAGNOSTIC */
3228                         if (error)
3229                                 return (error);
3230                         if ((is32bit && ((ctd32->cnt + weight) <=
3231                             (uint64_t)Q_GIVAL(k))) ||
3232                             (!is32bit && ((ctd64->cnt + weight) <=
3233                             (uint64_t)Q_GIVAL(k)))) {
3234                                 n++;
3235                                 /* random() produces 31 bits. */
3236                                 if (random() < (INT32_MAX / n))
3237                                         closest = cur;
3238                         }
3239                         sum += cnt;
3240                 }
3241         } while (closest == NULL &&
3242             (is32bit ? ARB_FULL(ctd32tree) : ARB_FULL(ctd64tree)) &&
3243             (error = stats_v1_vsd_tdgst_compress(vs_dtype, tdgst,
3244             attempt++)) == 0);
3245
3246         if (error)
3247                 return (error);
3248
3249         if (closest != NULL) {
3250                 /* Merge with an existing centroid. */
3251                 if (is32bit) {
3252                         ctd32 = (struct voistatdata_tdgstctd32 *)closest;
3253                         error = Q_QSUBQ(&x, ctd32->mu);
3254                         error = error ? error :
3255                             Q_QDIVI(&x, ctd32->cnt + weight);
3256                         if (error || (error = Q_QADDQ(&ctd32->mu, x))) {
3257 #ifdef DIAGNOSTIC
3258                                 KASSERT(!error, ("%s: unexpected error %d",
3259                                     __func__, error));
3260 #endif
3261                                 return (error);
3262                         }
3263                         ctd32->cnt += weight;
3264                         error = ARB_REINSERT(ctdth32, ctd32tree, ctd32) ==
3265                             NULL ? 0 : EALREADY;
3266 #ifdef DIAGNOSTIC
3267                         RB_REINSERT(rbctdth32,
3268                             &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32);
3269 #endif
3270                 } else {
3271                         ctd64 = (struct voistatdata_tdgstctd64 *)closest;
3272                         error = Q_QSUBQ(&x, ctd64->mu);
3273                         error = error ? error :
3274                             Q_QDIVI(&x, ctd64->cnt + weight);
3275                         if (error || (error = Q_QADDQ(&ctd64->mu, x))) {
3276                                 KASSERT(!error, ("%s: unexpected error %d",
3277                                     __func__, error));
3278                                 return (error);
3279                         }
3280                         ctd64->cnt += weight;
3281                         error = ARB_REINSERT(ctdth64, ctd64tree, ctd64) ==
3282                             NULL ? 0 : EALREADY;
3283 #ifdef DIAGNOSTIC
3284                         RB_REINSERT(rbctdth64,
3285                             &VSD(tdgstclust64, tdgst)->rbctdtree, ctd64);
3286 #endif
3287                 }
3288         } else {
3289                 /*
3290                  * Add a new centroid. If digest compression is working
3291                  * correctly, there should always be at least one free.
3292                  */
3293                 if (is32bit) {
3294                         ctd32 = ARB_GETFREE(ctd32tree, ctdlnk);
3295 #ifdef DIAGNOSTIC
3296                         KASSERT(ctd32 != NULL,
3297                             ("%s: t-digest@%p has no free centroids",
3298                             __func__, tdgst));
3299 #endif
3300                         if (ctd32 == NULL)
3301                                 return (EAGAIN);
3302                         if ((error = Q_QCPYVALQ(&ctd32->mu, x)))
3303                                 return (error);
3304                         ctd32->cnt = weight;
3305                         error = ARB_INSERT(ctdth32, ctd32tree, ctd32) == NULL ?
3306                             0 : EALREADY;
3307 #ifdef DIAGNOSTIC
3308                         RB_INSERT(rbctdth32,
3309                             &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32);
3310 #endif
3311                 } else {
3312                         ctd64 = ARB_GETFREE(ctd64tree, ctdlnk);
3313 #ifdef DIAGNOSTIC
3314                         KASSERT(ctd64 != NULL,
3315                             ("%s: t-digest@%p has no free centroids",
3316                             __func__, tdgst));
3317 #endif
3318                         if (ctd64 == NULL) /* Should not happen. */
3319                                 return (EAGAIN);
3320                         /* Direct assignment ok as both have same type/prec. */
3321                         ctd64->mu = x;
3322                         ctd64->cnt = weight;
3323                         error = ARB_INSERT(ctdth64, ctd64tree, ctd64) == NULL ?
3324                             0 : EALREADY;
3325 #ifdef DIAGNOSTIC
3326                         RB_INSERT(rbctdth64, &VSD(tdgstclust64,
3327                             tdgst)->rbctdtree, ctd64);
3328 #endif
3329                 }
3330         }
3331
3332         if (is32bit)
3333                 VSD(tdgstclust32, tdgst)->smplcnt += weight;
3334         else {
3335                 VSD(tdgstclust64, tdgst)->smplcnt += weight;
3336
3337 #ifdef DIAGNOSTIC
3338                 struct rbctdth64 *rbctdtree =
3339                     &VSD(tdgstclust64, tdgst)->rbctdtree;
3340                 struct voistatdata_tdgstctd64 *rbctd64;
3341                 int i = 0;
3342                 ARB_FOREACH(ctd64, ctdth64, ctd64tree) {
3343                         rbctd64 = (i == 0 ? RB_MIN(rbctdth64, rbctdtree) :
3344                             RB_NEXT(rbctdth64, rbctdtree, rbctd64));
3345
3346                         if (i >= ARB_CURNODES(ctd64tree)
3347                             || ctd64 != rbctd64
3348                             || ARB_MIN(ctdth64, ctd64tree) !=
3349                                RB_MIN(rbctdth64, rbctdtree)
3350                             || ARB_MAX(ctdth64, ctd64tree) !=
3351                                RB_MAX(rbctdth64, rbctdtree)
3352                             || ARB_LEFTIDX(ctd64, ctdlnk) !=
3353                                ARB_SELFIDX(ctd64tree, RB_LEFT(rbctd64, rblnk))
3354                             || ARB_RIGHTIDX(ctd64, ctdlnk) !=
3355                                ARB_SELFIDX(ctd64tree, RB_RIGHT(rbctd64, rblnk))
3356                             || ARB_PARENTIDX(ctd64, ctdlnk) !=
3357                                ARB_SELFIDX(ctd64tree,
3358                                RB_PARENT(rbctd64, rblnk))) {
3359                                 Q_TOSTR(ctd64->mu, -1, 10, qstr, sizeof(qstr));
3360                                 printf("ARB ctd=%3d p=%3d l=%3d r=%3d c=%2d "
3361                                     "mu=%s\n",
3362                                     (int)ARB_SELFIDX(ctd64tree, ctd64),
3363                                     ARB_PARENTIDX(ctd64, ctdlnk),
3364                                     ARB_LEFTIDX(ctd64, ctdlnk),
3365                                     ARB_RIGHTIDX(ctd64, ctdlnk),
3366                                     ARB_COLOR(ctd64, ctdlnk),
3367                                     qstr);
3368
3369                                 Q_TOSTR(rbctd64->mu, -1, 10, qstr,
3370                                     sizeof(qstr));
3371                                 printf(" RB ctd=%3d p=%3d l=%3d r=%3d c=%2d "
3372                                     "mu=%s\n",
3373                                     (int)ARB_SELFIDX(ctd64tree, rbctd64),
3374                                     (int)ARB_SELFIDX(ctd64tree,
3375                                       RB_PARENT(rbctd64, rblnk)),
3376                                     (int)ARB_SELFIDX(ctd64tree,
3377                                       RB_LEFT(rbctd64, rblnk)),
3378                                     (int)ARB_SELFIDX(ctd64tree,
3379                                       RB_RIGHT(rbctd64, rblnk)),
3380                                     RB_COLOR(rbctd64, rblnk),
3381                                     qstr);
3382
3383                                 panic("RB@%p and ARB@%p trees differ\n",
3384                                     rbctdtree, ctd64tree);
3385                         }
3386                         i++;
3387                 }
3388 #endif /* DIAGNOSTIC */
3389         }
3390
3391         return (error);
3392 }
3393
3394 static inline int
3395 stats_v1_voi_update_tdgst(enum vsd_dtype voi_dtype, struct voistatdata *voival,
3396     struct voistat *vs, struct voistatdata_tdgst *tdgst)
3397 {
3398         s64q_t x;
3399         int error;
3400
3401         error = 0;
3402
3403         switch (vs->dtype) {
3404         case VSD_DTYPE_TDGSTCLUST32:
3405                 /* Use same precision as the user's centroids. */
3406                 Q_INI(&x, 0, 0, Q_NFBITS(
3407                     ARB_CNODE(&VSD(tdgstclust32, tdgst)->ctdtree, 0)->mu));
3408                 break;
3409         case VSD_DTYPE_TDGSTCLUST64:
3410                 /* Use same precision as the user's centroids. */
3411                 Q_INI(&x, 0, 0, Q_NFBITS(
3412                     ARB_CNODE(&VSD(tdgstclust64, tdgst)->ctdtree, 0)->mu));
3413                 break;
3414         default:
3415                 KASSERT(vs->dtype == VSD_DTYPE_TDGSTCLUST32 ||
3416                     vs->dtype == VSD_DTYPE_TDGSTCLUST64,
3417                     ("%s: vs->dtype(%d) != VSD_DTYPE_TDGSTCLUST<32|64>",
3418                     __func__, vs->dtype));
3419                 return (EINVAL);
3420         }
3421
3422         /*
3423          * XXXLAS: Should have both a signed and unsigned 'x' variable to avoid
3424          * returning EOVERFLOW if the voival would have fit in a u64q_t.
3425          */
3426         switch (voi_dtype) {
3427         case VSD_DTYPE_INT_S32:
3428                 error = Q_QCPYVALI(&x, voival->int32.s32);
3429                 break;
3430         case VSD_DTYPE_INT_U32:
3431                 error = Q_QCPYVALI(&x, voival->int32.u32);
3432                 break;
3433         case VSD_DTYPE_INT_S64:
3434                 error = Q_QCPYVALI(&x, voival->int64.s64);
3435                 break;
3436         case VSD_DTYPE_INT_U64:
3437                 error = Q_QCPYVALI(&x, voival->int64.u64);
3438                 break;
3439         case VSD_DTYPE_INT_SLONG:
3440                 error = Q_QCPYVALI(&x, voival->intlong.slong);
3441                 break;
3442         case VSD_DTYPE_INT_ULONG:
3443                 error = Q_QCPYVALI(&x, voival->intlong.ulong);
3444                 break;
3445         case VSD_DTYPE_Q_S32:
3446                 error = Q_QCPYVALQ(&x, voival->q32.sq32);
3447                 break;
3448         case VSD_DTYPE_Q_U32:
3449                 error = Q_QCPYVALQ(&x, voival->q32.uq32);
3450                 break;
3451         case VSD_DTYPE_Q_S64:
3452                 error = Q_QCPYVALQ(&x, voival->q64.sq64);
3453                 break;
3454         case VSD_DTYPE_Q_U64:
3455                 error = Q_QCPYVALQ(&x, voival->q64.uq64);
3456                 break;
3457         default:
3458                 error = EINVAL;
3459                 break;
3460         }
3461
3462         if (error ||
3463             (error = stats_v1_vsd_tdgst_add(vs->dtype, tdgst, x, 1, 1)))
3464                 return (error);
3465
3466         vs->flags |= VS_VSDVALID;
3467         return (0);
3468 }
3469
3470 int
3471 stats_v1_voi_update(struct statsblobv1 *sb, int32_t voi_id,
3472     enum vsd_dtype voi_dtype, struct voistatdata *voival, uint32_t flags)
3473 {
3474         struct voi *v;
3475         struct voistat *vs;
3476         void *statevsd, *vsd;
3477         int error, i, tmperr;
3478
3479         error = 0;
3480
3481         if (sb == NULL || sb->abi != STATS_ABI_V1 || voi_id >= NVOIS(sb) ||
3482             voi_dtype == 0 || voi_dtype >= VSD_NUM_DTYPES || voival == NULL)
3483                 return (EINVAL);
3484         v = &sb->vois[voi_id];
3485         if (voi_dtype != v->dtype || v->id < 0 ||
3486             ((flags & SB_VOI_RELUPDATE) && !(v->flags & VOI_REQSTATE)))
3487                 return (EINVAL);
3488
3489         vs = BLOB_OFFSET(sb, v->stats_off);
3490         if (v->flags & VOI_REQSTATE)
3491                 statevsd = BLOB_OFFSET(sb, vs->data_off);
3492         else
3493                 statevsd = NULL;
3494
3495         if (flags & SB_VOI_RELUPDATE) {
3496                 switch (voi_dtype) {
3497                 case VSD_DTYPE_INT_S32:
3498                         voival->int32.s32 +=
3499                             VSD(voistate, statevsd)->prev.int32.s32;
3500                         break;
3501                 case VSD_DTYPE_INT_U32:
3502                         voival->int32.u32 +=
3503                             VSD(voistate, statevsd)->prev.int32.u32;
3504                         break;
3505                 case VSD_DTYPE_INT_S64:
3506                         voival->int64.s64 +=
3507                             VSD(voistate, statevsd)->prev.int64.s64;
3508                         break;
3509                 case VSD_DTYPE_INT_U64:
3510                         voival->int64.u64 +=
3511                             VSD(voistate, statevsd)->prev.int64.u64;
3512                         break;
3513                 case VSD_DTYPE_INT_SLONG:
3514                         voival->intlong.slong +=
3515                             VSD(voistate, statevsd)->prev.intlong.slong;
3516                         break;
3517                 case VSD_DTYPE_INT_ULONG:
3518                         voival->intlong.ulong +=
3519                             VSD(voistate, statevsd)->prev.intlong.ulong;
3520                         break;
3521                 case VSD_DTYPE_Q_S32:
3522                         error = Q_QADDQ(&voival->q32.sq32,
3523                             VSD(voistate, statevsd)->prev.q32.sq32);
3524                         break;
3525                 case VSD_DTYPE_Q_U32:
3526                         error = Q_QADDQ(&voival->q32.uq32,
3527                             VSD(voistate, statevsd)->prev.q32.uq32);
3528                         break;
3529                 case VSD_DTYPE_Q_S64:
3530                         error = Q_QADDQ(&voival->q64.sq64,
3531                             VSD(voistate, statevsd)->prev.q64.sq64);
3532                         break;
3533                 case VSD_DTYPE_Q_U64:
3534                         error = Q_QADDQ(&voival->q64.uq64,
3535                             VSD(voistate, statevsd)->prev.q64.uq64);
3536                         break;
3537                 default:
3538                         KASSERT(0, ("Unknown VOI data type %d", voi_dtype));
3539                         break;
3540                 }
3541         }
3542
3543         if (error)
3544                 return (error);
3545
3546         for (i = v->voistatmaxid; i > 0; i--) {
3547                 vs = &((struct voistat *)BLOB_OFFSET(sb, v->stats_off))[i];
3548                 if (vs->stype < 0)
3549                         continue;
3550
3551                 vsd = BLOB_OFFSET(sb, vs->data_off);
3552
3553                 switch (vs->stype) {
3554                 case VS_STYPE_MAX:
3555                         tmperr = stats_v1_voi_update_max(voi_dtype, voival,
3556                             vs, vsd);
3557                         break;
3558                 case VS_STYPE_MIN:
3559                         tmperr = stats_v1_voi_update_min(voi_dtype, voival,
3560                             vs, vsd);
3561                         break;
3562                 case VS_STYPE_SUM:
3563                         tmperr = stats_v1_voi_update_sum(voi_dtype, voival,
3564                             vs, vsd);
3565                         break;
3566                 case VS_STYPE_HIST:
3567                         tmperr = stats_v1_voi_update_hist(voi_dtype, voival,
3568                             vs, vsd);
3569                         break;
3570                 case VS_STYPE_TDGST:
3571                         tmperr = stats_v1_voi_update_tdgst(voi_dtype, voival,
3572                             vs, vsd);
3573                         break;
3574                 default:
3575                         KASSERT(0, ("Unknown VOI stat type %d", vs->stype));
3576                         break;
3577                 }
3578
3579                 if (tmperr) {
3580                         error = tmperr;
3581                         VS_INCERRS(vs);
3582                 }
3583         }
3584
3585         if (statevsd) {
3586                 switch (voi_dtype) {
3587                 case VSD_DTYPE_INT_S32:
3588                         VSD(voistate, statevsd)->prev.int32.s32 =
3589                             voival->int32.s32;
3590                         break;
3591                 case VSD_DTYPE_INT_U32:
3592                         VSD(voistate, statevsd)->prev.int32.u32 =
3593                             voival->int32.u32;
3594                         break;
3595                 case VSD_DTYPE_INT_S64:
3596                         VSD(voistate, statevsd)->prev.int64.s64 =
3597                             voival->int64.s64;
3598                         break;
3599                 case VSD_DTYPE_INT_U64:
3600                         VSD(voistate, statevsd)->prev.int64.u64 =
3601                             voival->int64.u64;
3602                         break;
3603                 case VSD_DTYPE_INT_SLONG:
3604                         VSD(voistate, statevsd)->prev.intlong.slong =
3605                             voival->intlong.slong;
3606                         break;
3607                 case VSD_DTYPE_INT_ULONG:
3608                         VSD(voistate, statevsd)->prev.intlong.ulong =
3609                             voival->intlong.ulong;
3610                         break;
3611                 case VSD_DTYPE_Q_S32:
3612                         error = Q_QCPYVALQ(
3613                             &VSD(voistate, statevsd)->prev.q32.sq32,
3614                             voival->q32.sq32);
3615                         break;
3616                 case VSD_DTYPE_Q_U32:
3617                         error = Q_QCPYVALQ(
3618                             &VSD(voistate, statevsd)->prev.q32.uq32,
3619                             voival->q32.uq32);
3620                         break;
3621                 case VSD_DTYPE_Q_S64:
3622                         error = Q_QCPYVALQ(
3623                             &VSD(voistate, statevsd)->prev.q64.sq64,
3624                             voival->q64.sq64);
3625                         break;
3626                 case VSD_DTYPE_Q_U64:
3627                         error = Q_QCPYVALQ(
3628                             &VSD(voistate, statevsd)->prev.q64.uq64,
3629                             voival->q64.uq64);
3630                         break;
3631                 default:
3632                         KASSERT(0, ("Unknown VOI data type %d", voi_dtype));
3633                         break;
3634                 }
3635         }
3636
3637         return (error);
3638 }
3639
3640 #ifdef _KERNEL
3641
3642 static void
3643 stats_init(void *arg)
3644 {
3645
3646 }
3647 SYSINIT(stats, SI_SUB_KDTRACE, SI_ORDER_FIRST, stats_init, NULL);
3648
3649 /*
3650  * Sysctl handler to display the list of available stats templates.
3651  */
3652 static int
3653 stats_tpl_list_available(SYSCTL_HANDLER_ARGS)
3654 {
3655         struct sbuf *s;
3656         int err, i;
3657
3658         err = 0;
3659
3660         /* We can tolerate ntpl being stale, so do not take the lock. */
3661         s = sbuf_new(NULL, NULL, /* +1 per tpl for , */
3662             ntpl * (STATS_TPL_MAX_STR_SPEC_LEN + 1), SBUF_FIXEDLEN);
3663         if (s == NULL)
3664                 return (ENOMEM);
3665
3666         TPL_LIST_RLOCK();
3667         for (i = 0; i < ntpl; i++) {
3668                 err = sbuf_printf(s, "%s\"%s\":%u", i ? "," : "",
3669                     tpllist[i]->mb->tplname, tpllist[i]->mb->tplhash);
3670                 if (err) {
3671                         /* Sbuf overflow condition. */
3672                         err = EOVERFLOW;
3673                         break;
3674                 }
3675         }
3676         TPL_LIST_RUNLOCK();
3677
3678         if (!err) {
3679                 sbuf_finish(s);
3680                 err = sysctl_handle_string(oidp, sbuf_data(s), 0, req);
3681         }
3682
3683         sbuf_delete(s);
3684         return (err);
3685 }
3686
3687 /*
3688  * Called by subsystem-specific sysctls to report and/or parse the list of
3689  * templates being sampled and their sampling rates. A stats_tpl_sr_cb_t
3690  * conformant function pointer must be passed in as arg1, which is used to
3691  * interact with the subsystem's stats template sample rates list. If arg2 > 0,
3692  * a zero-initialised allocation of arg2-sized contextual memory is
3693  * heap-allocated and passed in to all subsystem callbacks made during the
3694  * operation of stats_tpl_sample_rates().
3695  *
3696  * XXXLAS: Assumes templates are never removed, which is currently true but may
3697  * need to be reworked in future if dynamic template management becomes a
3698  * requirement e.g. to support kernel module based templates.
3699  */
3700 int
3701 stats_tpl_sample_rates(SYSCTL_HANDLER_ARGS)
3702 {
3703         char kvpair_fmt[16], tplspec_fmt[16];
3704         char tpl_spec[STATS_TPL_MAX_STR_SPEC_LEN];
3705         char tpl_name[TPL_MAX_NAME_LEN + 2]; /* +2 for "" */
3706         stats_tpl_sr_cb_t subsys_cb;
3707         void *subsys_ctx;
3708         char *buf, *new_rates_usr_str, *tpl_name_p;
3709         struct stats_tpl_sample_rate *rates;
3710         struct sbuf *s, _s;
3711         uint32_t cum_pct, pct, tpl_hash;
3712         int err, i, off, len, newlen, nrates;
3713
3714         buf = NULL;
3715         rates = NULL;
3716         err = nrates = 0;
3717         subsys_cb = (stats_tpl_sr_cb_t)arg1;
3718         KASSERT(subsys_cb != NULL, ("%s: subsys_cb == arg1 == NULL", __func__));
3719         if (arg2 > 0)
3720                 subsys_ctx = malloc(arg2, M_TEMP, M_WAITOK | M_ZERO);
3721         else
3722                 subsys_ctx = NULL;
3723
3724         /* Grab current count of subsystem rates. */
3725         err = subsys_cb(TPL_SR_UNLOCKED_GET, NULL, &nrates, subsys_ctx);
3726         if (err)
3727                 goto done;
3728
3729         /* +1 to ensure we can append '\0' post copyin, +5 per rate for =nnn, */
3730         len = max(req->newlen + 1, nrates * (STATS_TPL_MAX_STR_SPEC_LEN + 5));
3731
3732         if (req->oldptr != NULL || req->newptr != NULL)
3733                 buf = malloc(len, M_TEMP, M_WAITOK);
3734
3735         if (req->oldptr != NULL) {
3736                 if (nrates == 0) {
3737                         /* No rates, so return an empty string via oldptr. */
3738                         err = SYSCTL_OUT(req, "", 1);
3739                         if (err)
3740                                 goto done;
3741                         goto process_new;
3742                 }
3743
3744                 s = sbuf_new(&_s, buf, len, SBUF_FIXEDLEN | SBUF_INCLUDENUL);
3745
3746                 /* Grab locked count of, and ptr to, subsystem rates. */
3747                 err = subsys_cb(TPL_SR_RLOCKED_GET, &rates, &nrates,
3748                     subsys_ctx);
3749                 if (err)
3750                         goto done;
3751                 TPL_LIST_RLOCK();
3752                 for (i = 0; i < nrates && !err; i++) {
3753                         err = sbuf_printf(s, "%s\"%s\":%u=%u", i ? "," : "",
3754                             tpllist[rates[i].tpl_slot_id]->mb->tplname,
3755                             tpllist[rates[i].tpl_slot_id]->mb->tplhash,
3756                             rates[i].tpl_sample_pct);
3757                 }
3758                 TPL_LIST_RUNLOCK();
3759                 /* Tell subsystem that we're done with its rates list. */
3760                 err = subsys_cb(TPL_SR_RUNLOCK, &rates, &nrates, subsys_ctx);
3761                 if (err)
3762                         goto done;
3763
3764                 err = sbuf_finish(s);
3765                 if (err)
3766                         goto done; /* We lost a race for buf to be too small. */
3767
3768                 /* Return the rendered string data via oldptr. */
3769                 err = SYSCTL_OUT(req, sbuf_data(s), sbuf_len(s));
3770         } else {
3771                 /* Return the upper bound size for buffer sizing requests. */
3772                 err = SYSCTL_OUT(req, NULL, len);
3773         }
3774
3775 process_new:
3776         if (err || req->newptr == NULL)
3777                 goto done;
3778
3779         newlen = req->newlen - req->newidx;
3780         err = SYSCTL_IN(req, buf, newlen);
3781         if (err)
3782                 goto done;
3783
3784         /*
3785          * Initialise format strings at run time.
3786          *
3787          * Write the max template spec string length into the
3788          * template_spec=percent key-value pair parsing format string as:
3789          *     " %<width>[^=]=%u %n"
3790          *
3791          * Write the max template name string length into the tplname:tplhash
3792          * parsing format string as:
3793          *     "%<width>[^:]:%u"
3794          *
3795          * Subtract 1 for \0 appended by sscanf().
3796          */
3797         sprintf(kvpair_fmt, " %%%zu[^=]=%%u %%n", sizeof(tpl_spec) - 1);
3798         sprintf(tplspec_fmt, "%%%zu[^:]:%%u", sizeof(tpl_name) - 1);
3799
3800         /*
3801          * Parse each CSV key-value pair specifying a template and its sample
3802          * percentage. Whitespace either side of a key-value pair is ignored.
3803          * Templates can be specified by name, hash, or name and hash per the
3804          * following formats (chars in [] are optional):
3805          *    ["]<tplname>["]=<percent>
3806          *    :hash=pct
3807          *    ["]<tplname>["]:hash=<percent>
3808          */
3809         cum_pct = nrates = 0;
3810         rates = NULL;
3811         buf[newlen] = '\0'; /* buf is at least newlen+1 in size. */
3812         new_rates_usr_str = buf;
3813         while (isspace(*new_rates_usr_str))
3814                 new_rates_usr_str++; /* Skip leading whitespace. */
3815         while (*new_rates_usr_str != '\0') {
3816                 tpl_name_p = tpl_name;
3817                 tpl_name[0] = '\0';
3818                 tpl_hash = 0;
3819                 off = 0;
3820
3821                 /*
3822                  * Parse key-value pair which must perform 2 conversions, then
3823                  * parse the template spec to extract either name, hash, or name
3824                  * and hash depending on the three possible spec formats. The
3825                  * tplspec_fmt format specifier parses name or name and hash
3826                  * template specs, while the ":%u" format specifier parses
3827                  * hash-only template specs. If parsing is successfull, ensure
3828                  * the cumulative sampling percentage does not exceed 100.
3829                  */
3830                 err = EINVAL;
3831                 if (2 != sscanf(new_rates_usr_str, kvpair_fmt, tpl_spec, &pct,
3832                     &off))
3833                         break;
3834                 if ((1 > sscanf(tpl_spec, tplspec_fmt, tpl_name, &tpl_hash)) &&
3835                     (1 != sscanf(tpl_spec, ":%u", &tpl_hash)))
3836                         break;
3837                 if ((cum_pct += pct) > 100)
3838                         break;
3839                 err = 0;
3840
3841                 /* Strip surrounding "" from template name if present. */
3842                 len = strlen(tpl_name);
3843                 if (len > 0) {
3844                         if (tpl_name[len - 1] == '"')
3845                                 tpl_name[--len] = '\0';
3846                         if (tpl_name[0] == '"') {
3847                                 tpl_name_p++;
3848                                 len--;
3849                         }
3850                 }
3851
3852                 rates = stats_realloc(rates, 0, /* oldsz is unused in kernel. */
3853                     (nrates + 1) * sizeof(*rates), M_WAITOK);
3854                 rates[nrates].tpl_slot_id =
3855                     stats_tpl_fetch_allocid(len ? tpl_name_p : NULL, tpl_hash);
3856                 if (rates[nrates].tpl_slot_id < 0) {
3857                         err = -rates[nrates].tpl_slot_id;
3858                         break;
3859                 }
3860                 rates[nrates].tpl_sample_pct = pct;
3861                 nrates++;
3862                 new_rates_usr_str += off;
3863                 if (*new_rates_usr_str != ',')
3864                         break; /* End-of-input or malformed. */
3865                 new_rates_usr_str++; /* Move past comma to next pair. */
3866         }
3867
3868         if (!err) {
3869                 if ((new_rates_usr_str - buf) < newlen) {
3870                         /* Entire input has not been consumed. */
3871                         err = EINVAL;
3872                 } else {
3873                         /*
3874                          * Give subsystem the new rates. They'll return the
3875                          * appropriate rates pointer for us to garbage collect.
3876                          */
3877                         err = subsys_cb(TPL_SR_PUT, &rates, &nrates,
3878                             subsys_ctx);
3879                 }
3880         }
3881         stats_free(rates);
3882
3883 done:
3884         free(buf, M_TEMP);
3885         free(subsys_ctx, M_TEMP);
3886         return (err);
3887 }
3888
3889 SYSCTL_NODE(_kern, OID_AUTO, stats, CTLFLAG_RW, NULL,
3890     "stats(9) MIB");
3891
3892 SYSCTL_PROC(_kern_stats, OID_AUTO, templates, CTLTYPE_STRING|CTLFLAG_RD,
3893     NULL, 0, stats_tpl_list_available, "A",
3894     "list the name/hash of all available stats(9) templates");
3895
3896 #else /* ! _KERNEL */
3897
3898 static void __attribute__ ((constructor))
3899 stats_constructor(void)
3900 {
3901
3902         pthread_rwlock_init(&tpllistlock, NULL);
3903 }
3904
3905 static void __attribute__ ((destructor))
3906 stats_destructor(void)
3907 {
3908
3909         pthread_rwlock_destroy(&tpllistlock);
3910 }
3911
3912 #endif /* _KERNEL */