sys/kern/subr_stats.c

   1 /*-
   2  * Copyright (c) 2014-2018 Netflix, Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  * $FreeBSD$
  27  */
  28
  29 /*
  30  * Author: Lawrence Stewart <lstewart@netflix.com>
  31  */
  32
  33 #include <sys/cdefs.h>
  34 __FBSDID("$FreeBSD$");
  35
  36 #include <sys/param.h>
  37 #include <sys/arb.h>
  38 #include <sys/ctype.h>
  39 #include <sys/errno.h>
  40 #include <sys/hash.h>
  41 #include <sys/limits.h>
  42 #include <sys/malloc.h>
  43 #include <sys/qmath.h>
  44 #include <sys/sbuf.h>
  45 #if defined(DIAGNOSTIC)
  46 #include <sys/tree.h>
  47 #endif
  48 #include <sys/stats.h> /* Must come after qmath.h and arb.h */
  49 #include <sys/stddef.h>
  50 #include <sys/stdint.h>
  51 #include <sys/time.h>
  52
  53 #ifdef _KERNEL
  54 #include <sys/kernel.h>
  55 #include <sys/lock.h>
  56 #include <sys/rwlock.h>
  57 #include <sys/sysctl.h>
  58 #include <sys/systm.h>
  59 #else /* ! _KERNEL */
  60 #include <pthread.h>
  61 #include <stdbool.h>
  62 #include <stdio.h>
  63 #include <stdlib.h>
  64 #include <string.h>
  65 #endif /* _KERNEL */
  66
  67 struct voistatdata_voistate {
  68         /* Previous VOI value for diff calculation. */
  69         struct voistatdata_numeric prev;
  70 };
  71
  72 #define VS_VSDVALID     0x0001  /* Stat's voistatdata updated at least once. */
  73 struct voistat {
  74         int8_t          stype;          /* Type of stat e.g. VS_STYPE_SUM. */
  75         enum vsd_dtype  dtype : 8;      /* Data type of this stat's data. */
  76         uint16_t        data_off;       /* Blob offset for this stat's data. */
  77         uint16_t        dsz;            /* Size of stat's data. */
  78 #define VS_EBITS 8
  79         uint16_t        errs : VS_EBITS;/* Non-wrapping error count. */
  80         uint16_t        flags : 16 - VS_EBITS;
  81 };
  82 /* The voistat error count is capped to avoid wrapping. */
  83 #define VS_INCERRS(vs) do {                                             \
  84         if ((vs)->errs < (1U << VS_EBITS) - 1)                          \
  85                 (vs)->errs++;                                           \
  86 } while (0)
  87
  88 /*
  89  * Ideas for flags:
  90  *   - Global or entity specific (global would imply use of counter(9)?)
  91  *   - Whether to reset stats on read or not
  92  *   - Signal an overflow?
  93  *   - Compressed voistat array
  94  */
  95 #define VOI_REQSTATE    0x0001  /* VOI requires VS_STYPE_VOISTATE. */
  96 struct voi {
  97         int16_t         id;             /* VOI id. */
  98         enum vsd_dtype  dtype : 8;      /* Data type of the VOI itself. */
  99         int8_t          voistatmaxid;   /* Largest allocated voistat index. */
 100         uint16_t        stats_off;      /* Blob offset for this VOIs stats. */
 101         uint16_t        flags;
 102 };
 103
 104 /*
 105  * Memory for the entire blob is allocated as a slab and then offsets are
 106  * maintained to carve up the slab into sections holding different data types.
 107  *
 108  * Ideas for flags:
 109  * - Compressed voi array (trade off memory usage vs search time)
 110  * - Units of offsets (default bytes, flag for e.g. vm_page/KiB/Mib)
 111  */
 112 struct statsblobv1 {
 113         uint8_t         abi;
 114         uint8_t         endian;
 115         uint16_t        flags;
 116         uint16_t        maxsz;
 117         uint16_t        cursz;
 118         /* Fields from here down are opaque to consumers. */
 119         uint32_t        tplhash;        /* Base template hash ID. */
 120         uint16_t        stats_off;      /* voistat array blob offset. */
 121         uint16_t        statsdata_off;  /* voistatdata array blob offset. */
 122         sbintime_t      created;        /* Blob creation time. */
 123         sbintime_t      lastrst;        /* Time of last reset. */
 124         struct voi      vois[];         /* Array indexed by [voi_id]. */
 125 } __aligned(sizeof(void *));
 126 _Static_assert(offsetof(struct statsblobv1, cursz) +
 127     SIZEOF_MEMBER(struct statsblobv1, cursz) ==
 128     offsetof(struct statsblob, opaque),
 129     "statsblobv1 ABI mismatch");
 130
 131 struct statsblobv1_tpl {
 132         struct metablob         *mb;
 133         struct statsblobv1      *sb;
 134 };
 135
 136 /* Context passed to iterator callbacks. */
 137 struct sb_iter_ctx {
 138         void            *usrctx;        /* Caller supplied context. */
 139         uint32_t        flags;          /* Flags for current iteration. */
 140         int16_t         vslot;          /* struct voi slot index. */
 141         int8_t          vsslot;         /* struct voistat slot index. */
 142 };
 143
 144 struct sb_tostrcb_ctx {
 145         struct sbuf             *buf;
 146         struct statsblob_tpl    *tpl;
 147         enum sb_str_fmt fmt;
 148         uint32_t                flags;
 149 };
 150
 151 struct sb_visitcb_ctx {
 152         stats_blob_visitcb_t    cb;
 153         void                    *usrctx;
 154 };
 155
 156 /* Stats blob iterator callback. */
 157 typedef int (*stats_v1_blob_itercb_t)(struct statsblobv1 *sb, struct voi *v,
 158     struct voistat *vs, struct sb_iter_ctx *ctx);
 159
 160 #ifdef _KERNEL
 161 static struct rwlock tpllistlock;
 162 RW_SYSINIT(stats_tpl_list, &tpllistlock, "Stat template list lock");
 163 #define TPL_LIST_RLOCK() rw_rlock(&tpllistlock)
 164 #define TPL_LIST_RUNLOCK() rw_runlock(&tpllistlock)
 165 #define TPL_LIST_WLOCK() rw_wlock(&tpllistlock)
 166 #define TPL_LIST_WUNLOCK() rw_wunlock(&tpllistlock)
 167 #define TPL_LIST_LOCK_ASSERT() rw_assert(&tpllistlock, RA_LOCKED)
 168 #define TPL_LIST_RLOCK_ASSERT() rw_assert(&tpllistlock, RA_RLOCKED)
 169 #define TPL_LIST_WLOCK_ASSERT() rw_assert(&tpllistlock, RA_WLOCKED)
 170 MALLOC_DEFINE(M_STATS, "stats(9) related memory", "stats(9) related memory");
 171 #define stats_free(ptr) free((ptr), M_STATS)
 172 #else /* ! _KERNEL */
 173 static void stats_constructor(void);
 174 static void stats_destructor(void);
 175 static pthread_rwlock_t tpllistlock;
 176 #define TPL_LIST_UNLOCK() pthread_rwlock_unlock(&tpllistlock)
 177 #define TPL_LIST_RLOCK() pthread_rwlock_rdlock(&tpllistlock)
 178 #define TPL_LIST_RUNLOCK() TPL_LIST_UNLOCK()
 179 #define TPL_LIST_WLOCK() pthread_rwlock_wrlock(&tpllistlock)
 180 #define TPL_LIST_WUNLOCK() TPL_LIST_UNLOCK()
 181 #define TPL_LIST_LOCK_ASSERT() do { } while (0)
 182 #define TPL_LIST_RLOCK_ASSERT() do { } while (0)
 183 #define TPL_LIST_WLOCK_ASSERT() do { } while (0)
 184 #ifdef NDEBUG
 185 #define KASSERT(cond, msg) do {} while (0)
 186 #define stats_abort() do {} while (0)
 187 #else /* ! NDEBUG */
 188 #define KASSERT(cond, msg) do { \
 189         if (!(cond)) { \
 190                 panic msg; \
 191         } \
 192 } while (0)
 193 #define stats_abort() abort()
 194 #endif /* NDEBUG */
 195 #define stats_free(ptr) free(ptr)
 196 #define panic(fmt, ...) do { \
 197         fprintf(stderr, (fmt), ##__VA_ARGS__); \
 198         stats_abort(); \
 199 } while (0)
 200 #endif /* _KERNEL */
 201
 202 #define SB_V1_MAXSZ 65535
 203
 204 /* Obtain a blob offset pointer. */
 205 #define BLOB_OFFSET(sb, off) ((void *)(((uint8_t *)(sb)) + (off)))
 206
 207 /*
 208  * Number of VOIs in the blob's vois[] array. By virtue of struct voi being a
 209  * power of 2 size, we can shift instead of divide. The shift amount must be
 210  * updated if sizeof(struct voi) ever changes, which the assert should catch.
 211  */
 212 #define NVOIS(sb) ((int32_t)((((struct statsblobv1 *)(sb))->stats_off - \
 213     sizeof(struct statsblobv1)) >> 3))
 214 _Static_assert(sizeof(struct voi) == 8, "statsblobv1 voi ABI mismatch");
 215
 216 /* Try restrict names to alphanumeric and underscore to simplify JSON compat. */
 217 const char *vs_stype2name[VS_NUM_STYPES] = {
 218         [VS_STYPE_VOISTATE] = "VOISTATE",
 219         [VS_STYPE_SUM] = "SUM",
 220         [VS_STYPE_MAX] = "MAX",
 221         [VS_STYPE_MIN] = "MIN",
 222         [VS_STYPE_HIST] = "HIST",
 223         [VS_STYPE_TDGST] = "TDGST",
 224 };
 225
 226 const char *vs_stype2desc[VS_NUM_STYPES] = {
 227         [VS_STYPE_VOISTATE] = "VOI related state data (not a real stat)",
 228         [VS_STYPE_SUM] = "Simple arithmetic accumulator",
 229         [VS_STYPE_MAX] = "Maximum observed VOI value",
 230         [VS_STYPE_MIN] = "Minimum observed VOI value",
 231         [VS_STYPE_HIST] = "Histogram of observed VOI values",
 232         [VS_STYPE_TDGST] = "t-digest of observed VOI values",
 233 };
 234
 235 const char *vsd_dtype2name[VSD_NUM_DTYPES] = {
 236         [VSD_DTYPE_VOISTATE] = "VOISTATE",
 237         [VSD_DTYPE_INT_S32] = "INT_S32",
 238         [VSD_DTYPE_INT_U32] = "INT_U32",
 239         [VSD_DTYPE_INT_S64] = "INT_S64",
 240         [VSD_DTYPE_INT_U64] = "INT_U64",
 241         [VSD_DTYPE_INT_SLONG] = "INT_SLONG",
 242         [VSD_DTYPE_INT_ULONG] = "INT_ULONG",
 243         [VSD_DTYPE_Q_S32] = "Q_S32",
 244         [VSD_DTYPE_Q_U32] = "Q_U32",
 245         [VSD_DTYPE_Q_S64] = "Q_S64",
 246         [VSD_DTYPE_Q_U64] = "Q_U64",
 247         [VSD_DTYPE_CRHIST32] = "CRHIST32",
 248         [VSD_DTYPE_DRHIST32] = "DRHIST32",
 249         [VSD_DTYPE_DVHIST32] = "DVHIST32",
 250         [VSD_DTYPE_CRHIST64] = "CRHIST64",
 251         [VSD_DTYPE_DRHIST64] = "DRHIST64",
 252         [VSD_DTYPE_DVHIST64] = "DVHIST64",
 253         [VSD_DTYPE_TDGSTCLUST32] = "TDGSTCLUST32",
 254         [VSD_DTYPE_TDGSTCLUST64] = "TDGSTCLUST64",
 255 };
 256
 257 const size_t vsd_dtype2size[VSD_NUM_DTYPES] = {
 258         [VSD_DTYPE_VOISTATE] = sizeof(struct voistatdata_voistate),
 259         [VSD_DTYPE_INT_S32] = sizeof(struct voistatdata_int32),
 260         [VSD_DTYPE_INT_U32] = sizeof(struct voistatdata_int32),
 261         [VSD_DTYPE_INT_S64] = sizeof(struct voistatdata_int64),
 262         [VSD_DTYPE_INT_U64] = sizeof(struct voistatdata_int64),
 263         [VSD_DTYPE_INT_SLONG] = sizeof(struct voistatdata_intlong),
 264         [VSD_DTYPE_INT_ULONG] = sizeof(struct voistatdata_intlong),
 265         [VSD_DTYPE_Q_S32] = sizeof(struct voistatdata_q32),
 266         [VSD_DTYPE_Q_U32] = sizeof(struct voistatdata_q32),
 267         [VSD_DTYPE_Q_S64] = sizeof(struct voistatdata_q64),
 268         [VSD_DTYPE_Q_U64] = sizeof(struct voistatdata_q64),
 269         [VSD_DTYPE_CRHIST32] = sizeof(struct voistatdata_crhist32),
 270         [VSD_DTYPE_DRHIST32] = sizeof(struct voistatdata_drhist32),
 271         [VSD_DTYPE_DVHIST32] = sizeof(struct voistatdata_dvhist32),
 272         [VSD_DTYPE_CRHIST64] = sizeof(struct voistatdata_crhist64),
 273         [VSD_DTYPE_DRHIST64] = sizeof(struct voistatdata_drhist64),
 274         [VSD_DTYPE_DVHIST64] = sizeof(struct voistatdata_dvhist64),
 275         [VSD_DTYPE_TDGSTCLUST32] = sizeof(struct voistatdata_tdgstclust32),
 276         [VSD_DTYPE_TDGSTCLUST64] = sizeof(struct voistatdata_tdgstclust64),
 277 };
 278
 279 static const bool vsd_compoundtype[VSD_NUM_DTYPES] = {
 280         [VSD_DTYPE_VOISTATE] = true,
 281         [VSD_DTYPE_INT_S32] = false,
 282         [VSD_DTYPE_INT_U32] = false,
 283         [VSD_DTYPE_INT_S64] = false,
 284         [VSD_DTYPE_INT_U64] = false,
 285         [VSD_DTYPE_INT_SLONG] = false,
 286         [VSD_DTYPE_INT_ULONG] = false,
 287         [VSD_DTYPE_Q_S32] = false,
 288         [VSD_DTYPE_Q_U32] = false,
 289         [VSD_DTYPE_Q_S64] = false,
 290         [VSD_DTYPE_Q_U64] = false,
 291         [VSD_DTYPE_CRHIST32] = true,
 292         [VSD_DTYPE_DRHIST32] = true,
 293         [VSD_DTYPE_DVHIST32] = true,
 294         [VSD_DTYPE_CRHIST64] = true,
 295         [VSD_DTYPE_DRHIST64] = true,
 296         [VSD_DTYPE_DVHIST64] = true,
 297         [VSD_DTYPE_TDGSTCLUST32] = true,
 298         [VSD_DTYPE_TDGSTCLUST64] = true,
 299 };
 300
 301 const struct voistatdata_numeric numeric_limits[2][VSD_DTYPE_Q_U64 + 1] = {
 302         [LIM_MIN] = {
 303                 [VSD_DTYPE_VOISTATE] = {0},
 304                 [VSD_DTYPE_INT_S32] = {.int32 = {.s32 = INT32_MIN}},
 305                 [VSD_DTYPE_INT_U32] = {.int32 = {.u32 = 0}},
 306                 [VSD_DTYPE_INT_S64] = {.int64 = {.s64 = INT64_MIN}},
 307                 [VSD_DTYPE_INT_U64] = {.int64 = {.u64 = 0}},
 308                 [VSD_DTYPE_INT_SLONG] = {.intlong = {.slong = LONG_MIN}},
 309                 [VSD_DTYPE_INT_ULONG] = {.intlong = {.ulong = 0}},
 310                 [VSD_DTYPE_Q_S32] = {.q32 = {.sq32 = Q_IFMINVAL(INT32_MIN)}},
 311                 [VSD_DTYPE_Q_U32] = {.q32 = {.uq32 = 0}},
 312                 [VSD_DTYPE_Q_S64] = {.q64 = {.sq64 = Q_IFMINVAL(INT64_MIN)}},
 313                 [VSD_DTYPE_Q_U64] = {.q64 = {.uq64 = 0}},
 314         },
 315         [LIM_MAX] = {
 316                 [VSD_DTYPE_VOISTATE] = {0},
 317                 [VSD_DTYPE_INT_S32] = {.int32 = {.s32 = INT32_MAX}},
 318                 [VSD_DTYPE_INT_U32] = {.int32 = {.u32 = UINT32_MAX}},
 319                 [VSD_DTYPE_INT_S64] = {.int64 = {.s64 = INT64_MAX}},
 320                 [VSD_DTYPE_INT_U64] = {.int64 = {.u64 = UINT64_MAX}},
 321                 [VSD_DTYPE_INT_SLONG] = {.intlong = {.slong = LONG_MAX}},
 322                 [VSD_DTYPE_INT_ULONG] = {.intlong = {.ulong = ULONG_MAX}},
 323                 [VSD_DTYPE_Q_S32] = {.q32 = {.sq32 = Q_IFMAXVAL(INT32_MAX)}},
 324                 [VSD_DTYPE_Q_U32] = {.q32 = {.uq32 = Q_IFMAXVAL(UINT32_MAX)}},
 325                 [VSD_DTYPE_Q_S64] = {.q64 = {.sq64 = Q_IFMAXVAL(INT64_MAX)}},
 326                 [VSD_DTYPE_Q_U64] = {.q64 = {.uq64 = Q_IFMAXVAL(UINT64_MAX)}},
 327         }
 328 };
 329
 330 /* tpllistlock protects tpllist and ntpl */
 331 static uint32_t ntpl;
 332 static struct statsblob_tpl **tpllist;
 333
 334 static inline void * stats_realloc(void *ptr, size_t oldsz, size_t newsz,
 335     int flags);
 336 //static void stats_v1_blob_finalise(struct statsblobv1 *sb);
 337 static int stats_v1_blob_init_locked(struct statsblobv1 *sb, uint32_t tpl_id,
 338     uint32_t flags);
 339 static int stats_v1_blob_expand(struct statsblobv1 **sbpp, int newvoibytes,
 340     int newvoistatbytes, int newvoistatdatabytes);
 341 static void stats_v1_blob_iter(struct statsblobv1 *sb,
 342     stats_v1_blob_itercb_t icb, void *usrctx, uint32_t flags);
 343 static inline int stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype,
 344     struct voistatdata_tdgst *tdgst, s64q_t x, uint64_t weight, int attempt);
 345
 346 static inline int
 347 ctd32cmp(const struct voistatdata_tdgstctd32 *c1, const struct voistatdata_tdgstctd32 *c2)
 348 {
 349
 350         KASSERT(Q_PRECEQ(c1->mu, c2->mu),
 351             ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__,
 352             Q_RELPREC(c1->mu, c2->mu)));
 353
 354        return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1);
 355 }
 356 ARB_GENERATE_STATIC(ctdth32, voistatdata_tdgstctd32, ctdlnk, ctd32cmp);
 357
 358 static inline int
 359 ctd64cmp(const struct voistatdata_tdgstctd64 *c1, const struct voistatdata_tdgstctd64 *c2)
 360 {
 361
 362         KASSERT(Q_PRECEQ(c1->mu, c2->mu),
 363             ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__,
 364             Q_RELPREC(c1->mu, c2->mu)));
 365
 366        return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1);
 367 }
 368 ARB_GENERATE_STATIC(ctdth64, voistatdata_tdgstctd64, ctdlnk, ctd64cmp);
 369
 370 #ifdef DIAGNOSTIC
 371 RB_GENERATE_STATIC(rbctdth32, voistatdata_tdgstctd32, rblnk, ctd32cmp);
 372 RB_GENERATE_STATIC(rbctdth64, voistatdata_tdgstctd64, rblnk, ctd64cmp);
 373 #endif
 374
 375 static inline sbintime_t
 376 stats_sbinuptime(void)
 377 {
 378         sbintime_t sbt;
 379 #ifdef _KERNEL
 380
 381         sbt = sbinuptime();
 382 #else /* ! _KERNEL */
 383         struct timespec tp;
 384
 385         clock_gettime(CLOCK_MONOTONIC_FAST, &tp);
 386         sbt = tstosbt(tp);
 387 #endif /* _KERNEL */
 388
 389         return (sbt);
 390 }
 391
 392 static inline void *
 393 stats_realloc(void *ptr, size_t oldsz, size_t newsz, int flags)
 394 {
 395
 396 #ifdef _KERNEL
 397         /* Default to M_NOWAIT if neither M_NOWAIT or M_WAITOK are set. */
 398         if (!(flags & (M_WAITOK | M_NOWAIT)))
 399                 flags |= M_NOWAIT;
 400         ptr = realloc(ptr, newsz, M_STATS, flags);
 401 #else /* ! _KERNEL */
 402         ptr = realloc(ptr, newsz);
 403         if ((flags & M_ZERO) && ptr != NULL) {
 404                 if (oldsz == 0)
 405                         memset(ptr, '\0', newsz);
 406                 else if (newsz > oldsz)
 407                         memset(BLOB_OFFSET(ptr, oldsz), '\0', newsz - oldsz);
 408         }
 409 #endif /* _KERNEL */
 410
 411         return (ptr);
 412 }
 413
 414 static inline char *
 415 stats_strdup(const char *s,
 416 #ifdef _KERNEL
 417     int flags)
 418 {
 419         char *copy;
 420         size_t len;
 421
 422         if (!(flags & (M_WAITOK | M_NOWAIT)))
 423                 flags |= M_NOWAIT;
 424
 425         len = strlen(s) + 1;
 426         if ((copy = malloc(len, M_STATS, flags)) != NULL)
 427                 bcopy(s, copy, len);
 428
 429         return (copy);
 430 #else
 431     int flags __unused)
 432 {
 433         return (strdup(s));
 434 #endif
 435 }
 436
 437 static inline void
 438 stats_tpl_update_hash(struct statsblob_tpl *tpl)
 439 {
 440
 441         TPL_LIST_WLOCK_ASSERT();
 442         tpl->mb->tplhash = hash32_str(tpl->mb->tplname, 0);
 443         for (int voi_id = 0; voi_id < NVOIS(tpl->sb); voi_id++) {
 444                 if (tpl->mb->voi_meta[voi_id].name != NULL)
 445                         tpl->mb->tplhash = hash32_str(
 446                             tpl->mb->voi_meta[voi_id].name, tpl->mb->tplhash);
 447         }
 448         tpl->mb->tplhash = hash32_buf(tpl->sb, tpl->sb->cursz,
 449             tpl->mb->tplhash);
 450 }
 451
 452 static inline uint64_t
 453 stats_pow_u64(uint64_t base, uint64_t exp)
 454 {
 455         uint64_t result = 1;
 456
 457         while (exp) {
 458                 if (exp & 1)
 459                         result *= base;
 460                 exp >>= 1;
 461                 base *= base;
 462         }
 463
 464         return (result);
 465 }
 466
 467 static inline int
 468 stats_vss_hist_bkt_hlpr(struct vss_hist_hlpr_info *info, uint32_t curbkt,
 469     struct voistatdata_numeric *bkt_lb, struct voistatdata_numeric *bkt_ub)
 470 {
 471         uint64_t step = 0;
 472         int error = 0;
 473
 474         switch (info->scheme) {
 475         case BKT_LIN:
 476                 step = info->lin.stepinc;
 477                 break;
 478         case BKT_EXP:
 479                 step = stats_pow_u64(info->exp.stepbase,
 480                     info->exp.stepexp + curbkt);
 481                 break;
 482         case BKT_LINEXP:
 483                 {
 484                 uint64_t curstepexp = 1;
 485
 486                 switch (info->voi_dtype) {
 487                 case VSD_DTYPE_INT_S32:
 488                         while ((int32_t)stats_pow_u64(info->linexp.stepbase,
 489                             curstepexp) <= bkt_lb->int32.s32)
 490                                 curstepexp++;
 491                         break;
 492                 case VSD_DTYPE_INT_U32:
 493                         while ((uint32_t)stats_pow_u64(info->linexp.stepbase,
 494                             curstepexp) <= bkt_lb->int32.u32)
 495                                 curstepexp++;
 496                         break;
 497                 case VSD_DTYPE_INT_S64:
 498                         while ((int64_t)stats_pow_u64(info->linexp.stepbase,
 499                             curstepexp) <= bkt_lb->int64.s64)
 500                                 curstepexp++;
 501                         break;
 502                 case VSD_DTYPE_INT_U64:
 503                         while ((uint64_t)stats_pow_u64(info->linexp.stepbase,
 504                             curstepexp) <= bkt_lb->int64.u64)
 505                                 curstepexp++;
 506                         break;
 507                 case VSD_DTYPE_INT_SLONG:
 508                         while ((long)stats_pow_u64(info->linexp.stepbase,
 509                             curstepexp) <= bkt_lb->intlong.slong)
 510                                 curstepexp++;
 511                         break;
 512                 case VSD_DTYPE_INT_ULONG:
 513                         while ((unsigned long)stats_pow_u64(info->linexp.stepbase,
 514                             curstepexp) <= bkt_lb->intlong.ulong)
 515                                 curstepexp++;
 516                         break;
 517                 case VSD_DTYPE_Q_S32:
 518                         while ((s32q_t)stats_pow_u64(info->linexp.stepbase,
 519                             curstepexp) <= Q_GIVAL(bkt_lb->q32.sq32))
 520                         break;
 521                 case VSD_DTYPE_Q_U32:
 522                         while ((u32q_t)stats_pow_u64(info->linexp.stepbase,
 523                             curstepexp) <= Q_GIVAL(bkt_lb->q32.uq32))
 524                         break;
 525                 case VSD_DTYPE_Q_S64:
 526                         while ((s64q_t)stats_pow_u64(info->linexp.stepbase,
 527                             curstepexp) <= Q_GIVAL(bkt_lb->q64.sq64))
 528                                 curstepexp++;
 529                         break;
 530                 case VSD_DTYPE_Q_U64:
 531                         while ((u64q_t)stats_pow_u64(info->linexp.stepbase,
 532                             curstepexp) <= Q_GIVAL(bkt_lb->q64.uq64))
 533                                 curstepexp++;
 534                         break;
 535                 default:
 536                         break;
 537                 }
 538
 539                 step = stats_pow_u64(info->linexp.stepbase, curstepexp) /
 540                     info->linexp.linstepdiv;
 541                 if (step == 0)
 542                         step = 1;
 543                 break;
 544                 }
 545         default:
 546                 break;
 547         }
 548
 549         if (info->scheme == BKT_USR) {
 550                 *bkt_lb = info->usr.bkts[curbkt].lb;
 551                 *bkt_ub = info->usr.bkts[curbkt].ub;
 552         } else if (step != 0) {
 553                 switch (info->voi_dtype) {
 554                 case VSD_DTYPE_INT_S32:
 555                         bkt_ub->int32.s32 += (int32_t)step;
 556                         break;
 557                 case VSD_DTYPE_INT_U32:
 558                         bkt_ub->int32.u32 += (uint32_t)step;
 559                         break;
 560                 case VSD_DTYPE_INT_S64:
 561                         bkt_ub->int64.s64 += (int64_t)step;
 562                         break;
 563                 case VSD_DTYPE_INT_U64:
 564                         bkt_ub->int64.u64 += (uint64_t)step;
 565                         break;
 566                 case VSD_DTYPE_INT_SLONG:
 567                         bkt_ub->intlong.slong += (long)step;
 568                         break;
 569                 case VSD_DTYPE_INT_ULONG:
 570                         bkt_ub->intlong.ulong += (unsigned long)step;
 571                         break;
 572                 case VSD_DTYPE_Q_S32:
 573                         error = Q_QADDI(&bkt_ub->q32.sq32, step);
 574                         break;
 575                 case VSD_DTYPE_Q_U32:
 576                         error = Q_QADDI(&bkt_ub->q32.uq32, step);
 577                         break;
 578                 case VSD_DTYPE_Q_S64:
 579                         error = Q_QADDI(&bkt_ub->q64.sq64, step);
 580                         break;
 581                 case VSD_DTYPE_Q_U64:
 582                         error = Q_QADDI(&bkt_ub->q64.uq64, step);
 583                         break;
 584                 default:
 585                         break;
 586                 }
 587         } else { /* info->scheme != BKT_USR && step == 0 */
 588                 return (EINVAL);
 589         }
 590
 591         return (error);
 592 }
 593
 594 static uint32_t
 595 stats_vss_hist_nbkts_hlpr(struct vss_hist_hlpr_info *info)
 596 {
 597         struct voistatdata_numeric bkt_lb, bkt_ub;
 598         uint32_t nbkts;
 599         int done;
 600
 601         if (info->scheme == BKT_USR) {
 602                 /* XXXLAS: Setting info->{lb,ub} from macro is tricky. */
 603                 info->lb = info->usr.bkts[0].lb;
 604                 info->ub = info->usr.bkts[info->usr.nbkts - 1].lb;
 605         }
 606
 607         nbkts = 0;
 608         done = 0;
 609         bkt_ub = info->lb;
 610
 611         do {
 612                 bkt_lb = bkt_ub;
 613                 if (stats_vss_hist_bkt_hlpr(info, nbkts++, &bkt_lb, &bkt_ub))
 614                         return (0);
 615
 616                 if (info->scheme == BKT_USR)
 617                         done = (nbkts == info->usr.nbkts);
 618                 else {
 619                         switch (info->voi_dtype) {
 620                         case VSD_DTYPE_INT_S32:
 621                                 done = (bkt_ub.int32.s32 > info->ub.int32.s32);
 622                                 break;
 623                         case VSD_DTYPE_INT_U32:
 624                                 done = (bkt_ub.int32.u32 > info->ub.int32.u32);
 625                                 break;
 626                         case VSD_DTYPE_INT_S64:
 627                                 done = (bkt_ub.int64.s64 > info->ub.int64.s64);
 628                                 break;
 629                         case VSD_DTYPE_INT_U64:
 630                                 done = (bkt_ub.int64.u64 > info->ub.int64.u64);
 631                                 break;
 632                         case VSD_DTYPE_INT_SLONG:
 633                                 done = (bkt_ub.intlong.slong >
 634                                     info->ub.intlong.slong);
 635                                 break;
 636                         case VSD_DTYPE_INT_ULONG:
 637                                 done = (bkt_ub.intlong.ulong >
 638                                     info->ub.intlong.ulong);
 639                                 break;
 640                         case VSD_DTYPE_Q_S32:
 641                                 done = Q_QGTQ(bkt_ub.q32.sq32,
 642                                     info->ub.q32.sq32);
 643                                 break;
 644                         case VSD_DTYPE_Q_U32:
 645                                 done = Q_QGTQ(bkt_ub.q32.uq32,
 646                                     info->ub.q32.uq32);
 647                                 break;
 648                         case VSD_DTYPE_Q_S64:
 649                                 done = Q_QGTQ(bkt_ub.q64.sq64,
 650                                     info->ub.q64.sq64);
 651                                 break;
 652                         case VSD_DTYPE_Q_U64:
 653                                 done = Q_QGTQ(bkt_ub.q64.uq64,
 654                                     info->ub.q64.uq64);
 655                                 break;
 656                         default:
 657                                 return (0);
 658                         }
 659                 }
 660         } while (!done);
 661
 662         if (info->flags & VSD_HIST_LBOUND_INF)
 663                 nbkts++;
 664         if (info->flags & VSD_HIST_UBOUND_INF)
 665                 nbkts++;
 666
 667         return (nbkts);
 668 }
 669
 670 int
 671 stats_vss_hist_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss,
 672     struct vss_hist_hlpr_info *info)
 673 {
 674         struct voistatdata_hist *hist;
 675         struct voistatdata_numeric bkt_lb, bkt_ub, *lbinfbktlb, *lbinfbktub,
 676             *ubinfbktlb, *ubinfbktub;
 677         uint32_t bkt, nbkts, nloop;
 678
 679         if (vss == NULL || info == NULL || (info->flags &
 680         (VSD_HIST_LBOUND_INF|VSD_HIST_UBOUND_INF) && (info->hist_dtype ==
 681         VSD_DTYPE_DVHIST32 || info->hist_dtype == VSD_DTYPE_DVHIST64)))
 682                 return (EINVAL);
 683
 684         info->voi_dtype = voi_dtype;
 685
 686         if ((nbkts = stats_vss_hist_nbkts_hlpr(info)) == 0)
 687                 return (EINVAL);
 688
 689         switch (info->hist_dtype) {
 690         case VSD_DTYPE_CRHIST32:
 691                 vss->vsdsz = HIST_NBKTS2VSDSZ(crhist32, nbkts);
 692                 break;
 693         case VSD_DTYPE_DRHIST32:
 694                 vss->vsdsz = HIST_NBKTS2VSDSZ(drhist32, nbkts);
 695                 break;
 696         case VSD_DTYPE_DVHIST32:
 697                 vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist32, nbkts);
 698                 break;
 699         case VSD_DTYPE_CRHIST64:
 700                 vss->vsdsz = HIST_NBKTS2VSDSZ(crhist64, nbkts);
 701                 break;
 702         case VSD_DTYPE_DRHIST64:
 703                 vss->vsdsz = HIST_NBKTS2VSDSZ(drhist64, nbkts);
 704                 break;
 705         case VSD_DTYPE_DVHIST64:
 706                 vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist64, nbkts);
 707                 break;
 708         default:
 709                 return (EINVAL);
 710         }
 711
 712         vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO);
 713         if (vss->iv == NULL)
 714                 return (ENOMEM);
 715
 716         hist = (struct voistatdata_hist *)vss->iv;
 717         bkt_ub = info->lb;
 718
 719         for (bkt = (info->flags & VSD_HIST_LBOUND_INF), nloop = 0;
 720             bkt < nbkts;
 721             bkt++, nloop++) {
 722                 bkt_lb = bkt_ub;
 723                 if (stats_vss_hist_bkt_hlpr(info, nloop, &bkt_lb, &bkt_ub))
 724                         return (EINVAL);
 725
 726                 switch (info->hist_dtype) {
 727                 case VSD_DTYPE_CRHIST32:
 728                         VSD(crhist32, hist)->bkts[bkt].lb = bkt_lb;
 729                         break;
 730                 case VSD_DTYPE_DRHIST32:
 731                         VSD(drhist32, hist)->bkts[bkt].lb = bkt_lb;
 732                         VSD(drhist32, hist)->bkts[bkt].ub = bkt_ub;
 733                         break;
 734                 case VSD_DTYPE_DVHIST32:
 735                         VSD(dvhist32, hist)->bkts[bkt].val = bkt_lb;
 736                         break;
 737                 case VSD_DTYPE_CRHIST64:
 738                         VSD(crhist64, hist)->bkts[bkt].lb = bkt_lb;
 739                         break;
 740                 case VSD_DTYPE_DRHIST64:
 741                         VSD(drhist64, hist)->bkts[bkt].lb = bkt_lb;
 742                         VSD(drhist64, hist)->bkts[bkt].ub = bkt_ub;
 743                         break;
 744                 case VSD_DTYPE_DVHIST64:
 745                         VSD(dvhist64, hist)->bkts[bkt].val = bkt_lb;
 746                         break;
 747                 default:
 748                         return (EINVAL);
 749                 }
 750         }
 751
 752         lbinfbktlb = lbinfbktub = ubinfbktlb = ubinfbktub = NULL;
 753
 754         switch (info->hist_dtype) {
 755         case VSD_DTYPE_CRHIST32:
 756                 lbinfbktlb = &VSD(crhist32, hist)->bkts[0].lb;
 757                 ubinfbktlb = &VSD(crhist32, hist)->bkts[nbkts - 1].lb;
 758                 break;
 759         case VSD_DTYPE_DRHIST32:
 760                 lbinfbktlb = &VSD(drhist32, hist)->bkts[0].lb;
 761                 lbinfbktub = &VSD(drhist32, hist)->bkts[0].ub;
 762                 ubinfbktlb = &VSD(drhist32, hist)->bkts[nbkts - 1].lb;
 763                 ubinfbktub = &VSD(drhist32, hist)->bkts[nbkts - 1].ub;
 764                 break;
 765         case VSD_DTYPE_CRHIST64:
 766                 lbinfbktlb = &VSD(crhist64, hist)->bkts[0].lb;
 767                 ubinfbktlb = &VSD(crhist64, hist)->bkts[nbkts - 1].lb;
 768                 break;
 769         case VSD_DTYPE_DRHIST64:
 770                 lbinfbktlb = &VSD(drhist64, hist)->bkts[0].lb;
 771                 lbinfbktub = &VSD(drhist64, hist)->bkts[0].ub;
 772                 ubinfbktlb = &VSD(drhist64, hist)->bkts[nbkts - 1].lb;
 773                 ubinfbktub = &VSD(drhist64, hist)->bkts[nbkts - 1].ub;
 774                 break;
 775         case VSD_DTYPE_DVHIST32:
 776         case VSD_DTYPE_DVHIST64:
 777                 break;
 778         default:
 779                 return (EINVAL);
 780         }
 781
 782         if ((info->flags & VSD_HIST_LBOUND_INF) && lbinfbktlb) {
 783                 *lbinfbktlb = numeric_limits[LIM_MIN][info->voi_dtype];
 784                 /*
 785                  * Assignment from numeric_limit array for Q types assigns max
 786                  * possible integral/fractional value for underlying data type,
 787                  * but we must set control bits for this specific histogram per
 788                  * the user's choice of fractional bits, which we extract from
 789                  * info->lb.
 790                  */
 791                 if (info->voi_dtype == VSD_DTYPE_Q_S32 ||
 792                     info->voi_dtype == VSD_DTYPE_Q_U32) {
 793                         /* Signedness doesn't matter for setting control bits. */
 794                         Q_SCVAL(lbinfbktlb->q32.sq32,
 795                             Q_GCVAL(info->lb.q32.sq32));
 796                 } else if (info->voi_dtype == VSD_DTYPE_Q_S64 ||
 797                     info->voi_dtype == VSD_DTYPE_Q_U64) {
 798                         /* Signedness doesn't matter for setting control bits. */
 799                         Q_SCVAL(lbinfbktlb->q64.sq64,
 800                             Q_GCVAL(info->lb.q64.sq64));
 801                 }
 802                 if (lbinfbktub)
 803                         *lbinfbktub = info->lb;
 804         }
 805         if ((info->flags & VSD_HIST_UBOUND_INF) && ubinfbktlb) {
 806                 *ubinfbktlb = bkt_lb;
 807                 if (ubinfbktub) {
 808                         *ubinfbktub = numeric_limits[LIM_MAX][info->voi_dtype];
 809                         if (info->voi_dtype == VSD_DTYPE_Q_S32 ||
 810                             info->voi_dtype == VSD_DTYPE_Q_U32) {
 811                                 Q_SCVAL(ubinfbktub->q32.sq32,
 812                                     Q_GCVAL(info->lb.q32.sq32));
 813                         } else if (info->voi_dtype == VSD_DTYPE_Q_S64 ||
 814                             info->voi_dtype == VSD_DTYPE_Q_U64) {
 815                                 Q_SCVAL(ubinfbktub->q64.sq64,
 816                                     Q_GCVAL(info->lb.q64.sq64));
 817                         }
 818                 }
 819         }
 820
 821         return (0);
 822 }
 823
 824 int
 825 stats_vss_tdgst_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss,
 826     struct vss_tdgst_hlpr_info *info)
 827 {
 828         struct voistatdata_tdgst *tdgst;
 829         struct ctdth32 *ctd32tree;
 830         struct ctdth64 *ctd64tree;
 831         struct voistatdata_tdgstctd32 *ctd32;
 832         struct voistatdata_tdgstctd64 *ctd64;
 833
 834         info->voi_dtype = voi_dtype;
 835
 836         switch (info->tdgst_dtype) {
 837         case VSD_DTYPE_TDGSTCLUST32:
 838                 vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust32, info->nctds);
 839                 break;
 840         case VSD_DTYPE_TDGSTCLUST64:
 841                 vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust64, info->nctds);
 842                 break;
 843         default:
 844                 return (EINVAL);
 845         }
 846
 847         vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO);
 848         if (vss->iv == NULL)
 849                 return (ENOMEM);
 850
 851         tdgst = (struct voistatdata_tdgst *)vss->iv;
 852
 853         switch (info->tdgst_dtype) {
 854         case VSD_DTYPE_TDGSTCLUST32:
 855                 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
 856                 ARB_INIT(ctd32, ctdlnk, ctd32tree, info->nctds) {
 857                         Q_INI(&ctd32->mu, 0, 0, info->prec);
 858                 }
 859                 break;
 860         case VSD_DTYPE_TDGSTCLUST64:
 861                 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
 862                 ARB_INIT(ctd64, ctdlnk, ctd64tree, info->nctds) {
 863                         Q_INI(&ctd64->mu, 0, 0, info->prec);
 864                 }
 865                 break;
 866         default:
 867                 return (EINVAL);
 868         }
 869
 870         return (0);
 871 }
 872
 873 int
 874 stats_vss_numeric_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss,
 875     struct vss_numeric_hlpr_info *info)
 876 {
 877         struct voistatdata_numeric iv;
 878
 879         switch (vss->stype) {
 880         case VS_STYPE_SUM:
 881                 iv = stats_ctor_vsd_numeric(0);
 882                 break;
 883         case VS_STYPE_MIN:
 884                 iv = numeric_limits[LIM_MAX][voi_dtype];
 885                 break;
 886         case VS_STYPE_MAX:
 887                 iv = numeric_limits[LIM_MIN][voi_dtype];
 888                 break;
 889         default:
 890                 return (EINVAL);
 891         }
 892
 893         vss->iv = stats_realloc(NULL, 0, vsd_dtype2size[voi_dtype], 0);
 894         if (vss->iv == NULL)
 895                 return (ENOMEM);
 896
 897         vss->vs_dtype = voi_dtype;
 898         vss->vsdsz = vsd_dtype2size[voi_dtype];
 899         switch (voi_dtype) {
 900         case VSD_DTYPE_INT_S32:
 901                 *((int32_t *)vss->iv) = iv.int32.s32;
 902                 break;
 903         case VSD_DTYPE_INT_U32:
 904                 *((uint32_t *)vss->iv) = iv.int32.u32;
 905                 break;
 906         case VSD_DTYPE_INT_S64:
 907                 *((int64_t *)vss->iv) = iv.int64.s64;
 908                 break;
 909         case VSD_DTYPE_INT_U64:
 910                 *((uint64_t *)vss->iv) = iv.int64.u64;
 911                 break;
 912         case VSD_DTYPE_INT_SLONG:
 913                 *((long *)vss->iv) = iv.intlong.slong;
 914                 break;
 915         case VSD_DTYPE_INT_ULONG:
 916                 *((unsigned long *)vss->iv) = iv.intlong.ulong;
 917                 break;
 918         case VSD_DTYPE_Q_S32:
 919                 *((s32q_t *)vss->iv) = Q_SCVAL(iv.q32.sq32,
 920                     Q_CTRLINI(info->prec));
 921                 break;
 922         case VSD_DTYPE_Q_U32:
 923                 *((u32q_t *)vss->iv) = Q_SCVAL(iv.q32.uq32,
 924                     Q_CTRLINI(info->prec));
 925                 break;
 926         case VSD_DTYPE_Q_S64:
 927                 *((s64q_t *)vss->iv) = Q_SCVAL(iv.q64.sq64,
 928                     Q_CTRLINI(info->prec));
 929                 break;
 930         case VSD_DTYPE_Q_U64:
 931                 *((u64q_t *)vss->iv) = Q_SCVAL(iv.q64.uq64,
 932                     Q_CTRLINI(info->prec));
 933                 break;
 934         default:
 935                 break;
 936         }
 937
 938         return (0);
 939 }
 940
 941 int
 942 stats_vss_hlpr_init(enum vsd_dtype voi_dtype, uint32_t nvss,
 943     struct voistatspec *vss)
 944 {
 945         int i, ret;
 946
 947         for (i = nvss - 1; i >= 0; i--) {
 948                 if (vss[i].hlpr && (ret = vss[i].hlpr(voi_dtype, &vss[i],
 949                     vss[i].hlprinfo)) != 0)
 950                         return (ret);
 951         }
 952
 953         return (0);
 954 }
 955
 956 void
 957 stats_vss_hlpr_cleanup(uint32_t nvss, struct voistatspec *vss)
 958 {
 959         int i;
 960
 961         for (i = nvss - 1; i >= 0; i--) {
 962                 if (vss[i].hlpr) {
 963                         stats_free((void *)vss[i].iv);
 964                         vss[i].iv = NULL;
 965                 }
 966         }
 967 }
 968
 969 int
 970 stats_tpl_fetch(int tpl_id, struct statsblob_tpl **tpl)
 971 {
 972         int error;
 973
 974         error = 0;
 975
 976         TPL_LIST_WLOCK();
 977         if (tpl_id < 0 || tpl_id >= (int)ntpl) {
 978                 error = ENOENT;
 979         } else {
 980                 *tpl = tpllist[tpl_id];
 981                 /* XXXLAS: Acquire refcount on tpl. */
 982         }
 983         TPL_LIST_WUNLOCK();
 984
 985         return (error);
 986 }
 987
 988 int
 989 stats_tpl_fetch_allocid(const char *name, uint32_t hash)
 990 {
 991         int i, tpl_id;
 992
 993         tpl_id = -ESRCH;
 994
 995         TPL_LIST_RLOCK();
 996         for (i = ntpl - 1; i >= 0; i--) {
 997                 if (name != NULL) {
 998                         if (strlen(name) == strlen(tpllist[i]->mb->tplname) &&
 999                             strncmp(name, tpllist[i]->mb->tplname,
1000                             TPL_MAX_NAME_LEN) == 0 && (!hash || hash ==
1001                             tpllist[i]->mb->tplhash)) {
1002                                 tpl_id = i;
1003                                 break;
1004                         }
1005                 } else if (hash == tpllist[i]->mb->tplhash) {
1006                         tpl_id = i;
1007                         break;
1008                 }
1009         }
1010         TPL_LIST_RUNLOCK();
1011
1012         return (tpl_id);
1013 }
1014
1015 int
1016 stats_tpl_id2name(uint32_t tpl_id, char *buf, size_t len)
1017 {
1018         int error;
1019
1020         error = 0;
1021
1022         TPL_LIST_RLOCK();
1023         if (tpl_id < ntpl) {
1024                 if (buf != NULL && len > strlen(tpllist[tpl_id]->mb->tplname))
1025                         strlcpy(buf, tpllist[tpl_id]->mb->tplname, len);
1026                 else
1027                         error = EOVERFLOW;
1028         } else
1029                 error = ENOENT;
1030         TPL_LIST_RUNLOCK();
1031
1032         return (error);
1033 }
1034
1035 int
1036 stats_tpl_sample_rollthedice(struct stats_tpl_sample_rate *rates, int nrates,
1037     void *seed_bytes, size_t seed_len)
1038 {
1039         uint32_t cum_pct, rnd_pct;
1040         int i;
1041
1042         cum_pct = 0;
1043
1044         /*
1045          * Choose a pseudorandom or seeded number in range [0,100] and use
1046          * it to make a sampling decision and template selection where required.
1047          * If no seed is supplied, a PRNG is used to generate a pseudorandom
1048          * number so that every selection is independent. If a seed is supplied,
1049          * the caller desires random selection across different seeds, but
1050          * deterministic selection given the same seed. This is achieved by
1051          * hashing the seed and using the hash as the random number source.
1052          *
1053          * XXXLAS: Characterise hash function output distribution.
1054          */
1055         if (seed_bytes == NULL)
1056                 rnd_pct = random() / (INT32_MAX / 100);
1057         else
1058                 rnd_pct = hash32_buf(seed_bytes, seed_len, 0) /
1059                     (UINT32_MAX / 100U);
1060
1061         /*
1062          * We map the randomly selected percentage on to the interval [0,100]
1063          * consisting of the cumulatively summed template sampling percentages.
1064          * The difference between the cumulative sum of all template sampling
1065          * percentages and 100 is treated as a NULL assignment i.e. no stats
1066          * template will be assigned, and -1 returned instead.
1067          */
1068         for (i = 0; i < nrates; i++) {
1069                 cum_pct += rates[i].tpl_sample_pct;
1070
1071                 KASSERT(cum_pct <= 100, ("%s cum_pct %u > 100", __func__,
1072                     cum_pct));
1073                 if (rnd_pct > cum_pct || rates[i].tpl_sample_pct == 0)
1074                         continue;
1075
1076                 return (rates[i].tpl_slot_id);
1077         }
1078
1079         return (-1);
1080 }
1081
1082 int
1083 stats_v1_blob_clone(struct statsblobv1 **dst, size_t dstmaxsz,
1084     struct statsblobv1 *src, uint32_t flags)
1085 {
1086         int error;
1087
1088         error = 0;
1089
1090         if (src == NULL || dst == NULL ||
1091             src->cursz < sizeof(struct statsblob) ||
1092             ((flags & SB_CLONE_ALLOCDST) &&
1093             (flags & (SB_CLONE_USRDSTNOFAULT | SB_CLONE_USRDST)))) {
1094                 error = EINVAL;
1095         } else if (flags & SB_CLONE_ALLOCDST) {
1096                 *dst = stats_realloc(NULL, 0, src->cursz, 0);
1097                 if (*dst)
1098                         (*dst)->maxsz = dstmaxsz = src->cursz;
1099                 else
1100                         error = ENOMEM;
1101         } else if (*dst == NULL || dstmaxsz < sizeof(struct statsblob)) {
1102                 error = EINVAL;
1103         }
1104
1105         if (!error) {
1106                 size_t postcurszlen;
1107
1108                 /*
1109                  * Clone src into dst except for the maxsz field. If dst is too
1110                  * small to hold all of src, only copy src's header and return
1111                  * EOVERFLOW.
1112                  */
1113 #ifdef _KERNEL
1114                 if (flags & SB_CLONE_USRDSTNOFAULT)
1115                         copyout_nofault(src, *dst,
1116                             offsetof(struct statsblob, maxsz));
1117                 else if (flags & SB_CLONE_USRDST)
1118                         copyout(src, *dst, offsetof(struct statsblob, maxsz));
1119                 else
1120 #endif
1121                         memcpy(*dst, src, offsetof(struct statsblob, maxsz));
1122
1123                 if (dstmaxsz >= src->cursz) {
1124                         postcurszlen = src->cursz -
1125                             offsetof(struct statsblob, cursz);
1126                 } else {
1127                         error = EOVERFLOW;
1128                         postcurszlen = sizeof(struct statsblob) -
1129                             offsetof(struct statsblob, cursz);
1130                 }
1131 #ifdef _KERNEL
1132                 if (flags & SB_CLONE_USRDSTNOFAULT)
1133                         copyout_nofault(&(src->cursz), &((*dst)->cursz),
1134                             postcurszlen);
1135                 else if (flags & SB_CLONE_USRDST)
1136                         copyout(&(src->cursz), &((*dst)->cursz), postcurszlen);
1137                 else
1138 #endif
1139                         memcpy(&((*dst)->cursz), &(src->cursz), postcurszlen);
1140         }
1141
1142         return (error);
1143 }
1144
1145 int
1146 stats_v1_tpl_alloc(const char *name, uint32_t flags __unused)
1147 {
1148         struct statsblobv1_tpl *tpl, **newtpllist;
1149         struct statsblobv1 *tpl_sb;
1150         struct metablob *tpl_mb;
1151         int tpl_id;
1152
1153         if (name != NULL && strlen(name) > TPL_MAX_NAME_LEN)
1154                 return (-EINVAL);
1155
1156         if (name != NULL && stats_tpl_fetch_allocid(name, 0) >= 0)
1157                 return (-EEXIST);
1158
1159         tpl = stats_realloc(NULL, 0, sizeof(struct statsblobv1_tpl), M_ZERO);
1160         tpl_mb = stats_realloc(NULL, 0, sizeof(struct metablob), M_ZERO);
1161         tpl_sb = stats_realloc(NULL, 0, sizeof(struct statsblobv1), M_ZERO);
1162
1163         if (tpl_mb != NULL && name != NULL)
1164                 tpl_mb->tplname = stats_strdup(name, 0);
1165
1166         if (tpl == NULL || tpl_sb == NULL || tpl_mb == NULL ||
1167             tpl_mb->tplname == NULL) {
1168                 stats_free(tpl);
1169                 stats_free(tpl_sb);
1170                 if (tpl_mb != NULL) {
1171                         stats_free(tpl_mb->tplname);
1172                         stats_free(tpl_mb);
1173                 }
1174                 return (-ENOMEM);
1175         }
1176
1177         tpl->mb = tpl_mb;
1178         tpl->sb = tpl_sb;
1179
1180         tpl_sb->abi = STATS_ABI_V1;
1181         tpl_sb->endian =
1182 #if BYTE_ORDER == LITTLE_ENDIAN
1183             SB_LE;
1184 #elif BYTE_ORDER == BIG_ENDIAN
1185             SB_BE;
1186 #else
1187             SB_UE;
1188 #endif
1189         tpl_sb->cursz = tpl_sb->maxsz = sizeof(struct statsblobv1);
1190         tpl_sb->stats_off = tpl_sb->statsdata_off = sizeof(struct statsblobv1);
1191
1192         TPL_LIST_WLOCK();
1193         newtpllist = stats_realloc(tpllist, ntpl * sizeof(void *),
1194             (ntpl + 1) * sizeof(void *), 0);
1195         if (newtpllist != NULL) {
1196                 tpl_id = ntpl++;
1197                 tpllist = (struct statsblob_tpl **)newtpllist;
1198                 tpllist[tpl_id] = (struct statsblob_tpl *)tpl;
1199                 stats_tpl_update_hash(tpllist[tpl_id]);
1200         } else {
1201                 stats_free(tpl);
1202                 stats_free(tpl_sb);
1203                 if (tpl_mb != NULL) {
1204                         stats_free(tpl_mb->tplname);
1205                         stats_free(tpl_mb);
1206                 }
1207                 tpl_id = -ENOMEM;
1208         }
1209         TPL_LIST_WUNLOCK();
1210
1211         return (tpl_id);
1212 }
1213
1214 int
1215 stats_v1_tpl_add_voistats(uint32_t tpl_id, int32_t voi_id, const char *voi_name,
1216     enum vsd_dtype voi_dtype, uint32_t nvss, struct voistatspec *vss,
1217     uint32_t flags)
1218 {
1219         struct voi *voi;
1220         struct voistat *tmpstat;
1221         struct statsblobv1 *tpl_sb;
1222         struct metablob *tpl_mb;
1223         int error, i, newstatdataidx, newvoibytes, newvoistatbytes,
1224             newvoistatdatabytes, newvoistatmaxid;
1225         uint32_t nbytes;
1226
1227         if (voi_id < 0 || voi_dtype == 0 || voi_dtype >= VSD_NUM_DTYPES ||
1228             nvss == 0 || vss == NULL)
1229                 return (EINVAL);
1230
1231         error = nbytes = newvoibytes = newvoistatbytes =
1232             newvoistatdatabytes = 0;
1233         newvoistatmaxid = -1;
1234
1235         /* Calculate the number of bytes required for the new voistats. */
1236         for (i = nvss - 1; i >= 0; i--) {
1237                 if (vss[i].stype == 0 || vss[i].stype >= VS_NUM_STYPES ||
1238                     vss[i].vs_dtype == 0 || vss[i].vs_dtype >= VSD_NUM_DTYPES ||
1239                     vss[i].iv == NULL || vss[i].vsdsz == 0)
1240                         return (EINVAL);
1241                 if ((int)vss[i].stype > newvoistatmaxid)
1242                         newvoistatmaxid = vss[i].stype;
1243                 newvoistatdatabytes += vss[i].vsdsz;
1244         }
1245
1246         if (flags & SB_VOI_RELUPDATE) {
1247                 /* XXXLAS: VOI state bytes may need to vary based on stat types. */
1248                 newvoistatdatabytes += sizeof(struct voistatdata_voistate);
1249         }
1250         nbytes += newvoistatdatabytes;
1251
1252         TPL_LIST_WLOCK();
1253         if (tpl_id < ntpl) {
1254                 tpl_sb = (struct statsblobv1 *)tpllist[tpl_id]->sb;
1255                 tpl_mb = tpllist[tpl_id]->mb;
1256
1257                 if (voi_id >= NVOIS(tpl_sb) || tpl_sb->vois[voi_id].id == -1) {
1258                         /* Adding a new VOI and associated stats. */
1259                         if (voi_id >= NVOIS(tpl_sb)) {
1260                                 /* We need to grow the tpl_sb->vois array. */
1261                                 newvoibytes = (voi_id - (NVOIS(tpl_sb) - 1)) *
1262                                     sizeof(struct voi);
1263                                 nbytes += newvoibytes;
1264                         }
1265                         newvoistatbytes =
1266                             (newvoistatmaxid + 1) * sizeof(struct voistat);
1267                 } else {
1268                         /* Adding stats to an existing VOI. */
1269                         if (newvoistatmaxid >
1270                             tpl_sb->vois[voi_id].voistatmaxid) {
1271                                 newvoistatbytes = (newvoistatmaxid -
1272                                     tpl_sb->vois[voi_id].voistatmaxid) *
1273                                     sizeof(struct voistat);
1274                         }
1275                         /* XXXLAS: KPI does not yet support expanding VOIs. */
1276                         error = EOPNOTSUPP;
1277                 }
1278                 nbytes += newvoistatbytes;
1279
1280                 if (!error && newvoibytes > 0) {
1281                         struct voi_meta *voi_meta = tpl_mb->voi_meta;
1282
1283                         voi_meta = stats_realloc(voi_meta, voi_meta == NULL ?
1284                             0 : NVOIS(tpl_sb) * sizeof(struct voi_meta),
1285                             (1 + voi_id) * sizeof(struct voi_meta),
1286                             M_ZERO);
1287
1288                         if (voi_meta == NULL)
1289                                 error = ENOMEM;
1290                         else
1291                                 tpl_mb->voi_meta = voi_meta;
1292                 }
1293
1294                 if (!error) {
1295                         /* NB: Resizing can change where tpl_sb points. */
1296                         error = stats_v1_blob_expand(&tpl_sb, newvoibytes,
1297                             newvoistatbytes, newvoistatdatabytes);
1298                 }
1299
1300                 if (!error) {
1301                         tpl_mb->voi_meta[voi_id].name = stats_strdup(voi_name,
1302                             0);
1303                         if (tpl_mb->voi_meta[voi_id].name == NULL)
1304                                 error = ENOMEM;
1305                 }
1306
1307                 if (!error) {
1308                         /* Update the template list with the resized pointer. */
1309                         tpllist[tpl_id]->sb = (struct statsblob *)tpl_sb;
1310
1311                         /* Update the template. */
1312                         voi = &tpl_sb->vois[voi_id];
1313
1314                         if (voi->id < 0) {
1315                                 /* VOI is new and needs to be initialised. */
1316                                 voi->id = voi_id;
1317                                 voi->dtype = voi_dtype;
1318                                 voi->stats_off = tpl_sb->stats_off;
1319                                 if (flags & SB_VOI_RELUPDATE)
1320                                         voi->flags |= VOI_REQSTATE;
1321                         } else {
1322                                 /*
1323                                  * XXXLAS: When this else block is written, the
1324                                  * "KPI does not yet support expanding VOIs"
1325                                  * error earlier in this function can be
1326                                  * removed. What is required here is to shuffle
1327                                  * the voistat array such that the new stats for
1328                                  * the voi are contiguous, which will displace
1329                                  * stats for other vois that reside after the
1330                                  * voi being updated. The other vois then need
1331                                  * to have their stats_off adjusted post
1332                                  * shuffle.
1333                                  */
1334                         }
1335
1336                         voi->voistatmaxid = newvoistatmaxid;
1337                         newstatdataidx = 0;
1338
1339                         if (voi->flags & VOI_REQSTATE) {
1340                                 /* Initialise the voistate stat in slot 0. */
1341                                 tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off);
1342                                 tmpstat->stype = VS_STYPE_VOISTATE;
1343                                 tmpstat->flags = 0;
1344                                 tmpstat->dtype = VSD_DTYPE_VOISTATE;
1345                                 newstatdataidx = tmpstat->dsz =
1346                                     sizeof(struct voistatdata_numeric);
1347                                 tmpstat->data_off = tpl_sb->statsdata_off;
1348                         }
1349
1350                         for (i = 0; (uint32_t)i < nvss; i++) {
1351                                 tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off +
1352                                     (vss[i].stype * sizeof(struct voistat)));
1353                                 KASSERT(tmpstat->stype < 0, ("voistat %p "
1354                                     "already initialised", tmpstat));
1355                                 tmpstat->stype = vss[i].stype;
1356                                 tmpstat->flags = vss[i].flags;
1357                                 tmpstat->dtype = vss[i].vs_dtype;
1358                                 tmpstat->dsz = vss[i].vsdsz;
1359                                 tmpstat->data_off = tpl_sb->statsdata_off +
1360                                     newstatdataidx;
1361                                 memcpy(BLOB_OFFSET(tpl_sb, tmpstat->data_off),
1362                                     vss[i].iv, vss[i].vsdsz);
1363                                 newstatdataidx += vss[i].vsdsz;
1364                         }
1365
1366                         /* Update the template version hash. */
1367                         stats_tpl_update_hash(tpllist[tpl_id]);
1368                         /* XXXLAS: Confirm tpl name/hash pair remains unique. */
1369                 }
1370         } else
1371                 error = EINVAL;
1372         TPL_LIST_WUNLOCK();
1373
1374         return (error);
1375 }
1376
1377 struct statsblobv1 *
1378 stats_v1_blob_alloc(uint32_t tpl_id, uint32_t flags __unused)
1379 {
1380         struct statsblobv1 *sb;
1381         int error;
1382
1383         sb = NULL;
1384
1385         TPL_LIST_RLOCK();
1386         if (tpl_id < ntpl) {
1387                 sb = stats_realloc(NULL, 0, tpllist[tpl_id]->sb->maxsz, 0);
1388                 if (sb != NULL) {
1389                         sb->maxsz = tpllist[tpl_id]->sb->maxsz;
1390                         error = stats_v1_blob_init_locked(sb, tpl_id, 0);
1391                 } else
1392                         error = ENOMEM;
1393
1394                 if (error) {
1395                         stats_free(sb);
1396                         sb = NULL;
1397                 }
1398         }
1399         TPL_LIST_RUNLOCK();
1400
1401         return (sb);
1402 }
1403
1404 void
1405 stats_v1_blob_destroy(struct statsblobv1 *sb)
1406 {
1407
1408         stats_free(sb);
1409 }
1410
1411 int
1412 stats_v1_voistat_fetch_dptr(struct statsblobv1 *sb, int32_t voi_id,
1413     enum voi_stype stype, enum vsd_dtype *retdtype, struct voistatdata **retvsd,
1414     size_t *retvsdsz)
1415 {
1416         struct voi *v;
1417         struct voistat *vs;
1418
1419         if (retvsd == NULL || sb == NULL || sb->abi != STATS_ABI_V1 ||
1420             voi_id >= NVOIS(sb))
1421                 return (EINVAL);
1422
1423         v = &sb->vois[voi_id];
1424         if ((__typeof(v->voistatmaxid))stype > v->voistatmaxid)
1425                 return (EINVAL);
1426
1427         vs = BLOB_OFFSET(sb, v->stats_off + (stype * sizeof(struct voistat)));
1428         *retvsd = BLOB_OFFSET(sb, vs->data_off);
1429         if (retdtype != NULL)
1430                 *retdtype = vs->dtype;
1431         if (retvsdsz != NULL)
1432                 *retvsdsz = vs->dsz;
1433
1434         return (0);
1435 }
1436
1437 int
1438 stats_v1_blob_init(struct statsblobv1 *sb, uint32_t tpl_id, uint32_t flags)
1439 {
1440         int error;
1441
1442         error = 0;
1443
1444         TPL_LIST_RLOCK();
1445         if (sb == NULL || tpl_id >= ntpl) {
1446                 error = EINVAL;
1447         } else {
1448                 error = stats_v1_blob_init_locked(sb, tpl_id, flags);
1449         }
1450         TPL_LIST_RUNLOCK();
1451
1452         return (error);
1453 }
1454
1455 static inline int
1456 stats_v1_blob_init_locked(struct statsblobv1 *sb, uint32_t tpl_id,
1457     uint32_t flags __unused)
1458 {
1459         int error;
1460
1461         TPL_LIST_RLOCK_ASSERT();
1462         error = (sb->maxsz >= tpllist[tpl_id]->sb->cursz) ? 0 : EOVERFLOW;
1463         KASSERT(!error,
1464             ("sb %d instead of %d bytes", sb->maxsz, tpllist[tpl_id]->sb->cursz));
1465
1466         if (!error) {
1467                 memcpy(sb, tpllist[tpl_id]->sb, tpllist[tpl_id]->sb->cursz);
1468                 sb->created = sb->lastrst = stats_sbinuptime();
1469                 sb->tplhash = tpllist[tpl_id]->mb->tplhash;
1470         }
1471
1472         return (error);
1473 }
1474
1475 static int
1476 stats_v1_blob_expand(struct statsblobv1 **sbpp, int newvoibytes,
1477     int newvoistatbytes, int newvoistatdatabytes)
1478 {
1479         struct statsblobv1 *sb;
1480         struct voi *tmpvoi;
1481         struct voistat *tmpvoistat, *voistat_array;
1482         int error, i, idxnewvois, idxnewvoistats, nbytes, nvoistats;
1483
1484         KASSERT(newvoibytes % sizeof(struct voi) == 0,
1485             ("Bad newvoibytes %d", newvoibytes));
1486         KASSERT(newvoistatbytes % sizeof(struct voistat) == 0,
1487             ("Bad newvoistatbytes %d", newvoistatbytes));
1488
1489         error = ((newvoibytes % sizeof(struct voi) == 0) &&
1490             (newvoistatbytes % sizeof(struct voistat) == 0)) ? 0 : EINVAL;
1491         sb = *sbpp;
1492         nbytes = newvoibytes + newvoistatbytes + newvoistatdatabytes;
1493
1494         /*
1495          * XXXLAS: Required until we gain support for flags which alter the
1496          * units of size/offset fields in key structs.
1497          */
1498         if (!error && ((((int)sb->cursz) + nbytes) > SB_V1_MAXSZ))
1499                 error = EFBIG;
1500
1501         if (!error && (sb->cursz + nbytes > sb->maxsz)) {
1502                 /* Need to expand our blob. */
1503                 sb = stats_realloc(sb, sb->maxsz, sb->cursz + nbytes, M_ZERO);
1504                 if (sb != NULL) {
1505                         sb->maxsz = sb->cursz + nbytes;
1506                         *sbpp = sb;
1507                 } else
1508                     error = ENOMEM;
1509         }
1510
1511         if (!error) {
1512                 /*
1513                  * Shuffle memory within the expanded blob working from the end
1514                  * backwards, leaving gaps for the new voistat and voistatdata
1515                  * structs at the beginning of their respective blob regions,
1516                  * and for the new voi structs at the end of their blob region.
1517                  */
1518                 memmove(BLOB_OFFSET(sb, sb->statsdata_off + nbytes),
1519                     BLOB_OFFSET(sb, sb->statsdata_off),
1520                     sb->cursz - sb->statsdata_off);
1521                 memmove(BLOB_OFFSET(sb, sb->stats_off + newvoibytes +
1522                     newvoistatbytes), BLOB_OFFSET(sb, sb->stats_off),
1523                     sb->statsdata_off - sb->stats_off);
1524
1525                 /* First index of new voi/voistat structs to be initialised. */
1526                 idxnewvois = NVOIS(sb);
1527                 idxnewvoistats = (newvoistatbytes / sizeof(struct voistat)) - 1;
1528
1529                 /* Update housekeeping variables and offsets. */
1530                 sb->cursz += nbytes;
1531                 sb->stats_off += newvoibytes;
1532                 sb->statsdata_off += newvoibytes + newvoistatbytes;
1533
1534                 /* XXXLAS: Zeroing not strictly needed but aids debugging. */
1535                 memset(&sb->vois[idxnewvois], '\0', newvoibytes);
1536                 memset(BLOB_OFFSET(sb, sb->stats_off), '\0',
1537                     newvoistatbytes);
1538                 memset(BLOB_OFFSET(sb, sb->statsdata_off), '\0',
1539                     newvoistatdatabytes);
1540
1541                 /* Initialise new voi array members and update offsets. */
1542                 for (i = 0; i < NVOIS(sb); i++) {
1543                         tmpvoi = &sb->vois[i];
1544                         if (i >= idxnewvois) {
1545                                 tmpvoi->id = tmpvoi->voistatmaxid = -1;
1546                         } else if (tmpvoi->id > -1) {
1547                                 tmpvoi->stats_off += newvoibytes +
1548                                     newvoistatbytes;
1549                         }
1550                 }
1551
1552                 /* Initialise new voistat array members and update offsets. */
1553                 nvoistats = (sb->statsdata_off - sb->stats_off) /
1554                     sizeof(struct voistat);
1555                 voistat_array = BLOB_OFFSET(sb, sb->stats_off);
1556                 for (i = 0; i < nvoistats; i++) {
1557                         tmpvoistat = &voistat_array[i];
1558                         if (i <= idxnewvoistats) {
1559                                 tmpvoistat->stype = -1;
1560                         } else if (tmpvoistat->stype > -1) {
1561                                 tmpvoistat->data_off += nbytes;
1562                         }
1563                 }
1564         }
1565
1566         return (error);
1567 }
1568
1569 static void
1570 stats_v1_blob_finalise(struct statsblobv1 *sb __unused)
1571 {
1572
1573         /* XXXLAS: Fill this in. */
1574 }
1575
1576 static void
1577 stats_v1_blob_iter(struct statsblobv1 *sb, stats_v1_blob_itercb_t icb,
1578     void *usrctx, uint32_t flags)
1579 {
1580         struct voi *v;
1581         struct voistat *vs;
1582         struct sb_iter_ctx ctx;
1583         int i, j, firstvoi;
1584
1585         ctx.usrctx = usrctx;
1586         ctx.flags = SB_IT_FIRST_CB;
1587         firstvoi = 1;
1588
1589         for (i = 0; i < NVOIS(sb); i++) {
1590                 v = &sb->vois[i];
1591                 ctx.vslot = i;
1592                 ctx.vsslot = -1;
1593                 ctx.flags |= SB_IT_FIRST_VOISTAT;
1594
1595                 if (firstvoi)
1596                         ctx.flags |= SB_IT_FIRST_VOI;
1597                 else if (i == (NVOIS(sb) - 1))
1598                         ctx.flags |= SB_IT_LAST_VOI | SB_IT_LAST_CB;
1599
1600                 if (v->id < 0 && (flags & SB_IT_NULLVOI)) {
1601                         if (icb(sb, v, NULL, &ctx))
1602                                 return;
1603                         firstvoi = 0;
1604                         ctx.flags &= ~SB_IT_FIRST_CB;
1605                 }
1606
1607                 /* If NULL voi, v->voistatmaxid == -1 */
1608                 for (j = 0; j <= v->voistatmaxid; j++) {
1609                         vs = &((struct voistat *)BLOB_OFFSET(sb,
1610                             v->stats_off))[j];
1611                         if (vs->stype < 0 &&
1612                             !(flags & SB_IT_NULLVOISTAT))
1613                                 continue;
1614
1615                         if (j == v->voistatmaxid) {
1616                                 ctx.flags |= SB_IT_LAST_VOISTAT;
1617                                 if (i == (NVOIS(sb) - 1))
1618                                         ctx.flags |=
1619                                             SB_IT_LAST_CB;
1620                         } else
1621                                 ctx.flags &= ~SB_IT_LAST_CB;
1622
1623                         ctx.vsslot = j;
1624                         if (icb(sb, v, vs, &ctx))
1625                                 return;
1626
1627                         ctx.flags &= ~(SB_IT_FIRST_CB | SB_IT_FIRST_VOISTAT |
1628                             SB_IT_LAST_VOISTAT);
1629                 }
1630                 ctx.flags &= ~(SB_IT_FIRST_VOI | SB_IT_LAST_VOI);
1631         }
1632 }
1633
1634 static inline void
1635 stats_voistatdata_tdgst_tostr(enum vsd_dtype voi_dtype __unused,
1636     const struct voistatdata_tdgst *tdgst, enum vsd_dtype tdgst_dtype,
1637     size_t tdgst_dsz __unused, enum sb_str_fmt fmt, struct sbuf *buf, int objdump)
1638 {
1639         const struct ctdth32 *ctd32tree;
1640         const struct ctdth64 *ctd64tree;
1641         const struct voistatdata_tdgstctd32 *ctd32;
1642         const struct voistatdata_tdgstctd64 *ctd64;
1643         const char *fmtstr;
1644         uint64_t smplcnt, compcnt;
1645         int is32bit, qmaxstrlen;
1646         uint16_t maxctds, curctds;
1647
1648         switch (tdgst_dtype) {
1649         case VSD_DTYPE_TDGSTCLUST32:
1650                 smplcnt = CONSTVSD(tdgstclust32, tdgst)->smplcnt;
1651                 compcnt = CONSTVSD(tdgstclust32, tdgst)->compcnt;
1652                 maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree);
1653                 curctds = ARB_CURNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree);
1654                 ctd32tree = &CONSTVSD(tdgstclust32, tdgst)->ctdtree;
1655                 ctd32 = (objdump ? ARB_CNODE(ctd32tree, 0) :
1656                     ARB_CMIN(ctdth32, ctd32tree));
1657                 qmaxstrlen = (ctd32 == NULL) ? 1 : Q_MAXSTRLEN(ctd32->mu, 10);
1658                 is32bit = 1;
1659                 ctd64tree = NULL;
1660                 ctd64 = NULL;
1661                 break;
1662         case VSD_DTYPE_TDGSTCLUST64:
1663                 smplcnt = CONSTVSD(tdgstclust64, tdgst)->smplcnt;
1664                 compcnt = CONSTVSD(tdgstclust64, tdgst)->compcnt;
1665                 maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree);
1666                 curctds = ARB_CURNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree);
1667                 ctd64tree = &CONSTVSD(tdgstclust64, tdgst)->ctdtree;
1668                 ctd64 = (objdump ? ARB_CNODE(ctd64tree, 0) :
1669                     ARB_CMIN(ctdth64, ctd64tree));
1670                 qmaxstrlen = (ctd64 == NULL) ? 1 : Q_MAXSTRLEN(ctd64->mu, 10);
1671                 is32bit = 0;
1672                 ctd32tree = NULL;
1673                 ctd32 = NULL;
1674                 break;
1675         default:
1676                 return;
1677         }
1678
1679         switch (fmt) {
1680         case SB_STRFMT_FREEFORM:
1681                 fmtstr = "smplcnt=%ju, compcnt=%ju, maxctds=%hu, nctds=%hu";
1682                 break;
1683         case SB_STRFMT_JSON:
1684         default:
1685                 fmtstr =
1686                     "\"smplcnt\":%ju,\"compcnt\":%ju,\"maxctds\":%hu,"
1687                     "\"nctds\":%hu,\"ctds\":[";
1688                 break;
1689         }
1690         sbuf_printf(buf, fmtstr, (uintmax_t)smplcnt, (uintmax_t)compcnt,
1691             maxctds, curctds);
1692
1693         while ((is32bit ? NULL != ctd32 : NULL != ctd64)) {
1694                 char qstr[qmaxstrlen];
1695
1696                 switch (fmt) {
1697                 case SB_STRFMT_FREEFORM:
1698                         fmtstr = "\n\t\t\t\t";
1699                         break;
1700                 case SB_STRFMT_JSON:
1701                 default:
1702                         fmtstr = "{";
1703                         break;
1704                 }
1705                 sbuf_cat(buf, fmtstr);
1706
1707                 if (objdump) {
1708                         switch (fmt) {
1709                         case SB_STRFMT_FREEFORM:
1710                                 fmtstr = "ctd[%hu].";
1711                                 break;
1712                         case SB_STRFMT_JSON:
1713                         default:
1714                                 fmtstr = "\"ctd\":%hu,";
1715                                 break;
1716                         }
1717                         sbuf_printf(buf, fmtstr, is32bit ?
1718                             ARB_SELFIDX(ctd32tree, ctd32) :
1719                             ARB_SELFIDX(ctd64tree, ctd64));
1720                 }
1721
1722                 switch (fmt) {
1723                 case SB_STRFMT_FREEFORM:
1724                         fmtstr = "{mu=";
1725                         break;
1726                 case SB_STRFMT_JSON:
1727                 default:
1728                         fmtstr = "\"mu\":";
1729                         break;
1730                 }
1731                 sbuf_cat(buf, fmtstr);
1732                 Q_TOSTR((is32bit ? ctd32->mu : ctd64->mu), -1, 10, qstr,
1733                     sizeof(qstr));
1734                 sbuf_cat(buf, qstr);
1735
1736                 switch (fmt) {
1737                 case SB_STRFMT_FREEFORM:
1738                         fmtstr = is32bit ? ",cnt=%u}" : ",cnt=%ju}";
1739                         break;
1740                 case SB_STRFMT_JSON:
1741                 default:
1742                         fmtstr = is32bit ? ",\"cnt\":%u}" : ",\"cnt\":%ju}";
1743                         break;
1744                 }
1745                 sbuf_printf(buf, fmtstr,
1746                     is32bit ? ctd32->cnt : (uintmax_t)ctd64->cnt);
1747
1748                 if (is32bit)
1749                         ctd32 = (objdump ? ARB_CNODE(ctd32tree,
1750                             ARB_SELFIDX(ctd32tree, ctd32) + 1) :
1751                             ARB_CNEXT(ctdth32, ctd32tree, ctd32));
1752                 else
1753                         ctd64 = (objdump ? ARB_CNODE(ctd64tree,
1754                             ARB_SELFIDX(ctd64tree, ctd64) + 1) :
1755                             ARB_CNEXT(ctdth64, ctd64tree, ctd64));
1756
1757                 if (fmt == SB_STRFMT_JSON &&
1758                     (is32bit ? NULL != ctd32 : NULL != ctd64))
1759                         sbuf_putc(buf, ',');
1760         }
1761         if (fmt == SB_STRFMT_JSON)
1762                 sbuf_cat(buf, "]");
1763 }
1764
1765 static inline void
1766 stats_voistatdata_hist_tostr(enum vsd_dtype voi_dtype,
1767     const struct voistatdata_hist *hist, enum vsd_dtype hist_dtype,
1768     size_t hist_dsz, enum sb_str_fmt fmt, struct sbuf *buf, int objdump)
1769 {
1770         const struct voistatdata_numeric *bkt_lb, *bkt_ub;
1771         const char *fmtstr;
1772         int is32bit;
1773         uint16_t i, nbkts;
1774
1775         switch (hist_dtype) {
1776         case VSD_DTYPE_CRHIST32:
1777                 nbkts = HIST_VSDSZ2NBKTS(crhist32, hist_dsz);
1778                 is32bit = 1;
1779                 break;
1780         case VSD_DTYPE_DRHIST32:
1781                 nbkts = HIST_VSDSZ2NBKTS(drhist32, hist_dsz);
1782                 is32bit = 1;
1783                 break;
1784         case VSD_DTYPE_DVHIST32:
1785                 nbkts = HIST_VSDSZ2NBKTS(dvhist32, hist_dsz);
1786                 is32bit = 1;
1787                 break;
1788         case VSD_DTYPE_CRHIST64:
1789                 nbkts = HIST_VSDSZ2NBKTS(crhist64, hist_dsz);
1790                 is32bit = 0;
1791                 break;
1792         case VSD_DTYPE_DRHIST64:
1793                 nbkts = HIST_VSDSZ2NBKTS(drhist64, hist_dsz);
1794                 is32bit = 0;
1795                 break;
1796         case VSD_DTYPE_DVHIST64:
1797                 nbkts = HIST_VSDSZ2NBKTS(dvhist64, hist_dsz);
1798                 is32bit = 0;
1799                 break;
1800         default:
1801                 return;
1802         }
1803
1804         switch (fmt) {
1805         case SB_STRFMT_FREEFORM:
1806                 fmtstr = "nbkts=%hu, ";
1807                 break;
1808         case SB_STRFMT_JSON:
1809         default:
1810                 fmtstr = "\"nbkts\":%hu,";
1811                 break;
1812         }
1813         sbuf_printf(buf, fmtstr, nbkts);
1814
1815         switch (fmt) {
1816                 case SB_STRFMT_FREEFORM:
1817                         fmtstr = (is32bit ? "oob=%u" : "oob=%ju");
1818                         break;
1819                 case SB_STRFMT_JSON:
1820                 default:
1821                         fmtstr = (is32bit ? "\"oob\":%u,\"bkts\":[" :
1822                             "\"oob\":%ju,\"bkts\":[");
1823                         break;
1824         }
1825         sbuf_printf(buf, fmtstr, is32bit ? VSD_CONSTHIST_FIELDVAL(hist,
1826             hist_dtype, oob) : (uintmax_t)VSD_CONSTHIST_FIELDVAL(hist,
1827             hist_dtype, oob));
1828
1829         for (i = 0; i < nbkts; i++) {
1830                 switch (hist_dtype) {
1831                 case VSD_DTYPE_CRHIST32:
1832                 case VSD_DTYPE_CRHIST64:
1833                         bkt_lb = VSD_CONSTCRHIST_FIELDPTR(hist, hist_dtype,
1834                             bkts[i].lb);
1835                         if (i < nbkts - 1)
1836                                 bkt_ub = VSD_CONSTCRHIST_FIELDPTR(hist,
1837                                     hist_dtype, bkts[i + 1].lb);
1838                         else
1839                                 bkt_ub = &numeric_limits[LIM_MAX][voi_dtype];
1840                         break;
1841                 case VSD_DTYPE_DRHIST32:
1842                 case VSD_DTYPE_DRHIST64:
1843                         bkt_lb = VSD_CONSTDRHIST_FIELDPTR(hist, hist_dtype,
1844                             bkts[i].lb);
1845                         bkt_ub = VSD_CONSTDRHIST_FIELDPTR(hist, hist_dtype,
1846                             bkts[i].ub);
1847                         break;
1848                 case VSD_DTYPE_DVHIST32:
1849                 case VSD_DTYPE_DVHIST64:
1850                         bkt_lb = bkt_ub = VSD_CONSTDVHIST_FIELDPTR(hist,
1851                             hist_dtype, bkts[i].val);
1852                         break;
1853                 default:
1854                         break;
1855                 }
1856
1857                 switch (fmt) {
1858                 case SB_STRFMT_FREEFORM:
1859                         fmtstr = "\n\t\t\t\t";
1860                         break;
1861                 case SB_STRFMT_JSON:
1862                 default:
1863                         fmtstr = "{";
1864                         break;
1865                 }
1866                 sbuf_cat(buf, fmtstr);
1867
1868                 if (objdump) {
1869                         switch (fmt) {
1870                         case SB_STRFMT_FREEFORM:
1871                                 fmtstr = "bkt[%hu].";
1872                                 break;
1873                         case SB_STRFMT_JSON:
1874                         default:
1875                                 fmtstr = "\"bkt\":%hu,";
1876                                 break;
1877                         }
1878                         sbuf_printf(buf, fmtstr, i);
1879                 }
1880
1881                 switch (fmt) {
1882                 case SB_STRFMT_FREEFORM:
1883                         fmtstr = "{lb=";
1884                         break;
1885                 case SB_STRFMT_JSON:
1886                 default:
1887                         fmtstr = "\"lb\":";
1888                         break;
1889                 }
1890                 sbuf_cat(buf, fmtstr);
1891                 stats_voistatdata_tostr((const struct voistatdata *)bkt_lb,
1892                     voi_dtype, voi_dtype, sizeof(struct voistatdata_numeric),
1893                     fmt, buf, objdump);
1894
1895                 switch (fmt) {
1896                 case SB_STRFMT_FREEFORM:
1897                         fmtstr = ",ub=";
1898                         break;
1899                 case SB_STRFMT_JSON:
1900                 default:
1901                         fmtstr = ",\"ub\":";
1902                         break;
1903                 }
1904                 sbuf_cat(buf, fmtstr);
1905                 stats_voistatdata_tostr((const struct voistatdata *)bkt_ub,
1906                     voi_dtype, voi_dtype, sizeof(struct voistatdata_numeric),
1907                     fmt, buf, objdump);
1908
1909                 switch (fmt) {
1910                 case SB_STRFMT_FREEFORM:
1911                         fmtstr = is32bit ? ",cnt=%u}" : ",cnt=%ju}";
1912                         break;
1913                 case SB_STRFMT_JSON:
1914                 default:
1915                         fmtstr = is32bit ? ",\"cnt\":%u}" : ",\"cnt\":%ju}";
1916                         break;
1917                 }
1918                 sbuf_printf(buf, fmtstr, is32bit ?
1919                     VSD_CONSTHIST_FIELDVAL(hist, hist_dtype, bkts[i].cnt) :
1920                     (uintmax_t)VSD_CONSTHIST_FIELDVAL(hist, hist_dtype,
1921                     bkts[i].cnt));
1922
1923                 if (fmt == SB_STRFMT_JSON && i < nbkts - 1)
1924                         sbuf_putc(buf, ',');
1925         }
1926         if (fmt == SB_STRFMT_JSON)
1927                 sbuf_cat(buf, "]");
1928 }
1929
1930 int
1931 stats_voistatdata_tostr(const struct voistatdata *vsd, enum vsd_dtype voi_dtype,
1932     enum vsd_dtype vsd_dtype, size_t vsd_sz, enum sb_str_fmt fmt,
1933     struct sbuf *buf, int objdump)
1934 {
1935         const char *fmtstr;
1936
1937         if (vsd == NULL || buf == NULL || voi_dtype >= VSD_NUM_DTYPES ||
1938             vsd_dtype >= VSD_NUM_DTYPES || fmt >= SB_STRFMT_NUM_FMTS)
1939                 return (EINVAL);
1940
1941         switch (vsd_dtype) {
1942         case VSD_DTYPE_VOISTATE:
1943                 switch (fmt) {
1944                 case SB_STRFMT_FREEFORM:
1945                         fmtstr = "prev=";
1946                         break;
1947                 case SB_STRFMT_JSON:
1948                 default:
1949                         fmtstr = "\"prev\":";
1950                         break;
1951                 }
1952                 sbuf_cat(buf, fmtstr);
1953                 /*
1954                  * Render prev by passing it as *vsd and voi_dtype as vsd_dtype.
1955                  */
1956                 stats_voistatdata_tostr(
1957                     (const struct voistatdata *)&CONSTVSD(voistate, vsd)->prev,
1958                     voi_dtype, voi_dtype, vsd_sz, fmt, buf, objdump);
1959                 break;
1960         case VSD_DTYPE_INT_S32:
1961                 sbuf_printf(buf, "%d", vsd->int32.s32);
1962                 break;
1963         case VSD_DTYPE_INT_U32:
1964                 sbuf_printf(buf, "%u", vsd->int32.u32);
1965                 break;
1966         case VSD_DTYPE_INT_S64:
1967                 sbuf_printf(buf, "%jd", (intmax_t)vsd->int64.s64);
1968                 break;
1969         case VSD_DTYPE_INT_U64:
1970                 sbuf_printf(buf, "%ju", (uintmax_t)vsd->int64.u64);
1971                 break;
1972         case VSD_DTYPE_INT_SLONG:
1973                 sbuf_printf(buf, "%ld", vsd->intlong.slong);
1974                 break;
1975         case VSD_DTYPE_INT_ULONG:
1976                 sbuf_printf(buf, "%lu", vsd->intlong.ulong);
1977                 break;
1978         case VSD_DTYPE_Q_S32:
1979                 {
1980                 char qstr[Q_MAXSTRLEN(vsd->q32.sq32, 10)];
1981                 Q_TOSTR((s32q_t)vsd->q32.sq32, -1, 10, qstr, sizeof(qstr));
1982                 sbuf_cat(buf, qstr);
1983                 }
1984                 break;
1985         case VSD_DTYPE_Q_U32:
1986                 {
1987                 char qstr[Q_MAXSTRLEN(vsd->q32.uq32, 10)];
1988                 Q_TOSTR((u32q_t)vsd->q32.uq32, -1, 10, qstr, sizeof(qstr));
1989                 sbuf_cat(buf, qstr);
1990                 }
1991                 break;
1992         case VSD_DTYPE_Q_S64:
1993                 {
1994                 char qstr[Q_MAXSTRLEN(vsd->q64.sq64, 10)];
1995                 Q_TOSTR((s64q_t)vsd->q64.sq64, -1, 10, qstr, sizeof(qstr));
1996                 sbuf_cat(buf, qstr);
1997                 }
1998                 break;
1999         case VSD_DTYPE_Q_U64:
2000                 {
2001                 char qstr[Q_MAXSTRLEN(vsd->q64.uq64, 10)];
2002                 Q_TOSTR((u64q_t)vsd->q64.uq64, -1, 10, qstr, sizeof(qstr));
2003                 sbuf_cat(buf, qstr);
2004                 }
2005                 break;
2006         case VSD_DTYPE_CRHIST32:
2007         case VSD_DTYPE_DRHIST32:
2008         case VSD_DTYPE_DVHIST32:
2009         case VSD_DTYPE_CRHIST64:
2010         case VSD_DTYPE_DRHIST64:
2011         case VSD_DTYPE_DVHIST64:
2012                 stats_voistatdata_hist_tostr(voi_dtype, CONSTVSD(hist, vsd),
2013                     vsd_dtype, vsd_sz, fmt, buf, objdump);
2014                 break;
2015         case VSD_DTYPE_TDGSTCLUST32:
2016         case VSD_DTYPE_TDGSTCLUST64:
2017                 stats_voistatdata_tdgst_tostr(voi_dtype,
2018                     CONSTVSD(tdgst, vsd), vsd_dtype, vsd_sz, fmt, buf,
2019                     objdump);
2020                 break;
2021         default:
2022                 break;
2023         }
2024
2025         return (sbuf_error(buf));
2026 }
2027
2028 static void
2029 stats_v1_itercb_tostr_freeform(struct statsblobv1 *sb, struct voi *v,
2030     struct voistat *vs, struct sb_iter_ctx *ctx)
2031 {
2032         struct sb_tostrcb_ctx *sctx;
2033         struct metablob *tpl_mb;
2034         struct sbuf *buf;
2035         void *vsd;
2036         uint8_t dump;
2037
2038         sctx = ctx->usrctx;
2039         buf = sctx->buf;
2040         tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL;
2041         dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0);
2042
2043         if (ctx->flags & SB_IT_FIRST_CB) {
2044                 sbuf_printf(buf, "struct statsblobv1@%p", sb);
2045                 if (dump) {
2046                         sbuf_printf(buf, ", abi=%hhu, endian=%hhu, maxsz=%hu, "
2047                             "cursz=%hu, created=%jd, lastrst=%jd, flags=0x%04hx, "
2048                             "stats_off=%hu, statsdata_off=%hu",
2049                             sb->abi, sb->endian, sb->maxsz, sb->cursz,
2050                             sb->created, sb->lastrst, sb->flags, sb->stats_off,
2051                             sb->statsdata_off);
2052                 }
2053                 sbuf_printf(buf, ", tplhash=%u", sb->tplhash);
2054         }
2055
2056         if (ctx->flags & SB_IT_FIRST_VOISTAT) {
2057                 sbuf_printf(buf, "\n\tvois[%hd]: id=%hd", ctx->vslot, v->id);
2058                 if (v->id < 0)
2059                         return;
2060                 sbuf_printf(buf, ", name=\"%s\"", (tpl_mb == NULL) ? "" :
2061                     tpl_mb->voi_meta[v->id].name);
2062                 if (dump)
2063                     sbuf_printf(buf, ", flags=0x%04hx, dtype=%s, "
2064                     "voistatmaxid=%hhd, stats_off=%hu", v->flags,
2065                     vsd_dtype2name[v->dtype], v->voistatmaxid, v->stats_off);
2066         }
2067
2068         if (!dump && vs->stype <= 0)
2069                 return;
2070
2071         sbuf_printf(buf, "\n\t\tvois[%hd]stat[%hhd]: stype=", v->id, ctx->vsslot);
2072         if (vs->stype < 0) {
2073                 sbuf_printf(buf, "%hhd", vs->stype);
2074                 return;
2075         } else
2076                 sbuf_printf(buf, "%s, errs=%hu", vs_stype2name[vs->stype],
2077                     vs->errs);
2078         vsd = BLOB_OFFSET(sb, vs->data_off);
2079         if (dump)
2080                 sbuf_printf(buf, ", flags=0x%04x, dtype=%s, dsz=%hu, "
2081                     "data_off=%hu", vs->flags, vsd_dtype2name[vs->dtype],
2082                     vs->dsz, vs->data_off);
2083
2084         sbuf_printf(buf, "\n\t\t\tvoistatdata: ");
2085         stats_voistatdata_tostr(vsd, v->dtype, vs->dtype, vs->dsz,
2086             sctx->fmt, buf, dump);
2087 }
2088
2089 static void
2090 stats_v1_itercb_tostr_json(struct statsblobv1 *sb, struct voi *v, struct voistat *vs,
2091     struct sb_iter_ctx *ctx)
2092 {
2093         struct sb_tostrcb_ctx *sctx;
2094         struct metablob *tpl_mb;
2095         struct sbuf *buf;
2096         const char *fmtstr;
2097         void *vsd;
2098         uint8_t dump;
2099
2100         sctx = ctx->usrctx;
2101         buf = sctx->buf;
2102         tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL;
2103         dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0);
2104
2105         if (ctx->flags & SB_IT_FIRST_CB) {
2106                 sbuf_putc(buf, '{');
2107                 if (dump) {
2108                         sbuf_printf(buf, "\"abi\":%hhu,\"endian\":%hhu,"
2109                             "\"maxsz\":%hu,\"cursz\":%hu,\"created\":%jd,"
2110                             "\"lastrst\":%jd,\"flags\":%hu,\"stats_off\":%hu,"
2111                             "\"statsdata_off\":%hu,", sb->abi,
2112                             sb->endian, sb->maxsz, sb->cursz, sb->created,
2113                             sb->lastrst, sb->flags, sb->stats_off,
2114                             sb->statsdata_off);
2115                 }
2116
2117                 if (tpl_mb == NULL)
2118                         fmtstr = "\"tplname\":%s,\"tplhash\":%u,\"vois\":{";
2119                 else
2120                         fmtstr = "\"tplname\":\"%s\",\"tplhash\":%u,\"vois\":{";
2121
2122                 sbuf_printf(buf, fmtstr, tpl_mb ? tpl_mb->tplname : "null",
2123                     sb->tplhash);
2124         }
2125
2126         if (ctx->flags & SB_IT_FIRST_VOISTAT) {
2127                 if (dump) {
2128                         sbuf_printf(buf, "\"[%d]\":{\"id\":%d", ctx->vslot,
2129                             v->id);
2130                         if (v->id < 0) {
2131                                 sbuf_printf(buf, "},");
2132                                 return;
2133                         }
2134
2135                         if (tpl_mb == NULL)
2136                                 fmtstr = ",\"name\":%s,\"flags\":%hu,"
2137                                     "\"dtype\":\"%s\",\"voistatmaxid\":%hhd,"
2138                                     "\"stats_off\":%hu,";
2139                         else
2140                                 fmtstr = ",\"name\":\"%s\",\"flags\":%hu,"
2141                                     "\"dtype\":\"%s\",\"voistatmaxid\":%hhd,"
2142                                     "\"stats_off\":%hu,";
2143
2144                         sbuf_printf(buf, fmtstr, tpl_mb ?
2145                             tpl_mb->voi_meta[v->id].name : "null", v->flags,
2146                             vsd_dtype2name[v->dtype], v->voistatmaxid,
2147                             v->stats_off);
2148                 } else {
2149                         if (tpl_mb == NULL) {
2150                                 sbuf_printf(buf, "\"[%hd]\":{", v->id);
2151                         } else {
2152                                 sbuf_printf(buf, "\"%s\":{",
2153                                     tpl_mb->voi_meta[v->id].name);
2154                         }
2155                 }
2156                 sbuf_cat(buf, "\"stats\":{");
2157         }
2158
2159         vsd = BLOB_OFFSET(sb, vs->data_off);
2160         if (dump) {
2161                 sbuf_printf(buf, "\"[%hhd]\":", ctx->vsslot);
2162                 if (vs->stype < 0) {
2163                         sbuf_printf(buf, "{\"stype\":-1},");
2164                         return;
2165                 }
2166                 sbuf_printf(buf, "{\"stype\":\"%s\",\"errs\":%hu,\"flags\":%hu,"
2167                     "\"dtype\":\"%s\",\"data_off\":%hu,\"voistatdata\":{",
2168                     vs_stype2name[vs->stype], vs->errs, vs->flags,
2169                     vsd_dtype2name[vs->dtype], vs->data_off);
2170         } else if (vs->stype > 0) {
2171                 if (tpl_mb == NULL)
2172                         sbuf_printf(buf, "\"[%hhd]\":", vs->stype);
2173                 else
2174                         sbuf_printf(buf, "\"%s\":", vs_stype2name[vs->stype]);
2175         } else
2176                 return;
2177
2178         if ((vs->flags & VS_VSDVALID) || dump) {
2179                 if (!dump)
2180                         sbuf_printf(buf, "{\"errs\":%hu,", vs->errs);
2181                 /* Simple non-compound VSD types need a key. */
2182                 if (!vsd_compoundtype[vs->dtype])
2183                         sbuf_cat(buf, "\"val\":");
2184                 stats_voistatdata_tostr(vsd, v->dtype, vs->dtype, vs->dsz,
2185                     sctx->fmt, buf, dump);
2186                 sbuf_cat(buf, dump ? "}}" : "}");
2187         } else
2188                 sbuf_cat(buf, dump ? "null}" : "null");
2189
2190         if (ctx->flags & SB_IT_LAST_VOISTAT)
2191                 sbuf_cat(buf, "}}");
2192
2193         if (ctx->flags & SB_IT_LAST_CB)
2194                 sbuf_cat(buf, "}}");
2195         else
2196                 sbuf_putc(buf, ',');
2197 }
2198
2199 static int
2200 stats_v1_itercb_tostr(struct statsblobv1 *sb, struct voi *v, struct voistat *vs,
2201     struct sb_iter_ctx *ctx)
2202 {
2203         struct sb_tostrcb_ctx *sctx;
2204
2205         sctx = ctx->usrctx;
2206
2207         switch (sctx->fmt) {
2208         case SB_STRFMT_FREEFORM:
2209                 stats_v1_itercb_tostr_freeform(sb, v, vs, ctx);
2210                 break;
2211         case SB_STRFMT_JSON:
2212                 stats_v1_itercb_tostr_json(sb, v, vs, ctx);
2213                 break;
2214         default:
2215                 break;
2216         }
2217
2218         return (sbuf_error(sctx->buf));
2219 }
2220
2221 int
2222 stats_v1_blob_tostr(struct statsblobv1 *sb, struct sbuf *buf,
2223     enum sb_str_fmt fmt, uint32_t flags)
2224 {
2225         struct sb_tostrcb_ctx sctx;
2226         uint32_t iflags;
2227
2228         if (sb == NULL || sb->abi != STATS_ABI_V1 || buf == NULL ||
2229             fmt >= SB_STRFMT_NUM_FMTS)
2230                 return (EINVAL);
2231
2232         sctx.buf = buf;
2233         sctx.fmt = fmt;
2234         sctx.flags = flags;
2235
2236         if (flags & SB_TOSTR_META) {
2237                 if (stats_tpl_fetch(stats_tpl_fetch_allocid(NULL, sb->tplhash),
2238                     &sctx.tpl))
2239                         return (EINVAL);
2240         } else
2241                 sctx.tpl = NULL;
2242
2243         iflags = 0;
2244         if (flags & SB_TOSTR_OBJDUMP)
2245                 iflags |= (SB_IT_NULLVOI | SB_IT_NULLVOISTAT);
2246         stats_v1_blob_iter(sb, stats_v1_itercb_tostr, &sctx, iflags);
2247
2248         return (sbuf_error(buf));
2249 }
2250
2251 static int
2252 stats_v1_itercb_visit(struct statsblobv1 *sb, struct voi *v,
2253     struct voistat *vs, struct sb_iter_ctx *ctx)
2254 {
2255         struct sb_visitcb_ctx *vctx;
2256         struct sb_visit sbv;
2257
2258         vctx = ctx->usrctx;
2259
2260         sbv.tplhash = sb->tplhash;
2261         sbv.voi_id = v->id;
2262         sbv.voi_dtype = v->dtype;
2263         sbv.vs_stype = vs->stype;
2264         sbv.vs_dtype = vs->dtype;
2265         sbv.vs_dsz = vs->dsz;
2266         sbv.vs_data = BLOB_OFFSET(sb, vs->data_off);
2267         sbv.vs_errs = vs->errs;
2268         sbv.flags = ctx->flags & (SB_IT_FIRST_CB | SB_IT_LAST_CB |
2269             SB_IT_FIRST_VOI | SB_IT_LAST_VOI | SB_IT_FIRST_VOISTAT |
2270             SB_IT_LAST_VOISTAT);
2271
2272         return (vctx->cb(&sbv, vctx->usrctx));
2273 }
2274
2275 int
2276 stats_v1_blob_visit(struct statsblobv1 *sb, stats_blob_visitcb_t func,
2277     void *usrctx)
2278 {
2279         struct sb_visitcb_ctx vctx;
2280
2281         if (sb == NULL || sb->abi != STATS_ABI_V1 || func == NULL)
2282                 return (EINVAL);
2283
2284         vctx.cb = func;
2285         vctx.usrctx = usrctx;
2286
2287         stats_v1_blob_iter(sb, stats_v1_itercb_visit, &vctx, 0);
2288
2289         return (0);
2290 }
2291
2292 static int
2293 stats_v1_icb_reset_voistat(struct statsblobv1 *sb, struct voi *v __unused,
2294     struct voistat *vs, struct sb_iter_ctx *ctx __unused)
2295 {
2296         void *vsd;
2297
2298         if (vs->stype == VS_STYPE_VOISTATE)
2299                 return (0);
2300
2301         vsd = BLOB_OFFSET(sb, vs->data_off);
2302
2303         /* Perform the stat type's default reset action. */
2304         switch (vs->stype) {
2305         case VS_STYPE_SUM:
2306                 switch (vs->dtype) {
2307                 case VSD_DTYPE_Q_S32:
2308                         Q_SIFVAL(VSD(q32, vsd)->sq32, 0);
2309                         break;
2310                 case VSD_DTYPE_Q_U32:
2311                         Q_SIFVAL(VSD(q32, vsd)->uq32, 0);
2312                         break;
2313                 case VSD_DTYPE_Q_S64:
2314                         Q_SIFVAL(VSD(q64, vsd)->sq64, 0);
2315                         break;
2316                 case VSD_DTYPE_Q_U64:
2317                         Q_SIFVAL(VSD(q64, vsd)->uq64, 0);
2318                         break;
2319                 default:
2320                         bzero(vsd, vs->dsz);
2321                         break;
2322                 }
2323                 break;
2324         case VS_STYPE_MAX:
2325                 switch (vs->dtype) {
2326                 case VSD_DTYPE_Q_S32:
2327                         Q_SIFVAL(VSD(q32, vsd)->sq32,
2328                             Q_IFMINVAL(VSD(q32, vsd)->sq32));
2329                         break;
2330                 case VSD_DTYPE_Q_U32:
2331                         Q_SIFVAL(VSD(q32, vsd)->uq32,
2332                             Q_IFMINVAL(VSD(q32, vsd)->uq32));
2333                         break;
2334                 case VSD_DTYPE_Q_S64:
2335                         Q_SIFVAL(VSD(q64, vsd)->sq64,
2336                             Q_IFMINVAL(VSD(q64, vsd)->sq64));
2337                         break;
2338                 case VSD_DTYPE_Q_U64:
2339                         Q_SIFVAL(VSD(q64, vsd)->uq64,
2340                             Q_IFMINVAL(VSD(q64, vsd)->uq64));
2341                         break;
2342                 default:
2343                         memcpy(vsd, &numeric_limits[LIM_MIN][vs->dtype],
2344                             vs->dsz);
2345                         break;
2346                 }
2347                 break;
2348         case VS_STYPE_MIN:
2349                 switch (vs->dtype) {
2350                 case VSD_DTYPE_Q_S32:
2351                         Q_SIFVAL(VSD(q32, vsd)->sq32,
2352                             Q_IFMAXVAL(VSD(q32, vsd)->sq32));
2353                         break;
2354                 case VSD_DTYPE_Q_U32:
2355                         Q_SIFVAL(VSD(q32, vsd)->uq32,
2356                             Q_IFMAXVAL(VSD(q32, vsd)->uq32));
2357                         break;
2358                 case VSD_DTYPE_Q_S64:
2359                         Q_SIFVAL(VSD(q64, vsd)->sq64,
2360                             Q_IFMAXVAL(VSD(q64, vsd)->sq64));
2361                         break;
2362                 case VSD_DTYPE_Q_U64:
2363                         Q_SIFVAL(VSD(q64, vsd)->uq64,
2364                             Q_IFMAXVAL(VSD(q64, vsd)->uq64));
2365                         break;
2366                 default:
2367                         memcpy(vsd, &numeric_limits[LIM_MAX][vs->dtype],
2368                             vs->dsz);
2369                         break;
2370                 }
2371                 break;
2372         case VS_STYPE_HIST:
2373                 {
2374                 /* Reset bucket counts. */
2375                 struct voistatdata_hist *hist;
2376                 int i, is32bit;
2377                 uint16_t nbkts;
2378
2379                 hist = VSD(hist, vsd);
2380                 switch (vs->dtype) {
2381                 case VSD_DTYPE_CRHIST32:
2382                         nbkts = HIST_VSDSZ2NBKTS(crhist32, vs->dsz);
2383                         is32bit = 1;
2384                         break;
2385                 case VSD_DTYPE_DRHIST32:
2386                         nbkts = HIST_VSDSZ2NBKTS(drhist32, vs->dsz);
2387                         is32bit = 1;
2388                         break;
2389                 case VSD_DTYPE_DVHIST32:
2390                         nbkts = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz);
2391                         is32bit = 1;
2392                         break;
2393                 case VSD_DTYPE_CRHIST64:
2394                         nbkts = HIST_VSDSZ2NBKTS(crhist64, vs->dsz);
2395                         is32bit = 0;
2396                         break;
2397                 case VSD_DTYPE_DRHIST64:
2398                         nbkts = HIST_VSDSZ2NBKTS(drhist64, vs->dsz);
2399                         is32bit = 0;
2400                         break;
2401                 case VSD_DTYPE_DVHIST64:
2402                         nbkts = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz);
2403                         is32bit = 0;
2404                         break;
2405                 default:
2406                         return (0);
2407                 }
2408
2409                 bzero(VSD_HIST_FIELDPTR(hist, vs->dtype, oob),
2410                     is32bit ? sizeof(uint32_t) : sizeof(uint64_t));
2411                 for (i = nbkts - 1; i >= 0; i--) {
2412                         bzero(VSD_HIST_FIELDPTR(hist, vs->dtype,
2413                             bkts[i].cnt), is32bit ? sizeof(uint32_t) :
2414                             sizeof(uint64_t));
2415                 }
2416                 break;
2417                 }
2418         case VS_STYPE_TDGST:
2419                 {
2420                 /* Reset sample count centroids array/tree. */
2421                 struct voistatdata_tdgst *tdgst;
2422                 struct ctdth32 *ctd32tree;
2423                 struct ctdth64 *ctd64tree;
2424                 struct voistatdata_tdgstctd32 *ctd32;
2425                 struct voistatdata_tdgstctd64 *ctd64;
2426
2427                 tdgst = VSD(tdgst, vsd);
2428                 switch (vs->dtype) {
2429                 case VSD_DTYPE_TDGSTCLUST32:
2430                         VSD(tdgstclust32, tdgst)->smplcnt = 0;
2431                         VSD(tdgstclust32, tdgst)->compcnt = 0;
2432                         ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
2433                         ARB_INIT(ctd32, ctdlnk, ctd32tree,
2434                             ARB_MAXNODES(ctd32tree)) {
2435                                 ctd32->cnt = 0;
2436                                 Q_SIFVAL(ctd32->mu, 0);
2437                         }
2438 #ifdef DIAGNOSTIC
2439                         RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree);
2440 #endif
2441                 break;
2442                 case VSD_DTYPE_TDGSTCLUST64:
2443                         VSD(tdgstclust64, tdgst)->smplcnt = 0;
2444                         VSD(tdgstclust64, tdgst)->compcnt = 0;
2445                         ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
2446                         ARB_INIT(ctd64, ctdlnk, ctd64tree,
2447                             ARB_MAXNODES(ctd64tree)) {
2448                                 ctd64->cnt = 0;
2449                                 Q_SIFVAL(ctd64->mu, 0);
2450                         }
2451 #ifdef DIAGNOSTIC
2452                         RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree);
2453 #endif
2454                 break;
2455                 default:
2456                         return (0);
2457                 }
2458                 break;
2459                 }
2460         default:
2461                 KASSERT(0, ("Unknown VOI stat type %d", vs->stype));
2462                 break;
2463         }
2464
2465         vs->errs = 0;
2466         vs->flags &= ~VS_VSDVALID;
2467
2468         return (0);
2469 }
2470
2471 int
2472 stats_v1_blob_snapshot(struct statsblobv1 **dst, size_t dstmaxsz,
2473     struct statsblobv1 *src, uint32_t flags)
2474 {
2475         int error;
2476
2477         if (src != NULL && src->abi == STATS_ABI_V1) {
2478                 error = stats_v1_blob_clone(dst, dstmaxsz, src, flags);
2479                 if (!error) {
2480                         if (flags & SB_CLONE_RSTSRC) {
2481                                 stats_v1_blob_iter(src,
2482                                     stats_v1_icb_reset_voistat, NULL, 0);
2483                                 src->lastrst = stats_sbinuptime();
2484                         }
2485                         stats_v1_blob_finalise(*dst);
2486                 }
2487         } else
2488                 error = EINVAL;
2489
2490         return (error);
2491 }
2492
2493 static inline int
2494 stats_v1_voi_update_max(enum vsd_dtype voi_dtype __unused,
2495     struct voistatdata *voival, struct voistat *vs, void *vsd)
2496 {
2497         int error;
2498
2499         KASSERT(vs->dtype < VSD_NUM_DTYPES,
2500             ("Unknown VSD dtype %d", vs->dtype));
2501
2502         error = 0;
2503
2504         switch (vs->dtype) {
2505         case VSD_DTYPE_INT_S32:
2506                 if (VSD(int32, vsd)->s32 < voival->int32.s32) {
2507                         VSD(int32, vsd)->s32 = voival->int32.s32;
2508                         vs->flags |= VS_VSDVALID;
2509                 }
2510                 break;
2511         case VSD_DTYPE_INT_U32:
2512                 if (VSD(int32, vsd)->u32 < voival->int32.u32) {
2513                         VSD(int32, vsd)->u32 = voival->int32.u32;
2514                         vs->flags |= VS_VSDVALID;
2515                 }
2516                 break;
2517         case VSD_DTYPE_INT_S64:
2518                 if (VSD(int64, vsd)->s64 < voival->int64.s64) {
2519                         VSD(int64, vsd)->s64 = voival->int64.s64;
2520                         vs->flags |= VS_VSDVALID;
2521                 }
2522                 break;
2523         case VSD_DTYPE_INT_U64:
2524                 if (VSD(int64, vsd)->u64 < voival->int64.u64) {
2525                         VSD(int64, vsd)->u64 = voival->int64.u64;
2526                         vs->flags |= VS_VSDVALID;
2527                 }
2528                 break;
2529         case VSD_DTYPE_INT_SLONG:
2530                 if (VSD(intlong, vsd)->slong < voival->intlong.slong) {
2531                         VSD(intlong, vsd)->slong = voival->intlong.slong;
2532                         vs->flags |= VS_VSDVALID;
2533                 }
2534                 break;
2535         case VSD_DTYPE_INT_ULONG:
2536                 if (VSD(intlong, vsd)->ulong < voival->intlong.ulong) {
2537                         VSD(intlong, vsd)->ulong = voival->intlong.ulong;
2538                         vs->flags |= VS_VSDVALID;
2539                 }
2540                 break;
2541         case VSD_DTYPE_Q_S32:
2542                 if (Q_QLTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) &&
2543                     (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32,
2544                     voival->q32.sq32)))) {
2545                         vs->flags |= VS_VSDVALID;
2546                 }
2547                 break;
2548         case VSD_DTYPE_Q_U32:
2549                 if (Q_QLTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) &&
2550                     (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32,
2551                     voival->q32.uq32)))) {
2552                         vs->flags |= VS_VSDVALID;
2553                 }
2554                 break;
2555         case VSD_DTYPE_Q_S64:
2556                 if (Q_QLTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) &&
2557                     (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64,
2558                     voival->q64.sq64)))) {
2559                         vs->flags |= VS_VSDVALID;
2560                 }
2561                 break;
2562         case VSD_DTYPE_Q_U64:
2563                 if (Q_QLTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) &&
2564                     (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64,
2565                     voival->q64.uq64)))) {
2566                         vs->flags |= VS_VSDVALID;
2567                 }
2568                 break;
2569         default:
2570                 error = EINVAL;
2571                 break;
2572         }
2573
2574         return (error);
2575 }
2576
2577 static inline int
2578 stats_v1_voi_update_min(enum vsd_dtype voi_dtype __unused,
2579     struct voistatdata *voival, struct voistat *vs, void *vsd)
2580 {
2581         int error;
2582
2583         KASSERT(vs->dtype < VSD_NUM_DTYPES,
2584             ("Unknown VSD dtype %d", vs->dtype));
2585
2586         error = 0;
2587
2588         switch (vs->dtype) {
2589         case VSD_DTYPE_INT_S32:
2590                 if (VSD(int32, vsd)->s32 > voival->int32.s32) {
2591                         VSD(int32, vsd)->s32 = voival->int32.s32;
2592                         vs->flags |= VS_VSDVALID;
2593                 }
2594                 break;
2595         case VSD_DTYPE_INT_U32:
2596                 if (VSD(int32, vsd)->u32 > voival->int32.u32) {
2597                         VSD(int32, vsd)->u32 = voival->int32.u32;
2598                         vs->flags |= VS_VSDVALID;
2599                 }
2600                 break;
2601         case VSD_DTYPE_INT_S64:
2602                 if (VSD(int64, vsd)->s64 > voival->int64.s64) {
2603                         VSD(int64, vsd)->s64 = voival->int64.s64;
2604                         vs->flags |= VS_VSDVALID;
2605                 }
2606                 break;
2607         case VSD_DTYPE_INT_U64:
2608                 if (VSD(int64, vsd)->u64 > voival->int64.u64) {
2609                         VSD(int64, vsd)->u64 = voival->int64.u64;
2610                         vs->flags |= VS_VSDVALID;
2611                 }
2612                 break;
2613         case VSD_DTYPE_INT_SLONG:
2614                 if (VSD(intlong, vsd)->slong > voival->intlong.slong) {
2615                         VSD(intlong, vsd)->slong = voival->intlong.slong;
2616                         vs->flags |= VS_VSDVALID;
2617                 }
2618                 break;
2619         case VSD_DTYPE_INT_ULONG:
2620                 if (VSD(intlong, vsd)->ulong > voival->intlong.ulong) {
2621                         VSD(intlong, vsd)->ulong = voival->intlong.ulong;
2622                         vs->flags |= VS_VSDVALID;
2623                 }
2624                 break;
2625         case VSD_DTYPE_Q_S32:
2626                 if (Q_QGTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) &&
2627                     (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32,
2628                     voival->q32.sq32)))) {
2629                         vs->flags |= VS_VSDVALID;
2630                 }
2631                 break;
2632         case VSD_DTYPE_Q_U32:
2633                 if (Q_QGTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) &&
2634                     (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32,
2635                     voival->q32.uq32)))) {
2636                         vs->flags |= VS_VSDVALID;
2637                 }
2638                 break;
2639         case VSD_DTYPE_Q_S64:
2640                 if (Q_QGTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) &&
2641                     (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64,
2642                     voival->q64.sq64)))) {
2643                         vs->flags |= VS_VSDVALID;
2644                 }
2645                 break;
2646         case VSD_DTYPE_Q_U64:
2647                 if (Q_QGTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) &&
2648                     (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64,
2649                     voival->q64.uq64)))) {
2650                         vs->flags |= VS_VSDVALID;
2651                 }
2652                 break;
2653         default:
2654                 error = EINVAL;
2655                 break;
2656         }
2657
2658         return (error);
2659 }
2660
2661 static inline int
2662 stats_v1_voi_update_sum(enum vsd_dtype voi_dtype __unused,
2663     struct voistatdata *voival, struct voistat *vs, void *vsd)
2664 {
2665         int error;
2666
2667         KASSERT(vs->dtype < VSD_NUM_DTYPES,
2668             ("Unknown VSD dtype %d", vs->dtype));
2669
2670         error = 0;
2671
2672         switch (vs->dtype) {
2673         case VSD_DTYPE_INT_S32:
2674                 VSD(int32, vsd)->s32 += voival->int32.s32;
2675                 break;
2676         case VSD_DTYPE_INT_U32:
2677                 VSD(int32, vsd)->u32 += voival->int32.u32;
2678                 break;
2679         case VSD_DTYPE_INT_S64:
2680                 VSD(int64, vsd)->s64 += voival->int64.s64;
2681                 break;
2682         case VSD_DTYPE_INT_U64:
2683                 VSD(int64, vsd)->u64 += voival->int64.u64;
2684                 break;
2685         case VSD_DTYPE_INT_SLONG:
2686                 VSD(intlong, vsd)->slong += voival->intlong.slong;
2687                 break;
2688         case VSD_DTYPE_INT_ULONG:
2689                 VSD(intlong, vsd)->ulong += voival->intlong.ulong;
2690                 break;
2691         case VSD_DTYPE_Q_S32:
2692                 error = Q_QADDQ(&VSD(q32, vsd)->sq32, voival->q32.sq32);
2693                 break;
2694         case VSD_DTYPE_Q_U32:
2695                 error = Q_QADDQ(&VSD(q32, vsd)->uq32, voival->q32.uq32);
2696                 break;
2697         case VSD_DTYPE_Q_S64:
2698                 error = Q_QADDQ(&VSD(q64, vsd)->sq64, voival->q64.sq64);
2699                 break;
2700         case VSD_DTYPE_Q_U64:
2701                 error = Q_QADDQ(&VSD(q64, vsd)->uq64, voival->q64.uq64);
2702                 break;
2703         default:
2704                 error = EINVAL;
2705                 break;
2706         }
2707
2708         if (!error)
2709                 vs->flags |= VS_VSDVALID;
2710
2711         return (error);
2712 }
2713
2714 static inline int
2715 stats_v1_voi_update_hist(enum vsd_dtype voi_dtype, struct voistatdata *voival,
2716     struct voistat *vs, struct voistatdata_hist *hist)
2717 {
2718         struct voistatdata_numeric *bkt_lb, *bkt_ub;
2719         uint64_t *oob64, *cnt64;
2720         uint32_t *oob32, *cnt32;
2721         int error, i, found, is32bit, has_ub, eq_only;
2722
2723         error = 0;
2724
2725         switch (vs->dtype) {
2726         case VSD_DTYPE_CRHIST32:
2727                 i = HIST_VSDSZ2NBKTS(crhist32, vs->dsz);
2728                 is32bit = 1;
2729                 has_ub = eq_only = 0;
2730                 oob32 = &VSD(crhist32, hist)->oob;
2731                 break;
2732         case VSD_DTYPE_DRHIST32:
2733                 i = HIST_VSDSZ2NBKTS(drhist32, vs->dsz);
2734                 is32bit = has_ub = 1;
2735                 eq_only = 0;
2736                 oob32 = &VSD(drhist32, hist)->oob;
2737                 break;
2738         case VSD_DTYPE_DVHIST32:
2739                 i = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz);
2740                 is32bit = eq_only = 1;
2741                 has_ub = 0;
2742                 oob32 = &VSD(dvhist32, hist)->oob;
2743                 break;
2744         case VSD_DTYPE_CRHIST64:
2745                 i = HIST_VSDSZ2NBKTS(crhist64, vs->dsz);
2746                 is32bit = has_ub = eq_only = 0;
2747                 oob64 = &VSD(crhist64, hist)->oob;
2748                 break;
2749         case VSD_DTYPE_DRHIST64:
2750                 i = HIST_VSDSZ2NBKTS(drhist64, vs->dsz);
2751                 is32bit = eq_only = 0;
2752                 has_ub = 1;
2753                 oob64 = &VSD(drhist64, hist)->oob;
2754                 break;
2755         case VSD_DTYPE_DVHIST64:
2756                 i = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz);
2757                 is32bit = has_ub = 0;
2758                 eq_only = 1;
2759                 oob64 = &VSD(dvhist64, hist)->oob;
2760                 break;
2761         default:
2762                 return (EINVAL);
2763         }
2764         i--; /* Adjust for 0-based array index. */
2765
2766         /* XXXLAS: Should probably use a better bucket search algorithm. ARB? */
2767         for (found = 0; i >= 0 && !found; i--) {
2768                 switch (vs->dtype) {
2769                 case VSD_DTYPE_CRHIST32:
2770                         bkt_lb = &VSD(crhist32, hist)->bkts[i].lb;
2771                         cnt32 = &VSD(crhist32, hist)->bkts[i].cnt;
2772                         break;
2773                 case VSD_DTYPE_DRHIST32:
2774                         bkt_lb = &VSD(drhist32, hist)->bkts[i].lb;
2775                         bkt_ub = &VSD(drhist32, hist)->bkts[i].ub;
2776                         cnt32 = &VSD(drhist32, hist)->bkts[i].cnt;
2777                         break;
2778                 case VSD_DTYPE_DVHIST32:
2779                         bkt_lb = &VSD(dvhist32, hist)->bkts[i].val;
2780                         cnt32 = &VSD(dvhist32, hist)->bkts[i].cnt;
2781                         break;
2782                 case VSD_DTYPE_CRHIST64:
2783                         bkt_lb = &VSD(crhist64, hist)->bkts[i].lb;
2784                         cnt64 = &VSD(crhist64, hist)->bkts[i].cnt;
2785                         break;
2786                 case VSD_DTYPE_DRHIST64:
2787                         bkt_lb = &VSD(drhist64, hist)->bkts[i].lb;
2788                         bkt_ub = &VSD(drhist64, hist)->bkts[i].ub;
2789                         cnt64 = &VSD(drhist64, hist)->bkts[i].cnt;
2790                         break;
2791                 case VSD_DTYPE_DVHIST64:
2792                         bkt_lb = &VSD(dvhist64, hist)->bkts[i].val;
2793                         cnt64 = &VSD(dvhist64, hist)->bkts[i].cnt;
2794                         break;
2795                 default:
2796                         return (EINVAL);
2797                 }
2798
2799                 switch (voi_dtype) {
2800                 case VSD_DTYPE_INT_S32:
2801                         if (voival->int32.s32 >= bkt_lb->int32.s32) {
2802                                 if ((eq_only && voival->int32.s32 ==
2803                                     bkt_lb->int32.s32) ||
2804                                     (!eq_only && (!has_ub ||
2805                                     voival->int32.s32 < bkt_ub->int32.s32)))
2806                                         found = 1;
2807                         }
2808                         break;
2809                 case VSD_DTYPE_INT_U32:
2810                         if (voival->int32.u32 >= bkt_lb->int32.u32) {
2811                                 if ((eq_only && voival->int32.u32 ==
2812                                     bkt_lb->int32.u32) ||
2813                                     (!eq_only && (!has_ub ||
2814                                     voival->int32.u32 < bkt_ub->int32.u32)))
2815                                         found = 1;
2816                         }
2817                         break;
2818                 case VSD_DTYPE_INT_S64:
2819                         if (voival->int64.s64 >= bkt_lb->int64.s64)
2820                                 if ((eq_only && voival->int64.s64 ==
2821                                     bkt_lb->int64.s64) ||
2822                                     (!eq_only && (!has_ub ||
2823                                     voival->int64.s64 < bkt_ub->int64.s64)))
2824                                         found = 1;
2825                         break;
2826                 case VSD_DTYPE_INT_U64:
2827                         if (voival->int64.u64 >= bkt_lb->int64.u64)
2828                                 if ((eq_only && voival->int64.u64 ==
2829                                     bkt_lb->int64.u64) ||
2830                                     (!eq_only && (!has_ub ||
2831                                     voival->int64.u64 < bkt_ub->int64.u64)))
2832                                         found = 1;
2833                         break;
2834                 case VSD_DTYPE_INT_SLONG:
2835                         if (voival->intlong.slong >= bkt_lb->intlong.slong)
2836                                 if ((eq_only && voival->intlong.slong ==
2837                                     bkt_lb->intlong.slong) ||
2838                                     (!eq_only && (!has_ub ||
2839                                     voival->intlong.slong <
2840                                     bkt_ub->intlong.slong)))
2841                                         found = 1;
2842                         break;
2843                 case VSD_DTYPE_INT_ULONG:
2844                         if (voival->intlong.ulong >= bkt_lb->intlong.ulong)
2845                                 if ((eq_only && voival->intlong.ulong ==
2846                                     bkt_lb->intlong.ulong) ||
2847                                     (!eq_only && (!has_ub ||
2848                                     voival->intlong.ulong <
2849                                     bkt_ub->intlong.ulong)))
2850                                         found = 1;
2851                         break;
2852                 case VSD_DTYPE_Q_S32:
2853                         if (Q_QGEQ(voival->q32.sq32, bkt_lb->q32.sq32))
2854                                 if ((eq_only && Q_QEQ(voival->q32.sq32,
2855                                     bkt_lb->q32.sq32)) ||
2856                                     (!eq_only && (!has_ub ||
2857                                     Q_QLTQ(voival->q32.sq32,
2858                                     bkt_ub->q32.sq32))))
2859                                         found = 1;
2860                         break;
2861                 case VSD_DTYPE_Q_U32:
2862                         if (Q_QGEQ(voival->q32.uq32, bkt_lb->q32.uq32))
2863                                 if ((eq_only && Q_QEQ(voival->q32.uq32,
2864                                     bkt_lb->q32.uq32)) ||
2865                                     (!eq_only && (!has_ub ||
2866                                     Q_QLTQ(voival->q32.uq32,
2867                                     bkt_ub->q32.uq32))))
2868                                         found = 1;
2869                         break;
2870                 case VSD_DTYPE_Q_S64:
2871                         if (Q_QGEQ(voival->q64.sq64, bkt_lb->q64.sq64))
2872                                 if ((eq_only && Q_QEQ(voival->q64.sq64,
2873                                     bkt_lb->q64.sq64)) ||
2874                                     (!eq_only && (!has_ub ||
2875                                     Q_QLTQ(voival->q64.sq64,
2876                                     bkt_ub->q64.sq64))))
2877                                         found = 1;
2878                         break;
2879                 case VSD_DTYPE_Q_U64:
2880                         if (Q_QGEQ(voival->q64.uq64, bkt_lb->q64.uq64))
2881                                 if ((eq_only && Q_QEQ(voival->q64.uq64,
2882                                     bkt_lb->q64.uq64)) ||
2883                                     (!eq_only && (!has_ub ||
2884                                     Q_QLTQ(voival->q64.uq64,
2885                                     bkt_ub->q64.uq64))))
2886                                         found = 1;
2887                         break;
2888                 default:
2889                         break;
2890                 }
2891         }
2892
2893         if (found) {
2894                 if (is32bit)
2895                         *cnt32 += 1;
2896                 else
2897                         *cnt64 += 1;
2898         } else {
2899                 if (is32bit)
2900                         *oob32 += 1;
2901                 else
2902                         *oob64 += 1;
2903         }
2904
2905         vs->flags |= VS_VSDVALID;
2906         return (error);
2907 }
2908
2909 static inline int
2910 stats_v1_vsd_tdgst_compress(enum vsd_dtype vs_dtype,
2911     struct voistatdata_tdgst *tdgst, int attempt)
2912 {
2913         struct ctdth32 *ctd32tree;
2914         struct ctdth64 *ctd64tree;
2915         struct voistatdata_tdgstctd32 *ctd32;
2916         struct voistatdata_tdgstctd64 *ctd64;
2917         uint64_t ebits, idxmask;
2918         uint32_t bitsperidx, nebits;
2919         int error, idx, is32bit, maxctds, remctds, tmperr;
2920
2921         error = 0;
2922
2923         switch (vs_dtype) {
2924         case VSD_DTYPE_TDGSTCLUST32:
2925                 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
2926                 if (!ARB_FULL(ctd32tree))
2927                         return (0);
2928                 VSD(tdgstclust32, tdgst)->compcnt++;
2929                 maxctds = remctds = ARB_MAXNODES(ctd32tree);
2930                 ARB_RESET_TREE(ctd32tree, ctdth32, maxctds);
2931                 VSD(tdgstclust32, tdgst)->smplcnt = 0;
2932                 is32bit = 1;
2933                 ctd64tree = NULL;
2934                 ctd64 = NULL;
2935 #ifdef DIAGNOSTIC
2936                 RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree);
2937 #endif
2938                 break;
2939         case VSD_DTYPE_TDGSTCLUST64:
2940                 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
2941                 if (!ARB_FULL(ctd64tree))
2942                         return (0);
2943                 VSD(tdgstclust64, tdgst)->compcnt++;
2944                 maxctds = remctds = ARB_MAXNODES(ctd64tree);
2945                 ARB_RESET_TREE(ctd64tree, ctdth64, maxctds);
2946                 VSD(tdgstclust64, tdgst)->smplcnt = 0;
2947                 is32bit = 0;
2948                 ctd32tree = NULL;
2949                 ctd32 = NULL;
2950 #ifdef DIAGNOSTIC
2951                 RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree);
2952 #endif
2953                 break;
2954         default:
2955                 return (EINVAL);
2956         }
2957
2958         /*
2959          * Rebuild the t-digest ARB by pseudorandomly selecting centroids and
2960          * re-inserting the mu/cnt of each as a value and corresponding weight.
2961          */
2962
2963         /*
2964          * XXXCEM: random(9) is currently rand(3), not random(3).  rand(3)
2965          * RAND_MAX happens to be approximately 31 bits (range [0,
2966          * 0x7ffffffd]), so the math kinda works out.  When/if this portion of
2967          * the code is compiled in userspace, it gets the random(3) behavior,
2968          * which has expected range [0, 0x7fffffff].
2969          */
2970 #define bitsperrand 31
2971         ebits = 0;
2972         nebits = 0;
2973         bitsperidx = fls(maxctds);
2974         KASSERT(bitsperidx <= sizeof(ebits) << 3,
2975             ("%s: bitsperidx=%d, ebits=%d",
2976             __func__, bitsperidx, (int)(sizeof(ebits) << 3)));
2977         idxmask = (UINT64_C(1) << bitsperidx) - 1;
2978
2979         /* Initialise the free list with randomised centroid indices. */
2980         for (; remctds > 0; remctds--) {
2981                 while (nebits < bitsperidx) {
2982                         ebits |= ((uint64_t)random()) << nebits;
2983                         nebits += bitsperrand;
2984                         if (nebits > (sizeof(ebits) << 3))
2985                                 nebits = sizeof(ebits) << 3;
2986                 }
2987                 idx = ebits & idxmask;
2988                 nebits -= bitsperidx;
2989                 ebits >>= bitsperidx;
2990
2991                 /*
2992                  * Select the next centroid to put on the ARB free list. We
2993                  * start with the centroid at our randomly selected array index,
2994                  * and work our way forwards until finding one (the latter
2995                  * aspect reduces re-insertion randomness, but is good enough).
2996                  */
2997                 do {
2998                         if (idx >= maxctds)
2999                                 idx %= maxctds;
3000
3001                         if (is32bit)
3002                                 ctd32 = ARB_NODE(ctd32tree, idx);
3003                         else
3004                                 ctd64 = ARB_NODE(ctd64tree, idx);
3005                 } while ((is32bit ? ARB_ISFREE(ctd32, ctdlnk) :
3006                     ARB_ISFREE(ctd64, ctdlnk)) && ++idx);
3007
3008                 /* Put the centroid on the ARB free list. */
3009                 if (is32bit)
3010                         ARB_RETURNFREE(ctd32tree, ctd32, ctdlnk);
3011                 else
3012                         ARB_RETURNFREE(ctd64tree, ctd64, ctdlnk);
3013         }
3014
3015         /*
3016          * The free list now contains the randomised indices of every centroid.
3017          * Walk the free list from start to end, re-inserting each centroid's
3018          * mu/cnt. The tdgst_add() call may or may not consume the free centroid
3019          * we re-insert values from during each loop iteration, so we must latch
3020          * the index of the next free list centroid before the re-insertion
3021          * call. The previous loop above should have left the centroid pointer
3022          * pointing to the element at the head of the free list.
3023          */
3024         KASSERT((is32bit ?
3025             ARB_FREEIDX(ctd32tree) == ARB_SELFIDX(ctd32tree, ctd32) :
3026             ARB_FREEIDX(ctd64tree) == ARB_SELFIDX(ctd64tree, ctd64)),
3027             ("%s: t-digest ARB@%p free list bug", __func__,
3028             (is32bit ? (void *)ctd32tree : (void *)ctd64tree)));
3029         remctds = maxctds;
3030         while ((is32bit ? ctd32 != NULL : ctd64 != NULL)) {
3031                 tmperr = 0;
3032                 if (is32bit) {
3033                         s64q_t x;
3034
3035                         idx = ARB_NEXTFREEIDX(ctd32, ctdlnk);
3036                         /* Cloning a s32q_t into a s64q_t should never fail. */
3037                         tmperr = Q_QCLONEQ(&x, ctd32->mu);
3038                         tmperr = tmperr ? tmperr : stats_v1_vsd_tdgst_add(
3039                             vs_dtype, tdgst, x, ctd32->cnt, attempt);
3040                         ctd32 = ARB_NODE(ctd32tree, idx);
3041                         KASSERT(ctd32 == NULL || ARB_ISFREE(ctd32, ctdlnk),
3042                             ("%s: t-digest ARB@%p free list bug", __func__,
3043                             ctd32tree));
3044                 } else {
3045                         idx = ARB_NEXTFREEIDX(ctd64, ctdlnk);
3046                         tmperr = stats_v1_vsd_tdgst_add(vs_dtype, tdgst,
3047                             ctd64->mu, ctd64->cnt, attempt);
3048                         ctd64 = ARB_NODE(ctd64tree, idx);
3049                         KASSERT(ctd64 == NULL || ARB_ISFREE(ctd64, ctdlnk),
3050                             ("%s: t-digest ARB@%p free list bug", __func__,
3051                             ctd64tree));
3052                 }
3053                 /*
3054                  * This process should not produce errors, bugs notwithstanding.
3055                  * Just in case, latch any errors and attempt all re-insertions.
3056                  */
3057                 error = tmperr ? tmperr : error;
3058                 remctds--;
3059         }
3060
3061         KASSERT(remctds == 0, ("%s: t-digest ARB@%p free list bug", __func__,
3062             (is32bit ? (void *)ctd32tree : (void *)ctd64tree)));
3063
3064         return (error);
3065 }
3066
3067 static inline int
3068 stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype, struct voistatdata_tdgst *tdgst,
3069     s64q_t x, uint64_t weight, int attempt)
3070 {
3071 #ifdef DIAGNOSTIC
3072         char qstr[Q_MAXSTRLEN(x, 10)];
3073 #endif
3074         struct ctdth32 *ctd32tree;
3075         struct ctdth64 *ctd64tree;
3076         void *closest, *cur, *lb, *ub;
3077         struct voistatdata_tdgstctd32 *ctd32;
3078         struct voistatdata_tdgstctd64 *ctd64;
3079         uint64_t cnt, smplcnt, sum, tmpsum;
3080         s64q_t k, minz, q, z;
3081         int error, is32bit, n;
3082
3083         error = 0;
3084         minz = Q_INI(&z, 0, 0, Q_NFBITS(x));
3085
3086         switch (vs_dtype) {
3087         case VSD_DTYPE_TDGSTCLUST32:
3088                 if ((UINT32_MAX - weight) < VSD(tdgstclust32, tdgst)->smplcnt)
3089                         error = EOVERFLOW;
3090                 smplcnt = VSD(tdgstclust32, tdgst)->smplcnt;
3091                 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
3092                 is32bit = 1;
3093                 ctd64tree = NULL;
3094                 ctd64 = NULL;
3095                 break;
3096         case VSD_DTYPE_TDGSTCLUST64:
3097                 if ((UINT64_MAX - weight) < VSD(tdgstclust64, tdgst)->smplcnt)
3098                         error = EOVERFLOW;
3099                 smplcnt = VSD(tdgstclust64, tdgst)->smplcnt;
3100                 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
3101                 is32bit = 0;
3102                 ctd32tree = NULL;
3103                 ctd32 = NULL;
3104                 break;
3105         default:
3106                 error = EINVAL;
3107                 break;
3108         }
3109
3110         if (error)
3111                 return (error);
3112
3113         /*
3114          * Inspired by Ted Dunning's AVLTreeDigest.java
3115          */
3116         do {
3117 #if defined(DIAGNOSTIC)
3118                 KASSERT(attempt < 5,
3119                     ("%s: Too many attempts", __func__));
3120 #endif
3121                 if (attempt >= 5)
3122                         return (EAGAIN);
3123
3124                 Q_SIFVAL(minz, Q_IFMAXVAL(minz));
3125                 closest = ub = NULL;
3126                 sum = tmpsum = 0;
3127
3128                 if (is32bit)
3129                         lb = cur = (void *)(ctd32 = ARB_MIN(ctdth32, ctd32tree));
3130                 else
3131                         lb = cur = (void *)(ctd64 = ARB_MIN(ctdth64, ctd64tree));
3132
3133                 if (lb == NULL) /* Empty tree. */
3134                         lb = (is32bit ? (void *)ARB_ROOT(ctd32tree) :
3135                             (void *)ARB_ROOT(ctd64tree));
3136
3137                 /*
3138                  * Find the set of centroids with minimum distance to x and
3139                  * compute the sum of counts for all centroids with mean less
3140                  * than the first centroid in the set.
3141                  */
3142                 for (; cur != NULL;
3143                     cur = (is32bit ?
3144                     (void *)(ctd32 = ARB_NEXT(ctdth32, ctd32tree, ctd32)) :
3145                     (void *)(ctd64 = ARB_NEXT(ctdth64, ctd64tree, ctd64)))) {
3146                         if (is32bit) {
3147                                 cnt = ctd32->cnt;
3148                                 KASSERT(Q_PRECEQ(ctd32->mu, x),
3149                                     ("%s: Q_RELPREC(mu,x)=%d", __func__,
3150                                     Q_RELPREC(ctd32->mu, x)));
3151                                 /* Ok to assign as both have same precision. */
3152                                 z = ctd32->mu;
3153                         } else {
3154                                 cnt = ctd64->cnt;
3155                                 KASSERT(Q_PRECEQ(ctd64->mu, x),
3156                                     ("%s: Q_RELPREC(mu,x)=%d", __func__,
3157                                     Q_RELPREC(ctd64->mu, x)));
3158                                 /* Ok to assign as both have same precision. */
3159                                 z = ctd64->mu;
3160                         }
3161
3162                         error = Q_QSUBQ(&z, x);
3163 #if defined(DIAGNOSTIC)
3164                         KASSERT(!error, ("%s: unexpected error %d", __func__,
3165                             error));
3166 #endif
3167                         if (error)
3168                                 return (error);
3169
3170                         z = Q_QABS(z);
3171                         if (Q_QLTQ(z, minz)) {
3172                                 minz = z;
3173                                 lb = cur;
3174                                 sum = tmpsum;
3175                                 tmpsum += cnt;
3176                         } else if (Q_QGTQ(z, minz)) {
3177                                 ub = cur;
3178                                 break;
3179                         }
3180                 }
3181
3182                 cur = (is32bit ?
3183                     (void *)(ctd32 = (struct voistatdata_tdgstctd32 *)lb) :
3184                     (void *)(ctd64 = (struct voistatdata_tdgstctd64 *)lb));
3185
3186                 for (n = 0; cur != ub; cur = (is32bit ?
3187                     (void *)(ctd32 = ARB_NEXT(ctdth32, ctd32tree, ctd32)) :
3188                     (void *)(ctd64 = ARB_NEXT(ctdth64, ctd64tree, ctd64)))) {
3189                         if (is32bit)
3190                                 cnt = ctd32->cnt;
3191                         else
3192                                 cnt = ctd64->cnt;
3193
3194                         q = Q_CTRLINI(16);
3195                         if (smplcnt == 1)
3196                                 error = Q_QFRACI(&q, 1, 2);
3197                         else
3198                                 /* [ sum + ((cnt - 1) / 2) ] / (smplcnt - 1) */
3199                                 error = Q_QFRACI(&q, (sum << 1) + cnt - 1,
3200                                     (smplcnt - 1) << 1);
3201                         k = q;
3202                         /* k = q x 4 x samplcnt x attempt */
3203                         error |= Q_QMULI(&k, 4 * smplcnt * attempt);
3204                         /* k = k x (1 - q) */
3205                         error |= Q_QSUBI(&q, 1);
3206                         q = Q_QABS(q);
3207                         error |= Q_QMULQ(&k, q);
3208 #if defined(DIAGNOSTIC)
3209 #if !defined(_KERNEL)
3210                         double q_dbl, k_dbl, q2d, k2d;
3211                         q2d = Q_Q2D(q);
3212                         k2d = Q_Q2D(k);
3213                         q_dbl = smplcnt == 1 ? 0.5 :
3214                             (sum + ((cnt - 1)  / 2.0)) / (double)(smplcnt - 1);
3215                         k_dbl = 4 * smplcnt * q_dbl * (1.0 - q_dbl) * attempt;
3216                         /*
3217                          * If the difference between q and q_dbl is greater than
3218                          * the fractional precision of q, something is off.
3219                          * NB: q is holding the value of 1 - q
3220                          */
3221                         q_dbl = 1.0 - q_dbl;
3222                         KASSERT((q_dbl > q2d ? q_dbl - q2d : q2d - q_dbl) <
3223                             (1.05 * ((double)1 / (double)(1ULL << Q_NFBITS(q)))),
3224                             ("Q-type q bad precision"));
3225                         KASSERT((k_dbl > k2d ? k_dbl - k2d : k2d - k_dbl) <
3226                             1.0 + (0.01 * smplcnt),
3227                             ("Q-type k bad precision"));
3228 #endif /* !_KERNEL */
3229                         KASSERT(!error, ("%s: unexpected error %d", __func__,
3230                             error));
3231 #endif /* DIAGNOSTIC */
3232                         if (error)
3233                                 return (error);
3234                         if ((is32bit && ((ctd32->cnt + weight) <=
3235                             (uint64_t)Q_GIVAL(k))) ||
3236                             (!is32bit && ((ctd64->cnt + weight) <=
3237                             (uint64_t)Q_GIVAL(k)))) {
3238                                 n++;
3239                                 /* random() produces 31 bits. */
3240                                 if (random() < (INT32_MAX / n))
3241                                         closest = cur;
3242                         }
3243                         sum += cnt;
3244                 }
3245         } while (closest == NULL &&
3246             (is32bit ? ARB_FULL(ctd32tree) : ARB_FULL(ctd64tree)) &&
3247             (error = stats_v1_vsd_tdgst_compress(vs_dtype, tdgst,
3248             attempt++)) == 0);
3249
3250         if (error)
3251                 return (error);
3252
3253         if (closest != NULL) {
3254                 /* Merge with an existing centroid. */
3255                 if (is32bit) {
3256                         ctd32 = (struct voistatdata_tdgstctd32 *)closest;
3257                         error = Q_QSUBQ(&x, ctd32->mu);
3258                         error = error ? error :
3259                             Q_QDIVI(&x, ctd32->cnt + weight);
3260                         if (error || (error = Q_QADDQ(&ctd32->mu, x))) {
3261 #ifdef DIAGNOSTIC
3262                                 KASSERT(!error, ("%s: unexpected error %d",
3263                                     __func__, error));
3264 #endif
3265                                 return (error);
3266                         }
3267                         ctd32->cnt += weight;
3268                         error = ARB_REINSERT(ctdth32, ctd32tree, ctd32) ==
3269                             NULL ? 0 : EALREADY;
3270 #ifdef DIAGNOSTIC
3271                         RB_REINSERT(rbctdth32,
3272                             &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32);
3273 #endif
3274                 } else {
3275                         ctd64 = (struct voistatdata_tdgstctd64 *)closest;
3276                         error = Q_QSUBQ(&x, ctd64->mu);
3277                         error = error ? error :
3278                             Q_QDIVI(&x, ctd64->cnt + weight);
3279                         if (error || (error = Q_QADDQ(&ctd64->mu, x))) {
3280                                 KASSERT(!error, ("%s: unexpected error %d",
3281                                     __func__, error));
3282                                 return (error);
3283                         }
3284                         ctd64->cnt += weight;
3285                         error = ARB_REINSERT(ctdth64, ctd64tree, ctd64) ==
3286                             NULL ? 0 : EALREADY;
3287 #ifdef DIAGNOSTIC
3288                         RB_REINSERT(rbctdth64,
3289                             &VSD(tdgstclust64, tdgst)->rbctdtree, ctd64);
3290 #endif
3291                 }
3292         } else {
3293                 /*
3294                  * Add a new centroid. If digest compression is working
3295                  * correctly, there should always be at least one free.
3296                  */
3297                 if (is32bit) {
3298                         ctd32 = ARB_GETFREE(ctd32tree, ctdlnk);
3299 #ifdef DIAGNOSTIC
3300                         KASSERT(ctd32 != NULL,
3301                             ("%s: t-digest@%p has no free centroids",
3302                             __func__, tdgst));
3303 #endif
3304                         if (ctd32 == NULL)
3305                                 return (EAGAIN);
3306                         if ((error = Q_QCPYVALQ(&ctd32->mu, x)))
3307                                 return (error);
3308                         ctd32->cnt = weight;
3309                         error = ARB_INSERT(ctdth32, ctd32tree, ctd32) == NULL ?
3310                             0 : EALREADY;
3311 #ifdef DIAGNOSTIC
3312                         RB_INSERT(rbctdth32,
3313                             &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32);
3314 #endif
3315                 } else {
3316                         ctd64 = ARB_GETFREE(ctd64tree, ctdlnk);
3317 #ifdef DIAGNOSTIC
3318                         KASSERT(ctd64 != NULL,
3319                             ("%s: t-digest@%p has no free centroids",
3320                             __func__, tdgst));
3321 #endif
3322                         if (ctd64 == NULL) /* Should not happen. */
3323                                 return (EAGAIN);
3324                         /* Direct assignment ok as both have same type/prec. */
3325                         ctd64->mu = x;
3326                         ctd64->cnt = weight;
3327                         error = ARB_INSERT(ctdth64, ctd64tree, ctd64) == NULL ?
3328                             0 : EALREADY;
3329 #ifdef DIAGNOSTIC
3330                         RB_INSERT(rbctdth64, &VSD(tdgstclust64,
3331                             tdgst)->rbctdtree, ctd64);
3332 #endif
3333                 }
3334         }
3335
3336         if (is32bit)
3337                 VSD(tdgstclust32, tdgst)->smplcnt += weight;
3338         else {
3339                 VSD(tdgstclust64, tdgst)->smplcnt += weight;
3340
3341 #ifdef DIAGNOSTIC
3342                 struct rbctdth64 *rbctdtree =
3343                     &VSD(tdgstclust64, tdgst)->rbctdtree;
3344                 struct voistatdata_tdgstctd64 *rbctd64;
3345                 int i = 0;
3346                 ARB_FOREACH(ctd64, ctdth64, ctd64tree) {
3347                         rbctd64 = (i == 0 ? RB_MIN(rbctdth64, rbctdtree) :
3348                             RB_NEXT(rbctdth64, rbctdtree, rbctd64));
3349
3350                         if (i >= ARB_CURNODES(ctd64tree)
3351                             || ctd64 != rbctd64
3352                             || ARB_MIN(ctdth64, ctd64tree) !=
3353                                RB_MIN(rbctdth64, rbctdtree)
3354                             || ARB_MAX(ctdth64, ctd64tree) !=
3355                                RB_MAX(rbctdth64, rbctdtree)
3356                             || ARB_LEFTIDX(ctd64, ctdlnk) !=
3357                                ARB_SELFIDX(ctd64tree, RB_LEFT(rbctd64, rblnk))
3358                             || ARB_RIGHTIDX(ctd64, ctdlnk) !=
3359                                ARB_SELFIDX(ctd64tree, RB_RIGHT(rbctd64, rblnk))
3360                             || ARB_PARENTIDX(ctd64, ctdlnk) !=
3361                                ARB_SELFIDX(ctd64tree,
3362                                RB_PARENT(rbctd64, rblnk))) {
3363                                 Q_TOSTR(ctd64->mu, -1, 10, qstr, sizeof(qstr));
3364                                 printf("ARB ctd=%3d p=%3d l=%3d r=%3d c=%2d "
3365                                     "mu=%s\n",
3366                                     (int)ARB_SELFIDX(ctd64tree, ctd64),
3367                                     ARB_PARENTIDX(ctd64, ctdlnk),
3368                                     ARB_LEFTIDX(ctd64, ctdlnk),
3369                                     ARB_RIGHTIDX(ctd64, ctdlnk),
3370                                     ARB_COLOR(ctd64, ctdlnk),
3371                                     qstr);
3372
3373                                 Q_TOSTR(rbctd64->mu, -1, 10, qstr,
3374                                     sizeof(qstr));
3375                                 printf(" RB ctd=%3d p=%3d l=%3d r=%3d c=%2d "
3376                                     "mu=%s\n",
3377                                     (int)ARB_SELFIDX(ctd64tree, rbctd64),
3378                                     (int)ARB_SELFIDX(ctd64tree,
3379                                       RB_PARENT(rbctd64, rblnk)),
3380                                     (int)ARB_SELFIDX(ctd64tree,
3381                                       RB_LEFT(rbctd64, rblnk)),
3382                                     (int)ARB_SELFIDX(ctd64tree,
3383                                       RB_RIGHT(rbctd64, rblnk)),
3384                                     RB_COLOR(rbctd64, rblnk),
3385                                     qstr);
3386
3387                                 panic("RB@%p and ARB@%p trees differ\n",
3388                                     rbctdtree, ctd64tree);
3389                         }
3390                         i++;
3391                 }
3392 #endif /* DIAGNOSTIC */
3393         }
3394
3395         return (error);
3396 }
3397
3398 static inline int
3399 stats_v1_voi_update_tdgst(enum vsd_dtype voi_dtype, struct voistatdata *voival,
3400     struct voistat *vs, struct voistatdata_tdgst *tdgst)
3401 {
3402         s64q_t x;
3403         int error;
3404
3405         error = 0;
3406
3407         switch (vs->dtype) {
3408         case VSD_DTYPE_TDGSTCLUST32:
3409                 /* Use same precision as the user's centroids. */
3410                 Q_INI(&x, 0, 0, Q_NFBITS(
3411                     ARB_CNODE(&VSD(tdgstclust32, tdgst)->ctdtree, 0)->mu));
3412                 break;
3413         case VSD_DTYPE_TDGSTCLUST64:
3414                 /* Use same precision as the user's centroids. */
3415                 Q_INI(&x, 0, 0, Q_NFBITS(
3416                     ARB_CNODE(&VSD(tdgstclust64, tdgst)->ctdtree, 0)->mu));
3417                 break;
3418         default:
3419                 KASSERT(vs->dtype == VSD_DTYPE_TDGSTCLUST32 ||
3420                     vs->dtype == VSD_DTYPE_TDGSTCLUST64,
3421                     ("%s: vs->dtype(%d) != VSD_DTYPE_TDGSTCLUST<32|64>",
3422                     __func__, vs->dtype));
3423                 return (EINVAL);
3424         }
3425
3426         /*
3427          * XXXLAS: Should have both a signed and unsigned 'x' variable to avoid
3428          * returning EOVERFLOW if the voival would have fit in a u64q_t.
3429          */
3430         switch (voi_dtype) {
3431         case VSD_DTYPE_INT_S32:
3432                 error = Q_QCPYVALI(&x, voival->int32.s32);
3433                 break;
3434         case VSD_DTYPE_INT_U32:
3435                 error = Q_QCPYVALI(&x, voival->int32.u32);
3436                 break;
3437         case VSD_DTYPE_INT_S64:
3438                 error = Q_QCPYVALI(&x, voival->int64.s64);
3439                 break;
3440         case VSD_DTYPE_INT_U64:
3441                 error = Q_QCPYVALI(&x, voival->int64.u64);
3442                 break;
3443         case VSD_DTYPE_INT_SLONG:
3444                 error = Q_QCPYVALI(&x, voival->intlong.slong);
3445                 break;
3446         case VSD_DTYPE_INT_ULONG:
3447                 error = Q_QCPYVALI(&x, voival->intlong.ulong);
3448                 break;
3449         case VSD_DTYPE_Q_S32:
3450                 error = Q_QCPYVALQ(&x, voival->q32.sq32);
3451                 break;
3452         case VSD_DTYPE_Q_U32:
3453                 error = Q_QCPYVALQ(&x, voival->q32.uq32);
3454                 break;
3455         case VSD_DTYPE_Q_S64:
3456                 error = Q_QCPYVALQ(&x, voival->q64.sq64);
3457                 break;
3458         case VSD_DTYPE_Q_U64:
3459                 error = Q_QCPYVALQ(&x, voival->q64.uq64);
3460                 break;
3461         default:
3462                 error = EINVAL;
3463                 break;
3464         }
3465
3466         if (error ||
3467             (error = stats_v1_vsd_tdgst_add(vs->dtype, tdgst, x, 1, 1)))
3468                 return (error);
3469
3470         vs->flags |= VS_VSDVALID;
3471         return (0);
3472 }
3473
3474 int
3475 stats_v1_voi_update(struct statsblobv1 *sb, int32_t voi_id,
3476     enum vsd_dtype voi_dtype, struct voistatdata *voival, uint32_t flags)
3477 {
3478         struct voi *v;
3479         struct voistat *vs;
3480         void *statevsd, *vsd;
3481         int error, i, tmperr;
3482
3483         error = 0;
3484
3485         if (sb == NULL || sb->abi != STATS_ABI_V1 || voi_id >= NVOIS(sb) ||
3486             voi_dtype == 0 || voi_dtype >= VSD_NUM_DTYPES || voival == NULL)
3487                 return (EINVAL);
3488         v = &sb->vois[voi_id];
3489         if (voi_dtype != v->dtype || v->id < 0 ||
3490             ((flags & SB_VOI_RELUPDATE) && !(v->flags & VOI_REQSTATE)))
3491                 return (EINVAL);
3492
3493         vs = BLOB_OFFSET(sb, v->stats_off);
3494         if (v->flags & VOI_REQSTATE)
3495                 statevsd = BLOB_OFFSET(sb, vs->data_off);
3496         else
3497                 statevsd = NULL;
3498
3499         if (flags & SB_VOI_RELUPDATE) {
3500                 switch (voi_dtype) {
3501                 case VSD_DTYPE_INT_S32:
3502                         voival->int32.s32 +=
3503                             VSD(voistate, statevsd)->prev.int32.s32;
3504                         break;
3505                 case VSD_DTYPE_INT_U32:
3506                         voival->int32.u32 +=
3507                             VSD(voistate, statevsd)->prev.int32.u32;
3508                         break;
3509                 case VSD_DTYPE_INT_S64:
3510                         voival->int64.s64 +=
3511                             VSD(voistate, statevsd)->prev.int64.s64;
3512                         break;
3513                 case VSD_DTYPE_INT_U64:
3514                         voival->int64.u64 +=
3515                             VSD(voistate, statevsd)->prev.int64.u64;
3516                         break;
3517                 case VSD_DTYPE_INT_SLONG:
3518                         voival->intlong.slong +=
3519                             VSD(voistate, statevsd)->prev.intlong.slong;
3520                         break;
3521                 case VSD_DTYPE_INT_ULONG:
3522                         voival->intlong.ulong +=
3523                             VSD(voistate, statevsd)->prev.intlong.ulong;
3524                         break;
3525                 case VSD_DTYPE_Q_S32:
3526                         error = Q_QADDQ(&voival->q32.sq32,
3527                             VSD(voistate, statevsd)->prev.q32.sq32);
3528                         break;
3529                 case VSD_DTYPE_Q_U32:
3530                         error = Q_QADDQ(&voival->q32.uq32,
3531                             VSD(voistate, statevsd)->prev.q32.uq32);
3532                         break;
3533                 case VSD_DTYPE_Q_S64:
3534                         error = Q_QADDQ(&voival->q64.sq64,
3535                             VSD(voistate, statevsd)->prev.q64.sq64);
3536                         break;
3537                 case VSD_DTYPE_Q_U64:
3538                         error = Q_QADDQ(&voival->q64.uq64,
3539                             VSD(voistate, statevsd)->prev.q64.uq64);
3540                         break;
3541                 default:
3542                         KASSERT(0, ("Unknown VOI data type %d", voi_dtype));
3543                         break;
3544                 }
3545         }
3546
3547         if (error)
3548                 return (error);
3549
3550         for (i = v->voistatmaxid; i > 0; i--) {
3551                 vs = &((struct voistat *)BLOB_OFFSET(sb, v->stats_off))[i];
3552                 if (vs->stype < 0)
3553                         continue;
3554
3555                 vsd = BLOB_OFFSET(sb, vs->data_off);
3556
3557                 switch (vs->stype) {
3558                 case VS_STYPE_MAX:
3559                         tmperr = stats_v1_voi_update_max(voi_dtype, voival,
3560                             vs, vsd);
3561                         break;
3562                 case VS_STYPE_MIN:
3563                         tmperr = stats_v1_voi_update_min(voi_dtype, voival,
3564                             vs, vsd);
3565                         break;
3566                 case VS_STYPE_SUM:
3567                         tmperr = stats_v1_voi_update_sum(voi_dtype, voival,
3568                             vs, vsd);
3569                         break;
3570                 case VS_STYPE_HIST:
3571                         tmperr = stats_v1_voi_update_hist(voi_dtype, voival,
3572                             vs, vsd);
3573                         break;
3574                 case VS_STYPE_TDGST:
3575                         tmperr = stats_v1_voi_update_tdgst(voi_dtype, voival,
3576                             vs, vsd);
3577                         break;
3578                 default:
3579                         KASSERT(0, ("Unknown VOI stat type %d", vs->stype));
3580                         break;
3581                 }
3582
3583                 if (tmperr) {
3584                         error = tmperr;
3585                         VS_INCERRS(vs);
3586                 }
3587         }
3588
3589         if (statevsd) {
3590                 switch (voi_dtype) {
3591                 case VSD_DTYPE_INT_S32:
3592                         VSD(voistate, statevsd)->prev.int32.s32 =
3593                             voival->int32.s32;
3594                         break;
3595                 case VSD_DTYPE_INT_U32:
3596                         VSD(voistate, statevsd)->prev.int32.u32 =
3597                             voival->int32.u32;
3598                         break;
3599                 case VSD_DTYPE_INT_S64:
3600                         VSD(voistate, statevsd)->prev.int64.s64 =
3601                             voival->int64.s64;
3602                         break;
3603                 case VSD_DTYPE_INT_U64:
3604                         VSD(voistate, statevsd)->prev.int64.u64 =
3605                             voival->int64.u64;
3606                         break;
3607                 case VSD_DTYPE_INT_SLONG:
3608                         VSD(voistate, statevsd)->prev.intlong.slong =
3609                             voival->intlong.slong;
3610                         break;
3611                 case VSD_DTYPE_INT_ULONG:
3612                         VSD(voistate, statevsd)->prev.intlong.ulong =
3613                             voival->intlong.ulong;
3614                         break;
3615                 case VSD_DTYPE_Q_S32:
3616                         error = Q_QCPYVALQ(
3617                             &VSD(voistate, statevsd)->prev.q32.sq32,
3618                             voival->q32.sq32);
3619                         break;
3620                 case VSD_DTYPE_Q_U32:
3621                         error = Q_QCPYVALQ(
3622                             &VSD(voistate, statevsd)->prev.q32.uq32,
3623                             voival->q32.uq32);
3624                         break;
3625                 case VSD_DTYPE_Q_S64:
3626                         error = Q_QCPYVALQ(
3627                             &VSD(voistate, statevsd)->prev.q64.sq64,
3628                             voival->q64.sq64);
3629                         break;
3630                 case VSD_DTYPE_Q_U64:
3631                         error = Q_QCPYVALQ(
3632                             &VSD(voistate, statevsd)->prev.q64.uq64,
3633                             voival->q64.uq64);
3634                         break;
3635                 default:
3636                         KASSERT(0, ("Unknown VOI data type %d", voi_dtype));
3637                         break;
3638                 }
3639         }
3640
3641         return (error);
3642 }
3643
3644 #ifdef _KERNEL
3645
3646 static void
3647 stats_init(void *arg)
3648 {
3649
3650 }
3651 SYSINIT(stats, SI_SUB_KDTRACE, SI_ORDER_FIRST, stats_init, NULL);
3652
3653 /*
3654  * Sysctl handler to display the list of available stats templates.
3655  */
3656 static int
3657 stats_tpl_list_available(SYSCTL_HANDLER_ARGS)
3658 {
3659         struct sbuf *s;
3660         int err, i;
3661
3662         err = 0;
3663
3664         /* We can tolerate ntpl being stale, so do not take the lock. */
3665         s = sbuf_new(NULL, NULL, /* +1 per tpl for , */
3666             ntpl * (STATS_TPL_MAX_STR_SPEC_LEN + 1), SBUF_FIXEDLEN);
3667         if (s == NULL)
3668                 return (ENOMEM);
3669
3670         TPL_LIST_RLOCK();
3671         for (i = 0; i < ntpl; i++) {
3672                 err = sbuf_printf(s, "%s\"%s\":%u", i ? "," : "",
3673                     tpllist[i]->mb->tplname, tpllist[i]->mb->tplhash);
3674                 if (err) {
3675                         /* Sbuf overflow condition. */
3676                         err = EOVERFLOW;
3677                         break;
3678                 }
3679         }
3680         TPL_LIST_RUNLOCK();
3681
3682         if (!err) {
3683                 sbuf_finish(s);
3684                 err = sysctl_handle_string(oidp, sbuf_data(s), 0, req);
3685         }
3686
3687         sbuf_delete(s);
3688         return (err);
3689 }
3690
3691 /*
3692  * Called by subsystem-specific sysctls to report and/or parse the list of
3693  * templates being sampled and their sampling rates. A stats_tpl_sr_cb_t
3694  * conformant function pointer must be passed in as arg1, which is used to
3695  * interact with the subsystem's stats template sample rates list. If arg2 > 0,
3696  * a zero-initialised allocation of arg2-sized contextual memory is
3697  * heap-allocated and passed in to all subsystem callbacks made during the
3698  * operation of stats_tpl_sample_rates().
3699  *
3700  * XXXLAS: Assumes templates are never removed, which is currently true but may
3701  * need to be reworked in future if dynamic template management becomes a
3702  * requirement e.g. to support kernel module based templates.
3703  */
3704 int
3705 stats_tpl_sample_rates(SYSCTL_HANDLER_ARGS)
3706 {
3707         char kvpair_fmt[16], tplspec_fmt[16];
3708         char tpl_spec[STATS_TPL_MAX_STR_SPEC_LEN];
3709         char tpl_name[TPL_MAX_NAME_LEN + 2]; /* +2 for "" */
3710         stats_tpl_sr_cb_t subsys_cb;
3711         void *subsys_ctx;
3712         char *buf, *new_rates_usr_str, *tpl_name_p;
3713         struct stats_tpl_sample_rate *rates;
3714         struct sbuf *s, _s;
3715         uint32_t cum_pct, pct, tpl_hash;
3716         int err, i, off, len, newlen, nrates;
3717
3718         buf = NULL;
3719         rates = NULL;
3720         err = nrates = 0;
3721         subsys_cb = (stats_tpl_sr_cb_t)arg1;
3722         KASSERT(subsys_cb != NULL, ("%s: subsys_cb == arg1 == NULL", __func__));
3723         if (arg2 > 0)
3724                 subsys_ctx = malloc(arg2, M_TEMP, M_WAITOK | M_ZERO);
3725         else
3726                 subsys_ctx = NULL;
3727
3728         /* Grab current count of subsystem rates. */
3729         err = subsys_cb(TPL_SR_UNLOCKED_GET, NULL, &nrates, subsys_ctx);
3730         if (err)
3731                 goto done;
3732
3733         /* +1 to ensure we can append '\0' post copyin, +5 per rate for =nnn, */
3734         len = max(req->newlen + 1, nrates * (STATS_TPL_MAX_STR_SPEC_LEN + 5));
3735
3736         if (req->oldptr != NULL || req->newptr != NULL)
3737                 buf = malloc(len, M_TEMP, M_WAITOK);
3738
3739         if (req->oldptr != NULL) {
3740                 if (nrates == 0) {
3741                         /* No rates, so return an empty string via oldptr. */
3742                         err = SYSCTL_OUT(req, "", 1);
3743                         if (err)
3744                                 goto done;
3745                         goto process_new;
3746                 }
3747
3748                 s = sbuf_new(&_s, buf, len, SBUF_FIXEDLEN | SBUF_INCLUDENUL);
3749
3750                 /* Grab locked count of, and ptr to, subsystem rates. */
3751                 err = subsys_cb(TPL_SR_RLOCKED_GET, &rates, &nrates,
3752                     subsys_ctx);
3753                 if (err)
3754                         goto done;
3755                 TPL_LIST_RLOCK();
3756                 for (i = 0; i < nrates && !err; i++) {
3757                         err = sbuf_printf(s, "%s\"%s\":%u=%u", i ? "," : "",
3758                             tpllist[rates[i].tpl_slot_id]->mb->tplname,
3759                             tpllist[rates[i].tpl_slot_id]->mb->tplhash,
3760                             rates[i].tpl_sample_pct);
3761                 }
3762                 TPL_LIST_RUNLOCK();
3763                 /* Tell subsystem that we're done with its rates list. */
3764                 err = subsys_cb(TPL_SR_RUNLOCK, &rates, &nrates, subsys_ctx);
3765                 if (err)
3766                         goto done;
3767
3768                 err = sbuf_finish(s);
3769                 if (err)
3770                         goto done; /* We lost a race for buf to be too small. */
3771
3772                 /* Return the rendered string data via oldptr. */
3773                 err = SYSCTL_OUT(req, sbuf_data(s), sbuf_len(s));
3774         } else {
3775                 /* Return the upper bound size for buffer sizing requests. */
3776                 err = SYSCTL_OUT(req, NULL, len);
3777         }
3778
3779 process_new:
3780         if (err || req->newptr == NULL)
3781                 goto done;
3782
3783         newlen = req->newlen - req->newidx;
3784         err = SYSCTL_IN(req, buf, newlen);
3785         if (err)
3786                 goto done;
3787
3788         /*
3789          * Initialise format strings at run time.
3790          *
3791          * Write the max template spec string length into the
3792          * template_spec=percent key-value pair parsing format string as:
3793          *     " %<width>[^=]=%u %n"
3794          *
3795          * Write the max template name string length into the tplname:tplhash
3796          * parsing format string as:
3797          *     "%<width>[^:]:%u"
3798          *
3799          * Subtract 1 for \0 appended by sscanf().
3800          */
3801         sprintf(kvpair_fmt, " %%%zu[^=]=%%u %%n", sizeof(tpl_spec) - 1);
3802         sprintf(tplspec_fmt, "%%%zu[^:]:%%u", sizeof(tpl_name) - 1);
3803
3804         /*
3805          * Parse each CSV key-value pair specifying a template and its sample
3806          * percentage. Whitespace either side of a key-value pair is ignored.
3807          * Templates can be specified by name, hash, or name and hash per the
3808          * following formats (chars in [] are optional):
3809          *    ["]<tplname>["]=<percent>
3810          *    :hash=pct
3811          *    ["]<tplname>["]:hash=<percent>
3812          */
3813         cum_pct = nrates = 0;
3814         rates = NULL;
3815         buf[newlen] = '\0'; /* buf is at least newlen+1 in size. */
3816         new_rates_usr_str = buf;
3817         while (isspace(*new_rates_usr_str))
3818                 new_rates_usr_str++; /* Skip leading whitespace. */
3819         while (*new_rates_usr_str != '\0') {
3820                 tpl_name_p = tpl_name;
3821                 tpl_name[0] = '\0';
3822                 tpl_hash = 0;
3823                 off = 0;
3824
3825                 /*
3826                  * Parse key-value pair which must perform 2 conversions, then
3827                  * parse the template spec to extract either name, hash, or name
3828                  * and hash depending on the three possible spec formats. The
3829                  * tplspec_fmt format specifier parses name or name and hash
3830                  * template specs, while the ":%u" format specifier parses
3831                  * hash-only template specs. If parsing is successfull, ensure
3832                  * the cumulative sampling percentage does not exceed 100.
3833                  */
3834                 err = EINVAL;
3835                 if (2 != sscanf(new_rates_usr_str, kvpair_fmt, tpl_spec, &pct,
3836                     &off))
3837                         break;
3838                 if ((1 > sscanf(tpl_spec, tplspec_fmt, tpl_name, &tpl_hash)) &&
3839                     (1 != sscanf(tpl_spec, ":%u", &tpl_hash)))
3840                         break;
3841                 if ((cum_pct += pct) > 100)
3842                         break;
3843                 err = 0;
3844
3845                 /* Strip surrounding "" from template name if present. */
3846                 len = strlen(tpl_name);
3847                 if (len > 0) {
3848                         if (tpl_name[len - 1] == '"')
3849                                 tpl_name[--len] = '\0';
3850                         if (tpl_name[0] == '"') {
3851                                 tpl_name_p++;
3852                                 len--;
3853                         }
3854                 }
3855
3856                 rates = stats_realloc(rates, 0, /* oldsz is unused in kernel. */
3857                     (nrates + 1) * sizeof(*rates), M_WAITOK);
3858                 rates[nrates].tpl_slot_id =
3859                     stats_tpl_fetch_allocid(len ? tpl_name_p : NULL, tpl_hash);
3860                 if (rates[nrates].tpl_slot_id < 0) {
3861                         err = -rates[nrates].tpl_slot_id;
3862                         break;
3863                 }
3864                 rates[nrates].tpl_sample_pct = pct;
3865                 nrates++;
3866                 new_rates_usr_str += off;
3867                 if (*new_rates_usr_str != ',')
3868                         break; /* End-of-input or malformed. */
3869                 new_rates_usr_str++; /* Move past comma to next pair. */
3870         }
3871
3872         if (!err) {
3873                 if ((new_rates_usr_str - buf) < newlen) {
3874                         /* Entire input has not been consumed. */
3875                         err = EINVAL;
3876                 } else {
3877                         /*
3878                          * Give subsystem the new rates. They'll return the
3879                          * appropriate rates pointer for us to garbage collect.
3880                          */
3881                         err = subsys_cb(TPL_SR_PUT, &rates, &nrates,
3882                             subsys_ctx);
3883                 }
3884         }
3885         stats_free(rates);
3886
3887 done:
3888         free(buf, M_TEMP);
3889         free(subsys_ctx, M_TEMP);
3890         return (err);
3891 }
3892
3893 SYSCTL_NODE(_kern, OID_AUTO, stats, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
3894     "stats(9) MIB");
3895
3896 SYSCTL_PROC(_kern_stats, OID_AUTO, templates,
3897     CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
3898     stats_tpl_list_available, "A",
3899     "list the name/hash of all available stats(9) templates");
3900
3901 #else /* ! _KERNEL */
3902
3903 static void __attribute__ ((constructor))
3904 stats_constructor(void)
3905 {
3906
3907         pthread_rwlock_init(&tpllistlock, NULL);
3908 }
3909
3910 static void __attribute__ ((destructor))
3911 stats_destructor(void)
3912 {
3913
3914         pthread_rwlock_destroy(&tpllistlock);
3915 }
3916
3917 #endif /* _KERNEL */