2 * Copyright (c) 2014-2018 Netflix, Inc.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * Author: Lawrence Stewart <lstewart@netflix.com>
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD$");
36 #include <sys/param.h>
38 #include <sys/ctype.h>
39 #include <sys/errno.h>
41 #include <sys/limits.h>
42 #include <sys/malloc.h>
43 #include <sys/qmath.h>
45 #if defined(DIAGNOSTIC)
48 #include <sys/stats.h> /* Must come after qmath.h and arb.h */
49 #include <sys/stddef.h>
50 #include <sys/stdint.h>
54 #include <sys/kernel.h>
56 #include <sys/rwlock.h>
57 #include <sys/sysctl.h>
58 #include <sys/systm.h>
67 struct voistatdata_voistate {
68 /* Previous VOI value for diff calculation. */
69 struct voistatdata_numeric prev;
72 #define VS_VSDVALID 0x0001 /* Stat's voistatdata updated at least once. */
74 int8_t stype; /* Type of stat e.g. VS_STYPE_SUM. */
75 enum vsd_dtype dtype : 8; /* Data type of this stat's data. */
76 uint16_t data_off; /* Blob offset for this stat's data. */
77 uint16_t dsz; /* Size of stat's data. */
79 uint16_t errs : VS_EBITS;/* Non-wrapping error count. */
80 uint16_t flags : 16 - VS_EBITS;
82 /* The voistat error count is capped to avoid wrapping. */
83 #define VS_INCERRS(vs) do { \
84 if ((vs)->errs < (1U << VS_EBITS) - 1) \
90 * - Global or entity specific (global would imply use of counter(9)?)
91 * - Whether to reset stats on read or not
92 * - Signal an overflow?
93 * - Compressed voistat array
95 #define VOI_REQSTATE 0x0001 /* VOI requires VS_STYPE_VOISTATE. */
97 int16_t id; /* VOI id. */
98 enum vsd_dtype dtype : 8; /* Data type of the VOI itself. */
99 int8_t voistatmaxid; /* Largest allocated voistat index. */
100 uint16_t stats_off; /* Blob offset for this VOIs stats. */
105 * Memory for the entire blob is allocated as a slab and then offsets are
106 * maintained to carve up the slab into sections holding different data types.
109 * - Compressed voi array (trade off memory usage vs search time)
110 * - Units of offsets (default bytes, flag for e.g. vm_page/KiB/Mib)
118 /* Fields from here down are opaque to consumers. */
119 uint32_t tplhash; /* Base template hash ID. */
120 uint16_t stats_off; /* voistat array blob offset. */
121 uint16_t statsdata_off; /* voistatdata array blob offset. */
122 sbintime_t created; /* Blob creation time. */
123 sbintime_t lastrst; /* Time of last reset. */
124 struct voi vois[]; /* Array indexed by [voi_id]. */
125 } __aligned(sizeof(void *));
126 _Static_assert(offsetof(struct statsblobv1, cursz) +
127 SIZEOF_MEMBER(struct statsblobv1, cursz) == sizeof(struct statsblob),
128 "statsblobv1 ABI mismatch");
130 struct statsblobv1_tpl {
132 struct statsblobv1 *sb;
135 /* Context passed to iterator callbacks. */
137 void *usrctx; /* Caller supplied context. */
138 uint32_t flags; /* Flags for current iteration. */
139 int16_t vslot; /* struct voi slot index. */
140 int8_t vsslot; /* struct voistat slot index. */
143 struct sb_tostrcb_ctx {
145 struct statsblob_tpl *tpl;
150 struct sb_visitcb_ctx {
151 stats_blob_visitcb_t cb;
155 /* Stats blob iterator callback. */
156 typedef int (*stats_v1_blob_itercb_t)(struct statsblobv1 *sb, struct voi *v,
157 struct voistat *vs, struct sb_iter_ctx *ctx);
160 static struct rwlock tpllistlock;
161 RW_SYSINIT(stats_tpl_list, &tpllistlock, "Stat template list lock");
162 #define TPL_LIST_RLOCK() rw_rlock(&tpllistlock)
163 #define TPL_LIST_RUNLOCK() rw_runlock(&tpllistlock)
164 #define TPL_LIST_WLOCK() rw_wlock(&tpllistlock)
165 #define TPL_LIST_WUNLOCK() rw_wunlock(&tpllistlock)
166 #define TPL_LIST_LOCK_ASSERT() rw_assert(&tpllistlock, RA_LOCKED)
167 #define TPL_LIST_RLOCK_ASSERT() rw_assert(&tpllistlock, RA_RLOCKED)
168 #define TPL_LIST_WLOCK_ASSERT() rw_assert(&tpllistlock, RA_WLOCKED)
169 MALLOC_DEFINE(M_STATS, "stats(9) related memory", "stats(9) related memory");
170 #define stats_free(ptr) free((ptr), M_STATS)
171 #else /* ! _KERNEL */
172 static void stats_constructor(void);
173 static void stats_destructor(void);
174 static pthread_rwlock_t tpllistlock;
175 #define TPL_LIST_UNLOCK() pthread_rwlock_unlock(&tpllistlock)
176 #define TPL_LIST_RLOCK() pthread_rwlock_rdlock(&tpllistlock)
177 #define TPL_LIST_RUNLOCK() TPL_LIST_UNLOCK()
178 #define TPL_LIST_WLOCK() pthread_rwlock_wrlock(&tpllistlock)
179 #define TPL_LIST_WUNLOCK() TPL_LIST_UNLOCK()
180 #define TPL_LIST_LOCK_ASSERT() do { } while (0)
181 #define TPL_LIST_RLOCK_ASSERT() do { } while (0)
182 #define TPL_LIST_WLOCK_ASSERT() do { } while (0)
184 #define KASSERT(cond, msg) do {} while (0)
185 #define stats_abort() do {} while (0)
187 #define KASSERT(cond, msg) do { \
192 #define stats_abort() abort()
194 #define stats_free(ptr) free(ptr)
195 #define panic(fmt, ...) do { \
196 fprintf(stderr, (fmt), ##__VA_ARGS__); \
201 #define SB_V1_MAXSZ 65535
203 /* Obtain a blob offset pointer. */
204 #define BLOB_OFFSET(sb, off) ((void *)(((uint8_t *)(sb)) + (off)))
207 * Number of VOIs in the blob's vois[] array. By virtue of struct voi being a
208 * power of 2 size, we can shift instead of divide. The shift amount must be
209 * updated if sizeof(struct voi) ever changes, which the assert should catch.
211 #define NVOIS(sb) ((int32_t)((((struct statsblobv1 *)(sb))->stats_off - \
212 sizeof(struct statsblobv1)) >> 3))
213 _Static_assert(sizeof(struct voi) == 8, "statsblobv1 voi ABI mismatch");
215 /* Try restrict names to alphanumeric and underscore to simplify JSON compat. */
216 const char *vs_stype2name[VS_NUM_STYPES] = {
217 [VS_STYPE_VOISTATE] = "VOISTATE",
218 [VS_STYPE_SUM] = "SUM",
219 [VS_STYPE_MAX] = "MAX",
220 [VS_STYPE_MIN] = "MIN",
221 [VS_STYPE_HIST] = "HIST",
222 [VS_STYPE_TDGST] = "TDGST",
225 const char *vs_stype2desc[VS_NUM_STYPES] = {
226 [VS_STYPE_VOISTATE] = "VOI related state data (not a real stat)",
227 [VS_STYPE_SUM] = "Simple arithmetic accumulator",
228 [VS_STYPE_MAX] = "Maximum observed VOI value",
229 [VS_STYPE_MIN] = "Minimum observed VOI value",
230 [VS_STYPE_HIST] = "Histogram of observed VOI values",
231 [VS_STYPE_TDGST] = "t-digest of observed VOI values",
234 const char *vsd_dtype2name[VSD_NUM_DTYPES] = {
235 [VSD_DTYPE_VOISTATE] = "VOISTATE",
236 [VSD_DTYPE_INT_S32] = "INT_S32",
237 [VSD_DTYPE_INT_U32] = "INT_U32",
238 [VSD_DTYPE_INT_S64] = "INT_S64",
239 [VSD_DTYPE_INT_U64] = "INT_U64",
240 [VSD_DTYPE_INT_SLONG] = "INT_SLONG",
241 [VSD_DTYPE_INT_ULONG] = "INT_ULONG",
242 [VSD_DTYPE_Q_S32] = "Q_S32",
243 [VSD_DTYPE_Q_U32] = "Q_U32",
244 [VSD_DTYPE_Q_S64] = "Q_S64",
245 [VSD_DTYPE_Q_U64] = "Q_U64",
246 [VSD_DTYPE_CRHIST32] = "CRHIST32",
247 [VSD_DTYPE_DRHIST32] = "DRHIST32",
248 [VSD_DTYPE_DVHIST32] = "DVHIST32",
249 [VSD_DTYPE_CRHIST64] = "CRHIST64",
250 [VSD_DTYPE_DRHIST64] = "DRHIST64",
251 [VSD_DTYPE_DVHIST64] = "DVHIST64",
252 [VSD_DTYPE_TDGSTCLUST32] = "TDGSTCLUST32",
253 [VSD_DTYPE_TDGSTCLUST64] = "TDGSTCLUST64",
256 const size_t vsd_dtype2size[VSD_NUM_DTYPES] = {
257 [VSD_DTYPE_VOISTATE] = sizeof(struct voistatdata_voistate),
258 [VSD_DTYPE_INT_S32] = sizeof(struct voistatdata_int32),
259 [VSD_DTYPE_INT_U32] = sizeof(struct voistatdata_int32),
260 [VSD_DTYPE_INT_S64] = sizeof(struct voistatdata_int64),
261 [VSD_DTYPE_INT_U64] = sizeof(struct voistatdata_int64),
262 [VSD_DTYPE_INT_SLONG] = sizeof(struct voistatdata_intlong),
263 [VSD_DTYPE_INT_ULONG] = sizeof(struct voistatdata_intlong),
264 [VSD_DTYPE_Q_S32] = sizeof(struct voistatdata_q32),
265 [VSD_DTYPE_Q_U32] = sizeof(struct voistatdata_q32),
266 [VSD_DTYPE_Q_S64] = sizeof(struct voistatdata_q64),
267 [VSD_DTYPE_Q_U64] = sizeof(struct voistatdata_q64),
268 [VSD_DTYPE_CRHIST32] = sizeof(struct voistatdata_crhist32),
269 [VSD_DTYPE_DRHIST32] = sizeof(struct voistatdata_drhist32),
270 [VSD_DTYPE_DVHIST32] = sizeof(struct voistatdata_dvhist32),
271 [VSD_DTYPE_CRHIST64] = sizeof(struct voistatdata_crhist64),
272 [VSD_DTYPE_DRHIST64] = sizeof(struct voistatdata_drhist64),
273 [VSD_DTYPE_DVHIST64] = sizeof(struct voistatdata_dvhist64),
274 [VSD_DTYPE_TDGSTCLUST32] = sizeof(struct voistatdata_tdgstclust32),
275 [VSD_DTYPE_TDGSTCLUST64] = sizeof(struct voistatdata_tdgstclust64),
278 static const bool vsd_compoundtype[VSD_NUM_DTYPES] = {
279 [VSD_DTYPE_VOISTATE] = true,
280 [VSD_DTYPE_INT_S32] = false,
281 [VSD_DTYPE_INT_U32] = false,
282 [VSD_DTYPE_INT_S64] = false,
283 [VSD_DTYPE_INT_U64] = false,
284 [VSD_DTYPE_INT_SLONG] = false,
285 [VSD_DTYPE_INT_ULONG] = false,
286 [VSD_DTYPE_Q_S32] = false,
287 [VSD_DTYPE_Q_U32] = false,
288 [VSD_DTYPE_Q_S64] = false,
289 [VSD_DTYPE_Q_U64] = false,
290 [VSD_DTYPE_CRHIST32] = true,
291 [VSD_DTYPE_DRHIST32] = true,
292 [VSD_DTYPE_DVHIST32] = true,
293 [VSD_DTYPE_CRHIST64] = true,
294 [VSD_DTYPE_DRHIST64] = true,
295 [VSD_DTYPE_DVHIST64] = true,
296 [VSD_DTYPE_TDGSTCLUST32] = true,
297 [VSD_DTYPE_TDGSTCLUST64] = true,
300 const struct voistatdata_numeric numeric_limits[2][VSD_DTYPE_Q_U64 + 1] = {
302 [VSD_DTYPE_VOISTATE] = {0},
303 [VSD_DTYPE_INT_S32] = {.int32 = {.s32 = INT32_MIN}},
304 [VSD_DTYPE_INT_U32] = {.int32 = {.u32 = 0}},
305 [VSD_DTYPE_INT_S64] = {.int64 = {.s64 = INT64_MIN}},
306 [VSD_DTYPE_INT_U64] = {.int64 = {.u64 = 0}},
307 [VSD_DTYPE_INT_SLONG] = {.intlong = {.slong = LONG_MIN}},
308 [VSD_DTYPE_INT_ULONG] = {.intlong = {.ulong = 0}},
309 [VSD_DTYPE_Q_S32] = {.q32 = {.sq32 = Q_IFMINVAL(INT32_MIN)}},
310 [VSD_DTYPE_Q_U32] = {.q32 = {.uq32 = 0}},
311 [VSD_DTYPE_Q_S64] = {.q64 = {.sq64 = Q_IFMINVAL(INT64_MIN)}},
312 [VSD_DTYPE_Q_U64] = {.q64 = {.uq64 = 0}},
315 [VSD_DTYPE_VOISTATE] = {0},
316 [VSD_DTYPE_INT_S32] = {.int32 = {.s32 = INT32_MAX}},
317 [VSD_DTYPE_INT_U32] = {.int32 = {.u32 = UINT32_MAX}},
318 [VSD_DTYPE_INT_S64] = {.int64 = {.s64 = INT64_MAX}},
319 [VSD_DTYPE_INT_U64] = {.int64 = {.u64 = UINT64_MAX}},
320 [VSD_DTYPE_INT_SLONG] = {.intlong = {.slong = LONG_MAX}},
321 [VSD_DTYPE_INT_ULONG] = {.intlong = {.ulong = ULONG_MAX}},
322 [VSD_DTYPE_Q_S32] = {.q32 = {.sq32 = Q_IFMAXVAL(INT32_MAX)}},
323 [VSD_DTYPE_Q_U32] = {.q32 = {.uq32 = Q_IFMAXVAL(UINT32_MAX)}},
324 [VSD_DTYPE_Q_S64] = {.q64 = {.sq64 = Q_IFMAXVAL(INT64_MAX)}},
325 [VSD_DTYPE_Q_U64] = {.q64 = {.uq64 = Q_IFMAXVAL(UINT64_MAX)}},
329 /* tpllistlock protects tpllist and ntpl */
330 static uint32_t ntpl;
331 static struct statsblob_tpl **tpllist;
333 static inline void * stats_realloc(void *ptr, size_t oldsz, size_t newsz,
335 //static void stats_v1_blob_finalise(struct statsblobv1 *sb);
336 static int stats_v1_blob_init_locked(struct statsblobv1 *sb, uint32_t tpl_id,
338 static int stats_v1_blob_expand(struct statsblobv1 **sbpp, int newvoibytes,
339 int newvoistatbytes, int newvoistatdatabytes);
340 static void stats_v1_blob_iter(struct statsblobv1 *sb,
341 stats_v1_blob_itercb_t icb, void *usrctx, uint32_t flags);
342 static inline int stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype,
343 struct voistatdata_tdgst *tdgst, s64q_t x, uint64_t weight, int attempt);
346 ctd32cmp(const struct voistatdata_tdgstctd32 *c1, const struct voistatdata_tdgstctd32 *c2)
349 KASSERT(Q_PRECEQ(c1->mu, c2->mu),
350 ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__,
351 Q_RELPREC(c1->mu, c2->mu)));
353 return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1);
355 ARB_GENERATE_STATIC(ctdth32, voistatdata_tdgstctd32, ctdlnk, ctd32cmp);
358 ctd64cmp(const struct voistatdata_tdgstctd64 *c1, const struct voistatdata_tdgstctd64 *c2)
361 KASSERT(Q_PRECEQ(c1->mu, c2->mu),
362 ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__,
363 Q_RELPREC(c1->mu, c2->mu)));
365 return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1);
367 ARB_GENERATE_STATIC(ctdth64, voistatdata_tdgstctd64, ctdlnk, ctd64cmp);
370 RB_GENERATE_STATIC(rbctdth32, voistatdata_tdgstctd32, rblnk, ctd32cmp);
371 RB_GENERATE_STATIC(rbctdth64, voistatdata_tdgstctd64, rblnk, ctd64cmp);
374 static inline sbintime_t
375 stats_sbinuptime(void)
381 #else /* ! _KERNEL */
384 clock_gettime(CLOCK_MONOTONIC_FAST, &tp);
392 stats_realloc(void *ptr, size_t oldsz, size_t newsz, int flags)
396 /* Default to M_NOWAIT if neither M_NOWAIT or M_WAITOK are set. */
397 if (!(flags & (M_WAITOK | M_NOWAIT)))
399 ptr = realloc(ptr, newsz, M_STATS, flags);
400 #else /* ! _KERNEL */
401 ptr = realloc(ptr, newsz);
402 if ((flags & M_ZERO) && ptr != NULL) {
404 memset(ptr, '\0', newsz);
405 else if (newsz > oldsz)
406 memset(BLOB_OFFSET(ptr, oldsz), '\0', newsz - oldsz);
414 stats_strdup(const char *s,
421 if (!(flags & (M_WAITOK | M_NOWAIT)))
425 if ((copy = malloc(len, M_STATS, flags)) != NULL)
437 stats_tpl_update_hash(struct statsblob_tpl *tpl)
440 TPL_LIST_WLOCK_ASSERT();
441 tpl->mb->tplhash = hash32_str(tpl->mb->tplname, 0);
442 for (int voi_id = 0; voi_id < NVOIS(tpl->sb); voi_id++) {
443 if (tpl->mb->voi_meta[voi_id].name != NULL)
444 tpl->mb->tplhash = hash32_str(
445 tpl->mb->voi_meta[voi_id].name, tpl->mb->tplhash);
447 tpl->mb->tplhash = hash32_buf(tpl->sb, tpl->sb->cursz,
451 static inline uint64_t
452 stats_pow_u64(uint64_t base, uint64_t exp)
467 stats_vss_hist_bkt_hlpr(struct vss_hist_hlpr_info *info, uint32_t curbkt,
468 struct voistatdata_numeric *bkt_lb, struct voistatdata_numeric *bkt_ub)
473 switch (info->scheme) {
475 step = info->lin.stepinc;
478 step = stats_pow_u64(info->exp.stepbase,
479 info->exp.stepexp + curbkt);
483 uint64_t curstepexp = 1;
485 switch (info->voi_dtype) {
486 case VSD_DTYPE_INT_S32:
487 while ((int32_t)stats_pow_u64(info->linexp.stepbase,
488 curstepexp) <= bkt_lb->int32.s32)
491 case VSD_DTYPE_INT_U32:
492 while ((uint32_t)stats_pow_u64(info->linexp.stepbase,
493 curstepexp) <= bkt_lb->int32.u32)
496 case VSD_DTYPE_INT_S64:
497 while ((int64_t)stats_pow_u64(info->linexp.stepbase,
498 curstepexp) <= bkt_lb->int64.s64)
501 case VSD_DTYPE_INT_U64:
502 while ((uint64_t)stats_pow_u64(info->linexp.stepbase,
503 curstepexp) <= bkt_lb->int64.u64)
506 case VSD_DTYPE_INT_SLONG:
507 while ((long)stats_pow_u64(info->linexp.stepbase,
508 curstepexp) <= bkt_lb->intlong.slong)
511 case VSD_DTYPE_INT_ULONG:
512 while ((unsigned long)stats_pow_u64(info->linexp.stepbase,
513 curstepexp) <= bkt_lb->intlong.ulong)
516 case VSD_DTYPE_Q_S32:
517 while ((s32q_t)stats_pow_u64(info->linexp.stepbase,
518 curstepexp) <= Q_GIVAL(bkt_lb->q32.sq32))
520 case VSD_DTYPE_Q_U32:
521 while ((u32q_t)stats_pow_u64(info->linexp.stepbase,
522 curstepexp) <= Q_GIVAL(bkt_lb->q32.uq32))
524 case VSD_DTYPE_Q_S64:
525 while ((s64q_t)stats_pow_u64(info->linexp.stepbase,
526 curstepexp) <= Q_GIVAL(bkt_lb->q64.sq64))
529 case VSD_DTYPE_Q_U64:
530 while ((u64q_t)stats_pow_u64(info->linexp.stepbase,
531 curstepexp) <= Q_GIVAL(bkt_lb->q64.uq64))
538 step = stats_pow_u64(info->linexp.stepbase, curstepexp) /
539 info->linexp.linstepdiv;
548 if (info->scheme == BKT_USR) {
549 *bkt_lb = info->usr.bkts[curbkt].lb;
550 *bkt_ub = info->usr.bkts[curbkt].ub;
551 } else if (step != 0) {
552 switch (info->voi_dtype) {
553 case VSD_DTYPE_INT_S32:
554 bkt_ub->int32.s32 += (int32_t)step;
556 case VSD_DTYPE_INT_U32:
557 bkt_ub->int32.u32 += (uint32_t)step;
559 case VSD_DTYPE_INT_S64:
560 bkt_ub->int64.s64 += (int64_t)step;
562 case VSD_DTYPE_INT_U64:
563 bkt_ub->int64.u64 += (uint64_t)step;
565 case VSD_DTYPE_INT_SLONG:
566 bkt_ub->intlong.slong += (long)step;
568 case VSD_DTYPE_INT_ULONG:
569 bkt_ub->intlong.ulong += (unsigned long)step;
571 case VSD_DTYPE_Q_S32:
572 error = Q_QADDI(&bkt_ub->q32.sq32, step);
574 case VSD_DTYPE_Q_U32:
575 error = Q_QADDI(&bkt_ub->q32.uq32, step);
577 case VSD_DTYPE_Q_S64:
578 error = Q_QADDI(&bkt_ub->q64.sq64, step);
580 case VSD_DTYPE_Q_U64:
581 error = Q_QADDI(&bkt_ub->q64.uq64, step);
586 } else { /* info->scheme != BKT_USR && step == 0 */
594 stats_vss_hist_nbkts_hlpr(struct vss_hist_hlpr_info *info)
596 struct voistatdata_numeric bkt_lb, bkt_ub;
600 if (info->scheme == BKT_USR) {
601 /* XXXLAS: Setting info->{lb,ub} from macro is tricky. */
602 info->lb = info->usr.bkts[0].lb;
603 info->ub = info->usr.bkts[info->usr.nbkts - 1].lb;
612 if (stats_vss_hist_bkt_hlpr(info, nbkts++, &bkt_lb, &bkt_ub))
615 if (info->scheme == BKT_USR)
616 done = (nbkts == info->usr.nbkts);
618 switch (info->voi_dtype) {
619 case VSD_DTYPE_INT_S32:
620 done = (bkt_ub.int32.s32 > info->ub.int32.s32);
622 case VSD_DTYPE_INT_U32:
623 done = (bkt_ub.int32.u32 > info->ub.int32.u32);
625 case VSD_DTYPE_INT_S64:
626 done = (bkt_ub.int64.s64 > info->ub.int64.s64);
628 case VSD_DTYPE_INT_U64:
629 done = (bkt_ub.int64.u64 > info->ub.int64.u64);
631 case VSD_DTYPE_INT_SLONG:
632 done = (bkt_ub.intlong.slong >
633 info->ub.intlong.slong);
635 case VSD_DTYPE_INT_ULONG:
636 done = (bkt_ub.intlong.ulong >
637 info->ub.intlong.ulong);
639 case VSD_DTYPE_Q_S32:
640 done = Q_QGTQ(bkt_ub.q32.sq32,
643 case VSD_DTYPE_Q_U32:
644 done = Q_QGTQ(bkt_ub.q32.uq32,
647 case VSD_DTYPE_Q_S64:
648 done = Q_QGTQ(bkt_ub.q64.sq64,
651 case VSD_DTYPE_Q_U64:
652 done = Q_QGTQ(bkt_ub.q64.uq64,
661 if (info->flags & VSD_HIST_LBOUND_INF)
663 if (info->flags & VSD_HIST_UBOUND_INF)
670 stats_vss_hist_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss,
671 struct vss_hist_hlpr_info *info)
673 struct voistatdata_hist *hist;
674 struct voistatdata_numeric bkt_lb, bkt_ub, *lbinfbktlb, *lbinfbktub,
675 *ubinfbktlb, *ubinfbktub;
676 uint32_t bkt, nbkts, nloop;
678 if (vss == NULL || info == NULL || (info->flags &
679 (VSD_HIST_LBOUND_INF|VSD_HIST_UBOUND_INF) && (info->hist_dtype ==
680 VSD_DTYPE_DVHIST32 || info->hist_dtype == VSD_DTYPE_DVHIST64)))
683 info->voi_dtype = voi_dtype;
685 if ((nbkts = stats_vss_hist_nbkts_hlpr(info)) == 0)
688 switch (info->hist_dtype) {
689 case VSD_DTYPE_CRHIST32:
690 vss->vsdsz = HIST_NBKTS2VSDSZ(crhist32, nbkts);
692 case VSD_DTYPE_DRHIST32:
693 vss->vsdsz = HIST_NBKTS2VSDSZ(drhist32, nbkts);
695 case VSD_DTYPE_DVHIST32:
696 vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist32, nbkts);
698 case VSD_DTYPE_CRHIST64:
699 vss->vsdsz = HIST_NBKTS2VSDSZ(crhist64, nbkts);
701 case VSD_DTYPE_DRHIST64:
702 vss->vsdsz = HIST_NBKTS2VSDSZ(drhist64, nbkts);
704 case VSD_DTYPE_DVHIST64:
705 vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist64, nbkts);
711 vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO);
715 hist = (struct voistatdata_hist *)vss->iv;
718 for (bkt = (info->flags & VSD_HIST_LBOUND_INF), nloop = 0;
722 if (stats_vss_hist_bkt_hlpr(info, nloop, &bkt_lb, &bkt_ub))
725 switch (info->hist_dtype) {
726 case VSD_DTYPE_CRHIST32:
727 VSD(crhist32, hist)->bkts[bkt].lb = bkt_lb;
729 case VSD_DTYPE_DRHIST32:
730 VSD(drhist32, hist)->bkts[bkt].lb = bkt_lb;
731 VSD(drhist32, hist)->bkts[bkt].ub = bkt_ub;
733 case VSD_DTYPE_DVHIST32:
734 VSD(dvhist32, hist)->bkts[bkt].val = bkt_lb;
736 case VSD_DTYPE_CRHIST64:
737 VSD(crhist64, hist)->bkts[bkt].lb = bkt_lb;
739 case VSD_DTYPE_DRHIST64:
740 VSD(drhist64, hist)->bkts[bkt].lb = bkt_lb;
741 VSD(drhist64, hist)->bkts[bkt].ub = bkt_ub;
743 case VSD_DTYPE_DVHIST64:
744 VSD(dvhist64, hist)->bkts[bkt].val = bkt_lb;
751 lbinfbktlb = lbinfbktub = ubinfbktlb = ubinfbktub = NULL;
753 switch (info->hist_dtype) {
754 case VSD_DTYPE_CRHIST32:
755 lbinfbktlb = &VSD(crhist32, hist)->bkts[0].lb;
756 ubinfbktlb = &VSD(crhist32, hist)->bkts[nbkts - 1].lb;
758 case VSD_DTYPE_DRHIST32:
759 lbinfbktlb = &VSD(drhist32, hist)->bkts[0].lb;
760 lbinfbktub = &VSD(drhist32, hist)->bkts[0].ub;
761 ubinfbktlb = &VSD(drhist32, hist)->bkts[nbkts - 1].lb;
762 ubinfbktub = &VSD(drhist32, hist)->bkts[nbkts - 1].ub;
764 case VSD_DTYPE_CRHIST64:
765 lbinfbktlb = &VSD(crhist64, hist)->bkts[0].lb;
766 ubinfbktlb = &VSD(crhist64, hist)->bkts[nbkts - 1].lb;
768 case VSD_DTYPE_DRHIST64:
769 lbinfbktlb = &VSD(drhist64, hist)->bkts[0].lb;
770 lbinfbktub = &VSD(drhist64, hist)->bkts[0].ub;
771 ubinfbktlb = &VSD(drhist64, hist)->bkts[nbkts - 1].lb;
772 ubinfbktub = &VSD(drhist64, hist)->bkts[nbkts - 1].ub;
774 case VSD_DTYPE_DVHIST32:
775 case VSD_DTYPE_DVHIST64:
781 if ((info->flags & VSD_HIST_LBOUND_INF) && lbinfbktlb) {
782 *lbinfbktlb = numeric_limits[LIM_MIN][info->voi_dtype];
784 * Assignment from numeric_limit array for Q types assigns max
785 * possible integral/fractional value for underlying data type,
786 * but we must set control bits for this specific histogram per
787 * the user's choice of fractional bits, which we extract from
790 if (info->voi_dtype == VSD_DTYPE_Q_S32 ||
791 info->voi_dtype == VSD_DTYPE_Q_U32) {
792 /* Signedness doesn't matter for setting control bits. */
793 Q_SCVAL(lbinfbktlb->q32.sq32,
794 Q_GCVAL(info->lb.q32.sq32));
795 } else if (info->voi_dtype == VSD_DTYPE_Q_S64 ||
796 info->voi_dtype == VSD_DTYPE_Q_U64) {
797 /* Signedness doesn't matter for setting control bits. */
798 Q_SCVAL(lbinfbktlb->q64.sq64,
799 Q_GCVAL(info->lb.q64.sq64));
802 *lbinfbktub = info->lb;
804 if ((info->flags & VSD_HIST_UBOUND_INF) && ubinfbktlb) {
805 *ubinfbktlb = bkt_lb;
807 *ubinfbktub = numeric_limits[LIM_MAX][info->voi_dtype];
808 if (info->voi_dtype == VSD_DTYPE_Q_S32 ||
809 info->voi_dtype == VSD_DTYPE_Q_U32) {
810 Q_SCVAL(ubinfbktub->q32.sq32,
811 Q_GCVAL(info->lb.q32.sq32));
812 } else if (info->voi_dtype == VSD_DTYPE_Q_S64 ||
813 info->voi_dtype == VSD_DTYPE_Q_U64) {
814 Q_SCVAL(ubinfbktub->q64.sq64,
815 Q_GCVAL(info->lb.q64.sq64));
824 stats_vss_tdgst_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss,
825 struct vss_tdgst_hlpr_info *info)
827 struct voistatdata_tdgst *tdgst;
828 struct ctdth32 *ctd32tree;
829 struct ctdth64 *ctd64tree;
830 struct voistatdata_tdgstctd32 *ctd32;
831 struct voistatdata_tdgstctd64 *ctd64;
833 info->voi_dtype = voi_dtype;
835 switch (info->tdgst_dtype) {
836 case VSD_DTYPE_TDGSTCLUST32:
837 vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust32, info->nctds);
839 case VSD_DTYPE_TDGSTCLUST64:
840 vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust64, info->nctds);
846 vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO);
850 tdgst = (struct voistatdata_tdgst *)vss->iv;
852 switch (info->tdgst_dtype) {
853 case VSD_DTYPE_TDGSTCLUST32:
854 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
855 ARB_INIT(ctd32, ctdlnk, ctd32tree, info->nctds) {
856 Q_INI(&ctd32->mu, 0, 0, info->prec);
859 case VSD_DTYPE_TDGSTCLUST64:
860 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
861 ARB_INIT(ctd64, ctdlnk, ctd64tree, info->nctds) {
862 Q_INI(&ctd64->mu, 0, 0, info->prec);
873 stats_vss_numeric_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss,
874 struct vss_numeric_hlpr_info *info)
876 struct voistatdata_numeric iv;
878 switch (vss->stype) {
880 iv = stats_ctor_vsd_numeric(0);
883 iv = numeric_limits[LIM_MAX][voi_dtype];
886 iv = numeric_limits[LIM_MIN][voi_dtype];
892 vss->iv = stats_realloc(NULL, 0, vsd_dtype2size[voi_dtype], 0);
896 vss->vs_dtype = voi_dtype;
897 vss->vsdsz = vsd_dtype2size[voi_dtype];
899 case VSD_DTYPE_INT_S32:
900 *((int32_t *)vss->iv) = iv.int32.s32;
902 case VSD_DTYPE_INT_U32:
903 *((uint32_t *)vss->iv) = iv.int32.u32;
905 case VSD_DTYPE_INT_S64:
906 *((int64_t *)vss->iv) = iv.int64.s64;
908 case VSD_DTYPE_INT_U64:
909 *((uint64_t *)vss->iv) = iv.int64.u64;
911 case VSD_DTYPE_INT_SLONG:
912 *((long *)vss->iv) = iv.intlong.slong;
914 case VSD_DTYPE_INT_ULONG:
915 *((unsigned long *)vss->iv) = iv.intlong.ulong;
917 case VSD_DTYPE_Q_S32:
918 *((s32q_t *)vss->iv) = Q_SCVAL(iv.q32.sq32,
919 Q_CTRLINI(info->prec));
921 case VSD_DTYPE_Q_U32:
922 *((u32q_t *)vss->iv) = Q_SCVAL(iv.q32.uq32,
923 Q_CTRLINI(info->prec));
925 case VSD_DTYPE_Q_S64:
926 *((s64q_t *)vss->iv) = Q_SCVAL(iv.q64.sq64,
927 Q_CTRLINI(info->prec));
929 case VSD_DTYPE_Q_U64:
930 *((u64q_t *)vss->iv) = Q_SCVAL(iv.q64.uq64,
931 Q_CTRLINI(info->prec));
941 stats_vss_hlpr_init(enum vsd_dtype voi_dtype, uint32_t nvss,
942 struct voistatspec *vss)
946 for (i = nvss - 1; i >= 0; i--) {
947 if (vss[i].hlpr && (ret = vss[i].hlpr(voi_dtype, &vss[i],
948 vss[i].hlprinfo)) != 0)
956 stats_vss_hlpr_cleanup(uint32_t nvss, struct voistatspec *vss)
960 for (i = nvss - 1; i >= 0; i--) {
962 stats_free((void *)vss[i].iv);
969 stats_tpl_fetch(int tpl_id, struct statsblob_tpl **tpl)
976 if (tpl_id < 0 || tpl_id >= (int)ntpl) {
979 *tpl = tpllist[tpl_id];
980 /* XXXLAS: Acquire refcount on tpl. */
988 stats_tpl_fetch_allocid(const char *name, uint32_t hash)
995 for (i = ntpl - 1; i >= 0; i--) {
997 if (strlen(name) == strlen(tpllist[i]->mb->tplname) &&
998 strncmp(name, tpllist[i]->mb->tplname,
999 TPL_MAX_NAME_LEN) == 0 && (!hash || hash ==
1000 tpllist[i]->mb->tplhash)) {
1004 } else if (hash == tpllist[i]->mb->tplhash) {
1015 stats_tpl_id2name(uint32_t tpl_id, char *buf, size_t len)
1022 if (tpl_id < ntpl) {
1023 if (buf != NULL && len > strlen(tpllist[tpl_id]->mb->tplname))
1024 strlcpy(buf, tpllist[tpl_id]->mb->tplname, len);
1035 stats_tpl_sample_rollthedice(struct stats_tpl_sample_rate *rates, int nrates,
1036 void *seed_bytes, size_t seed_len)
1038 uint32_t cum_pct, rnd_pct;
1044 * Choose a pseudorandom or seeded number in range [0,100] and use
1045 * it to make a sampling decision and template selection where required.
1046 * If no seed is supplied, a PRNG is used to generate a pseudorandom
1047 * number so that every selection is independent. If a seed is supplied,
1048 * the caller desires random selection across different seeds, but
1049 * deterministic selection given the same seed. This is achieved by
1050 * hashing the seed and using the hash as the random number source.
1052 * XXXLAS: Characterise hash function output distribution.
1054 if (seed_bytes == NULL)
1055 rnd_pct = random() / (INT32_MAX / 100);
1057 rnd_pct = hash32_buf(seed_bytes, seed_len, 0) /
1058 (UINT32_MAX / 100U);
1061 * We map the randomly selected percentage on to the interval [0,100]
1062 * consisting of the cumulatively summed template sampling percentages.
1063 * The difference between the cumulative sum of all template sampling
1064 * percentages and 100 is treated as a NULL assignment i.e. no stats
1065 * template will be assigned, and -1 returned instead.
1067 for (i = 0; i < nrates; i++) {
1068 cum_pct += rates[i].tpl_sample_pct;
1070 KASSERT(cum_pct <= 100, ("%s cum_pct %u > 100", __func__,
1072 if (rnd_pct > cum_pct || rates[i].tpl_sample_pct == 0)
1075 return (rates[i].tpl_slot_id);
1082 stats_v1_blob_clone(struct statsblobv1 **dst, size_t dstmaxsz,
1083 struct statsblobv1 *src, uint32_t flags)
1089 if (src == NULL || dst == NULL ||
1090 src->cursz < sizeof(struct statsblob) ||
1091 ((flags & SB_CLONE_ALLOCDST) &&
1092 (flags & (SB_CLONE_USRDSTNOFAULT | SB_CLONE_USRDST)))) {
1094 } else if (flags & SB_CLONE_ALLOCDST) {
1095 *dst = stats_realloc(NULL, 0, src->cursz, 0);
1097 (*dst)->maxsz = dstmaxsz = src->cursz;
1100 } else if (*dst == NULL || dstmaxsz < sizeof(struct statsblob)) {
1105 size_t postcurszlen;
1108 * Clone src into dst except for the maxsz field. If dst is too
1109 * small to hold all of src, only copy src's header and return
1113 if (flags & SB_CLONE_USRDSTNOFAULT)
1114 copyout_nofault(src, *dst,
1115 offsetof(struct statsblob, maxsz));
1116 else if (flags & SB_CLONE_USRDST)
1117 copyout(src, *dst, offsetof(struct statsblob, maxsz));
1120 memcpy(*dst, src, offsetof(struct statsblob, maxsz));
1122 if (dstmaxsz >= src->cursz) {
1123 postcurszlen = src->cursz -
1124 offsetof(struct statsblob, cursz);
1127 postcurszlen = sizeof(struct statsblob) -
1128 offsetof(struct statsblob, cursz);
1131 if (flags & SB_CLONE_USRDSTNOFAULT)
1132 copyout_nofault(&(src->cursz), &((*dst)->cursz),
1134 else if (flags & SB_CLONE_USRDST)
1135 copyout(&(src->cursz), &((*dst)->cursz), postcurszlen);
1138 memcpy(&((*dst)->cursz), &(src->cursz), postcurszlen);
1145 stats_v1_tpl_alloc(const char *name, uint32_t flags __unused)
1147 struct statsblobv1_tpl *tpl, **newtpllist;
1148 struct statsblobv1 *tpl_sb;
1149 struct metablob *tpl_mb;
1152 if (name != NULL && strlen(name) > TPL_MAX_NAME_LEN)
1155 if (name != NULL && stats_tpl_fetch_allocid(name, 0) >= 0)
1158 tpl = stats_realloc(NULL, 0, sizeof(struct statsblobv1_tpl), M_ZERO);
1159 tpl_mb = stats_realloc(NULL, 0, sizeof(struct metablob), M_ZERO);
1160 tpl_sb = stats_realloc(NULL, 0, sizeof(struct statsblobv1), M_ZERO);
1162 if (tpl_mb != NULL && name != NULL)
1163 tpl_mb->tplname = stats_strdup(name, 0);
1165 if (tpl == NULL || tpl_sb == NULL || tpl_mb == NULL ||
1166 tpl_mb->tplname == NULL) {
1169 if (tpl_mb != NULL) {
1170 stats_free(tpl_mb->tplname);
1179 tpl_sb->abi = STATS_ABI_V1;
1181 #if BYTE_ORDER == LITTLE_ENDIAN
1183 #elif BYTE_ORDER == BIG_ENDIAN
1188 tpl_sb->cursz = tpl_sb->maxsz = sizeof(struct statsblobv1);
1189 tpl_sb->stats_off = tpl_sb->statsdata_off = sizeof(struct statsblobv1);
1192 newtpllist = stats_realloc(tpllist, ntpl * sizeof(void *),
1193 (ntpl + 1) * sizeof(void *), 0);
1194 if (newtpllist != NULL) {
1196 tpllist = (struct statsblob_tpl **)newtpllist;
1197 tpllist[tpl_id] = (struct statsblob_tpl *)tpl;
1198 stats_tpl_update_hash(tpllist[tpl_id]);
1202 if (tpl_mb != NULL) {
1203 stats_free(tpl_mb->tplname);
1214 stats_v1_tpl_add_voistats(uint32_t tpl_id, int32_t voi_id, const char *voi_name,
1215 enum vsd_dtype voi_dtype, uint32_t nvss, struct voistatspec *vss,
1219 struct voistat *tmpstat;
1220 struct statsblobv1 *tpl_sb;
1221 struct metablob *tpl_mb;
1222 int error, i, newstatdataidx, newvoibytes, newvoistatbytes,
1223 newvoistatdatabytes, newvoistatmaxid;
1226 if (voi_id < 0 || voi_dtype == 0 || voi_dtype >= VSD_NUM_DTYPES ||
1227 nvss == 0 || vss == NULL)
1230 error = nbytes = newvoibytes = newvoistatbytes =
1231 newvoistatdatabytes = 0;
1232 newvoistatmaxid = -1;
1234 /* Calculate the number of bytes required for the new voistats. */
1235 for (i = nvss - 1; i >= 0; i--) {
1236 if (vss[i].stype == 0 || vss[i].stype >= VS_NUM_STYPES ||
1237 vss[i].vs_dtype == 0 || vss[i].vs_dtype >= VSD_NUM_DTYPES ||
1238 vss[i].iv == NULL || vss[i].vsdsz == 0)
1240 if ((int)vss[i].stype > newvoistatmaxid)
1241 newvoistatmaxid = vss[i].stype;
1242 newvoistatdatabytes += vss[i].vsdsz;
1245 if (flags & SB_VOI_RELUPDATE) {
1246 /* XXXLAS: VOI state bytes may need to vary based on stat types. */
1247 newvoistatdatabytes += sizeof(struct voistatdata_voistate);
1249 nbytes += newvoistatdatabytes;
1252 if (tpl_id < ntpl) {
1253 tpl_sb = (struct statsblobv1 *)tpllist[tpl_id]->sb;
1254 tpl_mb = tpllist[tpl_id]->mb;
1256 if (voi_id >= NVOIS(tpl_sb) || tpl_sb->vois[voi_id].id == -1) {
1257 /* Adding a new VOI and associated stats. */
1258 if (voi_id >= NVOIS(tpl_sb)) {
1259 /* We need to grow the tpl_sb->vois array. */
1260 newvoibytes = (voi_id - (NVOIS(tpl_sb) - 1)) *
1262 nbytes += newvoibytes;
1265 (newvoistatmaxid + 1) * sizeof(struct voistat);
1267 /* Adding stats to an existing VOI. */
1268 if (newvoistatmaxid >
1269 tpl_sb->vois[voi_id].voistatmaxid) {
1270 newvoistatbytes = (newvoistatmaxid -
1271 tpl_sb->vois[voi_id].voistatmaxid) *
1272 sizeof(struct voistat);
1274 /* XXXLAS: KPI does not yet support expanding VOIs. */
1277 nbytes += newvoistatbytes;
1279 if (!error && newvoibytes > 0) {
1280 struct voi_meta *voi_meta = tpl_mb->voi_meta;
1282 voi_meta = stats_realloc(voi_meta, voi_meta == NULL ?
1283 0 : NVOIS(tpl_sb) * sizeof(struct voi_meta),
1284 (1 + voi_id) * sizeof(struct voi_meta),
1287 if (voi_meta == NULL)
1290 tpl_mb->voi_meta = voi_meta;
1294 /* NB: Resizing can change where tpl_sb points. */
1295 error = stats_v1_blob_expand(&tpl_sb, newvoibytes,
1296 newvoistatbytes, newvoistatdatabytes);
1300 tpl_mb->voi_meta[voi_id].name = stats_strdup(voi_name,
1302 if (tpl_mb->voi_meta[voi_id].name == NULL)
1307 /* Update the template list with the resized pointer. */
1308 tpllist[tpl_id]->sb = (struct statsblob *)tpl_sb;
1310 /* Update the template. */
1311 voi = &tpl_sb->vois[voi_id];
1314 /* VOI is new and needs to be initialised. */
1316 voi->dtype = voi_dtype;
1317 voi->stats_off = tpl_sb->stats_off;
1318 if (flags & SB_VOI_RELUPDATE)
1319 voi->flags |= VOI_REQSTATE;
1322 * XXXLAS: When this else block is written, the
1323 * "KPI does not yet support expanding VOIs"
1324 * error earlier in this function can be
1325 * removed. What is required here is to shuffle
1326 * the voistat array such that the new stats for
1327 * the voi are contiguous, which will displace
1328 * stats for other vois that reside after the
1329 * voi being updated. The other vois then need
1330 * to have their stats_off adjusted post
1335 voi->voistatmaxid = newvoistatmaxid;
1338 if (voi->flags & VOI_REQSTATE) {
1339 /* Initialise the voistate stat in slot 0. */
1340 tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off);
1341 tmpstat->stype = VS_STYPE_VOISTATE;
1343 tmpstat->dtype = VSD_DTYPE_VOISTATE;
1344 newstatdataidx = tmpstat->dsz =
1345 sizeof(struct voistatdata_numeric);
1346 tmpstat->data_off = tpl_sb->statsdata_off;
1349 for (i = 0; (uint32_t)i < nvss; i++) {
1350 tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off +
1351 (vss[i].stype * sizeof(struct voistat)));
1352 KASSERT(tmpstat->stype < 0, ("voistat %p "
1353 "already initialised", tmpstat));
1354 tmpstat->stype = vss[i].stype;
1355 tmpstat->flags = vss[i].flags;
1356 tmpstat->dtype = vss[i].vs_dtype;
1357 tmpstat->dsz = vss[i].vsdsz;
1358 tmpstat->data_off = tpl_sb->statsdata_off +
1360 memcpy(BLOB_OFFSET(tpl_sb, tmpstat->data_off),
1361 vss[i].iv, vss[i].vsdsz);
1362 newstatdataidx += vss[i].vsdsz;
1365 /* Update the template version hash. */
1366 stats_tpl_update_hash(tpllist[tpl_id]);
1367 /* XXXLAS: Confirm tpl name/hash pair remains unique. */
1376 struct statsblobv1 *
1377 stats_v1_blob_alloc(uint32_t tpl_id, uint32_t flags __unused)
1379 struct statsblobv1 *sb;
1385 if (tpl_id < ntpl) {
1386 sb = stats_realloc(NULL, 0, tpllist[tpl_id]->sb->maxsz, 0);
1388 sb->maxsz = tpllist[tpl_id]->sb->maxsz;
1389 error = stats_v1_blob_init_locked(sb, tpl_id, 0);
1404 stats_v1_blob_destroy(struct statsblobv1 *sb)
1411 stats_v1_voistat_fetch_dptr(struct statsblobv1 *sb, int32_t voi_id,
1412 enum voi_stype stype, enum vsd_dtype *retdtype, struct voistatdata **retvsd,
1418 if (retvsd == NULL || sb == NULL || sb->abi != STATS_ABI_V1 ||
1419 voi_id >= NVOIS(sb))
1422 v = &sb->vois[voi_id];
1423 if ((__typeof(v->voistatmaxid))stype > v->voistatmaxid)
1426 vs = BLOB_OFFSET(sb, v->stats_off + (stype * sizeof(struct voistat)));
1427 *retvsd = BLOB_OFFSET(sb, vs->data_off);
1428 if (retdtype != NULL)
1429 *retdtype = vs->dtype;
1430 if (retvsdsz != NULL)
1431 *retvsdsz = vs->dsz;
1437 stats_v1_blob_init(struct statsblobv1 *sb, uint32_t tpl_id, uint32_t flags)
1444 if (sb == NULL || tpl_id >= ntpl) {
1447 error = stats_v1_blob_init_locked(sb, tpl_id, flags);
1455 stats_v1_blob_init_locked(struct statsblobv1 *sb, uint32_t tpl_id,
1456 uint32_t flags __unused)
1460 TPL_LIST_RLOCK_ASSERT();
1461 error = (sb->maxsz >= tpllist[tpl_id]->sb->cursz) ? 0 : EOVERFLOW;
1463 ("sb %d instead of %d bytes", sb->maxsz, tpllist[tpl_id]->sb->cursz));
1466 memcpy(sb, tpllist[tpl_id]->sb, tpllist[tpl_id]->sb->cursz);
1467 sb->created = sb->lastrst = stats_sbinuptime();
1468 sb->tplhash = tpllist[tpl_id]->mb->tplhash;
1475 stats_v1_blob_expand(struct statsblobv1 **sbpp, int newvoibytes,
1476 int newvoistatbytes, int newvoistatdatabytes)
1478 struct statsblobv1 *sb;
1480 struct voistat *tmpvoistat, *voistat_array;
1481 int error, i, idxnewvois, idxnewvoistats, nbytes, nvoistats;
1483 KASSERT(newvoibytes % sizeof(struct voi) == 0,
1484 ("Bad newvoibytes %d", newvoibytes));
1485 KASSERT(newvoistatbytes % sizeof(struct voistat) == 0,
1486 ("Bad newvoistatbytes %d", newvoistatbytes));
1488 error = ((newvoibytes % sizeof(struct voi) == 0) &&
1489 (newvoistatbytes % sizeof(struct voistat) == 0)) ? 0 : EINVAL;
1491 nbytes = newvoibytes + newvoistatbytes + newvoistatdatabytes;
1494 * XXXLAS: Required until we gain support for flags which alter the
1495 * units of size/offset fields in key structs.
1497 if (!error && ((((int)sb->cursz) + nbytes) > SB_V1_MAXSZ))
1500 if (!error && (sb->cursz + nbytes > sb->maxsz)) {
1501 /* Need to expand our blob. */
1502 sb = stats_realloc(sb, sb->maxsz, sb->cursz + nbytes, M_ZERO);
1504 sb->maxsz = sb->cursz + nbytes;
1512 * Shuffle memory within the expanded blob working from the end
1513 * backwards, leaving gaps for the new voistat and voistatdata
1514 * structs at the beginning of their respective blob regions,
1515 * and for the new voi structs at the end of their blob region.
1517 memmove(BLOB_OFFSET(sb, sb->statsdata_off + nbytes),
1518 BLOB_OFFSET(sb, sb->statsdata_off),
1519 sb->cursz - sb->statsdata_off);
1520 memmove(BLOB_OFFSET(sb, sb->stats_off + newvoibytes +
1521 newvoistatbytes), BLOB_OFFSET(sb, sb->stats_off),
1522 sb->statsdata_off - sb->stats_off);
1524 /* First index of new voi/voistat structs to be initialised. */
1525 idxnewvois = NVOIS(sb);
1526 idxnewvoistats = (newvoistatbytes / sizeof(struct voistat)) - 1;
1528 /* Update housekeeping variables and offsets. */
1529 sb->cursz += nbytes;
1530 sb->stats_off += newvoibytes;
1531 sb->statsdata_off += newvoibytes + newvoistatbytes;
1533 /* XXXLAS: Zeroing not strictly needed but aids debugging. */
1534 memset(&sb->vois[idxnewvois], '\0', newvoibytes);
1535 memset(BLOB_OFFSET(sb, sb->stats_off), '\0',
1537 memset(BLOB_OFFSET(sb, sb->statsdata_off), '\0',
1538 newvoistatdatabytes);
1540 /* Initialise new voi array members and update offsets. */
1541 for (i = 0; i < NVOIS(sb); i++) {
1542 tmpvoi = &sb->vois[i];
1543 if (i >= idxnewvois) {
1544 tmpvoi->id = tmpvoi->voistatmaxid = -1;
1545 } else if (tmpvoi->id > -1) {
1546 tmpvoi->stats_off += newvoibytes +
1551 /* Initialise new voistat array members and update offsets. */
1552 nvoistats = (sb->statsdata_off - sb->stats_off) /
1553 sizeof(struct voistat);
1554 voistat_array = BLOB_OFFSET(sb, sb->stats_off);
1555 for (i = 0; i < nvoistats; i++) {
1556 tmpvoistat = &voistat_array[i];
1557 if (i <= idxnewvoistats) {
1558 tmpvoistat->stype = -1;
1559 } else if (tmpvoistat->stype > -1) {
1560 tmpvoistat->data_off += nbytes;
1569 stats_v1_blob_finalise(struct statsblobv1 *sb __unused)
1572 /* XXXLAS: Fill this in. */
1576 stats_v1_blob_iter(struct statsblobv1 *sb, stats_v1_blob_itercb_t icb,
1577 void *usrctx, uint32_t flags)
1581 struct sb_iter_ctx ctx;
1584 ctx.usrctx = usrctx;
1585 ctx.flags |= SB_IT_FIRST_CB;
1586 ctx.flags &= ~(SB_IT_FIRST_VOI | SB_IT_LAST_VOI | SB_IT_FIRST_VOISTAT |
1587 SB_IT_LAST_VOISTAT);
1590 for (i = 0; i < NVOIS(sb); i++) {
1594 ctx.flags |= SB_IT_FIRST_VOISTAT;
1597 ctx.flags |= SB_IT_FIRST_VOI;
1598 else if (i == (NVOIS(sb) - 1))
1599 ctx.flags |= SB_IT_LAST_VOI | SB_IT_LAST_CB;
1601 if (v->id < 0 && (flags & SB_IT_NULLVOI)) {
1602 if (icb(sb, v, NULL, &ctx))
1605 ctx.flags &= ~SB_IT_FIRST_CB;
1608 /* If NULL voi, v->voistatmaxid == -1 */
1609 for (j = 0; j <= v->voistatmaxid; j++) {
1610 vs = &((struct voistat *)BLOB_OFFSET(sb,
1612 if (vs->stype < 0 &&
1613 !(flags & SB_IT_NULLVOISTAT))
1616 if (j == v->voistatmaxid) {
1617 ctx.flags |= SB_IT_LAST_VOISTAT;
1618 if (i == (NVOIS(sb) - 1))
1622 ctx.flags &= ~SB_IT_LAST_CB;
1625 if (icb(sb, v, vs, &ctx))
1628 ctx.flags &= ~(SB_IT_FIRST_CB | SB_IT_FIRST_VOISTAT |
1629 SB_IT_LAST_VOISTAT);
1631 ctx.flags &= ~(SB_IT_FIRST_VOI | SB_IT_LAST_VOI);
1636 stats_voistatdata_tdgst_tostr(enum vsd_dtype voi_dtype __unused,
1637 const struct voistatdata_tdgst *tdgst, enum vsd_dtype tdgst_dtype,
1638 size_t tdgst_dsz __unused, enum sb_str_fmt fmt, struct sbuf *buf, int objdump)
1640 const struct ctdth32 *ctd32tree;
1641 const struct ctdth64 *ctd64tree;
1642 const struct voistatdata_tdgstctd32 *ctd32;
1643 const struct voistatdata_tdgstctd64 *ctd64;
1645 uint64_t smplcnt, compcnt;
1646 int is32bit, qmaxstrlen;
1647 uint16_t maxctds, curctds;
1649 switch (tdgst_dtype) {
1650 case VSD_DTYPE_TDGSTCLUST32:
1651 smplcnt = CONSTVSD(tdgstclust32, tdgst)->smplcnt;
1652 compcnt = CONSTVSD(tdgstclust32, tdgst)->compcnt;
1653 maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree);
1654 curctds = ARB_CURNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree);
1655 ctd32tree = &CONSTVSD(tdgstclust32, tdgst)->ctdtree;
1656 ctd32 = (objdump ? ARB_CNODE(ctd32tree, 0) :
1657 ARB_CMIN(ctdth32, ctd32tree));
1658 qmaxstrlen = (ctd32 == NULL) ? 1 : Q_MAXSTRLEN(ctd32->mu, 10);
1663 case VSD_DTYPE_TDGSTCLUST64:
1664 smplcnt = CONSTVSD(tdgstclust64, tdgst)->smplcnt;
1665 compcnt = CONSTVSD(tdgstclust64, tdgst)->compcnt;
1666 maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree);
1667 curctds = ARB_CURNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree);
1668 ctd64tree = &CONSTVSD(tdgstclust64, tdgst)->ctdtree;
1669 ctd64 = (objdump ? ARB_CNODE(ctd64tree, 0) :
1670 ARB_CMIN(ctdth64, ctd64tree));
1671 qmaxstrlen = (ctd64 == NULL) ? 1 : Q_MAXSTRLEN(ctd64->mu, 10);
1681 case SB_STRFMT_FREEFORM:
1682 fmtstr = "smplcnt=%ju, compcnt=%ju, maxctds=%hu, nctds=%hu";
1684 case SB_STRFMT_JSON:
1687 "\"smplcnt\":%ju,\"compcnt\":%ju,\"maxctds\":%hu,"
1688 "\"nctds\":%hu,\"ctds\":[";
1691 sbuf_printf(buf, fmtstr, (uintmax_t)smplcnt, (uintmax_t)compcnt,
1694 while ((is32bit ? NULL != ctd32 : NULL != ctd64)) {
1695 char qstr[qmaxstrlen];
1698 case SB_STRFMT_FREEFORM:
1699 fmtstr = "\n\t\t\t\t";
1701 case SB_STRFMT_JSON:
1706 sbuf_cat(buf, fmtstr);
1710 case SB_STRFMT_FREEFORM:
1711 fmtstr = "ctd[%hu].";
1713 case SB_STRFMT_JSON:
1715 fmtstr = "\"ctd\":%hu,";
1718 sbuf_printf(buf, fmtstr, is32bit ?
1719 ARB_SELFIDX(ctd32tree, ctd32) :
1720 ARB_SELFIDX(ctd64tree, ctd64));
1724 case SB_STRFMT_FREEFORM:
1727 case SB_STRFMT_JSON:
1732 sbuf_cat(buf, fmtstr);
1733 Q_TOSTR((is32bit ? ctd32->mu : ctd64->mu), -1, 10, qstr,
1735 sbuf_cat(buf, qstr);
1739 case SB_STRFMT_FREEFORM:
1740 fmtstr = is32bit ? ",cnt=%u}" : ",cnt=%ju}";
1742 case SB_STRFMT_JSON:
1744 fmtstr = is32bit ? ",\"cnt\":%u}" : ",\"cnt\":%ju}";
1747 sbuf_printf(buf, fmtstr,
1748 is32bit ? ctd32->cnt : (uintmax_t)ctd64->cnt);
1751 ctd32 = (objdump ? ARB_CNODE(ctd32tree,
1752 ARB_SELFIDX(ctd32tree, ctd32) + 1) :
1753 ARB_CNEXT(ctdth32, ctd32tree, ctd32));
1755 ctd64 = (objdump ? ARB_CNODE(ctd64tree,
1756 ARB_SELFIDX(ctd64tree, ctd64) + 1) :
1757 ARB_CNEXT(ctdth64, ctd64tree, ctd64));
1759 if (fmt == SB_STRFMT_JSON &&
1760 (is32bit ? NULL != ctd32 : NULL != ctd64))
1761 sbuf_putc(buf, ',');
1763 if (fmt == SB_STRFMT_JSON)
1768 stats_voistatdata_hist_tostr(enum vsd_dtype voi_dtype,
1769 const struct voistatdata_hist *hist, enum vsd_dtype hist_dtype,
1770 size_t hist_dsz, enum sb_str_fmt fmt, struct sbuf *buf, int objdump)
1772 const struct voistatdata_numeric *bkt_lb, *bkt_ub;
1777 switch (hist_dtype) {
1778 case VSD_DTYPE_CRHIST32:
1779 nbkts = HIST_VSDSZ2NBKTS(crhist32, hist_dsz);
1782 case VSD_DTYPE_DRHIST32:
1783 nbkts = HIST_VSDSZ2NBKTS(drhist32, hist_dsz);
1786 case VSD_DTYPE_DVHIST32:
1787 nbkts = HIST_VSDSZ2NBKTS(dvhist32, hist_dsz);
1790 case VSD_DTYPE_CRHIST64:
1791 nbkts = HIST_VSDSZ2NBKTS(crhist64, hist_dsz);
1794 case VSD_DTYPE_DRHIST64:
1795 nbkts = HIST_VSDSZ2NBKTS(drhist64, hist_dsz);
1798 case VSD_DTYPE_DVHIST64:
1799 nbkts = HIST_VSDSZ2NBKTS(dvhist64, hist_dsz);
1807 case SB_STRFMT_FREEFORM:
1808 fmtstr = "nbkts=%hu, ";
1810 case SB_STRFMT_JSON:
1812 fmtstr = "\"nbkts\":%hu,";
1815 sbuf_printf(buf, fmtstr, nbkts);
1818 case SB_STRFMT_FREEFORM:
1819 fmtstr = (is32bit ? "oob=%u" : "oob=%ju");
1821 case SB_STRFMT_JSON:
1823 fmtstr = (is32bit ? "\"oob\":%u,\"bkts\":[" :
1824 "\"oob\":%ju,\"bkts\":[");
1827 sbuf_printf(buf, fmtstr, is32bit ? VSD_CONSTHIST_FIELDVAL(hist,
1828 hist_dtype, oob) : (uintmax_t)VSD_CONSTHIST_FIELDVAL(hist,
1831 for (i = 0; i < nbkts; i++) {
1832 switch (hist_dtype) {
1833 case VSD_DTYPE_CRHIST32:
1834 case VSD_DTYPE_CRHIST64:
1835 bkt_lb = VSD_CONSTCRHIST_FIELDPTR(hist, hist_dtype,
1838 bkt_ub = VSD_CONSTCRHIST_FIELDPTR(hist,
1839 hist_dtype, bkts[i + 1].lb);
1841 bkt_ub = &numeric_limits[LIM_MAX][voi_dtype];
1843 case VSD_DTYPE_DRHIST32:
1844 case VSD_DTYPE_DRHIST64:
1845 bkt_lb = VSD_CONSTDRHIST_FIELDPTR(hist, hist_dtype,
1847 bkt_ub = VSD_CONSTDRHIST_FIELDPTR(hist, hist_dtype,
1850 case VSD_DTYPE_DVHIST32:
1851 case VSD_DTYPE_DVHIST64:
1852 bkt_lb = bkt_ub = VSD_CONSTDVHIST_FIELDPTR(hist,
1853 hist_dtype, bkts[i].val);
1860 case SB_STRFMT_FREEFORM:
1861 fmtstr = "\n\t\t\t\t";
1863 case SB_STRFMT_JSON:
1868 sbuf_cat(buf, fmtstr);
1872 case SB_STRFMT_FREEFORM:
1873 fmtstr = "bkt[%hu].";
1875 case SB_STRFMT_JSON:
1877 fmtstr = "\"bkt\":%hu,";
1880 sbuf_printf(buf, fmtstr, i);
1884 case SB_STRFMT_FREEFORM:
1887 case SB_STRFMT_JSON:
1892 sbuf_cat(buf, fmtstr);
1893 stats_voistatdata_tostr((const struct voistatdata *)bkt_lb,
1894 voi_dtype, voi_dtype, sizeof(struct voistatdata_numeric),
1898 case SB_STRFMT_FREEFORM:
1901 case SB_STRFMT_JSON:
1903 fmtstr = ",\"ub\":";
1906 sbuf_cat(buf, fmtstr);
1907 stats_voistatdata_tostr((const struct voistatdata *)bkt_ub,
1908 voi_dtype, voi_dtype, sizeof(struct voistatdata_numeric),
1912 case SB_STRFMT_FREEFORM:
1913 fmtstr = is32bit ? ",cnt=%u}" : ",cnt=%ju}";
1915 case SB_STRFMT_JSON:
1917 fmtstr = is32bit ? ",\"cnt\":%u}" : ",\"cnt\":%ju}";
1920 sbuf_printf(buf, fmtstr, is32bit ?
1921 VSD_CONSTHIST_FIELDVAL(hist, hist_dtype, bkts[i].cnt) :
1922 (uintmax_t)VSD_CONSTHIST_FIELDVAL(hist, hist_dtype,
1925 if (fmt == SB_STRFMT_JSON && i < nbkts - 1)
1926 sbuf_putc(buf, ',');
1928 if (fmt == SB_STRFMT_JSON)
1933 stats_voistatdata_tostr(const struct voistatdata *vsd, enum vsd_dtype voi_dtype,
1934 enum vsd_dtype vsd_dtype, size_t vsd_sz, enum sb_str_fmt fmt,
1935 struct sbuf *buf, int objdump)
1939 if (vsd == NULL || buf == NULL || voi_dtype >= VSD_NUM_DTYPES ||
1940 vsd_dtype >= VSD_NUM_DTYPES || fmt >= SB_STRFMT_NUM_FMTS)
1943 switch (vsd_dtype) {
1944 case VSD_DTYPE_VOISTATE:
1946 case SB_STRFMT_FREEFORM:
1949 case SB_STRFMT_JSON:
1951 fmtstr = "\"prev\":";
1954 sbuf_cat(buf, fmtstr);
1956 * Render prev by passing it as *vsd and voi_dtype as vsd_dtype.
1958 stats_voistatdata_tostr(
1959 (const struct voistatdata *)&CONSTVSD(voistate, vsd)->prev,
1960 voi_dtype, voi_dtype, vsd_sz, fmt, buf, objdump);
1962 case VSD_DTYPE_INT_S32:
1963 sbuf_printf(buf, "%d", vsd->int32.s32);
1965 case VSD_DTYPE_INT_U32:
1966 sbuf_printf(buf, "%u", vsd->int32.u32);
1968 case VSD_DTYPE_INT_S64:
1969 sbuf_printf(buf, "%jd", (intmax_t)vsd->int64.s64);
1971 case VSD_DTYPE_INT_U64:
1972 sbuf_printf(buf, "%ju", (uintmax_t)vsd->int64.u64);
1974 case VSD_DTYPE_INT_SLONG:
1975 sbuf_printf(buf, "%ld", vsd->intlong.slong);
1977 case VSD_DTYPE_INT_ULONG:
1978 sbuf_printf(buf, "%lu", vsd->intlong.ulong);
1980 case VSD_DTYPE_Q_S32:
1982 char qstr[Q_MAXSTRLEN(vsd->q32.sq32, 10)];
1983 Q_TOSTR((s32q_t)vsd->q32.sq32, -1, 10, qstr, sizeof(qstr));
1984 sbuf_cat(buf, qstr);
1987 case VSD_DTYPE_Q_U32:
1989 char qstr[Q_MAXSTRLEN(vsd->q32.uq32, 10)];
1990 Q_TOSTR((u32q_t)vsd->q32.uq32, -1, 10, qstr, sizeof(qstr));
1991 sbuf_cat(buf, qstr);
1994 case VSD_DTYPE_Q_S64:
1996 char qstr[Q_MAXSTRLEN(vsd->q64.sq64, 10)];
1997 Q_TOSTR((s64q_t)vsd->q64.sq64, -1, 10, qstr, sizeof(qstr));
1998 sbuf_cat(buf, qstr);
2001 case VSD_DTYPE_Q_U64:
2003 char qstr[Q_MAXSTRLEN(vsd->q64.uq64, 10)];
2004 Q_TOSTR((u64q_t)vsd->q64.uq64, -1, 10, qstr, sizeof(qstr));
2005 sbuf_cat(buf, qstr);
2008 case VSD_DTYPE_CRHIST32:
2009 case VSD_DTYPE_DRHIST32:
2010 case VSD_DTYPE_DVHIST32:
2011 case VSD_DTYPE_CRHIST64:
2012 case VSD_DTYPE_DRHIST64:
2013 case VSD_DTYPE_DVHIST64:
2014 stats_voistatdata_hist_tostr(voi_dtype, CONSTVSD(hist, vsd),
2015 vsd_dtype, vsd_sz, fmt, buf, objdump);
2017 case VSD_DTYPE_TDGSTCLUST32:
2018 case VSD_DTYPE_TDGSTCLUST64:
2019 stats_voistatdata_tdgst_tostr(voi_dtype,
2020 CONSTVSD(tdgst, vsd), vsd_dtype, vsd_sz, fmt, buf,
2027 return (sbuf_error(buf));
2031 stats_v1_itercb_tostr_freeform(struct statsblobv1 *sb, struct voi *v,
2032 struct voistat *vs, struct sb_iter_ctx *ctx)
2034 struct sb_tostrcb_ctx *sctx;
2035 struct metablob *tpl_mb;
2042 tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL;
2043 dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0);
2045 if (ctx->flags & SB_IT_FIRST_CB) {
2046 sbuf_printf(buf, "struct statsblobv1@%p", sb);
2048 sbuf_printf(buf, ", abi=%hhu, endian=%hhu, maxsz=%hu, "
2049 "cursz=%hu, created=%jd, lastrst=%jd, flags=0x%04hx, "
2050 "stats_off=%hu, statsdata_off=%hu",
2051 sb->abi, sb->endian, sb->maxsz, sb->cursz,
2052 sb->created, sb->lastrst, sb->flags, sb->stats_off,
2055 sbuf_printf(buf, ", tplhash=%u", sb->tplhash);
2058 if (ctx->flags & SB_IT_FIRST_VOISTAT) {
2059 sbuf_printf(buf, "\n\tvois[%hd]: id=%hd", ctx->vslot, v->id);
2062 sbuf_printf(buf, ", name=\"%s\"", (tpl_mb == NULL) ? "" :
2063 tpl_mb->voi_meta[v->id].name);
2065 sbuf_printf(buf, ", flags=0x%04hx, dtype=%s, "
2066 "voistatmaxid=%hhd, stats_off=%hu", v->flags,
2067 vsd_dtype2name[v->dtype], v->voistatmaxid, v->stats_off);
2070 if (!dump && vs->stype <= 0)
2073 sbuf_printf(buf, "\n\t\tvois[%hd]stat[%hhd]: stype=", v->id, ctx->vsslot);
2074 if (vs->stype < 0) {
2075 sbuf_printf(buf, "%hhd", vs->stype);
2078 sbuf_printf(buf, "%s, errs=%hu", vs_stype2name[vs->stype],
2080 vsd = BLOB_OFFSET(sb, vs->data_off);
2082 sbuf_printf(buf, ", flags=0x%04x, dtype=%s, dsz=%hu, "
2083 "data_off=%hu", vs->flags, vsd_dtype2name[vs->dtype],
2084 vs->dsz, vs->data_off);
2086 sbuf_printf(buf, "\n\t\t\tvoistatdata: ");
2087 stats_voistatdata_tostr(vsd, v->dtype, vs->dtype, vs->dsz,
2088 sctx->fmt, buf, dump);
2092 stats_v1_itercb_tostr_json(struct statsblobv1 *sb, struct voi *v, struct voistat *vs,
2093 struct sb_iter_ctx *ctx)
2095 struct sb_tostrcb_ctx *sctx;
2096 struct metablob *tpl_mb;
2104 tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL;
2105 dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0);
2107 if (ctx->flags & SB_IT_FIRST_CB) {
2108 sbuf_putc(buf, '{');
2110 sbuf_printf(buf, "\"abi\":%hhu,\"endian\":%hhu,"
2111 "\"maxsz\":%hu,\"cursz\":%hu,\"created\":%jd,"
2112 "\"lastrst\":%jd,\"flags\":%hu,\"stats_off\":%hu,"
2113 "\"statsdata_off\":%hu,", sb->abi,
2114 sb->endian, sb->maxsz, sb->cursz, sb->created,
2115 sb->lastrst, sb->flags, sb->stats_off,
2120 fmtstr = "\"tplname\":%s,\"tplhash\":%u,\"vois\":{";
2122 fmtstr = "\"tplname\":\"%s\",\"tplhash\":%u,\"vois\":{";
2124 sbuf_printf(buf, fmtstr, tpl_mb ? tpl_mb->tplname : "null",
2128 if (ctx->flags & SB_IT_FIRST_VOISTAT) {
2130 sbuf_printf(buf, "\"[%d]\":{\"id\":%d", ctx->vslot,
2133 sbuf_printf(buf, "},");
2138 fmtstr = ",\"name\":%s,\"flags\":%hu,"
2139 "\"dtype\":\"%s\",\"voistatmaxid\":%hhd,"
2140 "\"stats_off\":%hu,";
2142 fmtstr = ",\"name\":\"%s\",\"flags\":%hu,"
2143 "\"dtype\":\"%s\",\"voistatmaxid\":%hhd,"
2144 "\"stats_off\":%hu,";
2146 sbuf_printf(buf, fmtstr, tpl_mb ?
2147 tpl_mb->voi_meta[v->id].name : "null", v->flags,
2148 vsd_dtype2name[v->dtype], v->voistatmaxid,
2151 if (tpl_mb == NULL) {
2152 sbuf_printf(buf, "\"[%hd]\":{", v->id);
2154 sbuf_printf(buf, "\"%s\":{",
2155 tpl_mb->voi_meta[v->id].name);
2158 sbuf_cat(buf, "\"stats\":{");
2161 vsd = BLOB_OFFSET(sb, vs->data_off);
2163 sbuf_printf(buf, "\"[%hhd]\":", ctx->vsslot);
2164 if (vs->stype < 0) {
2165 sbuf_printf(buf, "{\"stype\":-1},");
2168 sbuf_printf(buf, "{\"stype\":\"%s\",\"errs\":%hu,\"flags\":%hu,"
2169 "\"dtype\":\"%s\",\"data_off\":%hu,\"voistatdata\":{",
2170 vs_stype2name[vs->stype], vs->errs, vs->flags,
2171 vsd_dtype2name[vs->dtype], vs->data_off);
2172 } else if (vs->stype > 0) {
2174 sbuf_printf(buf, "\"[%hhd]\":", vs->stype);
2176 sbuf_printf(buf, "\"%s\":", vs_stype2name[vs->stype]);
2180 if ((vs->flags & VS_VSDVALID) || dump) {
2182 sbuf_printf(buf, "{\"errs\":%hu,", vs->errs);
2183 /* Simple non-compound VSD types need a key. */
2184 if (!vsd_compoundtype[vs->dtype])
2185 sbuf_cat(buf, "\"val\":");
2186 stats_voistatdata_tostr(vsd, v->dtype, vs->dtype, vs->dsz,
2187 sctx->fmt, buf, dump);
2188 sbuf_cat(buf, dump ? "}}" : "}");
2190 sbuf_cat(buf, dump ? "null}" : "null");
2192 if (ctx->flags & SB_IT_LAST_VOISTAT)
2193 sbuf_cat(buf, "}}");
2195 if (ctx->flags & SB_IT_LAST_CB)
2196 sbuf_cat(buf, "}}");
2198 sbuf_putc(buf, ',');
2202 stats_v1_itercb_tostr(struct statsblobv1 *sb, struct voi *v, struct voistat *vs,
2203 struct sb_iter_ctx *ctx)
2205 struct sb_tostrcb_ctx *sctx;
2209 switch (sctx->fmt) {
2210 case SB_STRFMT_FREEFORM:
2211 stats_v1_itercb_tostr_freeform(sb, v, vs, ctx);
2213 case SB_STRFMT_JSON:
2214 stats_v1_itercb_tostr_json(sb, v, vs, ctx);
2220 return (sbuf_error(sctx->buf));
2224 stats_v1_blob_tostr(struct statsblobv1 *sb, struct sbuf *buf,
2225 enum sb_str_fmt fmt, uint32_t flags)
2227 struct sb_tostrcb_ctx sctx;
2230 if (sb == NULL || sb->abi != STATS_ABI_V1 || buf == NULL ||
2231 fmt >= SB_STRFMT_NUM_FMTS)
2238 if (flags & SB_TOSTR_META) {
2239 if (stats_tpl_fetch(stats_tpl_fetch_allocid(NULL, sb->tplhash),
2246 if (flags & SB_TOSTR_OBJDUMP)
2247 iflags |= (SB_IT_NULLVOI | SB_IT_NULLVOISTAT);
2248 stats_v1_blob_iter(sb, stats_v1_itercb_tostr, &sctx, iflags);
2250 return (sbuf_error(buf));
2254 stats_v1_itercb_visit(struct statsblobv1 *sb, struct voi *v,
2255 struct voistat *vs, struct sb_iter_ctx *ctx)
2257 struct sb_visitcb_ctx *vctx;
2258 struct sb_visit sbv;
2262 sbv.tplhash = sb->tplhash;
2264 sbv.voi_dtype = v->dtype;
2265 sbv.vs_stype = vs->stype;
2266 sbv.vs_dtype = vs->dtype;
2267 sbv.vs_dsz = vs->dsz;
2268 sbv.vs_data = BLOB_OFFSET(sb, vs->data_off);
2269 sbv.vs_errs = vs->errs;
2270 sbv.flags = ctx->flags & (SB_IT_FIRST_CB | SB_IT_LAST_CB |
2271 SB_IT_FIRST_VOI | SB_IT_LAST_VOI | SB_IT_FIRST_VOISTAT |
2272 SB_IT_LAST_VOISTAT);
2274 return (vctx->cb(&sbv, vctx->usrctx));
2278 stats_v1_blob_visit(struct statsblobv1 *sb, stats_blob_visitcb_t func,
2281 struct sb_visitcb_ctx vctx;
2283 if (sb == NULL || sb->abi != STATS_ABI_V1 || func == NULL)
2287 vctx.usrctx = usrctx;
2289 stats_v1_blob_iter(sb, stats_v1_itercb_visit, &vctx, 0);
2295 stats_v1_icb_reset_voistat(struct statsblobv1 *sb, struct voi *v __unused,
2296 struct voistat *vs, struct sb_iter_ctx *ctx __unused)
2300 if (vs->stype == VS_STYPE_VOISTATE)
2303 vsd = BLOB_OFFSET(sb, vs->data_off);
2305 /* Perform the stat type's default reset action. */
2306 switch (vs->stype) {
2308 switch (vs->dtype) {
2309 case VSD_DTYPE_Q_S32:
2310 Q_SIFVAL(VSD(q32, vsd)->sq32, 0);
2312 case VSD_DTYPE_Q_U32:
2313 Q_SIFVAL(VSD(q32, vsd)->uq32, 0);
2315 case VSD_DTYPE_Q_S64:
2316 Q_SIFVAL(VSD(q64, vsd)->sq64, 0);
2318 case VSD_DTYPE_Q_U64:
2319 Q_SIFVAL(VSD(q64, vsd)->uq64, 0);
2322 bzero(vsd, vs->dsz);
2327 switch (vs->dtype) {
2328 case VSD_DTYPE_Q_S32:
2329 Q_SIFVAL(VSD(q32, vsd)->sq32,
2330 Q_IFMINVAL(VSD(q32, vsd)->sq32));
2332 case VSD_DTYPE_Q_U32:
2333 Q_SIFVAL(VSD(q32, vsd)->uq32,
2334 Q_IFMINVAL(VSD(q32, vsd)->uq32));
2336 case VSD_DTYPE_Q_S64:
2337 Q_SIFVAL(VSD(q64, vsd)->sq64,
2338 Q_IFMINVAL(VSD(q64, vsd)->sq64));
2340 case VSD_DTYPE_Q_U64:
2341 Q_SIFVAL(VSD(q64, vsd)->uq64,
2342 Q_IFMINVAL(VSD(q64, vsd)->uq64));
2345 memcpy(vsd, &numeric_limits[LIM_MIN][vs->dtype],
2351 switch (vs->dtype) {
2352 case VSD_DTYPE_Q_S32:
2353 Q_SIFVAL(VSD(q32, vsd)->sq32,
2354 Q_IFMAXVAL(VSD(q32, vsd)->sq32));
2356 case VSD_DTYPE_Q_U32:
2357 Q_SIFVAL(VSD(q32, vsd)->uq32,
2358 Q_IFMAXVAL(VSD(q32, vsd)->uq32));
2360 case VSD_DTYPE_Q_S64:
2361 Q_SIFVAL(VSD(q64, vsd)->sq64,
2362 Q_IFMAXVAL(VSD(q64, vsd)->sq64));
2364 case VSD_DTYPE_Q_U64:
2365 Q_SIFVAL(VSD(q64, vsd)->uq64,
2366 Q_IFMAXVAL(VSD(q64, vsd)->uq64));
2369 memcpy(vsd, &numeric_limits[LIM_MAX][vs->dtype],
2376 /* Reset bucket counts. */
2377 struct voistatdata_hist *hist;
2381 hist = VSD(hist, vsd);
2382 switch (vs->dtype) {
2383 case VSD_DTYPE_CRHIST32:
2384 nbkts = HIST_VSDSZ2NBKTS(crhist32, vs->dsz);
2387 case VSD_DTYPE_DRHIST32:
2388 nbkts = HIST_VSDSZ2NBKTS(drhist32, vs->dsz);
2391 case VSD_DTYPE_DVHIST32:
2392 nbkts = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz);
2395 case VSD_DTYPE_CRHIST64:
2396 nbkts = HIST_VSDSZ2NBKTS(crhist64, vs->dsz);
2399 case VSD_DTYPE_DRHIST64:
2400 nbkts = HIST_VSDSZ2NBKTS(drhist64, vs->dsz);
2403 case VSD_DTYPE_DVHIST64:
2404 nbkts = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz);
2411 bzero(VSD_HIST_FIELDPTR(hist, vs->dtype, oob),
2412 is32bit ? sizeof(uint32_t) : sizeof(uint64_t));
2413 for (i = nbkts - 1; i >= 0; i--) {
2414 bzero(VSD_HIST_FIELDPTR(hist, vs->dtype,
2415 bkts[i].cnt), is32bit ? sizeof(uint32_t) :
2420 case VS_STYPE_TDGST:
2422 /* Reset sample count centroids array/tree. */
2423 struct voistatdata_tdgst *tdgst;
2424 struct ctdth32 *ctd32tree;
2425 struct ctdth64 *ctd64tree;
2426 struct voistatdata_tdgstctd32 *ctd32;
2427 struct voistatdata_tdgstctd64 *ctd64;
2429 tdgst = VSD(tdgst, vsd);
2430 switch (vs->dtype) {
2431 case VSD_DTYPE_TDGSTCLUST32:
2432 VSD(tdgstclust32, tdgst)->smplcnt = 0;
2433 VSD(tdgstclust32, tdgst)->compcnt = 0;
2434 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
2435 ARB_INIT(ctd32, ctdlnk, ctd32tree,
2436 ARB_MAXNODES(ctd32tree)) {
2438 Q_SIFVAL(ctd32->mu, 0);
2441 RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree);
2444 case VSD_DTYPE_TDGSTCLUST64:
2445 VSD(tdgstclust64, tdgst)->smplcnt = 0;
2446 VSD(tdgstclust64, tdgst)->compcnt = 0;
2447 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
2448 ARB_INIT(ctd64, ctdlnk, ctd64tree,
2449 ARB_MAXNODES(ctd64tree)) {
2451 Q_SIFVAL(ctd64->mu, 0);
2454 RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree);
2463 KASSERT(0, ("Unknown VOI stat type %d", vs->stype));
2468 vs->flags &= ~VS_VSDVALID;
2474 stats_v1_blob_snapshot(struct statsblobv1 **dst, size_t dstmaxsz,
2475 struct statsblobv1 *src, uint32_t flags)
2479 if (src != NULL && src->abi == STATS_ABI_V1) {
2480 error = stats_v1_blob_clone(dst, dstmaxsz, src, flags);
2482 if (flags & SB_CLONE_RSTSRC) {
2483 stats_v1_blob_iter(src,
2484 stats_v1_icb_reset_voistat, NULL, 0);
2485 src->lastrst = stats_sbinuptime();
2487 stats_v1_blob_finalise(*dst);
2496 stats_v1_voi_update_max(enum vsd_dtype voi_dtype __unused,
2497 struct voistatdata *voival, struct voistat *vs, void *vsd)
2501 KASSERT(vs->dtype < VSD_NUM_DTYPES,
2502 ("Unknown VSD dtype %d", vs->dtype));
2506 switch (vs->dtype) {
2507 case VSD_DTYPE_INT_S32:
2508 if (VSD(int32, vsd)->s32 < voival->int32.s32) {
2509 VSD(int32, vsd)->s32 = voival->int32.s32;
2510 vs->flags |= VS_VSDVALID;
2513 case VSD_DTYPE_INT_U32:
2514 if (VSD(int32, vsd)->u32 < voival->int32.u32) {
2515 VSD(int32, vsd)->u32 = voival->int32.u32;
2516 vs->flags |= VS_VSDVALID;
2519 case VSD_DTYPE_INT_S64:
2520 if (VSD(int64, vsd)->s64 < voival->int64.s64) {
2521 VSD(int64, vsd)->s64 = voival->int64.s64;
2522 vs->flags |= VS_VSDVALID;
2525 case VSD_DTYPE_INT_U64:
2526 if (VSD(int64, vsd)->u64 < voival->int64.u64) {
2527 VSD(int64, vsd)->u64 = voival->int64.u64;
2528 vs->flags |= VS_VSDVALID;
2531 case VSD_DTYPE_INT_SLONG:
2532 if (VSD(intlong, vsd)->slong < voival->intlong.slong) {
2533 VSD(intlong, vsd)->slong = voival->intlong.slong;
2534 vs->flags |= VS_VSDVALID;
2537 case VSD_DTYPE_INT_ULONG:
2538 if (VSD(intlong, vsd)->ulong < voival->intlong.ulong) {
2539 VSD(intlong, vsd)->ulong = voival->intlong.ulong;
2540 vs->flags |= VS_VSDVALID;
2543 case VSD_DTYPE_Q_S32:
2544 if (Q_QLTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) &&
2545 (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32,
2546 voival->q32.sq32)))) {
2547 vs->flags |= VS_VSDVALID;
2550 case VSD_DTYPE_Q_U32:
2551 if (Q_QLTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) &&
2552 (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32,
2553 voival->q32.uq32)))) {
2554 vs->flags |= VS_VSDVALID;
2557 case VSD_DTYPE_Q_S64:
2558 if (Q_QLTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) &&
2559 (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64,
2560 voival->q64.sq64)))) {
2561 vs->flags |= VS_VSDVALID;
2564 case VSD_DTYPE_Q_U64:
2565 if (Q_QLTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) &&
2566 (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64,
2567 voival->q64.uq64)))) {
2568 vs->flags |= VS_VSDVALID;
2580 stats_v1_voi_update_min(enum vsd_dtype voi_dtype __unused,
2581 struct voistatdata *voival, struct voistat *vs, void *vsd)
2585 KASSERT(vs->dtype < VSD_NUM_DTYPES,
2586 ("Unknown VSD dtype %d", vs->dtype));
2590 switch (vs->dtype) {
2591 case VSD_DTYPE_INT_S32:
2592 if (VSD(int32, vsd)->s32 > voival->int32.s32) {
2593 VSD(int32, vsd)->s32 = voival->int32.s32;
2594 vs->flags |= VS_VSDVALID;
2597 case VSD_DTYPE_INT_U32:
2598 if (VSD(int32, vsd)->u32 > voival->int32.u32) {
2599 VSD(int32, vsd)->u32 = voival->int32.u32;
2600 vs->flags |= VS_VSDVALID;
2603 case VSD_DTYPE_INT_S64:
2604 if (VSD(int64, vsd)->s64 > voival->int64.s64) {
2605 VSD(int64, vsd)->s64 = voival->int64.s64;
2606 vs->flags |= VS_VSDVALID;
2609 case VSD_DTYPE_INT_U64:
2610 if (VSD(int64, vsd)->u64 > voival->int64.u64) {
2611 VSD(int64, vsd)->u64 = voival->int64.u64;
2612 vs->flags |= VS_VSDVALID;
2615 case VSD_DTYPE_INT_SLONG:
2616 if (VSD(intlong, vsd)->slong > voival->intlong.slong) {
2617 VSD(intlong, vsd)->slong = voival->intlong.slong;
2618 vs->flags |= VS_VSDVALID;
2621 case VSD_DTYPE_INT_ULONG:
2622 if (VSD(intlong, vsd)->ulong > voival->intlong.ulong) {
2623 VSD(intlong, vsd)->ulong = voival->intlong.ulong;
2624 vs->flags |= VS_VSDVALID;
2627 case VSD_DTYPE_Q_S32:
2628 if (Q_QGTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) &&
2629 (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32,
2630 voival->q32.sq32)))) {
2631 vs->flags |= VS_VSDVALID;
2634 case VSD_DTYPE_Q_U32:
2635 if (Q_QGTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) &&
2636 (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32,
2637 voival->q32.uq32)))) {
2638 vs->flags |= VS_VSDVALID;
2641 case VSD_DTYPE_Q_S64:
2642 if (Q_QGTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) &&
2643 (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64,
2644 voival->q64.sq64)))) {
2645 vs->flags |= VS_VSDVALID;
2648 case VSD_DTYPE_Q_U64:
2649 if (Q_QGTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) &&
2650 (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64,
2651 voival->q64.uq64)))) {
2652 vs->flags |= VS_VSDVALID;
2664 stats_v1_voi_update_sum(enum vsd_dtype voi_dtype __unused,
2665 struct voistatdata *voival, struct voistat *vs, void *vsd)
2669 KASSERT(vs->dtype < VSD_NUM_DTYPES,
2670 ("Unknown VSD dtype %d", vs->dtype));
2674 switch (vs->dtype) {
2675 case VSD_DTYPE_INT_S32:
2676 VSD(int32, vsd)->s32 += voival->int32.s32;
2678 case VSD_DTYPE_INT_U32:
2679 VSD(int32, vsd)->u32 += voival->int32.u32;
2681 case VSD_DTYPE_INT_S64:
2682 VSD(int64, vsd)->s64 += voival->int64.s64;
2684 case VSD_DTYPE_INT_U64:
2685 VSD(int64, vsd)->u64 += voival->int64.u64;
2687 case VSD_DTYPE_INT_SLONG:
2688 VSD(intlong, vsd)->slong += voival->intlong.slong;
2690 case VSD_DTYPE_INT_ULONG:
2691 VSD(intlong, vsd)->ulong += voival->intlong.ulong;
2693 case VSD_DTYPE_Q_S32:
2694 error = Q_QADDQ(&VSD(q32, vsd)->sq32, voival->q32.sq32);
2696 case VSD_DTYPE_Q_U32:
2697 error = Q_QADDQ(&VSD(q32, vsd)->uq32, voival->q32.uq32);
2699 case VSD_DTYPE_Q_S64:
2700 error = Q_QADDQ(&VSD(q64, vsd)->sq64, voival->q64.sq64);
2702 case VSD_DTYPE_Q_U64:
2703 error = Q_QADDQ(&VSD(q64, vsd)->uq64, voival->q64.uq64);
2711 vs->flags |= VS_VSDVALID;
2717 stats_v1_voi_update_hist(enum vsd_dtype voi_dtype, struct voistatdata *voival,
2718 struct voistat *vs, struct voistatdata_hist *hist)
2720 struct voistatdata_numeric *bkt_lb, *bkt_ub;
2721 uint64_t *oob64, *cnt64;
2722 uint32_t *oob32, *cnt32;
2723 int error, i, found, is32bit, has_ub, eq_only;
2727 switch (vs->dtype) {
2728 case VSD_DTYPE_CRHIST32:
2729 i = HIST_VSDSZ2NBKTS(crhist32, vs->dsz);
2731 has_ub = eq_only = 0;
2732 oob32 = &VSD(crhist32, hist)->oob;
2734 case VSD_DTYPE_DRHIST32:
2735 i = HIST_VSDSZ2NBKTS(drhist32, vs->dsz);
2736 is32bit = has_ub = 1;
2738 oob32 = &VSD(drhist32, hist)->oob;
2740 case VSD_DTYPE_DVHIST32:
2741 i = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz);
2742 is32bit = eq_only = 1;
2744 oob32 = &VSD(dvhist32, hist)->oob;
2746 case VSD_DTYPE_CRHIST64:
2747 i = HIST_VSDSZ2NBKTS(crhist64, vs->dsz);
2748 is32bit = has_ub = eq_only = 0;
2749 oob64 = &VSD(crhist64, hist)->oob;
2751 case VSD_DTYPE_DRHIST64:
2752 i = HIST_VSDSZ2NBKTS(drhist64, vs->dsz);
2753 is32bit = eq_only = 0;
2755 oob64 = &VSD(drhist64, hist)->oob;
2757 case VSD_DTYPE_DVHIST64:
2758 i = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz);
2759 is32bit = has_ub = 0;
2761 oob64 = &VSD(dvhist64, hist)->oob;
2766 i--; /* Adjust for 0-based array index. */
2768 /* XXXLAS: Should probably use a better bucket search algorithm. ARB? */
2769 for (found = 0; i >= 0 && !found; i--) {
2770 switch (vs->dtype) {
2771 case VSD_DTYPE_CRHIST32:
2772 bkt_lb = &VSD(crhist32, hist)->bkts[i].lb;
2773 cnt32 = &VSD(crhist32, hist)->bkts[i].cnt;
2775 case VSD_DTYPE_DRHIST32:
2776 bkt_lb = &VSD(drhist32, hist)->bkts[i].lb;
2777 bkt_ub = &VSD(drhist32, hist)->bkts[i].ub;
2778 cnt32 = &VSD(drhist32, hist)->bkts[i].cnt;
2780 case VSD_DTYPE_DVHIST32:
2781 bkt_lb = &VSD(dvhist32, hist)->bkts[i].val;
2782 cnt32 = &VSD(dvhist32, hist)->bkts[i].cnt;
2784 case VSD_DTYPE_CRHIST64:
2785 bkt_lb = &VSD(crhist64, hist)->bkts[i].lb;
2786 cnt64 = &VSD(crhist64, hist)->bkts[i].cnt;
2788 case VSD_DTYPE_DRHIST64:
2789 bkt_lb = &VSD(drhist64, hist)->bkts[i].lb;
2790 bkt_ub = &VSD(drhist64, hist)->bkts[i].ub;
2791 cnt64 = &VSD(drhist64, hist)->bkts[i].cnt;
2793 case VSD_DTYPE_DVHIST64:
2794 bkt_lb = &VSD(dvhist64, hist)->bkts[i].val;
2795 cnt64 = &VSD(dvhist64, hist)->bkts[i].cnt;
2801 switch (voi_dtype) {
2802 case VSD_DTYPE_INT_S32:
2803 if (voival->int32.s32 >= bkt_lb->int32.s32) {
2804 if ((eq_only && voival->int32.s32 ==
2805 bkt_lb->int32.s32) ||
2806 (!eq_only && (!has_ub ||
2807 voival->int32.s32 < bkt_ub->int32.s32)))
2811 case VSD_DTYPE_INT_U32:
2812 if (voival->int32.u32 >= bkt_lb->int32.u32) {
2813 if ((eq_only && voival->int32.u32 ==
2814 bkt_lb->int32.u32) ||
2815 (!eq_only && (!has_ub ||
2816 voival->int32.u32 < bkt_ub->int32.u32)))
2820 case VSD_DTYPE_INT_S64:
2821 if (voival->int64.s64 >= bkt_lb->int64.s64)
2822 if ((eq_only && voival->int64.s64 ==
2823 bkt_lb->int64.s64) ||
2824 (!eq_only && (!has_ub ||
2825 voival->int64.s64 < bkt_ub->int64.s64)))
2828 case VSD_DTYPE_INT_U64:
2829 if (voival->int64.u64 >= bkt_lb->int64.u64)
2830 if ((eq_only && voival->int64.u64 ==
2831 bkt_lb->int64.u64) ||
2832 (!eq_only && (!has_ub ||
2833 voival->int64.u64 < bkt_ub->int64.u64)))
2836 case VSD_DTYPE_INT_SLONG:
2837 if (voival->intlong.slong >= bkt_lb->intlong.slong)
2838 if ((eq_only && voival->intlong.slong ==
2839 bkt_lb->intlong.slong) ||
2840 (!eq_only && (!has_ub ||
2841 voival->intlong.slong <
2842 bkt_ub->intlong.slong)))
2845 case VSD_DTYPE_INT_ULONG:
2846 if (voival->intlong.ulong >= bkt_lb->intlong.ulong)
2847 if ((eq_only && voival->intlong.ulong ==
2848 bkt_lb->intlong.ulong) ||
2849 (!eq_only && (!has_ub ||
2850 voival->intlong.ulong <
2851 bkt_ub->intlong.ulong)))
2854 case VSD_DTYPE_Q_S32:
2855 if (Q_QGEQ(voival->q32.sq32, bkt_lb->q32.sq32))
2856 if ((eq_only && Q_QEQ(voival->q32.sq32,
2857 bkt_lb->q32.sq32)) ||
2858 (!eq_only && (!has_ub ||
2859 Q_QLTQ(voival->q32.sq32,
2860 bkt_ub->q32.sq32))))
2863 case VSD_DTYPE_Q_U32:
2864 if (Q_QGEQ(voival->q32.uq32, bkt_lb->q32.uq32))
2865 if ((eq_only && Q_QEQ(voival->q32.uq32,
2866 bkt_lb->q32.uq32)) ||
2867 (!eq_only && (!has_ub ||
2868 Q_QLTQ(voival->q32.uq32,
2869 bkt_ub->q32.uq32))))
2872 case VSD_DTYPE_Q_S64:
2873 if (Q_QGEQ(voival->q64.sq64, bkt_lb->q64.sq64))
2874 if ((eq_only && Q_QEQ(voival->q64.sq64,
2875 bkt_lb->q64.sq64)) ||
2876 (!eq_only && (!has_ub ||
2877 Q_QLTQ(voival->q64.sq64,
2878 bkt_ub->q64.sq64))))
2881 case VSD_DTYPE_Q_U64:
2882 if (Q_QGEQ(voival->q64.uq64, bkt_lb->q64.uq64))
2883 if ((eq_only && Q_QEQ(voival->q64.uq64,
2884 bkt_lb->q64.uq64)) ||
2885 (!eq_only && (!has_ub ||
2886 Q_QLTQ(voival->q64.uq64,
2887 bkt_ub->q64.uq64))))
2907 vs->flags |= VS_VSDVALID;
2912 stats_v1_vsd_tdgst_compress(enum vsd_dtype vs_dtype,
2913 struct voistatdata_tdgst *tdgst, int attempt)
2915 struct ctdth32 *ctd32tree;
2916 struct ctdth64 *ctd64tree;
2917 struct voistatdata_tdgstctd32 *ctd32;
2918 struct voistatdata_tdgstctd64 *ctd64;
2919 uint64_t ebits, idxmask;
2920 uint32_t bitsperidx, nebits;
2921 int error, idx, is32bit, maxctds, remctds, tmperr;
2926 case VSD_DTYPE_TDGSTCLUST32:
2927 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
2928 if (!ARB_FULL(ctd32tree))
2930 VSD(tdgstclust32, tdgst)->compcnt++;
2931 maxctds = remctds = ARB_MAXNODES(ctd32tree);
2932 ARB_RESET_TREE(ctd32tree, ctdth32, maxctds);
2933 VSD(tdgstclust32, tdgst)->smplcnt = 0;
2938 RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree);
2941 case VSD_DTYPE_TDGSTCLUST64:
2942 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
2943 if (!ARB_FULL(ctd64tree))
2945 VSD(tdgstclust64, tdgst)->compcnt++;
2946 maxctds = remctds = ARB_MAXNODES(ctd64tree);
2947 ARB_RESET_TREE(ctd64tree, ctdth64, maxctds);
2948 VSD(tdgstclust64, tdgst)->smplcnt = 0;
2953 RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree);
2961 * Rebuild the t-digest ARB by pseudorandomly selecting centroids and
2962 * re-inserting the mu/cnt of each as a value and corresponding weight.
2965 #define bitsperrand 31 /* Per random(3). */
2968 bitsperidx = fls(maxctds);
2969 KASSERT(bitsperidx <= sizeof(ebits) << 3,
2970 ("%s: bitsperidx=%d, ebits=%d",
2971 __func__, bitsperidx, (int)(sizeof(ebits) << 3)));
2972 idxmask = (UINT64_C(1) << bitsperidx) - 1;
2973 srandom(stats_sbinuptime());
2975 /* Initialise the free list with randomised centroid indices. */
2976 for (; remctds > 0; remctds--) {
2977 while (nebits < bitsperidx) {
2978 ebits |= ((uint64_t)random()) << nebits;
2979 nebits += bitsperrand;
2980 if (nebits > (sizeof(ebits) << 3))
2981 nebits = sizeof(ebits) << 3;
2983 idx = ebits & idxmask;
2984 nebits -= bitsperidx;
2985 ebits >>= bitsperidx;
2988 * Select the next centroid to put on the ARB free list. We
2989 * start with the centroid at our randomly selected array index,
2990 * and work our way forwards until finding one (the latter
2991 * aspect reduces re-insertion randomness, but is good enough).
2998 ctd32 = ARB_NODE(ctd32tree, idx);
3000 ctd64 = ARB_NODE(ctd64tree, idx);
3001 } while ((is32bit ? ARB_ISFREE(ctd32, ctdlnk) :
3002 ARB_ISFREE(ctd64, ctdlnk)) && ++idx);
3004 /* Put the centroid on the ARB free list. */
3006 ARB_RETURNFREE(ctd32tree, ctd32, ctdlnk);
3008 ARB_RETURNFREE(ctd64tree, ctd64, ctdlnk);
3012 * The free list now contains the randomised indices of every centroid.
3013 * Walk the free list from start to end, re-inserting each centroid's
3014 * mu/cnt. The tdgst_add() call may or may not consume the free centroid
3015 * we re-insert values from during each loop iteration, so we must latch
3016 * the index of the next free list centroid before the re-insertion
3017 * call. The previous loop above should have left the centroid pointer
3018 * pointing to the element at the head of the free list.
3021 ARB_FREEIDX(ctd32tree) == ARB_SELFIDX(ctd32tree, ctd32) :
3022 ARB_FREEIDX(ctd64tree) == ARB_SELFIDX(ctd64tree, ctd64)),
3023 ("%s: t-digest ARB@%p free list bug", __func__,
3024 (is32bit ? (void *)ctd32tree : (void *)ctd64tree)));
3026 while ((is32bit ? ctd32 != NULL : ctd64 != NULL)) {
3031 idx = ARB_NEXTFREEIDX(ctd32, ctdlnk);
3032 /* Cloning a s32q_t into a s64q_t should never fail. */
3033 tmperr = Q_QCLONEQ(&x, ctd32->mu);
3034 tmperr = tmperr ? tmperr : stats_v1_vsd_tdgst_add(
3035 vs_dtype, tdgst, x, ctd32->cnt, attempt);
3036 ctd32 = ARB_NODE(ctd32tree, idx);
3037 KASSERT(ctd32 == NULL || ARB_ISFREE(ctd32, ctdlnk),
3038 ("%s: t-digest ARB@%p free list bug", __func__,
3041 idx = ARB_NEXTFREEIDX(ctd64, ctdlnk);
3042 tmperr = stats_v1_vsd_tdgst_add(vs_dtype, tdgst,
3043 ctd64->mu, ctd64->cnt, attempt);
3044 ctd64 = ARB_NODE(ctd64tree, idx);
3045 KASSERT(ctd64 == NULL || ARB_ISFREE(ctd64, ctdlnk),
3046 ("%s: t-digest ARB@%p free list bug", __func__,
3050 * This process should not produce errors, bugs notwithstanding.
3051 * Just in case, latch any errors and attempt all re-insertions.
3053 error = tmperr ? tmperr : error;
3057 KASSERT(remctds == 0, ("%s: t-digest ARB@%p free list bug", __func__,
3058 (is32bit ? (void *)ctd32tree : (void *)ctd64tree)));
3064 stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype, struct voistatdata_tdgst *tdgst,
3065 s64q_t x, uint64_t weight, int attempt)
3068 char qstr[Q_MAXSTRLEN(x, 10)];
3070 struct ctdth32 *ctd32tree;
3071 struct ctdth64 *ctd64tree;
3072 void *closest, *cur, *lb, *ub;
3073 struct voistatdata_tdgstctd32 *ctd32;
3074 struct voistatdata_tdgstctd64 *ctd64;
3075 uint64_t cnt, smplcnt, sum, tmpsum;
3076 s64q_t k, minz, q, z;
3077 int error, is32bit, n;
3080 minz = Q_INI(&z, 0, 0, Q_NFBITS(x));
3083 case VSD_DTYPE_TDGSTCLUST32:
3084 if ((UINT32_MAX - weight) < VSD(tdgstclust32, tdgst)->smplcnt)
3086 smplcnt = VSD(tdgstclust32, tdgst)->smplcnt;
3087 ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree;
3092 case VSD_DTYPE_TDGSTCLUST64:
3093 if ((UINT64_MAX - weight) < VSD(tdgstclust64, tdgst)->smplcnt)
3095 smplcnt = VSD(tdgstclust64, tdgst)->smplcnt;
3096 ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree;
3110 * Inspired by Ted Dunning's AVLTreeDigest.java
3113 #if defined(DIAGNOSTIC)
3114 KASSERT(attempt < 5,
3115 ("%s: Too many attempts", __func__));
3120 Q_SIFVAL(minz, Q_IFMAXVAL(minz));
3121 closest = ub = NULL;
3125 lb = cur = (void *)(ctd32 = ARB_MIN(ctdth32, ctd32tree));
3127 lb = cur = (void *)(ctd64 = ARB_MIN(ctdth64, ctd64tree));
3129 if (lb == NULL) /* Empty tree. */
3130 lb = (is32bit ? (void *)ARB_ROOT(ctd32tree) :
3131 (void *)ARB_ROOT(ctd64tree));
3134 * Find the set of centroids with minimum distance to x and
3135 * compute the sum of counts for all centroids with mean less
3136 * than the first centroid in the set.
3140 (void *)(ctd32 = ARB_NEXT(ctdth32, ctd32tree, ctd32)) :
3141 (void *)(ctd64 = ARB_NEXT(ctdth64, ctd64tree, ctd64)))) {
3144 KASSERT(Q_PRECEQ(ctd32->mu, x),
3145 ("%s: Q_RELPREC(mu,x)=%d", __func__,
3146 Q_RELPREC(ctd32->mu, x)));
3147 /* Ok to assign as both have same precision. */
3151 KASSERT(Q_PRECEQ(ctd64->mu, x),
3152 ("%s: Q_RELPREC(mu,x)=%d", __func__,
3153 Q_RELPREC(ctd64->mu, x)));
3154 /* Ok to assign as both have same precision. */
3158 error = Q_QSUBQ(&z, x);
3159 #if defined(DIAGNOSTIC)
3160 KASSERT(!error, ("%s: unexpected error %d", __func__,
3167 if (Q_QLTQ(z, minz)) {
3172 } else if (Q_QGTQ(z, minz)) {
3179 (void *)(ctd32 = (struct voistatdata_tdgstctd32 *)lb) :
3180 (void *)(ctd64 = (struct voistatdata_tdgstctd64 *)lb));
3182 for (n = 0; cur != ub; cur = (is32bit ?
3183 (void *)(ctd32 = ARB_NEXT(ctdth32, ctd32tree, ctd32)) :
3184 (void *)(ctd64 = ARB_NEXT(ctdth64, ctd64tree, ctd64)))) {
3192 error = Q_QFRACI(&q, 1, 2);
3194 /* [ sum + ((cnt - 1) / 2) ] / (smplcnt - 1) */
3195 error = Q_QFRACI(&q, (sum << 1) + cnt - 1,
3196 (smplcnt - 1) << 1);
3198 /* k = q x 4 x samplcnt x attempt */
3199 error |= Q_QMULI(&k, 4 * smplcnt * attempt);
3200 /* k = k x (1 - q) */
3201 error |= Q_QSUBI(&q, 1);
3203 error |= Q_QMULQ(&k, q);
3204 #if defined(DIAGNOSTIC)
3205 #if !defined(_KERNEL)
3206 double q_dbl, k_dbl, q2d, k2d;
3209 q_dbl = smplcnt == 1 ? 0.5 :
3210 (sum + ((cnt - 1) / 2.0)) / (double)(smplcnt - 1);
3211 k_dbl = 4 * smplcnt * q_dbl * (1.0 - q_dbl) * attempt;
3213 * If the difference between q and q_dbl is greater than
3214 * the fractional precision of q, something is off.
3215 * NB: q is holding the value of 1 - q
3217 q_dbl = 1.0 - q_dbl;
3218 KASSERT((q_dbl > q2d ? q_dbl - q2d : q2d - q_dbl) <
3219 (1.05 * ((double)1 / (double)(1ULL << Q_NFBITS(q)))),
3220 ("Q-type q bad precision"));
3221 KASSERT((k_dbl > k2d ? k_dbl - k2d : k2d - k_dbl) <
3222 1.0 + (0.01 * smplcnt),
3223 ("Q-type k bad precision"));
3224 #endif /* !_KERNEL */
3225 KASSERT(!error, ("%s: unexpected error %d", __func__,
3227 #endif /* DIAGNOSTIC */
3230 if ((is32bit && ((ctd32->cnt + weight) <=
3231 (uint64_t)Q_GIVAL(k))) ||
3232 (!is32bit && ((ctd64->cnt + weight) <=
3233 (uint64_t)Q_GIVAL(k)))) {
3235 /* random() produces 31 bits. */
3236 if (random() < (INT32_MAX / n))
3241 } while (closest == NULL &&
3242 (is32bit ? ARB_FULL(ctd32tree) : ARB_FULL(ctd64tree)) &&
3243 (error = stats_v1_vsd_tdgst_compress(vs_dtype, tdgst,
3249 if (closest != NULL) {
3250 /* Merge with an existing centroid. */
3252 ctd32 = (struct voistatdata_tdgstctd32 *)closest;
3253 error = Q_QSUBQ(&x, ctd32->mu);
3254 error = error ? error :
3255 Q_QDIVI(&x, ctd32->cnt + weight);
3256 if (error || (error = Q_QADDQ(&ctd32->mu, x))) {
3258 KASSERT(!error, ("%s: unexpected error %d",
3263 ctd32->cnt += weight;
3264 error = ARB_REINSERT(ctdth32, ctd32tree, ctd32) ==
3265 NULL ? 0 : EALREADY;
3267 RB_REINSERT(rbctdth32,
3268 &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32);
3271 ctd64 = (struct voistatdata_tdgstctd64 *)closest;
3272 error = Q_QSUBQ(&x, ctd64->mu);
3273 error = error ? error :
3274 Q_QDIVI(&x, ctd64->cnt + weight);
3275 if (error || (error = Q_QADDQ(&ctd64->mu, x))) {
3276 KASSERT(!error, ("%s: unexpected error %d",
3280 ctd64->cnt += weight;
3281 error = ARB_REINSERT(ctdth64, ctd64tree, ctd64) ==
3282 NULL ? 0 : EALREADY;
3284 RB_REINSERT(rbctdth64,
3285 &VSD(tdgstclust64, tdgst)->rbctdtree, ctd64);
3290 * Add a new centroid. If digest compression is working
3291 * correctly, there should always be at least one free.
3294 ctd32 = ARB_GETFREE(ctd32tree, ctdlnk);
3296 KASSERT(ctd32 != NULL,
3297 ("%s: t-digest@%p has no free centroids",
3302 if ((error = Q_QCPYVALQ(&ctd32->mu, x)))
3304 ctd32->cnt = weight;
3305 error = ARB_INSERT(ctdth32, ctd32tree, ctd32) == NULL ?
3308 RB_INSERT(rbctdth32,
3309 &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32);
3312 ctd64 = ARB_GETFREE(ctd64tree, ctdlnk);
3314 KASSERT(ctd64 != NULL,
3315 ("%s: t-digest@%p has no free centroids",
3318 if (ctd64 == NULL) /* Should not happen. */
3320 /* Direct assignment ok as both have same type/prec. */
3322 ctd64->cnt = weight;
3323 error = ARB_INSERT(ctdth64, ctd64tree, ctd64) == NULL ?
3326 RB_INSERT(rbctdth64, &VSD(tdgstclust64,
3327 tdgst)->rbctdtree, ctd64);
3333 VSD(tdgstclust32, tdgst)->smplcnt += weight;
3335 VSD(tdgstclust64, tdgst)->smplcnt += weight;
3338 struct rbctdth64 *rbctdtree =
3339 &VSD(tdgstclust64, tdgst)->rbctdtree;
3340 struct voistatdata_tdgstctd64 *rbctd64;
3342 ARB_FOREACH(ctd64, ctdth64, ctd64tree) {
3343 rbctd64 = (i == 0 ? RB_MIN(rbctdth64, rbctdtree) :
3344 RB_NEXT(rbctdth64, rbctdtree, rbctd64));
3346 if (i >= ARB_CURNODES(ctd64tree)
3348 || ARB_MIN(ctdth64, ctd64tree) !=
3349 RB_MIN(rbctdth64, rbctdtree)
3350 || ARB_MAX(ctdth64, ctd64tree) !=
3351 RB_MAX(rbctdth64, rbctdtree)
3352 || ARB_LEFTIDX(ctd64, ctdlnk) !=
3353 ARB_SELFIDX(ctd64tree, RB_LEFT(rbctd64, rblnk))
3354 || ARB_RIGHTIDX(ctd64, ctdlnk) !=
3355 ARB_SELFIDX(ctd64tree, RB_RIGHT(rbctd64, rblnk))
3356 || ARB_PARENTIDX(ctd64, ctdlnk) !=
3357 ARB_SELFIDX(ctd64tree,
3358 RB_PARENT(rbctd64, rblnk))) {
3359 Q_TOSTR(ctd64->mu, -1, 10, qstr, sizeof(qstr));
3360 printf("ARB ctd=%3d p=%3d l=%3d r=%3d c=%2d "
3362 (int)ARB_SELFIDX(ctd64tree, ctd64),
3363 ARB_PARENTIDX(ctd64, ctdlnk),
3364 ARB_LEFTIDX(ctd64, ctdlnk),
3365 ARB_RIGHTIDX(ctd64, ctdlnk),
3366 ARB_COLOR(ctd64, ctdlnk),
3369 Q_TOSTR(rbctd64->mu, -1, 10, qstr,
3371 printf(" RB ctd=%3d p=%3d l=%3d r=%3d c=%2d "
3373 (int)ARB_SELFIDX(ctd64tree, rbctd64),
3374 (int)ARB_SELFIDX(ctd64tree,
3375 RB_PARENT(rbctd64, rblnk)),
3376 (int)ARB_SELFIDX(ctd64tree,
3377 RB_LEFT(rbctd64, rblnk)),
3378 (int)ARB_SELFIDX(ctd64tree,
3379 RB_RIGHT(rbctd64, rblnk)),
3380 RB_COLOR(rbctd64, rblnk),
3383 panic("RB@%p and ARB@%p trees differ\n",
3384 rbctdtree, ctd64tree);
3388 #endif /* DIAGNOSTIC */
3395 stats_v1_voi_update_tdgst(enum vsd_dtype voi_dtype, struct voistatdata *voival,
3396 struct voistat *vs, struct voistatdata_tdgst *tdgst)
3403 switch (vs->dtype) {
3404 case VSD_DTYPE_TDGSTCLUST32:
3405 /* Use same precision as the user's centroids. */
3406 Q_INI(&x, 0, 0, Q_NFBITS(
3407 ARB_CNODE(&VSD(tdgstclust32, tdgst)->ctdtree, 0)->mu));
3409 case VSD_DTYPE_TDGSTCLUST64:
3410 /* Use same precision as the user's centroids. */
3411 Q_INI(&x, 0, 0, Q_NFBITS(
3412 ARB_CNODE(&VSD(tdgstclust64, tdgst)->ctdtree, 0)->mu));
3415 KASSERT(vs->dtype == VSD_DTYPE_TDGSTCLUST32 ||
3416 vs->dtype == VSD_DTYPE_TDGSTCLUST64,
3417 ("%s: vs->dtype(%d) != VSD_DTYPE_TDGSTCLUST<32|64>",
3418 __func__, vs->dtype));
3423 * XXXLAS: Should have both a signed and unsigned 'x' variable to avoid
3424 * returning EOVERFLOW if the voival would have fit in a u64q_t.
3426 switch (voi_dtype) {
3427 case VSD_DTYPE_INT_S32:
3428 error = Q_QCPYVALI(&x, voival->int32.s32);
3430 case VSD_DTYPE_INT_U32:
3431 error = Q_QCPYVALI(&x, voival->int32.u32);
3433 case VSD_DTYPE_INT_S64:
3434 error = Q_QCPYVALI(&x, voival->int64.s64);
3436 case VSD_DTYPE_INT_U64:
3437 error = Q_QCPYVALI(&x, voival->int64.u64);
3439 case VSD_DTYPE_INT_SLONG:
3440 error = Q_QCPYVALI(&x, voival->intlong.slong);
3442 case VSD_DTYPE_INT_ULONG:
3443 error = Q_QCPYVALI(&x, voival->intlong.ulong);
3445 case VSD_DTYPE_Q_S32:
3446 error = Q_QCPYVALQ(&x, voival->q32.sq32);
3448 case VSD_DTYPE_Q_U32:
3449 error = Q_QCPYVALQ(&x, voival->q32.uq32);
3451 case VSD_DTYPE_Q_S64:
3452 error = Q_QCPYVALQ(&x, voival->q64.sq64);
3454 case VSD_DTYPE_Q_U64:
3455 error = Q_QCPYVALQ(&x, voival->q64.uq64);
3463 (error = stats_v1_vsd_tdgst_add(vs->dtype, tdgst, x, 1, 1)))
3466 vs->flags |= VS_VSDVALID;
3471 stats_v1_voi_update(struct statsblobv1 *sb, int32_t voi_id,
3472 enum vsd_dtype voi_dtype, struct voistatdata *voival, uint32_t flags)
3476 void *statevsd, *vsd;
3477 int error, i, tmperr;
3481 if (sb == NULL || sb->abi != STATS_ABI_V1 || voi_id >= NVOIS(sb) ||
3482 voi_dtype == 0 || voi_dtype >= VSD_NUM_DTYPES || voival == NULL)
3484 v = &sb->vois[voi_id];
3485 if (voi_dtype != v->dtype || v->id < 0 ||
3486 ((flags & SB_VOI_RELUPDATE) && !(v->flags & VOI_REQSTATE)))
3489 vs = BLOB_OFFSET(sb, v->stats_off);
3490 if (v->flags & VOI_REQSTATE)
3491 statevsd = BLOB_OFFSET(sb, vs->data_off);
3495 if (flags & SB_VOI_RELUPDATE) {
3496 switch (voi_dtype) {
3497 case VSD_DTYPE_INT_S32:
3498 voival->int32.s32 +=
3499 VSD(voistate, statevsd)->prev.int32.s32;
3501 case VSD_DTYPE_INT_U32:
3502 voival->int32.u32 +=
3503 VSD(voistate, statevsd)->prev.int32.u32;
3505 case VSD_DTYPE_INT_S64:
3506 voival->int64.s64 +=
3507 VSD(voistate, statevsd)->prev.int64.s64;
3509 case VSD_DTYPE_INT_U64:
3510 voival->int64.u64 +=
3511 VSD(voistate, statevsd)->prev.int64.u64;
3513 case VSD_DTYPE_INT_SLONG:
3514 voival->intlong.slong +=
3515 VSD(voistate, statevsd)->prev.intlong.slong;
3517 case VSD_DTYPE_INT_ULONG:
3518 voival->intlong.ulong +=
3519 VSD(voistate, statevsd)->prev.intlong.ulong;
3521 case VSD_DTYPE_Q_S32:
3522 error = Q_QADDQ(&voival->q32.sq32,
3523 VSD(voistate, statevsd)->prev.q32.sq32);
3525 case VSD_DTYPE_Q_U32:
3526 error = Q_QADDQ(&voival->q32.uq32,
3527 VSD(voistate, statevsd)->prev.q32.uq32);
3529 case VSD_DTYPE_Q_S64:
3530 error = Q_QADDQ(&voival->q64.sq64,
3531 VSD(voistate, statevsd)->prev.q64.sq64);
3533 case VSD_DTYPE_Q_U64:
3534 error = Q_QADDQ(&voival->q64.uq64,
3535 VSD(voistate, statevsd)->prev.q64.uq64);
3538 KASSERT(0, ("Unknown VOI data type %d", voi_dtype));
3546 for (i = v->voistatmaxid; i > 0; i--) {
3547 vs = &((struct voistat *)BLOB_OFFSET(sb, v->stats_off))[i];
3551 vsd = BLOB_OFFSET(sb, vs->data_off);
3553 switch (vs->stype) {
3555 tmperr = stats_v1_voi_update_max(voi_dtype, voival,
3559 tmperr = stats_v1_voi_update_min(voi_dtype, voival,
3563 tmperr = stats_v1_voi_update_sum(voi_dtype, voival,
3567 tmperr = stats_v1_voi_update_hist(voi_dtype, voival,
3570 case VS_STYPE_TDGST:
3571 tmperr = stats_v1_voi_update_tdgst(voi_dtype, voival,
3575 KASSERT(0, ("Unknown VOI stat type %d", vs->stype));
3586 switch (voi_dtype) {
3587 case VSD_DTYPE_INT_S32:
3588 VSD(voistate, statevsd)->prev.int32.s32 =
3591 case VSD_DTYPE_INT_U32:
3592 VSD(voistate, statevsd)->prev.int32.u32 =
3595 case VSD_DTYPE_INT_S64:
3596 VSD(voistate, statevsd)->prev.int64.s64 =
3599 case VSD_DTYPE_INT_U64:
3600 VSD(voistate, statevsd)->prev.int64.u64 =
3603 case VSD_DTYPE_INT_SLONG:
3604 VSD(voistate, statevsd)->prev.intlong.slong =
3605 voival->intlong.slong;
3607 case VSD_DTYPE_INT_ULONG:
3608 VSD(voistate, statevsd)->prev.intlong.ulong =
3609 voival->intlong.ulong;
3611 case VSD_DTYPE_Q_S32:
3613 &VSD(voistate, statevsd)->prev.q32.sq32,
3616 case VSD_DTYPE_Q_U32:
3618 &VSD(voistate, statevsd)->prev.q32.uq32,
3621 case VSD_DTYPE_Q_S64:
3623 &VSD(voistate, statevsd)->prev.q64.sq64,
3626 case VSD_DTYPE_Q_U64:
3628 &VSD(voistate, statevsd)->prev.q64.uq64,
3632 KASSERT(0, ("Unknown VOI data type %d", voi_dtype));
3643 stats_init(void *arg)
3647 SYSINIT(stats, SI_SUB_KDTRACE, SI_ORDER_FIRST, stats_init, NULL);
3650 * Sysctl handler to display the list of available stats templates.
3653 stats_tpl_list_available(SYSCTL_HANDLER_ARGS)
3660 /* We can tolerate ntpl being stale, so do not take the lock. */
3661 s = sbuf_new(NULL, NULL, /* +1 per tpl for , */
3662 ntpl * (STATS_TPL_MAX_STR_SPEC_LEN + 1), SBUF_FIXEDLEN);
3667 for (i = 0; i < ntpl; i++) {
3668 err = sbuf_printf(s, "%s\"%s\":%u", i ? "," : "",
3669 tpllist[i]->mb->tplname, tpllist[i]->mb->tplhash);
3671 /* Sbuf overflow condition. */
3680 err = sysctl_handle_string(oidp, sbuf_data(s), 0, req);
3688 * Called by subsystem-specific sysctls to report and/or parse the list of
3689 * templates being sampled and their sampling rates. A stats_tpl_sr_cb_t
3690 * conformant function pointer must be passed in as arg1, which is used to
3691 * interact with the subsystem's stats template sample rates list. If arg2 > 0,
3692 * a zero-initialised allocation of arg2-sized contextual memory is
3693 * heap-allocated and passed in to all subsystem callbacks made during the
3694 * operation of stats_tpl_sample_rates().
3696 * XXXLAS: Assumes templates are never removed, which is currently true but may
3697 * need to be reworked in future if dynamic template management becomes a
3698 * requirement e.g. to support kernel module based templates.
3701 stats_tpl_sample_rates(SYSCTL_HANDLER_ARGS)
3703 char kvpair_fmt[16], tplspec_fmt[16];
3704 char tpl_spec[STATS_TPL_MAX_STR_SPEC_LEN];
3705 char tpl_name[TPL_MAX_NAME_LEN + 2]; /* +2 for "" */
3706 stats_tpl_sr_cb_t subsys_cb;
3708 char *buf, *new_rates_usr_str, *tpl_name_p;
3709 struct stats_tpl_sample_rate *rates;
3711 uint32_t cum_pct, pct, tpl_hash;
3712 int err, i, off, len, newlen, nrates;
3717 subsys_cb = (stats_tpl_sr_cb_t)arg1;
3718 KASSERT(subsys_cb != NULL, ("%s: subsys_cb == arg1 == NULL", __func__));
3720 subsys_ctx = malloc(arg2, M_TEMP, M_WAITOK | M_ZERO);
3724 /* Grab current count of subsystem rates. */
3725 err = subsys_cb(TPL_SR_UNLOCKED_GET, NULL, &nrates, subsys_ctx);
3729 /* +1 to ensure we can append '\0' post copyin, +5 per rate for =nnn, */
3730 len = max(req->newlen + 1, nrates * (STATS_TPL_MAX_STR_SPEC_LEN + 5));
3732 if (req->oldptr != NULL || req->newptr != NULL)
3733 buf = malloc(len, M_TEMP, M_WAITOK);
3735 if (req->oldptr != NULL) {
3737 /* No rates, so return an empty string via oldptr. */
3738 err = SYSCTL_OUT(req, "", 1);
3744 s = sbuf_new(&_s, buf, len, SBUF_FIXEDLEN | SBUF_INCLUDENUL);
3746 /* Grab locked count of, and ptr to, subsystem rates. */
3747 err = subsys_cb(TPL_SR_RLOCKED_GET, &rates, &nrates,
3752 for (i = 0; i < nrates && !err; i++) {
3753 err = sbuf_printf(s, "%s\"%s\":%u=%u", i ? "," : "",
3754 tpllist[rates[i].tpl_slot_id]->mb->tplname,
3755 tpllist[rates[i].tpl_slot_id]->mb->tplhash,
3756 rates[i].tpl_sample_pct);
3759 /* Tell subsystem that we're done with its rates list. */
3760 err = subsys_cb(TPL_SR_RUNLOCK, &rates, &nrates, subsys_ctx);
3764 err = sbuf_finish(s);
3766 goto done; /* We lost a race for buf to be too small. */
3768 /* Return the rendered string data via oldptr. */
3769 err = SYSCTL_OUT(req, sbuf_data(s), sbuf_len(s));
3771 /* Return the upper bound size for buffer sizing requests. */
3772 err = SYSCTL_OUT(req, NULL, len);
3776 if (err || req->newptr == NULL)
3779 newlen = req->newlen - req->newidx;
3780 err = SYSCTL_IN(req, buf, newlen);
3785 * Initialise format strings at run time.
3787 * Write the max template spec string length into the
3788 * template_spec=percent key-value pair parsing format string as:
3789 * " %<width>[^=]=%u %n"
3791 * Write the max template name string length into the tplname:tplhash
3792 * parsing format string as:
3795 * Subtract 1 for \0 appended by sscanf().
3797 sprintf(kvpair_fmt, " %%%zu[^=]=%%u %%n", sizeof(tpl_spec) - 1);
3798 sprintf(tplspec_fmt, "%%%zu[^:]:%%u", sizeof(tpl_name) - 1);
3801 * Parse each CSV key-value pair specifying a template and its sample
3802 * percentage. Whitespace either side of a key-value pair is ignored.
3803 * Templates can be specified by name, hash, or name and hash per the
3804 * following formats (chars in [] are optional):
3805 * ["]<tplname>["]=<percent>
3807 * ["]<tplname>["]:hash=<percent>
3809 cum_pct = nrates = 0;
3811 buf[newlen] = '\0'; /* buf is at least newlen+1 in size. */
3812 new_rates_usr_str = buf;
3813 while (isspace(*new_rates_usr_str))
3814 new_rates_usr_str++; /* Skip leading whitespace. */
3815 while (*new_rates_usr_str != '\0') {
3816 tpl_name_p = tpl_name;
3822 * Parse key-value pair which must perform 2 conversions, then
3823 * parse the template spec to extract either name, hash, or name
3824 * and hash depending on the three possible spec formats. The
3825 * tplspec_fmt format specifier parses name or name and hash
3826 * template specs, while the ":%u" format specifier parses
3827 * hash-only template specs. If parsing is successfull, ensure
3828 * the cumulative sampling percentage does not exceed 100.
3831 if (2 != sscanf(new_rates_usr_str, kvpair_fmt, tpl_spec, &pct,
3834 if ((1 > sscanf(tpl_spec, tplspec_fmt, tpl_name, &tpl_hash)) &&
3835 (1 != sscanf(tpl_spec, ":%u", &tpl_hash)))
3837 if ((cum_pct += pct) > 100)
3841 /* Strip surrounding "" from template name if present. */
3842 len = strlen(tpl_name);
3844 if (tpl_name[len - 1] == '"')
3845 tpl_name[--len] = '\0';
3846 if (tpl_name[0] == '"') {
3852 rates = stats_realloc(rates, 0, /* oldsz is unused in kernel. */
3853 (nrates + 1) * sizeof(*rates), M_WAITOK);
3854 rates[nrates].tpl_slot_id =
3855 stats_tpl_fetch_allocid(len ? tpl_name_p : NULL, tpl_hash);
3856 if (rates[nrates].tpl_slot_id < 0) {
3857 err = -rates[nrates].tpl_slot_id;
3860 rates[nrates].tpl_sample_pct = pct;
3862 new_rates_usr_str += off;
3863 if (*new_rates_usr_str != ',')
3864 break; /* End-of-input or malformed. */
3865 new_rates_usr_str++; /* Move past comma to next pair. */
3869 if ((new_rates_usr_str - buf) < newlen) {
3870 /* Entire input has not been consumed. */
3874 * Give subsystem the new rates. They'll return the
3875 * appropriate rates pointer for us to garbage collect.
3877 err = subsys_cb(TPL_SR_PUT, &rates, &nrates,
3885 free(subsys_ctx, M_TEMP);
3889 SYSCTL_NODE(_kern, OID_AUTO, stats, CTLFLAG_RW, NULL,
3892 SYSCTL_PROC(_kern_stats, OID_AUTO, templates, CTLTYPE_STRING|CTLFLAG_RD,
3893 NULL, 0, stats_tpl_list_available, "A",
3894 "list the name/hash of all available stats(9) templates");
3896 #else /* ! _KERNEL */
3898 static void __attribute__ ((constructor))
3899 stats_constructor(void)
3902 pthread_rwlock_init(&tpllistlock, NULL);
3905 static void __attribute__ ((destructor))
3906 stats_destructor(void)
3909 pthread_rwlock_destroy(&tpllistlock);
3912 #endif /* _KERNEL */