1 #!/usr/bin/env @PYTHON_SHEBANG@
3 # Print out statistics for all cached dmu buffers. This information
4 # is available through the dbufs kstat and may be post-processed as
5 # needed by the script.
9 # The contents of this file are subject to the terms of the
10 # Common Development and Distribution License, Version 1.0 only
11 # (the "License"). You may not use this file except in compliance
14 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
15 # or http://www.opensolaris.org/os/licensing.
16 # See the License for the specific language governing permissions
17 # and limitations under the License.
19 # When distributing Covered Code, include this CDDL HEADER in each
20 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
21 # If applicable, add the following below this CDDL HEADER, with the
22 # fields enclosed by brackets "[]" replaced with your own identifying
23 # information: Portions Copyright [yyyy] [name of copyright owner]
27 # Copyright (C) 2013 Lawrence Livermore National Security, LLC.
28 # Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
30 # This script must remain compatible with Python 2.6+ and Python 3.4+.
38 bhdr = ["pool", "objset", "object", "level", "blkid", "offset", "dbsize"]
39 bxhdr = ["pool", "objset", "object", "level", "blkid", "offset", "dbsize",
40 "meta", "state", "dbholds", "dbc", "list", "atype", "flags",
41 "count", "asize", "access", "mru", "gmru", "mfu", "gmfu", "l2",
42 "l2_dattr", "l2_asize", "l2_comp", "aholds", "dtype", "btype",
43 "data_bs", "meta_bs", "bsize", "lvls", "dholds", "blocks", "dsize"]
44 bincompat = ["cached", "direct", "indirect", "bonus", "spill"]
46 dhdr = ["pool", "objset", "object", "dtype", "cached"]
47 dxhdr = ["pool", "objset", "object", "dtype", "btype", "data_bs", "meta_bs",
48 "bsize", "lvls", "dholds", "blocks", "dsize", "cached", "direct",
49 "indirect", "bonus", "spill"]
50 dincompat = ["level", "blkid", "offset", "dbsize", "meta", "state", "dbholds",
51 "dbc", "list", "atype", "flags", "count", "asize", "access",
52 "mru", "gmru", "mfu", "gmfu", "l2", "l2_dattr", "l2_asize",
55 thdr = ["pool", "objset", "dtype", "cached"]
56 txhdr = ["pool", "objset", "dtype", "cached", "direct", "indirect",
58 tincompat = ["object", "level", "blkid", "offset", "dbsize", "meta", "state",
59 "dbc", "dbholds", "list", "atype", "flags", "count", "asize",
60 "access", "mru", "gmru", "mfu", "gmfu", "l2", "l2_dattr",
61 "l2_asize", "l2_comp", "aholds", "btype", "data_bs", "meta_bs",
62 "bsize", "lvls", "dholds", "blocks", "dsize"]
65 # hdr: [size, scale, description]
66 "pool": [15, -1, "pool name"],
67 "objset": [6, -1, "dataset identification number"],
68 "object": [10, -1, "object number"],
69 "level": [5, -1, "indirection level of buffer"],
70 "blkid": [8, -1, "block number of buffer"],
71 "offset": [12, 1024, "offset in object of buffer"],
72 "dbsize": [7, 1024, "size of buffer"],
73 "meta": [4, -1, "is this buffer metadata?"],
74 "state": [5, -1, "state of buffer (read, cached, etc)"],
75 "dbholds": [7, 1000, "number of holds on buffer"],
76 "dbc": [3, -1, "in dbuf cache"],
77 "list": [4, -1, "which ARC list contains this buffer"],
78 "atype": [7, -1, "ARC header type (data or metadata)"],
79 "flags": [9, -1, "ARC read flags"],
80 "count": [5, -1, "ARC data count"],
81 "asize": [7, 1024, "size of this ARC buffer"],
82 "access": [10, -1, "time this ARC buffer was last accessed"],
83 "mru": [5, 1000, "hits while on the ARC's MRU list"],
84 "gmru": [5, 1000, "hits while on the ARC's MRU ghost list"],
85 "mfu": [5, 1000, "hits while on the ARC's MFU list"],
86 "gmfu": [5, 1000, "hits while on the ARC's MFU ghost list"],
87 "l2": [5, 1000, "hits while on the L2ARC"],
88 "l2_dattr": [8, -1, "L2ARC disk address/offset"],
89 "l2_asize": [8, 1024, "L2ARC alloc'd size (depending on compression)"],
90 "l2_comp": [21, -1, "L2ARC compression algorithm for buffer"],
91 "aholds": [6, 1000, "number of holds on this ARC buffer"],
92 "dtype": [27, -1, "dnode type"],
93 "btype": [27, -1, "bonus buffer type"],
94 "data_bs": [7, 1024, "data block size"],
95 "meta_bs": [7, 1024, "metadata block size"],
96 "bsize": [6, 1024, "bonus buffer size"],
97 "lvls": [6, -1, "number of indirection levels"],
98 "dholds": [6, 1000, "number of holds on dnode"],
99 "blocks": [8, 1000, "number of allocated blocks"],
100 "dsize": [12, 1024, "size of dnode"],
101 "cached": [6, 1024, "bytes cached for all blocks"],
102 "direct": [6, 1024, "bytes cached for direct blocks"],
103 "indirect": [8, 1024, "bytes cached for indirect blocks"],
104 "bonus": [5, 1024, "bytes cached for bonus buffer"],
105 "spill": [5, 1024, "bytes cached for spill block"],
110 sep = " " # Default separator is 2 spaces
111 cmd = ("Usage: dbufstat [-bdhnrtvx] [-i file] [-f fields] [-o file] "
112 "[-s string] [-F filter]\n")
116 def print_incompat_helper(incompat):
118 for key in sorted(incompat):
120 sys.stderr.write("\t")
122 sys.stderr.write(",\n\t")
125 sys.stderr.write(", ")
127 sys.stderr.write("%s" % key)
130 sys.stderr.write("\n\n")
133 def detailed_usage():
134 sys.stderr.write("%s\n" % cmd)
136 sys.stderr.write("Field definitions incompatible with '-b' option:\n")
137 print_incompat_helper(bincompat)
139 sys.stderr.write("Field definitions incompatible with '-d' option:\n")
140 print_incompat_helper(dincompat)
142 sys.stderr.write("Field definitions incompatible with '-t' option:\n")
143 print_incompat_helper(tincompat)
145 sys.stderr.write("Field definitions are as follows:\n")
146 for key in sorted(cols.keys()):
147 sys.stderr.write("%11s : %s\n" % (key, cols[key][2]))
148 sys.stderr.write("\n")
154 sys.stderr.write("%s\n" % cmd)
155 sys.stderr.write("\t -b : Print table of information for each dbuf\n")
156 sys.stderr.write("\t -d : Print table of information for each dnode\n")
157 sys.stderr.write("\t -h : Print this help message\n")
158 sys.stderr.write("\t -n : Exclude header from output\n")
159 sys.stderr.write("\t -r : Print raw values\n")
160 sys.stderr.write("\t -t : Print table of information for each dnode type"
162 sys.stderr.write("\t -v : List all possible field headers and definitions"
164 sys.stderr.write("\t -x : Print extended stats\n")
165 sys.stderr.write("\t -i : Redirect input from the specified file\n")
166 sys.stderr.write("\t -f : Specify specific fields to print (see -v)\n")
167 sys.stderr.write("\t -o : Redirect output to the specified file\n")
168 sys.stderr.write("\t -s : Override default field separator with custom "
169 "character or string\n")
170 sys.stderr.write("\t -F : Filter output by value or regex\n")
171 sys.stderr.write("\nExamples:\n")
172 sys.stderr.write("\tdbufstat -d -o /tmp/d.log\n")
173 sys.stderr.write("\tdbufstat -t -s \",\" -o /tmp/t.log\n")
174 sys.stderr.write("\tdbufstat -v\n")
175 sys.stderr.write("\tdbufstat -d -f pool,object,objset,dsize,cached\n")
176 sys.stderr.write("\tdbufstat -bx -F dbc=1,objset=54,pool=testpool\n")
177 sys.stderr.write("\n")
182 def prettynum(sz, scale, num=0):
185 suffix = [' ', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']
189 if raw or scale == -1:
190 return "%*s" % (sz, num)
192 # Rounding error, return 0
196 while num > scale and index < 5:
202 return "%*d" % (sz, num)
204 if (save / scale) < 10:
205 return "%*.1f%s" % (sz - 1, num, suffix[index])
207 return "%*d%s" % (sz - 1, num, suffix[index])
216 sys.stdout.write("%s%s" % (
217 prettynum(cols[col][0], cols[col][1], v[col]), sep))
218 sys.stdout.write("\n")
220 if e.errno == errno.EPIPE:
230 sys.stdout.write("%*s%s" % (cols[col][0], col, sep))
231 sys.stdout.write("\n")
233 if e.errno == errno.EPIPE:
237 def get_typestring(t):
241 "DMU_OT_OBJECT_DIRECTORY",
242 "DMU_OT_OBJECT_ARRAY",
243 "DMU_OT_PACKED_NVLIST",
244 "DMU_OT_PACKED_NVLIST_SIZE",
248 "DMU_OT_SPACE_MAP_HEADER",
257 "DMU_OT_DSL_DIR_CHILD_MAP",
258 "DMU_OT_DSL_DS_SNAP_MAP",
260 "DMU_OT_DSL_DATASET",
264 "DMU_OT_PLAIN_FILE_CONTENTS",
265 "DMU_OT_DIRECTORY_CONTENTS",
266 "DMU_OT_MASTER_NODE",
267 "DMU_OT_UNLINKED_SET",
271 # other; for testing only!
272 "DMU_OT_PLAIN_OTHER",
273 "DMU_OT_UINT64_OTHER",
277 "DMU_OT_SPA_HISTORY",
278 "DMU_OT_SPA_HISTORY_OFFSETS",
285 "DMU_OT_NEXT_CLONES",
287 "DMU_OT_USERGROUP_USED",
288 "DMU_OT_USERGROUP_QUOTA",
293 "DMU_OT_SA_MASTER_NODE",
294 "DMU_OT_SA_ATTR_REGISTRATION",
295 "DMU_OT_SA_ATTR_LAYOUTS",
299 "DMU_OT_DEADLIST_HDR",
301 "DMU_OT_BPOBJ_SUBOBJ"]
303 0x80: "DMU_OTN_UINT8_DATA",
304 0xc0: "DMU_OTN_UINT8_METADATA",
305 0x81: "DMU_OTN_UINT16_DATA",
306 0xc1: "DMU_OTN_UINT16_METADATA",
307 0x82: "DMU_OTN_UINT32_DATA",
308 0xc2: "DMU_OTN_UINT32_METADATA",
309 0x83: "DMU_OTN_UINT64_DATA",
310 0xc3: "DMU_OTN_UINT64_METADATA",
311 0x84: "DMU_OTN_ZAP_DATA",
312 0xc4: "DMU_OTN_ZAP_METADATA",
313 0xa0: "DMU_OTN_UINT8_ENC_DATA",
314 0xe0: "DMU_OTN_UINT8_ENC_METADATA",
315 0xa1: "DMU_OTN_UINT16_ENC_DATA",
316 0xe1: "DMU_OTN_UINT16_ENC_METADATA",
317 0xa2: "DMU_OTN_UINT32_ENC_DATA",
318 0xe2: "DMU_OTN_UINT32_ENC_METADATA",
319 0xa3: "DMU_OTN_UINT64_ENC_DATA",
320 0xe3: "DMU_OTN_UINT64_ENC_METADATA",
321 0xa4: "DMU_OTN_ZAP_ENC_DATA",
322 0xe4: "DMU_OTN_ZAP_ENC_METADATA"}
324 # If "-rr" option is used, don't convert to string representation
329 if t < len(ot_strings):
332 return otn_strings[t]
333 except (IndexError, KeyError):
337 def get_compstring(c):
338 comp_strings = ["ZIO_COMPRESS_INHERIT", "ZIO_COMPRESS_ON",
339 "ZIO_COMPRESS_OFF", "ZIO_COMPRESS_LZJB",
340 "ZIO_COMPRESS_EMPTY", "ZIO_COMPRESS_GZIP_1",
341 "ZIO_COMPRESS_GZIP_2", "ZIO_COMPRESS_GZIP_3",
342 "ZIO_COMPRESS_GZIP_4", "ZIO_COMPRESS_GZIP_5",
343 "ZIO_COMPRESS_GZIP_6", "ZIO_COMPRESS_GZIP_7",
344 "ZIO_COMPRESS_GZIP_8", "ZIO_COMPRESS_GZIP_9",
345 "ZIO_COMPRESS_ZLE", "ZIO_COMPRESS_LZ4",
346 "ZIO_COMPRESS_ZSTD", "ZIO_COMPRESS_FUNCTION"]
348 # If "-rr" option is used, don't convert to string representation
353 return comp_strings[c]
358 def parse_line(line, labels):
364 # These are "special" fields computed in the update_dict
365 # function, prevent KeyError exception on labels[col] for these.
366 if col not in ['bonus', 'cached', 'direct', 'indirect', 'spill']:
367 val = line[labels[col]]
369 if col in ['pool', 'flags']:
371 elif col in ['dtype', 'btype']:
372 new[col] = get_typestring(int(val))
373 elif col in ['l2_comp']:
374 new[col] = get_compstring(int(val))
381 def update_dict(d, k, line, labels):
382 pool = line[labels['pool']]
383 objset = line[labels['objset']]
384 key = line[labels[k]]
386 dbsize = int(line[labels['dbsize']])
387 blkid = int(line[labels['blkid']])
388 level = int(line[labels['level']])
393 if objset not in d[pool]:
394 d[pool][objset] = dict()
396 if key not in d[pool][objset]:
397 d[pool][objset][key] = parse_line(line, labels)
398 d[pool][objset][key]['bonus'] = 0
399 d[pool][objset][key]['cached'] = 0
400 d[pool][objset][key]['direct'] = 0
401 d[pool][objset][key]['indirect'] = 0
402 d[pool][objset][key]['spill'] = 0
404 d[pool][objset][key]['cached'] += dbsize
407 d[pool][objset][key]['bonus'] += dbsize
409 d[pool][objset][key]['spill'] += dbsize
412 d[pool][objset][key]['direct'] += dbsize
414 d[pool][objset][key]['indirect'] += dbsize
419 def skip_line(vals, filters):
421 Determines if a line should be skipped during printing
422 based on a set of filters
424 if len(filters) == 0:
429 val = prettynum(cols[key][0], cols[key][1], vals[key]).strip()
430 # we want a full match here
431 if re.match("(?:" + filters[key] + r")\Z", val) is None:
437 def print_dict(d, filters, noheader):
440 for pool in list(d.keys()):
441 for objset in list(d[pool].keys()):
442 for v in list(d[pool][objset].values()):
443 if not skip_line(v, filters):
447 def dnodes_build_dict(filehandle):
451 # First 3 lines are header information, skip the first two
455 # The third line contains the labels and index locations
456 for i, v in enumerate(next(filehandle).split()):
459 # The rest of the file is buffer information
460 for line in filehandle:
461 update_dict(dnodes, 'object', line.split(), labels)
466 def types_build_dict(filehandle):
470 # First 3 lines are header information, skip the first two
474 # The third line contains the labels and index locations
475 for i, v in enumerate(next(filehandle).split()):
478 # The rest of the file is buffer information
479 for line in filehandle:
480 update_dict(types, 'dtype', line.split(), labels)
485 def buffers_print_all(filehandle, filters, noheader):
488 # First 3 lines are header information, skip the first two
492 # The third line contains the labels and index locations
493 for i, v in enumerate(next(filehandle).split()):
499 # The rest of the file is buffer information
500 for line in filehandle:
501 vals = parse_line(line.split(), labels)
502 if not skip_line(vals, filters):
524 opts, args = getopt.getopt(
526 "bdf:hi:o:rs:tvxF:n",
545 for opt, arg in opts:
546 if opt in ('-b', '--buffers'):
548 if opt in ('-d', '--dnodes'):
550 if opt in ('-f', '--columns'):
552 if opt in ('-h', '--help'):
554 if opt in ('-i', '--infile'):
556 if opt in ('-o', '--outfile'):
558 if opt in ('-r', '--raw'):
560 if opt in ('-s', '--separator'):
562 if opt in ('-t', '--types'):
564 if opt in ('-v', '--verbose'):
566 if opt in ('-x', '--extended'):
568 if opt in ('-n', '--noheader'):
570 if opt in ('-F', '--filter'):
571 fils = [x.strip() for x in arg.split(",")]
574 f = [x.strip() for x in fil.split("=")]
577 sys.stderr.write("Invalid filter '%s'.\n" % fil)
581 sys.stderr.write("Invalid field '%s' in filter.\n" % f[0])
585 sys.stderr.write("Field '%s' specified multiple times in "
590 re.compile("(?:" + f[1] + r")\Z")
592 sys.stderr.write("Invalid regex for field '%s' in "
598 if hflag or (xflag and desired_cols):
604 # Ensure at most only one of b, d, or t flags are set
605 if (bflag and dflag) or (bflag and tflag) or (dflag and tflag):
609 hdr = bxhdr if xflag else bhdr
611 hdr = txhdr if xflag else thdr
612 else: # Even if dflag is False, it's the default if none set
614 hdr = dxhdr if xflag else dhdr
617 hdr = desired_cols.split(",")
624 elif ((bflag and bincompat and ele in bincompat) or
625 (dflag and dincompat and ele in dincompat) or
626 (tflag and tincompat and ele in tincompat)):
630 sys.stderr.write("Invalid column definition! -- %s\n" % invalid)
633 if len(incompat) > 0:
634 sys.stderr.write("Incompatible field specified! -- %s\n" %
640 tmp = open(ofile, "w")
644 sys.stderr.write("Cannot open %s for writing\n" % ofile)
648 ifile = '/proc/spl/kstat/zfs/dbufs'
652 tmp = open(ifile, "r")
655 sys.stderr.write("Cannot open %s for reading\n" % ifile)
659 buffers_print_all(sys.stdin, filters, nflag)
662 print_dict(dnodes_build_dict(sys.stdin), filters, nflag)
665 print_dict(types_build_dict(sys.stdin), filters, nflag)
668 if __name__ == '__main__':