6 # This file and its contents are supplied under the terms of the
7 # Common Development and Distribution License ("CDDL"), version 1.0.
8 # You may only use this file in accordance with the terms of version
11 # A full copy of the text of the CDDL should have accompanied this
12 # source. A copy of the CDDL is also available via the Internet at
13 # http://www.illumos.org/license/CDDL.
19 # Copyright (c) 2015 by Delphix. All rights reserved.
20 # Copyright (C) 2016 Lawrence Livermore National Security, LLC.
21 # Copyright (c) 2017, Intel Corporation.
25 SCRIPT_COMMON=common.sh
26 if [[ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]]; then
27 . "${BASE_DIR}/${SCRIPT_COMMON}"
29 echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
32 # shellcheck disable=SC2034
36 DEFAULTWORKDIR=/var/tmp
37 DEFAULTCOREDIR=/var/tmp/zloop
43 $0 [-hl] [-c <dump directory>] [-f <vdev directory>]
44 [-m <max core dumps>] [-s <vdev size>] [-t <timeout>]
45 [-I <max iterations>] [-- [extra ztest parameters]]
47 This script runs ztest repeatedly with randomized arguments.
48 If a crash is encountered, the ztest logs, any associated
49 vdev files, and core file (if one exists) are moved to the
50 output directory ($DEFAULTCOREDIR by default). Any options
51 after the -- end-of-options marker will be passed to ztest.
54 -c Specify a core dump directory to use.
55 -f Specify working directory for ztest vdev files.
56 -h Print this help message.
57 -l Create 'ztest.core.N' symlink to core directory.
58 -m Max number of core dumps to allow before exiting.
59 -s Size of vdev devices.
60 -t Total time to loop for, in seconds. If not provided,
62 -I Max number of iterations to loop before exiting.
70 echo "Command failed: $*"
81 read -r origcorepattern </proc/sys/kernel/core_pattern
82 coreglob="$(grep -E -o '^([^|%[:space:]]*)' /proc/sys/kernel/core_pattern)*"
84 if [[ $coreglob = "*" ]]; then
85 echo "Setting core file pattern..."
86 echo "core" > /proc/sys/kernel/core_pattern
87 coreglob="$(grep -E -o '^([^|%[:space:]]*)' \
88 /proc/sys/kernel/core_pattern)*"
98 # shellcheck disable=SC2012,SC2086
99 ls -tr1 $coreglob 2>/dev/null | head -1
104 # shellcheck disable=SC2154
106 core_id=$($GDB --batch -c "$1" | grep "Core was generated by" | \
108 if [[ "$core_id" == *"zdb "* ]]; then
109 # shellcheck disable=SC2154
118 if [[ $ztrc -ne 0 ]] || [[ -f "$core" ]]; then
119 df -h "$workdir" >>ztest.out
120 coreid=$(date "+zloop-%y%m%d-%H%M%S")
121 foundcrashes=$((foundcrashes + 1))
124 zdbcmd="$ZDB -U "$workdir/zpool.cache" -dddMmDDG ztest"
125 zdbdebug=$($zdbcmd 2>&1)
126 echo -e "$zdbcmd\n" >>ztest.zdb
127 echo "$zdbdebug" >>ztest.zdb
129 dest=$coredir/$coreid
130 or_die mkdir -p "$dest/vdev"
132 if [[ $symlink -ne 0 ]]; then
133 or_die ln -sf "$dest" "ztest.core.${foundcrashes}"
136 echo "*** ztest crash found - moving logs to $dest"
138 or_die mv ztest.history ztest.zdb ztest.out "$dest/"
139 or_die mv "$workdir/"ztest* "$dest/vdev/"
141 if [[ -e "$workdir/zpool.cache" ]]; then
142 or_die mv "$workdir/zpool.cache" "$dest/vdev/"
146 if [[ -f "$core" ]]; then
147 coreprog=$(core_prog "$core")
148 coredebug=$($GDB --batch --quiet \
149 -ex "set print thread-events off" \
150 -ex "printf \"*\n* Backtrace \n*\n\"" \
152 -ex "printf \"*\n* Libraries \n*\n\"" \
153 -ex "info sharedlib" \
154 -ex "printf \"*\n* Threads (full) \n*\n\"" \
156 -ex "printf \"*\n* Backtraces \n*\n\"" \
157 -ex "thread apply all bt" \
158 -ex "printf \"*\n* Backtraces (full) \n*\n\"" \
159 -ex "thread apply all bt full" \
160 -ex "quit" "$coreprog" "$core" 2>&1 | \
163 # Dump core + logs to stored directory
164 echo "$coredebug" >>"$dest/ztest.gdb"
165 or_die mv "$core" "$dest/"
167 # Record info in cores logfile
168 echo "*** core @ $coredir/$coreid/$core:" | \
172 if [[ $coremax -gt 0 ]] &&
173 [[ $foundcrashes -ge $coremax ]]; then
174 echo "exiting... max $coremax allowed cores"
183 # expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args]
184 coredir=$DEFAULTCOREDIR
185 basedir=$DEFAULTWORKDIR
192 while getopts ":ht:m:I:s:c:f:l" opt; do
194 t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;;
195 m ) [[ $OPTARG -gt 0 ]] && coremax=$OPTARG ;;
196 I ) [[ -n $OPTARG ]] && iterations=$OPTARG ;;
197 s ) [[ -n $OPTARG ]] && size=$OPTARG ;;
198 c ) [[ -n $OPTARG ]] && coredir=$OPTARG ;;
199 f ) [[ -n $OPTARG ]] && basedir=$(readlink -f "$OPTARG") ;;
204 * ) echo "Invalid argument: -$OPTARG";
209 # pass remaining arguments on to ztest
210 shift $((OPTIND - 1))
214 export ASAN_OPTIONS=abort_on_error=true:halt_on_error=true:allocator_may_return_null=true:disable_coredump=false:detect_stack_use_after_return=true
215 export UBSAN_OPTIONS=abort_on_error=true:halt_on_error=true:print_stacktrace=true
217 if [[ -f "$(core_file)" ]]; then
218 echo -n "There's a core dump here you might want to look at first... "
224 if [[ ! -d $coredir ]]; then
225 echo "core dump directory ($coredir) does not exist, creating it."
226 or_die mkdir -p "$coredir"
229 if [[ ! -w $coredir ]]; then
230 echo "core dump directory ($coredir) is not writable."
234 or_die rm -f ztest.history ztest.zdb ztest.cores
236 ztrc=0 # ztest return value
237 foundcrashes=0 # number of crashes found so far
238 starttime=$(date +%s)
242 # if no timeout was specified, loop forever.
243 while (( timeout == 0 )) || (( curtime <= (starttime + timeout) )); do
244 if (( iterations > 0 )) && (( iteration++ == iterations )); then
250 # start each run with an empty directory
251 workdir="$basedir/$rundir"
252 or_die rm -rf "$workdir"
253 or_die mkdir "$workdir"
255 # switch between three types of configs
256 # 1/3 basic, 1/3 raidz mix, and 1/3 draid mix
257 choice=$((RANDOM % 3))
259 # ashift range 9 - 15
260 align=$(((RANDOM % 2) * 3 + 9))
262 # randomly use special classes
263 class="special=random"
265 if [[ $choice -eq 0 ]]; then
274 elif [[ $choice -eq 1 ]]; then
275 # fully randomized mirror/raidz (sans dRAID)
276 parity=$(((RANDOM % 3) + 1))
277 mirrors=$(((RANDOM % 3) * 1))
280 raid_children=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2)))
281 vdevs=$(((RANDOM % 3) + 3))
284 # fully randomized dRAID (sans mirror/raidz)
285 parity=$(((RANDOM % 3) + 1))
287 draid_data=$(((RANDOM % 8) + 3))
288 draid_spares=$(((RANDOM % 2) + parity))
289 stripe=$((draid_data + parity))
290 extra=$((draid_spares + (RANDOM % 4)))
291 raid_children=$(((((RANDOM % 4) + 1) * stripe) + extra))
292 vdevs=$((RANDOM % 3))
296 zopt="$zopt -K $raid_type"
297 zopt="$zopt -m $mirrors"
298 zopt="$zopt -r $raid_children"
299 zopt="$zopt -D $draid_data"
300 zopt="$zopt -S $draid_spares"
301 zopt="$zopt -R $parity"
302 zopt="$zopt -v $vdevs"
303 zopt="$zopt -a $align"
304 zopt="$zopt -C $class"
305 zopt="$zopt -s $size"
306 zopt="$zopt -f $workdir"
308 cmd="$ZTEST $zopt $*"
309 echo "$(date '+%m/%d %T') $cmd" | tee -a ztest.history ztest.out
310 $cmd >>ztest.out 2>&1
312 grep -E '===|WARNING' ztest.out >>ztest.history
319 echo "zloop finished, $foundcrashes crashes found"
321 # restore core pattern.
324 echo "$origcorepattern" > /proc/sys/kernel/core_pattern
332 if [[ $foundcrashes -gt 0 ]]; then