6 # This file and its contents are supplied under the terms of the
7 # Common Development and Distribution License ("CDDL"), version 1.0.
8 # You may only use this file in accordance with the terms of version
11 # A full copy of the text of the CDDL should have accompanied this
12 # source. A copy of the CDDL is also available via the Internet at
13 # http://www.illumos.org/license/CDDL.
19 # Copyright (c) 2015 by Delphix. All rights reserved.
20 # Copyright (C) 2016 Lawrence Livermore National Security, LLC.
23 basedir="$(dirname $0)"
25 SCRIPT_COMMON=common.sh
26 if [ -f "${basedir}/${SCRIPT_COMMON}" ]; then
27 . "${basedir}/${SCRIPT_COMMON}"
29 echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
34 DEFAULTWORKDIR=/var/tmp
35 DEFAULTCOREDIR=/var/tmp/zloop
39 echo -e "\n$0 [-t <timeout>] [-c <dump directory>]" \
40 "[ -- [extra ztest parameters]]\n" \
42 " This script runs ztest repeatedly with randomized arguments.\n" \
43 " If a crash is encountered, the ztest logs, any associated\n" \
44 " vdev files, and core file (if one exists) are moved to the\n" \
45 " output directory ($DEFAULTCOREDIR by default). Any options\n" \
46 " after the -- end-of-options marker will be passed to ztest.\n" \
49 " -t Total time to loop for, in seconds. If not provided,\n" \
50 " zloop runs forever.\n" \
51 " -f Specify working directory for ztest vdev files.\n" \
52 " -c Specify a core dump directory to use.\n" \
53 " -h Print this help message.\n" \
60 if [[ $? -ne 0 ]]; then
61 echo "Command failed: $@"
67 origcorepattern="$(cat /proc/sys/kernel/core_pattern)"
68 coreglob="$(egrep -o '^([^|%[:space:]]*)' /proc/sys/kernel/core_pattern)*"
70 if [[ $coreglob = "*" ]]; then
71 echo "Setting core file pattern..."
72 echo "core" > /proc/sys/kernel/core_pattern
73 coreglob="$(egrep -o '^([^|%[:space:]]*)' \
74 /proc/sys/kernel/core_pattern)*"
79 printf "%s" "$(ls -tr1 $coreglob 2> /dev/null | head -1)"
85 core_id=$($GDB --batch -c $1 | grep "Core was generated by" | \
87 if [[ "$core_id" =~ "zdb " ]]; then
96 if [[ $ztrc -ne 0 ]] || [[ -f "$core" ]]; then
97 coreid=$(date "+zloop-%y%m%d-%H%M%S")
98 foundcrashes=$(($foundcrashes + 1))
100 dest=$coredir/$coreid
101 or_die mkdir -p $dest
102 or_die mkdir -p $dest/vdev
104 echo "*** ztest crash found - moving logs to $dest"
106 or_die mv ztest.history $dest/
107 or_die mv ztest.ddt $dest/
108 or_die mv ztest.out $dest/
109 or_die mv $workdir/ztest* $dest/vdev/
110 or_die mv $workdir/zpool.cache $dest/vdev/
113 if [[ -f "$core" ]]; then
114 coreprog=$(core_prog $core)
115 corestatus=$($GDB --batch --quiet \
116 -ex "set print thread-events off" \
117 -ex "printf \"*\n* Backtrace \n*\n\"" \
119 -ex "printf \"*\n* Libraries \n*\n\"" \
120 -ex "info sharedlib" \
121 -ex "printf \"*\n* Threads (full) \n*\n\"" \
123 -ex "printf \"*\n* Backtraces \n*\n\"" \
124 -ex "thread apply all bt" \
125 -ex "printf \"*\n* Backtraces (full) \n*\n\"" \
126 -ex "thread apply all bt full" \
127 -ex "quit" $coreprog "$core" | grep -v "New LWP")
129 # Dump core + logs to stored directory
130 echo "$corestatus" >>$dest/status
131 or_die mv "$core" $dest/
133 # Record info in cores logfile
134 echo "*** core @ $coredir/$coreid/$core:" | \
136 echo "$corestatus" | tee -a ztest.cores
137 echo "" | tee -a ztest.cores
144 # expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args]
145 coredir=$DEFAULTCOREDIR
146 workdir=$DEFAULTWORKDIR
148 while getopts ":ht:c:f:" opt; do
150 t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;;
151 c ) [[ $OPTARG ]] && coredir=$OPTARG ;;
152 f ) [[ $OPTARG ]] && workdir=$(readlink -f $OPTARG) ;;
156 * ) echo "Invalid argument: -$OPTARG";
161 # pass remaining arguments on to ztest
162 shift $((OPTIND - 1))
167 if [[ -f "$(core_file)" ]]; then
168 echo -n "There's a core dump here you might want to look at first... "
173 if [[ ! -d $coredir ]]; then
174 echo "core dump directory ($coredir) does not exist, creating it."
175 or_die mkdir -p $coredir
178 if [[ ! -w $coredir ]]; then
179 echo "core dump directory ($coredir) is not writable."
183 or_die rm -f ztest.history
184 or_die rm -f ztest.ddt
185 or_die rm -f ztest.cores
187 ztrc=0 # ztest return value
188 foundcrashes=0 # number of crashes found so far
189 starttime=$(date +%s)
192 # if no timeout was specified, loop forever.
193 while [[ $timeout -eq 0 ]] || [[ $curtime -le $(($starttime + $timeout)) ]]; do
196 # switch between common arrangements & fully randomized
197 if [[ $((RANDOM % 2)) -eq 0 ]]; then
203 mirrors=$(((RANDOM % 3) * 1))
204 parity=$(((RANDOM % 3) + 1))
205 raidz=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2)))
206 vdevs=$(((RANDOM % 3) + 3))
208 align=$(((RANDOM % 2) * 3 + 9))
209 runtime=$((RANDOM % 100))
210 passtime=$((RANDOM % (runtime / 3 + 1) + 10))
213 zopt="$zopt -m $mirrors"
214 zopt="$zopt -r $raidz"
215 zopt="$zopt -R $parity"
216 zopt="$zopt -v $vdevs"
217 zopt="$zopt -a $align"
218 zopt="$zopt -T $runtime"
219 zopt="$zopt -P $passtime"
220 zopt="$zopt -s $size"
221 zopt="$zopt -f $workdir"
223 cmd="$ZTEST $zopt $@"
224 desc="$(date '+%m/%d %T') $cmd"
225 echo "$desc" | tee -a ztest.history
226 echo "$desc" >>ztest.out
227 $cmd >>ztest.out 2>&1
229 egrep '===|WARNING' ztest.out >>ztest.history
230 $ZDB -U $workdir/zpool.cache -DD ztest >>ztest.ddt
237 echo "zloop finished, $foundcrashes crashes found"
239 #restore core pattern
240 echo "$origcorepattern" > /proc/sys/kernel/core_pattern
244 if [[ $foundcrashes -gt 0 ]]; then