tests/zfs-tests/tests/functional/mmp/mmp.kshlib

   1 #
   2 # CDDL HEADER START
   3 #
   4 # The contents of this file are subject to the terms of the
   5 # Common Development and Distribution License (the "License").
   6 # You may not use this file except in compliance with the License.
   7 #
   8 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9 # or http://www.opensolaris.org/os/licensing.
  10 # See the License for the specific language governing permissions
  11 # and limitations under the License.
  12 #
  13 # When distributing Covered Code, include this CDDL HEADER in each
  14 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15 # If applicable, add the following below this CDDL HEADER, with the
  16 # fields enclosed by brackets "[]" replaced with your own identifying
  17 # information: Portions Copyright [yyyy] [name of copyright owner]
  18 #
  19 # CDDL HEADER END
  20 #
  21
  22 #
  23 # Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
  24 # Use is subject to license terms.
  25 #
  26
  27 . $STF_SUITE/include/libtest.shlib
  28 . $STF_SUITE/tests/functional/mmp/mmp.cfg
  29
  30
  31 function check_pool_import # pool opts token keyword
  32 {
  33         typeset pool=${1:-$MMP_POOL}
  34         typeset opts=$2
  35         typeset token=$3
  36         typeset keyword=$4
  37
  38         zpool import $opts 2>&1 | \
  39             nawk -v token="$token:" '($1==token) {print $0}' | \
  40             grep -i "$keyword" > /dev/null 2>&1
  41
  42         return $?
  43 }
  44
  45 function is_pool_imported # pool opts
  46 {
  47         typeset pool=${1:-$MMP_POOL}
  48         typeset opts=$2
  49
  50         check_pool_import "$pool" "$opts" "status" \
  51             "The pool is currently imported"
  52         return $?
  53 }
  54
  55 function wait_pool_imported # pool opts
  56 {
  57         typeset pool=${1:-$MMP_POOL}
  58         typeset opts=$2
  59
  60         while is_pool_imported "$pool" "$opts"; do
  61                 log_must sleep 5
  62         done
  63
  64         return 0
  65 }
  66
  67 function try_pool_import # pool opts message
  68 {
  69         typeset pool=${1:-$MMP_POOL}
  70         typeset opts=$2
  71         typeset msg=$3
  72
  73         zpool import $opts $pool 2>&1 | grep -i "$msg"
  74
  75         return $?
  76 }
  77
  78 function mmp_set_hostid
  79 {
  80         typeset hostid=$1
  81
  82         zgenhostid $1
  83
  84         if [ $(hostid) != "$hostid" ]; then
  85                 return 1
  86         fi
  87
  88         return 0
  89 }
  90
  91 function mmp_clear_hostid
  92 {
  93         rm -f $HOSTID_FILE
  94 }
  95
  96 function mmp_pool_create_simple # pool dir
  97 {
  98         typeset pool=${1:-$MMP_POOL}
  99         typeset dir=${2:-$MMP_DIR}
 100
 101         log_must mkdir -p $dir
 102         log_must rm -f $dir/*
 103         log_must truncate -s $MINVDEVSIZE $dir/vdev1 $dir/vdev2
 104
 105         log_must mmp_clear_hostid
 106         log_must mmp_set_hostid $HOSTID1
 107         log_must zpool create -f -o cachefile=$MMP_CACHE $pool \
 108             mirror $dir/vdev1 $dir/vdev2
 109         log_must zpool set multihost=on $pool
 110 }
 111
 112 function mmp_pool_create # pool dir
 113 {
 114         typeset pool=${1:-$MMP_POOL}
 115         typeset dir=${2:-$MMP_DIR}
 116         typeset opts="-VVVVV -T120 -M -k0 -f $dir -E -p $pool"
 117
 118         mmp_pool_create_simple $pool $dir
 119
 120         log_must mv $MMP_CACHE ${MMP_CACHE}.stale
 121         log_must zpool export $pool
 122         log_must mmp_clear_hostid
 123         log_must mmp_set_hostid $HOSTID2
 124
 125         log_note "Starting ztest in the background as hostid $HOSTID1"
 126         log_must eval "ZFS_HOSTID=$HOSTID1 ztest $opts >$MMP_ZTEST_LOG 2>&1 &"
 127
 128         while ! is_pool_imported "$pool" "-d $dir"; do
 129                 log_must pgrep ztest
 130                 log_must sleep 5
 131         done
 132 }
 133
 134 function mmp_pool_destroy # pool dir
 135 {
 136         typeset pool=${1:-$MMP_POOL}
 137         typeset dir=${2:-$MMP_DIR}
 138
 139         ZTESTPID=$(pgrep ztest)
 140         if [ -n "$ZTESTPID" ]; then
 141                 log_must kill $ZTESTPID
 142                 wait $ZTESTPID
 143         fi
 144
 145         if poolexists $pool; then
 146                 destroy_pool $pool
 147         fi
 148
 149         log_must rm -f $dir/*
 150         mmp_clear_hostid
 151 }
 152
 153 function mmp_pool_set_hostid # pool hostid
 154 {
 155         typeset pool=$1
 156         typeset hostid=$2
 157
 158         log_must mmp_clear_hostid
 159         log_must mmp_set_hostid $hostid
 160         log_must zpool export $pool
 161         log_must zpool import $pool
 162
 163         return 0
 164 }
 165 # Return the number of seconds the activity check portion of the import process
 166 # will take.  Does not include the time to find devices and assemble a config.
 167 # Note that the activity check may be skipped, e.g. if the pool and host
 168 # hostid's match, but this will return non-zero because mmp_* are populated.
 169 function seconds_mmp_waits_for_activity
 170 {
 171         typeset pool=$1
 172         typeset devpath=$2
 173
 174         typeset seconds=0
 175         typeset devices=${#DISK[@]}
 176         typeset import_intervals=$(get_tunable MULTIHOST_IMPORT_INTERVALS)
 177         typeset import_interval=$(get_tunable MULTIHOST_INTERVAL)
 178         typeset tmpfile=$(mktemp)
 179         typeset mmp_fail
 180         typeset mmp_write
 181         typeset mmp_delay
 182
 183         log_must zdb -e -p $devpath $pool >$tmpfile 2>/dev/null
 184         mmp_fail=$(awk '/mmp_fail/ {print $NF}' $tmpfile)
 185         mmp_write=$(awk '/mmp_write/ {print $NF}' $tmpfile)
 186         mmp_delay=$(awk '/mmp_delay/ {print $NF}' $tmpfile)
 187         if [ -f $tmpfile ]; then
 188                 rm $tmpfile
 189         fi
 190
 191         # In order of preference:
 192         if [ -n $mmp_fail -a -n $mmp_write ]; then
 193                 seconds=$((2*mmp_fail*mmp_write/1000))
 194         elif [ -n $mmp_delay ]; then
 195                 # MMP V0: Based on mmp_delay from the best Uberblock
 196                 seconds=$((import_intervals*devices*mmp_delay/1000000000))
 197         else
 198                 # Non-MMP aware: Based on zfs_multihost_interval and import_intervals
 199                 seconds=$((import_intervals*import_interval/1000))
 200         fi
 201
 202         echo $seconds
 203 }
 204
 205 function import_no_activity_check # pool opts
 206 {
 207         typeset pool=$1
 208         typeset opts=$2
 209
 210         typeset max_duration=$((MMP_TEST_DURATION_DEFAULT-1))
 211
 212         SECONDS=0
 213         zpool import $opts $pool
 214         typeset rc=$?
 215
 216         if [[ $SECONDS -gt $max_duration ]]; then
 217                 log_fail "ERROR: import_no_activity_check unexpected activity \
 218 check (${SECONDS}s gt $max_duration)"
 219         fi
 220
 221         return $rc
 222 }
 223
 224 function import_activity_check # pool opts act_test_duration
 225 {
 226         typeset pool=$1
 227         typeset opts=$2
 228         typeset min_duration=${3:-$MMP_TEST_DURATION_DEFAULT}
 229
 230         SECONDS=0
 231         zpool import $opts $pool
 232         typeset rc=$?
 233
 234         if [[ $SECONDS -le $min_duration ]]; then
 235                 log_fail "ERROR: import_activity_check expected activity check \
 236 (${SECONDS}s le min_duration $min_duration)"
 237         fi
 238
 239         return $rc
 240 }
 241
 242 function clear_mmp_history
 243 {
 244         log_must set_tunable64 MULTIHOST_HISTORY $MMP_HISTORY_OFF
 245         log_must set_tunable64 MULTIHOST_HISTORY $MMP_HISTORY
 246 }
 247
 248 function count_skipped_mmp_writes # pool duration
 249 {
 250         typeset pool=$1
 251         typeset -i duration=$2
 252         typeset hist_path="/proc/spl/kstat/zfs/$pool/multihost"
 253
 254         sleep $duration
 255         awk 'BEGIN {count=0}; $NF == "-" {count++}; END {print count};' "$hist_path"
 256 }
 257
 258 function count_mmp_writes # pool duration
 259 {
 260         typeset pool=$1
 261         typeset -i duration=$2
 262         typeset hist_path="/proc/spl/kstat/zfs/$pool/multihost"
 263
 264         sleep $duration
 265         awk 'BEGIN {count=0}; $NF != "-" {count++}; END {print count};' "$hist_path"
 266 }
 267
 268 function summarize_uberblock_mmp # device
 269 {
 270         typeset device=$1
 271
 272         zdb -luuuu $device | awk '
 273         BEGIN                           {write_fail_present=0; write_fail_missing=0; uber_invalid=0;}
 274         /Uberblock\[[0-9][0-9]*\]/      {delay=-99; write=-99; fail=-99; total++; if (/invalid/) {uber_invalid++};};
 275         /mmp_fail/                      {fail=$3};
 276         /mmp_seq/                       {seq=$3};
 277         /mmp_write/                     {write=$3};
 278         /mmp_delay/                     {delay=$3; if (delay==0) {delay_zero++};};
 279         /mmp_valid/ && delay>0 && write>0 && fail>0 {write_fail_present++};
 280         /mmp_valid/ && delay>0 && (write<=0 || fail<=0) {write_fail_missing++};
 281         /mmp_valid/ && delay>0 && write<=0 {write_missing++};
 282         /mmp_valid/ && delay>0 && fail<=0 {fail_missing++};
 283         /mmp_valid/ && delay>0 && seq>0 {seq_nonzero++};
 284         END {
 285                 print "total_uberblocks " total;
 286                 print "delay_zero " delay_zero;
 287                 print "write_fail_present " write_fail_present;
 288                 print "write_fail_missing " write_fail_missing;
 289                 print "write_missing " write_missing;
 290                 print "fail_missing " fail_missing;
 291                 print "seq_nonzero " seq_nonzero;
 292                 print "uberblock_invalid " uber_invalid;
 293         }'
 294 }
 295
 296 function count_mmp_write_fail_present # device
 297 {
 298         typeset device=$1
 299
 300         summarize_uberblock_mmp $device | awk '/write_fail_present/ {print $NF}'
 301 }
 302
 303 function count_mmp_write_fail_missing # device
 304 {
 305         typeset device=$1
 306
 307         summarize_uberblock_mmp $device | awk '/write_fail_missing/ {print $NF}'
 308 }
 309
 310 function verify_mmp_write_fail_present # device
 311 {
 312         typeset device=$1
 313
 314         count=$(count_mmp_write_fail_present $device)
 315         log_note "present count: $count"
 316         if [ $count -eq 0 ]; then
 317                 summarize_uberblock_mmp $device
 318                 log_note "----- snip -----"
 319                 zdb -luuuu $device
 320                 log_note "----- snip -----"
 321                 log_fail "No Uberblocks contain valid mmp_write and fail values"
 322         fi
 323
 324         count=$(count_mmp_write_fail_missing $device)
 325         log_note "missing count: $count"
 326         if [ $count -gt 0 ]; then
 327                 summarize_uberblock_mmp $device
 328                 log_note "----- snip -----"
 329                 zdb -luuuu $device
 330                 log_note "----- snip -----"
 331                 log_fail "Uberblocks missing mmp_write or mmp_fail"
 332         fi
 333 }