scripts/zfs-tests.sh

   1 #!/bin/sh
   2 # shellcheck disable=SC2154
   3 #
   4 # CDDL HEADER START
   5 #
   6 # The contents of this file are subject to the terms of the
   7 # Common Development and Distribution License, Version 1.0 only
   8 # (the "License").  You may not use this file except in compliance
   9 # with the License.
  10 #
  11 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  12 # or https://opensource.org/licenses/CDDL-1.0.
  13 # See the License for the specific language governing permissions
  14 # and limitations under the License.
  15 #
  16 # When distributing Covered Code, include this CDDL HEADER in each
  17 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  18 # If applicable, add the following below this CDDL HEADER, with the
  19 # fields enclosed by brackets "[]" replaced with your own identifying
  20 # information: Portions Copyright [yyyy] [name of copyright owner]
  21 #
  22 # CDDL HEADER END
  23 #
  24
  25 #
  26 # Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
  27 #
  28
  29 SCRIPT_COMMON=${SCRIPT_COMMON:-${0%/*}/common.sh}
  30 . "${SCRIPT_COMMON}" || exit
  31
  32 PROG=zfs-tests.sh
  33 VERBOSE="no"
  34 QUIET=""
  35 DEBUG=""
  36 CLEANUP="yes"
  37 CLEANUPALL="no"
  38 KMSG=""
  39 LOOPBACK="yes"
  40 STACK_TRACER="no"
  41 FILESIZE="4G"
  42 DEFAULT_RUNFILES="common.run,$(uname | tr '[:upper:]' '[:lower:]').run"
  43 RUNFILES=${RUNFILES:-$DEFAULT_RUNFILES}
  44 FILEDIR=${FILEDIR:-/var/tmp}
  45 DISKS=${DISKS:-""}
  46 SINGLETEST=""
  47 SINGLETESTUSER="root"
  48 TAGS=""
  49 ITERATIONS=1
  50 ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh"
  51 ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh"
  52 UNAME=$(uname)
  53 RERUN=""
  54 KMEMLEAK=""
  55
  56 # Override some defaults if on FreeBSD
  57 if [ "$UNAME" = "FreeBSD" ] ; then
  58         TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DMESG"}
  59         LOSETUP=/sbin/mdconfig
  60         DMSETUP=/sbin/gpart
  61 else
  62         ZFS_MMP="$STF_SUITE/callbacks/zfs_mmp.ksh"
  63         TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DBGMSG:$ZFS_DMESG:$ZFS_MMP"}
  64         LOSETUP=${LOSETUP:-/sbin/losetup}
  65         DMSETUP=${DMSETUP:-/sbin/dmsetup}
  66 fi
  67
  68 #
  69 # Log an informational message when additional verbosity is enabled.
  70 #
  71 msg() {
  72         if [ "$VERBOSE" = "yes" ]; then
  73                 echo "$@"
  74         fi
  75 }
  76
  77 #
  78 # Log a failure message, cleanup, and return an error.
  79 #
  80 fail() {
  81         echo "$PROG: $1" >&2
  82         cleanup
  83         exit 1
  84 }
  85
  86 cleanup_freebsd_loopback() {
  87         for TEST_LOOPBACK in ${LOOPBACKS}; do
  88                 if [ -c "/dev/${TEST_LOOPBACK}" ]; then
  89                         sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}" ||
  90                             echo "Failed to destroy: ${TEST_LOOPBACK}"
  91                 fi
  92         done
  93 }
  94
  95 cleanup_linux_loopback() {
  96         for TEST_LOOPBACK in ${LOOPBACKS}; do
  97                 LOOP_DEV="${TEST_LOOPBACK##*/}"
  98                 DM_DEV=$(sudo "${DMSETUP}" ls 2>/dev/null | \
  99                     awk -v l="${LOOP_DEV}" '$0 ~ l {print $1}')
 100
 101                 if [ -n "$DM_DEV" ]; then
 102                         sudo "${DMSETUP}" remove "${DM_DEV}" ||
 103                             echo "Failed to remove: ${DM_DEV}"
 104                 fi
 105
 106                 if [ -n "${TEST_LOOPBACK}" ]; then
 107                         sudo "${LOSETUP}" -d "${TEST_LOOPBACK}" ||
 108                             echo "Failed to remove: ${TEST_LOOPBACK}"
 109                 fi
 110         done
 111 }
 112
 113 #
 114 # Attempt to remove loopback devices and files which where created earlier
 115 # by this script to run the test framework.  The '-k' option may be passed
 116 # to the script to suppress cleanup for debugging purposes.
 117 #
 118 cleanup() {
 119         if [ "$CLEANUP" = "no" ]; then
 120                 return 0
 121         fi
 122
 123
 124         if [ "$LOOPBACK" = "yes" ]; then
 125                 if [ "$UNAME" = "FreeBSD" ] ; then
 126                         cleanup_freebsd_loopback
 127                 else
 128                         cleanup_linux_loopback
 129                 fi
 130         fi
 131
 132         # shellcheck disable=SC2086
 133         rm -f ${FILES} >/dev/null 2>&1
 134
 135         if [ "$STF_PATH_REMOVE" = "yes" ] && [ -d "$STF_PATH" ]; then
 136                 rm -Rf "$STF_PATH"
 137         fi
 138 }
 139 trap cleanup EXIT
 140
 141 #
 142 # Attempt to remove all testpools (testpool.XXX), unopened dm devices,
 143 # loopback devices, and files.  This is a useful way to cleanup a previous
 144 # test run failure which has left the system in an unknown state.  This can
 145 # be dangerous and should only be used in a dedicated test environment.
 146 #
 147 cleanup_all() {
 148         TEST_POOLS=$(ASAN_OPTIONS=detect_leaks=false "$ZPOOL" list -Ho name | grep testpool)
 149         if [ "$UNAME" = "FreeBSD" ] ; then
 150                 TEST_LOOPBACKS=$(sudo "${LOSETUP}" -l)
 151         else
 152                 TEST_LOOPBACKS=$("${LOSETUP}" -a | awk -F: '/file-vdev/ {print $1}')
 153         fi
 154         TEST_FILES=$(ls "${FILEDIR}"/file-vdev* /var/tmp/file-vdev* 2>/dev/null)
 155
 156         msg
 157         msg "--- Cleanup ---"
 158         # shellcheck disable=2116,2086
 159         msg "Removing pool(s):     $(echo ${TEST_POOLS})"
 160         for TEST_POOL in $TEST_POOLS; do
 161                 sudo env ASAN_OPTIONS=detect_leaks=false "$ZPOOL" destroy "${TEST_POOL}"
 162         done
 163
 164         if [ "$UNAME" != "FreeBSD" ] ; then
 165                 msg "Removing all dm(s):   $(sudo "${DMSETUP}" ls |
 166                     grep loop | tr '\n' ' ')"
 167                 sudo "${DMSETUP}" remove_all
 168         fi
 169
 170         # shellcheck disable=2116,2086
 171         msg "Removing loopback(s): $(echo ${TEST_LOOPBACKS})"
 172         for TEST_LOOPBACK in $TEST_LOOPBACKS; do
 173                 if [ "$UNAME" = "FreeBSD" ] ; then
 174                         sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}"
 175                 else
 176                         sudo "${LOSETUP}" -d "${TEST_LOOPBACK}"
 177                 fi
 178         done
 179
 180         # shellcheck disable=2116,2086
 181         msg "Removing files(s):    $(echo ${TEST_FILES})"
 182         # shellcheck disable=2086
 183         sudo rm -f ${TEST_FILES}
 184 }
 185
 186 #
 187 # Takes a name as the only arguments and looks for the following variations
 188 # on that name.  If one is found it is returned.
 189 #
 190 # $RUNFILE_DIR/<name>
 191 # $RUNFILE_DIR/<name>.run
 192 # <name>
 193 # <name>.run
 194 #
 195 find_runfile() {
 196         NAME=$1
 197
 198         if [ -f "$RUNFILE_DIR/$NAME" ]; then
 199                 echo "$RUNFILE_DIR/$NAME"
 200         elif [ -f "$RUNFILE_DIR/$NAME.run" ]; then
 201                 echo "$RUNFILE_DIR/$NAME.run"
 202         elif [ -f "$NAME" ]; then
 203                 echo "$NAME"
 204         elif [ -f "$NAME.run" ]; then
 205                 echo "$NAME.run"
 206         else
 207                 return 1
 208         fi
 209 }
 210
 211 #
 212 # Symlink file if it appears under any of the given paths.
 213 #
 214 create_links() {
 215         dir_list="$1"
 216         file_list="$2"
 217
 218         [ -n "$STF_PATH" ] || fail "STF_PATH wasn't correctly set"
 219
 220         for i in $file_list; do
 221                 for j in $dir_list; do
 222                         [ ! -e "$STF_PATH/$i" ] || continue
 223
 224                         if [ ! -d "$j/$i" ] && [ -e "$j/$i" ]; then
 225                                 ln -sf "$j/$i" "$STF_PATH/$i" || \
 226                                     fail "Couldn't link $i"
 227                                 break
 228                         fi
 229                 done
 230
 231                 [ ! -e "$STF_PATH/$i" ] && \
 232                     STF_MISSING_BIN="$STF_MISSING_BIN $i"
 233         done
 234         STF_MISSING_BIN=${STF_MISSING_BIN# }
 235 }
 236
 237 #
 238 # Constrain the path to limit the available binaries to a known set.
 239 # When running in-tree a top level ./bin/ directory is created for
 240 # convenience, otherwise a temporary directory is used.
 241 #
 242 constrain_path() {
 243         . "$STF_SUITE/include/commands.cfg"
 244
 245         # On FreeBSD, base system zfs utils are in /sbin and OpenZFS utils
 246         # install to /usr/local/sbin. To avoid testing the wrong utils we
 247         # need /usr/local to come before / in the path search order.
 248         SYSTEM_DIRS="/usr/local/bin /usr/local/sbin"
 249         SYSTEM_DIRS="$SYSTEM_DIRS /usr/bin /usr/sbin /bin /sbin $LIBEXEC_DIR"
 250
 251         if [ "$INTREE" = "yes" ]; then
 252                 # Constrained path set to $(top_builddir)/tests/zfs-tests/bin
 253                 STF_PATH="$BIN_DIR"
 254                 STF_PATH_REMOVE="no"
 255                 STF_MISSING_BIN=""
 256                 if [ ! -d "$STF_PATH" ]; then
 257                         mkdir "$STF_PATH"
 258                         chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
 259                 fi
 260
 261                 # Special case links for standard zfs utilities
 262                 create_links "$CMD_DIR" "$ZFS_FILES"
 263
 264                 # Special case links for zfs test suite utilities
 265                 create_links "$CMD_DIR/tests/zfs-tests/cmd" "$ZFSTEST_FILES"
 266         else
 267                 # Constrained path set to /var/tmp/constrained_path.*
 268                 SYSTEMDIR=${SYSTEMDIR:-/var/tmp/constrained_path.XXXXXX}
 269                 STF_PATH=$(mktemp -d "$SYSTEMDIR")
 270                 STF_PATH_REMOVE="yes"
 271                 STF_MISSING_BIN=""
 272
 273                 chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
 274
 275                 # Special case links for standard zfs utilities
 276                 create_links "$SYSTEM_DIRS" "$ZFS_FILES"
 277
 278                 # Special case links for zfs test suite utilities
 279                 create_links "$STF_SUITE/bin" "$ZFSTEST_FILES"
 280         fi
 281
 282         # Standard system utilities
 283         SYSTEM_FILES="$SYSTEM_FILES_COMMON"
 284         if [ "$UNAME" = "FreeBSD" ] ; then
 285                 SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_FREEBSD"
 286         else
 287                 SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_LINUX"
 288         fi
 289         create_links "$SYSTEM_DIRS" "$SYSTEM_FILES"
 290
 291         # Exceptions
 292         if [ "$UNAME" = "Linux" ] ; then
 293                 ln -fs /sbin/fsck.ext4 "$STF_PATH/fsck"
 294                 ln -fs /sbin/mkfs.ext4 "$STF_PATH/newfs"
 295                 ln -fs "$STF_PATH/gzip" "$STF_PATH/compress"
 296                 ln -fs "$STF_PATH/gunzip" "$STF_PATH/uncompress"
 297         elif [ "$UNAME" = "FreeBSD" ] ; then
 298                 ln -fs /usr/local/bin/ksh93 "$STF_PATH/ksh"
 299         fi
 300 }
 301
 302 #
 303 # Output a useful usage message.
 304 #
 305 usage() {
 306 cat << EOF
 307 USAGE:
 308 $0 [-hvqxkfS] [-s SIZE] [-r RUNFILES] [-t PATH] [-u USER]
 309
 310 DESCRIPTION:
 311         ZFS Test Suite launch script
 312
 313 OPTIONS:
 314         -h          Show this message
 315         -v          Verbose zfs-tests.sh output
 316         -q          Quiet test-runner output
 317         -D          Debug; show all test output immediately (noisy)
 318         -x          Remove all testpools, dm, lo, and files (unsafe)
 319         -k          Disable cleanup after test failure
 320         -K          Log test names to /dev/kmsg
 321         -f          Use files only, disables block device tests
 322         -S          Enable stack tracer (negative performance impact)
 323         -c          Only create and populate constrained path
 324         -R          Automatically rerun failing tests
 325         -m          Enable kmemleak reporting (Linux only)
 326         -n NFSFILE  Use the nfsfile to determine the NFS configuration
 327         -I NUM      Number of iterations
 328         -d DIR      Use world-writable DIR for files and loopback devices
 329         -s SIZE     Use vdevs of SIZE (default: 4G)
 330         -r RUNFILES Run tests in RUNFILES (default: ${DEFAULT_RUNFILES})
 331         -t PATH|NAME  Run single test at PATH relative to test suite,
 332                         or search for test by NAME
 333         -T TAGS     Comma separated list of tags (default: 'functional')
 334         -u USER     Run single test as USER (default: root)
 335
 336 EXAMPLES:
 337 # Run the default ($(echo "${DEFAULT_RUNFILES}" | sed 's/\.run//')) suite of tests and output the configuration used.
 338 $0 -v
 339
 340 # Run a smaller suite of tests designed to run more quickly.
 341 $0 -r linux-fast
 342
 343 # Run a single test
 344 $0 -t tests/functional/cli_root/zfs_bookmark/zfs_bookmark_cliargs.ksh
 345
 346 # Run a single test by name
 347 $0 -t zfs_bookmark_cliargs
 348
 349 # Cleanup a previous run of the test suite prior to testing, run the
 350 # default ($(echo "${DEFAULT_RUNFILES}" | sed 's/\.run//')) suite of tests and perform no cleanup on exit.
 351 $0 -x
 352
 353 EOF
 354 }
 355
 356 while getopts 'hvqxkKfScRmn:d:Ds:r:?t:T:u:I:' OPTION; do
 357         case $OPTION in
 358         h)
 359                 usage
 360                 exit 1
 361                 ;;
 362         v)
 363                 VERBOSE="yes"
 364                 ;;
 365         q)
 366                 QUIET="yes"
 367                 ;;
 368         x)
 369                 CLEANUPALL="yes"
 370                 ;;
 371         k)
 372                 CLEANUP="no"
 373                 ;;
 374         K)
 375                 KMSG="yes"
 376                 ;;
 377         f)
 378                 LOOPBACK="no"
 379                 ;;
 380         S)
 381                 STACK_TRACER="yes"
 382                 ;;
 383         c)
 384                 constrain_path
 385                 exit
 386                 ;;
 387         R)
 388                 RERUN="yes"
 389                 ;;
 390         m)
 391                 KMEMLEAK="yes"
 392                 ;;
 393         n)
 394                 nfsfile=$OPTARG
 395                 [ -f "$nfsfile" ] || fail "Cannot read file: $nfsfile"
 396                 export NFS=1
 397                 . "$nfsfile"
 398                 ;;
 399         d)
 400                 FILEDIR="$OPTARG"
 401                 ;;
 402         D)
 403                 DEBUG="yes"
 404                 ;;
 405         I)
 406                 ITERATIONS="$OPTARG"
 407                 if [ "$ITERATIONS" -le 0 ]; then
 408                         fail "Iterations must be greater than 0."
 409                 fi
 410                 ;;
 411         s)
 412                 FILESIZE="$OPTARG"
 413                 ;;
 414         r)
 415                 RUNFILES="$OPTARG"
 416                 ;;
 417         t)
 418                 if [ -n "$SINGLETEST" ]; then
 419                         fail "-t can only be provided once."
 420                 fi
 421                 SINGLETEST="$OPTARG"
 422                 ;;
 423         T)
 424                 TAGS="$OPTARG"
 425                 ;;
 426         u)
 427                 SINGLETESTUSER="$OPTARG"
 428                 ;;
 429         ?)
 430                 usage
 431                 exit
 432                 ;;
 433         *)
 434                 ;;
 435         esac
 436 done
 437
 438 shift $((OPTIND-1))
 439
 440 FILES=${FILES:-"$FILEDIR/file-vdev0 $FILEDIR/file-vdev1 $FILEDIR/file-vdev2"}
 441 LOOPBACKS=${LOOPBACKS:-""}
 442
 443 if [ -n "$SINGLETEST" ]; then
 444         if [ -n "$TAGS" ]; then
 445                 fail "-t and -T are mutually exclusive."
 446         fi
 447         RUNFILE_DIR="/var/tmp"
 448         RUNFILES="zfs-tests.$$.run"
 449         [ -n "$QUIET" ] && SINGLEQUIET="True" || SINGLEQUIET="False"
 450
 451         cat >"${RUNFILE_DIR}/${RUNFILES}" << EOF
 452 [DEFAULT]
 453 pre =
 454 quiet = $SINGLEQUIET
 455 pre_user = root
 456 user = $SINGLETESTUSER
 457 timeout = 600
 458 post_user = root
 459 post =
 460 outputdir = /var/tmp/test_results
 461 EOF
 462         if [ "$SINGLETEST" = "${SINGLETEST%/*}" ] ; then
 463                 NEWSINGLETEST=$(find "$STF_SUITE" -name "$SINGLETEST*" -print -quit)
 464                 if [ -z "$NEWSINGLETEST" ] ; then
 465                         fail "couldn't find test matching '$SINGLETEST'"
 466                 fi
 467                 SINGLETEST=$NEWSINGLETEST
 468         fi
 469
 470         SINGLETESTDIR="${SINGLETEST%/*}"
 471         SETUPDIR="$SINGLETESTDIR"
 472         [ "${SETUPDIR#/}" = "$SETUPDIR" ] && SETUPDIR="$STF_SUITE/$SINGLETESTDIR"
 473         [ -x "$SETUPDIR/setup.ksh"   ] && SETUPSCRIPT="setup"     || SETUPSCRIPT=
 474         [ -x "$SETUPDIR/cleanup.ksh" ] && CLEANUPSCRIPT="cleanup" || CLEANUPSCRIPT=
 475
 476         SINGLETESTFILE="${SINGLETEST##*/}"
 477         cat >>"${RUNFILE_DIR}/${RUNFILES}" << EOF
 478
 479 [$SINGLETESTDIR]
 480 tests = ['$SINGLETESTFILE']
 481 pre = $SETUPSCRIPT
 482 post = $CLEANUPSCRIPT
 483 tags = ['functional']
 484 EOF
 485 fi
 486
 487 #
 488 # Use default tag if none was specified
 489 #
 490 TAGS=${TAGS:='functional'}
 491
 492 #
 493 # Attempt to locate the runfiles describing the test workload.
 494 #
 495 R=""
 496 IFS=,
 497 for RUNFILE in $RUNFILES; do
 498         if [ -n "$RUNFILE" ]; then
 499                 SAVED_RUNFILE="$RUNFILE"
 500                 RUNFILE=$(find_runfile "$RUNFILE") ||
 501                         fail "Cannot find runfile: $SAVED_RUNFILE"
 502                 R="$R,$RUNFILE"
 503         fi
 504
 505         if [ ! -r "$RUNFILE" ]; then
 506                 fail "Cannot read runfile: $RUNFILE"
 507         fi
 508 done
 509 unset IFS
 510 RUNFILES=${R#,}
 511
 512 #
 513 # This script should not be run as root.  Instead the test user, which may
 514 # be a normal user account, needs to be configured such that it can
 515 # run commands via sudo passwordlessly.
 516 #
 517 if [ "$(id -u)" = "0" ]; then
 518         fail "This script must not be run as root."
 519 fi
 520
 521 if [ "$(sudo id -un)" != "root" ]; then
 522         fail "Passwordless sudo access required."
 523 fi
 524
 525 #
 526 # Constrain the available binaries to a known set.
 527 #
 528 constrain_path
 529
 530 #
 531 # Check if ksh exists
 532 #
 533 if [ "$UNAME" = "FreeBSD" ]; then
 534         sudo ln -fs /usr/local/bin/ksh93 /bin/ksh
 535 fi
 536 [ -e "$STF_PATH/ksh" ] || fail "This test suite requires ksh."
 537 [ -e "$STF_SUITE/include/default.cfg" ] || fail \
 538     "Missing $STF_SUITE/include/default.cfg file."
 539
 540 #
 541 # Verify the ZFS module stack is loaded.
 542 #
 543 if [ "$STACK_TRACER" = "yes" ]; then
 544         sudo "${ZFS_SH}" -S >/dev/null 2>&1
 545 else
 546         sudo "${ZFS_SH}" >/dev/null 2>&1
 547 fi
 548
 549 #
 550 # Attempt to cleanup all previous state for a new test run.
 551 #
 552 if [ "$CLEANUPALL" = "yes" ]; then
 553         cleanup_all
 554 fi
 555
 556 #
 557 # By default preserve any existing pools
 558 #
 559 if [ -z "${KEEP}" ]; then
 560         KEEP="$(ASAN_OPTIONS=detect_leaks=false "$ZPOOL" list -Ho name | tr -s '[:space:]' ' ')"
 561         if [ -z "${KEEP}" ]; then
 562                 KEEP="rpool"
 563         fi
 564 else
 565         KEEP="$(echo "$KEEP" | tr -s '[:space:]' ' ')"
 566 fi
 567
 568 #
 569 # NOTE: The following environment variables are undocumented
 570 # and should be used for testing purposes only:
 571 #
 572 # __ZFS_POOL_EXCLUDE - don't iterate over the pools it lists
 573 # __ZFS_POOL_RESTRICT - iterate only over the pools it lists
 574 #
 575 # See libzfs/libzfs_config.c for more information.
 576 #
 577 __ZFS_POOL_EXCLUDE="$KEEP"
 578
 579 . "$STF_SUITE/include/default.cfg"
 580
 581 #
 582 # No DISKS have been provided so a basic file or loopback based devices
 583 # must be created for the test suite to use.
 584 #
 585 if [ -z "${DISKS}" ]; then
 586         #
 587         # If this is a performance run, prevent accidental use of
 588         # loopback devices.
 589         #
 590         [ "$TAGS" = "perf" ] && fail "Running perf tests without disks."
 591
 592         #
 593         # Create sparse files for the test suite.  These may be used
 594         # directory or have loopback devices layered on them.
 595         #
 596         for TEST_FILE in ${FILES}; do
 597                 [ -f "$TEST_FILE" ] && fail "Failed file exists: ${TEST_FILE}"
 598                 truncate -s "${FILESIZE}" "${TEST_FILE}" ||
 599                     fail "Failed creating: ${TEST_FILE} ($?)"
 600         done
 601
 602         #
 603         # If requested setup loopback devices backed by the sparse files.
 604         #
 605         if [ "$LOOPBACK" = "yes" ]; then
 606                 test -x "$LOSETUP" || fail "$LOSETUP utility must be installed"
 607
 608                 for TEST_FILE in ${FILES}; do
 609                         if [ "$UNAME" = "FreeBSD" ] ; then
 610                                 MDDEVICE=$(sudo "${LOSETUP}" -a -t vnode -f "${TEST_FILE}")
 611                                 if [ -z "$MDDEVICE" ] ; then
 612                                         fail "Failed: ${TEST_FILE} -> loopback"
 613                                 fi
 614                                 DISKS="$DISKS $MDDEVICE"
 615                                 LOOPBACKS="$LOOPBACKS $MDDEVICE"
 616                         else
 617                                 TEST_LOOPBACK=$(sudo "${LOSETUP}" --show -f "${TEST_FILE}") ||
 618                                     fail "Failed: ${TEST_FILE} -> ${TEST_LOOPBACK}"
 619                                 BASELOOPBACK="${TEST_LOOPBACK##*/}"
 620                                 DISKS="$DISKS $BASELOOPBACK"
 621                                 LOOPBACKS="$LOOPBACKS $TEST_LOOPBACK"
 622                         fi
 623                 done
 624                 DISKS=${DISKS# }
 625                 LOOPBACKS=${LOOPBACKS# }
 626         else
 627                 DISKS="$FILES"
 628         fi
 629 fi
 630
 631 #
 632 # It may be desirable to test with fewer disks than the default when running
 633 # the performance tests, but the functional tests require at least three.
 634 #
 635 NUM_DISKS=$(echo "${DISKS}" | awk '{print NF}')
 636 if [ "$TAGS" != "perf" ]; then
 637         [ "$NUM_DISKS" -lt 3 ] && fail "Not enough disks ($NUM_DISKS/3 minimum)"
 638 fi
 639
 640 #
 641 # Disable SELinux until the ZFS Test Suite has been updated accordingly.
 642 #
 643 if command -v setenforce >/dev/null; then
 644         sudo setenforce permissive >/dev/null 2>&1
 645 fi
 646
 647 #
 648 # Enable internal ZFS debug log and clear it.
 649 #
 650 if [ -e /sys/module/zfs/parameters/zfs_dbgmsg_enable ]; then
 651         sudo sh -c "echo 1 >/sys/module/zfs/parameters/zfs_dbgmsg_enable"
 652         sudo sh -c "echo 0 >/proc/spl/kstat/zfs/dbgmsg"
 653 fi
 654
 655 msg
 656 msg "--- Configuration ---"
 657 msg "Runfiles:        $RUNFILES"
 658 msg "STF_TOOLS:       $STF_TOOLS"
 659 msg "STF_SUITE:       $STF_SUITE"
 660 msg "STF_PATH:        $STF_PATH"
 661 msg "FILEDIR:         $FILEDIR"
 662 msg "FILES:           $FILES"
 663 msg "LOOPBACKS:       $LOOPBACKS"
 664 msg "DISKS:           $DISKS"
 665 msg "NUM_DISKS:       $NUM_DISKS"
 666 msg "FILESIZE:        $FILESIZE"
 667 msg "ITERATIONS:      $ITERATIONS"
 668 msg "TAGS:            $TAGS"
 669 msg "STACK_TRACER:    $STACK_TRACER"
 670 msg "Keep pool(s):    $KEEP"
 671 msg "Missing util(s): $STF_MISSING_BIN"
 672 msg ""
 673
 674 export STF_TOOLS
 675 export STF_SUITE
 676 export STF_PATH
 677 export DISKS
 678 export FILEDIR
 679 export KEEP
 680 export __ZFS_POOL_EXCLUDE
 681 export TESTFAIL_CALLBACKS
 682
 683 mktemp_file() {
 684         if [ "$UNAME" = "FreeBSD" ]; then
 685                 mktemp -u "${FILEDIR}/$1.XXXXXX"
 686         else
 687                 mktemp -ut "$1.XXXXXX" -p "$FILEDIR"
 688         fi
 689 }
 690 mkdir -p "$FILEDIR" || :
 691 RESULTS_FILE=$(mktemp_file zts-results)
 692 REPORT_FILE=$(mktemp_file zts-report)
 693
 694 #
 695 # Run all the tests as specified.
 696 #
 697 msg "${TEST_RUNNER}" \
 698     "${QUIET:+-q}" \
 699     "${DEBUG:+-D}" \
 700     "${KMEMLEAK:+-m}" \
 701     "${KMSG:+-K}" \
 702     "-c \"${RUNFILES}\"" \
 703     "-T \"${TAGS}\"" \
 704     "-i \"${STF_SUITE}\"" \
 705     "-I \"${ITERATIONS}\""
 706 { PATH=$STF_PATH \
 707     ${TEST_RUNNER} \
 708     ${QUIET:+-q} \
 709     ${DEBUG:+-D} \
 710     ${KMEMLEAK:+-m} \
 711     ${KMSG:+-K} \
 712     -c "${RUNFILES}" \
 713     -T "${TAGS}" \
 714     -i "${STF_SUITE}" \
 715     -I "${ITERATIONS}" \
 716     2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE"
 717 read -r RUNRESULT <"$REPORT_FILE"
 718
 719 #
 720 # Analyze the results.
 721 #
 722 ${ZTS_REPORT} ${RERUN:+--no-maybes} "$RESULTS_FILE" >"$REPORT_FILE"
 723 RESULT=$?
 724
 725 if [ "$RESULT" -eq "2" ] && [ -n "$RERUN" ]; then
 726         MAYBES="$($ZTS_REPORT --list-maybes)"
 727         TEMP_RESULTS_FILE=$(mktemp_file zts-results-tmp)
 728         TEST_LIST=$(mktemp_file test-list)
 729         grep "^Test:.*\[FAIL\]" "$RESULTS_FILE" >"$TEMP_RESULTS_FILE"
 730         for test_name in $MAYBES; do
 731                 grep "$test_name " "$TEMP_RESULTS_FILE" >>"$TEST_LIST"
 732         done
 733         { PATH=$STF_PATH \
 734             ${TEST_RUNNER} \
 735                 ${QUIET:+-q} \
 736                 ${DEBUG:+-D} \
 737                 ${KMEMLEAK:+-m} \
 738             -c "${RUNFILES}" \
 739             -T "${TAGS}" \
 740             -i "${STF_SUITE}" \
 741             -I "${ITERATIONS}" \
 742             -l "${TEST_LIST}" \
 743             2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE"
 744         read -r RUNRESULT <"$REPORT_FILE"
 745         #
 746         # Analyze the results.
 747         #
 748         ${ZTS_REPORT} --no-maybes "$RESULTS_FILE" >"$REPORT_FILE"
 749         RESULT=$?
 750 fi
 751
 752
 753 cat "$REPORT_FILE"
 754
 755 RESULTS_DIR=$(awk '/^Log directory/ { print $3 }' "$RESULTS_FILE")
 756 if [ -d "$RESULTS_DIR" ]; then
 757         cat "$RESULTS_FILE" "$REPORT_FILE" >"$RESULTS_DIR/results"
 758 fi
 759
 760 rm -f "$RESULTS_FILE" "$REPORT_FILE" "$TEST_LIST" "$TEMP_RESULTS_FILE"
 761
 762 if [ -n "$SINGLETEST" ]; then
 763         rm -f "$RUNFILES" >/dev/null 2>&1
 764 fi
 765
 766 [ "$RUNRESULT" -gt 3 ] && exit "$RUNRESULT" || exit "$RESULT"