qemu/tests/qemu-iotests/common.rc

#!/usr/bin/env bash
#
# Copyright (C) 2009 Red Hat, Inc.
# Copyright (c) 2000-2006 Silicon Graphics, Inc.  All Rights Reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

SED=
for sed in sed gsed; do
    ($sed --version | grep 'GNU sed') > /dev/null 2>&1
    if [ "$?" -eq 0 ]; then
        SED=$sed
        break
    fi
done
if [ -z "$SED" ]; then
    echo "$0: GNU sed not found"
    exit 1
fi

dd()
{
   if [ "$HOSTOS" == "Linux" ]
   then
        command dd --help | grep noxfer > /dev/null 2>&1

        if [ "$?" -eq 0 ]
            then
                command dd status=noxfer $@
            else
                command dd $@
            fi
   else
        command dd $@
   fi
}

# poke_file 'test.img' 512 '\xff\xfe'
poke_file()
{
    printf "$3" | dd "of=$1" bs=1 "seek=$2" conv=notrunc &>/dev/null
}

# poke_file_le $img_filename $offset $byte_width $value
# Example: poke_file_le "$TEST_IMG" 512 2 65534
poke_file_le()
{
    local img=$1 ofs=$2 len=$3 val=$4 str=''

    while ((len--)); do
        str+=$(printf '\\x%02x' $((val & 0xff)))
        val=$((val >> 8))
    done

    poke_file "$img" "$ofs" "$str"
}

# poke_file_be $img_filename $offset $byte_width $value
# Example: poke_file_be "$TEST_IMG" 512 2 65279
poke_file_be()
{
    local img=$1 ofs=$2 len=$3 val=$4
    local str=$(printf "%0$((len * 2))x\n" $val | sed 's/\(..\)/\\x\1/g')

    poke_file "$img" "$ofs" "$str"
}

# peek_file_le 'test.img' 512 2 => 65534
peek_file_le()
{
    local val=0 shift=0 byte

    # coreutils' od --endian is not portable, so manually assemble bytes.
    for byte in $(od -j"$2" -N"$3" -An -v -tu1 "$1"); do
        val=$(( val | (byte << shift) ))
        shift=$((shift + 8))
    done
    printf %llu $val
}

# peek_file_be 'test.img' 512 2 => 65279
peek_file_be()
{
    local val=0 byte

    # coreutils' od --endian is not portable, so manually assemble bytes.
    for byte in $(od -j"$2" -N"$3" -An -v -tu1 "$1"); do
        val=$(( (val << 8) | byte ))
    done
    printf %llu $val
}

# peek_file_raw 'test.img' 512 2 => '\xff\xfe'. Do not use if the raw data
# is likely to contain \0 or trailing \n.
peek_file_raw()
{
    dd if="$1" bs=1 skip="$2" count="$3" status=none
}

config=common.config
test -f $config || config=../common.config
if ! test -f $config
then
    echo "$0: failed to find common.config"
    exit 1
fi
if ! . $config
    then
    echo "$0: failed to source common.config"
    exit 1
fi

# Set the variables to the empty string to turn Valgrind off
# for specific processes, e.g.
# $ VALGRIND_QEMU_IO= ./check -qcow2 -valgrind 015

: ${VALGRIND_QEMU_VM=$VALGRIND_QEMU}
: ${VALGRIND_QEMU_IMG=$VALGRIND_QEMU}
: ${VALGRIND_QEMU_IO=$VALGRIND_QEMU}
: ${VALGRIND_QEMU_NBD=$VALGRIND_QEMU}
: ${VALGRIND_QSD=$VALGRIND_QEMU}

# The Valgrind own parameters may be set with
# its environment variable VALGRIND_OPTS, e.g.
# $ VALGRIND_OPTS="--leak-check=yes" ./check -qcow2 -valgrind 015

_qemu_proc_exec()
{
    local VALGRIND_LOGFILE="$1"
    shift
    if [[ "${VALGRIND_QEMU}" == "y" && "${NO_VALGRIND}" != "y" ]]; then
        exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$@"
    else
        exec "$@"
    fi
}

_qemu_proc_valgrind_log()
{
    local VALGRIND_LOGFILE="$1"
    local RETVAL="$2"
    if [[ "${VALGRIND_QEMU}" == "y" && "${NO_VALGRIND}" != "y" ]]; then
        if [ $RETVAL == 99 ]; then
            cat "${VALGRIND_LOGFILE}"
        fi
        rm -f "${VALGRIND_LOGFILE}"
    fi
}

_qemu_wrapper()
{
    local VALGRIND_LOGFILE="${TEST_DIR}"/$$.valgrind
    (
        if [ -n "${QEMU_NEED_PID}" ]; then
            echo $BASHPID > "${QEMU_TEST_DIR}/qemu-${_QEMU_HANDLE}.pid"
        fi

        GDB=""
        if [ -n "${GDB_OPTIONS}" ]; then
            GDB="gdbserver ${GDB_OPTIONS}"
        fi

        VALGRIND_QEMU="${VALGRIND_QEMU_VM}" _qemu_proc_exec "${VALGRIND_LOGFILE}" \
            $GDB "$QEMU_PROG" $QEMU_OPTIONS "$@"
    )
    RETVAL=$?
    _qemu_proc_valgrind_log "${VALGRIND_LOGFILE}" $RETVAL
    return $RETVAL
}

_qemu_img_wrapper()
{
    local VALGRIND_LOGFILE="${TEST_DIR}"/$$.valgrind
    (
        VALGRIND_QEMU="${VALGRIND_QEMU_IMG}" _qemu_proc_exec "${VALGRIND_LOGFILE}" \
            "$QEMU_IMG_PROG" $QEMU_IMG_OPTIONS "$@"
    )
    RETVAL=$?
    _qemu_proc_valgrind_log "${VALGRIND_LOGFILE}" $RETVAL
    return $RETVAL
}

_qemu_io_wrapper()
{
    local VALGRIND_LOGFILE="${TEST_DIR}"/$$.valgrind
    local QEMU_IO_ARGS="$QEMU_IO_OPTIONS"
    if [ "$IMGOPTSSYNTAX" = "true" ]; then
        QEMU_IO_ARGS="--image-opts $QEMU_IO_ARGS"
        if [ -n "$IMGKEYSECRET" ]; then
            QEMU_IO_ARGS="--object secret,id=keysec0,data=$IMGKEYSECRET $QEMU_IO_ARGS"
        fi
    fi
    (
        VALGRIND_QEMU="${VALGRIND_QEMU_IO}" _qemu_proc_exec "${VALGRIND_LOGFILE}" \
            "$QEMU_IO_PROG" $QEMU_IO_ARGS "$@"
    )
    RETVAL=$?
    _qemu_proc_valgrind_log "${VALGRIND_LOGFILE}" $RETVAL
    return $RETVAL
}

_qemu_nbd_wrapper()
{
    local VALGRIND_LOGFILE="${TEST_DIR}"/$$.valgrind
    (
        VALGRIND_QEMU="${VALGRIND_QEMU_NBD}" _qemu_proc_exec "${VALGRIND_LOGFILE}" \
            "$QEMU_NBD_PROG" --pid-file="${QEMU_TEST_DIR}/qemu-nbd.pid" \
             $QEMU_NBD_OPTIONS "$@"
    )
    RETVAL=$?
    _qemu_proc_valgrind_log "${VALGRIND_LOGFILE}" $RETVAL
    return $RETVAL
}

_qemu_storage_daemon_wrapper()
{
    local VALGRIND_LOGFILE="${TEST_DIR}"/$$.valgrind
    (
        if [ -n "${QSD_NEED_PID}" ]; then
            echo $BASHPID > "${QEMU_TEST_DIR}/qemu-storage-daemon.pid"
        fi
        VALGRIND_QEMU="${VALGRIND_QSD}" _qemu_proc_exec "${VALGRIND_LOGFILE}" \
            "$QSD_PROG" $QSD_OPTIONS "$@"
    )
    RETVAL=$?
    _qemu_proc_valgrind_log "${VALGRIND_LOGFILE}" $RETVAL
    return $RETVAL
}

# Valgrind bug #409141 https://bugs.kde.org/show_bug.cgi?id=409141
# Until valgrind 3.16+ is ubiquitous, we must work around a hang in
# valgrind when issuing sigkill. Disable valgrind for this invocation.
_NO_VALGRIND()
{
    NO_VALGRIND="y" "$@"
}

export QEMU=_qemu_wrapper
export QEMU_IMG=_qemu_img_wrapper
export QEMU_IO=_qemu_io_wrapper
export QEMU_NBD=_qemu_nbd_wrapper
export QSD=_qemu_storage_daemon_wrapper

if [ "$IMGOPTSSYNTAX" = "true" ]; then
    DRIVER="driver=$IMGFMT"
    QEMU_IMG_EXTRA_ARGS="--image-opts $QEMU_IMG_EXTRA_ARGS"
    if [ -n "$IMGKEYSECRET" ]; then
        QEMU_IMG_EXTRA_ARGS="--object secret,id=keysec0,data=$IMGKEYSECRET $QEMU_IMG_EXTRA_ARGS"
    fi
    if [ "$IMGFMT" = "luks" ]; then
        DRIVER="$DRIVER,key-secret=keysec0"
    fi
    if [ "$IMGPROTO" = "file" ]; then
        TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
        TEST_IMG="$DRIVER,file.filename=$TEST_DIR/t.$IMGFMT"
    elif [ "$IMGPROTO" = "nbd" ]; then
        TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
        TEST_IMG="$DRIVER,file.driver=nbd,file.type=unix"
        TEST_IMG="$TEST_IMG,file.path=$SOCK_DIR/nbd"
    elif [ "$IMGPROTO" = "fuse" ]; then
        TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
        TEST_IMG="$DRIVER,file.filename=$SOCK_DIR/fuse-t.$IMGFMT"
    elif [ "$IMGPROTO" = "ssh" ]; then
        TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
        TEST_IMG="$DRIVER,file.driver=ssh,file.host=127.0.0.1,file.path=$TEST_IMG_FILE"
    elif [ "$IMGPROTO" = "nfs" ]; then
        TEST_DIR="$DRIVER,file.driver=nfs,file.filename=nfs://127.0.0.1/$TEST_DIR"
        TEST_IMG=$TEST_DIR/t.$IMGFMT
    else
        TEST_IMG="$DRIVER,file.driver=$IMGPROTO,file.filename=$TEST_DIR/t.$IMGFMT"
    fi
else
    QEMU_IMG_EXTRA_ARGS=
    if [ "$IMGPROTO" = "file" ]; then
        TEST_IMG=$TEST_DIR/t.$IMGFMT
    elif [ "$IMGPROTO" = "nbd" ]; then
        TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
        TEST_IMG="nbd+unix:///?socket=$SOCK_DIR/nbd"
    elif [ "$IMGPROTO" = "fuse" ]; then
        TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
        TEST_IMG="$SOCK_DIR/fuse-t.$IMGFMT"
    elif [ "$IMGPROTO" = "ssh" ]; then
        TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
        REMOTE_TEST_DIR="ssh://\\($USER@\\)\\?127.0.0.1\\(:[0-9]\\+\\)\\?$TEST_DIR"
        TEST_IMG="ssh://127.0.0.1$TEST_IMG_FILE"
    elif [ "$IMGPROTO" = "nfs" ]; then
        TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
        REMOTE_TEST_DIR="nfs://127.0.0.1$TEST_DIR"
        TEST_IMG="nfs://127.0.0.1$TEST_IMG_FILE"
    else
        TEST_IMG=$IMGPROTO:$TEST_DIR/t.$IMGFMT
    fi
fi
ORIG_TEST_IMG_FILE=$TEST_IMG_FILE
ORIG_TEST_IMG="$TEST_IMG"

FUSE_PIDS=()
FUSE_EXPORTS=()

if [ -z "$TEST_DIR" ]; then
        TEST_DIR=$PWD/scratch
fi

QEMU_TEST_DIR="${TEST_DIR}"

if [ ! -e "$TEST_DIR" ]; then
        mkdir "$TEST_DIR"
fi

if [ ! -d "$TEST_DIR" ]; then
    echo "common.rc: Error: \$TEST_DIR ($TEST_DIR) is not a directory"
    exit 1
fi

if [ -z "$REMOTE_TEST_DIR" ]; then
    REMOTE_TEST_DIR="$TEST_DIR"
fi

if [ ! -d "$SAMPLE_IMG_DIR" ]; then
    echo "common.rc: Error: \$SAMPLE_IMG_DIR ($SAMPLE_IMG_DIR) is not a directory"
    exit 1
fi

_use_sample_img()
{
    SAMPLE_IMG_FILE="${1%\.bz2}"
    TEST_IMG="$TEST_DIR/$SAMPLE_IMG_FILE"
    bzcat "$SAMPLE_IMG_DIR/$1" > "$TEST_IMG"
    if [ $? -ne 0 ]
    then
        echo "_use_sample_img error, cannot extract '$SAMPLE_IMG_DIR/$1'"
        exit 1
    fi
}

_stop_nbd_server()
{
    if [ -f "${QEMU_TEST_DIR}/qemu-nbd.pid" ]; then
        local QEMU_NBD_PID
        read QEMU_NBD_PID < "${QEMU_TEST_DIR}/qemu-nbd.pid"
        kill ${QEMU_NBD_PID}
        rm -f "${QEMU_TEST_DIR}/qemu-nbd.pid" "$SOCK_DIR/nbd"
    fi
}

# Gets the data_file value from IMGOPTS and replaces the '$TEST_IMG'
# pattern by '$1'
# Caution: The replacement is done with sed, so $1 must be escaped
#          properly.  (The delimiter is '#'.)
_get_data_file()
{
    if ! echo "$IMGOPTS" | grep -q 'data_file='; then
        return 1
    fi

    echo "$IMGOPTS" | sed -e 's/.*data_file=\([^,]*\).*/\1/' \
                    | sed -e "s#\\\$TEST_IMG#$1#"
}

# Translate a $TEST_IMG to its corresponding $TEST_IMG_FILE for
# different protocols
_test_img_to_test_img_file()
{
    case "$IMGPROTO" in
        file)
            echo "$1"
            ;;

        fuse)
            echo "$1" | sed -e "s#$SOCK_DIR/fuse-#$TEST_DIR/#"
            ;;

        nfs)
            echo "$1" | sed -e "s#nfs://127.0.0.1##"
            ;;

        ssh)
            echo "$1" | \
                sed -e "s#ssh://\\($USER@\\)\\?127.0.0.1\\(:[0-9]\\+\\)\\?##"
            ;;

        *)
            return 1
            ;;
    esac
}

_make_test_img()
{
    # extra qemu-img options can be added by tests
    # at least one argument (the image size) needs to be added
    local extra_img_options=""
    local optstr=""
    local img_name=""
    local use_backing=0
    local backing_file=""
    local object_options=""
    local opts_param=false
    local misc_params=()

    if [[ $IMGPROTO == fuse && $TEST_IMG == $SOCK_DIR/fuse-* ]]; then
        # The caller may be trying to overwrite an existing image
        _rm_test_img "$TEST_IMG"
    fi

    if [ -z "$TEST_IMG_FILE" ]; then
        img_name=$TEST_IMG
    elif [ "$IMGOPTSSYNTAX" != "true" -a \
           "$TEST_IMG_FILE" = "$ORIG_TEST_IMG_FILE" ]; then
        # Handle cases of tests only updating TEST_IMG, but not TEST_IMG_FILE
        img_name=$(_test_img_to_test_img_file "$TEST_IMG")
        if [ "$?" != 0 ]; then
            img_name=$TEST_IMG_FILE
        fi
    else
        # $TEST_IMG_FILE is not the default value, so it definitely has been
        # modified by the test
        img_name=$TEST_IMG_FILE
    fi

    if [ -n "$IMGOPTS" ]; then
        imgopts_expanded=$(echo "$IMGOPTS" | sed -e "s#\\\$TEST_IMG#$img_name#")
        optstr=$(_optstr_add "$optstr" "$imgopts_expanded")
    fi
    if [ -n "$IMGKEYSECRET" ]; then
        object_options="--object secret,id=keysec0,data=$IMGKEYSECRET"
        optstr=$(_optstr_add "$optstr" "key-secret=keysec0")
    fi

    for param; do
        if [ "$use_backing" = "1" -a -z "$backing_file" ]; then
            backing_file=$param
            continue
        elif $opts_param; then
            optstr=$(_optstr_add "$optstr" "$param")
            opts_param=false
            continue
        fi

        case "$param" in
            -b)
                use_backing=1
                ;;

            -o)
                opts_param=true
                ;;

            --no-opts)
                optstr=""
                ;;

            *)
                misc_params=("${misc_params[@]}" "$param")
                ;;
        esac
    done

    if [ \( "$IMGFMT" = "qcow2" -o "$IMGFMT" = "qed" \) -a -n "$CLUSTER_SIZE" ]; then
        optstr=$(_optstr_add "$optstr" "cluster_size=$CLUSTER_SIZE")
    fi

    if [ -n "$optstr" ]; then
        extra_img_options="-o $optstr $extra_img_options"
    fi

    if [ $IMGPROTO = "nbd" ]; then
        _stop_nbd_server
    fi

    # XXX(hch): have global image options?
    (
     if [ $use_backing = 1 ]; then
        $QEMU_IMG create $object_options -f $IMGFMT $extra_img_options -b "$backing_file" "$img_name" "${misc_params[@]}" 2>&1
     else
        $QEMU_IMG create $object_options -f $IMGFMT $extra_img_options "$img_name" "${misc_params[@]}" 2>&1
     fi
    ) | _filter_img_create

    # Start an NBD server on the image file, which is what we'll be talking to.
    # Once NBD gains resize support, we may also want to use -f raw at the
    # server and interpret format over NBD, but for now, the format is
    # interpreted at the server and raw data sent over NBD.
    if [ $IMGPROTO = "nbd" ]; then
        # Pass a sufficiently high number to -e that should be enough for all
        # tests
        eval "$QEMU_NBD -v -t -k '$SOCK_DIR/nbd' -f $IMGFMT -e 42 -x '' $TEST_IMG_FILE >/dev/null &"
        sleep 1 # FIXME: qemu-nbd needs to be listening before we continue
    fi

    if [ $IMGPROTO = "fuse" -a -f "$img_name" ]; then
        local export_mp
        local pid
        local pidfile
        local timeout

        export_mp=$(echo "$img_name" | sed -e "s#$TEST_DIR/#$SOCK_DIR/fuse-#")
        if ! echo "$export_mp" | grep -q "^$SOCK_DIR"; then
            echo 'Cannot use FUSE exports with images outside of TEST_DIR' >&2
            return 1
        fi

        touch "$export_mp"
        rm -f "$SOCK_DIR/fuse-output"

        # Usually, users would export formatted nodes.  But we present fuse as a
        # protocol-level driver here, so we have to leave the format to the
        # client.
        # Switch off allow-other, because in general we do not need it for
        # iotests.  The default allow-other=auto has the downside of printing a
        # fusermount error on its first attempt if allow_other is not
        # permissible, which we would need to filter.
        QSD_NEED_PID=y $QSD \
              --blockdev file,node-name=export-node,filename=$img_name,discard=unmap \
              --export fuse,id=fuse-export,node-name=export-node,mountpoint="$export_mp",writable=on,growable=on,allow-other=off \
              &

        pidfile="$QEMU_TEST_DIR/qemu-storage-daemon.pid"

        # Wait for the PID file
        while [ ! -f "$pidfile" ]; do
            sleep 0.5
        done

        pid=$(cat "$pidfile")
        rm -f "$pidfile"

        FUSE_PIDS+=($pid)
        FUSE_EXPORTS+=("$export_mp")
    fi
}

_rm_test_img()
{
    local img=$1

    if [[ $IMGPROTO == fuse && $img == $SOCK_DIR/fuse-* ]]; then
        # Drop a FUSE export
        local df_output
        local i
        local image_file
        local index=''
        local timeout

        for i in "${!FUSE_EXPORTS[@]}"; do
            if [ "${FUSE_EXPORTS[i]}" = "$img" ]; then
                index=$i
                break
            fi
        done

        if [ -z "$index" ]; then
            # Probably gone already
            return 0
        fi

        kill "${FUSE_PIDS[index]}"

        # Wait until the mount is gone
        timeout=10 # *0.5 s
        while true; do
            # Will show the mount point; if the mount is still there,
            # it will be $img.
            df_output=$(df "$img" 2>/dev/null)

            # But df may also show an error ("Transpoint endpoint not
            # connected"), so retry in such cases
            if [ -n "$df_output" ]; then
                if ! echo "$df_output" | grep -q "$img"; then
                    break
                fi
            fi

            sleep 0.5

            timeout=$((timeout - 1))
            if [ "$timeout" = 0 ]; then
                echo 'Failed to take down FUSE export' >&2
                return 1
            fi
        done

        rm -f "$img"

        unset "FUSE_PIDS[$index]"
        unset "FUSE_EXPORTS[$index]"

        image_file=$(echo "$img" | sed -e "s#$SOCK_DIR/fuse-#$TEST_DIR/#")
        _rm_test_img "$image_file"
        return
    fi

    if [ "$IMGFMT" = "vmdk" ]; then
        # Remove all the extents for vmdk
        "$QEMU_IMG" info "$img" 2>/dev/null | grep 'filename:' | cut -f 2 -d: \
            | xargs -I {} rm -f "{}"
    elif [ "$IMGFMT" = "qcow2" ]; then
        # Remove external data file
        if data_file=$(_get_data_file "$img"); then
            rm -f "$data_file"
        fi
    fi
    rm -f "$img"
}

_cleanup_test_img()
{
    case "$IMGPROTO" in

        nbd)
            _stop_nbd_server
            rm -f "$TEST_IMG_FILE"
            ;;

        fuse)
            local mp

            for mp in "${FUSE_EXPORTS[@]}"; do
                _rm_test_img "$mp"
            done

            FUSE_PIDS=()
            FUSE_EXPORTS=()
            ;;

        file)
            _rm_test_img "$TEST_DIR/t.$IMGFMT"
            _rm_test_img "$TEST_DIR/t.$IMGFMT.orig"
            _rm_test_img "$TEST_DIR/t.$IMGFMT.base"
            if [ -n "$SAMPLE_IMG_FILE" ]
            then
                rm -f "$TEST_DIR/$SAMPLE_IMG_FILE"
                SAMPLE_IMG_FILE=
                TEST_IMG="$ORIG_TEST_IMG"
            fi
            ;;

        rbd)
            rbd --no-progress rm "$TEST_DIR/t.$IMGFMT" > /dev/null
            ;;

    esac
}

_check_test_img()
{
    (
        if [ "$IMGOPTSSYNTAX" = "true" ]; then
            $QEMU_IMG check $QEMU_IMG_EXTRA_ARGS "$@" "$TEST_IMG" 2>&1
        else
            $QEMU_IMG check "$@" -f $IMGFMT "$TEST_IMG" 2>&1
        fi
    ) | _filter_testdir | _filter_qemu_img_check

    # return real qemu_img check status, to analyze in
    # _check_test_img_ignore_leaks
    return ${PIPESTATUS[0]}
}

_check_test_img_ignore_leaks()
{
    out=$(_check_test_img "$@")
    status=$?
    if [ $status = 3 ]; then
        # This must correspond to success output in dump_human_image_check()
        echo "No errors were found on the image."
        return 0
    fi
    echo "$out"
    return $status
}

_img_info()
{
    if [[ "$1" == "--format-specific" ]]; then
        local format_specific=1
        shift
    else
        local format_specific=0
    fi

    discard=0
    regex_json_spec_start='^ *"format-specific": \{'
    $QEMU_IMG info $QEMU_IMG_EXTRA_ARGS "$@" "$TEST_IMG" 2>&1 | \
        sed -e "s#$REMOTE_TEST_DIR#TEST_DIR#g" \
            -e "s#$IMGPROTO:$TEST_DIR#TEST_DIR#g" \
            -e "s#$TEST_DIR#TEST_DIR#g" \
            -e "s#$SOCK_DIR/fuse-#TEST_DIR/#g" \
            -e "s#$IMGFMT#IMGFMT#g" \
            -e 's/\(compression type: \)\(zlib\|zstd\)/\1COMPRESSION_TYPE/' \
            -e "/^disk size:/ D" \
            -e "/actual-size/ D" | \
        while IFS='' read -r line; do
            if [[ $format_specific == 1 ]]; then
                discard=0
            elif [[ $line == "Format specific information:" ]]; then
                discard=1
            elif [[ $line =~ $regex_json_spec_start ]]; then
                discard=2
                regex_json_spec_end="^${line%%[^ ]*}\\},? *$"
            fi
            if [[ $discard == 0 ]]; then
                echo "$line"
            elif [[ $discard == 1 && ! $line ]]; then
                echo
                discard=0
            elif [[ $discard == 2 && $line =~ $regex_json_spec_end ]]; then
                discard=0
            fi
        done
}

# bail out, setting up .notrun file
#
_notrun()
{
    echo "$*" >"$OUTPUT_DIR/$seq.notrun"
    echo "$seq not run: $*"
    status=0
    exit
}

# bail out, setting up .casenotrun file
# The function _casenotrun() is used as a notifier. It is the
# caller's responsibility to make skipped a particular test.
#
_casenotrun()
{
    echo "    [case not run] $*" >>"$OUTPUT_DIR/$seq.casenotrun"
}

# just plain bail out
#
_fail()
{
    echo "$*" | tee -a "$OUTPUT_DIR/$seq.full"
    echo "(see $seq.full for details)"
    status=1
    exit 1
}

# tests whether $IMGFMT is one of the supported image formats for a test
#
_supported_fmt()
{
    # "generic" is suitable for most image formats. For some formats it doesn't
    # work, however (most notably read-only formats), so they can opt out by
    # setting IMGFMT_GENERIC to false.
    for f; do
        if [ "$f" = "$IMGFMT" -o "$f" = "generic" -a "$IMGFMT_GENERIC" = "true" ]; then
            if [ "$IMGFMT" = "luks" ]; then
                _require_working_luks
            fi
            return
        fi
    done

    _notrun "not suitable for this image format: $IMGFMT"
}

# tests whether $IMGFMT is one of the unsupported image format for a test
#
_unsupported_fmt()
{
    for f; do
        if [ "$f" = "$IMGFMT" ]; then
            _notrun "not suitable for this image format: $IMGFMT"
        fi
    done
}

# tests whether $IMGPROTO is one of the supported image protocols for a test
#
_supported_proto()
{
    for f; do
        if [ "$f" = "$IMGPROTO" -o "$f" = "generic" ]; then
            return
        fi
    done

    _notrun "not suitable for this image protocol: $IMGPROTO"
}

# tests whether $IMGPROTO is specified as an unsupported image protocol for a test
#
_unsupported_proto()
{
    for f; do
        if [ "$f" = "$IMGPROTO" ]; then
            _notrun "not suitable for this image protocol: $IMGPROTO"
            return
        fi
    done
}

# tests whether the host OS is one of the supported OSes for a test
#
_supported_os()
{
    for h
    do
        if [ "$h" = "$HOSTOS" ]
        then
            return
        fi
    done

    _notrun "not suitable for this OS: $HOSTOS"
}

_supported_cache_modes()
{
    for mode; do
        if [ "$mode" = "$CACHEMODE" ]; then
            return
        fi
    done
    _notrun "not suitable for cache mode: $CACHEMODE"
}

# Check whether the filesystem supports O_DIRECT
_check_o_direct()
{
    testfile="$TEST_DIR"/_check_o_direct
    $QEMU_IMG create -f raw "$testfile" 1M > /dev/null
    out=$($QEMU_IO -f raw -t none -c quit "$testfile" 2>&1)
    rm -f "$testfile"

    [[ "$out" != *"O_DIRECT"* ]]
}

_require_o_direct()
{
    if ! _check_o_direct; then
        _notrun "file system on $TEST_DIR does not support O_DIRECT"
    fi
}

_check_cache_mode()
{
    if [ $CACHEMODE == "none" ] || [ $CACHEMODE == "directsync" ]; then
        _require_o_direct
    fi
}

_check_cache_mode

# $1 - cache mode to use by default
# $2 - (optional) cache mode to use by default if O_DIRECT is not supported
_default_cache_mode()
{
    if $CACHEMODE_IS_DEFAULT; then
        if [ -z "$2" ] || _check_o_direct; then
            CACHEMODE="$1"
        else
            CACHEMODE="$2"
        fi
        QEMU_IO="$QEMU_IO --cache $CACHEMODE"
        _check_cache_mode
        return
    fi
}
_supported_aio_modes()
{
    for mode; do
        if [ "$mode" = "$AIOMODE" ]; then
            return
        fi
    done
    _notrun "not suitable for aio mode: $AIOMODE"
}
_default_aio_mode()
{
    AIOMODE="$1"
    QEMU_IO="$QEMU_IO --aio $1"
}

_unsupported_imgopts()
{
    for bad_opt
    do
        # Add a space so tests can match for whitespace that marks the
        # end of an option (\b or \> are not portable)
        if echo "$IMGOPTS " | grep -q 2>/dev/null "$bad_opt"
        then
            _notrun "not suitable for image option: $bad_opt"
        fi
    done
}

# Caution: Overwrites $TEST_DIR/t.luks
_require_working_luks()
{
    file="$TEST_DIR/t.luks"

    output=$(
        $QEMU_IMG create -f luks \
            --object secret,id=sec0,data=hunter0 \
            -o key-secret=sec0 \
            -o iter-time=10 \
            "$file" \
            1M \
            2>&1
    )
    status=$?

    IMGFMT='luks' _rm_test_img "$file"

    if [ $status != 0 ]; then
        reason=$(echo "$output" | grep "$file:" | $SED -e "s#.*$file: *##")
        if [ -z "$reason" ]; then
            reason="Failed to create a LUKS image"
        fi
        _notrun "$reason"
    fi
}

# this test requires that a specified command (executable) exists
#
_require_command()
{
    if [ "$1" = "QEMU" ]; then
        c=$QEMU_PROG
    elif [ "$1" = "QEMU_IMG" ]; then
        c=$QEMU_IMG_PROG
    elif [ "$1" = "QEMU_IO" ]; then
        c=$QEMU_IO_PROG
    elif [ "$1" = "QEMU_NBD" ]; then
        c=$QEMU_NBD_PROG
    else
        eval c=\$$1
    fi
    [ -x "$c" ] || _notrun "$1 utility required, skipped this test"
}

# Check that a set of drivers has been whitelisted in the QEMU binary
#
_require_drivers()
{
    available=$($QEMU -drive format=help | \
                sed -e '/Supported formats:/!d' -e 's/Supported formats://')
    for driver
    do
        if ! echo "$available" | grep -q " $driver\( \|$\)"; then
            _notrun "$driver not available"
        fi
    done
}

# Check that we have a file system that allows huge (but very sparse) files
#
_require_large_file()
{
    if ! truncate --size="$1" "$TEST_IMG"; then
        _notrun "file system on $TEST_DIR does not support large enough files"
    fi
    rm "$TEST_IMG"
}

# Check that a set of devices is available in the QEMU binary
#
_require_devices()
{
    available=$($QEMU -M none -device help | \
                grep ^name | sed -e 's/^name "//' -e 's/".*$//')
    for device
    do
        if ! echo "$available" | grep -q "$device" ; then
            _notrun "$device not available"
        fi
    done
}

_require_one_device_of()
{
    available=$($QEMU -M none -device help | \
                grep ^name | sed -e 's/^name "//' -e 's/".*$//')
    for device
    do
        if echo "$available" | grep -q "$device" ; then
            return
        fi
    done
    _notrun "$* not available"
}

_qcow2_dump_header()
{
    if [[ "$1" == "--no-filter-compression" ]]; then
        local filter_compression=0
        shift
    else
        local filter_compression=1
    fi

    img="$1"
    if [ -z "$img" ]; then
        img="$TEST_IMG"
    fi

    if [[ $filter_compression == 0 ]]; then
        $PYTHON qcow2.py "$img" dump-header
    else
        $PYTHON qcow2.py "$img" dump-header | _filter_qcow2_compression_type_bit
    fi
}

# make sure this script returns success
true