mirror of
https://git.haproxy.org/git/haproxy.git/
synced 2025-12-21 17:40:59 +01:00
1ed2c9da2c
The script for running regression tests is modified to improve the
visibility of skipped tests.
Previously, the reasons for skipping tests were only visible during the
test discovery phase when grepping the vtc (REQUIRE, EXCLUDE, etc).
But reg-tests skipped by vtest with the 'feature cmd' keywords were not
listed.
This change introduces the following:
- vtest does not remove the logs itself anymore, because it is not
able to let the log available when a test is skipped. So the -L
parameter is now always passed to vtest
- All skipped tests during the discovery phase are now logged to a
'skipped.log' file within the test directory
- The script now parses vtest logs to find tests that were skipped
due to missing features (via the 'feature cmd' in .vtc files)
and adds them to the skipped list.
439 lines
12 KiB
Bash
Executable File
439 lines
12 KiB
Bash
Executable File
#!/bin/sh
|
|
|
|
_help()
|
|
{
|
|
cat << EOF
|
|
### run-regtests.sh ###
|
|
Running run-regtests.sh --help shows this information about how to use it
|
|
|
|
Run without parameters to run all tests in the current folder (including subfolders)
|
|
run-regtests.sh
|
|
|
|
Provide paths to run tests from (including subfolders):
|
|
run-regtests.sh ./tests1 ./tests2
|
|
|
|
Parameters:
|
|
--j <NUM>, To run vtest with multiple jobs / threads for a faster overall result
|
|
run-regtests.sh ./fasttest --j 16
|
|
|
|
--v, to run verbose
|
|
run-regtests.sh --v, disables the default vtest 'quiet' parameter
|
|
|
|
--debug to show test logs on standard output (implies --v)
|
|
run-regtests.sh --debug
|
|
|
|
--keep-logs to keep all log directories (by default kept if test fails)
|
|
run-regtests.sh --keep-logs
|
|
|
|
--vtestparams <ARGS>, passes custom ARGS to vtest
|
|
run-regtests.sh --vtestparams "-n 10"
|
|
|
|
--type <reg tests types> filter the types of the tests to be run, depending on
|
|
the commented REGTESTS_TYPE variable value in each VTC file.
|
|
The value of REGTESTS_TYPE supported are: default, slow, bug, broken, devel
|
|
and experimental. When not specified, it is set to 'default' as default value.
|
|
|
|
run-regtest.sh --type slow,default
|
|
|
|
--clean to cleanup previous reg-tests log directories and exit
|
|
run-regtests.sh --clean
|
|
|
|
Including text below into a .vtc file will check for its requirements
|
|
related to haproxy's target and compilation options
|
|
# Below targets are not capable of completing this test successfully
|
|
#EXCLUDE_TARGET=freebsd, abns sockets are not available on freebsd
|
|
|
|
#EXCLUDE_TARGETS=dos,freebsd,windows
|
|
|
|
# Below option is required to complete this test successfully
|
|
#REQUIRE_OPTION=OPENSSL, this test needs OPENSSL compiled in.
|
|
#REQUIRE_OPTIONS=ZLIB|SLZ,OPENSSL,LUA
|
|
|
|
#REQUIRE_SERVICE=prometheus-exporter
|
|
#REQUIRE_SERVICES=prometheus-exporter,foo
|
|
|
|
Configure environment variables to set the haproxy and vtest binaries to use
|
|
setenv HAPROXY_PROGRAM /usr/local/sbin/haproxy
|
|
setenv VTEST_PROGRAM /usr/local/bin/vtest
|
|
setenv HAPROXY_ARGS "-dM -de -m 50"
|
|
or
|
|
export HAPROXY_PROGRAM=/usr/local/sbin/haproxy
|
|
export VTEST_PROGRAM=/usr/local/bin/vtest
|
|
export HAPROXY_ARGS="-dM -de -m 50"
|
|
EOF
|
|
exit 0
|
|
}
|
|
|
|
add_range_to_test_list()
|
|
{
|
|
level0="*.vtc"
|
|
level1="h*.vtc"
|
|
level2="s*.vtc"
|
|
level3="l*.vtc"
|
|
level4="b*.vtc"
|
|
level5="k*.vtc"
|
|
level6="e*.vtc"
|
|
|
|
new_range=$(echo $1 | tr '-' ' ')
|
|
non_digit=$(echo $new_range | grep '[^0-9 ]')
|
|
if [ -n "$non_digit" ] ; then
|
|
return
|
|
fi
|
|
if [ "$new_range" = "$1" ] ; then
|
|
if [ $1 -gt 6 ] ; then
|
|
return
|
|
fi
|
|
eval echo '$'level$1
|
|
return
|
|
fi
|
|
if [ -z "$new_range" ] ; then
|
|
return
|
|
fi
|
|
list=
|
|
for l in $(seq $new_range) ; do
|
|
if [ -n "l" ] ; then
|
|
if [ -z "$list" ] ; then
|
|
list="$(eval echo '$'level${l})"
|
|
else
|
|
list="$list $(eval echo '$'level${l})"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
echo $list
|
|
}
|
|
|
|
_startswith() {
|
|
_str="$1"
|
|
_sub="$2"
|
|
echo "$_str" | grep "^$_sub" >/dev/null 2>&1
|
|
}
|
|
|
|
_findtests() {
|
|
set -f
|
|
|
|
REGTESTS_TYPES="${REGTESTS_TYPES:-default,bug,devel,slow}"
|
|
any_test=$(echo $REGTESTS_TYPES | grep -cw "any")
|
|
for i in $( find "$1" -name *.vtc ); do
|
|
skiptest=
|
|
OLDIFS="$IFS"; IFS="$LINEFEED"
|
|
set -- $(grep '^#[0-9A-Z_]*=' "$i")
|
|
IFS="$OLDIFS"
|
|
|
|
require_options="";
|
|
require_services=""; exclude_targets=""; regtest_type=""
|
|
requiredoption=""; requiredservice=""; excludedtarget="";
|
|
|
|
while [ $# -gt 0 ]; do
|
|
v="$1"; v="${v#*=}"
|
|
case "$1" in
|
|
"#REQUIRE_OPTIONS="*) require_options="$v" ;;
|
|
"#REQUIRE_SERVICES="*) require_services="$v" ;;
|
|
"#EXCLUDE_TARGETS="*) exclude_targets="$v" ;;
|
|
"#REGTEST_TYPE="*) regtest_type="$v" ;;
|
|
"#REQUIRE_OPTION="*) requiredoption="${v%,*}" ;;
|
|
"#REQUIRE_SERVICE="*) required_service="${v%,*}" ;;
|
|
"#EXCLUDE_TARGET="*) excludedtarget="${v%,*}" ;;
|
|
# Note: any new variable declared here must be initialized above.
|
|
esac
|
|
shift
|
|
done
|
|
|
|
if [ $any_test -ne 1 ] ; then
|
|
if [ -z $regtest_type ] ; then
|
|
regtest_type=default
|
|
fi
|
|
if ! $(echo $REGTESTS_TYPES | grep -wq $regtest_type) ; then
|
|
echo " Skipped $i because its type '"$regtest_type"' is excluded" >> "${TESTDIR}/skipped.log"
|
|
skiptest=1
|
|
fi
|
|
fi
|
|
|
|
if [ -n "$requiredoption" ]; then
|
|
require_options="$require_options,$requiredoption"
|
|
fi
|
|
|
|
if [ -n "$requiredservice" ]; then
|
|
require_services="$require_services,$requiredservice"
|
|
fi
|
|
|
|
if [ -n "$excludedtarget" ]; then
|
|
exclude_targets="$exclude_targets,$excludedtarget"
|
|
fi
|
|
|
|
IFS=","; set -- $require_options; IFS=$OLDIFS; require_options="$*"
|
|
IFS=","; set -- $require_services; IFS=$OLDIFS; require_services="$*"
|
|
IFS=","; set -- $exclude_targets; IFS=$OLDIFS; exclude_targets="$*"
|
|
|
|
for excludedtarget in $exclude_targets; do
|
|
if [ "$excludedtarget" = "$TARGET" ]; then
|
|
echo " Skipped $i because haproxy is compiled for the excluded target $TARGET" >> "${TESTDIR}/skipped.log"
|
|
skiptest=1
|
|
fi
|
|
done
|
|
|
|
for requiredoption in $require_options; do
|
|
IFS="|"; set -- $requiredoption; IFS=$OLDIFS; alternatives="$*"
|
|
found=
|
|
for alt in $alternatives; do
|
|
if [ -z "${FEATURES_PATTERN##* +$alt *}" ]; then
|
|
found=1;
|
|
fi
|
|
done
|
|
if [ -z $found ]; then
|
|
echo " Skipped $i because haproxy is not compiled with the required option $requiredoption" >> "${TESTDIR}/skipped.log"
|
|
skiptest=1
|
|
fi
|
|
done
|
|
|
|
for requiredservice in $require_services; do
|
|
IFS="|"; set -- $requiredservice; IFS=$OLDIFS; alternatives="$*"
|
|
found=
|
|
for alt in $alternatives; do
|
|
if [ -z "${SERVICES_PATTERN##* $alt *}" ]; then
|
|
found=1;
|
|
fi
|
|
done
|
|
if [ -z $found ]; then
|
|
echo " Skipped $i because haproxy is not compiled with the required service $requiredservice" >> "${TESTDIR}/skipped.log"
|
|
skiptest=1
|
|
fi
|
|
done
|
|
|
|
if [ -z $skiptest ]; then
|
|
echo " Add test: $i"
|
|
testlist="$testlist $i"
|
|
fi
|
|
done
|
|
}
|
|
|
|
_cleanup()
|
|
{
|
|
DIRS=$(find "${TESTDIR}" -maxdepth 1 -type d -name "haregtests-*" -exec basename {} \; 2>/dev/null)
|
|
if [ -z "${DIRS}" ]; then
|
|
echo "No reg-tests log directory found"
|
|
else
|
|
echo "Cleanup following reg-tests log directories:"
|
|
for d in ${DIRS}; do
|
|
echo " o ${TESTDIR}/$d"
|
|
done
|
|
read -p "Continue (y/n)?" reply
|
|
case "$reply" in
|
|
y|Y)
|
|
for d in ${DIRS}; do
|
|
rm -r "${TESTDIR}/$d"
|
|
done
|
|
echo "done"
|
|
exit 0
|
|
;;
|
|
*)
|
|
echo "aborted"
|
|
exit 1
|
|
;;
|
|
esac
|
|
fi
|
|
}
|
|
|
|
|
|
_process() {
|
|
while [ ${#} -gt 0 ]; do
|
|
if _startswith "$1" "-"; then
|
|
case "${1}" in
|
|
--j)
|
|
jobcount="$2"
|
|
shift
|
|
;;
|
|
--vtestparams)
|
|
vtestparams="$2"
|
|
shift
|
|
;;
|
|
--v)
|
|
verbose=""
|
|
;;
|
|
--debug)
|
|
verbose=""
|
|
debug="-v"
|
|
;;
|
|
--keep-logs)
|
|
keep_logs=1
|
|
;;
|
|
--type)
|
|
REGTESTS_TYPES="$2"
|
|
shift
|
|
;;
|
|
--clean)
|
|
_cleanup
|
|
exit 0
|
|
;;
|
|
--help)
|
|
_help
|
|
;;
|
|
*)
|
|
echo "Unknown parameter : $1"
|
|
exit 1
|
|
;;
|
|
esac
|
|
else
|
|
REGTESTS="${REGTESTS} $1"
|
|
fi
|
|
shift 1
|
|
done
|
|
}
|
|
|
|
# compute a version from up to 4 sub-version components, each multiplied
|
|
# by a power of 1000, and padded left with 0, 1 or 2 zeroes.
|
|
_version() {
|
|
OLDIFS="$IFS"; IFS="."; set -- $*; IFS="$OLDIFS"
|
|
set -- ${1%%[!0-9]*} 000${2%%[!0-9]*} 000${3%%[!0-9]*} 000${4%%[!0-9]*}
|
|
prf2=${2%???}; prf3=${3%???}; prf4=${4%???}
|
|
echo ${1}${2#$prf2}${3#$prf3}${4#$prf4}
|
|
}
|
|
|
|
|
|
HAPROXY_PROGRAM="${HAPROXY_PROGRAM:-${PWD}/haproxy}"
|
|
HAPROXY_ARGS="${HAPROXY_ARGS--dM -dI -dW}"
|
|
VTEST_PROGRAM="${VTEST_PROGRAM:-vtest}"
|
|
VTEST_TIMEOUT="${VTEST_TIMEOUT:-10}"
|
|
TESTDIR="${TMPDIR:-/tmp}"
|
|
REGTESTS=""
|
|
LINEFEED="
|
|
"
|
|
|
|
jobcount=""
|
|
verbose="-q"
|
|
debug=""
|
|
keep_logs=0
|
|
testlist=""
|
|
|
|
_process "$@";
|
|
|
|
echo ""
|
|
echo "########################## Preparing to run tests ##########################"
|
|
|
|
preparefailed=
|
|
if ! [ -x "$(command -v $HAPROXY_PROGRAM)" ]; then
|
|
echo "haproxy not found in path, please specify HAPROXY_PROGRAM environment variable"
|
|
preparefailed=1
|
|
fi
|
|
if ! [ -x "$(command -v $VTEST_PROGRAM)" ]; then
|
|
echo "vtest not found in path, please specify VTEST_PROGRAM environment variable"
|
|
preparefailed=1
|
|
fi
|
|
if [ $preparefailed ]; then
|
|
exit 1
|
|
fi
|
|
|
|
{ read HAPROXY_VERSION; read TARGET; read FEATURES; read SERVICES; } << EOF
|
|
$($HAPROXY_PROGRAM $HAPROXY_ARGS -vv | grep -E 'HA-?Proxy version|TARGET.*=|^Feature|^Available services' | sed 's/.* [:=] //')
|
|
EOF
|
|
|
|
HAPROXY_VERSION=$(echo $HAPROXY_VERSION | cut -d " " -f 3)
|
|
echo "Testing with haproxy version: $HAPROXY_VERSION"
|
|
|
|
PROJECT_VERSION=$(${MAKE:-make} version 2>&1 | grep -E '^VERSION:|^SUBVERS:'|cut -f2 -d' '|tr -d '\012')
|
|
if [ -z "${PROJECT_VERSION}${MAKE}" ]; then
|
|
# try again with gmake, just in case
|
|
PROJECT_VERSION=$(gmake version 2>&1 | grep -E '^VERSION:|^SUBVERS:'|cut -f2 -d' '|tr -d '\012')
|
|
fi
|
|
|
|
FEATURES_PATTERN=" $FEATURES "
|
|
SERVICES_PATTERN=" $SERVICES "
|
|
|
|
TESTRUNDATETIME="$(date '+%Y-%m-%d_%H-%M-%S')"
|
|
|
|
mkdir -p "$TESTDIR" || exit 1
|
|
TESTDIR=$(mktemp -d "$TESTDIR/haregtests-$TESTRUNDATETIME.XXXXXX") || exit 1
|
|
|
|
export TMPDIR="$TESTDIR"
|
|
export HAPROXY_PROGRAM="$HAPROXY_PROGRAM"
|
|
if [ -n "$HAPROXY_ARGS" ]; then
|
|
export HAPROXY_ARGS
|
|
fi
|
|
|
|
echo "Target : $TARGET"
|
|
echo "Options : $FEATURES"
|
|
echo "Services : $SERVICES"
|
|
|
|
echo "########################## Gathering tests to run ##########################"
|
|
|
|
if [ -z "$REGTESTS" ]; then
|
|
_findtests reg-tests/
|
|
else
|
|
for t in $REGTESTS; do
|
|
_findtests $t
|
|
done
|
|
fi
|
|
|
|
echo "########################## Starting vtest ##########################"
|
|
echo "Testing with haproxy version: $HAPROXY_VERSION"
|
|
|
|
if [ -n "$PROJECT_VERSION" -a "$PROJECT_VERSION" != "$HAPROXY_VERSION" ]; then
|
|
echo "Warning: version does not match the current tree ($PROJECT_VERSION)"
|
|
fi
|
|
|
|
_vtresult=0
|
|
if [ -n "$testlist" ]; then
|
|
if [ -n "$jobcount" ]; then
|
|
jobcount="-j $jobcount"
|
|
fi
|
|
cmd="$VTEST_PROGRAM -b $((2<<20)) -k -t ${VTEST_TIMEOUT} -L $verbose $debug $jobcount $vtestparams $testlist"
|
|
eval $cmd
|
|
_vtresult=$?
|
|
else
|
|
echo "No tests found that meet the required criteria"
|
|
fi
|
|
|
|
if [ -d "${TESTDIR}" ]; then
|
|
# look for tests skipped by vtest
|
|
find "${TESTDIR}" -type f -name "LOG" | while read logfile; do
|
|
REASON=$(grep "SKIPPING test" "$logfile")
|
|
if [ -n "$REASON" ]; then
|
|
infofile="$(dirname "$logfile")/INFO"
|
|
if [ -e "$infofile" ]; then
|
|
vtc_path=$(sed 's/^Test case: //' "$infofile" )
|
|
if [ -n "$vtc_path" ]; then
|
|
echo " Skipped $vtc_path (feature cmd)" >> "${TESTDIR}/skipped.log"
|
|
fi
|
|
fi
|
|
fi
|
|
done
|
|
|
|
if [ $keep_logs -eq 0 ]; then
|
|
# remove logs for successful tests
|
|
find "$TESTDIR" -type d -name "vtc.*" | while read vtcdir; do
|
|
# errors are starting with ----
|
|
grep -q "^----" ${vtcdir}/LOG || rm -fr "${vtcdir}"
|
|
done
|
|
fi
|
|
|
|
if [ $_vtresult -eq 0 ]; then
|
|
# all tests were successful, removing tempdir (the last part.)
|
|
# ignore errors is the directory is not empty or if it does not exist
|
|
rmdir "$TESTDIR" 2>/dev/null
|
|
fi
|
|
|
|
# show failed tests
|
|
if [ -d "${TESTDIR}" ]; then
|
|
echo "########################## Gathering results ##########################"
|
|
export TESTDIR
|
|
find "$TESTDIR" -type d -name "vtc.*" -exec sh -c 'for i; do
|
|
if [ ! -e "$i/LOG" ] ; then continue; fi
|
|
|
|
cat <<- EOF | tee -a "$TESTDIR/failedtests.log"
|
|
$(echo "###### $(cat "$i/INFO") ######")
|
|
$(echo "## test results in: \"$i\"")
|
|
$(echo "## test log file: $i/LOG")
|
|
$(grep -E -- "^(----|\* diag)" "$i/LOG")
|
|
EOF
|
|
done' sh {} +
|
|
fi
|
|
|
|
echo "########################## Listing skipped tests ####################"
|
|
count=$(wc -l < "${TESTDIR}/skipped.log")
|
|
cat "${TESTDIR}/skipped.log" | sort -n
|
|
echo "Total skipped tests: $count"
|
|
|
|
fi # if TESTDIR
|
|
|
|
exit $_vtresult
|