diff --git a/Makefile b/Makefile index dd5274fa9..f598e26c8 100644 --- a/Makefile +++ b/Makefile @@ -166,6 +166,7 @@ ERR = #### May be used to force running a specific set of reg-tests REG_TEST_FILES = +REG_TEST_SCRIPT=./scripts/run-regtests.sh #### Compiler-specific flags that may be used to disable some negative over- # optimization or to silence some warnings. -fno-strict-aliasing is needed with @@ -967,17 +968,19 @@ endif # Target to run the regression testing script files. reg-tests: - $(Q)./scripts/run-regtests.sh --LEVEL "$(LEVEL)" $(REGTEST_ARGS) $(REG_TEST_FILES) + $(Q)$(REG_TEST_SCRIPT) --type "$(REGTESTS_TYPES)" $(REGTEST_ARGS) $(REG_TEST_FILES) .PHONY: $(REGTEST_ARGS) reg-tests-help: @echo - @echo "To launch the reg tests for haproxy, first export to your environment VTEST_PROGRAM variable to point to your vtest program:" + @echo "To launch the reg tests for haproxy, first export to your environment " + @echo "VTEST_PROGRAM variable to point to your vtest program:" @echo " $$ export VTEST_PROGRAM=/opt/local/bin/vtest" @echo "or" @echo " $$ setenv VTEST_PROGRAM /opt/local/bin/vtest" @echo - @echo "The same thing may be done to set your haproxy program with HAPROXY_PROGRAM but with ./haproxy as default value." + @echo "The same thing may be done to set your haproxy program with HAPROXY_PROGRAM " + @echo "but with ./haproxy as default value." @echo @echo "To run all the tests:" @echo " $$ make reg-tests" @@ -985,16 +988,23 @@ reg-tests-help: @echo "You can also set the programs to be used on the command line:" @echo " $$ VTEST_PROGRAM=<...> HAPROXY_PROGRAM=<...> make reg-tests" @echo - @echo "To run tests with specific levels:" - @echo " $$ LEVEL=1,3,4 make reg-tests #list of levels" - @echo " $$ LEVEL=1-3,5-6 make reg-tests #list of range of levels" + @echo "To run tests with specific types:" + @echo " $$ REGTESTS_TYPES=slow,default make reg-tests" @echo - @echo "About the levels:" - @echo " LEVEL 1 scripts are dedicated to pure haproxy compliance tests (prefixed with 'h' letter)." - @echo " LEVEL 2 scripts are slow scripts (prefixed with 's' letter)." - @echo " LEVEL 3 scripts are low interest scripts (prefixed with 'l' letter)." - @echo " LEVEL 4 scripts are in relation with bugs they help to reproduce (prefixed with 'b' letter)." - @echo " LEVEL 5 scripts are scripts triggering known broken behaviors for which there is still no fix (prefixed with 'k' letter)." - @echo " LEVEL 6 scripts are experimental, typically used to develop new scripts (prefixed with 'e' lettre)." + @echo "with 'any' as default value for REGTESTS_TYPES variable." + @echo + @echo "About the reg test types:" + @echo " any : all the tests without distinction (this is the default" + @echo " value of REGTESTS_TYPES." + @echo " default : dedicated to pure haproxy compliance tests." + @echo " slow : scripts which take non negligible time to run." + @echo " bug : scripts in relation with bugs they help to reproduce." + @echo " broken : scripts triggering known broken behaviors for which" + @echo " there is still no fix." + @echo " experimental: for scripts which are experimental, typically used to" + @echo " develop new scripts." + @echo + @echo "Note that 'reg-tests' target run '"$(REG_TEST_SCRIPT)"' script" + @echo "(see --help option of this script for more information)." .PHONY: reg-tests reg-tests-help diff --git a/reg-tests/checks/s00000.vtc b/reg-tests/checks/s00000.vtc index 93575d347..a9b3a81d3 100644 --- a/reg-tests/checks/s00000.vtc +++ b/reg-tests/checks/s00000.vtc @@ -3,6 +3,7 @@ feature ignore_unknown_macro #REQUIRE_VERSION=1.8 #EXCLUDE_TARGETS=freebsd +#REGTEST_TYPE=slow # This script test health-checks for four backends with one server by backend. # A syslog server is attached to each backend to check the syslog messages diff --git a/reg-tests/checks/s00001.vtc b/reg-tests/checks/s00001.vtc index 2e38867c8..b37e86836 100644 --- a/reg-tests/checks/s00001.vtc +++ b/reg-tests/checks/s00001.vtc @@ -8,6 +8,7 @@ feature ignore_unknown_macro #REQUIRE_VERSION=1.8 #EXCLUDE_TARGETS=freebsd +#REGTEST_TYPE=slow # Note that the first syslog message received is: "Proxy started." syslog S -repeat 21 -level notice { diff --git a/reg-tests/checks/s00002.vtc b/reg-tests/checks/s00002.vtc index 2449ba752..84fd6b170 100644 --- a/reg-tests/checks/s00002.vtc +++ b/reg-tests/checks/s00002.vtc @@ -3,6 +3,7 @@ feature ignore_unknown_macro #REQUIRE_VERSION=1.8 #EXCLUDE_TARGETS=freebsd +#REGTEST_TYPE=slow # This script start 40 servers named s0 up to s39. # For 0 <= i <= 19: diff --git a/reg-tests/checks/s00003.vtc b/reg-tests/checks/s00003.vtc index 6ce10b0e2..ffd31235e 100644 --- a/reg-tests/checks/s00003.vtc +++ b/reg-tests/checks/s00003.vtc @@ -1,6 +1,8 @@ varnishtest "Check: smptchk option" feature ignore_unknown_macro +#REGTEST_TYPE=slow + barrier b cond 3 syslog S1 -level notice { diff --git a/reg-tests/compression/s00000.vtc b/reg-tests/compression/s00000.vtc index 6baddeae1..c3be36e66 100644 --- a/reg-tests/compression/s00000.vtc +++ b/reg-tests/compression/s00000.vtc @@ -3,6 +3,7 @@ varnishtest "Compression validation" #REQUIRE_VERSION=1.6 #REQUIRE_OPTIONS=ZLIB|SLZ,LUA +#REGTEST_TYPE=slow feature ignore_unknown_macro diff --git a/reg-tests/connection/b00000.vtc b/reg-tests/connection/b00000.vtc index b5b06297b..ad4f010e2 100644 --- a/reg-tests/connection/b00000.vtc +++ b/reg-tests/connection/b00000.vtc @@ -13,6 +13,8 @@ varnishtest "PROXY protocol random failures" feature ignore_unknown_macro +#REGTEST_TYPE=bug + syslog Slog_1 -repeat 8 -level info { recv expect ~ "Connect from .* to ${h1_ssl_addr}:${h1_ssl_port}" diff --git a/reg-tests/http-rules/b00000.vtc b/reg-tests/http-rules/b00000.vtc index 897c3b48b..8fd39f766 100644 --- a/reg-tests/http-rules/b00000.vtc +++ b/reg-tests/http-rules/b00000.vtc @@ -11,6 +11,8 @@ varnishtest "map_regm get_trash_chunk test" feature ignore_unknown_macro #REQUIRE_VERSION=1.6 +#REGTEST_TYPE=bug + syslog S1 -level notice { recv expect ~ "[^:\\[ ]\\[${h1_pid}\\]: Proxy (fe|be)1 started." diff --git a/reg-tests/log/b00000.vtc b/reg-tests/log/b00000.vtc index bd4ba0fd3..228c3fdc2 100644 --- a/reg-tests/log/b00000.vtc +++ b/reg-tests/log/b00000.vtc @@ -18,6 +18,8 @@ # Note that the problem doesn't happen when using %[src] or %[dst] since # all sample expressions set LW_XPRT. +#REGTEST_TYPE=bug + varnishtest "Wrong ip/port logging" feature ignore_unknown_macro diff --git a/reg-tests/lua/b00000.vtc b/reg-tests/lua/b00000.vtc index 577329eb3..65d4cbbb8 100644 --- a/reg-tests/lua/b00000.vtc +++ b/reg-tests/lua/b00000.vtc @@ -32,6 +32,7 @@ varnishtest "Basic LUA test h00000" #REQUIRE_OPTIONS=LUA +#REGTEST_TYPE=bug feature ignore_unknown_macro diff --git a/reg-tests/lua/b00001.vtc b/reg-tests/lua/b00001.vtc index 588a5e30e..427613362 100644 --- a/reg-tests/lua/b00001.vtc +++ b/reg-tests/lua/b00001.vtc @@ -21,6 +21,7 @@ varnishtest "LUA bug" #REQUIRE_OPTIONS=LUA +#REGTEST_TYPE=bug feature ignore_unknown_macro diff --git a/reg-tests/lua/b00002.vtc b/reg-tests/lua/b00002.vtc index e1fe7f335..ecb9955c0 100644 --- a/reg-tests/lua/b00002.vtc +++ b/reg-tests/lua/b00002.vtc @@ -1,5 +1,6 @@ varnishtest "Lua: txn:get_priv() scope" #REQUIRE_OPTIONS=LUA +#REGTEST_TYPE=bug feature ignore_unknown_macro diff --git a/reg-tests/lua/b00003.vtc b/reg-tests/lua/b00003.vtc index 3fd70ebc6..26e71ad44 100644 --- a/reg-tests/lua/b00003.vtc +++ b/reg-tests/lua/b00003.vtc @@ -15,6 +15,7 @@ varnishtest "possible CLOSE-WAIT with '\n' headers" #REQUIRE_OPTIONS=LUA +#REGTEST_TYPE=bug feature ignore_unknown_macro diff --git a/reg-tests/mailers/k_healthcheckmail.vtc b/reg-tests/mailers/k_healthcheckmail.vtc index d3af35899..ed8c2f15e 100644 --- a/reg-tests/mailers/k_healthcheckmail.vtc +++ b/reg-tests/mailers/k_healthcheckmail.vtc @@ -1,6 +1,7 @@ varnishtest "Lua: txn:get_priv() scope" #REQUIRE_OPTIONS=LUA #REQUIRE_VERSION=1.6 +#REGTEST_TYPE=broken feature ignore_unknown_macro diff --git a/reg-tests/peers/s_basic_sync.vtc b/reg-tests/peers/s_basic_sync.vtc index ecf8fd059..3dddafa20 100644 --- a/reg-tests/peers/s_basic_sync.vtc +++ b/reg-tests/peers/s_basic_sync.vtc @@ -2,6 +2,7 @@ vtest "Basic test for peers protocol" feature ignore_unknown_macro #REQUIRE_VERSION=2.0 +#REGTEST_TYPE=slow haproxy h1 -arg "-L A" -conf { defaults diff --git a/reg-tests/peers/s_tls_basic_sync.vtc b/reg-tests/peers/s_tls_basic_sync.vtc index 41e874b4d..d69b01d0a 100644 --- a/reg-tests/peers/s_tls_basic_sync.vtc +++ b/reg-tests/peers/s_tls_basic_sync.vtc @@ -3,6 +3,8 @@ vtest "Basic test for peers protocol over SSL/TLS" #REQUIRE_VERSION=2.0 feature ignore_unknown_macro +#REGTEST_TYPE=slow + haproxy h1 -arg "-L A" -conf { defaults timeout client 1s diff --git a/reg-tests/seamless-reload/b00000.vtc b/reg-tests/seamless-reload/b00000.vtc index f533860fa..e29312ca9 100644 --- a/reg-tests/seamless-reload/b00000.vtc +++ b/reg-tests/seamless-reload/b00000.vtc @@ -15,6 +15,7 @@ feature ignore_unknown_macro #REQUIRE_VERSION=1.8 # abns@ sockets are not available on freebsd #EXCLUDE_TARGETS=freebsd +#REGTEST_TYPE=bug haproxy h1 -W -conf { global diff --git a/reg-tests/server/b00000.vtc b/reg-tests/server/b00000.vtc index 831a63d26..cf6b809f1 100644 --- a/reg-tests/server/b00000.vtc +++ b/reg-tests/server/b00000.vtc @@ -2,6 +2,8 @@ varnishtest "Set server FQDN via CLI crash" feature ignore_unknown_macro +#REGTEST_TYPE=bug + # Do nothing. Is there only to create s1_* macros server s1 { } -start diff --git a/reg-tests/spoe/b00000.vtc b/reg-tests/spoe/b00000.vtc index 76b41d60b..ce4e16fa6 100644 --- a/reg-tests/spoe/b00000.vtc +++ b/reg-tests/spoe/b00000.vtc @@ -5,6 +5,8 @@ # segmentaion fault when first errors are catched, when the "filter spoe" line is # parsed. +#REGTEST_TYPE=bug + varnishtest "SPOE bug: missing configuration file" feature ignore_unknown_macro diff --git a/reg-tests/ssl/b00000.vtc b/reg-tests/ssl/b00000.vtc index 9171c4554..9f0bdd789 100644 --- a/reg-tests/ssl/b00000.vtc +++ b/reg-tests/ssl/b00000.vtc @@ -15,6 +15,7 @@ # #define SSL_set_app_data(s,arg) (SSL_set_ex_data(s,0,(char *)arg)) # #define SSL_get_app_data(s) (SSL_get_ex_data(s,0)) +#REGTEST_TYPE=bug varnishtest "OpenSSL bug: Random crashes" #REQUIRE_OPTIONS=OPENSSL diff --git a/reg-tests/stick-table/b00000.vtc b/reg-tests/stick-table/b00000.vtc index 58851939d..ca84c0bef 100644 --- a/reg-tests/stick-table/b00000.vtc +++ b/reg-tests/stick-table/b00000.vtc @@ -1,5 +1,7 @@ # Shipped with the commit fixing the bug. +#REGTEST_TYPE=bug + varnishtest "Stick Table: Crash when accessing unknown key." feature ignore_unknown_macro diff --git a/reg-tests/stick-table/b00001.vtc b/reg-tests/stick-table/b00001.vtc index 3435712ef..d4fb8620d 100644 --- a/reg-tests/stick-table/b00001.vtc +++ b/reg-tests/stick-table/b00001.vtc @@ -14,6 +14,8 @@ # # This should be backported to 1.8 +#REGTEST_TYPE=bug + varnishtest "stick-tables: Test expirations when used with table_*" # As some macros for haproxy are used in this file, this line is mandatory. diff --git a/scripts/run-regtests.sh b/scripts/run-regtests.sh index 84a6c1c8f..27f738056 100755 --- a/scripts/run-regtests.sh +++ b/scripts/run-regtests.sh @@ -28,6 +28,13 @@ _help() --vtestparams , passes custom ARGS to vtest run-regtests.sh --vtestparams "-n 10" + --type filter the types of the tests to be run, depending on + the commented REGTESTS_TYPE variable value in each VTC file. + The value of REGTESTS_TYPE supported are: default, slow, bug, broken and + experimental. When not specified, it is set to 'default' as default value. + + run-regtest.sh --type slow,default + --clean to cleanup previous reg-tests log directories and exit run-regtests.sh --clean @@ -105,39 +112,6 @@ add_range_to_test_list() echo $list } - -build_test_list() -{ - # Remove any spacing character - LEVEL="$(echo $LEVEL | tr -d ' ')" - # Replave any comma character by a space character - LEVEL="$(echo $LEVEL | tr ',' ' ')" - list= - for range in $LEVEL ; do - if [ -z "$list" ] ; then - list=$(add_range_to_test_list $range) - else - list="$list $(add_range_to_test_list $range)" - fi - done - - echo $list -} - -build_find_expr() -{ - expr= - for i in $@; do - if [ -z "$expr" ] ; then - expr="-name \"$i\"" - else - expr="$expr -o -name \"$i\"" - fi - done - - echo $expr -} - _startswith() { _str="$1" _sub="$2" @@ -146,20 +120,25 @@ _startswith() { _findtests() { set -f - LEVEL=${LEVEL:-0}; - list=$(build_test_list "$LEVEL") - if [ -z "$list" ] ; then - echo "Invalid level specification '"$LEVEL"' or no file was found." - exit 1 - fi - EXPR=$(build_find_expr $list) - for i in $( find "$1" $(eval echo $EXPR) ); do + REGTESTS_TYPES="${REGTESTS_TYPES:-any}" + any_test=$(echo $REGTESTS_TYPES | grep -cw "any") + for i in $( find "$1" -name *.vtc ); do skiptest= require_version="$(sed -ne 's/^#REQUIRE_VERSION=//p' "$i")" require_version_below="$(sed -ne 's/^#REQUIRE_VERSION_BELOW=//p' "$i")" require_options="$(sed -ne 's/^#REQUIRE_OPTIONS=//p' "$i" | sed -e 's/,/ /g')" exclude_targets="$(sed -ne 's/^#EXCLUDE_TARGETS=//p' "$i" | sed -e 's/,/ /g')" + if [ $any_test -ne 1 ] ; then + regtest_type="$(sed -ne 's/^#REGTEST_TYPE=//p' "$i")" + if [ -z $regtest_type ] ; then + regtest_type=default + fi + if ! $(echo $REGTESTS_TYPES | grep -wq $regtest_type) ; then + echo " Skip $i because its type '"$regtest_type"' is excluded" + skiptest=1 + fi + fi requiredoption="$(sed -ne 's/^#REQUIRE_OPTION=//p' "$i" | sed -e 's/,.*//')" if [ -n "$requiredoption" ]; then @@ -264,10 +243,10 @@ _process() { --keep-logs) keep_logs="-L" ;; - --LEVEL) - LEVEL="$2" - shift - ;; + --type) + REGTESTS_TYPES="$2" + shift + ;; --use-htx) no_htx="" ;;