tests/e2e: add E2E_INCLUDE and rename E2E_SKIP to E2E_EXCLUDE

CHANGELOG.md

@@ -66,3 +66,4 @@
 - Now building with Go 1.15
   - same for the k3d-tools code
 - updated dependencies (including Docker v20.10)
+- tests/e2e: add E2E_INCLUDE and rename E2E_SKIP to E2E_EXCLUDE

Makefile

@@ -46,7 +46,8 @@ REC_DIRS := cmd
 
 ########## Test Settings ##########
 E2E_LOG_LEVEL ?= WARN
-E2E_SKIP ?=
+E2E_INCLUDE ?=
+E2E_EXCLUDE ?=
 E2E_EXTRA ?=
 E2E_RUNNER_START_TIMEOUT ?= 10
 E2E_HELPER_IMAGE_TAG ?=
@@ -169,7 +170,7 @@ test:
 
 e2e: build-docker-dind
 	@echo "Running e2e tests in k3d:$(K3D_IMAGE_TAG)"
-	LOG_LEVEL="$(E2E_LOG_LEVEL)" E2E_SKIP="$(E2E_SKIP)" E2E_EXTRA="$(E2E_EXTRA)" E2E_RUNNER_START_TIMEOUT=$(E2E_RUNNER_START_TIMEOUT) E2E_HELPER_IMAGE_TAG="$(E2E_HELPER_IMAGE_TAG)" tests/dind.sh "${K3D_IMAGE_TAG}-dind"
+	LOG_LEVEL="$(E2E_LOG_LEVEL)" E2E_INCLUDE="$(E2E_INCLUDE)" E2E_EXCLUDE="$(E2E_EXCLUDE)" E2E_EXTRA="$(E2E_EXTRA)" E2E_RUNNER_START_TIMEOUT=$(E2E_RUNNER_START_TIMEOUT) E2E_HELPER_IMAGE_TAG="$(E2E_HELPER_IMAGE_TAG)" tests/dind.sh "${K3D_IMAGE_TAG}-dind"
 
 ci-tests: fmt check e2e
 

tests/assets/config_test_simple.yaml

@@ -27,6 +27,9 @@ labels:
     nodeFilters:
       - server[0]
       - loadbalancer
+registries:
+  create: true
+  use: []
 
 options:
   k3d:

tests/dind.sh

@@ -19,7 +19,8 @@ k3de2e=$(docker run -d \
           -e EXE="$K3D_EXE" \
           -e CI="true" \
           -e LOG_LEVEL="$LOG_LEVEL" \
-          -e E2E_SKIP="$E2E_SKIP" \
+          -e E2E_INCLUDE="$E2E_INCLUDE" \
+          -e E2E_EXCLUDE="$E2E_EXCLUDE" \
           -e E2E_EXTRA="$E2E_EXTRA" \
           --add-host "k3d-registrytest-registry:127.0.0.1" \
           --name "k3d-e2e-runner-$TIMESTAMP" \

tests/runner.sh

@@ -3,7 +3,8 @@
 CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 [ -d "$CURR_DIR" ] || { echo "FATAL: no current dir (maybe running in zsh?)";  exit 1; }
 
-: "${E2E_SKIP:=""}"
+: "${E2E_INCLUDE:=""}"
+: "${E2E_EXCLUDE:=""}"
 : "${E2E_EXTRA:=""}"
 
 # shellcheck source=./common.sh
@@ -22,9 +23,34 @@ section "BASIC TESTS"
 for i in $CURR_DIR/test_*.sh ; do
   base=$(basename "$i" .sh)
   skip=false
-  for skiptest in "${E2E_SKIP[@]}"; do
-    [[ "$skiptest" =~ (^| )${base}($| ) ]] && skip=true
+  included=false
+  excluded=false
+
+  # prepare to skip test, if it's in the exclusion list
+  for excludetest in "${E2E_EXCLUDE[@]}"; do
+    [[ "$excludetest" =~ (^| )${base}($| ) ]] && excluded=true
   done
+
+  # (re-)add test to list, if it's on inclusion list
+  for includetest in "${E2E_INCLUDE[@]}"; do
+    [[ "$includetest" =~ (^| )${base}($| ) ]] && included=true
+  done
+
+  if  [[ -z "${E2E_INCLUDE}" ]]; then # no explicit include list given
+    if $excluded; then # test is on explicit exclude list
+      skip=true
+    fi
+  else
+    if $included && $excluded; then # test is in both lists, so we include it
+      warn "Test ${base} is in both, exclude and include list. Include list takes precedence."
+      skip=false
+    fi
+    if ! $included; then # test is not in include list -> skip
+      skip=true
+    fi
+  fi
+
+  # skip or run test
   if [ "$skip" = true ]; then
     highlight "***** Skipping $base *****"
   else
@@ -33,11 +59,12 @@ for i in $CURR_DIR/test_*.sh ; do
   fi
 done
 
+# Additional (extra) tests
 if [[ -n "$E2E_EXTRA" ]]; then
   section "EXTRA TESTS"
   for i in $CURR_DIR/extra_test_*.sh ; do
     base=$(basename "$i" .sh)
-    if [[ $E2E_SKIP =~ (^| )$base($| ) ]]; then
+    if [[ $E2E_EXCLUDE =~ (^| )$base($| ) ]]; then
       highlight "***** Skipping $base *****"
     else
       highlight "***** Running $base *****"