ci-automation: Add retest cycle breaking functionality

ci-automation/ci_automation_common.sh

@@ -10,6 +10,8 @@ source ci-automation/ci-config.env
 : ${PIGZ:=pigz}
 : ${docker:=docker}
 
+: ${TEST_WORK_DIR:='__TESTS__'}
+
 function init_submodules() {
     git submodule init
     git submodule update
@@ -225,6 +227,32 @@ function docker_image_from_registry_or_buildcache() {
 }
 # --
 
+# Called by vendor test in case of complete failure not eligible for
+# reruns (like trying to run tests on unsupported architecture).
+function break_retest_cycle() {
+    local work_dir=$(dirname "${PWD}")
+    local dir=$(basename "${work_dir}")
+
+    if [[ "${dir}" != "${TEST_WORK_DIR}" ]]; then
+        echo "Not breaking retest cycle, expected test work dir to be a parent directory" >&2
+        return
+    fi
+    touch "${work_dir}/break_retests"
+}
+# --
+
+# Called by test runner to see if the retest cycle should be broken.
+function retest_cycle_broken() {
+    # Using the reverse boolean logic here!
+    local broken=1
+    if [[ -f "${TEST_WORK_DIR}/break_retests" ]]; then
+        broken=0
+        rm -f "${TEST_WORK_DIR}/break_retests"
+    fi
+    return ${broken}
+}
+# --
+
 # Substitutes fields in the passed template and prints the
 # result. Followed by the template, the parameters used for
 # replacement are in alphabetical order: arch, channel, proto and

ci-automation/test.sh

@@ -122,7 +122,7 @@ function test_run() {
     local docker_vernum
     docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
 
-    local work_dir="__TESTS__"
+    local work_dir="${TEST_WORK_DIR}"
     local tests_dir="${work_dir}/${image}"
     mkdir -p "${tests_dir}"
 
@@ -182,6 +182,16 @@ function test_run() {
             break
         fi
 
+        if retest_cycle_broken; then
+            echo "########### Test cycle requested to break ###########"
+            echo "Failed tests: $failed_tests"
+            echo "-----------"
+            # not really a success, but don't print a message about
+            # exhaused reruns and giving up
+            success=true
+            break
+        fi
+
         echo "########### Some tests failed and will be re-run (${retry} / ${retries}). ###########"
         echo "Failed tests: $failed_tests"
         echo "-----------"

ci-automation/vendor-testing/digitalocean.sh

@@ -29,6 +29,7 @@ if [[ "${arch}" == "arm64" ]]; then
     echo "  ---" >> "${tapfile}"
     echo "  ERROR: ARM64 tests not supported on Digital Ocean." | tee -a "${tapfile}"
     echo "  ..." >> "${tapfile}"
+    break_retest_cycle
     exit 1
 fi