From a5b958fd0763cf372fb31da4fa02acc24f05780e Mon Sep 17 00:00:00 2001
From: Thilo Fromm <thilo@kinvolk.io>
Date: Fri, 18 Feb 2022 14:40:18 +0100
Subject: [PATCH] ci-automation/test.sh: fix reruns, set retry to 20

Signed-off-by: Thilo Fromm <thilo@kinvolk.io>
---
 ci-automation/ci-config.env |  2 +-
 ci-automation/test.sh       | 16 +++++++++-------
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/ci-automation/ci-config.env b/ci-automation/ci-config.env
index 3e4f347802..34ca3eaf21 100644
--- a/ci-automation/ci-config.env
+++ b/ci-automation/ci-config.env
@@ -25,4 +25,4 @@ CONTAINER_IMAGE_ROOT="/home/sdk/build/images"
 
 # Image / vendor tests settings
 QEMU_IMAGE_NAME="flatcar_production_image.bin"
-QEMU_PARALLEL=4
+QEMU_PARALLEL=20
diff --git a/ci-automation/test.sh b/ci-automation/test.sh
index 9fbc760fc8..b488ea3f0c 100644
--- a/ci-automation/test.sh
+++ b/ci-automation/test.sh
@@ -35,7 +35,7 @@
 #   3. List of tests / test patterns. Defaults to "*" (all tests).
 #      All positional arguments after the first 2 (see above) are tests / patterns of tests to run.
 #
-#   MAX_RETRIES. Environment variable. Number of re-runs to overcome transient failures. Defaults to 999.
+#   MAX_RETRIES. Environment variable. Number of re-runs to overcome transient failures. Defaults to 20.
 #
 # OUTPUT:
 #
@@ -58,7 +58,7 @@ function test_run() {
         set -- '*'
     fi
 
-    local retries="${MAX_RETRIES:-999}"
+    local retries="${MAX_RETRIES:-20}"
 
     source ci-automation/tapfile_helper_lib.sh
     source ci-automation/ci_automation_common.sh
@@ -84,9 +84,11 @@ function test_run() {
     local success=false
     for retry in $(seq "${retries}"); do
         local tapfile="results-run-${retry}.tap"
-        local failfile="failed-run-${retry}."
+        local failfile="failed-run-${retry}.txt"
 
-        set -o noglob
+        # Ignore retcode since tests are flaky. We'll re-run failed tests and
+        #  determine success based on test results (tapfile).
+        set +e -o noglob
         ./run_sdk_container -x ./ci-cleanup.sh \
             -n "${container_name}" -C "${packages_image}" -v "${vernum}" \
             ci-automation/vendor-testing/"${image}".sh \
@@ -95,16 +97,16 @@ function test_run() {
                 "${vernum}" \
                 "${tapfile}" \
                 $@
-        set +o noglob
+        set -e +o noglob
 
         ./run_sdk_container -x ./ci-cleanup.sh \
             -n "${container_name}" -C "${packages_image}" -v "${vernum}" \
             ci-automation/test_update_reruns.sh \
                 "${tests_dir}/${tapfile}" "${image}" "${retry}" \
-                "${tests_dir}/failed-run-${retry}.txt"
+                "${tests_dir}/${failfile}"
 
         local failed_tests
-        failed_tests="$(cat "${tests_dir}/failed-run-${retry}.txt")"
+        failed_tests="$(cat "${tests_dir}/${failfile}")"
         if [ -z "$failed_tests" ] ; then
             echo "########### All tests succeeded. ###########"
             success=true