From e92521c166e20e45f24ac8d50b3ea71346350549 Mon Sep 17 00:00:00 2001
From: Thilo Fromm <thilo@kinvolk.io>
Date: Wed, 23 Feb 2022 10:58:48 +0100
Subject: [PATCH] ci-automation/test*: per-image summary, honor parallel

Signed-off-by: Thilo Fromm <thilo@kinvolk.io>
---
 ci-automation/ci-config.env         |  2 +-
 ci-automation/test.sh               | 23 +++++++++++++++++------
 ci-automation/test_update_reruns.sh | 16 +++++++++++-----
 3 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/ci-automation/ci-config.env b/ci-automation/ci-config.env
index deac6763cf..3a61831e35 100644
--- a/ci-automation/ci-config.env
+++ b/ci-automation/ci-config.env
@@ -29,4 +29,4 @@ CONTAINER_IMAGE_ROOT="/home/sdk/build/images"
 
 # Image / vendor tests settings
 QEMU_IMAGE_NAME="flatcar_production_image.bin"
-QEMU_PARALLEL=20
+QEMU_PARALLEL="${PARALLEL_TESTS:-20}"
diff --git a/ci-automation/test.sh b/ci-automation/test.sh
index 7921d79d36..e122f92798 100644
--- a/ci-automation/test.sh
+++ b/ci-automation/test.sh
@@ -46,7 +46,9 @@
 #        - a "detailed" report which also contains error messages of transient failures which succeeded after re-runs.
 #        These reports will be updated after each (re-)run of each vendor, making the test job safe
 #          to abort at any point - the previous runs' results won't be lost.
-#   2. "./ci-cleanup.sh" with commands to clean up temporary build resources,
+#   2. All intermediate kola tap reports, kola debug output, and merged tap reports (from 1.) published
+#        to buildcache at testing/[VERSION]/[ARCH]/[IMAGE]
+#   3. "./ci-cleanup.sh" with commands to clean up temporary build resources,
 #        to be run after this step finishes / when this step is aborted.
 #
 #
@@ -126,13 +128,14 @@ function test_run() {
     local work_dir="__TESTS__"
     local tests_dir="${work_dir}/${image}"
     mkdir -p "${tests_dir}"
-    echo "sudo rm -rf '${tests_dir}'" >> ci-cleanup.sh
 
     local container_name="flatcar-tests-${arch}-${docker_vernum}-${image}"
 
     # Make the torcx artifacts available to test implementation
     __prepare_torcx "${arch}" "${vernum}" "${work_dir}"
 
+    local tap_merged_summary="results-${image}.tap"
+    local tap_merged_detailed="results-${image}-detailed.tap"
     local retry=""
     local success=false
     for retry in $(seq "${retries}"); do
@@ -155,8 +158,11 @@ function test_run() {
         ./run_sdk_container -x ./ci-cleanup.sh \
             -n "${container_name}" -C "${sdk_image}" -v "${vernum}" \
             ci-automation/test_update_reruns.sh \
-                "${tests_dir}/${tapfile}" "${image}" "${retry}" \
-                "${tests_dir}/${failfile}"
+                "${arch}" "${vernum}" "${image}" "${retry}" \
+                "${tests_dir}/${tapfile}" \
+                "${tests_dir}/${failfile}" \
+                "${tap_merged_summary}" \
+                "${tap_merged_detailed}"
 
         local failed_tests
         failed_tests="$(cat "${tests_dir}/${failfile}")"
@@ -172,14 +178,19 @@ function test_run() {
         set -- $failed_tests
     done
 
+
     if ! $success; then
         echo "########### All re-runs exhausted ($retries). Giving up. ###########"
     fi
 
-    # publish TAP files to build cache
+    # publish kola output, TAP files to build cache
+    copy_to_buildcache "testing/${vernum}/${arch}/${image}" \
+        "${tests_dir}/_kola_temp"
     copy_to_buildcache "testing/${vernum}/${arch}/${image}" \
         "${tests_dir}/"*.tap
     copy_to_buildcache "testing/${vernum}/${arch}/${image}" \
-        "${tests_dir}/_kola_temp"
+        "${tap_merged_summary}"
+    copy_to_buildcache "testing/${vernum}/${arch}/${image}" \
+        "${tap_merged_detailed}"
 }
 # --
diff --git a/ci-automation/test_update_reruns.sh b/ci-automation/test_update_reruns.sh
index 8a3eb078a7..e6a530b4aa 100755
--- a/ci-automation/test_update_reruns.sh
+++ b/ci-automation/test_update_reruns.sh
@@ -10,11 +10,17 @@
 
 set -euo pipefail
 
-tapfile="$1"
-image="$2"
-retry="$3"
-outfile="$4"
+arch="$1"
+vernum="$2"
+image="$3"
+retry="$4"
+tapfile="$5"
+failfile="$6"
+merged_summary="$7"
+merged_detailed="$8"
 
 source ci-automation/tapfile_helper_lib.sh
 tap_ingest_tapfile "${tapfile}" "${image}" "${retry}"
-tap_failed_tests_for_vendor "${image}" | tee "${outfile}"
+tap_failed_tests_for_vendor "${image}" > "${failfile}"
+tap_generate_report "${arch}" "${version}"  > "${merged_summary}"
+tap_generate_report "${arch}" "${version}"  "true" > "${merged_detailed}"