From 6c77ebde54aa5a20270f78922c240620052e8e57 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 30 Aug 2022 11:17:00 +0200 Subject: [PATCH 1/4] ci-automation: Break test cycle properly Create a tapfile and break out of the loop. --- ci-automation/vendor-testing/qemu_update.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci-automation/vendor-testing/qemu_update.sh b/ci-automation/vendor-testing/qemu_update.sh index 2d8dea3a39..1ce875f684 100755 --- a/ci-automation/vendor-testing/qemu_update.sh +++ b/ci-automation/vendor-testing/qemu_update.sh @@ -13,7 +13,12 @@ set -euo pipefail source ci-automation/vendor_test.sh if [ "$*" != "" ] && [ "$*" != "*" ] && [ "$*" != "cl.update.payload" ]; then - echo "Only cl.update.payload is supported, got '$*'" + echo "1..1" > "${CIA_TAPFILE}" + echo "not ok - all qemu update tests" >> "${CIA_TAPFILE}" + echo " ---" >> "${CIA_TAPFILE}" + echo " ERROR: Only cl.update.payload is supported, got '$*'." | tee -a "${CIA_TAPFILE}" + echo " ..." >> "${CIA_TAPFILE}" + break_retest_cycle exit 1 fi From 9e05a07a7702c0108579cd330cc3f7d33ce18f94 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 30 Aug 2022 11:24:32 +0200 Subject: [PATCH 2/4] ci-automation: Return 1 on broken cycle We have set success to true when the test cycle was broken, which was a hacky way to avoid printing the give up message. But this setting success to true also meant that the script returned with status 0, which is wrong. Add another variable for controlling printing the give up message. --- ci-automation/test.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci-automation/test.sh b/ci-automation/test.sh index 0dd0fce80c..c49993f869 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -154,6 +154,7 @@ function _test_run_impl() { local tap_merged_detailed="results-${image}-detailed.tap" local retry="" local success=false + local print_give_up=true # A job on each worker prunes old mantle images (docker image prune) echo "docker rm -f '${container_name}'" >> ./ci-cleanup.sh @@ -193,6 +194,7 @@ function _test_run_impl() { if [ -z "$failed_tests" ] ; then echo "########### All tests succeeded. ###########" success=true + print_give_up=false break fi @@ -200,9 +202,7 @@ function _test_run_impl() { echo "########### Test cycle requested to break ###########" echo "Failed tests: $failed_tests" echo "-----------" - # not really a success, but don't print a message about - # exhaused reruns and giving up - success=true + print_give_up=false break fi @@ -213,7 +213,7 @@ function _test_run_impl() { done - if ! $success; then + if ${print_give_up}; then echo "########### All re-runs exhausted ($retries). Giving up. ###########" fi From 8cd06230ba823015bfda9ec7b43565a0e319bf76 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 30 Aug 2022 11:27:10 +0200 Subject: [PATCH 3/4] ci-automation: Print failed tests nicer Instead of printing failed tests like this: Failed tests: kubeadm.v1.25.0.cilium.base kubeadm.v1.24.1.cilium.base Do it like this: Failed tests: kubeadm.v1.25.0.cilium.base kubeadm.v1.24.1.cilium.base --- ci-automation/test.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci-automation/test.sh b/ci-automation/test.sh index c49993f869..9e5c3688de 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -200,14 +200,16 @@ function _test_run_impl() { if retest_cycle_broken; then echo "########### Test cycle requested to break ###########" - echo "Failed tests: $failed_tests" + echo "Failed tests:" + echo "${failed_tests}" echo "-----------" print_give_up=false break fi echo "########### Some tests failed and will be re-run (${retry} / ${retries}). ###########" - echo "Failed tests: $failed_tests" + echo "Failed tests:" + echo "${failed_tests}" echo "-----------" set -- $failed_tests done From 8b52a9b04ca31e0223fa1da5ad58f0a9a31e8e14 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 30 Aug 2022 12:27:59 +0200 Subject: [PATCH 4/4] ci-automation: Use an array for storing failed tests --- ci-automation/test.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ci-automation/test.sh b/ci-automation/test.sh index 9e5c3688de..f5d14d05bb 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -155,6 +155,7 @@ function _test_run_impl() { local retry="" local success=false local print_give_up=true + local failed_tests=() # A job on each worker prunes old mantle images (docker image prune) echo "docker rm -f '${container_name}'" >> ./ci-cleanup.sh @@ -189,9 +190,8 @@ function _test_run_impl() { "${tap_merged_summary}" \ "${tap_merged_detailed}" - local failed_tests - failed_tests="$(cat "${tests_dir}/${failfile}")" - if [ -z "$failed_tests" ] ; then + readarray -t failed_tests <"${tests_dir}/${failfile}" + if [ "${#failed_tests[@]}" -eq 0 ] ; then echo "########### All tests succeeded. ###########" success=true print_give_up=false @@ -201,7 +201,7 @@ function _test_run_impl() { if retest_cycle_broken; then echo "########### Test cycle requested to break ###########" echo "Failed tests:" - echo "${failed_tests}" + printf '%s\n' "${failed_tests[@]}" echo "-----------" print_give_up=false break @@ -209,9 +209,9 @@ function _test_run_impl() { echo "########### Some tests failed and will be re-run (${retry} / ${retries}). ###########" echo "Failed tests:" - echo "${failed_tests}" + printf '%s\n' "${failed_tests[@]}" echo "-----------" - set -- $failed_tests + set -- "${failed_tests[@]}" done