From db4aa480f79e583e5948e3e81381379f79ad1aaf Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 30 Aug 2022 11:17:00 +0200 Subject: [PATCH 1/4] ci-automation: Break test cycle properly Create a tapfile and break out of the loop. --- ci-automation/vendor-testing/qemu_update.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci-automation/vendor-testing/qemu_update.sh b/ci-automation/vendor-testing/qemu_update.sh index 2d8dea3a39..1ce875f684 100755 --- a/ci-automation/vendor-testing/qemu_update.sh +++ b/ci-automation/vendor-testing/qemu_update.sh @@ -13,7 +13,12 @@ set -euo pipefail source ci-automation/vendor_test.sh if [ "$*" != "" ] && [ "$*" != "*" ] && [ "$*" != "cl.update.payload" ]; then - echo "Only cl.update.payload is supported, got '$*'" + echo "1..1" > "${CIA_TAPFILE}" + echo "not ok - all qemu update tests" >> "${CIA_TAPFILE}" + echo " ---" >> "${CIA_TAPFILE}" + echo " ERROR: Only cl.update.payload is supported, got '$*'." | tee -a "${CIA_TAPFILE}" + echo " ..." >> "${CIA_TAPFILE}" + break_retest_cycle exit 1 fi From a8bb461a16064b4cc94d6fa0ab1ceb9488392e6c Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 30 Aug 2022 11:24:32 +0200 Subject: [PATCH 2/4] ci-automation: Return 1 on broken cycle We have set success to true when the test cycle was broken, which was a hacky way to avoid printing the give up message. But this setting success to true also meant that the script returned with status 0, which is wrong. Add another variable for controlling printing the give up message. --- ci-automation/test.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci-automation/test.sh b/ci-automation/test.sh index 0dd0fce80c..c49993f869 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -154,6 +154,7 @@ function _test_run_impl() { local tap_merged_detailed="results-${image}-detailed.tap" local retry="" local success=false + local print_give_up=true # A job on each worker prunes old mantle images (docker image prune) echo "docker rm -f '${container_name}'" >> ./ci-cleanup.sh @@ -193,6 +194,7 @@ function _test_run_impl() { if [ -z "$failed_tests" ] ; then echo "########### All tests succeeded. ###########" success=true + print_give_up=false break fi @@ -200,9 +202,7 @@ function _test_run_impl() { echo "########### Test cycle requested to break ###########" echo "Failed tests: $failed_tests" echo "-----------" - # not really a success, but don't print a message about - # exhaused reruns and giving up - success=true + print_give_up=false break fi @@ -213,7 +213,7 @@ function _test_run_impl() { done - if ! $success; then + if ${print_give_up}; then echo "########### All re-runs exhausted ($retries). Giving up. ###########" fi From fb8ac440af3ee7be4bedaf2a2e61c55f88343fd2 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 30 Aug 2022 11:27:10 +0200 Subject: [PATCH 3/4] ci-automation: Print failed tests nicer Instead of printing failed tests like this: Failed tests: kubeadm.v1.25.0.cilium.base kubeadm.v1.24.1.cilium.base Do it like this: Failed tests: kubeadm.v1.25.0.cilium.base kubeadm.v1.24.1.cilium.base --- ci-automation/test.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci-automation/test.sh b/ci-automation/test.sh index c49993f869..9e5c3688de 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -200,14 +200,16 @@ function _test_run_impl() { if retest_cycle_broken; then echo "########### Test cycle requested to break ###########" - echo "Failed tests: $failed_tests" + echo "Failed tests:" + echo "${failed_tests}" echo "-----------" print_give_up=false break fi echo "########### Some tests failed and will be re-run (${retry} / ${retries}). ###########" - echo "Failed tests: $failed_tests" + echo "Failed tests:" + echo "${failed_tests}" echo "-----------" set -- $failed_tests done From 5a44d40e989f9ca8122df41e3338564e8b6299e4 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 30 Aug 2022 12:27:59 +0200 Subject: [PATCH 4/4] ci-automation: Use an array for storing failed tests --- ci-automation/test.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ci-automation/test.sh b/ci-automation/test.sh index 9e5c3688de..f5d14d05bb 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -155,6 +155,7 @@ function _test_run_impl() { local retry="" local success=false local print_give_up=true + local failed_tests=() # A job on each worker prunes old mantle images (docker image prune) echo "docker rm -f '${container_name}'" >> ./ci-cleanup.sh @@ -189,9 +190,8 @@ function _test_run_impl() { "${tap_merged_summary}" \ "${tap_merged_detailed}" - local failed_tests - failed_tests="$(cat "${tests_dir}/${failfile}")" - if [ -z "$failed_tests" ] ; then + readarray -t failed_tests <"${tests_dir}/${failfile}" + if [ "${#failed_tests[@]}" -eq 0 ] ; then echo "########### All tests succeeded. ###########" success=true print_give_up=false @@ -201,7 +201,7 @@ function _test_run_impl() { if retest_cycle_broken; then echo "########### Test cycle requested to break ###########" echo "Failed tests:" - echo "${failed_tests}" + printf '%s\n' "${failed_tests[@]}" echo "-----------" print_give_up=false break @@ -209,9 +209,9 @@ function _test_run_impl() { echo "########### Some tests failed and will be re-run (${retry} / ${retries}). ###########" echo "Failed tests:" - echo "${failed_tests}" + printf '%s\n' "${failed_tests[@]}" echo "-----------" - set -- $failed_tests + set -- "${failed_tests[@]}" done