ci-automation: Refactor running kola tests

ci-automation/vendor-testing/aws.sh

@@ -22,6 +22,7 @@ CIA_OUTPUT_MAIN_INSTANCE="${aws_instance_type}"
 CIA_OUTPUT_ALL_TESTS=( "${@}" )
 CIA_OUTPUT_EXTRA_INSTANCES=( "${more_aws_instance_types[@]}" )
 CIA_OUTPUT_EXTRA_INSTANCE_TESTS=( 'cl.internet' )
+CIA_OUTPUT_TIMEOUT=6h
 
 query_kola_tests() {
     shift; # ignore the instance type
@@ -50,23 +51,17 @@ fi
 
 run_kola_tests() {
     local instance_type="${1}"; shift
-    local instance_tapfile="${1}"; shift
 
-    timeout --signal=SIGQUIT 6h \
-        kola run \
-         --board="${board}" \
-         --basename="${image_name}" \
-         --channel="${CIA_CHANNEL}" \
-         --offering='basic' \
-         --parallel="${AWS_PARALLEL}" \
-         --platform=aws \
-         --aws-ami="${AWS_AMI_ID}" \
-         --aws-region="${AWS_REGION}" \
-         --aws-type="${instance_type}" \
-         --aws-iam-profile="${AWS_IAM_PROFILE}" \
-         --tapfile="${instance_tapfile}" \
-         --torcx-manifest="${CIA_TORCX_MANIFEST}" \
-         "${@}"
+    kola_run \
+        --basename="${image_name}" \
+        --offering='basic' \
+        --parallel="${AWS_PARALLEL}" \
+        --platform=aws \
+        --aws-ami="${AWS_AMI_ID}" \
+        --aws-region="${AWS_REGION}" \
+        --aws-type="${instance_type}" \
+        --aws-iam-profile="${AWS_IAM_PROFILE}" \
+        "${@}"
 }
 
 # these are set in ci-config.env

ci-automation/vendor-testing/azure.sh

@@ -27,6 +27,8 @@ CIA_OUTPUT_MAIN_INSTANCE="${azure_instance_type}"
 CIA_OUTPUT_ALL_TESTS=( "${@}" )
 CIA_OUTPUT_EXTRA_INSTANCES=( "${other_instance_types[@]}" )
 CIA_OUTPUT_EXTRA_INSTANCE_TESTS=( 'cl.internet' )
+# Align timeout with ore azure gc --duration parameter
+CIA_OUTPUT_TIMEOUT=6h
 
 query_kola_tests() {
     shift; # ignore the instance type
@@ -55,33 +57,27 @@ fi
 
 run_kola_tests() {
     local instance_type="${1}"; shift
-    local instance_tapfile="${1}"; shift
     local hyperv_gen="V2"
     if [ "${instance_type}" = "V1" ]; then
         hyperv_gen="V1"
         instance_type="${azure_instance_type}"
     fi
 
-    # Align timeout with ore azure gc --duration parameter
-    timeout --signal=SIGQUIT 6h \
-      kola run \
-      --board="${board}" \
-      --basename="${basename}" \
-      --parallel="${AZURE_PARALLEL}" \
-      --offering=basic \
-      --platform=azure \
-      --azure-image-file="${AZURE_IMAGE_NAME}" \
-      --azure-location="${AZURE_LOCATION}" \
-      --azure-profile="${azure_profile_config_file}" \
-      --azure-auth="${azure_auth_config_file}" \
-      --torcx-manifest="${CIA_TORCX_MANIFEST}" \
-      --tapfile="${instance_tapfile}" \
-      --azure-size="${instance_type}" \
-      --azure-hyper-v-generation="${hyperv_gen}" \
-      ${AZURE_USE_GALLERY} \
-      ${azure_vnet_subnet_name:+--azure-vnet-subnet-name=${azure_vnet_subnet_name}} \
-      ${AZURE_USE_PRIVATE_IPS:+--azure-use-private-ips=${AZURE_USE_PRIVATE_IPS}} \
-      "${@}"
+    kola_run \
+        --basename="${basename}" \
+        --parallel="${AZURE_PARALLEL}" \
+        --offering=basic \
+        --platform=azure \
+        --azure-image-file="${AZURE_IMAGE_NAME}" \
+        --azure-location="${AZURE_LOCATION}" \
+        --azure-profile="${azure_profile_config_file}" \
+        --azure-auth="${azure_auth_config_file}" \
+        --azure-size="${instance_type}" \
+        --azure-hyper-v-generation="${hyperv_gen}" \
+        ${AZURE_USE_GALLERY} \
+        ${azure_vnet_subnet_name:+--azure-vnet-subnet-name=${azure_vnet_subnet_name}} \
+        ${AZURE_USE_PRIVATE_IPS:+--azure-use-private-ips=${AZURE_USE_PRIVATE_IPS}} \
+        "${@}"
 }
 
 run_default_kola_tests

ci-automation/vendor-testing/digitalocean.sh

@@ -26,6 +26,7 @@ CIA_OUTPUT_MAIN_INSTANCE="${DIGITALOCEAN_MACHINE_SIZE}"
 CIA_OUTPUT_ALL_TESTS=( "${@}" )
 CIA_OUTPUT_EXTRA_INSTANCES=()
 CIA_OUTPUT_EXTRA_INSTANCE_TESTS=()
+CIA_OUTPUT_TIMEOUT=4h
 
 query_kola_tests() {
     shift; # ignore the instance type
@@ -50,10 +51,8 @@ trap 'ore do delete-image \
 
 run_kola_tests() {
     local instance_type="${1}"; shift
-    local instance_tapfile="${1}"; shift
 
-    timeout --signal=SIGQUIT 4h\
-    kola run \
+    kola_run \
         --do-size="${instance_type}" \
         --do-region="${DIGITALOCEAN_REGION}" \
         --basename="${image_name}" \
@@ -61,9 +60,6 @@ run_kola_tests() {
         --do-image="${image_name}" \
         --parallel="${DIGITALOCEAN_PARALLEL}" \
         --platform=do \
-        --channel="${CIA_CHANNEL}" \
-        --tapfile="${instance_tapfile}" \
-        --torcx-manifest="${CIA_TORCX_MANIFEST}" \
         "${@}"
 }
 

ci-automation/vendor-testing/equinix_metal.sh

@@ -24,39 +24,33 @@ CIA_OUTPUT_MAIN_INSTANCE="${EQUINIXMETAL_INSTANCE_TYPE}"
 CIA_OUTPUT_ALL_TESTS=( "${@}" )
 CIA_OUTPUT_EXTRA_INSTANCES=( "${MORE_INSTANCE_TYPES[@]}" )
 CIA_OUTPUT_EXTRA_INSTANCE_TESTS=( 'cl.internet' )
+# The maximum is 6h coming from the ore GC duration parameter
+CIA_OUTPUT_TIMEOUT=6h
 
 query_kola_tests() {
     shift; # ignore the instance type
     kola list --platform=equinixmetal --filter "${@}"
 }
 
-# The maximum is 6h coming from the ore GC duration parameter
-timeout=6h
-
 BASE_URL="http://${BUILDCACHE_SERVER}/images/${CIA_ARCH}/${CIA_VERNUM}"
 
 run_kola_tests() {
     local instance_type="${1}"; shift
-    local instance_tapfile="${1}"; shift
 
-    timeout --signal=SIGQUIT "${timeout}" \
-        kola run \
-          --board="${CIA_ARCH}-usr" \
-          --basename="ci-${CIA_VERNUM/+/-}-${CIA_ARCH}" \
-          --platform=equinixmetal \
-          --tapfile="${instance_tapfile}" \
-          --parallel="${EQUINIXMETAL_PARALLEL}" \
-          --torcx-manifest="${CIA_TORCX_MANIFEST}" \
-          --equinixmetal-image-url="${BASE_URL}/${EQUINIXMETAL_IMAGE_NAME}" \
-          --equinixmetal-installer-image-kernel-url="${BASE_URL}/${PXE_KERNEL_NAME}" \
-          --equinixmetal-installer-image-cpio-url="${BASE_URL}/${PXE_IMAGE_NAME}" \
-          --equinixmetal-metro="${equinixmetal_metro}" \
-          --equinixmetal-plan="${instance_type}" \
-          --equinixmetal-project="${EQUINIXMETAL_PROJECT}" \
-          --equinixmetal-storage-url="${EQUINIXMETAL_STORAGE_URL}" \
-          --gce-json-key=<(set +x; echo "${GCP_JSON_KEY}" | base64 --decode) \
-          --equinixmetal-api-key="${EQUINIXMETAL_KEY}" \
-          "${@}"
+    kola_run \
+        --basename="ci-${CIA_VERNUM/+/-}-${CIA_ARCH}" \
+        --platform=equinixmetal \
+        --parallel="${EQUINIXMETAL_PARALLEL}" \
+        --equinixmetal-image-url="${BASE_URL}/${EQUINIXMETAL_IMAGE_NAME}" \
+        --equinixmetal-installer-image-kernel-url="${BASE_URL}/${PXE_KERNEL_NAME}" \
+        --equinixmetal-installer-image-cpio-url="${BASE_URL}/${PXE_IMAGE_NAME}" \
+        --equinixmetal-metro="${equinixmetal_metro}" \
+        --equinixmetal-plan="${instance_type}" \
+        --equinixmetal-project="${EQUINIXMETAL_PROJECT}" \
+        --equinixmetal-storage-url="${EQUINIXMETAL_STORAGE_URL}" \
+        --gce-json-key=<(echo "${GCP_JSON_KEY}" | base64 --decode) \
+        --equinixmetal-api-key="${EQUINIXMETAL_KEY}" \
+        "${@}"
 }
 
 run_default_kola_tests

ci-automation/vendor-testing/gce.sh

@@ -26,6 +26,7 @@ CIA_OUTPUT_MAIN_INSTANCE='default'
 CIA_OUTPUT_ALL_TESTS=( "${@}" )
 CIA_OUTPUT_EXTRA_INSTANCES=( 'gvnic' )
 CIA_OUTPUT_EXTRA_INSTANCE_TESTS=( 'cl.internet' )
+CIA_OUTPUT_TIMEOUT=6h
 
 query_kola_tests() {
     shift; # ignore the instance type
@@ -56,13 +57,11 @@ trap 'ore gcloud delete-images \
 
 run_kola_tests() {
     local instance_type="${1}"; shift
-    local instance_tapfile="${1}"; shift
     local extra_arg=()
     if [ "${instance_type}" = "gvnic" ]; then
         extra_arg+=("--gce-gvnic")
     fi
-    timeout --signal=SIGQUIT 6h \
-    kola run \
+    kola_run \
         --basename="${image_name}" \
         --gce-image="${image_name}" \
         --gce-json-key="${GCP_JSON_KEY_PATH}" \
@@ -70,9 +69,6 @@ run_kola_tests() {
         "${extra_arg[@]}" \
         --parallel="${GCE_PARALLEL}" \
         --platform=gce \
-        --channel="${CIA_CHANNEL}" \
-        --tapfile="${instance_tapfile}" \
-        --torcx-manifest="${CIA_TORCX_MANIFEST}" \
         "${@}"
 }
 

ci-automation/vendor-testing/openstack.sh

@@ -26,6 +26,7 @@ CIA_OUTPUT_MAIN_INSTANCE='default'
 CIA_OUTPUT_ALL_TESTS=( "${@}" )
 CIA_OUTPUT_EXTRA_INSTANCES=()
 CIA_OUTPUT_EXTRA_INSTANCE_TESTS=()
+CIA_OUTPUT_TIMEOUT=2h
 
 query_kola_tests() {
     shift; # ignore the instance type
@@ -54,25 +55,20 @@ kola_test_basename="${kola_test_basename//[+.]/-}"
 
 run_kola_tests() {
     shift # ignore the instance type
-    local instance_tapfile="${1}"; shift
 
-    timeout --signal=SIGQUIT 2h kola run \
-            --board="${CIA_ARCH}-usr" \
-            --parallel="${OPENSTACK_PARALLEL}" \
-            --tapfile="${instance_tapfile}" \
-            --channel="${CIA_CHANNEL}" \
-            --torcx-manifest="${CIA_TORCX_MANIFEST}" \
-            --basename="${kola_test_basename}" \
-            --platform=openstack \
-            --openstack-network=public \
-            --openstack-domain=default \
-            --openstack-flavor=flatcar-flavor \
-            --openstack-user="${OPENSTACK_USER}" \
-            --openstack-host="${OPENSTACK_HOST}" \
-            --openstack-keyfile="${openstack_keyfile}" \
-            --openstack-image="${IMAGE_ID}" \
-            --openstack-config-file="${config_file}" \
-            "${@}"
+    kola_run \
+        --parallel="${OPENSTACK_PARALLEL}" \
+        --basename="${kola_test_basename}" \
+        --platform=openstack \
+        --openstack-network=public \
+        --openstack-domain=default \
+        --openstack-flavor=flatcar-flavor \
+        --openstack-user="${OPENSTACK_USER}" \
+        --openstack-host="${OPENSTACK_HOST}" \
+        --openstack-keyfile="${openstack_keyfile}" \
+        --openstack-image="${IMAGE_ID}" \
+        --openstack-config-file="${config_file}" \
+        "${@}"
 }
 
 run_default_kola_tests

ci-automation/vendor-testing/qemu.sh

@@ -54,18 +54,14 @@ fi
 
 run_kola_tests() {
     shift # ignore the instance type
-    local instance_tapfile="${1}"; shift
 
-    kola run \
-         --board="${CIA_ARCH}-usr" \
-         --parallel="${QEMU_PARALLEL}" \
-         --platform=qemu \
-         --qemu-bios="${bios}" \
-         --qemu-image="${QEMU_IMAGE_NAME}" \
-         --tapfile="${instance_tapfile}" \
-         --torcx-manifest="${CIA_TORCX_MANIFEST}" \
-         --qemu-skip-mangle \
-         "${@}"
+    kola_run \
+        --parallel="${QEMU_PARALLEL}" \
+        --platform=qemu \
+        --qemu-bios="${bios}" \
+        --qemu-image="${QEMU_IMAGE_NAME}" \
+        --qemu-skip-mangle \
+        "${@}"
 }
 
 run_default_kola_tests

ci-automation/vendor-testing/qemu_update.sh

@@ -91,7 +91,6 @@ fi
 
 run_kola_tests() {
     local instance_type="${1}"; shift;
-    local instance_tapfile="${1}"; shift
     local image
     if [ "${instance_type}" = "previous" ]; then
         image="tmp/flatcar_production_image_previous.bin"
@@ -102,14 +101,11 @@ run_kola_tests() {
         exit 1
     fi
 
-    kola run \
-      --board="${CIA_ARCH}-usr" \
+    kola_run \
       --parallel="${QEMU_PARALLEL}" \
       --platform=qemu \
       --qemu-bios="${bios}" \
       --qemu-image="${image}" \
-      --tapfile="${instance_tapfile}" \
-      --torcx-manifest="${CIA_TORCX_MANIFEST}" \
       --update-payload=tmp/flatcar_test_update.gz \
       --qemu-skip-mangle \
       cl.update.payload

ci-automation/vendor-testing/vmware.sh

@@ -26,6 +26,7 @@ CIA_OUTPUT_MAIN_INSTANCE='default'
 CIA_OUTPUT_ALL_TESTS=( "${@}" )
 CIA_OUTPUT_EXTRA_INSTANCES=()
 CIA_OUTPUT_EXTRA_INSTANCE_TESTS=()
+CIA_OUTPUT_TIMEOUT=2h
 
 query_kola_tests() {
     shift; # ignore the instance type
@@ -63,19 +64,14 @@ trap 'ore esx --esx-config-file "${config_file}" remove-vms \
 
 run_kola_tests() {
     shift # ignore the instance type
-    local instance_tapfile="${1}"; shift
 
-    timeout --signal=SIGQUIT 2h kola run \
-            --board="${CIA_ARCH}-usr" \
-            --basename="${kola_test_basename}" \
-            --channel="${CIA_CHANNEL}" \
-            --platform=esx \
-            --tapfile="${instance_tapfile}" \
-            --parallel="${VMWARE_ESX_PARALLEL}" \
-            --torcx-manifest="${CIA_TORCX_MANIFEST}" \
-            --esx-config-file "${config_file}" \
-            --esx-ova-path "${VMWARE_ESX_IMAGE_NAME}" \
-            "${@}"
+    kola_run \
+        --basename="${kola_test_basename}" \
+        --platform=esx \
+        --parallel="${VMWARE_ESX_PARALLEL}" \
+        --esx-config-file "${config_file}" \
+        --esx-ova-path "${VMWARE_ESX_IMAGE_NAME}" \
+        "${@}"
 }
 
 run_default_kola_tests

ci-automation/vendor_test.sh

@@ -398,12 +398,8 @@ function run_kola_tests_on_instances() {
             tests_on_instances_running=1
             (
                 local instance_tapfile="instance_${instance_type}_validate.tap"
-                set +e
-                set -x
                 local output
-                output=$(run_kola_tests "${instance_type}" "${instance_tapfile}" "${instance_tests[@]}" 2>&1)
-                set +x
-                set -e
+                output=$(run_kola_tests_internal "${instance_type}" "${instance_tapfile}" "${instance_tests[@]}" 2>&1)
                 local escaped_instance_type
                 escaped_instance_type="$(sed -e 's/[\/&]/\\&/g' <<<"${instance_type}")"
                 printf "=== START ${instance_type} ===\n%s\n=== END ${instance_type} ===\n" "$(sed -e "s/^/${escaped_instance_type}: /g" <<<"${output}")"
@@ -416,14 +412,7 @@ function run_kola_tests_on_instances() {
 
     filter_out_prefixed_tests main_tests "${@}"
     if [[ "${#main_tests[@]}" -gt 0 ]]; then
-        # run in a subshell, so the set -x and set +e do not pollute
-        # the outer environment
-        (
-            set +e
-            set -x
-            run_kola_tests "${main_instance_type}" "${main_tapfile}" "${main_tests[@]}"
-            true
-        )
+        run_kola_tests_internal "${main_instance_type}" "${main_tapfile}" "${main_tests[@]}"
     fi
 
     if [[ "${tests_on_instances_running}" -eq 1 ]]; then
@@ -433,6 +422,15 @@ function run_kola_tests_on_instances() {
     fi
 }
 
+# An internal function that invokes the user-defined run_kola_tests
+# callback. It defines the CIA_INTERNAL_TAPFILE variable, which is
+# used by the kola_run function.
+function run_kola_tests_internal() {
+    local instance_type="${1}"; shift
+    local CIA_INTERNAL_TAPFILE="${1}"; shift
+    run_kola_tests "${instance_type}" "${@}"
+}
+
 # Runs the user-defined query_kola_tests callback and massages its
 # output so only a list of tests is printed.
 function run_query_kola_tests() {
@@ -464,3 +462,43 @@ function run_default_kola_tests() {
         '--' \
         "${CIA_OUTPUT_ALL_TESTS[@]}"
 }
+
+# Invokes kola. Does it with a timeout if CIA_OUTPUT_TIMEOUT is not
+# empty. All the parameters passed to this function are forwarded to
+# "kola run". There's no need to specify "--board", "--channel",
+# "--torcx-manifest" and "--tapfile" parameters - this function will
+# pass those based on the various CIA_ variables. Usable only inside
+# run_kola_tests callback used by run_default_kola_tests or
+# run_kola_tests_on_instances.
+function kola_run() {
+    local common_opts
+
+    common_opts=(
+        --board="${CIA_ARCH}-usr"
+        --tapfile="${CIA_INTERNAL_TAPFILE}"
+        --torcx-manifest="${CIA_TORCX_MANIFEST}"
+        --channel="${CIA_CHANNEL}"
+    )
+    if [[ -n "${CIA_OUTPUT_TIMEOUT:-}" ]]; then
+        unsafe_code_section \
+            timeout --signal=SIGQUIT "${CIA_OUTPUT_TIMEOUT}" \
+                kola run "${common_opts[@]}" "${@}"
+    else
+        unsafe_code_section \
+            kola run "${common_opts[@]}" "${@}"
+    fi
+    # In case of timeout, the tapfile might be still missing. But
+    # since we were ignoring the error at the time, handle_flaky_setup
+    # didn't run, so do it now.
+    if [[ ! -e "${CIA_INTERNAL_TAPFILE}" ]]; then
+        handle_flaky_setup
+    fi
+}
+
+function unsafe_code_section() {
+    # Run in a subshell to avoid executing the err handler.
+    (
+        echo "${@}"
+        "${@}" || :
+    )
+}