From d416ab759aa55dd547dd2a8e921398e7607a2bd6 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Tue, 8 Nov 2022 14:01:46 +0100
Subject: [PATCH] ci-automation: Opt-in the vendor tests into handling flaky
 setup

---
 ci-automation/vendor-testing/aws.sh           | 15 +++++++----
 ci-automation/vendor-testing/azure.sh         | 25 ++++++++++-------
 ci-automation/vendor-testing/digitalocean.sh  | 10 +++++++
 ci-automation/vendor-testing/equinix_metal.sh | 15 +++++++----
 ci-automation/vendor-testing/gce.sh           | 15 +++++++----
 ci-automation/vendor-testing/openstack.sh     | 10 +++++++
 ci-automation/vendor-testing/qemu.sh          | 10 +++++++
 ci-automation/vendor-testing/qemu_update.sh   | 27 +++++++++++--------
 ci-automation/vendor-testing/vmware.sh        | 10 +++++++
 9 files changed, 101 insertions(+), 36 deletions(-)

diff --git a/ci-automation/vendor-testing/aws.sh b/ci-automation/vendor-testing/aws.sh
index ae843a5efc..fadaf05701 100755
--- a/ci-automation/vendor-testing/aws.sh
+++ b/ci-automation/vendor-testing/aws.sh
@@ -18,6 +18,16 @@ aws_instance_type="${!aws_instance_type_var}"
 more_aws_instance_types_var="AWS_${CIA_ARCH}_MORE_INSTANCE_TYPES"
 mapfile -t more_aws_instance_types < <(tr ' ' '\n' <<<"${!more_aws_instance_types_var}")
 
+CIA_OUTPUT_MAIN_INSTANCE="${aws_instance_type}"
+CIA_OUTPUT_ALL_TESTS=( "${@}" )
+CIA_OUTPUT_EXTRA_INSTANCES=( "${more_aws_instance_types[@]}" )
+CIA_OUTPUT_EXTRA_INSTANCE_TESTS=( 'cl.internet' )
+
+query_kola_tests() {
+    shift; # ignore the instance type
+    kola list --platform=aws --filter "${@}"
+}
+
 image_file='flatcar_production_ami_image.bin'
 tarball="${image_file}.bz2"
 
@@ -59,11 +69,6 @@ run_kola_tests() {
          "${@}"
 }
 
-query_kola_tests() {
-    shift; # ignore the instance type
-    kola list --platform=aws --filter "${@}"
-}
-
 # these are set in ci-config.env
 export AWS_ACCESS_KEY_ID
 export AWS_SECRET_ACCESS_KEY
diff --git a/ci-automation/vendor-testing/azure.sh b/ci-automation/vendor-testing/azure.sh
index 17081b3598..18ba7f4b89 100755
--- a/ci-automation/vendor-testing/azure.sh
+++ b/ci-automation/vendor-testing/azure.sh
@@ -18,6 +18,21 @@ azure_instance_type_var="AZURE_${CIA_ARCH}_MACHINE_SIZE"
 azure_instance_type="${!azure_instance_type_var}"
 azure_vnet_subnet_name="jenkins-vnet-${AZURE_LOCATION}"
 
+other_instance_types=()
+if [[ "${CIA_ARCH}" = 'amd64' ]]; then
+    other_instance_types+=('V1')
+fi
+
+CIA_OUTPUT_MAIN_INSTANCE="${azure_instance_type}"
+CIA_OUTPUT_ALL_TESTS=( "${@}" )
+CIA_OUTPUT_EXTRA_INSTANCES=( "${other_instance_types[@]}" )
+CIA_OUTPUT_EXTRA_INSTANCE_TESTS=( 'cl.internet' )
+
+query_kola_tests() {
+    shift; # ignore the instance type
+    kola list --platform=azure --filter "${@}"
+}
+
 azure_profile_config_file=''
 secret_to_file azure_profile_config_file "${AZURE_PROFILE}"
 azure_auth_config_file=''
@@ -69,16 +84,6 @@ run_kola_tests() {
       "${@}"
 }
 
-query_kola_tests() {
-    shift; # ignore the instance type
-    kola list --platform=azure --filter "${@}"
-}
-
-other_instance_types=()
-if [[ "${CIA_ARCH}" = 'amd64' ]]; then
-    other_instance_types+=('V1')
-fi
-
 run_kola_tests_on_instances \
     "${azure_instance_type}" \
     "${CIA_TAPFILE}" \
diff --git a/ci-automation/vendor-testing/digitalocean.sh b/ci-automation/vendor-testing/digitalocean.sh
index 44fa3c41ee..24ff23cb64 100755
--- a/ci-automation/vendor-testing/digitalocean.sh
+++ b/ci-automation/vendor-testing/digitalocean.sh
@@ -22,6 +22,16 @@ if [[ "${CIA_ARCH}" == "arm64" ]]; then
     exit 1
 fi
 
+CIA_OUTPUT_MAIN_INSTANCE="${DIGITALOCEAN_MACHINE_SIZE}"
+CIA_OUTPUT_ALL_TESTS=( "${@}" )
+CIA_OUTPUT_EXTRA_INSTANCES=()
+CIA_OUTPUT_EXTRA_INSTANCE_TESTS=()
+
+query_kola_tests() {
+    shift; # ignore the instance type
+    kola list --platform=do --filter "${@}"
+}
+
 image_name="ci-${CIA_VERNUM//+/-}"
 image_url="$(url_from_template "${DIGITALOCEAN_IMAGE_URL_TEMPLATE}" "${CIA_ARCH}" "${CIA_CHANNEL}" 'https' "${CIA_VERNUM}")"
 
diff --git a/ci-automation/vendor-testing/equinix_metal.sh b/ci-automation/vendor-testing/equinix_metal.sh
index 163fbbe58b..41f8a7e5c5 100755
--- a/ci-automation/vendor-testing/equinix_metal.sh
+++ b/ci-automation/vendor-testing/equinix_metal.sh
@@ -20,6 +20,16 @@ EQUINIXMETAL_INSTANCE_TYPE="${!EQUINIXMETAL_INSTANCE_TYPE_VAR}"
 MORE_INSTANCE_TYPES_VAR="EQUINIXMETAL_${CIA_ARCH}_MORE_INSTANCE_TYPES"
 mapfile -t MORE_INSTANCE_TYPES < <(tr ' ' '\n' <<<"${!MORE_INSTANCE_TYPES_VAR}")
 
+CIA_OUTPUT_MAIN_INSTANCE="${EQUINIXMETAL_INSTANCE_TYPE}"
+CIA_OUTPUT_ALL_TESTS=( "${@}" )
+CIA_OUTPUT_EXTRA_INSTANCES=( "${MORE_INSTANCE_TYPES[@]}" )
+CIA_OUTPUT_EXTRA_INSTANCE_TESTS=( 'cl.internet' )
+
+query_kola_tests() {
+    shift; # ignore the instance type
+    kola list --platform=equinixmetal --filter "${@}"
+}
+
 # The maximum is 6h coming from the ore GC duration parameter
 timeout=6h
 
@@ -49,11 +59,6 @@ run_kola_tests() {
           "${@}"
 }
 
-query_kola_tests() {
-    shift; # ignore the instance type
-    kola list --platform=equinixmetal --filter "${@}"
-}
-
 run_kola_tests_on_instances \
     "${EQUINIXMETAL_INSTANCE_TYPE}" \
     "${CIA_TAPFILE}" \
diff --git a/ci-automation/vendor-testing/gce.sh b/ci-automation/vendor-testing/gce.sh
index 2ab0fd1284..b56d91c195 100755
--- a/ci-automation/vendor-testing/gce.sh
+++ b/ci-automation/vendor-testing/gce.sh
@@ -22,6 +22,16 @@ if [[ "${CIA_ARCH}" == "arm64" ]]; then
     exit 1
 fi
 
+CIA_OUTPUT_MAIN_INSTANCE='default'
+CIA_OUTPUT_ALL_TESTS=( "${@}" )
+CIA_OUTPUT_EXTRA_INSTANCES=( 'gvnic' )
+CIA_OUTPUT_EXTRA_INSTANCE_TESTS=( 'cl.internet' )
+
+query_kola_tests() {
+    shift; # ignore the instance type
+    kola list --platform=gce --filter "${@}"
+}
+
 GCP_JSON_KEY_PATH=''
 secret_to_file GCP_JSON_KEY_PATH "${GCP_JSON_KEY}"
 
@@ -66,11 +76,6 @@ run_kola_tests() {
         "${@}"
 }
 
-query_kola_tests() {
-    shift; # ignore the instance type
-    kola list --platform=gce --filter "${@}"
-}
-
 run_kola_tests_on_instances \
     "default" \
     "${CIA_TAPFILE}" \
diff --git a/ci-automation/vendor-testing/openstack.sh b/ci-automation/vendor-testing/openstack.sh
index b197f87283..a21151f503 100755
--- a/ci-automation/vendor-testing/openstack.sh
+++ b/ci-automation/vendor-testing/openstack.sh
@@ -22,6 +22,16 @@ if [[ "${CIA_ARCH}" == "arm64" ]]; then
     exit 1
 fi
 
+CIA_OUTPUT_MAIN_INSTANCE='default'
+CIA_OUTPUT_ALL_TESTS=( "${@}" )
+CIA_OUTPUT_EXTRA_INSTANCES=()
+CIA_OUTPUT_EXTRA_INSTANCE_TESTS=()
+
+query_kola_tests() {
+    shift; # ignore the instance type
+    kola list --platform=openstack --filter "${@}"
+}
+
 # OPENSTACK_CREDS, OPENSTACK_USER, OPENSTACK_HOST, OPENSTACK_KEYFILE should be provided by sdk_container/.env
 config_file=''
 secret_to_file config_file "${OPENSTACK_CREDS}"
diff --git a/ci-automation/vendor-testing/qemu.sh b/ci-automation/vendor-testing/qemu.sh
index 99e9d2a9c2..04ed91de73 100755
--- a/ci-automation/vendor-testing/qemu.sh
+++ b/ci-automation/vendor-testing/qemu.sh
@@ -21,6 +21,16 @@ if [ "${CIA_ARCH}" = "arm64" ] && [ "${CIA_TESTSCRIPT}" != "qemu_uefi.sh" ] ; th
     exit 1
 fi
 
+CIA_OUTPUT_MAIN_INSTANCE='default'
+CIA_OUTPUT_ALL_TESTS=( "${@}" )
+CIA_OUTPUT_EXTRA_INSTANCES=()
+CIA_OUTPUT_EXTRA_INSTANCE_TESTS=()
+
+query_kola_tests() {
+    shift; # ignore the instance type
+    kola list --platform=qemu --filter "${@}"
+}
+
 # Fetch image and BIOS if not present
 if [ -f "${QEMU_IMAGE_NAME}" ] ; then
     echo "++++ ${CIA_TESTSCRIPT}: Using existing ${QEMU_IMAGE_NAME} for testing ${CIA_VERNUM} (${CIA_ARCH}) ++++"
diff --git a/ci-automation/vendor-testing/qemu_update.sh b/ci-automation/vendor-testing/qemu_update.sh
index f8fbce53b3..396cd7d594 100755
--- a/ci-automation/vendor-testing/qemu_update.sh
+++ b/ci-automation/vendor-testing/qemu_update.sh
@@ -23,6 +23,22 @@ if [ "$*" != "" ] && [ "$*" != "*" ] && [[ "$*" != *"cl.update.payload" ]]; then
     exit 1
 fi
 
+CIA_OUTPUT_MAIN_INSTANCE='previous'
+CIA_OUTPUT_ALL_TESTS=( 'cl.update.payload' )
+CIA_OUTPUT_EXTRA_INSTANCES=( 'first_dual' )
+CIA_OUTPUT_EXTRA_INSTANCE_TESTS=( 'cl.update.payload' )
+
+query_kola_tests() {
+    shift; # ignore the instance type
+    local arg
+    arg="${*}"
+    if [ "${arg}" != "" ]; then
+      # Empty calls are ok, which mean no tests, but otherwise we restrict the tests to run
+      arg="cl.update.payload"
+    fi
+    kola list --platform=qemu --filter "${arg}"
+}
+
 mkdir -p tmp/
 if [ -f tmp/flatcar_test_update.gz ] ; then
     echo "++++ ${CIA_TESTSCRIPT}: Using existing ./tmp/flatcar_test_update.gz for testing ${CIA_VERNUM} (${CIA_ARCH}) ++++"
@@ -73,17 +89,6 @@ if [ "${CIA_ARCH}" = "arm64" ]; then
     fi
 fi
 
-query_kola_tests() {
-    shift; # ignore the instance type
-    local arg
-    arg="${*}"
-    if [ "${arg}" != "" ]; then
-      # Empty calls are ok, which mean no tests, but otherwise we restrict the tests to run
-      arg="cl.update.payload"
-    fi
-    kola list --platform=qemu --filter "${arg}"
-}
-
 run_kola_tests() {
     local instance_type="${1}"; shift;
     local instance_tapfile="${1}"; shift
diff --git a/ci-automation/vendor-testing/vmware.sh b/ci-automation/vendor-testing/vmware.sh
index e387f2eb46..3ce79056c0 100755
--- a/ci-automation/vendor-testing/vmware.sh
+++ b/ci-automation/vendor-testing/vmware.sh
@@ -22,6 +22,16 @@ if [[ "${CIA_ARCH}" == "arm64" ]]; then
     exit 1
 fi
 
+CIA_OUTPUT_MAIN_INSTANCE='default'
+CIA_OUTPUT_ALL_TESTS=( "${@}" )
+CIA_OUTPUT_EXTRA_INSTANCES=()
+CIA_OUTPUT_EXTRA_INSTANCE_TESTS=()
+
+query_kola_tests() {
+    shift; # ignore the instance type
+    kola list --platform=esx --filter "${@}"
+}
+
 # Fetch image if not present.
 if [ -f "${VMWARE_ESX_IMAGE_NAME}" ] ; then
     echo "++++ ${CIA_TESTSCRIPT}: Using existing ${VMWARE_ESX_IMAGE_NAME} for testing ${CIA_VERNUM} (${CIA_ARCH}) ++++"