From d65966eb7621ddf3f8604086c6129e7a9e9bee76 Mon Sep 17 00:00:00 2001
From: Dongsu Park <dpark@linux.microsoft.com>
Date: Wed, 13 Apr 2022 18:31:39 +0200
Subject: [PATCH 1/5] ci-automation: Add WIP AWS test script for CI automation

---
 ci-automation/ci-config.env         |  8 ++++
 ci-automation/vendor-testing/aws.sh | 72 +++++++++++++++++++++++++++++
 2 files changed, 80 insertions(+)
 create mode 100644 ci-automation/vendor-testing/aws.sh

diff --git a/ci-automation/ci-config.env b/ci-automation/ci-config.env
index 2282b66b6f..8faef2747a 100644
--- a/ci-automation/ci-config.env
+++ b/ci-automation/ci-config.env
@@ -102,3 +102,11 @@ DIGITALOCEAN_PARALLEL="${PARALLEL_TESTS:-8}"
 VMWARE_ESX_PARALLEL="${PARALLEL_TESTS:-4}"
 # VMWARE_ESX_CREDS should come from sdk_container/.env and must be
 # base64-encoded.
+
+# -- AWS --
+
+AWS_INSTANCE_TYPE="${AWS_INSTANCE_TYPE:-t3.small}"
+AWS_OFFER="${AWS_OFFER:-basic}"
+AWS_OEM_SUFFIX="_${AWS_OFFER}"
+AWS_IAM_PROFILE="${AWS_IAM_PROFILE:-ciauto-test}"
+AWS_REGION="${AWS_REGION:-us-east-1}"
diff --git a/ci-automation/vendor-testing/aws.sh b/ci-automation/vendor-testing/aws.sh
new file mode 100644
index 0000000000..fffcc4c1b1
--- /dev/null
+++ b/ci-automation/vendor-testing/aws.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+# Copyright (c) 2022 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -euo pipefail
+
+# Test execution script for the qemu vendor image.
+# This script is supposed to run in the SDK container.
+
+work_dir="$1"; shift
+arch="$1"; shift
+vernum="$1"; shift
+tapfile="$1"; shift
+
+AWS_BOARD="${arch}-usr"
+AWS_CHANNEL="$(get_git_channel)"
+AWS_IMAGE_NAME="ci-${vernum}"
+
+if [[ "${arch}" == "arm64-usr" ]]; then
+    AWS_INSTANCE_TYPE="a1.large"
+fi
+
+# $@ now contains tests / test patterns to run
+
+source ci-automation/ci_automation_common.sh
+
+mkdir -p "${work_dir}"
+cd "${work_dir}"
+
+testscript="$(basename "$0")"
+
+if [[ "${AWS_AMI_ID}" == "" ]]; then
+    [ -s verify.asc ] && verify_key=--verify-key=verify.asc || verify_key=
+
+    echo "++++ ${testscript}: downloading flatcar_production_ami_vmdk${AWS_OEM_SUFFIX}_image.vmdk.bz2 for ${vernum} (${arch}) ++++"
+    copy_from_buildcache "images/${arch}/${vernum}/flatcar_production_ami_vmdk${AWS_OEM_SUFFIX}_image.vmdk.bz2" .
+
+    bunzip2 "${work_dir}/flatcar_production_ami_vmdk${AWS_OEM_SUFFIX}_image.vmdk.bz2"
+
+    # FIXME: need to check if it is ok to run ore
+    AWS_BUCKET="flatcar-kola-ami-import-${AWS_REGION}"
+    trap 'bin/ore -d aws delete --region="${AWS_REGION}" --name="${AWS_IMAGE_NAME}" --ami-name="${AWS_IMAGE_NAME}" --file="${work_dir}/flatcar_production_ami_vmdk${AWS_OEM_SUFFIX}_image.vmdk" --bucket "s3://${AWS_BUCKET}/${AWS_BOARD}/"; rm -r ${work_dir}/' EXIT
+    bin/ore aws initialize --region="${AWS_REGION}" --bucket "${AWS_BUCKET}"
+    AWS_AMI_ID=$(bin/ore aws upload --force --region="${AWS_REGION}" --name=${AWS_IMAGE_NAME} --ami-name="${AWS_IMAGE_NAME}" --ami- description="Flatcar Test ${AWS_IMAGE_NAME}" --file="${work_dir}/flatcar_production_ami_vmdk${AWS_OEM_SUFFIX}_image.vmdk" --bucket  "s3://${AWS_BUCKET}/${AWS_BOARD}/" | jq -r .HVM)
+    echo "Created new AMI ${AWS_AMI_ID} (will be removed after testing)"
+fi
+
+
+# AWS timeout
+timeout=6h
+
+set -x
+set -o noglob
+
+sudo timeout --signal=SIGQUIT ${timeout} bin/kola run \
+    --board="${AWS_BOARD}" \
+    --basename="${AWS_IMAGE_NAME}" \
+    --channel="${AWS_CHANNEL}" \
+    --offering="${AWS_OFFER}" \
+    --parallel=${PARALLEL_TESTS} \
+    --platform=aws \
+    --aws-ami="${AWS_AMI_ID}" \
+    --aws-region="${AWS_REGION}" \
+    --aws-type="${AWS_INSTANCE_TYPE}" \
+    --aws-iam-profile="${AWS_IAM_PROFILE}" \
+    --tapfile="${tapfile}" \
+    --torcx-manifest=torcx_manifest.json \
+    $@
+
+set +o noglob
+set +x

From 5e259c89f0bf689bb9226c251ca77f249de2935e Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Thu, 5 May 2022 12:50:11 +0200
Subject: [PATCH 2/5] ci-automation: Let vendor scripts know if this is their
 first run

I will need it to correctly handle test reruns as we will need to
handle passed test names differently on first runs than on reruns.
---
 ci-automation/test.sh        | 4 ++++
 ci-automation/vendor_test.sh | 9 +++++++++
 2 files changed, 13 insertions(+)

diff --git a/ci-automation/test.sh b/ci-automation/test.sh
index c66fffeb09..2810ca9faf 100644
--- a/ci-automation/test.sh
+++ b/ci-automation/test.sh
@@ -146,6 +146,9 @@ function test_run() {
     local success=false
     # A job on each worker prunes old mantle images (docker image prune)
     echo "docker rm -f '${container_name}'" >> ./ci-cleanup.sh
+
+    # Vendor tests may need to know if it is a first run or a rerun
+    touch "${work_dir}/first_run"
     for retry in $(seq "${retries}"); do
         local tapfile="results-run-${retry}.tap"
         local failfile="failed-run-${retry}.txt"
@@ -164,6 +167,7 @@ function test_run() {
                 \"${tapfile}\" \
                 $@"
         set -e
+        rm -f "${work_dir}/first_run"
 
         docker run --pull always --rm --name="${container_name}" --privileged --net host -v /dev:/dev \
           -w /work -v "$PWD":/work "${mantle_ref}" \
diff --git a/ci-automation/vendor_test.sh b/ci-automation/vendor_test.sh
index 0aac38a641..9e8d25a5ea 100644
--- a/ci-automation/vendor_test.sh
+++ b/ci-automation/vendor_test.sh
@@ -51,6 +51,9 @@
 #   Path to the Torcx manifest. Usually passed to kola through the
 #   --torcx-manifest parameter.
 #
+# CIA_FIRST_RUN:
+#   1 if this is a first run, 0 if it is a rerun of failed tests.
+#
 #
 # After this script is sourced, the parameters in ${@} specify test
 # cases / test case patterns to run.
@@ -90,6 +93,11 @@ fi
 ciavts_tapfile="${PWD}/${ciavts_work_dir}/${ciavts_tapfile}"
 ciavts_torcx_manifest="${PWD}/${ciavts_main_work_dir}/torcx_manifest.json"
 
+ciavts_first_run=0
+if [[ -f "${ciavts_main_work_dir}/first_run" ]]; then
+    ciavts_first_run=1
+fi
+
 echo "++++ Running ${ciavts_testscript} inside ${ciavts_work_dir} ++++"
 
 cd "${ciavts_work_dir}"
@@ -102,6 +110,7 @@ CIA_TESTSCRIPT="${ciavts_testscript}"
 CIA_GIT_VERSION="${ciavts_git_version}"
 CIA_BUILD_TYPE="${ciavts_type}"
 CIA_TORCX_MANIFEST="${ciavts_torcx_manifest}"
+CIA_FIRST_RUN="${ciavts_first_run}"
 
 # Unset all variables with ciavts_ prefix now.
 unset -v "${!ciavts_@}"

From 3a19d7ed272f288d2bb3513e1416a42f59fec145 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Thu, 5 May 2022 12:53:56 +0200
Subject: [PATCH 3/5] ci-automation: Fix passing multiple test names to vendor
 scripts

---
 ci-automation/test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci-automation/test.sh b/ci-automation/test.sh
index 2810ca9faf..8d7b48facd 100644
--- a/ci-automation/test.sh
+++ b/ci-automation/test.sh
@@ -165,7 +165,7 @@ function test_run() {
                 \"${arch}\" \
                 \"${vernum}\" \
                 \"${tapfile}\" \
-                $@"
+                $*"
         set -e
         rm -f "${work_dir}/first_run"
 

From 36be73c633a51606aa3ebbd14dd2925ae51a58d7 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Thu, 5 May 2022 12:56:24 +0200
Subject: [PATCH 4/5] ci-automation: Add helper functions for running tests on
 multiple instances

---
 ci-automation/vendor_test.sh | 269 +++++++++++++++++++++++++++++++++++
 1 file changed, 269 insertions(+)

diff --git a/ci-automation/vendor_test.sh b/ci-automation/vendor_test.sh
index 9e8d25a5ea..77fbd7b4f3 100644
--- a/ci-automation/vendor_test.sh
+++ b/ci-automation/vendor_test.sh
@@ -114,3 +114,272 @@ CIA_FIRST_RUN="${ciavts_first_run}"
 
 # Unset all variables with ciavts_ prefix now.
 unset -v "${!ciavts_@}"
+
+# Prefixes all test names in the tap file with a given prefix, so the
+# test name like "cl.basic" will become "extra-test.[${prefix}].cl.basic".
+#
+# Typical use:
+#   prefix_tap_file "${instance_type}" "${tapfile}"
+#
+# Parameters:
+# 1 - prefix
+# 2 - tap file, modified in place
+function prefix_tap_file() {
+    local prefix="${1}"; shift
+    local tap_file="${1}"; shift
+    # drop the dots from prefix
+    local actual_prefix="extra-test.[${prefix}]."
+
+    sed --in-place --expression 's/^\(\s*\(not\)\?\s*ok[^-]*\s*-\s*\)\(\S\)/\1'"${actual_prefix}"'\3/g' "${tap_file}"
+}
+
+# Filters the test names, so it puts only the real names of the
+# prefixed tests into the chosen variable. For example for prefix
+# "foo", it will ignore the test name like "cl.basic", but will print
+# "cl.internet" for a test name like "extra-test.[foo].cl.internet".
+# "*" is treated specially - it will be inserted into the chosen
+# variable if it is passed.
+#
+# Typical use:
+#   filter_prefixed_tests tests_to_run "${instance_type}" "${@}"
+#   if [[ "${#tests_to_run[@]}" -gt 0 ]]; then …; fi
+#
+# Parameters:
+# 1 - name of an array variable where the filtering results will be stored
+# 2 - prefix
+# @ - test names
+function filter_prefixed_tests() {
+    local var_name="${1}"; shift
+    local prefix="${1}"; shift
+    # rest of the parameters are test names
+    local -n results="${var_name}"
+    local name
+    local stripped_name
+    for name; do
+        stripped_name="${name#extra-test.\[${prefix}\].}"
+        if [[ "${stripped_name}" != "${name}" ]]; then
+            results+=( "${stripped_name}" )
+            continue
+        elif [[ "${name}" = '*' ]]; then
+            results+=( '*' )
+        fi
+    done
+}
+
+# Filters out the extra tests from the passed test names. Ignored test
+# names begin with "extra-test.". The results of the filtering are
+# inserted into the chosen variable.
+#
+# Typical use:
+#   filter_out_prefixed_tests tests_to_run "${@}"
+#   if [[ "${#tests_to_run[@]}" -gt 0 ]]; then …; fi
+#
+# Parameters:
+# 1 - name of an array variable where the filtering results will be stored
+# @ - test names
+function filter_out_prefixed_tests() {
+    local var_name="${1}"; shift
+    local -n results="${var_name}"
+    local name
+    for name; do
+        if [[ "${name#extra-test.}" = "${name}" ]]; then
+            results+=( "${name}" )
+        fi
+    done
+}
+
+# Merges into the first (main) tap file the contents of other tap
+# files. It is very simple - the function assumes that all the tap
+# files begin with a line like:
+#
+# 1..${number_of_tests}
+#
+# Other lines that are processed should begin like:
+#
+# (not)? ok - ${test_name}
+#
+# Any other lines are copied verbatim.
+#
+# The other tap files should already be preprocessed by
+# prefix_tap_file to avoid duplicated test names.
+#
+# Typical use:
+#   merge_tap_files "${tap_file}" extra-validation-*.tap
+#   rm -f extra-validation-*.tap
+#
+# Parameters:
+# 1 - main tap file
+# @ - other tap files
+function merge_tap_files() {
+    local main_tap_file="${1}"; shift
+    # rest of the parameters are other tap files
+
+    local main_test_count=0
+    if [[ -f "${main_tap_file}" ]]; then
+        main_test_count=$(head --lines=1 "${main_tap_file}" | grep --only-matching '[0-9]\+$')
+    fi
+    local other_test_count
+    local other_tap_file
+    local tmp_tap_file="${main_tap_file}.mtf.tmp"
+    for other_tap_file; do
+        if [[ ! -f "${other_tap_file}" ]]; then
+            continue
+        fi
+        other_test_count=$(head --lines=1 "${other_tap_file}" | grep --only-matching '[0-9]\+$' || echo 0 )
+        ((main_test_count+=other_test_count))
+    done
+    echo "1..${main_test_count}" >"${tmp_tap_file}"
+    if [[ -f "${main_tap_file}" ]]; then
+        tail --lines=+2 "${main_tap_file}" >>"${tmp_tap_file}"
+    fi
+    for other_tap_file; do
+        if [[ ! -f "${other_tap_file}" ]]; then
+            continue
+        fi
+        tail --lines=+2 "${other_tap_file}" >>"${tmp_tap_file}"
+    done
+    mv --force "${tmp_tap_file}" "${main_tap_file}"
+}
+
+# Runs or reruns the tests on the main instance and other
+# instances. Other instances usually run a subset of tests only.
+#
+# For this function to work, the caller needs to define two functions
+# beforehand:
+#
+# run_kola_tests that takes the following parameters:
+# 1 - instance type
+# 2 - tap file
+# @ - tests to run
+#
+# query_kola_tests that takes the following parameters:
+# 1 - instance type
+# @ - tests to run
+# This function should print the names of the tests to run. Every line
+# of the output should have one test name to run. Any other cruft in
+# the line will be ignored.
+#
+# Typical use:
+# function run_kola_tests() {
+#     local instance_type="${1}"; shift
+#     local tap_file="${1}"; shift
+#     kola run … "${@}"
+# }
+#
+# function query_kola_tests() {
+#     local instance_type="${1}"; shift
+#     kola list … "${@}"
+# }
+#
+# args=(
+#     "${main_instance}"
+#     "${CIA_TAPFILE}"
+#     "${CIA_FIRST_RUN}"
+#     "${other_instance_types[@]}"
+#     '--'
+#     'cl.internet'
+#     '--'
+#     "${tests_to_run[@]}"
+# )
+# run_kola_tests_on_instances "${args[@]}"
+#
+# Parameters:
+# 1 - main instance type - there all the tests are being run
+# 2 - main tap file
+# 3 - if this is first run (1 if it is, 0 if it is a rerun)
+# @ - other instance types followed by double dash (--) followed by
+#     test names for other instances to filter from the tests to be
+#     run followed by double dash, followed by tests to be run or
+#     rerun
+function run_kola_tests_on_instances() {
+    local main_instance_type="${1}"; shift
+    local main_tapfile="${1}"; shift
+    local is_first_run="${1}"; shift
+    local other_instance_types=()
+    local other_tests=()
+    local arg
+
+    while [[ "${#}" -gt 0 ]]; do
+        arg="${1}"; shift
+        if [[ "${arg}" = '--' ]]; then
+            break
+        fi
+        other_instance_types+=( "${arg}" )
+    done
+
+    while [[ "${#}" -gt 0 ]]; do
+        arg="${1}"; shift
+        if [[ "${arg}" = '--' ]]; then
+            break
+        fi
+        other_tests+=( "${arg}" )
+    done
+
+    # rest of the parameters are tests to be run or rerun
+
+    local instance_type
+    local queried_tests
+    local instance_tests=()
+    local tests_on_instances_running=0
+    local other_tests_for_fgrep
+    other_tests_for_fgrep="$(printf '%s\n' "${other_tests[@]}")"
+
+    for instance_type in "${other_instance_types[@]}"; do
+        # On first run we usually pass the canonical test names like
+        # cl.basic, cl.internet or *, so we decide which tests should
+        # be run on the other instances based on this list. On the
+        # other hand, the rerun will contain names of the failed tests
+        # only, and those are specific - if a test failed on the main
+        # instance, the name of the test will be like cl.basic; if a
+        # test failed on other instance, the name of the test will be
+        # like extra-test.[…].cl.basic. So in case of reruns, we want
+        # to filter the extra tests first then we decide which tests
+        # should be run.
+        if [[ "${is_first_run}" -eq 1 ]]; then
+            set -o noglob # noglob should not be necessary, as
+                          # query_kola_tests shouldn't return a
+                          # wildcard, but better to be safe than sorry
+            queried_tests="$(query_kola_tests "${instance_type}" "${@}")"
+            instance_tests=( $(grep --only-matching --fixed-strings "${other_tests_for_fgrep}" <<<"${queried_tests}" || :) )
+            set +o noglob
+        else
+            filter_prefixed_tests instance_tests "${instance_type}" "${@}"
+        fi
+        if [[ "${#instance_tests[@]}" -gt 0 ]]; then
+            tests_on_instances_running=1
+            (
+                local instance_tapfile="instance_${instance_type}_validate.tap"
+                set +e
+                set -x
+                local output
+                output=$(run_kola_tests "${instance_type}" "${instance_tapfile}" "${instance_tests[@]}" 2>&1)
+                set +x
+                set -e
+                local escaped_instance_type
+                escaped_instance_type="$(sed -e 's/[\/&]/\\&/g' <<<"${instance_type}")"
+                printf "=== START ${instance_type} ===\n%s\n=== END ${instance_type} ===\n" "$(sed -e "s/^/${escaped_instance_type}: /g" <<<"${output}")"
+                prefix_tap_file "${instance_type}" "${instance_tapfile}"
+            ) &
+        fi
+    done
+
+    local -a main_tests
+
+    filter_out_prefixed_tests main_tests "${@}"
+    if [[ "${#main_tests[@]}" -gt 0 ]]; then
+        # run in a subshell, so the set -x and set +e do not pollute
+        # the outer environment
+        (
+            set +e
+            set -x
+            run_kola_tests "${main_instance_type}" "${main_tapfile}" "${main_tests[@]}"
+            true
+        )
+    fi
+
+    if [[ "${tests_on_instances_running}" -eq 1 ]]; then
+        wait
+        merge_tap_files "${main_tapfile}" 'instance_'*'_validate.tap'
+        rm -f 'instance_'*'_validate.tap'
+    fi
+}

From 47eb233fc4392002b82907a258aaabf67df98a0a Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Thu, 5 May 2022 12:55:03 +0200
Subject: [PATCH 5/5] ci-automation: Make AWS test script to work

---
 ci-automation/ci-config.env         |  19 +++--
 ci-automation/vendor-testing/aws.sh | 116 +++++++++++++++-------------
 2 files changed, 77 insertions(+), 58 deletions(-)
 mode change 100644 => 100755 ci-automation/vendor-testing/aws.sh

diff --git a/ci-automation/ci-config.env b/ci-automation/ci-config.env
index 8faef2747a..2875c8f5b1 100644
--- a/ci-automation/ci-config.env
+++ b/ci-automation/ci-config.env
@@ -105,8 +105,17 @@ VMWARE_ESX_PARALLEL="${PARALLEL_TESTS:-4}"
 
 # -- AWS --
 
-AWS_INSTANCE_TYPE="${AWS_INSTANCE_TYPE:-t3.small}"
-AWS_OFFER="${AWS_OFFER:-basic}"
-AWS_OEM_SUFFIX="_${AWS_OFFER}"
-AWS_IAM_PROFILE="${AWS_IAM_PROFILE:-ciauto-test}"
-AWS_REGION="${AWS_REGION:-us-east-1}"
+: ${AWS_amd64_INSTANCE_TYPE:="t3.small"}
+# Space separated list of instance types. On those instances the
+# cl.internet kola test will be run if this test is selected to run.
+: ${AWS_amd64_MORE_INSTANCE_TYPES:="m4.2xlarge"}
+: ${AWS_arm64_INSTANCE_TYPE:="a1.large"}
+# Space separated list of instance types. On those instances the
+# cl.internet kola test will be run if this test is selected to run.
+: ${AWS_arm64_MORE_INSTANCE_TYPES:=""}
+: ${AWS_IAM_PROFILE:="ciauto-test"}
+: ${AWS_REGION:="us-east-1"}
+: ${AWS_AMI_ID:=""}
+AWS_PARALLEL="${PARALLEL_TESTS:-8}"
+# AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY should come from
+# sdk_container/.env
diff --git a/ci-automation/vendor-testing/aws.sh b/ci-automation/vendor-testing/aws.sh
old mode 100644
new mode 100755
index fffcc4c1b1..ac5043064f
--- a/ci-automation/vendor-testing/aws.sh
+++ b/ci-automation/vendor-testing/aws.sh
@@ -5,68 +5,78 @@
 
 set -euo pipefail
 
-# Test execution script for the qemu vendor image.
-# This script is supposed to run in the SDK container.
+# Test execution script for the AWS vendor image.
+# This script is supposed to run in the mantle container.
 
-work_dir="$1"; shift
-arch="$1"; shift
-vernum="$1"; shift
-tapfile="$1"; shift
+source ci-automation/vendor_test.sh
 
-AWS_BOARD="${arch}-usr"
-AWS_CHANNEL="$(get_git_channel)"
-AWS_IMAGE_NAME="ci-${vernum}"
+board="${CIA_ARCH}-usr"
+escaped_vernum="${CIA_VERNUM//+/-}"
+image_name="ci-${escaped_vernum}-${CIA_ARCH}"
+aws_instance_type_var="AWS_${CIA_ARCH}_INSTANCE_TYPE"
+aws_instance_type="${!aws_instance_type_var}"
+more_aws_instance_types_var="AWS_${CIA_ARCH}_MORE_INSTANCE_TYPES"
+set -o noglob # there shouldn't be any instance types with asterisks
+              # in it, but…
+more_aws_instance_types=( ${!more_aws_instance_types_var} )
+set +o noglob
 
-if [[ "${arch}" == "arm64-usr" ]]; then
-    AWS_INSTANCE_TYPE="a1.large"
-fi
-
-# $@ now contains tests / test patterns to run
-
-source ci-automation/ci_automation_common.sh
-
-mkdir -p "${work_dir}"
-cd "${work_dir}"
-
-testscript="$(basename "$0")"
+vmdk='flatcar_production_ami_vmdk_image.vmdk'
+tarball="${vmdk}.bz2"
 
 if [[ "${AWS_AMI_ID}" == "" ]]; then
-    [ -s verify.asc ] && verify_key=--verify-key=verify.asc || verify_key=
+    if [[ -f "${vmdk}" ]]; then
+        echo "++++ ${CIA_TESTSCRIPT}: using existing ${vmdk} for ${CIA_VERNUM} (${CIA_ARCH}) ++++"
+    else
+        echo "++++ ${CIA_TESTSCRIPT}: downloading ${tarball} for ${CIA_VERNUM} (${CIA_ARCH}) ++++"
+        copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/${tarball}" .
+        lbunzip2 "${tarball}"
+    fi
 
-    echo "++++ ${testscript}: downloading flatcar_production_ami_vmdk${AWS_OEM_SUFFIX}_image.vmdk.bz2 for ${vernum} (${arch}) ++++"
-    copy_from_buildcache "images/${arch}/${vernum}/flatcar_production_ami_vmdk${AWS_OEM_SUFFIX}_image.vmdk.bz2" .
-
-    bunzip2 "${work_dir}/flatcar_production_ami_vmdk${AWS_OEM_SUFFIX}_image.vmdk.bz2"
-
-    # FIXME: need to check if it is ok to run ore
-    AWS_BUCKET="flatcar-kola-ami-import-${AWS_REGION}"
-    trap 'bin/ore -d aws delete --region="${AWS_REGION}" --name="${AWS_IMAGE_NAME}" --ami-name="${AWS_IMAGE_NAME}" --file="${work_dir}/flatcar_production_ami_vmdk${AWS_OEM_SUFFIX}_image.vmdk" --bucket "s3://${AWS_BUCKET}/${AWS_BOARD}/"; rm -r ${work_dir}/' EXIT
-    bin/ore aws initialize --region="${AWS_REGION}" --bucket "${AWS_BUCKET}"
-    AWS_AMI_ID=$(bin/ore aws upload --force --region="${AWS_REGION}" --name=${AWS_IMAGE_NAME} --ami-name="${AWS_IMAGE_NAME}" --ami- description="Flatcar Test ${AWS_IMAGE_NAME}" --file="${work_dir}/flatcar_production_ami_vmdk${AWS_OEM_SUFFIX}_image.vmdk" --bucket  "s3://${AWS_BUCKET}/${AWS_BOARD}/" | jq -r .HVM)
-    echo "Created new AMI ${AWS_AMI_ID} (will be removed after testing)"
+    aws_bucket="flatcar-kola-ami-import-${AWS_REGION}"
+    aws_s3_path="s3://${aws_bucket}/${escaped_vernum}/${board}/"
+    trap 'ore -d aws delete --region="${AWS_REGION}" --board="${board}" --name="${image_name}" --ami-name="${image_name}" --file="${vmdk}" --bucket "${aws_s3_path}"' EXIT
+    ore aws initialize --region="${AWS_REGION}" --bucket "${aws_bucket}"
+    AWS_AMI_ID=$(ore aws upload --force --region="${AWS_REGION}" --board="${board}" --name="${image_name}" --ami-name="${image_name}" --ami-description="Flatcar Test ${image_name}" --file="${vmdk}" --bucket "${aws_s3_path}" | jq -r .HVM)
+    echo "++++ ${CIA_TESTSCRIPT}: created new AMI ${AWS_AMI_ID} (will be removed after testing) ++++"
 fi
 
+run_kola_tests() {
+    local instance_type="${1}"; shift
+    local instance_tapfile="${1}"; shift
 
-# AWS timeout
-timeout=6h
+    timeout --signal=SIGQUIT 6h \
+        kola run \
+         --board="${board}" \
+         --basename="${image_name}" \
+         --channel="${CIA_CHANNEL}" \
+         --offering='basic' \
+         --parallel="${AWS_PARALLEL}" \
+         --platform=aws \
+         --aws-ami="${AWS_AMI_ID}" \
+         --aws-region="${AWS_REGION}" \
+         --aws-type="${instance_type}" \
+         --aws-iam-profile="${AWS_IAM_PROFILE}" \
+         --tapfile="${instance_tapfile}" \
+         --torcx-manifest="${CIA_TORCX_MANIFEST}" \
+         "${@}"
+}
 
-set -x
-set -o noglob
+query_kola_tests() {
+    shift; # ignore the instance type
+    kola list --platform=aws --filter "${@}"
+}
 
-sudo timeout --signal=SIGQUIT ${timeout} bin/kola run \
-    --board="${AWS_BOARD}" \
-    --basename="${AWS_IMAGE_NAME}" \
-    --channel="${AWS_CHANNEL}" \
-    --offering="${AWS_OFFER}" \
-    --parallel=${PARALLEL_TESTS} \
-    --platform=aws \
-    --aws-ami="${AWS_AMI_ID}" \
-    --aws-region="${AWS_REGION}" \
-    --aws-type="${AWS_INSTANCE_TYPE}" \
-    --aws-iam-profile="${AWS_IAM_PROFILE}" \
-    --tapfile="${tapfile}" \
-    --torcx-manifest=torcx_manifest.json \
-    $@
+# these are set in ci-config.env
+export AWS_ACCESS_KEY_ID
+export AWS_SECRET_ACCESS_KEY
 
-set +o noglob
-set +x
+run_kola_tests_on_instances \
+    "${aws_instance_type}" \
+    "${CIA_TAPFILE}" \
+    "${CIA_FIRST_RUN}" \
+    "${more_aws_instance_types[@]}" \
+    '--' \
+    'cl.internet' \
+    '--' \
+    "${@}"