mirror of
https://github.com/flatcar/scripts.git
synced 2025-10-31 08:11:03 +01:00
386 lines
12 KiB
Bash
386 lines
12 KiB
Bash
# Copyright (c) 2021 The Flatcar Maintainers.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
# Vendor test helper script. Sourced by vendor tests. Does some
|
|
# initial setup.
|
|
#
|
|
#
|
|
# The initial setup consist of creating the vendor working directory
|
|
# for the vendor test script, specifying the variables described below
|
|
# and changing the current working directory to the vendor working
|
|
# directory.
|
|
#
|
|
#
|
|
# The vendor test script is expected to keep all artifacts it produces
|
|
# in its current working directory.
|
|
#
|
|
#
|
|
# The script specifies the following variables for the vendor test
|
|
# script to use:
|
|
#
|
|
# CIA_VERNUM:
|
|
# Image version. In case of developer builds it comes with a suffix,
|
|
# so it looks like "3217.0.0+nightly-20220422-0155". For release
|
|
# builds the version will be without suffix, so it looks like
|
|
# "3217.0.0". Whether the build is a release or a developer one is
|
|
# reflected in CIA_BUILD_TYPE variable described below.
|
|
#
|
|
# CIA_ARCH:
|
|
# Architecture to test. Currently it is either "amd64" or "arm64".
|
|
#
|
|
# CIA_TAPFILE:
|
|
# Where the TAP reports should be written. Usually just passed to
|
|
# kola throught the --tapfile parameter.
|
|
#
|
|
# CIA_CHANNEL:
|
|
# A channel. Either "alpha", "beta", "stable" or "lts". Used to find
|
|
# the last release for the update check.
|
|
#
|
|
# CIA_TESTSCRIPT:
|
|
# Name of the vendor script. May be useful in some messages.
|
|
#
|
|
# CIA_GIT_VERSION:
|
|
# The most recent tag for the current commit.
|
|
#
|
|
# CIA_BUILD_TYPE:
|
|
# It's either "release" or "developer", based on the CIA_VERNUM
|
|
# variable.
|
|
#
|
|
# CIA_FIRST_RUN:
|
|
# 1 if this is a first run, 0 if it is a rerun of failed tests.
|
|
#
|
|
#
|
|
# After this script is sourced, the parameters in ${@} specify test
|
|
# cases / test case patterns to run.
|
|
|
|
|
|
# "ciavts" stands for Continuous Integration Automation Vendor Test
|
|
# Setup. This prefix is used to easily unset all the variables with
|
|
# this prefix before leaving this file.
|
|
|
|
ciavts_main_work_dir="${1}"; shift
|
|
ciavts_work_dir="${1}"; shift
|
|
ciavts_arch="${1}"; shift
|
|
ciavts_vernum="${1}"; shift
|
|
ciavts_tapfile="${1}"; shift
|
|
|
|
# $@ now contains tests / test patterns to run
|
|
|
|
source ci-automation/ci_automation_common.sh
|
|
|
|
mkdir -p "${ciavts_work_dir}"
|
|
|
|
ciavts_testscript=$(basename "${0}")
|
|
ciavts_git_version=$(cat "${ciavts_main_work_dir}/git_version")
|
|
ciavts_channel=$(cat "${ciavts_main_work_dir}/git_channel")
|
|
if [[ "${ciavts_channel}" = 'developer' ]]; then
|
|
ciavts_channel='alpha'
|
|
fi
|
|
# If vernum is like 3200.0.0+whatever, it's a developer build,
|
|
# otherwise it's a release build.
|
|
ciavts_type='developer'
|
|
if [[ "${ciavts_vernum%%+*}" = "${ciavts_vernum}" ]]; then
|
|
ciavts_type='release'
|
|
fi
|
|
|
|
# Make these paths absolute to avoid problems when changing
|
|
# directories.
|
|
ciavts_tapfile="${PWD}/${ciavts_work_dir}/${ciavts_tapfile}"
|
|
|
|
ciavts_first_run=0
|
|
if [[ -f "${ciavts_main_work_dir}/first_run" ]]; then
|
|
ciavts_first_run=1
|
|
fi
|
|
|
|
echo "++++ Running ${ciavts_testscript} inside ${ciavts_work_dir} ++++"
|
|
|
|
cd "${ciavts_work_dir}"
|
|
|
|
CIA_VERNUM="${ciavts_vernum}"
|
|
CIA_ARCH="${ciavts_arch}"
|
|
CIA_TAPFILE="${ciavts_tapfile}"
|
|
CIA_CHANNEL="${ciavts_channel}"
|
|
CIA_TESTSCRIPT="${ciavts_testscript}"
|
|
CIA_GIT_VERSION="${ciavts_git_version}"
|
|
CIA_BUILD_TYPE="${ciavts_type}"
|
|
CIA_FIRST_RUN="${ciavts_first_run}"
|
|
|
|
# Unset all variables with ciavts_ prefix now.
|
|
unset -v "${!ciavts_@}"
|
|
|
|
# Prefixes all test names in the tap file with a given prefix, so the
|
|
# test name like "cl.basic" will become "extra-test.[${prefix}].cl.basic".
|
|
#
|
|
# Typical use:
|
|
# prefix_tap_file "${instance_type}" "${tapfile}"
|
|
#
|
|
# Parameters:
|
|
# 1 - prefix
|
|
# 2 - tap file, modified in place
|
|
function prefix_tap_file() {
|
|
local prefix="${1}"; shift
|
|
local tap_file="${1}"; shift
|
|
# drop the dots from prefix
|
|
local actual_prefix="extra-test.[${prefix}]."
|
|
|
|
sed --in-place --expression 's/^\(\s*\(not\)\?\s*ok[^-]*\s*-\s*\)\(\S\)/\1'"${actual_prefix}"'\3/g' "${tap_file}"
|
|
}
|
|
|
|
# Filters the test names, so it puts only the real names of the
|
|
# prefixed tests into the chosen variable. For example for prefix
|
|
# "foo", it will ignore the test name like "cl.basic", but will print
|
|
# "cl.internet" for a test name like "extra-test.[foo].cl.internet".
|
|
# "*" is treated specially - it will be inserted into the chosen
|
|
# variable if it is passed.
|
|
#
|
|
# Typical use:
|
|
# filter_prefixed_tests tests_to_run "${instance_type}" "${@}"
|
|
# if [[ "${#tests_to_run[@]}" -gt 0 ]]; then …; fi
|
|
#
|
|
# Parameters:
|
|
# 1 - name of an array variable where the filtering results will be stored
|
|
# 2 - prefix
|
|
# @ - test names
|
|
function filter_prefixed_tests() {
|
|
local var_name="${1}"; shift
|
|
local prefix="${1}"; shift
|
|
# rest of the parameters are test names
|
|
local -n results="${var_name}"
|
|
local name
|
|
local stripped_name
|
|
# clear the array, so it will contain results of current filtering
|
|
# only
|
|
results=()
|
|
for name; do
|
|
stripped_name="${name#extra-test.\[${prefix}\].}"
|
|
if [[ "${stripped_name}" != "${name}" ]]; then
|
|
results+=( "${stripped_name}" )
|
|
continue
|
|
elif [[ "${name}" = '*' ]]; then
|
|
results+=( '*' )
|
|
fi
|
|
done
|
|
}
|
|
|
|
# Filters out the extra tests from the passed test names. Ignored test
|
|
# names begin with "extra-test.". The results of the filtering are
|
|
# inserted into the chosen variable.
|
|
#
|
|
# Typical use:
|
|
# filter_out_prefixed_tests tests_to_run "${@}"
|
|
# if [[ "${#tests_to_run[@]}" -gt 0 ]]; then …; fi
|
|
#
|
|
# Parameters:
|
|
# 1 - name of an array variable where the filtering results will be stored
|
|
# @ - test names
|
|
function filter_out_prefixed_tests() {
|
|
local var_name="${1}"; shift
|
|
local -n results="${var_name}"
|
|
local name
|
|
# clear the array, so it will contain results of current filtering
|
|
# only
|
|
results=()
|
|
for name; do
|
|
if [[ "${name#extra-test.}" = "${name}" ]]; then
|
|
results+=( "${name}" )
|
|
fi
|
|
done
|
|
}
|
|
|
|
# Merges into the first (main) tap file the contents of other tap
|
|
# files. It is very simple - the function assumes that all the tap
|
|
# files begin with a line like:
|
|
#
|
|
# 1..${number_of_tests}
|
|
#
|
|
# Other lines that are processed should begin like:
|
|
#
|
|
# (not)? ok - ${test_name}
|
|
#
|
|
# Any other lines are copied verbatim.
|
|
#
|
|
# The other tap files should already be preprocessed by
|
|
# prefix_tap_file to avoid duplicated test names.
|
|
#
|
|
# Typical use:
|
|
# merge_tap_files "${tap_file}" extra-validation-*.tap
|
|
# rm -f extra-validation-*.tap
|
|
#
|
|
# Parameters:
|
|
# 1 - main tap file
|
|
# @ - other tap files
|
|
function merge_tap_files() {
|
|
local main_tap_file="${1}"; shift
|
|
# rest of the parameters are other tap files
|
|
|
|
local main_test_count=0
|
|
if [[ -f "${main_tap_file}" ]]; then
|
|
main_test_count=$(head --lines=1 "${main_tap_file}" | grep --only-matching '[0-9]\+$')
|
|
fi
|
|
local other_test_count
|
|
local other_tap_file
|
|
local tmp_tap_file="${main_tap_file}.mtf.tmp"
|
|
for other_tap_file; do
|
|
if [[ ! -f "${other_tap_file}" ]]; then
|
|
continue
|
|
fi
|
|
other_test_count=$(head --lines=1 "${other_tap_file}" | grep --only-matching '[0-9]\+$' || echo 0 )
|
|
((main_test_count+=other_test_count))
|
|
done
|
|
echo "1..${main_test_count}" >"${tmp_tap_file}"
|
|
if [[ -f "${main_tap_file}" ]]; then
|
|
tail --lines=+2 "${main_tap_file}" >>"${tmp_tap_file}"
|
|
fi
|
|
for other_tap_file; do
|
|
if [[ ! -f "${other_tap_file}" ]]; then
|
|
continue
|
|
fi
|
|
tail --lines=+2 "${other_tap_file}" >>"${tmp_tap_file}"
|
|
done
|
|
mv --force "${tmp_tap_file}" "${main_tap_file}"
|
|
}
|
|
|
|
# Runs or reruns the tests on the main instance and other
|
|
# instances. Other instances usually run a subset of tests only.
|
|
#
|
|
# For this function to work, the caller needs to define two functions
|
|
# beforehand:
|
|
#
|
|
# run_kola_tests that takes the following parameters:
|
|
# 1 - instance type
|
|
# 2 - tap file
|
|
# @ - tests to run
|
|
#
|
|
# query_kola_tests that takes the following parameters:
|
|
# 1 - instance type
|
|
# @ - tests to run
|
|
# This function should print the names of the tests to run. Every line
|
|
# of the output should have one test name to run. Any other cruft in
|
|
# the line will be ignored.
|
|
#
|
|
# Typical use:
|
|
# function run_kola_tests() {
|
|
# local instance_type="${1}"; shift
|
|
# local tap_file="${1}"; shift
|
|
# kola run … "${@}"
|
|
# }
|
|
#
|
|
# function query_kola_tests() {
|
|
# local instance_type="${1}"; shift
|
|
# kola list … "${@}"
|
|
# }
|
|
#
|
|
# args=(
|
|
# "${main_instance}"
|
|
# "${CIA_TAPFILE}"
|
|
# "${CIA_FIRST_RUN}"
|
|
# "${other_instance_types[@]}"
|
|
# '--'
|
|
# 'cl.internet'
|
|
# '--'
|
|
# "${tests_to_run[@]}"
|
|
# )
|
|
# run_kola_tests_on_instances "${args[@]}"
|
|
#
|
|
# Parameters:
|
|
# 1 - main instance type - there all the tests are being run
|
|
# 2 - main tap file
|
|
# 3 - if this is first run (1 if it is, 0 if it is a rerun)
|
|
# @ - other instance types followed by double dash (--) followed by
|
|
# test names for other instances to filter from the tests to be
|
|
# run followed by double dash, followed by tests to be run or
|
|
# rerun
|
|
function run_kola_tests_on_instances() {
|
|
local main_instance_type="${1}"; shift
|
|
local main_tapfile="${1}"; shift
|
|
local is_first_run="${1}"; shift
|
|
local other_instance_types=()
|
|
local other_tests=()
|
|
local arg
|
|
|
|
while [[ "${#}" -gt 0 ]]; do
|
|
arg="${1}"; shift
|
|
if [[ "${arg}" = '--' ]]; then
|
|
break
|
|
fi
|
|
other_instance_types+=( "${arg}" )
|
|
done
|
|
|
|
while [[ "${#}" -gt 0 ]]; do
|
|
arg="${1}"; shift
|
|
if [[ "${arg}" = '--' ]]; then
|
|
break
|
|
fi
|
|
other_tests+=( "${arg}" )
|
|
done
|
|
|
|
# rest of the parameters are tests to be run or rerun
|
|
|
|
local instance_type
|
|
local queried_tests
|
|
local instance_tests=()
|
|
local tests_on_instances_running=0
|
|
local other_tests_for_fgrep
|
|
other_tests_for_fgrep="$(printf '%s\n' "${other_tests[@]}")"
|
|
|
|
for instance_type in "${other_instance_types[@]}"; do
|
|
# On first run we usually pass the canonical test names like
|
|
# cl.basic, cl.internet or *, so we decide which tests should
|
|
# be run on the other instances based on this list. On the
|
|
# other hand, the rerun will contain names of the failed tests
|
|
# only, and those are specific - if a test failed on the main
|
|
# instance, the name of the test will be like cl.basic; if a
|
|
# test failed on other instance, the name of the test will be
|
|
# like extra-test.[…].cl.basic. So in case of reruns, we want
|
|
# to filter the extra tests first then we decide which tests
|
|
# should be run.
|
|
if [[ "${is_first_run}" -eq 1 ]]; then
|
|
set -o noglob # noglob should not be necessary, as
|
|
# query_kola_tests shouldn't return a
|
|
# wildcard, but better to be safe than sorry
|
|
queried_tests="$(query_kola_tests "${instance_type}" "${@}")"
|
|
instance_tests=( $(grep --only-matching --fixed-strings "${other_tests_for_fgrep}" <<<"${queried_tests}" || :) )
|
|
set +o noglob
|
|
else
|
|
filter_prefixed_tests instance_tests "${instance_type}" "${@}"
|
|
fi
|
|
if [[ "${#instance_tests[@]}" -gt 0 ]]; then
|
|
tests_on_instances_running=1
|
|
(
|
|
local instance_tapfile="instance_${instance_type}_validate.tap"
|
|
set +e
|
|
set -x
|
|
local output
|
|
output=$(run_kola_tests "${instance_type}" "${instance_tapfile}" "${instance_tests[@]}" 2>&1)
|
|
set +x
|
|
set -e
|
|
local escaped_instance_type
|
|
escaped_instance_type="$(sed -e 's/[\/&]/\\&/g' <<<"${instance_type}")"
|
|
printf "=== START ${instance_type} ===\n%s\n=== END ${instance_type} ===\n" "$(sed -e "s/^/${escaped_instance_type}: /g" <<<"${output}")"
|
|
prefix_tap_file "${instance_type}" "${instance_tapfile}"
|
|
) &
|
|
fi
|
|
done
|
|
|
|
local main_tests=()
|
|
|
|
filter_out_prefixed_tests main_tests "${@}"
|
|
if [[ "${#main_tests[@]}" -gt 0 ]]; then
|
|
# run in a subshell, so the set -x and set +e do not pollute
|
|
# the outer environment
|
|
(
|
|
set +e
|
|
set -x
|
|
run_kola_tests "${main_instance_type}" "${main_tapfile}" "${main_tests[@]}"
|
|
true
|
|
)
|
|
fi
|
|
|
|
if [[ "${tests_on_instances_running}" -eq 1 ]]; then
|
|
wait
|
|
merge_tap_files "${main_tapfile}" 'instance_'*'_validate.tap'
|
|
rm -f 'instance_'*'_validate.tap'
|
|
fi
|
|
}
|