Merge pull request #293 from flatcar-linux/krnowak/ciauto-boilerplate

ci-automation: Reduce boilerplate in vendor tests
This commit is contained in:
Krzesimir Nowak 2022-05-06 08:54:46 +02:00 committed by GitHub
commit 4adf8df751
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 197 additions and 135 deletions

View File

@ -60,12 +60,19 @@
# torcx tarball on the build cache (for the docker.torcx-manifest-pkgs test).
#
# Vendor specific scripts are called with the following positional arguments:
# 1 - working directory for the tests.
# 1 - Toplevel tests directory
# It contains some additional files needed for running the tests (like torcx manifest or file with channel information).
# 2 - Working directory for the tests.
# The vendor script is expected to keep all artifacts it produces in that directory.
# 2 - Architecture to test.
# 3 - version number to test.
# 4 - output TAP file.
# 3 - Architecture to test.
# 4 - Version number to test.
# 5 - Output TAP file.
# All following arguments specify test cases / test case patterns to run.
#
# The vendor tests should source ci-automation/vendor_test.sh script
# as a first step - it will do some common steps that the vendor
# script would need to make anyway. For more information, please refer
# to the vendor_test.sh file.
set -euo pipefail
@ -107,6 +114,7 @@ function test_run() {
source ci-automation/tapfile_helper_lib.sh
source ci-automation/ci_automation_common.sh
source sdk_lib/sdk_container_common.sh
init_submodules
source sdk_container/.repo/manifests/version.txt
@ -118,6 +126,13 @@ function test_run() {
local tests_dir="${work_dir}/${image}"
mkdir -p "${tests_dir}"
# Store git version and git channel as files inside ${work_dir}.
# This information might not be available inside the docker
# container if this directory is not a main git repo, but rather a
# git worktree.
get_git_version >"${work_dir}/git_version"
get_git_channel >"${work_dir}/git_channel"
local container_name="flatcar-tests-${arch}-${docker_vernum}-${image}"
local mantle_ref
mantle_ref=$(cat sdk_container/.repo/manifests/mantle-container)
@ -137,17 +152,18 @@ function test_run() {
# Ignore retcode since tests are flaky. We'll re-run failed tests and
# determine success based on test results (tapfile).
set +e -o noglob
set +e
touch sdk_container/.env
docker run --pull always --rm --name="${container_name}" --privileged --net host -v /dev:/dev \
-w /work -v "$PWD":/work "${mantle_ref}" \
bash -c "set -o noglob && source sdk_container/.env && ci-automation/vendor-testing/\"${image}\".sh \
bash -c "set -o noglob && source sdk_container/.env && ci-automation/vendor-testing/${image}.sh \
\"${work_dir}\" \
\"${tests_dir}\" \
\"${arch}\" \
\"${vernum}\" \
\"${tapfile}\" \
$@"
set -e +o noglob
set -e
docker run --pull always --rm --name="${container_name}" --privileged --net host -v /dev:/dev \
-w /work -v "$PWD":/work "${mantle_ref}" \

View File

@ -6,38 +6,24 @@
set -euo pipefail
# Test execution script for the Equinix Metal vendor image.
# This script is supposed to run in the SDK container.
# This script is supposed to run in the mantle container.
# This script requires "pxe" Jenkins job.
work_dir="$1"; shift
arch="$1"; shift
vernum="$1"; shift
tapfile="$1"; shift
# $@ now contains tests / test patterns to run
source ci-automation/ci_automation_common.sh
# required for get_git_channel
source sdk_lib/sdk_container_common.sh
channel="$(get_git_channel)"
mkdir -p "${work_dir}"
cd "${work_dir}"
source ci-automation/vendor_test.sh
# Equinix Metal ARM server are not yet hourly available in the default `SV` metro
equinixmetal_metro_var="EQUINIXMETAL_${arch}_METRO"
equinixmetal_metro_var="EQUINIXMETAL_${CIA_ARCH}_METRO"
equinixmetal_metro="${!equinixmetal_metro_var}"
EQUINIXMETAL_INSTANCE_TYPE_VAR="EQUINIXMETAL_${arch}_INSTANCE_TYPE"
EQUINIXMETAL_INSTANCE_TYPE_VAR="EQUINIXMETAL_${CIA_ARCH}_INSTANCE_TYPE"
EQUINIXMETAL_INSTANCE_TYPE="${!EQUINIXMETAL_INSTANCE_TYPE_VAR}"
MORE_INSTANCE_TYPES_VAR="EQUINIXMETAL_${arch}_MORE_INSTANCE_TYPES"
MORE_INSTANCE_TYPES_VAR="EQUINIXMETAL_${CIA_ARCH}_MORE_INSTANCE_TYPES"
MORE_INSTANCE_TYPES=( ${!MORE_INSTANCE_TYPES_VAR} )
# The maximum is 6h coming from the ore GC duration parameter
timeout=6h
BASE_URL="http://${BUILDCACHE_SERVER}/images/${arch}/${vernum}"
BASE_URL="http://${BUILDCACHE_SERVER}/images/${CIA_ARCH}/${CIA_VERNUM}"
run_equinix_metal_kola_test() {
local instance_type="${1}"
@ -45,12 +31,12 @@ run_equinix_metal_kola_test() {
timeout --signal=SIGQUIT "${timeout}" \
kola run \
--board="${arch}-usr" \
--basename="ci-${vernum/+/-}" \
--board="${CIA_ARCH}-usr" \
--basename="ci-${CIA_VERNUM/+/-}" \
--platform=equinixmetal \
--tapfile="${instance_tapfile}" \
--parallel="${EQUINIXMETAL_PARALLEL}" \
--torcx-manifest=../torcx_manifest.json \
--torcx-manifest="${CIA_TORCX_MANIFEST}" \
--equinixmetal-image-url="${BASE_URL}/${EQUINIXMETAL_IMAGE_NAME}" \
--equinixmetal-installer-image-kernel-url="${BASE_URL}/${PXE_KERNEL_NAME}" \
--equinixmetal-installer-image-cpio-url="${BASE_URL}/${PXE_IMAGE_NAME}" \
@ -109,11 +95,11 @@ fi
ARGS="$*"
if [[ -n "${ARGS// }" ]]; then
set -x
run_equinix_metal_kola_test "${EQUINIXMETAL_INSTANCE_TYPE}" "${tapfile}" "${@}"
run_equinix_metal_kola_test "${EQUINIXMETAL_INSTANCE_TYPE}" "${CIA_TAPFILE}" "${@}"
set +x
fi
if [[ "${run_more_tests}" -eq 1 ]]; then
wait
cat validate_*.tap >>"${tapfile}"
cat validate_*.tap >>"${CIA_TAPFILE}"
fi

View File

@ -8,60 +8,43 @@ set -euo pipefail
# Test execution script for the GCE vendor image.
# This script is supposed to run in the mantle container.
work_dir="$1"; shift
arch="$1"; shift
vernum="$1"; shift
tapfile="$1"; shift
# $@ now contains tests / test patterns to run
source ci-automation/ci_automation_common.sh
source sdk_lib/sdk_container_common.sh
mkdir -p "${work_dir}"
cd "${work_dir}"
source ci-automation/vendor_test.sh
# We never run GCE on arm64, so for now fail it as an
# unsupported option.
if [[ "${arch}" == "arm64" ]]; then
echo "1..1" > "${tapfile}"
echo "not ok - all GCE tests" >> "${tapfile}"
echo " ---" >> "${tapfile}"
echo " ERROR: ARM64 tests not supported on GCE." | tee -a "${tapfile}"
echo " ..." >> "${tapfile}"
if [[ "${CIA_ARCH}" == "arm64" ]]; then
echo "1..1" > "${CIA_TAPFILE}"
echo "not ok - all GCE tests" >> "${CIA_TAPFILE}"
echo " ---" >> "${CIA_TAPFILE}"
echo " ERROR: ARM64 tests not supported on GCE." | tee -a "${CIA_TAPFILE}"
echo " ..." >> "${CIA_TAPFILE}"
exit 1
fi
channel="$(get_git_channel)"
if [[ "${channel}" = 'developer' ]]; then
channel='alpha'
fi
testscript="$(basename "$0")"
# Create temp file and delete it immediately
echo "${GCP_JSON_KEY}" | base64 --decode > /tmp/gcp_auth
exec {gcp_auth}</tmp/gcp_auth
rm /tmp/gcp_auth
GCP_JSON_KEY_PATH="/proc/$$/fd/${gcp_auth}"
copy_from_buildcache "images/${arch}/${vernum}/${GCE_IMAGE_NAME}" .
copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/${GCE_IMAGE_NAME}" .
gcloud auth activate-service-account --key-file "${GCP_JSON_KEY_PATH}"
gsutil rm -r "${GCE_GCS_IMAGE_UPLOAD}/${arch}-usr/${vernum}" || true
gsutil cp "${GCE_IMAGE_NAME}" "${GCE_GCS_IMAGE_UPLOAD}/${arch}-usr/${vernum}/${GCE_IMAGE_NAME}"
gsutil rm -r "${GCE_GCS_IMAGE_UPLOAD}/${CIA_ARCH}-usr/${CIA_VERNUM}" || true
gsutil cp "${GCE_IMAGE_NAME}" "${GCE_GCS_IMAGE_UPLOAD}/${CIA_ARCH}-usr/${CIA_VERNUM}/${GCE_IMAGE_NAME}"
family="ci"
image_name="${family}-${vernum//[+.]/-}"
image_name="${family}-${CIA_VERNUM//[+.]/-}"
ore gcloud delete-images --json-key="${GCP_JSON_KEY_PATH}" "${image_name}" || true
ore gcloud create-image \
--board="${arch}-usr" \
--board="${CIA_ARCH}-usr" \
--family="${family}" \
--json-key="${GCP_JSON_KEY_PATH}" \
--source-root="${GCE_GCS_IMAGE_UPLOAD}" \
--source-name="${GCE_IMAGE_NAME}" \
--version="${vernum}"
--version="${CIA_VERNUM}"
trap 'ore gcloud delete-images \
--json-key="${GCP_JSON_KEY_PATH}" \
"${image_name}" ; gsutil rm -r "${GCE_GCS_IMAGE_UPLOAD}/${arch}-usr/${vernum}" || true' EXIT
"${image_name}" ; gsutil rm -r "${GCE_GCS_IMAGE_UPLOAD}/${CIA_ARCH}-usr/${CIA_VERNUM}" || true' EXIT
set -x
@ -73,9 +56,9 @@ timeout --signal=SIGQUIT 6h \
--gce-machinetype="${GCE_MACHINE_TYPE}" \
--parallel="${GCE_PARALLEL}" \
--platform=gce \
--channel="${channel}" \
--tapfile="${tapfile}" \
--torcx-manifest='../torcx_manifest.json' \
--channel="${CIA_CHANNEL}" \
--tapfile="${CIA_TAPFILE}" \
--torcx-manifest="${CIA_TORCX_MANIFEST}" \
"${@}"
set +x

View File

@ -6,63 +6,49 @@
set -euo pipefail
# Test execution script for the qemu vendor image.
# This script is supposed to run in the SDK container.
# This script is supposed to run in the mantle container.
work_dir="$1"; shift
arch="$1"; shift
vernum="$1"; shift
tapfile="$1"; shift
# $@ now contains tests / test patterns to run
source ci-automation/ci_automation_common.sh
mkdir -p "${work_dir}"
cd "${work_dir}"
testscript="$(basename "$0")"
source ci-automation/vendor_test.sh
# ARM64 qemu tests only supported on UEFI
if [ "${arch}" = "arm64" ] && [ "${testscript}" != "qemu_uefi.sh" ] ; then
echo "1..1" > "${tapfile}"
echo "not ok - all qemu tests" >> "${tapfile}"
echo " ---" >> "${tapfile}"
echo " ERROR: ARM64 tests only supported on qemu_uefi." | tee -a "${tapfile}"
echo " ..." >> "${tapfile}"
if [ "${CIA_ARCH}" = "arm64" ] && [ "${CIA_TESTSCRIPT}" != "qemu_uefi.sh" ] ; then
echo "1..1" > "${CIA_TAPFILE}"
echo "not ok - all qemu tests" >> "${CIA_TAPFILE}"
echo " ---" >> "${CIA_TAPFILE}"
echo " ERROR: ARM64 tests only supported on qemu_uefi." | tee -a "${CIA_TAPFILE}"
echo " ..." >> "${CIA_TAPFILE}"
exit 1
fi
# Fetch image and BIOS if not present
if [ -f "${QEMU_IMAGE_NAME}" ] ; then
echo "++++ ${testscript}: Using existing ${work_dir}/${QEMU_IMAGE_NAME} for testing ${vernum} (${arch}) ++++"
echo "++++ ${CIA_TESTSCRIPT}: Using existing ./${QEMU_IMAGE_NAME} for testing ${CIA_VERNUM} (${CIA_ARCH}) ++++"
else
echo "++++ ${testscript}: downloading ${QEMU_IMAGE_NAME} for ${vernum} (${arch}) ++++"
copy_from_buildcache "images/${arch}/${vernum}/${QEMU_IMAGE_NAME}" .
echo "++++ ${CIA_TESTSCRIPT}: downloading ${QEMU_IMAGE_NAME} for ${CIA_VERNUM} (${CIA_ARCH}) ++++"
copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/${QEMU_IMAGE_NAME}" .
fi
bios="${QEMU_BIOS}"
if [ "${testscript}" = "qemu_uefi.sh" ] ; then
if [ "${CIA_TESTSCRIPT}" = "qemu_uefi.sh" ] ; then
bios="${QEMU_UEFI_BIOS}"
if [ -f "${bios}" ] ; then
echo "++++ ${testscript}: Using existing ${work_dir}/${bios} ++++"
echo "++++ ${CIA_TESTSCRIPT}: Using existing ./${bios} ++++"
else
echo "++++ ${testscript}: downloading ${bios} for ${vernum} (${arch}) ++++"
copy_from_buildcache "images/${arch}/${vernum}/${bios}" .
echo "++++ ${CIA_TESTSCRIPT}: downloading ${bios} for ${CIA_VERNUM} (${CIA_ARCH}) ++++"
copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/${bios}" .
fi
fi
set -x
set -o noglob
sudo kola run \
--board="${arch}-usr" \
kola run \
--board="${CIA_ARCH}-usr" \
--parallel="${QEMU_PARALLEL}" \
--platform=qemu \
--qemu-bios=${bios} \
--qemu-bios="${bios}" \
--qemu-image="${QEMU_IMAGE_NAME}" \
--tapfile="${tapfile}" \
--torcx-manifest=../torcx_manifest.json \
$@
--tapfile="${CIA_TAPFILE}" \
--torcx-manifest="${CIA_TORCX_MANIFEST}" \
"${@}"
set +o noglob
set +x

View File

@ -8,67 +8,51 @@ set -euo pipefail
# Test execution script for the update payload using the previous
# release as starting point, and doing a second update from the current
# build to itself again.
# This script is supposed to run in the SDK container.
# This script is supposed to run in the mantle container.
work_dir="$1"; shift
arch="$1"; shift
vernum="$1"; shift
tapfile="$1"; shift
source ci-automation/vendor_test.sh
# $@ now contains tests / test patterns to run
if [ "$@" != "" ] && [ "$@" != "*" ] && [ "$@" != "cl.update.payload" ]; then
echo "Only cl.update.payload is supported, got '$@'"
if [ "$*" != "" ] && [ "$*" != "*" ] && [ "$*" != "cl.update.payload" ]; then
echo "Only cl.update.payload is supported, got '$*'"
exit 1
fi
source ci-automation/ci_automation_common.sh
source sdk_lib/sdk_container_common.sh
mkdir -p "${work_dir}"
cd "${work_dir}"
mkdir -p tmp/
if [ -f tmp/flatcar_test_update.gz ] ; then
echo "++++ QEMU test: Using existing ${work_dir}/tmp/flatcar_test_update.gz for testing ${vernum} (${arch}) ++++"
echo "++++ ${CIA_TESTSCRIPT}: Using existing ./tmp/flatcar_test_update.gz for testing ${CIA_VERNUM} (${CIA_ARCH}) ++++"
else
echo "++++ QEMU test: downloading flatcar_test_update.gz for ${vernum} (${arch}) ++++"
copy_from_buildcache "images/${arch}/${vernum}/flatcar_test_update.gz" tmp/
echo "++++ ${CIA_TESTSCRIPT}: downloading flatcar_test_update.gz for ${CIA_VERNUM} (${CIA_ARCH}) ++++"
copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/flatcar_test_update.gz" tmp/
fi
ON_CHANNEL="$(get_git_channel)"
if [ "${ON_CHANNEL}" = "developer" ]; then
# For main/dev builds we compare to last alpha release
ON_CHANNEL="alpha"
fi
if [ -f tmp/flatcar_production_image_previous.bin ] ; then
echo "++++ QEMU test: Using existing ${work_dir}/tmp/flatcar_production_image_previous.bin for testing update to ${vernum} (${arch}) from previous ${ON_CHANNEL} ++++"
echo "++++ ${CIA_TESTSCRIPT}: Using existing ./tmp/flatcar_production_image_previous.bin for testing update to ${CIA_VERNUM} (${CIA_ARCH}) from previous ${CIA_CHANNEL} ++++"
else
echo "++++ QEMU test: downloading flatcar_production_image_previous.bin from previous ${ON_CHANNEL} ++++"
echo "++++ ${CIA_TESTSCRIPT}: downloading flatcar_production_image_previous.bin from previous ${CIA_CHANNEL} ++++"
rm -f tmp/flatcar_production_image_previous.bin.bz2
curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 "https://${ON_CHANNEL}.release.flatcar-linux.net/${arch}-usr/current/flatcar_production_image.bin.bz2"
curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 "https://${CIA_CHANNEL}.release.flatcar-linux.net/${CIA_ARCH}-usr/current/flatcar_production_image.bin.bz2"
mv flatcar_production_image.bin.bz2 tmp/flatcar_production_image_previous.bin.bz2
lbunzip2 -k -f tmp/flatcar_production_image_previous.bin.bz2
fi
bios="${QEMU_BIOS}"
if [ "${arch}" = "arm64" ]; then
if [ "${CIA_ARCH}" = "arm64" ]; then
bios="${QEMU_UEFI_BIOS}"
if [ -f "${bios}" ] ; then
echo "++++ qemu_update.sh: Using existing ${work_dir}/${bios} ++++"
echo "++++ qemu_update.sh: Using existing ./${bios} ++++"
else
echo "++++ qemu_update.sh: downloading ${bios} for ${vernum} (${arch}) ++++"
copy_from_buildcache "images/${arch}/${vernum}/${bios}" .
echo "++++ qemu_update.sh: downloading ${bios} for ${CIA_VERNUM} (${CIA_ARCH}) ++++"
copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/${bios}" .
fi
fi
sudo kola run \
--board="${arch}-usr" \
kola run \
--board="${CIA_ARCH}-usr" \
--parallel="${QEMU_PARALLEL}" \
--platform=qemu \
--qemu-bios="${bios}" \
--qemu-image=tmp/flatcar_production_image_previous.bin \
--tapfile="${tapfile}" \
--torcx-manifest=../torcx_manifest.json \
--tapfile="${CIA_TAPFILE}" \
--torcx-manifest="${CIA_TORCX_MANIFEST}" \
--update-payload=tmp/flatcar_test_update.gz \
cl.update.payload

View File

@ -0,0 +1,107 @@
# Copyright (c) 2021 The Flatcar Maintainers.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Vendor test helper script. Sourced by vendor tests. Does some
# initial setup.
#
#
# The initial setup consist of creating the vendor working directory
# for the vendor test script, specifying the variables described below
# and changing the current working directory to the vendor working
# directory.
#
#
# The vendor test script is expected to keep all artifacts it produces
# in its current working directory.
#
#
# The script specifies the following variables for the vendor test
# script to use:
#
# CIA_VERNUM:
# Image version. In case of developer builds it comes with a suffix,
# so it looks like "3217.0.0+nightly-20220422-0155". For release
# builds the version will be without suffix, so it looks like
# "3217.0.0". Whether the build is a release or a developer one is
# reflected in CIA_BUILD_TYPE variable described below.
#
# CIA_ARCH:
# Architecture to test. Currently it is either "amd64" or "arm64".
#
# CIA_TAPFILE:
# Where the TAP reports should be written. Usually just passed to
# kola throught the --tapfile parameter.
#
# CIA_CHANNEL:
# A channel. Either "alpha", "beta", "stable" or "lts". Used to find
# the last release for the update check.
#
# CIA_TESTSCRIPT:
# Name of the vendor script. May be useful in some messages.
#
# CIA_GIT_VERSION:
# The most recent tag for the current commit.
#
# CIA_BUILD_TYPE:
# It's either "release" or "developer", based on the CIA_VERNUM
# variable.
#
# CIA_TORCX_MANIFEST:
# Path to the Torcx manifest. Usually passed to kola through the
# --torcx-manifest parameter.
#
#
# After this script is sourced, the parameters in ${@} specify test
# cases / test case patterns to run.
# "ciavts" stands for Continuous Integration Automation Vendor Test
# Setup. This prefix is used to easily unset all the variables with
# this prefix before leaving this file.
ciavts_main_work_dir="${1}"; shift
ciavts_work_dir="${1}"; shift
ciavts_arch="${1}"; shift
ciavts_vernum="${1}"; shift
ciavts_tapfile="${1}"; shift
# $@ now contains tests / test patterns to run
source ci-automation/ci_automation_common.sh
mkdir -p "${ciavts_work_dir}"
ciavts_testscript=$(basename "${0}")
ciavts_git_version=$(cat "${ciavts_main_work_dir}/git_version")
ciavts_channel=$(cat "${ciavts_main_work_dir}/git_channel")
if [[ "${ciavts_channel}" = 'developer' ]]; then
ciavts_channel='alpha'
fi
# If vernum is like 3200.0.0+whatever, it's a developer build,
# otherwise it's a release build.
ciavts_type='developer'
if [[ "${ciavts_vernum%%+*}" = "${ciavts_vernum}" ]]; then
ciavts_type='release'
fi
# Make these paths absolute to avoid problems when changing
# directories.
ciavts_tapfile="${PWD}/${ciavts_work_dir}/${ciavts_tapfile}"
ciavts_torcx_manifest="${PWD}/${ciavts_main_work_dir}/torcx_manifest.json"
echo "++++ Running ${ciavts_testscript} inside ${ciavts_work_dir} ++++"
cd "${ciavts_work_dir}"
CIA_VERNUM="${ciavts_vernum}"
CIA_ARCH="${ciavts_arch}"
CIA_TAPFILE="${ciavts_tapfile}"
CIA_CHANNEL="${ciavts_channel}"
CIA_TESTSCRIPT="${ciavts_testscript}"
CIA_GIT_VERSION="${ciavts_git_version}"
CIA_BUILD_TYPE="${ciavts_type}"
CIA_TORCX_MANIFEST="${ciavts_torcx_manifest}"
# Unset all variables with ciavts_ prefix now.
unset -v "${!ciavts_@}"