From 1d6f38a72e475705269343bcc4b27c5aeabae8b1 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 22 Apr 2022 11:56:43 +0200 Subject: [PATCH] ci-automation: Reduce boilerplate in vendor tests Move the common setup to the vendor_test.sh script, which will be sourced by the vendor scripts. --- ci-automation/test.sh | 30 ++++-- ci-automation/vendor-testing/qemu.sh | 56 ++++------ ci-automation/vendor-testing/qemu_update.sh | 52 ++++------ ci-automation/vendor_test.sh | 107 ++++++++++++++++++++ 4 files changed, 169 insertions(+), 76 deletions(-) create mode 100644 ci-automation/vendor_test.sh diff --git a/ci-automation/test.sh b/ci-automation/test.sh index f51d62ce27..fa7c6c3720 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -60,12 +60,19 @@ # torcx tarball on the build cache (for the docker.torcx-manifest-pkgs test). # # Vendor specific scripts are called with the following positional arguments: -# 1 - working directory for the tests. +# 1 - Toplevel tests directory +# It contains some additional files needed for running the tests (like torcx manifest or file with channel information). +# 2 - Working directory for the tests. # The vendor script is expected to keep all artifacts it produces in that directory. -# 2 - Architecture to test. -# 3 - version number to test. -# 4 - output TAP file. +# 3 - Architecture to test. +# 4 - Version number to test. +# 5 - Output TAP file. # All following arguments specify test cases / test case patterns to run. +# +# The vendor tests should source ci-automation/vendor_test.sh script +# as a first step - it will do some common steps that the vendor +# script would need to make anyway. For more information, please refer +# to the vendor_test.sh file. set -euo pipefail @@ -107,6 +114,7 @@ function test_run() { source ci-automation/tapfile_helper_lib.sh source ci-automation/ci_automation_common.sh + source sdk_lib/sdk_container_common.sh init_submodules source sdk_container/.repo/manifests/version.txt @@ -118,6 +126,13 @@ function test_run() { local tests_dir="${work_dir}/${image}" mkdir -p "${tests_dir}" + # Store git version and git channel as files inside ${work_dir}. + # This information might not be available inside the docker + # container if this directory is not a main git repo, but rather a + # git worktree. + get_git_version >"${work_dir}/git_version" + get_git_channel >"${work_dir}/git_channel" + local container_name="flatcar-tests-${arch}-${docker_vernum}-${image}" local mantle_ref mantle_ref=$(cat sdk_container/.repo/manifests/mantle-container) @@ -137,17 +152,18 @@ function test_run() { # Ignore retcode since tests are flaky. We'll re-run failed tests and # determine success based on test results (tapfile). - set +e -o noglob + set +e touch sdk_container/.env docker run --pull always --rm --name="${container_name}" --privileged --net host -v /dev:/dev \ -w /work -v "$PWD":/work "${mantle_ref}" \ - bash -c "set -o noglob && source sdk_container/.env && ci-automation/vendor-testing/\"${image}\".sh \ + bash -c "set -o noglob && source sdk_container/.env && ci-automation/vendor-testing/${image}.sh \ + \"${work_dir}\" \ \"${tests_dir}\" \ \"${arch}\" \ \"${vernum}\" \ \"${tapfile}\" \ $@" - set -e +o noglob + set -e docker run --pull always --rm --name="${container_name}" --privileged --net host -v /dev:/dev \ -w /work -v "$PWD":/work "${mantle_ref}" \ diff --git a/ci-automation/vendor-testing/qemu.sh b/ci-automation/vendor-testing/qemu.sh index 568d7dc03b..4143609239 100755 --- a/ci-automation/vendor-testing/qemu.sh +++ b/ci-automation/vendor-testing/qemu.sh @@ -6,63 +6,49 @@ set -euo pipefail # Test execution script for the qemu vendor image. -# This script is supposed to run in the SDK container. +# This script is supposed to run in the mantle container. -work_dir="$1"; shift -arch="$1"; shift -vernum="$1"; shift -tapfile="$1"; shift - -# $@ now contains tests / test patterns to run - -source ci-automation/ci_automation_common.sh - -mkdir -p "${work_dir}" -cd "${work_dir}" - -testscript="$(basename "$0")" +source ci-automation/vendor_test.sh # ARM64 qemu tests only supported on UEFI -if [ "${arch}" = "arm64" ] && [ "${testscript}" != "qemu_uefi.sh" ] ; then - echo "1..1" > "${tapfile}" - echo "not ok - all qemu tests" >> "${tapfile}" - echo " ---" >> "${tapfile}" - echo " ERROR: ARM64 tests only supported on qemu_uefi." | tee -a "${tapfile}" - echo " ..." >> "${tapfile}" +if [ "${CIA_ARCH}" = "arm64" ] && [ "${CIA_TESTSCRIPT}" != "qemu_uefi.sh" ] ; then + echo "1..1" > "${CIA_TAPFILE}" + echo "not ok - all qemu tests" >> "${CIA_TAPFILE}" + echo " ---" >> "${CIA_TAPFILE}" + echo " ERROR: ARM64 tests only supported on qemu_uefi." | tee -a "${CIA_TAPFILE}" + echo " ..." >> "${CIA_TAPFILE}" exit 1 fi # Fetch image and BIOS if not present if [ -f "${QEMU_IMAGE_NAME}" ] ; then - echo "++++ ${testscript}: Using existing ${work_dir}/${QEMU_IMAGE_NAME} for testing ${vernum} (${arch}) ++++" + echo "++++ ${CIA_TESTSCRIPT}: Using existing ./${QEMU_IMAGE_NAME} for testing ${CIA_VERNUM} (${CIA_ARCH}) ++++" else - echo "++++ ${testscript}: downloading ${QEMU_IMAGE_NAME} for ${vernum} (${arch}) ++++" - copy_from_buildcache "images/${arch}/${vernum}/${QEMU_IMAGE_NAME}" . + echo "++++ ${CIA_TESTSCRIPT}: downloading ${QEMU_IMAGE_NAME} for ${CIA_VERNUM} (${CIA_ARCH}) ++++" + copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/${QEMU_IMAGE_NAME}" . fi bios="${QEMU_BIOS}" -if [ "${testscript}" = "qemu_uefi.sh" ] ; then +if [ "${CIA_TESTSCRIPT}" = "qemu_uefi.sh" ] ; then bios="${QEMU_UEFI_BIOS}" if [ -f "${bios}" ] ; then - echo "++++ ${testscript}: Using existing ${work_dir}/${bios} ++++" + echo "++++ ${CIA_TESTSCRIPT}: Using existing ./${bios} ++++" else - echo "++++ ${testscript}: downloading ${bios} for ${vernum} (${arch}) ++++" - copy_from_buildcache "images/${arch}/${vernum}/${bios}" . + echo "++++ ${CIA_TESTSCRIPT}: downloading ${bios} for ${CIA_VERNUM} (${CIA_ARCH}) ++++" + copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/${bios}" . fi fi set -x -set -o noglob -sudo kola run \ - --board="${arch}-usr" \ +kola run \ + --board="${CIA_ARCH}-usr" \ --parallel="${QEMU_PARALLEL}" \ --platform=qemu \ - --qemu-bios=${bios} \ + --qemu-bios="${bios}" \ --qemu-image="${QEMU_IMAGE_NAME}" \ - --tapfile="${tapfile}" \ - --torcx-manifest=../torcx_manifest.json \ - $@ + --tapfile="${CIA_TAPFILE}" \ + --torcx-manifest="${CIA_TORCX_MANIFEST}" \ + "${@}" -set +o noglob set +x diff --git a/ci-automation/vendor-testing/qemu_update.sh b/ci-automation/vendor-testing/qemu_update.sh index 2ecc26e992..a21df0c822 100755 --- a/ci-automation/vendor-testing/qemu_update.sh +++ b/ci-automation/vendor-testing/qemu_update.sh @@ -8,67 +8,51 @@ set -euo pipefail # Test execution script for the update payload using the previous # release as starting point, and doing a second update from the current # build to itself again. -# This script is supposed to run in the SDK container. +# This script is supposed to run in the mantle container. -work_dir="$1"; shift -arch="$1"; shift -vernum="$1"; shift -tapfile="$1"; shift +source ci-automation/vendor_test.sh -# $@ now contains tests / test patterns to run - -if [ "$@" != "" ] && [ "$@" != "*" ] && [ "$@" != "cl.update.payload" ]; then - echo "Only cl.update.payload is supported, got '$@'" +if [ "$*" != "" ] && [ "$*" != "*" ] && [ "$*" != "cl.update.payload" ]; then + echo "Only cl.update.payload is supported, got '$*'" exit 1 fi -source ci-automation/ci_automation_common.sh -source sdk_lib/sdk_container_common.sh - -mkdir -p "${work_dir}" -cd "${work_dir}" - mkdir -p tmp/ if [ -f tmp/flatcar_test_update.gz ] ; then - echo "++++ QEMU test: Using existing ${work_dir}/tmp/flatcar_test_update.gz for testing ${vernum} (${arch}) ++++" + echo "++++ ${CIA_TESTSCRIPT}: Using existing ./tmp/flatcar_test_update.gz for testing ${CIA_VERNUM} (${CIA_ARCH}) ++++" else - echo "++++ QEMU test: downloading flatcar_test_update.gz for ${vernum} (${arch}) ++++" - copy_from_buildcache "images/${arch}/${vernum}/flatcar_test_update.gz" tmp/ + echo "++++ ${CIA_TESTSCRIPT}: downloading flatcar_test_update.gz for ${CIA_VERNUM} (${CIA_ARCH}) ++++" + copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/flatcar_test_update.gz" tmp/ fi -ON_CHANNEL="$(get_git_channel)" -if [ "${ON_CHANNEL}" = "developer" ]; then - # For main/dev builds we compare to last alpha release - ON_CHANNEL="alpha" -fi if [ -f tmp/flatcar_production_image_previous.bin ] ; then - echo "++++ QEMU test: Using existing ${work_dir}/tmp/flatcar_production_image_previous.bin for testing update to ${vernum} (${arch}) from previous ${ON_CHANNEL} ++++" + echo "++++ ${CIA_TESTSCRIPT}: Using existing ./tmp/flatcar_production_image_previous.bin for testing update to ${CIA_VERNUM} (${CIA_ARCH}) from previous ${CIA_CHANNEL} ++++" else - echo "++++ QEMU test: downloading flatcar_production_image_previous.bin from previous ${ON_CHANNEL} ++++" + echo "++++ ${CIA_TESTSCRIPT}: downloading flatcar_production_image_previous.bin from previous ${CIA_CHANNEL} ++++" rm -f tmp/flatcar_production_image_previous.bin.bz2 - curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 "https://${ON_CHANNEL}.release.flatcar-linux.net/${arch}-usr/current/flatcar_production_image.bin.bz2" + curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 "https://${CIA_CHANNEL}.release.flatcar-linux.net/${CIA_ARCH}-usr/current/flatcar_production_image.bin.bz2" mv flatcar_production_image.bin.bz2 tmp/flatcar_production_image_previous.bin.bz2 lbunzip2 -k -f tmp/flatcar_production_image_previous.bin.bz2 fi bios="${QEMU_BIOS}" -if [ "${arch}" = "arm64" ]; then +if [ "${CIA_ARCH}" = "arm64" ]; then bios="${QEMU_UEFI_BIOS}" if [ -f "${bios}" ] ; then - echo "++++ qemu_update.sh: Using existing ${work_dir}/${bios} ++++" + echo "++++ qemu_update.sh: Using existing ./${bios} ++++" else - echo "++++ qemu_update.sh: downloading ${bios} for ${vernum} (${arch}) ++++" - copy_from_buildcache "images/${arch}/${vernum}/${bios}" . + echo "++++ qemu_update.sh: downloading ${bios} for ${CIA_VERNUM} (${CIA_ARCH}) ++++" + copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/${bios}" . fi fi -sudo kola run \ - --board="${arch}-usr" \ +kola run \ + --board="${CIA_ARCH}-usr" \ --parallel="${QEMU_PARALLEL}" \ --platform=qemu \ --qemu-bios="${bios}" \ --qemu-image=tmp/flatcar_production_image_previous.bin \ - --tapfile="${tapfile}" \ - --torcx-manifest=../torcx_manifest.json \ + --tapfile="${CIA_TAPFILE}" \ + --torcx-manifest="${CIA_TORCX_MANIFEST}" \ --update-payload=tmp/flatcar_test_update.gz \ cl.update.payload diff --git a/ci-automation/vendor_test.sh b/ci-automation/vendor_test.sh new file mode 100644 index 0000000000..0aac38a641 --- /dev/null +++ b/ci-automation/vendor_test.sh @@ -0,0 +1,107 @@ +# Copyright (c) 2021 The Flatcar Maintainers. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Vendor test helper script. Sourced by vendor tests. Does some +# initial setup. +# +# +# The initial setup consist of creating the vendor working directory +# for the vendor test script, specifying the variables described below +# and changing the current working directory to the vendor working +# directory. +# +# +# The vendor test script is expected to keep all artifacts it produces +# in its current working directory. +# +# +# The script specifies the following variables for the vendor test +# script to use: +# +# CIA_VERNUM: +# Image version. In case of developer builds it comes with a suffix, +# so it looks like "3217.0.0+nightly-20220422-0155". For release +# builds the version will be without suffix, so it looks like +# "3217.0.0". Whether the build is a release or a developer one is +# reflected in CIA_BUILD_TYPE variable described below. +# +# CIA_ARCH: +# Architecture to test. Currently it is either "amd64" or "arm64". +# +# CIA_TAPFILE: +# Where the TAP reports should be written. Usually just passed to +# kola throught the --tapfile parameter. +# +# CIA_CHANNEL: +# A channel. Either "alpha", "beta", "stable" or "lts". Used to find +# the last release for the update check. +# +# CIA_TESTSCRIPT: +# Name of the vendor script. May be useful in some messages. +# +# CIA_GIT_VERSION: +# The most recent tag for the current commit. +# +# CIA_BUILD_TYPE: +# It's either "release" or "developer", based on the CIA_VERNUM +# variable. +# +# CIA_TORCX_MANIFEST: +# Path to the Torcx manifest. Usually passed to kola through the +# --torcx-manifest parameter. +# +# +# After this script is sourced, the parameters in ${@} specify test +# cases / test case patterns to run. + + +# "ciavts" stands for Continuous Integration Automation Vendor Test +# Setup. This prefix is used to easily unset all the variables with +# this prefix before leaving this file. + +ciavts_main_work_dir="${1}"; shift +ciavts_work_dir="${1}"; shift +ciavts_arch="${1}"; shift +ciavts_vernum="${1}"; shift +ciavts_tapfile="${1}"; shift + +# $@ now contains tests / test patterns to run + +source ci-automation/ci_automation_common.sh + +mkdir -p "${ciavts_work_dir}" + +ciavts_testscript=$(basename "${0}") +ciavts_git_version=$(cat "${ciavts_main_work_dir}/git_version") +ciavts_channel=$(cat "${ciavts_main_work_dir}/git_channel") +if [[ "${ciavts_channel}" = 'developer' ]]; then + ciavts_channel='alpha' +fi +# If vernum is like 3200.0.0+whatever, it's a developer build, +# otherwise it's a release build. +ciavts_type='developer' +if [[ "${ciavts_vernum%%+*}" = "${ciavts_vernum}" ]]; then + ciavts_type='release' +fi + +# Make these paths absolute to avoid problems when changing +# directories. +ciavts_tapfile="${PWD}/${ciavts_work_dir}/${ciavts_tapfile}" +ciavts_torcx_manifest="${PWD}/${ciavts_main_work_dir}/torcx_manifest.json" + +echo "++++ Running ${ciavts_testscript} inside ${ciavts_work_dir} ++++" + +cd "${ciavts_work_dir}" + +CIA_VERNUM="${ciavts_vernum}" +CIA_ARCH="${ciavts_arch}" +CIA_TAPFILE="${ciavts_tapfile}" +CIA_CHANNEL="${ciavts_channel}" +CIA_TESTSCRIPT="${ciavts_testscript}" +CIA_GIT_VERSION="${ciavts_git_version}" +CIA_BUILD_TYPE="${ciavts_type}" +CIA_TORCX_MANIFEST="${ciavts_torcx_manifest}" + +# Unset all variables with ciavts_ prefix now. +unset -v "${!ciavts_@}"