From 0fa985b8721855e1b7b4dc651b3e6f121488edbb Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Mon, 21 Feb 2022 16:30:30 +0100 Subject: [PATCH] ci-automation/test.sh: stage torcx manifest Signed-off-by: Thilo Fromm --- ci-automation/packages.sh | 11 ++++-- ci-automation/test.sh | 51 +++++++++++++++++++++++++--- ci-automation/vendor-testing/qemu.sh | 10 ++++-- 3 files changed, 62 insertions(+), 10 deletions(-) diff --git a/ci-automation/packages.sh b/ci-automation/packages.sh index 9e5efe743c..5e6a7fce15 100644 --- a/ci-automation/packages.sh +++ b/ci-automation/packages.sh @@ -103,10 +103,14 @@ function packages_build() { ./build_packages --board="${arch}-usr" \ --torcx_output_root="${CONTAINER_TORCX_ROOT}" - # copy torcx manifest for publishing + # copy torcx manifest and docker tarball for publishing + local torcx_tmp="__build__/torcx_tmp" + rm -rf "${torcx_tmp}" + mkdir "${torcx_tmp}" ./run_sdk_container -n "${packages_container}" -v "${version}" \ -C "${sdk_image}" \ - cp "${CONTAINER_TORCX_ROOT}/amd64-usr/latest/torcx_manifest.json" __build__/ + cp -r "${CONTAINER_TORCX_ROOT}/" \ + "${torcx_tmp}" # run_sdk_container updates the version file, use that version from here on source sdk_container/.repo/manifests/version.txt @@ -118,7 +122,8 @@ function packages_build() { docker_commit_to_buildcache "${packages_container}" "${packages_image}" "${docker_vernum}" # Publish torcx manifest to "images" cache so tests can pull it later. - copy_to_buildcache "images/${arch}/${vernum}/" __build__/torcx_manifest.json + copy_to_buildcache "images/${arch}/${vernum}/torcx" \ + __build__/torcx_tmp/pkgs/${arch}-usr/docker/*/*.torcx.tgz update_and_push_version "${version}" } diff --git a/ci-automation/test.sh b/ci-automation/test.sh index 7e948bf721..86b3077cfe 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -23,7 +23,7 @@ # OR # - available via build cache server "/containers/[VERSION]/flatcar-sdk-[ARCH]-[VERSION].tar.gz" # (dev SDK) -# 4. Vendor image and torcx manifest to run tests for are available on buildcache +# 4. Vendor image and torcx docker tarball + manifest to run tests for are available on buildcache # ( images/[ARCH]/[FLATCAR_VERSION]/ ) # # INPUT: @@ -48,9 +48,51 @@ # to abort at any point - the previous runs' results won't be lost. # 2. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. +# +# +# LOW-LEVEL / VENDOR SPECIFIC scripts API +# +# Vendor scripts are provided with their own sub-directory and are expected to CD into there before +# creating any artifacts (see vendor script argument 1 below). +# The torcx manifest is supplied in +# ../ +# relative to the vendor sub-directory. The manifest is updated to include a URL pointing to the docker +# torcx tarball on the build cache (for the docker.torcx-manifest-pkgs test). +# +# Vendor specific scripts are called with the following positional arguments: +# 1 - working directory for the tests. +# The vendor script is expected to keep all artifacts it produces in that directory. +# 2 - Architecture to test. +# 3 - version number to test. +# 4 - output TAP file. +# All following arguments specify test cases / test case patterns to run. set -euo pipefail +# Download torcx package and manifest, add build cache URL to manifest +# so the docker.torcx-manifest-pkgs test can use it. +function __prepare_torcx() { + local arch="$1" + local vernum="$2" + local workdir="$3" + + copy_from_buildcache "images/${arch}/${vernum}/torcx/torcx_manifest.json" "${workdir}" + + local docker_pkg + docker_pkg="$(basename \ + "$(jq -r ".value.packages[0].versions[0].locations[0].path" \ + ${workdir}/torcx_manifest.json)")" + + # Add docker package URL on build cache to manifest + jq ".value.packages[0].versions[0].locations += [{\"url\" : \"https://${BUILDCACHE_SERVER}/images/${arch}/${vernum}/${docker_pkg}\"}]" \ + "${workdir}/torcx_manifest.json" \ + > "${workdir}/torcx_manifest_new.json" + + mv "${workdir}/torcx_manifest.json" "${workdir}/torcx_manifest.json.original" + mv "${workdir}/torcx_manifest_new.json" "${workdir}/torcx_manifest.json" +} +# -- + function test_run() { local arch="$1" ; shift local image="$1"; shift @@ -80,14 +122,15 @@ function test_run() { local sdk_image="$(docker_image_fullname "${sdk_name}" "${docker_sdk_vernum}")" echo "docker image rm -f '${sdk_image}'" >> ./ci-cleanup.sh - local tests_dir="__TESTS__/${image}" + local work_dir="__TESTS__" + local tests_dir="${work_dir}/${image}" mkdir -p "${tests_dir}" echo "sudo rm -rf '${tests_dir}'" >> ci-cleanup.sh local container_name="flatcar-tests-${arch}-${docker_vernum}-${image}" - # Make the torcx manifest available to test implementation - copy_from_buildcache "images/${arch}/${vernum}/torcx_manifest.json" "${tests_dir}" + # Make the torcx artifacts available to test implementation + __prepare_torcx "${arch}" "${vernum}" "${work_dir}" local retry="" local success=false diff --git a/ci-automation/vendor-testing/qemu.sh b/ci-automation/vendor-testing/qemu.sh index 8c40b87159..9f174a127c 100755 --- a/ci-automation/vendor-testing/qemu.sh +++ b/ci-automation/vendor-testing/qemu.sh @@ -20,8 +20,12 @@ source ci-automation/ci_automation_common.sh mkdir -p "${work_dir}" cd "${work_dir}" -echo "++++ QEMU test: downloading ${QEMU_IMAGE_NAME} for ${vernum} (${arch}) ++++" -copy_from_buildcache "images/${arch}/${vernum}/${QEMU_IMAGE_NAME}" . +if [ -f "${QEMU_IMAGE_NAME}" ] ; then + echo "++++ QEMU test: Using existing ${work_dir}/${QEMU_IMAGE_NAME} for testing ${vernum} (${arch}) ++++" +else + echo "++++ QEMU test: downloading ${QEMU_IMAGE_NAME} for ${vernum} (${arch}) ++++" + copy_from_buildcache "images/${arch}/${vernum}/${QEMU_IMAGE_NAME}" . +fi set -o noglob @@ -32,7 +36,7 @@ sudo kola run \ --qemu-bios=/usr/share/qemu/bios-256k.bin \ --qemu-image="${QEMU_IMAGE_NAME}" \ --tapfile="${tapfile}" \ - --torcx-manifest=torcx_manifest.json \ + --torcx-manifest=../torcx_manifest.json \ $@ set +o noglob