From cafa385164c0ade0387b6f4ab9595f130bdddae3 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Fri, 18 Feb 2022 15:35:49 +0100 Subject: [PATCH] ci-automation: publish torcx json and use in tests This change updates the package build script to publish the torcx manifest file to the build cache so it can be used by tests. It also updates the generic test script to use the SDK container instead of the packages container image, and to download and use the torcx manifest from the build cache. Signed-off-by: Thilo Fromm --- ci-automation/packages.sh | 11 +++++++++- ci-automation/test.sh | 31 ++++++++++++++++++---------- ci-automation/vendor-testing/qemu.sh | 2 +- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/ci-automation/packages.sh b/ci-automation/packages.sh index a3786bd21e..9e5efe743c 100644 --- a/ci-automation/packages.sh +++ b/ci-automation/packages.sh @@ -48,7 +48,8 @@ # OUTPUT: # # 1. Exported container image "flatcar-packages-[ARCH]-[VERSION].tar.gz" with binary packages -# pushed to buildcache. +# pushed to buildcache, and torcx_manifest.json pushed to "images/${arch}/${vernum}/" +# (for use with tests). # 2. Updated scripts repository # - version tag w/ submodules # - sdk_container/.repo/manifests/version.txt denotes new FLATCAR OS version @@ -102,6 +103,11 @@ function packages_build() { ./build_packages --board="${arch}-usr" \ --torcx_output_root="${CONTAINER_TORCX_ROOT}" + # copy torcx manifest for publishing + ./run_sdk_container -n "${packages_container}" -v "${version}" \ + -C "${sdk_image}" \ + cp "${CONTAINER_TORCX_ROOT}/amd64-usr/latest/torcx_manifest.json" __build__/ + # run_sdk_container updates the version file, use that version from here on source sdk_container/.repo/manifests/version.txt local vernum="${FLATCAR_VERSION}" @@ -111,6 +117,9 @@ function packages_build() { # generate image + push to build cache docker_commit_to_buildcache "${packages_container}" "${packages_image}" "${docker_vernum}" + # Publish torcx manifest to "images" cache so tests can pull it later. + copy_to_buildcache "images/${arch}/${vernum}/" __build__/torcx_manifest.json + update_and_push_version "${version}" } # -- diff --git a/ci-automation/test.sh b/ci-automation/test.sh index b488ea3f0c..924a791f4b 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -10,8 +10,7 @@ # Test scenarios runner stub. # This script will run test scenarios for a single image type. -# Tests will be started inside a container based on the packages container image -# (which contains the torcx manifest). +# Tests will be started inside the SDK container. # This script is generic and will use a vendor-specific test runner from # "ci-automation/vendor-testing/.sh. # @@ -19,10 +18,13 @@ # # 1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt # 2. Scripts repo version tag of OS image version to be built is available and checked out. -# 3. Flatcar packages container is available via build cache server -# from "/containers/[VERSION]/flatcar-packages-[ARCH]-[FLATCAR_VERSION].tar.gz" -# or present locally. Container must contain binary packages and torcx artefacts. -# 4. Vendor image(s) to run tests for are available on buildcache ( images/[ARCH]/[FLATCAR_VERSION]/ ) +# 2. SDK container is either +# - available via ghcr.io/flatcar-linux/flatcar-sdk-[ARCH]:[VERSION] (official SDK release) +# OR +# - available via build cache server "/containers/[VERSION]/flatcar-sdk-[ARCH]-[VERSION].tar.gz" +# (dev SDK) +# 4. Vendor image and torcx manifest to run tests for are available on buildcache +# ( images/[ARCH]/[FLATCAR_VERSION]/ ) # # INPUT: # @@ -69,10 +71,14 @@ function test_run() { local docker_vernum docker_vernum="$(vernum_to_docker_image_version "${vernum}")" - local packages="flatcar-packages-${arch}" - local packages_image="${packages}:${docker_vernum}" + # Get SDK from either the registry or import from build cache + local sdk_version="${FLATCAR_SDK_VERSION}" + local sdk_name="flatcar-sdk-${arch}" + local docker_sdk_vernum="$(vernum_to_docker_image_version "${sdk_version}")" - docker_image_from_buildcache "${packages}" "${docker_vernum}" + docker_image_from_registry_or_buildcache "${sdk_name}" "${docker_sdk_vernum}" + local sdk_image="$(docker_image_fullname "${sdk_name}" "${docker_sdk_vernum}")" + echo "docker image rm -f '${sdk_image}'" >> ./ci-cleanup.sh local tests_dir="__TESTS__/${image}" mkdir -p "${tests_dir}" @@ -80,6 +86,9 @@ function test_run() { local container_name="flatcar-tests-${arch}-${docker_vernum}-${image}" + # Make the torcx manifest available to test implementation + copy_from_buildcache "images/${arch}/${vernum}/torcx_manifest.json" "${tests_dir}" + local retry="" local success=false for retry in $(seq "${retries}"); do @@ -90,7 +99,7 @@ function test_run() { # determine success based on test results (tapfile). set +e -o noglob ./run_sdk_container -x ./ci-cleanup.sh \ - -n "${container_name}" -C "${packages_image}" -v "${vernum}" \ + -n "${container_name}" -C "${sdk_image}" -v "${vernum}" \ ci-automation/vendor-testing/"${image}".sh \ "${tests_dir}" \ "${arch}" \ @@ -100,7 +109,7 @@ function test_run() { set -e +o noglob ./run_sdk_container -x ./ci-cleanup.sh \ - -n "${container_name}" -C "${packages_image}" -v "${vernum}" \ + -n "${container_name}" -C "${sdk_image}" -v "${vernum}" \ ci-automation/test_update_reruns.sh \ "${tests_dir}/${tapfile}" "${image}" "${retry}" \ "${tests_dir}/${failfile}" diff --git a/ci-automation/vendor-testing/qemu.sh b/ci-automation/vendor-testing/qemu.sh index b66daa2064..8c40b87159 100755 --- a/ci-automation/vendor-testing/qemu.sh +++ b/ci-automation/vendor-testing/qemu.sh @@ -32,7 +32,7 @@ sudo kola run \ --qemu-bios=/usr/share/qemu/bios-256k.bin \ --qemu-image="${QEMU_IMAGE_NAME}" \ --tapfile="${tapfile}" \ - --torcx-manifest="${CONTAINER_TORCX_ROOT}/${arch}-usr/latest/torcx_manifest.json" \ + --torcx-manifest=torcx_manifest.json \ $@ set +o noglob