diff --git a/ci-automation/README.md b/ci-automation/README.md index 9899580c6a..aef5b76fa3 100644 --- a/ci-automation/README.md +++ b/ci-automation/README.md @@ -77,8 +77,8 @@ image_build amd64 4. Packages build (`packages.sh`): Build OS image packages and generate a new container image (containing both SDK and packages). 5. Packages are published and the generic OS image is built. 1. Binary packages are published (`push_pkgs.sh`) to the build cache, making them available to developers who base their work on the main branch. - 2. Image build (`image.sh`): Using the container from 3., build an OS image and torcx store, and generate a new container image with everything in it. -6. VMs build (`vms.sh`). Using the packages+torcx+image container from 4., build vendor images. Results are vendor-specific OS images. + 2. Image build (`image.sh`): Using the container from 3., build an OS image, and generate a new container image with everything in it. +6. VMs build (`vms.sh`). Using the packages+image container from 4., build vendor images. Results are vendor-specific OS images. ``` .---------. .------------. .--------. @@ -97,7 +97,7 @@ image_build amd64 | alpha-3499.0.0-dev23 `--------´ | | |`- sdk + OS packages -->| | | container image | - | | torcx manifest | + | | | | ______v_______ | | ( publish pkgs ) | | `------------´ | @@ -129,7 +129,7 @@ Testing is implemented in two layers: Testing relies on the SDK container and will use tools / test suites from the SDK. The low-level vendor / image specific script (layer 2. in the list above) runs inside the SDK. -Testing will use the vendor image published by `vms.sh` from buildcache, and the torcx manifest published by `packages.sh`. +Testing will use the vendor image published by `vms.sh` from buildcache. Additionally, a script library is provided (at `ci-automation/tapfile_helper_lib.sh`) to help handling `.tap` test result files produced by test runs. Library functions may be used to merge the result of multiple test runs (e.g. for multiple image types / vendors) into a single test result report. diff --git a/ci-automation/ci-config.env b/ci-automation/ci-config.env index 39fdb46939..52768fb8ee 100644 --- a/ci-automation/ci-config.env +++ b/ci-automation/ci-config.env @@ -28,7 +28,6 @@ CI_GIT_AUTHOR="flatcar-ci" CI_GIT_EMAIL="infra+ci@flatcar-linux.org" # build artifacts go here (in container) -CONTAINER_TORCX_ROOT="/home/sdk/trunk/src/build/torcx" CONTAINER_IMAGE_ROOT="/home/sdk/trunk/src/build/images" # Set it to "1" or "true" or "t" or "y" or "yes" to always run a full diff --git a/ci-automation/image.sh b/ci-automation/image.sh index 92e9a22cd5..0cb5b7e118 100644 --- a/ci-automation/image.sh +++ b/ci-automation/image.sh @@ -17,7 +17,7 @@ # 2. Scripts repo version tag of OS image version to be built is available and checked out. # 3. Flatcar packages container is available via build cache server # from "/containers/[VERSION]/flatcar-packages-[ARCH]-[FLATCAR_VERSION].tar.gz" -# or present locally. Container must contain binary packages and torcx artefacts. +# or present locally. Container must contain binary packages. # # INPUT: # @@ -37,7 +37,7 @@ # # OUTPUT: # -# 1. OS image, dev container, related artifacts, and torcx packages pushed to buildcache. +# 1. OS image, dev container, and related artifacts pushed to buildcache. # 2. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. # 3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache. @@ -84,17 +84,12 @@ function _image_build_impl() { official_arg="--noofficial" fi - local torcx_root_tar="torcx_root.tar.zst" apply_local_patches - copy_from_buildcache "images/${arch}/${vernum}/torcx/${torcx_root_tar}" . # build image and related artifacts ./run_sdk_container -x ./ci-cleanup.sh -n "${image_container}" -C "${packages_image}" \ -v "${vernum}" \ - mkdir -p "${CONTAINER_IMAGE_ROOT}" "${CONTAINER_TORCX_ROOT}" - ./run_sdk_container -n "${image_container}" -C "${packages_image}" \ - -v "${vernum}" \ - tar --zstd -xf "${torcx_root_tar}" -C "${CONTAINER_TORCX_ROOT}" + mkdir -p "${CONTAINER_IMAGE_ROOT}" ./run_sdk_container -n "${image_container}" -C "${packages_image}" \ -v "${vernum}" \ ./set_official --board="${arch}-usr" "${official_arg}" @@ -103,7 +98,7 @@ function _image_build_impl() { ./build_image --board="${arch}-usr" --group="${channel}" \ --output_root="${CONTAINER_IMAGE_ROOT}" \ --only_store_compressed \ - --torcx_root="${CONTAINER_TORCX_ROOT}" prodtar container + prodtar container # copy resulting images + push to buildcache local images_out="images/" diff --git a/ci-automation/packages.sh b/ci-automation/packages.sh index 1d4faf42c8..e6e984f7f3 100644 --- a/ci-automation/packages.sh +++ b/ci-automation/packages.sh @@ -41,8 +41,7 @@ # OUTPUT: # # 1. Exported container image "flatcar-packages-[ARCH]-[VERSION].tar.gz" with binary packages -# pushed to buildcache, and torcx_manifest.json pushed to "images/${arch}/${vernum}/" -# (for use with tests). +# pushed to buildcache (for use with tests). # 2. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. # 3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache. @@ -81,38 +80,14 @@ function _packages_build_impl() { local vernum="${FLATCAR_VERSION}" local docker_vernum="$(vernum_to_docker_image_version "${vernum}")" local packages_container="flatcar-packages-${arch}-${docker_vernum}" - local torcx_pkg_url="https://${BUILDCACHE_SERVER}/images/${arch}/${vernum}/torcx" source sdk_lib/sdk_container_common.sh - if is_official "${vernum}"; then - # A channel returned by get_git_channel should not ever be - # "developer" here, because it's an official build done from - # one of the maintenance branches. So if the channel happens - # to be "developer", then you are doing it wrong (releasing - # from the main branch?). - torcx_pkg_url="https://$(get_git_channel).release.flatcar-linux.net/${arch}-usr/${vernum}/torcx" - fi - apply_local_patches - # Build packages; store packages and torcx output in container - ./run_sdk_container -x ./ci-cleanup.sh -n "${packages_container}" -v "${vernum}" \ + # Build packages; store packages in container + ./run_sdk_container -x ./ci-cleanup.sh -n "${packages_container}" -v "${vernum}" \ -C "${sdk_image}" \ - mkdir -p "${CONTAINER_TORCX_ROOT}" - ./run_sdk_container -n "${packages_container}" -v "${vernum}" \ - -C "${sdk_image}" \ - ./build_packages --board="${arch}-usr" \ - --torcx_output_root="${CONTAINER_TORCX_ROOT}" \ - --torcx_extra_pkg_url="${torcx_pkg_url}" - - # copy torcx manifest and docker tarball for publishing - local torcx_tmp="__build__/torcx_tmp" - rm -rf "${torcx_tmp}" - mkdir "${torcx_tmp}" - ./run_sdk_container -n "${packages_container}" -v "${vernum}" \ - -C "${sdk_image}" \ - cp -r "${CONTAINER_TORCX_ROOT}/" \ - "${torcx_tmp}" + ./build_packages --board="${arch}-usr" # run_sdk_container updates the version file, use that version from here on source sdk_container/.repo/manifests/version.txt @@ -122,22 +97,5 @@ function _packages_build_impl() { # generate image + push to build cache docker_commit_to_buildcache "${packages_container}" "${packages_image}" "${docker_vernum}" - - # publish torcx output root for consumption by build_image - local torcx_root_tar="torcx_root.tar.zst" - tar --zstd -cpf "${torcx_root_tar}" -C "${torcx_tmp}/torcx" . - copy_to_buildcache "images/${arch}/${vernum}/torcx" "${torcx_root_tar}" - - # Publish torcx manifest and docker tarball to "images" cache so tests can pull it later. - create_digests "${SIGNER}" \ - "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json" \ - "${torcx_tmp}/torcx/pkgs/${arch}-usr/docker/"*/*.torcx.tgz - sign_artifacts "${SIGNER}" \ - "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json"* \ - "${torcx_tmp}/torcx/pkgs/${arch}-usr/docker/"*/*.torcx.tgz* - copy_to_buildcache "images/${arch}/${vernum}/torcx" \ - "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json"* - copy_to_buildcache "images/${arch}/${vernum}/torcx" \ - "${torcx_tmp}/torcx/pkgs/${arch}-usr/docker/"*/*.torcx.tgz* } # -- diff --git a/ci-automation/push_pkgs.sh b/ci-automation/push_pkgs.sh index 86439c599b..0cb1a0187c 100644 --- a/ci-automation/push_pkgs.sh +++ b/ci-automation/push_pkgs.sh @@ -19,7 +19,7 @@ # 2. Scripts repo version tag of OS image version to be built is available and checked out. # 3. Flatcar packages container is available via build cache server # from "/containers/[VERSION]/flatcar-packages-[ARCH]-[FLATCAR_VERSION].tar.gz" -# or present locally. Container must contain binary packages and torcx artifacts. +# or present locally. Container must contain binary packages. # # INPUT: # diff --git a/ci-automation/release.sh b/ci-automation/release.sh index 930fd9d904..90fb1c59db 100644 --- a/ci-automation/release.sh +++ b/ci-automation/release.sh @@ -16,7 +16,7 @@ # 1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt # 2. Scripts repo version tag of OS image version to be built is available and checked out. # 3. Mantle container docker image reference is stored in sdk_container/.repo/manifests/mantle-container. -# 4. Vendor image and torcx docker tarball + manifest to run tests for are available on buildcache +# 4. Vendor image to run tests for are available on buildcache # ( images/[ARCH]/[FLATCAR_VERSION]/ ) # 5. SDK container is either # - available via ghcr.io/flatcar/flatcar-sdk-[ARCH]:[VERSION] (official SDK release) diff --git a/ci-automation/test.sh b/ci-automation/test.sh index c3b14fe2f3..bdddb4a3b4 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -19,7 +19,7 @@ # 1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt # 2. Scripts repo version tag of OS image version to be built is available and checked out. # 3. Mantle container docker image reference is stored in sdk_container/.repo/manifests/mantle-container. -# 4. Vendor image and torcx docker tarball + manifest to run tests for are available on buildcache +# 4. Vendor image to run tests for are available on buildcache # ( images/[ARCH]/[FLATCAR_VERSION]/ ) # # INPUT: @@ -54,14 +54,10 @@ # # Vendor scripts are provided with their own sub-directory and are expected to CD into there before # creating any artifacts (see vendor script argument 1 below). -# The torcx manifest is supplied in -# ../ -# relative to the vendor sub-directory. The manifest is updated to include a URL pointing to the docker -# torcx tarball on the build cache (for the docker.torcx-manifest-pkgs test). # # Vendor specific scripts are called with the following positional arguments: # 1 - Toplevel tests directory -# It contains some additional files needed for running the tests (like torcx manifest or file with channel information). +# It contains some additional files needed for running the tests (like file with channel information). # 2 - Working directory for the tests. # The vendor script is expected to keep all artifacts it produces in that directory. # 3 - Architecture to test. @@ -74,46 +70,6 @@ # script would need to make anyway. For more information, please refer # to the vendor_test.sh file. -# Download torcx manifest and modify URLs pointing to the origin -# server to point to the build cache. This is because the tests for -# releases are run before artifacts are uploaded to the origin -# server. This would make kola's docker.torcx-manifest-pkgs test to -# fail. -function __prepare_torcx() { - local arch="$1" - local vernum="$2" - local workdir="$3" - - copy_from_buildcache "images/${arch}/${vernum}/torcx/torcx_manifest.json" "${workdir}" - - # Change URLs from: - # - # https://${channel}.release.flatcar-linux.net/${arch}-usr/${vernum}/torcx/… - # - # to: - # - # https://bincache.flatcar-linux.net/images/${arch}/${vernum}/torcx/… - # - # This is done in two parts - replacing host part and arch part. - # - # Replace 'https://${channel}.release.flatcar-linux.net/' with - # 'https://bincache.flatcar-linux.net/' matching the initial "url" - # JSON key too. - local host_match='\("url":\s*"https://\)[a-z]\+\.release\([^/]\+/\)' - local host_replace='\1bincache\2' - # Replace '${arch}-usr/` part with 'images/${arch}/'. - local arch_match='\([a-z0-9]\+\)-usr/' - local arch_replace='images/\3/' - sed \ - -e "s#${host_match}${arch_match}#${host_replace}${arch_replace}#g" \ - "${workdir}/torcx_manifest.json" \ - >"${workdir}/torcx_manifest_new.json" - - mv "${workdir}/torcx_manifest.json" "${workdir}/torcx_manifest.json.original" - mv "${workdir}/torcx_manifest_new.json" "${workdir}/torcx_manifest.json" -} -# -- - function __escape_multiple() { local out_array_arg_name="${1}"; shift # rest are args to be escape and appended into the array named @@ -176,9 +132,6 @@ function _test_run_impl() { local mantle_ref mantle_ref=$(cat sdk_container/.repo/manifests/mantle-container) - # Make the torcx artifacts available to test implementation - __prepare_torcx "${arch}" "${vernum}" "${work_dir}" - local tap_merged_summary="results-${image}" local tap_merged_detailed="results-${image}-detailed" local retry="" @@ -214,7 +167,7 @@ function _test_run_impl() { # determine success based on test results (tapfile). set +e touch sdk_container/.env - docker run --pull always --rm --name="${container_name}" --privileged --net host -v /dev:/dev \ + docker run --rm --name="${container_name}" --privileged --net host -v /dev:/dev \ -w /work -v "$PWD":/work "${mantle_ref}" \ bash -c "git config --global --add safe.directory /work && \ source sdk_container/.env && \ @@ -223,7 +176,7 @@ function _test_run_impl() { rm -f "${work_dir}/first_run" # Note: git safe.directory is not set in this run as it does not use git - docker run --pull always --rm --name="${container_name}" --privileged --net host -v /dev:/dev \ + docker run --rm --name="${container_name}" --privileged --net host -v /dev:/dev \ -w /work -v "$PWD":/work "${mantle_ref}" \ ci-automation/test_update_reruns.sh \ "${arch}" "${vernum}" "${image}" "${retry}" \ diff --git a/ci-automation/vendor-testing/aws.sh b/ci-automation/vendor-testing/aws.sh index f9e3c34f06..f33f668b8e 100755 --- a/ci-automation/vendor-testing/aws.sh +++ b/ci-automation/vendor-testing/aws.sh @@ -58,7 +58,6 @@ run_kola_tests() { --aws-type="${instance_type}" \ --aws-iam-profile="${AWS_IAM_PROFILE}" \ --tapfile="${instance_tapfile}" \ - --torcx-manifest="${CIA_TORCX_MANIFEST}" \ "${@}" } diff --git a/ci-automation/vendor-testing/azure.sh b/ci-automation/vendor-testing/azure.sh index 6fc84ff303..bc7adf16ae 100755 --- a/ci-automation/vendor-testing/azure.sh +++ b/ci-automation/vendor-testing/azure.sh @@ -53,7 +53,6 @@ run_kola_tests() { --azure-image-file="${AZURE_IMAGE_NAME}" \ --azure-location="${AZURE_LOCATION}" \ --azure-identity \ - --torcx-manifest="${CIA_TORCX_MANIFEST}" \ --tapfile="${instance_tapfile}" \ --azure-size="${instance_type}" \ --azure-hyper-v-generation="${hyperv_gen}" \ diff --git a/ci-automation/vendor-testing/digitalocean.sh b/ci-automation/vendor-testing/digitalocean.sh index 3ca82ac2e4..14e100ed29 100755 --- a/ci-automation/vendor-testing/digitalocean.sh +++ b/ci-automation/vendor-testing/digitalocean.sh @@ -51,7 +51,6 @@ timeout --signal=SIGQUIT 4h\ --platform=do \ --channel="${CIA_CHANNEL}" \ --tapfile="${CIA_TAPFILE}" \ - --torcx-manifest="${CIA_TORCX_MANIFEST}" \ "${@}" set +x diff --git a/ci-automation/vendor-testing/equinix_metal.sh b/ci-automation/vendor-testing/equinix_metal.sh index 5f4440fd12..8bab854110 100755 --- a/ci-automation/vendor-testing/equinix_metal.sh +++ b/ci-automation/vendor-testing/equinix_metal.sh @@ -36,7 +36,6 @@ run_kola_tests() { --platform=equinixmetal \ --tapfile="${instance_tapfile}" \ --parallel="${EQUINIXMETAL_PARALLEL}" \ - --torcx-manifest="${CIA_TORCX_MANIFEST}" \ --equinixmetal-image-url="${BASE_URL}/${EQUINIXMETAL_IMAGE_NAME}" \ --equinixmetal-installer-image-kernel-url="${BASE_URL}/${PXE_KERNEL_NAME}" \ --equinixmetal-installer-image-cpio-url="${BASE_URL}/${PXE_IMAGE_NAME}" \ diff --git a/ci-automation/vendor-testing/gce.sh b/ci-automation/vendor-testing/gce.sh index 2ab0fd1284..ec26fbe57f 100755 --- a/ci-automation/vendor-testing/gce.sh +++ b/ci-automation/vendor-testing/gce.sh @@ -62,7 +62,6 @@ run_kola_tests() { --platform=gce \ --channel="${CIA_CHANNEL}" \ --tapfile="${instance_tapfile}" \ - --torcx-manifest="${CIA_TORCX_MANIFEST}" \ "${@}" } diff --git a/ci-automation/vendor-testing/openstack.sh b/ci-automation/vendor-testing/openstack.sh index caaa7fe3b1..4efa6d4907 100755 --- a/ci-automation/vendor-testing/openstack.sh +++ b/ci-automation/vendor-testing/openstack.sh @@ -49,7 +49,6 @@ timeout --signal=SIGQUIT 2h kola run \ --parallel="${OPENSTACK_PARALLEL}" \ --tapfile="${CIA_TAPFILE}" \ --channel="${CIA_CHANNEL}" \ - --torcx-manifest="${CIA_TORCX_MANIFEST}" \ --basename="${kola_test_basename}" \ --platform=openstack \ --openstack-network=public \ diff --git a/ci-automation/vendor-testing/qemu.sh b/ci-automation/vendor-testing/qemu.sh index bd1763656a..7b426889e4 100755 --- a/ci-automation/vendor-testing/qemu.sh +++ b/ci-automation/vendor-testing/qemu.sh @@ -64,7 +64,6 @@ kola run \ --qemu-bios="${bios}" \ --qemu-image="${QEMU_IMAGE_NAME}" \ --tapfile="${CIA_TAPFILE}" \ - --torcx-manifest="${CIA_TORCX_MANIFEST}" \ ${QEMU_KOLA_SKIP_MANGLE:+--qemu-skip-mangle} \ "${devcontainer_opts[@]}" \ "${@}" diff --git a/ci-automation/vendor-testing/qemu_update.sh b/ci-automation/vendor-testing/qemu_update.sh index d4cf2d8105..5232b09938 100755 --- a/ci-automation/vendor-testing/qemu_update.sh +++ b/ci-automation/vendor-testing/qemu_update.sh @@ -115,7 +115,6 @@ run_kola_tests() { --qemu-bios="${bios}" \ --qemu-image="${image}" \ --tapfile="${instance_tapfile}" \ - --torcx-manifest="${CIA_TORCX_MANIFEST}" \ --update-payload="${QEMU_UPDATE_PAYLOAD}" \ ${QEMU_KOLA_SKIP_MANGLE:+--qemu-skip-mangle} \ "${tests[@]}" diff --git a/ci-automation/vendor-testing/vmware.sh b/ci-automation/vendor-testing/vmware.sh index 399fbfc897..ce0ca1eeff 100755 --- a/ci-automation/vendor-testing/vmware.sh +++ b/ci-automation/vendor-testing/vmware.sh @@ -60,7 +60,6 @@ sudo timeout --signal=SIGQUIT 2h kola run \ --platform=esx \ --tapfile="${CIA_TAPFILE}" \ --parallel="${VMWARE_ESX_PARALLEL}" \ - --torcx-manifest="${CIA_TORCX_MANIFEST}" \ --esx-config-file "${config_file}" \ --esx-ova-path "${VMWARE_ESX_IMAGE_NAME}" \ "${@}" diff --git a/ci-automation/vendor_test.sh b/ci-automation/vendor_test.sh index f0d657e36d..60081a6772 100644 --- a/ci-automation/vendor_test.sh +++ b/ci-automation/vendor_test.sh @@ -47,10 +47,6 @@ # It's either "release" or "developer", based on the CIA_VERNUM # variable. # -# CIA_TORCX_MANIFEST: -# Path to the Torcx manifest. Usually passed to kola through the -# --torcx-manifest parameter. -# # CIA_FIRST_RUN: # 1 if this is a first run, 0 if it is a rerun of failed tests. # @@ -91,7 +87,6 @@ fi # Make these paths absolute to avoid problems when changing # directories. ciavts_tapfile="${PWD}/${ciavts_work_dir}/${ciavts_tapfile}" -ciavts_torcx_manifest="${PWD}/${ciavts_main_work_dir}/torcx_manifest.json" ciavts_first_run=0 if [[ -f "${ciavts_main_work_dir}/first_run" ]]; then @@ -109,7 +104,6 @@ CIA_CHANNEL="${ciavts_channel}" CIA_TESTSCRIPT="${ciavts_testscript}" CIA_GIT_VERSION="${ciavts_git_version}" CIA_BUILD_TYPE="${ciavts_type}" -CIA_TORCX_MANIFEST="${ciavts_torcx_manifest}" CIA_FIRST_RUN="${ciavts_first_run}" # Unset all variables with ciavts_ prefix now.