ci-automation: remove torcx artifact handling and tests

2025-08-06 20:47:00 +02:00 · 2023-10-12 10:17:10 +02:00 · 2023-10-12 10:17:10 +02:00 · e85ca4984c
commit e85ca4984c
parent 89555ed2bd
17 changed files with 18 additions and 128 deletions
--- a/ci-automation/README.md
+++ b/ci-automation/README.md
@ -77,8 +77,8 @@ image_build amd64
 4. Packages build (`packages.sh`): Build OS image packages and generate a new container image (containing both SDK and packages).
 5. Packages are published and the generic OS image is built.
   1. Binary packages are published (`push_pkgs.sh`) to the build cache, making them available to developers who base their work on the main branch.
-   2. Image build (`image.sh`): Using the container from 3., build an OS image and torcx store, and generate a new container image with everything in it.
-6. VMs build (`vms.sh`). Using the packages+torcx+image container from 4., build vendor images. Results are vendor-specific OS images.
+   2. Image build (`image.sh`): Using the container from 3., build an OS image, and generate a new container image with everything in it.
+6. VMs build (`vms.sh`). Using the packages+image container from 4., build vendor images. Results are vendor-specific OS images.

 ```
       .---------.                     .------------.             .--------.
@ -97,7 +97,7 @@ image_build amd64
            |    alpha-3499.0.0-dev23     `--------´                   |
            |                                 |`- sdk + OS packages -->|
            |                                 |    container image     |
-            |                                 |    torcx manifest      |
+            |                                 |                        |
            |                           ______v_______                 |
            |                          ( publish pkgs )                |
            |                           `------------´                 |
@ -129,7 +129,7 @@ Testing is implemented in two layers:

 Testing relies on the SDK container and will use tools / test suites from the SDK.
 The low-level vendor / image specific script (layer 2. in the list above) runs inside the SDK.
-Testing will use the vendor image published by `vms.sh` from buildcache, and the torcx manifest published by `packages.sh`.
+Testing will use the vendor image published by `vms.sh` from buildcache.

 Additionally, a script library is provided (at `ci-automation/tapfile_helper_lib.sh`) to help handling `.tap` test result files produced by test runs.
 Library functions may be used to merge the result of multiple test runs (e.g. for multiple image types / vendors) into a single test result report.
--- a/ci-automation/ci-config.env
+++ b/ci-automation/ci-config.env
@ -28,7 +28,6 @@ CI_GIT_AUTHOR="flatcar-ci"
 CI_GIT_EMAIL="infra+ci@flatcar-linux.org"

 # build artifacts go here (in container)
-CONTAINER_TORCX_ROOT="/home/sdk/trunk/src/build/torcx"
 CONTAINER_IMAGE_ROOT="/home/sdk/trunk/src/build/images"

 # Set it to "1" or "true" or "t" or "y" or "yes" to always run a full
--- a/ci-automation/image.sh
+++ b/ci-automation/image.sh
@ -17,7 +17,7 @@
 #   2. Scripts repo version tag of OS image version to be built is available and checked out.
 #   3. Flatcar packages container is available via build cache server
 #       from "/containers/[VERSION]/flatcar-packages-[ARCH]-[FLATCAR_VERSION].tar.gz"
-#       or present locally. Container must contain binary packages and torcx artefacts.
+#       or present locally. Container must contain binary packages.
 #
 # INPUT:
 #
@ -37,7 +37,7 @@
 #
 # OUTPUT:
 #
-#   1. OS image, dev container, related artifacts, and torcx packages pushed to buildcache.
+#   1. OS image, dev container, and related artifacts pushed to buildcache.
 #   2. "./ci-cleanup.sh" with commands to clean up temporary build resources,
 #        to be run after this step finishes / when this step is aborted.
 #   3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache.
@ -84,17 +84,12 @@ function _image_build_impl() {
            official_arg="--noofficial"
    fi

-    local torcx_root_tar="torcx_root.tar.zst"
    apply_local_patches
-    copy_from_buildcache "images/${arch}/${vernum}/torcx/${torcx_root_tar}" .

    # build image and related artifacts
    ./run_sdk_container -x ./ci-cleanup.sh -n "${image_container}" -C "${packages_image}" \
            -v "${vernum}" \
-            mkdir -p "${CONTAINER_IMAGE_ROOT}" "${CONTAINER_TORCX_ROOT}"
-    ./run_sdk_container -n "${image_container}" -C "${packages_image}" \
-            -v "${vernum}" \
-            tar --zstd -xf "${torcx_root_tar}" -C "${CONTAINER_TORCX_ROOT}"
+            mkdir -p "${CONTAINER_IMAGE_ROOT}"
    ./run_sdk_container -n "${image_container}" -C "${packages_image}" \
            -v "${vernum}" \
            ./set_official --board="${arch}-usr" "${official_arg}"
@ -103,7 +98,7 @@ function _image_build_impl() {
            ./build_image --board="${arch}-usr" --group="${channel}" \
                          --output_root="${CONTAINER_IMAGE_ROOT}" \
                          --only_store_compressed \
-                          --torcx_root="${CONTAINER_TORCX_ROOT}" prodtar container
+                          prodtar container

    # copy resulting images + push to buildcache
    local images_out="images/"
--- a/ci-automation/packages.sh
+++ b/ci-automation/packages.sh
@ -41,8 +41,7 @@
 # OUTPUT:
 #
 #   1. Exported container image "flatcar-packages-[ARCH]-[VERSION].tar.gz" with binary packages
-#       pushed to buildcache, and torcx_manifest.json pushed to "images/${arch}/${vernum}/"
-#       (for use with tests).
+#       pushed to buildcache (for use with tests).
 #   2. "./ci-cleanup.sh" with commands to clean up temporary build resources,
 #        to be run after this step finishes / when this step is aborted.
 #   3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache.
@ -81,38 +80,14 @@ function _packages_build_impl() {
    local vernum="${FLATCAR_VERSION}"
    local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
    local packages_container="flatcar-packages-${arch}-${docker_vernum}"
-    local torcx_pkg_url="https://${BUILDCACHE_SERVER}/images/${arch}/${vernum}/torcx"

    source sdk_lib/sdk_container_common.sh

-    if is_official "${vernum}"; then
-        # A channel returned by get_git_channel should not ever be
-        # "developer" here, because it's an official build done from
-        # one of the maintenance branches. So if the channel happens
-        # to be "developer", then you are doing it wrong (releasing
-        # from the main branch?).
-        torcx_pkg_url="https://$(get_git_channel).release.flatcar-linux.net/${arch}-usr/${vernum}/torcx"
-    fi
-
    apply_local_patches
-    # Build packages; store packages and torcx output in container
-    ./run_sdk_container -x ./ci-cleanup.sh -n "${packages_container}" -v "${vernum}" \
+    # Build packages; store packages in container
+    ./run_sdk_container -x ./ci-cleanup.sh  -n "${packages_container}" -v "${vernum}" \
        -C "${sdk_image}" \
-        mkdir -p "${CONTAINER_TORCX_ROOT}"
-    ./run_sdk_container -n "${packages_container}" -v "${vernum}" \
-        -C "${sdk_image}" \
-        ./build_packages --board="${arch}-usr" \
-            --torcx_output_root="${CONTAINER_TORCX_ROOT}" \
-            --torcx_extra_pkg_url="${torcx_pkg_url}"
-
-    # copy torcx manifest and docker tarball for publishing
-    local torcx_tmp="__build__/torcx_tmp"
-    rm -rf "${torcx_tmp}"
-    mkdir "${torcx_tmp}"
-    ./run_sdk_container -n "${packages_container}" -v "${vernum}" \
-        -C "${sdk_image}" \
-        cp -r "${CONTAINER_TORCX_ROOT}/" \
-        "${torcx_tmp}"
+        ./build_packages --board="${arch}-usr"

    # run_sdk_container updates the version file, use that version from here on
    source sdk_container/.repo/manifests/version.txt
@ -122,22 +97,5 @@ function _packages_build_impl() {

    # generate image + push to build cache
    docker_commit_to_buildcache "${packages_container}" "${packages_image}" "${docker_vernum}"
-
-    # publish torcx output root for consumption by build_image
-    local torcx_root_tar="torcx_root.tar.zst"
-    tar --zstd -cpf "${torcx_root_tar}" -C "${torcx_tmp}/torcx" .
-    copy_to_buildcache "images/${arch}/${vernum}/torcx" "${torcx_root_tar}"
-
-    # Publish torcx manifest and docker tarball to "images" cache so tests can pull it later.
-    create_digests "${SIGNER}" \
-        "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json" \
-        "${torcx_tmp}/torcx/pkgs/${arch}-usr/docker/"*/*.torcx.tgz
-    sign_artifacts "${SIGNER}" \
-        "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json"* \
-        "${torcx_tmp}/torcx/pkgs/${arch}-usr/docker/"*/*.torcx.tgz*
-    copy_to_buildcache "images/${arch}/${vernum}/torcx" \
-        "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json"*
-    copy_to_buildcache "images/${arch}/${vernum}/torcx" \
-        "${torcx_tmp}/torcx/pkgs/${arch}-usr/docker/"*/*.torcx.tgz*
 }
 # --
--- a/ci-automation/push_pkgs.sh
+++ b/ci-automation/push_pkgs.sh
@ -19,7 +19,7 @@
 #   2. Scripts repo version tag of OS image version to be built is available and checked out.
 #   3. Flatcar packages container is available via build cache server
 #       from "/containers/[VERSION]/flatcar-packages-[ARCH]-[FLATCAR_VERSION].tar.gz"
-#       or present locally. Container must contain binary packages and torcx artifacts.
+#       or present locally. Container must contain binary packages.
 #
 # INPUT:
 #
--- a/ci-automation/release.sh
+++ b/ci-automation/release.sh
@ -16,7 +16,7 @@
 #   1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt
 #   2. Scripts repo version tag of OS image version to be built is available and checked out.
 #   3. Mantle container docker image reference is stored in sdk_container/.repo/manifests/mantle-container.
-#   4. Vendor image and torcx docker tarball + manifest to run tests for are available on buildcache
+#   4. Vendor image to run tests for are available on buildcache
 #         ( images/[ARCH]/[FLATCAR_VERSION]/ )
 #   5. SDK container is either
 #       - available via ghcr.io/flatcar/flatcar-sdk-[ARCH]:[VERSION] (official SDK release)
--- a/ci-automation/test.sh
+++ b/ci-automation/test.sh
@ -19,7 +19,7 @@
 #   1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt
 #   2. Scripts repo version tag of OS image version to be built is available and checked out.
 #   3. Mantle container docker image reference is stored in sdk_container/.repo/manifests/mantle-container.
-#   4. Vendor image and torcx docker tarball + manifest to run tests for are available on buildcache
+#   4. Vendor image to run tests for are available on buildcache
 #         ( images/[ARCH]/[FLATCAR_VERSION]/ )
 #
 # INPUT:
@ -54,14 +54,10 @@
 #
 # Vendor scripts are provided with their own sub-directory and are expected to CD into there before
 #  creating any artifacts (see vendor script argument 1 below).
-# The torcx manifest is supplied in
-#   ../
-# relative to the vendor sub-directory. The manifest is updated to include a URL pointing to the docker
-#  torcx tarball on the build cache (for the docker.torcx-manifest-pkgs test).
 #
 # Vendor specific scripts are called with the following positional arguments:
 # 1 - Toplevel tests directory
-#     It contains some additional files needed for running the tests (like torcx manifest or file with channel information).
+#     It contains some additional files needed for running the tests (like file with channel information).
 # 2 - Working directory for the tests.
 #     The vendor script is expected to keep all artifacts it produces in that directory.
 # 3 - Architecture to test.
@ -74,46 +70,6 @@
 # script would need to make anyway. For more information, please refer
 # to the vendor_test.sh file.

-# Download torcx manifest and modify URLs pointing to the origin
-# server to point to the build cache. This is because the tests for
-# releases are run before artifacts are uploaded to the origin
-# server. This would make kola's docker.torcx-manifest-pkgs test to
-# fail.
-function __prepare_torcx() {
-    local arch="$1"
-    local vernum="$2"
-    local workdir="$3"
-
-    copy_from_buildcache "images/${arch}/${vernum}/torcx/torcx_manifest.json" "${workdir}"
-
-    # Change URLs from:
-    #
-    # https://${channel}.release.flatcar-linux.net/${arch}-usr/${vernum}/torcx/…
-    #
-    # to:
-    #
-    # https://bincache.flatcar-linux.net/images/${arch}/${vernum}/torcx/…
-    #
-    # This is done in two parts - replacing host part and arch part.
-    #
-    # Replace 'https://${channel}.release.flatcar-linux.net/' with
-    # 'https://bincache.flatcar-linux.net/' matching the initial "url"
-    # JSON key too.
-    local host_match='\("url":\s*"https://\)[a-z]\+\.release\([^/]\+/\)'
-    local host_replace='\1bincache\2'
-    # Replace '${arch}-usr/` part with 'images/${arch}/'.
-    local arch_match='\([a-z0-9]\+\)-usr/'
-    local arch_replace='images/\3/'
-    sed \
-        -e "s#${host_match}${arch_match}#${host_replace}${arch_replace}#g" \
-        "${workdir}/torcx_manifest.json" \
-        >"${workdir}/torcx_manifest_new.json"
-
-    mv "${workdir}/torcx_manifest.json" "${workdir}/torcx_manifest.json.original"
-    mv "${workdir}/torcx_manifest_new.json" "${workdir}/torcx_manifest.json"
-}
-# --
-
 function __escape_multiple() {
    local out_array_arg_name="${1}"; shift
    # rest are args to be escape and appended into the array named
@ -176,9 +132,6 @@ function _test_run_impl() {
    local mantle_ref
    mantle_ref=$(cat sdk_container/.repo/manifests/mantle-container)

-    # Make the torcx artifacts available to test implementation
-    __prepare_torcx "${arch}" "${vernum}" "${work_dir}"
-
    local tap_merged_summary="results-${image}"
    local tap_merged_detailed="results-${image}-detailed"
    local retry=""
@ -214,7 +167,7 @@ function _test_run_impl() {
        #  determine success based on test results (tapfile).
        set +e
        touch sdk_container/.env
-        docker run --pull always --rm --name="${container_name}" --privileged --net host -v /dev:/dev \
+        docker run --rm --name="${container_name}" --privileged --net host -v /dev:/dev \
          -w /work -v "$PWD":/work "${mantle_ref}" \
         bash -c "git config --global --add safe.directory /work && \
                  source sdk_container/.env && \
@ -223,7 +176,7 @@ function _test_run_impl() {
        rm -f "${work_dir}/first_run"

        # Note: git safe.directory is not set in this run as it does not use git
-        docker run --pull always --rm --name="${container_name}" --privileged --net host -v /dev:/dev \
+        docker run --rm --name="${container_name}" --privileged --net host -v /dev:/dev \
          -w /work -v "$PWD":/work "${mantle_ref}" \
            ci-automation/test_update_reruns.sh \
                "${arch}" "${vernum}" "${image}" "${retry}" \
--- a/ci-automation/vendor-testing/aws.sh
+++ b/ci-automation/vendor-testing/aws.sh
@ -58,7 +58,6 @@ run_kola_tests() {
         --aws-type="${instance_type}" \
         --aws-iam-profile="${AWS_IAM_PROFILE}" \
         --tapfile="${instance_tapfile}" \
-         --torcx-manifest="${CIA_TORCX_MANIFEST}" \
         "${@}"
 }

--- a/ci-automation/vendor-testing/azure.sh
+++ b/ci-automation/vendor-testing/azure.sh
@ -53,7 +53,6 @@ run_kola_tests() {
      --azure-image-file="${AZURE_IMAGE_NAME}" \
      --azure-location="${AZURE_LOCATION}" \
      --azure-identity \
-      --torcx-manifest="${CIA_TORCX_MANIFEST}" \
      --tapfile="${instance_tapfile}" \
      --azure-size="${instance_type}" \
      --azure-hyper-v-generation="${hyperv_gen}" \
--- a/ci-automation/vendor-testing/digitalocean.sh
+++ b/ci-automation/vendor-testing/digitalocean.sh
@ -51,7 +51,6 @@ timeout --signal=SIGQUIT 4h\
    --platform=do \
    --channel="${CIA_CHANNEL}" \
    --tapfile="${CIA_TAPFILE}" \
-    --torcx-manifest="${CIA_TORCX_MANIFEST}" \
    "${@}"

 set +x
--- a/ci-automation/vendor-testing/equinix_metal.sh
+++ b/ci-automation/vendor-testing/equinix_metal.sh
@ -36,7 +36,6 @@ run_kola_tests() {
          --platform=equinixmetal \
          --tapfile="${instance_tapfile}" \
          --parallel="${EQUINIXMETAL_PARALLEL}" \
-          --torcx-manifest="${CIA_TORCX_MANIFEST}" \
          --equinixmetal-image-url="${BASE_URL}/${EQUINIXMETAL_IMAGE_NAME}" \
          --equinixmetal-installer-image-kernel-url="${BASE_URL}/${PXE_KERNEL_NAME}" \
          --equinixmetal-installer-image-cpio-url="${BASE_URL}/${PXE_IMAGE_NAME}" \
--- a/ci-automation/vendor-testing/gce.sh
+++ b/ci-automation/vendor-testing/gce.sh
@ -62,7 +62,6 @@ run_kola_tests() {
        --platform=gce \
        --channel="${CIA_CHANNEL}" \
        --tapfile="${instance_tapfile}" \
-        --torcx-manifest="${CIA_TORCX_MANIFEST}" \
        "${@}"
 }

--- a/ci-automation/vendor-testing/openstack.sh
+++ b/ci-automation/vendor-testing/openstack.sh
@ -49,7 +49,6 @@ timeout --signal=SIGQUIT 2h kola run \
  --parallel="${OPENSTACK_PARALLEL}" \
  --tapfile="${CIA_TAPFILE}" \
  --channel="${CIA_CHANNEL}" \
-  --torcx-manifest="${CIA_TORCX_MANIFEST}" \
  --basename="${kola_test_basename}" \
  --platform=openstack \
  --openstack-network=public \
--- a/ci-automation/vendor-testing/qemu.sh
+++ b/ci-automation/vendor-testing/qemu.sh
@ -64,7 +64,6 @@ kola run \
    --qemu-bios="${bios}" \
    --qemu-image="${QEMU_IMAGE_NAME}" \
    --tapfile="${CIA_TAPFILE}" \
-    --torcx-manifest="${CIA_TORCX_MANIFEST}" \
    ${QEMU_KOLA_SKIP_MANGLE:+--qemu-skip-mangle} \
    "${devcontainer_opts[@]}" \
    "${@}"
--- a/ci-automation/vendor-testing/qemu_update.sh
+++ b/ci-automation/vendor-testing/qemu_update.sh
@ -115,7 +115,6 @@ run_kola_tests() {
      --qemu-bios="${bios}" \
      --qemu-image="${image}" \
      --tapfile="${instance_tapfile}" \
-      --torcx-manifest="${CIA_TORCX_MANIFEST}" \
      --update-payload="${QEMU_UPDATE_PAYLOAD}" \
      ${QEMU_KOLA_SKIP_MANGLE:+--qemu-skip-mangle} \
      "${tests[@]}"
--- a/ci-automation/vendor-testing/vmware.sh
+++ b/ci-automation/vendor-testing/vmware.sh
@ -60,7 +60,6 @@ sudo timeout --signal=SIGQUIT 2h kola run \
    --platform=esx \
    --tapfile="${CIA_TAPFILE}" \
    --parallel="${VMWARE_ESX_PARALLEL}" \
-    --torcx-manifest="${CIA_TORCX_MANIFEST}" \
    --esx-config-file "${config_file}" \
    --esx-ova-path "${VMWARE_ESX_IMAGE_NAME}" \
    "${@}"
--- a/ci-automation/vendor_test.sh
+++ b/ci-automation/vendor_test.sh
@ -47,10 +47,6 @@
 #   It's either "release" or "developer", based on the CIA_VERNUM
 #   variable.
 #
-# CIA_TORCX_MANIFEST:
-#   Path to the Torcx manifest. Usually passed to kola through the
-#   --torcx-manifest parameter.
-#
 # CIA_FIRST_RUN:
 #   1 if this is a first run, 0 if it is a rerun of failed tests.
 #
@ -91,7 +87,6 @@ fi
 # Make these paths absolute to avoid problems when changing
 # directories.
 ciavts_tapfile="${PWD}/${ciavts_work_dir}/${ciavts_tapfile}"
-ciavts_torcx_manifest="${PWD}/${ciavts_main_work_dir}/torcx_manifest.json"

 ciavts_first_run=0
 if [[ -f "${ciavts_main_work_dir}/first_run" ]]; then
@ -109,7 +104,6 @@ CIA_CHANNEL="${ciavts_channel}"
 CIA_TESTSCRIPT="${ciavts_testscript}"
 CIA_GIT_VERSION="${ciavts_git_version}"
 CIA_BUILD_TYPE="${ciavts_type}"
-CIA_TORCX_MANIFEST="${ciavts_torcx_manifest}"
 CIA_FIRST_RUN="${ciavts_first_run}"

 # Unset all variables with ciavts_ prefix now.