ci-automation: remove torcx artifact handling and tests

ci-automation/README.md

@@ -77,8 +77,8 @@ image_build amd64
 4. Packages build (`packages.sh`): Build OS image packages and generate a new container image (containing both SDK and packages).
 5. Packages are published and the generic OS image is built.
    1. Binary packages are published (`push_pkgs.sh`) to the build cache, making them available to developers who base their work on the main branch.
-   2. Image build (`image.sh`): Using the container from 3., build an OS image and torcx store, and generate a new container image with everything in it.
+   2. Image build (`image.sh`): Using the container from 3., build an OS image, and generate a new container image with everything in it.
-6. VMs build (`vms.sh`). Using the packages+torcx+image container from 4., build vendor images. Results are vendor-specific OS images.
+6. VMs build (`vms.sh`). Using the packages+image container from 4., build vendor images. Results are vendor-specific OS images.
 
 ```
        .---------.                     .------------.             .--------.
@@ -97,7 +97,7 @@ image_build amd64
             |    alpha-3499.0.0-dev23     `--------´                   |
             |                                 |`- sdk + OS packages -->|
             |                                 |    container image     |
-            |                                 |    torcx manifest      |
+            |                                 |                        |
             |                           ______v_______                 |
             |                          ( publish pkgs )                |
             |                           `------------´                 |
@@ -129,7 +129,7 @@ Testing is implemented in two layers:
 
 Testing relies on the SDK container and will use tools / test suites from the SDK.
 The low-level vendor / image specific script (layer 2. in the list above) runs inside the SDK.
-Testing will use the vendor image published by `vms.sh` from buildcache, and the torcx manifest published by `packages.sh`.
+Testing will use the vendor image published by `vms.sh` from buildcache.
 
 Additionally, a script library is provided (at `ci-automation/tapfile_helper_lib.sh`) to help handling `.tap` test result files produced by test runs.
 Library functions may be used to merge the result of multiple test runs (e.g. for multiple image types / vendors) into a single test result report.

ci-automation/ci-config.env

@@ -28,7 +28,6 @@ CI_GIT_AUTHOR="flatcar-ci"
 CI_GIT_EMAIL="infra+ci@flatcar-linux.org"
 
 # build artifacts go here (in container)
-CONTAINER_TORCX_ROOT="/home/sdk/trunk/src/build/torcx"
 CONTAINER_IMAGE_ROOT="/home/sdk/trunk/src/build/images"
 
 # Set it to "1" or "true" or "t" or "y" or "yes" to always run a full

ci-automation/image.sh

@@ -17,7 +17,7 @@
 #   2. Scripts repo version tag of OS image version to be built is available and checked out.
 #   3. Flatcar packages container is available via build cache server
 #       from "/containers/[VERSION]/flatcar-packages-[ARCH]-[FLATCAR_VERSION].tar.gz"
-#       or present locally. Container must contain binary packages and torcx artefacts.
+#       or present locally. Container must contain binary packages.
 #
 # INPUT:
 #
@@ -37,7 +37,7 @@
 #
 # OUTPUT:
 #
-#   1. OS image, dev container, related artifacts, and torcx packages pushed to buildcache.
+#   1. OS image, dev container, and related artifacts pushed to buildcache.
 #   2. "./ci-cleanup.sh" with commands to clean up temporary build resources,
 #        to be run after this step finishes / when this step is aborted.
 #   3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache.
@@ -84,17 +84,12 @@ function _image_build_impl() {
             official_arg="--noofficial"
     fi
 
-    local torcx_root_tar="torcx_root.tar.zst"
     apply_local_patches
-    copy_from_buildcache "images/${arch}/${vernum}/torcx/${torcx_root_tar}" .
 
     # build image and related artifacts
     ./run_sdk_container -x ./ci-cleanup.sh -n "${image_container}" -C "${packages_image}" \
             -v "${vernum}" \
-            mkdir -p "${CONTAINER_IMAGE_ROOT}" "${CONTAINER_TORCX_ROOT}"
+            mkdir -p "${CONTAINER_IMAGE_ROOT}"
-    ./run_sdk_container -n "${image_container}" -C "${packages_image}" \
-            -v "${vernum}" \
-            tar --zstd -xf "${torcx_root_tar}" -C "${CONTAINER_TORCX_ROOT}"
     ./run_sdk_container -n "${image_container}" -C "${packages_image}" \
             -v "${vernum}" \
             ./set_official --board="${arch}-usr" "${official_arg}"
@@ -103,7 +98,7 @@ function _image_build_impl() {
             ./build_image --board="${arch}-usr" --group="${channel}" \
                           --output_root="${CONTAINER_IMAGE_ROOT}" \
                           --only_store_compressed \
-                          --torcx_root="${CONTAINER_TORCX_ROOT}" prodtar container
+                          prodtar container
 
     # copy resulting images + push to buildcache
     local images_out="images/"

ci-automation/packages.sh

@@ -41,8 +41,7 @@
 # OUTPUT:
 #
 #   1. Exported container image "flatcar-packages-[ARCH]-[VERSION].tar.gz" with binary packages
-#       pushed to buildcache, and torcx_manifest.json pushed to "images/${arch}/${vernum}/"
+#       pushed to buildcache (for use with tests).
-#       (for use with tests).
 #   2. "./ci-cleanup.sh" with commands to clean up temporary build resources,
 #        to be run after this step finishes / when this step is aborted.
 #   3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache.
@@ -81,38 +80,14 @@ function _packages_build_impl() {
     local vernum="${FLATCAR_VERSION}"
     local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
     local packages_container="flatcar-packages-${arch}-${docker_vernum}"
-    local torcx_pkg_url="https://${BUILDCACHE_SERVER}/images/${arch}/${vernum}/torcx"
 
     source sdk_lib/sdk_container_common.sh
 
-    if is_official "${vernum}"; then
-        # A channel returned by get_git_channel should not ever be
-        # "developer" here, because it's an official build done from
-        # one of the maintenance branches. So if the channel happens
-        # to be "developer", then you are doing it wrong (releasing
-        # from the main branch?).
-        torcx_pkg_url="https://$(get_git_channel).release.flatcar-linux.net/${arch}-usr/${vernum}/torcx"
-    fi
-
     apply_local_patches
-    # Build packages; store packages and torcx output in container
+    # Build packages; store packages in container
     ./run_sdk_container -x ./ci-cleanup.sh  -n "${packages_container}" -v "${vernum}" \
         -C "${sdk_image}" \
-        mkdir -p "${CONTAINER_TORCX_ROOT}"
+        ./build_packages --board="${arch}-usr"
-    ./run_sdk_container -n "${packages_container}" -v "${vernum}" \
-        -C "${sdk_image}" \
-        ./build_packages --board="${arch}-usr" \
-            --torcx_output_root="${CONTAINER_TORCX_ROOT}" \
-            --torcx_extra_pkg_url="${torcx_pkg_url}"
-
-    # copy torcx manifest and docker tarball for publishing
-    local torcx_tmp="__build__/torcx_tmp"
-    rm -rf "${torcx_tmp}"
-    mkdir "${torcx_tmp}"
-    ./run_sdk_container -n "${packages_container}" -v "${vernum}" \
-        -C "${sdk_image}" \
-        cp -r "${CONTAINER_TORCX_ROOT}/" \
-        "${torcx_tmp}"
 
     # run_sdk_container updates the version file, use that version from here on
     source sdk_container/.repo/manifests/version.txt
@@ -122,22 +97,5 @@ function _packages_build_impl() {
 
     # generate image + push to build cache
     docker_commit_to_buildcache "${packages_container}" "${packages_image}" "${docker_vernum}"
-
-    # publish torcx output root for consumption by build_image
-    local torcx_root_tar="torcx_root.tar.zst"
-    tar --zstd -cpf "${torcx_root_tar}" -C "${torcx_tmp}/torcx" .
-    copy_to_buildcache "images/${arch}/${vernum}/torcx" "${torcx_root_tar}"
-
-    # Publish torcx manifest and docker tarball to "images" cache so tests can pull it later.
-    create_digests "${SIGNER}" \
-        "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json" \
-        "${torcx_tmp}/torcx/pkgs/${arch}-usr/docker/"*/*.torcx.tgz
-    sign_artifacts "${SIGNER}" \
-        "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json"* \
-        "${torcx_tmp}/torcx/pkgs/${arch}-usr/docker/"*/*.torcx.tgz*
-    copy_to_buildcache "images/${arch}/${vernum}/torcx" \
-        "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json"*
-    copy_to_buildcache "images/${arch}/${vernum}/torcx" \
-        "${torcx_tmp}/torcx/pkgs/${arch}-usr/docker/"*/*.torcx.tgz*
 }
 # --

ci-automation/push_pkgs.sh

@@ -19,7 +19,7 @@
 #   2. Scripts repo version tag of OS image version to be built is available and checked out.
 #   3. Flatcar packages container is available via build cache server
 #       from "/containers/[VERSION]/flatcar-packages-[ARCH]-[FLATCAR_VERSION].tar.gz"
-#       or present locally. Container must contain binary packages and torcx artifacts.
+#       or present locally. Container must contain binary packages.
 #
 # INPUT:
 #

ci-automation/release.sh

@@ -16,7 +16,7 @@
 #   1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt
 #   2. Scripts repo version tag of OS image version to be built is available and checked out.
 #   3. Mantle container docker image reference is stored in sdk_container/.repo/manifests/mantle-container.
-#   4. Vendor image and torcx docker tarball + manifest to run tests for are available on buildcache
+#   4. Vendor image to run tests for are available on buildcache
 #         ( images/[ARCH]/[FLATCAR_VERSION]/ )
 #   5. SDK container is either
 #       - available via ghcr.io/flatcar/flatcar-sdk-[ARCH]:[VERSION] (official SDK release)

ci-automation/test.sh

@@ -19,7 +19,7 @@
 #   1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt
 #   2. Scripts repo version tag of OS image version to be built is available and checked out.
 #   3. Mantle container docker image reference is stored in sdk_container/.repo/manifests/mantle-container.
-#   4. Vendor image and torcx docker tarball + manifest to run tests for are available on buildcache
+#   4. Vendor image to run tests for are available on buildcache
 #         ( images/[ARCH]/[FLATCAR_VERSION]/ )
 #
 # INPUT:
@@ -54,14 +54,10 @@
 #
 # Vendor scripts are provided with their own sub-directory and are expected to CD into there before
 #  creating any artifacts (see vendor script argument 1 below).
-# The torcx manifest is supplied in
-#   ../
-# relative to the vendor sub-directory. The manifest is updated to include a URL pointing to the docker
-#  torcx tarball on the build cache (for the docker.torcx-manifest-pkgs test).
 #
 # Vendor specific scripts are called with the following positional arguments:
 # 1 - Toplevel tests directory
-#     It contains some additional files needed for running the tests (like torcx manifest or file with channel information).
+#     It contains some additional files needed for running the tests (like file with channel information).
 # 2 - Working directory for the tests.
 #     The vendor script is expected to keep all artifacts it produces in that directory.
 # 3 - Architecture to test.
@@ -74,46 +70,6 @@
 # script would need to make anyway. For more information, please refer
 # to the vendor_test.sh file.
 
-# Download torcx manifest and modify URLs pointing to the origin
-# server to point to the build cache. This is because the tests for
-# releases are run before artifacts are uploaded to the origin
-# server. This would make kola's docker.torcx-manifest-pkgs test to
-# fail.
-function __prepare_torcx() {
-    local arch="$1"
-    local vernum="$2"
-    local workdir="$3"
-
-    copy_from_buildcache "images/${arch}/${vernum}/torcx/torcx_manifest.json" "${workdir}"
-
-    # Change URLs from:
-    #
-    # https://${channel}.release.flatcar-linux.net/${arch}-usr/${vernum}/torcx/…
-    #
-    # to:
-    #
-    # https://bincache.flatcar-linux.net/images/${arch}/${vernum}/torcx/…
-    #
-    # This is done in two parts - replacing host part and arch part.
-    #
-    # Replace 'https://${channel}.release.flatcar-linux.net/' with
-    # 'https://bincache.flatcar-linux.net/' matching the initial "url"
-    # JSON key too.
-    local host_match='\("url":\s*"https://\)[a-z]\+\.release\([^/]\+/\)'
-    local host_replace='\1bincache\2'
-    # Replace '${arch}-usr/` part with 'images/${arch}/'.
-    local arch_match='\([a-z0-9]\+\)-usr/'
-    local arch_replace='images/\3/'
-    sed \
-        -e "s#${host_match}${arch_match}#${host_replace}${arch_replace}#g" \
-        "${workdir}/torcx_manifest.json" \
-        >"${workdir}/torcx_manifest_new.json"
-
-    mv "${workdir}/torcx_manifest.json" "${workdir}/torcx_manifest.json.original"
-    mv "${workdir}/torcx_manifest_new.json" "${workdir}/torcx_manifest.json"
-}
-# --
-
 function __escape_multiple() {
     local out_array_arg_name="${1}"; shift
     # rest are args to be escape and appended into the array named
@@ -176,9 +132,6 @@ function _test_run_impl() {
     local mantle_ref
     mantle_ref=$(cat sdk_container/.repo/manifests/mantle-container)
 
-    # Make the torcx artifacts available to test implementation
-    __prepare_torcx "${arch}" "${vernum}" "${work_dir}"
-
     local tap_merged_summary="results-${image}"
     local tap_merged_detailed="results-${image}-detailed"
     local retry=""
@@ -214,7 +167,7 @@ function _test_run_impl() {
         #  determine success based on test results (tapfile).
         set +e
         touch sdk_container/.env
-        docker run --pull always --rm --name="${container_name}" --privileged --net host -v /dev:/dev \
+        docker run --rm --name="${container_name}" --privileged --net host -v /dev:/dev \
           -w /work -v "$PWD":/work "${mantle_ref}" \
          bash -c "git config --global --add safe.directory /work && \
                   source sdk_container/.env && \
@@ -223,7 +176,7 @@ function _test_run_impl() {
         rm -f "${work_dir}/first_run"
 
         # Note: git safe.directory is not set in this run as it does not use git
-        docker run --pull always --rm --name="${container_name}" --privileged --net host -v /dev:/dev \
+        docker run --rm --name="${container_name}" --privileged --net host -v /dev:/dev \
           -w /work -v "$PWD":/work "${mantle_ref}" \
             ci-automation/test_update_reruns.sh \
                 "${arch}" "${vernum}" "${image}" "${retry}" \

ci-automation/vendor-testing/aws.sh

@@ -58,7 +58,6 @@ run_kola_tests() {
          --aws-type="${instance_type}" \
          --aws-iam-profile="${AWS_IAM_PROFILE}" \
          --tapfile="${instance_tapfile}" \
-         --torcx-manifest="${CIA_TORCX_MANIFEST}" \
          "${@}"
 }
 

ci-automation/vendor-testing/azure.sh

@@ -53,7 +53,6 @@ run_kola_tests() {
       --azure-image-file="${AZURE_IMAGE_NAME}" \
       --azure-location="${AZURE_LOCATION}" \
       --azure-identity \
-      --torcx-manifest="${CIA_TORCX_MANIFEST}" \
       --tapfile="${instance_tapfile}" \
       --azure-size="${instance_type}" \
       --azure-hyper-v-generation="${hyperv_gen}" \

ci-automation/vendor-testing/digitalocean.sh

@@ -51,7 +51,6 @@ timeout --signal=SIGQUIT 4h\
     --platform=do \
     --channel="${CIA_CHANNEL}" \
     --tapfile="${CIA_TAPFILE}" \
-    --torcx-manifest="${CIA_TORCX_MANIFEST}" \
     "${@}"
 
 set +x

ci-automation/vendor-testing/equinix_metal.sh

@@ -36,7 +36,6 @@ run_kola_tests() {
           --platform=equinixmetal \
           --tapfile="${instance_tapfile}" \
           --parallel="${EQUINIXMETAL_PARALLEL}" \
-          --torcx-manifest="${CIA_TORCX_MANIFEST}" \
           --equinixmetal-image-url="${BASE_URL}/${EQUINIXMETAL_IMAGE_NAME}" \
           --equinixmetal-installer-image-kernel-url="${BASE_URL}/${PXE_KERNEL_NAME}" \
           --equinixmetal-installer-image-cpio-url="${BASE_URL}/${PXE_IMAGE_NAME}" \

ci-automation/vendor-testing/gce.sh

@@ -62,7 +62,6 @@ run_kola_tests() {
         --platform=gce \
         --channel="${CIA_CHANNEL}" \
         --tapfile="${instance_tapfile}" \
-        --torcx-manifest="${CIA_TORCX_MANIFEST}" \
         "${@}"
 }
 

ci-automation/vendor-testing/openstack.sh

@@ -49,7 +49,6 @@ timeout --signal=SIGQUIT 2h kola run \
   --parallel="${OPENSTACK_PARALLEL}" \
   --tapfile="${CIA_TAPFILE}" \
   --channel="${CIA_CHANNEL}" \
-  --torcx-manifest="${CIA_TORCX_MANIFEST}" \
   --basename="${kola_test_basename}" \
   --platform=openstack \
   --openstack-network=public \

ci-automation/vendor-testing/qemu.sh

@@ -64,7 +64,6 @@ kola run \
     --qemu-bios="${bios}" \
     --qemu-image="${QEMU_IMAGE_NAME}" \
     --tapfile="${CIA_TAPFILE}" \
-    --torcx-manifest="${CIA_TORCX_MANIFEST}" \
     ${QEMU_KOLA_SKIP_MANGLE:+--qemu-skip-mangle} \
     "${devcontainer_opts[@]}" \
     "${@}"

ci-automation/vendor-testing/qemu_update.sh

@@ -115,7 +115,6 @@ run_kola_tests() {
       --qemu-bios="${bios}" \
       --qemu-image="${image}" \
       --tapfile="${instance_tapfile}" \
-      --torcx-manifest="${CIA_TORCX_MANIFEST}" \
       --update-payload="${QEMU_UPDATE_PAYLOAD}" \
       ${QEMU_KOLA_SKIP_MANGLE:+--qemu-skip-mangle} \
       "${tests[@]}"

ci-automation/vendor-testing/vmware.sh

@@ -60,7 +60,6 @@ sudo timeout --signal=SIGQUIT 2h kola run \
     --platform=esx \
     --tapfile="${CIA_TAPFILE}" \
     --parallel="${VMWARE_ESX_PARALLEL}" \
-    --torcx-manifest="${CIA_TORCX_MANIFEST}" \
     --esx-config-file "${config_file}" \
     --esx-ova-path "${VMWARE_ESX_IMAGE_NAME}" \
     "${@}"

ci-automation/vendor_test.sh

@@ -47,10 +47,6 @@
 #   It's either "release" or "developer", based on the CIA_VERNUM
 #   variable.
 #
-# CIA_TORCX_MANIFEST:
-#   Path to the Torcx manifest. Usually passed to kola through the
-#   --torcx-manifest parameter.
-#
 # CIA_FIRST_RUN:
 #   1 if this is a first run, 0 if it is a rerun of failed tests.
 #
@@ -91,7 +87,6 @@ fi
 # Make these paths absolute to avoid problems when changing
 # directories.
 ciavts_tapfile="${PWD}/${ciavts_work_dir}/${ciavts_tapfile}"
-ciavts_torcx_manifest="${PWD}/${ciavts_main_work_dir}/torcx_manifest.json"
 
 ciavts_first_run=0
 if [[ -f "${ciavts_main_work_dir}/first_run" ]]; then
@@ -109,7 +104,6 @@ CIA_CHANNEL="${ciavts_channel}"
 CIA_TESTSCRIPT="${ciavts_testscript}"
 CIA_GIT_VERSION="${ciavts_git_version}"
 CIA_BUILD_TYPE="${ciavts_type}"
-CIA_TORCX_MANIFEST="${ciavts_torcx_manifest}"
 CIA_FIRST_RUN="${ciavts_first_run}"
 
 # Unset all variables with ciavts_ prefix now.