From 1b3e9ef1885e3f1136739eaa084156fddbaa9cbf Mon Sep 17 00:00:00 2001 From: Kai Lueke Date: Mon, 27 Jun 2022 18:30:13 +0200 Subject: [PATCH 1/2] ci-automation: Use the package container for VM image building The image job builds an image container that is multiple GBs big and takes >10 mins to be loaded in the vms job. The vms job can also do its work by running from the packages container from the packages job when it fetchs the built image from bincache first and assuming the images job copies it there. Skip generating the image container and instead use the packages container for VM image building by copying the image folder first to bincache and then retrieving it from there. While reworking this we also address the issue that the VMs container had used the same name for both architectures, causing a race when both run in parallel on the same worker. --- ci-automation/ci_automation_common.sh | 12 +++++++ ci-automation/image.sh | 18 ++++++---- ci-automation/vms.sh | 49 +++++++++++++++------------ 3 files changed, 51 insertions(+), 28 deletions(-) diff --git a/ci-automation/ci_automation_common.sh b/ci-automation/ci_automation_common.sh index 25e47c12df..adc2b85d77 100644 --- a/ci-automation/ci_automation_common.sh +++ b/ci-automation/ci_automation_common.sh @@ -122,6 +122,18 @@ function gen_sshcmd() { } # -- +function copy_dir_from_buildcache() { + local remote_path="${BUILDCACHE_PATH_PREFIX}/$1" + local local_path="$2" + + local sshcmd="$(gen_sshcmd)" + mkdir -p "${local_path}" + rsync --partial -a -e "${sshcmd}" "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}:${remote_path}" \ + "${local_path}" +} + +# -- + function copy_to_buildcache() { local remote_path="${BUILDCACHE_PATH_PREFIX}/$1" shift diff --git a/ci-automation/image.sh b/ci-automation/image.sh index 9fff6b8e10..8f0943e414 100644 --- a/ci-automation/image.sh +++ b/ci-automation/image.sh @@ -35,10 +35,7 @@ # # OUTPUT: # -# 1. Exported container image with OS image, dev container, and related artifacts at -# /home/sdk/image/[ARCH], torcx packages at /home/sdk/torcx -# named "flatcar-images-[ARCH]-[FLATCAR_VERSION].tar.gz" -# pushed to buildcache. +# 1. OS image, dev container, related artifacts, and torcx packages pushed to buildcache. # 2. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. # 3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache. @@ -85,7 +82,7 @@ function _image_build_impl() { official_arg="--noofficial" fi - # build image and store it in the container + # build image and related artifacts ./run_sdk_container -x ./ci-cleanup.sh -n "${image_container}" -C "${packages_image}" \ -v "${vernum}" \ mkdir -p "${CONTAINER_IMAGE_ROOT}" @@ -98,7 +95,14 @@ function _image_build_impl() { --output_root="${CONTAINER_IMAGE_ROOT}" \ --torcx_root="${CONTAINER_TORCX_ROOT}" prodtar container - # rename container and push to build cache - docker_commit_to_buildcache "${image_container}" "${image}" "${docker_vernum}" + # copy resulting images + push to buildcache + local images_out="images/" + rm -rf "${images_out}" + ./run_sdk_container -n "${image_container}" -C "${packages_image}" \ + -v "${vernum}" \ + mv "${CONTAINER_IMAGE_ROOT}/${arch}-usr/" "./${images_out}/" + + sign_artifacts "${SIGNER}" "images/latest/"* + copy_to_buildcache "images/${arch}/${vernum}/" "images/latest/"* } # -- diff --git a/ci-automation/vms.sh b/ci-automation/vms.sh index 7710d8a134..f575d5dfc3 100644 --- a/ci-automation/vms.sh +++ b/ci-automation/vms.sh @@ -9,15 +9,16 @@ # vm_build() should be called w/ the positional INPUT parameters below. # Vendor images build automation stub. -# This script will build one or more vendor images ("vm") using a pre-built image container. +# This script will build one or more vendor images ("vm") using a pre-built packages container. # # PREREQUISITES: # # 1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt # 2. Scripts repo version tag of OS image version to be built is available and checked out. -# 3. Flatcar image container is available via build cache server +# 3. Flatcar packages container is available via build cache server # from "/containers/[VERSION]/flatcar-images-[ARCH]-[FLATCAR_VERSION].tar.gz" -# or present locally. Must contain packages and image. +# or present locally. Must contain packages. +# 4. The generic Flatcar image must be present in build cache server. # # INPUT: # @@ -66,19 +67,13 @@ function _vm_build_impl() { local vernum="${FLATCAR_VERSION}" local docker_vernum="$(vernum_to_docker_image_version "${vernum}")" - local image="flatcar-images-${arch}" - local image_image="${image}:${docker_vernum}" - local vms_container="flatcar-vms-${docker_vernum}" + local packages="flatcar-packages-${arch}" + local packages_image="${packages}:${docker_vernum}" - docker_image_from_buildcache "${image}" "${docker_vernum}" + docker_image_from_buildcache "${packages}" "${docker_vernum}" - # clean up dangling containers from previous builds - docker container rm -f "${vms_container}" || true - - local images_out="images/" - rm -rf "${images_out}" - - echo "docker container rm -f '${vms_container}'" >> ci-cleanup.sh + local vms="flatcar-vms-${arch}" + local vms_container="${vms}-${docker_vernum}" # automatically add PXE to formats if we build for Equinix Metal (packet). local has_packet=0 @@ -103,26 +98,38 @@ function _vm_build_impl() { # Keep compatibility with SDK scripts where "equinix_metal" remains unknown. formats=$(echo "$formats" | tr ' ' '\n' | sed 's/equinix_metal/packet/g') + local images_in="images-in/" + rm -rf "${images_in}" + copy_dir_from_buildcache "images/${arch}/${vernum}/" "${images_in}" + ./run_sdk_container -x ./ci-cleanup.sh -n "${vms_container}" -C "${packages_image}" \ + -v "${vernum}" \ + mkdir -p "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest" + ./run_sdk_container -n "${vms_container}" -C "${packages_image}" \ + -v "${vernum}" \ + mv "${images_in}" "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest-input" + for format in ${formats}; do echo " ################### VENDOR '${format}' ################### " COMPRESSION_FORMAT="bz2" if [[ "${format}" =~ ^(openstack|openstack_mini|digitalocean)$ ]];then COMPRESSION_FORMAT="gz,bz2" fi - ./run_sdk_container -n "${vms_container}" -C "${image_image}" \ + ./run_sdk_container -n "${vms_container}" -C "${packages_image}" \ -v "${vernum}" \ ./image_to_vm.sh --format "${format}" --board="${arch}-usr" \ - --from "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest" \ + --from "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest-input" \ + --to "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest" \ --image_compression_formats="${COMPRESSION_FORMAT}" done # copy resulting images + push to buildcache - ./run_sdk_container -n "${vms_container}" \ + local images_out="images/" + rm -rf "${images_out}" + ./run_sdk_container -n "${vms_container}" -C "${packages_image}" \ -v "${vernum}" \ - cp --reflink=auto -R "${CONTAINER_IMAGE_ROOT}/${arch}-usr/" "./${images_out}/" + mv "${CONTAINER_IMAGE_ROOT}/${arch}-usr/" "./${images_out}/" - cd "images/latest" - sign_artifacts "${SIGNER}" * - copy_to_buildcache "images/${arch}/${vernum}/" * + sign_artifacts "${SIGNER}" "images/latest/"* + copy_to_buildcache "images/${arch}/${vernum}/" "images/latest/"* } # -- From c1f1404df83a74cd3ae45e1fe4e8f414e6711b4a Mon Sep 17 00:00:00 2001 From: Kai Lueke Date: Mon, 27 Jun 2022 18:34:10 +0200 Subject: [PATCH 2/2] ci-automation: Run package-diff to report image changes The original pipeline has package-diff commands to print out image differences compared to the last release. This is used for the release Go/No-Go QA checks. Add the same logic to the new pipeline. --- ci-automation/image.sh | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/ci-automation/image.sh b/ci-automation/image.sh index 8f0943e414..8d419ebafa 100644 --- a/ci-automation/image.sh +++ b/ci-automation/image.sh @@ -104,5 +104,41 @@ function _image_build_impl() { sign_artifacts "${SIGNER}" "images/latest/"* copy_to_buildcache "images/${arch}/${vernum}/" "images/latest/"* + + ( + set +x + # Don't fail the whole job + set +e + echo "===================================================================" + export BOARD_A="${arch}-usr" + export FROM_A="release" + export VERSION_A="current" + if [ "${channel}" = "developer" ]; then + export CHANNEL_A="alpha" + else + export CHANNEL_A="${channel}" + fi + export FROM_B="file://${PWD}/images/latest" + # Use the directory directly (and BOARD_B and CHANNEL_B are unused) + export VERSION_B="." + echo "== Image differences compared to ${CHANNEL_A} ${VERSION_A} ==" + rm -f package-diff + curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 "https://raw.githubusercontent.com/flatcar-linux/flatcar-build-scripts/master/package-diff" + chmod +x package-diff + echo "Package updates, compared to ${CHANNEL_A} ${VERSION_A}:" + FILE=flatcar_production_image_packages.txt ./package-diff "${VERSION_A}" "${VERSION_B}" + echo + echo "Image file changes, compared to ${CHANNEL_A} ${VERSION_A}:" + FILE=flatcar_production_image_contents.txt FILESONLY=1 CUTKERNEL=1 ./package-diff "${VERSION_A}" "${VERSION_B}" + echo + echo "Image kernel config changes, compared to ${CHANNEL_A} ${VERSION_A}:" + FILE=flatcar_production_image_kernel_config.txt ./package-diff "${VERSION_A}" "${VERSION_B}" + echo + echo "Image file size change (includes /boot, /usr and the default rootfs partitions), compared to ${CHANNEL_A} ${VERSION_A}:" + FILE=flatcar_production_image_contents.txt CALCSIZE=1 ./package-diff "${VERSION_A}" "${VERSION_B}" + echo + BASE_URL="http://${BUILDCACHE_SERVER}/images/${arch}/${vernum}" + echo "Image URL: ${BASE_URL}/flatcar_production_image.bin.bz2" + ) } # --