From ef9b0ff820249333774c5e50e1a36749f3bc4a05 Mon Sep 17 00:00:00 2001
From: Kai Lueke <kailuke@microsoft.com>
Date: Fri, 1 Jul 2022 16:46:32 +0200
Subject: [PATCH] ci-automation: Only store compressed images

The new build pipeline compresses images already but uploaded both the
compressed and uncompressed files because the whole build folder gets
uploaded.
Add a new flag "--only_store_compressed" to the image generation which
deletes the uncompressed file after compression is done. Uncompressed
images are still supported if specified in the flag
"image_compression_formats".

Closes https://github.com/flatcar-linux/Flatcar/issues/793
---
 build_library/release_util.sh        | 10 ++++++++++
 ci-automation/image.sh               |  3 +++
 ci-automation/vendor-testing/qemu.sh |  4 +++-
 ci-automation/vms.sh                 |  4 +++-
 4 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/build_library/release_util.sh b/build_library/release_util.sh
index 1c9cb23323..16d33f2c93 100644
--- a/build_library/release_util.sh
+++ b/build_library/release_util.sh
@@ -42,6 +42,8 @@ DEFINE_string sign_digests "" \
   "Sign image DIGESTS files with the given GPG key."
 DEFINE_string image_compression_formats "${DEFAULT_IMAGE_COMPRESSION_FORMAT}" \
   "Compress the resulting images using thise formats. This option acceps a list of comma separated values. Options are: none, bz2, gz, zip, zst"
+DEFINE_boolean only_store_compressed ${FLAGS_TRUE} \
+  "Delete input file when compressing, except when 'none' is part of the compression formats or the generic image is the input"
 
 
 compress_file() {
@@ -113,6 +115,14 @@ compress_disk_images() {
                     processed_format["${format}"]=1
                 fi
             done
+            # If requested, delete the input file after compression (only if 'none' is not part of the formats)
+            # Exclude the generic image and update payload because they are needed for generating other formats
+            if [ "${FLAGS_only_store_compressed}" -eq "${FLAGS_TRUE}" ] &&
+               [ "${filename##*/}" != "flatcar_production_image.bin" ] &&
+               [ "${filename##*/}" != "flatcar_production_update.bin" ] &&
+               ! echo "${FORMATS[@]}" | grep -q "none"; then
+                rm "${filename}"
+            fi
         else
             local_extra_files+=( "${filename}" )            
         fi
diff --git a/ci-automation/image.sh b/ci-automation/image.sh
index 8d419ebafa..e1d796b093 100644
--- a/ci-automation/image.sh
+++ b/ci-automation/image.sh
@@ -93,6 +93,7 @@ function _image_build_impl() {
             -v "${vernum}" \
             ./build_image --board="${arch}-usr" --group="${channel}" \
                           --output_root="${CONTAINER_IMAGE_ROOT}" \
+                          --only_store_compressed \
                           --torcx_root="${CONTAINER_TORCX_ROOT}" prodtar container
 
     # copy resulting images + push to buildcache
@@ -102,6 +103,8 @@ function _image_build_impl() {
             -v "${vernum}" \
             mv "${CONTAINER_IMAGE_ROOT}/${arch}-usr/" "./${images_out}/"
 
+    # Delete uncompressed generic image before signing and upload
+    rm "images/latest/flatcar_production_image.bin" "images/latest/flatcar_production_update.bin"
     sign_artifacts "${SIGNER}" "images/latest/"*
     copy_to_buildcache "images/${arch}/${vernum}/" "images/latest/"*
 
diff --git a/ci-automation/vendor-testing/qemu.sh b/ci-automation/vendor-testing/qemu.sh
index 4143609239..8977f254eb 100755
--- a/ci-automation/vendor-testing/qemu.sh
+++ b/ci-automation/vendor-testing/qemu.sh
@@ -25,7 +25,9 @@ if [ -f "${QEMU_IMAGE_NAME}" ] ; then
     echo "++++ ${CIA_TESTSCRIPT}: Using existing ./${QEMU_IMAGE_NAME} for testing ${CIA_VERNUM} (${CIA_ARCH}) ++++"
 else
     echo "++++ ${CIA_TESTSCRIPT}: downloading ${QEMU_IMAGE_NAME} for ${CIA_VERNUM} (${CIA_ARCH}) ++++"
-    copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/${QEMU_IMAGE_NAME}" .
+    rm -f "${QEMU_IMAGE_NAME}.bz2"
+    copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/${QEMU_IMAGE_NAME}.bz2" .
+    lbunzip2 "${QEMU_IMAGE_NAME}.bz2"
 fi
 
 bios="${QEMU_BIOS}"
diff --git a/ci-automation/vms.sh b/ci-automation/vms.sh
index f575d5dfc3..b832ab0084 100644
--- a/ci-automation/vms.sh
+++ b/ci-automation/vms.sh
@@ -101,6 +101,7 @@ function _vm_build_impl() {
     local images_in="images-in/"
     rm -rf "${images_in}"
     copy_dir_from_buildcache "images/${arch}/${vernum}/" "${images_in}"
+    lbunzip2 "${images_in}/flatcar_production_image.bin.bz2"
     ./run_sdk_container -x ./ci-cleanup.sh -n "${vms_container}" -C "${packages_image}" \
             -v "${vernum}" \
             mkdir -p "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest"
@@ -119,7 +120,8 @@ function _vm_build_impl() {
             ./image_to_vm.sh --format "${format}" --board="${arch}-usr" \
                 --from "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest-input" \
                 --to "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest" \
-                --image_compression_formats="${COMPRESSION_FORMAT}"
+                --image_compression_formats="${COMPRESSION_FORMAT}" \
+                --only_store_compressed
     done
 
     # copy resulting images + push to buildcache