Merge pull request #233 from gabriel-samfira/add-configurable-compression

Make image compression format configurable

build_library/build_image_util.sh

@@ -58,10 +58,22 @@ extract_update() {
   local image_name="$1"
   local disk_layout="$2"
   local update_path="${BUILD_DIR}/${image_name%_image.bin}_update.bin"
+  local digest_path="${update_path}.DIGESTS"
 
   "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout="${disk_layout}" \
     extract "${BUILD_DIR}/${image_name}" "USR-A" "${update_path}"
-  upload_image "${update_path}"
+
+  # Compress image
+  files_to_evaluate+=( "${update_path}" )
+  declare -a compressed_images
+  declare -a extra_files
+  compress_disk_images files_to_evaluate compressed_images extra_files
+
+  # Upload compressed image
+  upload_image -d "${digest_path}" "${compressed_images[@]}" "${extra_files[@]}"
+
+  # Upload legacy digests
+  upload_legacy_digests "${digest_path}" compressed_images
 
   # For production as well as dev builds we generate a dev-key-signed update
   # payload for running tests (the signature won't be accepted by production systems).
@@ -105,7 +117,18 @@ generate_update() {
       -new_kernel "${image_kernel}" \
       -out_file "${update}.gz"
 
-  upload_image -d "${update}.DIGESTS" "${update}".{bin,gz,zip}
+  # Compress image
+  declare -a files_to_evaluate
+  declare -a compressed_images
+  declare -a extra_files
+  files_to_evaluate+=( "${update}.bin" )
+  compress_disk_images files_to_evaluate compressed_images extra_files
+
+  # Upload images
+  upload_image -d "${update}.DIGESTS" "${update}".{gz,zip} "${compressed_images[@]}" "${extra_files[@]}"
+
+  # Upload legacy digests
+  upload_legacy_digests "${update}.DIGESTS" compressed_images
 }
 
 # ldconfig cannot generate caches for non-native arches.

build_library/dev_container_util.sh

@@ -108,9 +108,21 @@ create_dev_container() {
   systemd_enable "${root_fs_dir}" "multi-user.target" "remount-usr.service"
 
   finish_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${image_contents}"
-  upload_image -d "${BUILD_DIR}/${image_name}.bz2.DIGESTS" \
+
+  declare -a files_to_evaluate
+  declare -a compressed_images
+  declare -a extra_files
+
+  files_to_evaluate+=( "${BUILD_DIR}/${image_name}" )
+  compress_disk_images files_to_evaluate compressed_images extra_files
+
+  upload_image -d "${BUILD_DIR}/${image_name}.DIGESTS" \
       "${BUILD_DIR}/${image_contents}" \
       "${BUILD_DIR}/${image_packages}" \
       "${BUILD_DIR}/${image_licenses}" \
-      "${BUILD_DIR}/${image_name}"
+      "${compressed_images[@]}" \
+      "${extra_files[@]}"
+
+  # Upload legacy digests
+  upload_legacy_digests "${BUILD_DIR}/${image_name}.DIGESTS" compressed_images
 }

build_library/modify_image_util.sh

@@ -83,7 +83,21 @@ finish_modify_image() {
     cleanup_mounts "${ROOT_FS_DIR}"
     trap - EXIT
 
-    upload_image "${DST_IMAGE}"
+
+    declare -a files_to_evaluate
+    declare -a compressed_images
+    declare -a extra_files
+
+    files_to_evaluate+=( "${DST_IMAGE}" )
+    compress_disk_images files_to_evaluate compressed_images extra_files
+
+    upload_image -d "${DST_IMAGE}.DIGESTS" \
+        "${compressed_images[@]}" \
+        "${extra_files[@]}"
+
+    # Upload legacy digests
+    upload_legacy_digests "${DST_IMAGE}.DIGESTS" compressed_images
+
     for filename in "${EXTRA_FILES[@]}"; do
         if [[ -e "${BUILD_DIR}/${filename}" ]]; then
             upload_image "${BUILD_DIR}/${filename}"

build_library/prod_image_util.sh

@@ -138,17 +138,27 @@ EOF
     "${BUILD_DIR}/${image_contents}"
     "${BUILD_DIR}/${image_packages}"
     "${BUILD_DIR}/${image_licenses}"
-    "${BUILD_DIR}/${image_name}"
     "${BUILD_DIR}/${image_kernel}"
     "${BUILD_DIR}/${image_pcr_policy}"
     "${BUILD_DIR}/${image_grub}"
     "${BUILD_DIR}/${image_kconfig}"
   )
+
+  local files_to_evaluate=( "${BUILD_DIR}/${image_name}" )
+  declare -a compressed_images
+  declare -a extra_files
+  compress_disk_images files_to_evaluate compressed_images extra_files
+  to_upload+=( "${compressed_images[@]}" )
+  to_upload+=( "${extra_files[@]}" )
+
   # FIXME(bgilbert): no shim on arm64
   if [[ -f "${BUILD_DIR}/${image_shim}" ]]; then
     to_upload+=("${BUILD_DIR}/${image_shim}")
   fi
-  upload_image -d "${BUILD_DIR}/${image_name}.bz2.DIGESTS" "${to_upload[@]}"
+  upload_image -d "${BUILD_DIR}/${image_name}.DIGESTS" "${to_upload[@]}"
+
+  # Upload legacy digests
+  upload_legacy_digests "${BUILD_DIR}/${image_name}.DIGESTS" compressed_images
 }
 
 create_prod_tar() {

build_library/release_util.sh

@@ -7,6 +7,7 @@ UPLOAD_ROOT=
 UPLOAD_PATH=
 TORCX_UPLOAD_ROOT=
 UPLOAD_DEFAULT=${FLAGS_FALSE}
+DEFAULT_IMAGE_COMPRESSION_FORMAT="bz2"
 
 # Default upload root can be overridden from the environment.
 _user="${USER}"
@@ -15,9 +16,6 @@ _user="${USER}"
 : ${FLATCAR_TORCX_UPLOAD_ROOT:=${FLATCAR_UPLOAD_ROOT}/torcx}
 unset _user
 
-IMAGE_ZIPPER="lbzip2 --compress --keep"
-IMAGE_ZIPEXT=".bz2"
-
 DEFINE_boolean parallel ${FLAGS_TRUE} \
   "Enable parallelism in gsutil."
 DEFINE_boolean upload ${UPLOAD_DEFAULT} \
@@ -42,6 +40,103 @@ DEFINE_string sign "" \
   "Sign all files to be uploaded with the given GPG key."
 DEFINE_string sign_digests "" \
   "Sign image DIGESTS files with the given GPG key."
+DEFINE_string image_compression_formats "${DEFAULT_IMAGE_COMPRESSION_FORMAT}" \
+  "Compress the resulting images using thise formats. This option acceps a list of comma separated values. Options are: none, bz2, gz, zip, zst"
+
+
+compress_file() {
+    local filepath="$1"
+    local compression_format="$2"
+
+    [ ! -f "${filepath}" ] && die "Image file ${filepath} does not exist"
+    [ -z "${compression_format}" ] && die "compression format parameter is mandatory"
+
+    case "${compression_format}" in
+    "none"|"")
+        echo -n "${filepath}"
+        return 0
+        ;;
+    "bz2")
+        IMAGE_ZIPPER="lbzip2 --compress --keep"
+        ;;
+    "gz")
+        IMAGE_ZIPPER="pigz --keep"
+        ;;
+    "zip")
+        IMAGE_ZIPPER="pigz --keep --zip"
+        ;;
+    "zst")
+       IMAGE_ZIPPER="zstd --format=zstd -k -q --no-progress"
+       ;;
+    *)
+        die "Unsupported compression format ${compression_format}"
+        ;;
+    esac
+
+    ${IMAGE_ZIPPER} -f "${filepath}" 2>&1 >/dev/null || die "failed to compress ${filepath}"
+
+    echo -n "${filepath}.${compression_format}"
+}
+
+compress_disk_images() {
+    # An array of files that are to be evaluated and possibly compressed if images are
+    # among them.
+    local -n local_files_to_evaluate="$1"
+
+    # An array that will hold the path on disk to the resulting disk image archives.
+    # Multiple compression formats may be requested, so this array may hold
+    # multiple archives for the same image.
+    local -n local_resulting_archives="$2"
+
+    # Files that did not match the filter for disk images.
+    local -n local_extra_files="$3"
+
+    info "Compressing images"
+    # We want to compress images, but we also want to remove the uncompressed files
+    # from the list of uploadable files.
+    for filename in "${local_files_to_evaluate[@]}"; do
+        if [[ "${filename}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then
+            # Parse the formats as an array. This will yield an extra empty
+            # array element at the end.
+            readarray -td, FORMATS<<<"${FLAGS_image_compression_formats},"
+            # unset the last element
+            unset 'FORMATS[-1]'
+
+            # An associative array we set an element on whenever we process a format.
+            # This way we don't process the same format twice. A unique for array elements.
+            declare -A processed_format
+            for format in "${FORMATS[@]}";do
+                if [ -z "${processed_format[${format}]}" ]; then
+                    info "Compressing ${filename##*/} to ${format}"
+                    COMPRESSED_FILENAME=$(compress_file "${filename}" "${format}")
+                    local_resulting_archives+=( "$COMPRESSED_FILENAME" )
+                    processed_format["${format}"]=1
+                fi
+            done
+        else
+            local_extra_files+=( "${filename}" )            
+        fi
+    done
+}
+
+upload_legacy_digests() {
+    [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0
+
+    local local_digest_file="$1"
+    local -n local_compressed_files="$2"
+
+    [[ "${#local_compressed_files[@]}" -gt 0 ]] || return 0
+
+    # Upload legacy digests
+    declare -a digests_to_upload
+    for file in "${local_compressed_files[@]}";do
+        legacy_digest_file="${file}.DIGESTS"
+        cp "${local_digest_file}" "${legacy_digest_file}"
+        digests_to_upload+=( "${legacy_digest_file}" )
+    done
+    local def_upload_path="${UPLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}"
+    upload_files "digests" "${def_upload_path}" "" "${digests_to_upload[@]}"
+}
 
 check_gsutil_opts() {
     [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0
@@ -213,15 +308,7 @@ upload_image() {
         if [[ ! -f "${filename}" ]]; then
             die "File '${filename}' does not exist!"
         fi
-
-        # Compress disk images
-        if [[ "${filename}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then
-            info "Compressing ${filename##*/}"
-            $IMAGE_ZIPPER -f "${filename}"
-            uploads+=( "${filename}${IMAGE_ZIPEXT}" )
-        else
-            uploads+=( "${filename}" )
-        fi
+        uploads+=( "${filename}" )
     done
 
     if [[ -z "${digests}" ]]; then

build_library/vm_image_util.sh

@@ -1199,45 +1199,46 @@ vm_cleanup() {
 }
 
 vm_upload() {
+
+    declare -a legacy_uploads
+    declare -a uploadable_files
+    declare -a compressed_images
+    declare -a image_files
+    declare -a digest_uploads
+
+    compress_disk_images VM_GENERATED_FILES compressed_images uploadable_files
+
+    if [ "${#compressed_images[@]}" -gt 0 ]; then
+        uploadable_files+=( "${compressed_images[@]}" )
+        legacy_uploads+=( "${compressed_images[@]}" )
+    fi
+
     local digests="$(_dst_dir)/$(_dst_name .DIGESTS)"
-    upload_image -d "${digests}" "${VM_GENERATED_FILES[@]}"
+    upload_image -d "${digests}" "${uploadable_files[@]}"
 
     [[ -e "${digests}" ]] || return 0
 
-    # FIXME(marineam): Temporary alternate name for .DIGESTS
-    # This used to be derived from the first file listed in
-    # ${VM_GENERATED_FILES[@]}", usually $VM_DST_IMG or similar.
-    # Since not everything actually uploads $VM_DST_IMG this was not very
-    # consistent and relying on ordering was breakable.
-    # Now the common prefix, output by $(_dst_name) is used above.
-    # Some download/install scripts may still refer to the old name.
-    local uploaded legacy_uploaded
-    for uploaded in "${VM_GENERATED_FILES[@]}"; do
-        if [[ "${uploaded}" == "${VM_DST_IMG}" ]]; then
-            legacy_uploaded="$(_dst_dir)/$(basename ${VM_DST_IMG})"
-            break
+    # Since depending on the ordering of $VM_GENERATED_FILES is brittle only
+    # use it if $VM_DST_IMG isn't included in the uploaded files.
+    if [ "${#legacy_uploads[@]}" -eq 0 ];then
+        legacy_uploads+=( "${VM_GENERATED_FILES[0]}" )
+    fi
+
+    for legacy_upload in "${legacy_uploads[@]}";do
+        local legacy_digest_file="${legacy_upload}.DIGESTS"
+        [[ "${legacy_digest_file}" == "${digests}" ]] && continue
+
+        cp "${digests}" "${legacy_digest_file}"
+        digest_uploads+=( "${legacy_digest_file}" )
+
+        if [[ -e "${digests}.asc" ]]; then
+            digest_uploads+=( "${legacy_digest_file}.asc" )
+            cp "${digests}.asc" "${legacy_digest_file}.asc"
         fi
     done
 
-    # Since depending on the ordering of $VM_GENERATED_FILES is brittle only
-    # use it if $VM_DST_IMG isn't included in the uploaded files.
-    if [[ -z "${legacy_uploaded}" ]]; then
-        legacy_uploaded="${VM_GENERATED_FILES[0]}"
-    fi
-
-    # If upload_images compressed $legacy_uploaded be sure to add .bz2
-    if [[ "${legacy_uploaded}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then
-        legacy_uploaded+="${IMAGE_ZIPEXT}"
-    fi
-
-    local legacy_digests="${legacy_uploaded}.DIGESTS"
-    [[ "${legacy_digests}" != "${digests}" ]] || return 0
-
-    local legacy_uploads=( "${legacy_digests}" )
-    cp "${digests}" "${legacy_digests}"
-    if [[ -e "${digests}.asc" ]]; then
-        legacy_uploads+=( "${legacy_digests}.asc" )
-        cp "${digests}.asc" "${legacy_digests}.asc"
+    if [ "${#digest_uploads[@]}" -gt 0 ];then
+        legacy_uploads+=( "${digest_uploads[@]}" )
     fi
 
     local def_upload_path="${UPLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}"

changelog/changes/2022-02-22-configurable-image-compression.md

@@ -0,0 +1,3 @@
+- OpenStack: In addition to the `bz2` image, a `gz` compressed image is published. This allows Glance to directly consume the images by simply passing in the URL of the image.
+- DigitalOcean: In addition to the `bz2` image, a `gz` compressed image is published. This helps against hitting the compression timeout that sometimes lets the image import fail.
+- SDK: The image compression format is now configurable. Supported formats are: `bz2`, `gz`, `zip`, `none`, `zst`. Selecting the image format can now be done by passing the `--image_compression_formats` option. This flag gets a comma separated list of formats.

jenkins/vms.sh

@@ -113,6 +113,12 @@ for FORMAT in ${FORMATS}; do
     UPLOAD_ROOT=${UPLOAD_PRIVATE_ROOT}
   fi
 
+  COMPRESSION_FORMAT="bz2"
+
+  if [[ "${FORMAT}" =~ ^(openstack|openstack_mini|digitalocean)$ ]];then
+    COMPRESSION_FORMAT="gz,bz2"
+  fi
+
   script image_to_vm.sh \
     --board="${BOARD}" \
     --format="${FORMAT}" \
@@ -124,6 +130,7 @@ for FORMAT in ${FORMATS}; do
     --sign_digests="${SIGNING_USER}" \
     --download_root="${DOWNLOAD_ROOT}" \
     --upload_root="${UPLOAD_ROOT}" \
+    --image_compression_formats="${COMPRESSION_FORMAT}" \
     --upload \
     ${PRIVATE_UPLOAD_OPT}
 done