From 571de4064b73008b47cbf7d14b805c9a4fce1484 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Tue, 22 Feb 2022 15:31:26 +0000 Subject: [PATCH 1/7] Make image compression format configurable This change adds a new flag called --image_compression_format which allows us to output the final VM image as one of the supported formats: bz2 (default), gz, zip or none if the compression format is "none" or "", the image will not be compressed. Signed-off-by: Gabriel Adrian Samfira --- build_library/release_util.sh | 40 +++++++++++++++++++++++++++++----- build_library/vm_image_util.sh | 4 +++- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/build_library/release_util.sh b/build_library/release_util.sh index f1ecc8acc0..9905645c33 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -7,6 +7,7 @@ UPLOAD_ROOT= UPLOAD_PATH= TORCX_UPLOAD_ROOT= UPLOAD_DEFAULT=${FLAGS_FALSE} +DEFAULT_IMAGE_COMPRESSION_FORMAT="bz2" # Default upload root can be overridden from the environment. _user="${USER}" @@ -15,9 +16,6 @@ _user="${USER}" : ${FLATCAR_TORCX_UPLOAD_ROOT:=${FLATCAR_UPLOAD_ROOT}/torcx} unset _user -IMAGE_ZIPPER="lbzip2 --compress --keep" -IMAGE_ZIPEXT=".bz2" - DEFINE_boolean parallel ${FLAGS_TRUE} \ "Enable parallelism in gsutil." DEFINE_boolean upload ${UPLOAD_DEFAULT} \ @@ -42,6 +40,38 @@ DEFINE_string sign "" \ "Sign all files to be uploaded with the given GPG key." DEFINE_string sign_digests "" \ "Sign image DIGESTS files with the given GPG key." +DEFINE_string image_compression_format "${DEFAULT_IMAGE_COMPRESSION_FORMAT}" \ + "Compress the resulting images using this format. Options are: none, bz2, gz, zip" + + +compress_file() { + local filepath="$1" + + [ ! -f "${filepath}" ] && die "Image file ${filepath} does not exist" + + case "${FLAGS_image_compression_format}" in + "none"|"") + echo -n "${filepath}" + return 0 + ;; + "bz2") + IMAGE_ZIPPER="lbzip2 --compress --keep" + ;; + "gz") + IMAGE_ZIPPER="pigz --keep" + ;; + "zip") + IMAGE_ZIPPER="pigz --keep --zip" + ;; + *) + die "Unsupported compression format ${FLAGS_image_compression_format}" + ;; + esac + + ${IMAGE_ZIPPER} -f "${filepath}" 2>&1 >/dev/null || die "failed to compress ${filepath}" + + echo -n "${filepath}.${FLAGS_image_compression_format}" +} check_gsutil_opts() { [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0 @@ -217,8 +247,8 @@ upload_image() { # Compress disk images if [[ "${filename}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then info "Compressing ${filename##*/}" - $IMAGE_ZIPPER -f "${filename}" - uploads+=( "${filename}${IMAGE_ZIPEXT}" ) + COMPRESSED_FILENAME=$(compress_file "${filename}") + uploads+=( "$COMPRESSED_FILENAME" ) else uploads+=( "${filename}" ) fi diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index 6b5ad1744d..a2798c8d76 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -1227,7 +1227,9 @@ vm_upload() { # If upload_images compressed $legacy_uploaded be sure to add .bz2 if [[ "${legacy_uploaded}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then - legacy_uploaded+="${IMAGE_ZIPEXT}" + if ! [[ "${FLAGS_image_compression_format}" =~ ^(""|none)$ ]]; then + legacy_uploaded+=".${FLAGS_image_compression_format}" + fi fi local legacy_digests="${legacy_uploaded}.DIGESTS" From f0a535271c285a83d6df1f42010170a13540d2e6 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Tue, 22 Feb 2022 15:33:25 +0000 Subject: [PATCH 2/7] Use gzip compression for OpenStack images This change makes the Jenkins job output openstack images using gzip compression format. This allows OpenStack users to directly consume images by simply specifying the URL to the image. Glance will then download the image, unarchive it and add it to it's catalogue. Fixes #575 Signed-off-by: Gabriel Adrian Samfira --- .../changes/2022-02-22-configurable-image-compression.md | 2 ++ jenkins/vms.sh | 7 +++++++ 2 files changed, 9 insertions(+) create mode 100644 changelog/changes/2022-02-22-configurable-image-compression.md diff --git a/changelog/changes/2022-02-22-configurable-image-compression.md b/changelog/changes/2022-02-22-configurable-image-compression.md new file mode 100644 index 0000000000..a06c7a31eb --- /dev/null +++ b/changelog/changes/2022-02-22-configurable-image-compression.md @@ -0,0 +1,2 @@ +- Image compression format is now configurable. Supported formats are: bz2, gz, zip, none. Selecting the image format can now be done by passing the ```--image_compression_format``` option to ```image_to_vm.sh``` +- The Jenkins ```vms.sh``` script sets the image compression format to ```gz``` for OpenStack images, which allows Glance to directly consume the images by simply passing in the URL of the image. diff --git a/jenkins/vms.sh b/jenkins/vms.sh index 4beb63653e..06e3972eb2 100755 --- a/jenkins/vms.sh +++ b/jenkins/vms.sh @@ -113,6 +113,12 @@ for FORMAT in ${FORMATS}; do UPLOAD_ROOT=${UPLOAD_PRIVATE_ROOT} fi + COMPRESSION_FORMAT="bz2" + + if [[ "${FORMAT}" =~ ^(openstack|openstack_mini)$ ]];then + COMPRESSION_FORMAT="gz" + fi + script image_to_vm.sh \ --board="${BOARD}" \ --format="${FORMAT}" \ @@ -124,6 +130,7 @@ for FORMAT in ${FORMATS}; do --sign_digests="${SIGNING_USER}" \ --download_root="${DOWNLOAD_ROOT}" \ --upload_root="${UPLOAD_ROOT}" \ + --image_compression_format="${COMPRESSION_FORMAT}" --upload \ ${PRIVATE_UPLOAD_OPT} done From f126debb37bfc7b3e0dd1bfdb8e6295ccc165043 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Wed, 23 Feb 2022 16:42:53 +0000 Subject: [PATCH 3/7] Allow specifying multiple compression formats Add the ability to specify a comma separated list of compression formats. Signed-off-by: Gabriel Adrian Samfira --- build_library/build_image_util.sh | 27 ++++- build_library/dev_container_util.sh | 16 ++- build_library/modify_image_util.sh | 16 ++- build_library/prod_image_util.sh | 14 ++- build_library/release_util.sh | 106 +++++++++++++++--- build_library/vm_image_util.sh | 67 ++++++----- ...22-02-22-configurable-image-compression.md | 4 +- jenkins/vms.sh | 2 +- 8 files changed, 195 insertions(+), 57 deletions(-) diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index a31931cb82..e99d266b05 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -58,10 +58,22 @@ extract_update() { local image_name="$1" local disk_layout="$2" local update_path="${BUILD_DIR}/${image_name%_image.bin}_update.bin" + local digest_path="${update_path}.DIGESTS" "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout="${disk_layout}" \ extract "${BUILD_DIR}/${image_name}" "USR-A" "${update_path}" - upload_image "${update_path}" + + # Compress image + files_to_evaluate+=( "${update_path}" ) + declare -a compressed_images + declare -a extra_files + compress_disk_images files_to_evaluate compressed_images extra_files + + # Upload compressed image + upload_image -d "${digest_path}" "${compressed_images[@]}" "${extra_files[@]}" + + # Upload legacy digests + upload_legacy_digests "${digest_path}" compressed_images # For production as well as dev builds we generate a dev-key-signed update # payload for running tests (the signature won't be accepted by production systems). @@ -105,7 +117,18 @@ generate_update() { -new_kernel "${image_kernel}" \ -out_file "${update}.gz" - upload_image -d "${update}.DIGESTS" "${update}".{bin,gz,zip} + # Compress image + declare -a files_to_evaluate + declare -a compressed_images + declare -a extra_files + files_to_evaluate+=( "${update}.bin" ) + compress_disk_images files_to_evaluate compressed_images extra_files + + # Upload images + upload_image -d "${update}.DIGESTS" "${update}".{gz,zip} "${compressed_images[@]}" "${extra_files[@]}" + + # Upload legacy digests + upload_legacy_digests "${update}.DIGESTS" compressed_images } # ldconfig cannot generate caches for non-native arches. diff --git a/build_library/dev_container_util.sh b/build_library/dev_container_util.sh index 9e8f9d8346..34a028baef 100755 --- a/build_library/dev_container_util.sh +++ b/build_library/dev_container_util.sh @@ -108,9 +108,21 @@ create_dev_container() { systemd_enable "${root_fs_dir}" "multi-user.target" "remount-usr.service" finish_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${image_contents}" - upload_image -d "${BUILD_DIR}/${image_name}.bz2.DIGESTS" \ + + declare -a files_to_evaluate + declare -a compressed_images + declare -a extra_files + + files_to_evaluate+=( "${BUILD_DIR}/${image_name}" ) + compress_disk_images files_to_evaluate compressed_images extra_files + + upload_image -d "${BUILD_DIR}/${image_name}.DIGESTS" \ "${BUILD_DIR}/${image_contents}" \ "${BUILD_DIR}/${image_packages}" \ "${BUILD_DIR}/${image_licenses}" \ - "${BUILD_DIR}/${image_name}" + "${compressed_images[@]}" \ + "${extra_files[@]}" + + # Upload legacy digests + upload_legacy_digests "${BUILD_DIR}/${image_name}.DIGESTS" compressed_images } diff --git a/build_library/modify_image_util.sh b/build_library/modify_image_util.sh index f8ed535762..53fa0cc81e 100755 --- a/build_library/modify_image_util.sh +++ b/build_library/modify_image_util.sh @@ -83,7 +83,21 @@ finish_modify_image() { cleanup_mounts "${ROOT_FS_DIR}" trap - EXIT - upload_image "${DST_IMAGE}" + + declare -a files_to_evaluate + declare -a compressed_images + declare -a extra_files + + files_to_evaluate+=( "${DST_IMAGE}" ) + compress_disk_images files_to_evaluate compressed_images extra_files + + upload_image -d "${DST_IMAGE}.DIGESTS" \ + "${compressed_images[@]}" \ + "${extra_files[@]}" + + # Upload legacy digests + upload_legacy_digests "${DST_IMAGE}.DIGESTS" compressed_images + for filename in "${EXTRA_FILES[@]}"; do if [[ -e "${BUILD_DIR}/${filename}" ]]; then upload_image "${BUILD_DIR}/${filename}" diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 509e350aa4..935c9aaf49 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -138,17 +138,27 @@ EOF "${BUILD_DIR}/${image_contents}" "${BUILD_DIR}/${image_packages}" "${BUILD_DIR}/${image_licenses}" - "${BUILD_DIR}/${image_name}" "${BUILD_DIR}/${image_kernel}" "${BUILD_DIR}/${image_pcr_policy}" "${BUILD_DIR}/${image_grub}" "${BUILD_DIR}/${image_kconfig}" ) + + local files_to_evaluate=( "${BUILD_DIR}/${image_name}" ) + declare -a compressed_images + declare -a extra_files + compress_disk_images files_to_evaluate compressed_images extra_files + to_upload+=( "${compressed_images[@]}" ) + to_upload+=( "${extra_files[@]}" ) + # FIXME(bgilbert): no shim on arm64 if [[ -f "${BUILD_DIR}/${image_shim}" ]]; then to_upload+=("${BUILD_DIR}/${image_shim}") fi - upload_image -d "${BUILD_DIR}/${image_name}.bz2.DIGESTS" "${to_upload[@]}" + upload_image -d "${BUILD_DIR}/${image_name}.DIGESTS" "${to_upload[@]}" + + # Upload legacy digests + upload_legacy_digests "${BUILD_DIR}/${image_name}.DIGESTS" compressed_images } create_prod_tar() { diff --git a/build_library/release_util.sh b/build_library/release_util.sh index 9905645c33..56c6ca0d71 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -40,16 +40,18 @@ DEFINE_string sign "" \ "Sign all files to be uploaded with the given GPG key." DEFINE_string sign_digests "" \ "Sign image DIGESTS files with the given GPG key." -DEFINE_string image_compression_format "${DEFAULT_IMAGE_COMPRESSION_FORMAT}" \ - "Compress the resulting images using this format. Options are: none, bz2, gz, zip" +DEFINE_string image_compression_formats "${DEFAULT_IMAGE_COMPRESSION_FORMAT}" \ + "Compress the resulting images using thise formats. This option acceps a list of comma separated values. Options are: none, bz2, gz, zip, zstd" compress_file() { local filepath="$1" + local compression_format="$2" [ ! -f "${filepath}" ] && die "Image file ${filepath} does not exist" + [ -z "${compression_format}" ] && die "compression format parameter is mandatory" - case "${FLAGS_image_compression_format}" in + case "${compression_format}" in "none"|"") echo -n "${filepath}" return 0 @@ -63,14 +65,77 @@ compress_file() { "zip") IMAGE_ZIPPER="pigz --keep --zip" ;; + "zstd") + IMAGE_ZIPPER="zstd --format=zstd -k -q -f --no-progress -o ${filepath}.${compression_format}" + ;; *) - die "Unsupported compression format ${FLAGS_image_compression_format}" + die "Unsupported compression format ${compression_format}" ;; esac ${IMAGE_ZIPPER} -f "${filepath}" 2>&1 >/dev/null || die "failed to compress ${filepath}" - echo -n "${filepath}.${FLAGS_image_compression_format}" + echo -n "${filepath}.${compression_format}" +} + +compress_disk_images() { + # An array of files that are to be evaluated and possibly compressed if images are + # among them. + local -n local_files_to_evaluate="$1" + + # An array that will hold the path on disk to the resulting disk image archives. + # Multiple compression formats may be requested, so this array may hold + # multiple archives for the same image. + local -n local_resulting_archives="$2" + + # Files that did not match the filter for disk images. + local -n local_extra_files="$3" + + info "Compressing images" + # We want to compress images, but we also want to remove the uncompressed files + # from the list of uploadable files. + for filename in "${local_files_to_evaluate[@]}"; do + if [[ "${filename}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then + # Parse the formats as an array. This will yield an extra empty + # array element at the end. + readarray -td, FORMATS<<<"${FLAGS_image_compression_formats}," + # unset the last element + unset 'FORMATS[-1]' + + # An associative array we set an element on whenever we process a format. + # This way we don't process the same format twice. A unique for array elements. + declare -A processed_format + for format in "${FORMATS[@]}";do + if [ -z "${processed_format[${format}]}" ]; then + info "Compressing ${filename##*/} to ${format}" + COMPRESSED_FILENAME=$(compress_file "${filename}" "${format}") + local_resulting_archives+=( "$COMPRESSED_FILENAME" ) + processed_format["${format}"]=1 + fi + done + else + local_extra_files+=( "${filename}" ) + fi + done +} + +upload_legacy_digests() { + [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0 + + local local_digest_file="$1" + local -n local_compressed_files="$2" + + [[ "${#local_compressed_files[@]}" -gt 0 ]] || return 0 + + # Upload legacy digests + declare -a digests_to_upload + for file in "${local_compressed_files[@]}";do + legacy_digest_file="${file}.DIGESTS" + cp "${local_digest_file}" "${legacy_digest_file}" + digests_to_upload+=( "${legacy_digest_file}" ) + done + local def_upload_path="${UPLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}" + upload_files "digests" "${def_upload_path}" "" "${digests_to_upload[@]}" } check_gsutil_opts() { @@ -243,15 +308,30 @@ upload_image() { if [[ ! -f "${filename}" ]]; then die "File '${filename}' does not exist!" fi - + uploads+=( "${filename}" ) # Compress disk images - if [[ "${filename}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then - info "Compressing ${filename##*/}" - COMPRESSED_FILENAME=$(compress_file "${filename}") - uploads+=( "$COMPRESSED_FILENAME" ) - else - uploads+=( "${filename}" ) - fi + #if [[ "${filename}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then + # # Parse the formats as an array. This will yield an extra empty + # # array element at the end. + # readarray -td, FORMATS <<<"${FLAGS_image_compression_formats}," + # # unset the last element + # unset 'FORMATS[-1]' + # + # # An associative array we set an element on whenever we process a format. + # # This way we don't process the same format twice. A unique for array elements. + # declare -A processed_format + # for format in ${FORMATS[@]} + # do + # if [ ! -z ${processed_format[${format}]} ]; then + # info "Compressing ${filename##*/} to ${format}" + # COMPRESSED_FILENAME=$(compress_file "${filename}" "${format}") + # uploads+=( "$COMPRESSED_FILENAME" ) + # processed_format[${format}]=1 + # fi + # done + #else + # uploads+=( "${filename}" ) + #fi done if [[ -z "${digests}" ]]; then diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index a2798c8d76..27e23b41fc 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -1199,47 +1199,46 @@ vm_cleanup() { } vm_upload() { + + declare -a legacy_uploads + declare -a uploadable_files + declare -a compressed_images + declare -a image_files + declare -a digest_uploads + + compress_disk_images VM_GENERATED_FILES compressed_images uploadable_files + + if [ "${#compressed_images[@]}" -gt 0 ]; then + uploadable_files+=( "${compressed_images[@]}" ) + legacy_uploads+=( "${compressed_images[@]}" ) + fi + local digests="$(_dst_dir)/$(_dst_name .DIGESTS)" - upload_image -d "${digests}" "${VM_GENERATED_FILES[@]}" + upload_image -d "${digests}" "${uploadable_files[@]}" [[ -e "${digests}" ]] || return 0 - # FIXME(marineam): Temporary alternate name for .DIGESTS - # This used to be derived from the first file listed in - # ${VM_GENERATED_FILES[@]}", usually $VM_DST_IMG or similar. - # Since not everything actually uploads $VM_DST_IMG this was not very - # consistent and relying on ordering was breakable. - # Now the common prefix, output by $(_dst_name) is used above. - # Some download/install scripts may still refer to the old name. - local uploaded legacy_uploaded - for uploaded in "${VM_GENERATED_FILES[@]}"; do - if [[ "${uploaded}" == "${VM_DST_IMG}" ]]; then - legacy_uploaded="$(_dst_dir)/$(basename ${VM_DST_IMG})" - break + # Since depending on the ordering of $VM_GENERATED_FILES is brittle only + # use it if $VM_DST_IMG isn't included in the uploaded files. + if [ "${#legacy_uploads[@]}" -eq 0 ];then + legacy_uploads+=( "${VM_GENERATED_FILES[0]}" ) + fi + + for legacy_upload in "${legacy_uploads[@]}";do + local legacy_digest_file="${legacy_upload}.DIGESTS" + [[ "${legacy_digest_file}" == "${digests}" ]] && continue + + cp "${digests}" "${legacy_digest_file}" + digest_uploads+=( "${legacy_digest_file}" ) + + if [[ -e "${digests}.asc" ]]; then + digest_uploads+=( "${legacy_digest_file}.asc" ) + cp "${digests}.asc" "${legacy_digest_file}.asc" fi done - # Since depending on the ordering of $VM_GENERATED_FILES is brittle only - # use it if $VM_DST_IMG isn't included in the uploaded files. - if [[ -z "${legacy_uploaded}" ]]; then - legacy_uploaded="${VM_GENERATED_FILES[0]}" - fi - - # If upload_images compressed $legacy_uploaded be sure to add .bz2 - if [[ "${legacy_uploaded}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then - if ! [[ "${FLAGS_image_compression_format}" =~ ^(""|none)$ ]]; then - legacy_uploaded+=".${FLAGS_image_compression_format}" - fi - fi - - local legacy_digests="${legacy_uploaded}.DIGESTS" - [[ "${legacy_digests}" != "${digests}" ]] || return 0 - - local legacy_uploads=( "${legacy_digests}" ) - cp "${digests}" "${legacy_digests}" - if [[ -e "${digests}.asc" ]]; then - legacy_uploads+=( "${legacy_digests}.asc" ) - cp "${digests}.asc" "${legacy_digests}.asc" + if [ "${#digest_uploads[@]}" -gt 0 ];then + legacy_uploads+=( "${digest_uploads[@]}" ) fi local def_upload_path="${UPLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}" diff --git a/changelog/changes/2022-02-22-configurable-image-compression.md b/changelog/changes/2022-02-22-configurable-image-compression.md index a06c7a31eb..a0a19f82fc 100644 --- a/changelog/changes/2022-02-22-configurable-image-compression.md +++ b/changelog/changes/2022-02-22-configurable-image-compression.md @@ -1,2 +1,2 @@ -- Image compression format is now configurable. Supported formats are: bz2, gz, zip, none. Selecting the image format can now be done by passing the ```--image_compression_format``` option to ```image_to_vm.sh``` -- The Jenkins ```vms.sh``` script sets the image compression format to ```gz``` for OpenStack images, which allows Glance to directly consume the images by simply passing in the URL of the image. +- OpenStack: In addition to the `bz2` image, a `gz` compressed image is published. This allows Glance to directly consume the images by simply passing in the URL of the image. +- SDK: The image compression format is now configurable. Supported formats are: `bz2`, `gz`, `zip`, `none`, `zstd`. Selecting the image format can now be done by passing the `--image_compression_formats` option. This flag gets a comma separated list of formats. diff --git a/jenkins/vms.sh b/jenkins/vms.sh index 06e3972eb2..9b6aad48f8 100755 --- a/jenkins/vms.sh +++ b/jenkins/vms.sh @@ -130,7 +130,7 @@ for FORMAT in ${FORMATS}; do --sign_digests="${SIGNING_USER}" \ --download_root="${DOWNLOAD_ROOT}" \ --upload_root="${UPLOAD_ROOT}" \ - --image_compression_format="${COMPRESSION_FORMAT}" + --image_compression_format="${COMPRESSION_FORMAT}" \ --upload \ ${PRIVATE_UPLOAD_OPT} done From 71e4f1c660588eac5aae347dc67576de9d9955ab Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Tue, 8 Mar 2022 08:07:47 +0000 Subject: [PATCH 4/7] Remove commented code --- build_library/release_util.sh | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/build_library/release_util.sh b/build_library/release_util.sh index 56c6ca0d71..9f3e766798 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -309,29 +309,6 @@ upload_image() { die "File '${filename}' does not exist!" fi uploads+=( "${filename}" ) - # Compress disk images - #if [[ "${filename}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then - # # Parse the formats as an array. This will yield an extra empty - # # array element at the end. - # readarray -td, FORMATS <<<"${FLAGS_image_compression_formats}," - # # unset the last element - # unset 'FORMATS[-1]' - # - # # An associative array we set an element on whenever we process a format. - # # This way we don't process the same format twice. A unique for array elements. - # declare -A processed_format - # for format in ${FORMATS[@]} - # do - # if [ ! -z ${processed_format[${format}]} ]; then - # info "Compressing ${filename##*/} to ${format}" - # COMPRESSED_FILENAME=$(compress_file "${filename}" "${format}") - # uploads+=( "$COMPRESSED_FILENAME" ) - # processed_format[${format}]=1 - # fi - # done - #else - # uploads+=( "${filename}" ) - #fi done if [[ -z "${digests}" ]]; then From e88f12c69cd78041d2994d8883907ec37a32e7ef Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Tue, 8 Mar 2022 12:27:18 +0000 Subject: [PATCH 5/7] Rename zstd to zst Rename sztd to zst and amend the changelog. The zstd binary generates a compressed file with the .zst extension by default. Signed-off-by: Gabriel Adrian Samfira --- build_library/release_util.sh | 6 +++--- .../changes/2022-02-22-configurable-image-compression.md | 2 +- jenkins/vms.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/build_library/release_util.sh b/build_library/release_util.sh index 9f3e766798..1c9cb23323 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -41,7 +41,7 @@ DEFINE_string sign "" \ DEFINE_string sign_digests "" \ "Sign image DIGESTS files with the given GPG key." DEFINE_string image_compression_formats "${DEFAULT_IMAGE_COMPRESSION_FORMAT}" \ - "Compress the resulting images using thise formats. This option acceps a list of comma separated values. Options are: none, bz2, gz, zip, zstd" + "Compress the resulting images using thise formats. This option acceps a list of comma separated values. Options are: none, bz2, gz, zip, zst" compress_file() { @@ -65,8 +65,8 @@ compress_file() { "zip") IMAGE_ZIPPER="pigz --keep --zip" ;; - "zstd") - IMAGE_ZIPPER="zstd --format=zstd -k -q -f --no-progress -o ${filepath}.${compression_format}" + "zst") + IMAGE_ZIPPER="zstd --format=zstd -k -q --no-progress" ;; *) die "Unsupported compression format ${compression_format}" diff --git a/changelog/changes/2022-02-22-configurable-image-compression.md b/changelog/changes/2022-02-22-configurable-image-compression.md index a0a19f82fc..3112faf1a1 100644 --- a/changelog/changes/2022-02-22-configurable-image-compression.md +++ b/changelog/changes/2022-02-22-configurable-image-compression.md @@ -1,2 +1,2 @@ - OpenStack: In addition to the `bz2` image, a `gz` compressed image is published. This allows Glance to directly consume the images by simply passing in the URL of the image. -- SDK: The image compression format is now configurable. Supported formats are: `bz2`, `gz`, `zip`, `none`, `zstd`. Selecting the image format can now be done by passing the `--image_compression_formats` option. This flag gets a comma separated list of formats. +- SDK: The image compression format is now configurable. Supported formats are: `bz2`, `gz`, `zip`, `none`, `zst`. Selecting the image format can now be done by passing the `--image_compression_formats` option. This flag gets a comma separated list of formats. diff --git a/jenkins/vms.sh b/jenkins/vms.sh index 9b6aad48f8..494f7f1bb1 100755 --- a/jenkins/vms.sh +++ b/jenkins/vms.sh @@ -116,7 +116,7 @@ for FORMAT in ${FORMATS}; do COMPRESSION_FORMAT="bz2" if [[ "${FORMAT}" =~ ^(openstack|openstack_mini)$ ]];then - COMPRESSION_FORMAT="gz" + COMPRESSION_FORMAT="gz,bz2" fi script image_to_vm.sh \ @@ -130,7 +130,7 @@ for FORMAT in ${FORMATS}; do --sign_digests="${SIGNING_USER}" \ --download_root="${DOWNLOAD_ROOT}" \ --upload_root="${UPLOAD_ROOT}" \ - --image_compression_format="${COMPRESSION_FORMAT}" \ + --image_compression_formats="${COMPRESSION_FORMAT}" \ --upload \ ${PRIVATE_UPLOAD_OPT} done From 99801d24cdb8701570a581959503c8abe00e966d Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 9 Mar 2022 17:19:07 +0200 Subject: [PATCH 6/7] Update jenkins/vms.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Kai Lüke --- jenkins/vms.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/vms.sh b/jenkins/vms.sh index 494f7f1bb1..dc0bc23395 100755 --- a/jenkins/vms.sh +++ b/jenkins/vms.sh @@ -115,7 +115,7 @@ for FORMAT in ${FORMATS}; do COMPRESSION_FORMAT="bz2" - if [[ "${FORMAT}" =~ ^(openstack|openstack_mini)$ ]];then + if [[ "${FORMAT}" =~ ^(openstack|openstack_mini|digitalocean)$ ]];then COMPRESSION_FORMAT="gz,bz2" fi From b7d51af5dce1c8b653d5a776c731590cf18af2ad Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 9 Mar 2022 17:26:05 +0200 Subject: [PATCH 7/7] Update changelog/changes/2022-02-22-configurable-image-compression.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Kai Lüke --- changelog/changes/2022-02-22-configurable-image-compression.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog/changes/2022-02-22-configurable-image-compression.md b/changelog/changes/2022-02-22-configurable-image-compression.md index 3112faf1a1..a4982bd916 100644 --- a/changelog/changes/2022-02-22-configurable-image-compression.md +++ b/changelog/changes/2022-02-22-configurable-image-compression.md @@ -1,2 +1,3 @@ - OpenStack: In addition to the `bz2` image, a `gz` compressed image is published. This allows Glance to directly consume the images by simply passing in the URL of the image. +- DigitalOcean: In addition to the `bz2` image, a `gz` compressed image is published. This helps against hitting the compression timeout that sometimes lets the image import fail. - SDK: The image compression format is now configurable. Supported formats are: `bz2`, `gz`, `zip`, `none`, `zst`. Selecting the image format can now be done by passing the `--image_compression_formats` option. This flag gets a comma separated list of formats.