From 5a14212c65e3b48481669f98bae0c66996212c6b Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Tue, 22 Feb 2022 15:31:26 +0000 Subject: [PATCH 1/7] Make image compression format configurable This change adds a new flag called --image_compression_format which allows us to output the final VM image as one of the supported formats: bz2 (default), gz, zip or none if the compression format is "none" or "", the image will not be compressed. Signed-off-by: Gabriel Adrian Samfira --- build_library/release_util.sh | 40 +++++++++++++++++++++++++++++----- build_library/vm_image_util.sh | 4 +++- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/build_library/release_util.sh b/build_library/release_util.sh index f1ecc8acc0..9905645c33 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -7,6 +7,7 @@ UPLOAD_ROOT= UPLOAD_PATH= TORCX_UPLOAD_ROOT= UPLOAD_DEFAULT=${FLAGS_FALSE} +DEFAULT_IMAGE_COMPRESSION_FORMAT="bz2" # Default upload root can be overridden from the environment. _user="${USER}" @@ -15,9 +16,6 @@ _user="${USER}" : ${FLATCAR_TORCX_UPLOAD_ROOT:=${FLATCAR_UPLOAD_ROOT}/torcx} unset _user -IMAGE_ZIPPER="lbzip2 --compress --keep" -IMAGE_ZIPEXT=".bz2" - DEFINE_boolean parallel ${FLAGS_TRUE} \ "Enable parallelism in gsutil." DEFINE_boolean upload ${UPLOAD_DEFAULT} \ @@ -42,6 +40,38 @@ DEFINE_string sign "" \ "Sign all files to be uploaded with the given GPG key." DEFINE_string sign_digests "" \ "Sign image DIGESTS files with the given GPG key." +DEFINE_string image_compression_format "${DEFAULT_IMAGE_COMPRESSION_FORMAT}" \ + "Compress the resulting images using this format. Options are: none, bz2, gz, zip" + + +compress_file() { + local filepath="$1" + + [ ! -f "${filepath}" ] && die "Image file ${filepath} does not exist" + + case "${FLAGS_image_compression_format}" in + "none"|"") + echo -n "${filepath}" + return 0 + ;; + "bz2") + IMAGE_ZIPPER="lbzip2 --compress --keep" + ;; + "gz") + IMAGE_ZIPPER="pigz --keep" + ;; + "zip") + IMAGE_ZIPPER="pigz --keep --zip" + ;; + *) + die "Unsupported compression format ${FLAGS_image_compression_format}" + ;; + esac + + ${IMAGE_ZIPPER} -f "${filepath}" 2>&1 >/dev/null || die "failed to compress ${filepath}" + + echo -n "${filepath}.${FLAGS_image_compression_format}" +} check_gsutil_opts() { [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0 @@ -217,8 +247,8 @@ upload_image() { # Compress disk images if [[ "${filename}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then info "Compressing ${filename##*/}" - $IMAGE_ZIPPER -f "${filename}" - uploads+=( "${filename}${IMAGE_ZIPEXT}" ) + COMPRESSED_FILENAME=$(compress_file "${filename}") + uploads+=( "$COMPRESSED_FILENAME" ) else uploads+=( "${filename}" ) fi diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index 8e72eda86f..f0d76eed91 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -1203,7 +1203,9 @@ vm_upload() { # If upload_images compressed $legacy_uploaded be sure to add .bz2 if [[ "${legacy_uploaded}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then - legacy_uploaded+="${IMAGE_ZIPEXT}" + if ! [[ "${FLAGS_image_compression_format}" =~ ^(""|none)$ ]]; then + legacy_uploaded+=".${FLAGS_image_compression_format}" + fi fi local legacy_digests="${legacy_uploaded}.DIGESTS" From 7a7eec19adeb9ff29f73ccef70da2d3a998c4e4a Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Tue, 22 Feb 2022 15:33:25 +0000 Subject: [PATCH 2/7] Use gzip compression for OpenStack images This change makes the Jenkins job output openstack images using gzip compression format. This allows OpenStack users to directly consume images by simply specifying the URL to the image. Glance will then download the image, unarchive it and add it to it's catalogue. Fixes #575 Signed-off-by: Gabriel Adrian Samfira --- .../changes/2022-02-22-configurable-image-compression.md | 2 ++ jenkins/vms.sh | 7 +++++++ 2 files changed, 9 insertions(+) create mode 100644 changelog/changes/2022-02-22-configurable-image-compression.md diff --git a/changelog/changes/2022-02-22-configurable-image-compression.md b/changelog/changes/2022-02-22-configurable-image-compression.md new file mode 100644 index 0000000000..a06c7a31eb --- /dev/null +++ b/changelog/changes/2022-02-22-configurable-image-compression.md @@ -0,0 +1,2 @@ +- Image compression format is now configurable. Supported formats are: bz2, gz, zip, none. Selecting the image format can now be done by passing the ```--image_compression_format``` option to ```image_to_vm.sh``` +- The Jenkins ```vms.sh``` script sets the image compression format to ```gz``` for OpenStack images, which allows Glance to directly consume the images by simply passing in the URL of the image. diff --git a/jenkins/vms.sh b/jenkins/vms.sh index 20bd67031e..1ebcd1970b 100755 --- a/jenkins/vms.sh +++ b/jenkins/vms.sh @@ -113,6 +113,12 @@ for FORMAT in ${FORMATS}; do UPLOAD_ROOT=${UPLOAD_PRIVATE_ROOT} fi + COMPRESSION_FORMAT="bz2" + + if [[ "${FORMAT}" =~ ^(openstack|openstack_mini)$ ]];then + COMPRESSION_FORMAT="gz" + fi + script image_to_vm.sh \ --board="${BOARD}" \ --format="${FORMAT}" \ @@ -124,6 +130,7 @@ for FORMAT in ${FORMATS}; do --sign_digests="${SIGNING_USER}" \ --download_root="${DOWNLOAD_ROOT}" \ --upload_root="${UPLOAD_ROOT}" \ + --image_compression_format="${COMPRESSION_FORMAT}" --upload \ ${PRIVATE_UPLOAD_OPT} done From 43c031ae005ca29605ebfb9c1315bdbcc18ae9ae Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Wed, 23 Feb 2022 16:42:53 +0000 Subject: [PATCH 3/7] Allow specifying multiple compression formats Add the ability to specify a comma separated list of compression formats. Signed-off-by: Gabriel Adrian Samfira --- build_library/build_image_util.sh | 27 ++++- build_library/dev_container_util.sh | 16 ++- build_library/modify_image_util.sh | 16 ++- build_library/prod_image_util.sh | 14 ++- build_library/release_util.sh | 106 +++++++++++++++--- build_library/vm_image_util.sh | 67 ++++++----- ...22-02-22-configurable-image-compression.md | 4 +- jenkins/vms.sh | 2 +- 8 files changed, 195 insertions(+), 57 deletions(-) diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 4e29cadf88..54b027558d 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -58,10 +58,22 @@ extract_update() { local image_name="$1" local disk_layout="$2" local update_path="${BUILD_DIR}/${image_name%_image.bin}_update.bin" + local digest_path="${update_path}.DIGESTS" "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout="${disk_layout}" \ extract "${BUILD_DIR}/${image_name}" "USR-A" "${update_path}" - upload_image "${update_path}" + + # Compress image + files_to_evaluate+=( "${update_path}" ) + declare -a compressed_images + declare -a extra_files + compress_disk_images files_to_evaluate compressed_images extra_files + + # Upload compressed image + upload_image -d "${digest_path}" "${compressed_images[@]}" "${extra_files[@]}" + + # Upload legacy digests + upload_legacy_digests "${digest_path}" compressed_images # For production as well as dev builds we generate a dev-key-signed update # payload for running tests (the signature won't be accepted by production systems). @@ -105,7 +117,18 @@ generate_update() { -new_kernel "${image_kernel}" \ -out_file "${update}.gz" - upload_image -d "${update}.DIGESTS" "${update}".{bin,gz,zip} + # Compress image + declare -a files_to_evaluate + declare -a compressed_images + declare -a extra_files + files_to_evaluate+=( "${update}.bin" ) + compress_disk_images files_to_evaluate compressed_images extra_files + + # Upload images + upload_image -d "${update}.DIGESTS" "${update}".{gz,zip} "${compressed_images[@]}" "${extra_files[@]}" + + # Upload legacy digests + upload_legacy_digests "${update}.DIGESTS" compressed_images } # ldconfig cannot generate caches for non-native arches. diff --git a/build_library/dev_container_util.sh b/build_library/dev_container_util.sh index 9e8f9d8346..34a028baef 100755 --- a/build_library/dev_container_util.sh +++ b/build_library/dev_container_util.sh @@ -108,9 +108,21 @@ create_dev_container() { systemd_enable "${root_fs_dir}" "multi-user.target" "remount-usr.service" finish_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${image_contents}" - upload_image -d "${BUILD_DIR}/${image_name}.bz2.DIGESTS" \ + + declare -a files_to_evaluate + declare -a compressed_images + declare -a extra_files + + files_to_evaluate+=( "${BUILD_DIR}/${image_name}" ) + compress_disk_images files_to_evaluate compressed_images extra_files + + upload_image -d "${BUILD_DIR}/${image_name}.DIGESTS" \ "${BUILD_DIR}/${image_contents}" \ "${BUILD_DIR}/${image_packages}" \ "${BUILD_DIR}/${image_licenses}" \ - "${BUILD_DIR}/${image_name}" + "${compressed_images[@]}" \ + "${extra_files[@]}" + + # Upload legacy digests + upload_legacy_digests "${BUILD_DIR}/${image_name}.DIGESTS" compressed_images } diff --git a/build_library/modify_image_util.sh b/build_library/modify_image_util.sh index f8ed535762..53fa0cc81e 100755 --- a/build_library/modify_image_util.sh +++ b/build_library/modify_image_util.sh @@ -83,7 +83,21 @@ finish_modify_image() { cleanup_mounts "${ROOT_FS_DIR}" trap - EXIT - upload_image "${DST_IMAGE}" + + declare -a files_to_evaluate + declare -a compressed_images + declare -a extra_files + + files_to_evaluate+=( "${DST_IMAGE}" ) + compress_disk_images files_to_evaluate compressed_images extra_files + + upload_image -d "${DST_IMAGE}.DIGESTS" \ + "${compressed_images[@]}" \ + "${extra_files[@]}" + + # Upload legacy digests + upload_legacy_digests "${DST_IMAGE}.DIGESTS" compressed_images + for filename in "${EXTRA_FILES[@]}"; do if [[ -e "${BUILD_DIR}/${filename}" ]]; then upload_image "${BUILD_DIR}/${filename}" diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 509e350aa4..935c9aaf49 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -138,17 +138,27 @@ EOF "${BUILD_DIR}/${image_contents}" "${BUILD_DIR}/${image_packages}" "${BUILD_DIR}/${image_licenses}" - "${BUILD_DIR}/${image_name}" "${BUILD_DIR}/${image_kernel}" "${BUILD_DIR}/${image_pcr_policy}" "${BUILD_DIR}/${image_grub}" "${BUILD_DIR}/${image_kconfig}" ) + + local files_to_evaluate=( "${BUILD_DIR}/${image_name}" ) + declare -a compressed_images + declare -a extra_files + compress_disk_images files_to_evaluate compressed_images extra_files + to_upload+=( "${compressed_images[@]}" ) + to_upload+=( "${extra_files[@]}" ) + # FIXME(bgilbert): no shim on arm64 if [[ -f "${BUILD_DIR}/${image_shim}" ]]; then to_upload+=("${BUILD_DIR}/${image_shim}") fi - upload_image -d "${BUILD_DIR}/${image_name}.bz2.DIGESTS" "${to_upload[@]}" + upload_image -d "${BUILD_DIR}/${image_name}.DIGESTS" "${to_upload[@]}" + + # Upload legacy digests + upload_legacy_digests "${BUILD_DIR}/${image_name}.DIGESTS" compressed_images } create_prod_tar() { diff --git a/build_library/release_util.sh b/build_library/release_util.sh index 9905645c33..56c6ca0d71 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -40,16 +40,18 @@ DEFINE_string sign "" \ "Sign all files to be uploaded with the given GPG key." DEFINE_string sign_digests "" \ "Sign image DIGESTS files with the given GPG key." -DEFINE_string image_compression_format "${DEFAULT_IMAGE_COMPRESSION_FORMAT}" \ - "Compress the resulting images using this format. Options are: none, bz2, gz, zip" +DEFINE_string image_compression_formats "${DEFAULT_IMAGE_COMPRESSION_FORMAT}" \ + "Compress the resulting images using thise formats. This option acceps a list of comma separated values. Options are: none, bz2, gz, zip, zstd" compress_file() { local filepath="$1" + local compression_format="$2" [ ! -f "${filepath}" ] && die "Image file ${filepath} does not exist" + [ -z "${compression_format}" ] && die "compression format parameter is mandatory" - case "${FLAGS_image_compression_format}" in + case "${compression_format}" in "none"|"") echo -n "${filepath}" return 0 @@ -63,14 +65,77 @@ compress_file() { "zip") IMAGE_ZIPPER="pigz --keep --zip" ;; + "zstd") + IMAGE_ZIPPER="zstd --format=zstd -k -q -f --no-progress -o ${filepath}.${compression_format}" + ;; *) - die "Unsupported compression format ${FLAGS_image_compression_format}" + die "Unsupported compression format ${compression_format}" ;; esac ${IMAGE_ZIPPER} -f "${filepath}" 2>&1 >/dev/null || die "failed to compress ${filepath}" - echo -n "${filepath}.${FLAGS_image_compression_format}" + echo -n "${filepath}.${compression_format}" +} + +compress_disk_images() { + # An array of files that are to be evaluated and possibly compressed if images are + # among them. + local -n local_files_to_evaluate="$1" + + # An array that will hold the path on disk to the resulting disk image archives. + # Multiple compression formats may be requested, so this array may hold + # multiple archives for the same image. + local -n local_resulting_archives="$2" + + # Files that did not match the filter for disk images. + local -n local_extra_files="$3" + + info "Compressing images" + # We want to compress images, but we also want to remove the uncompressed files + # from the list of uploadable files. + for filename in "${local_files_to_evaluate[@]}"; do + if [[ "${filename}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then + # Parse the formats as an array. This will yield an extra empty + # array element at the end. + readarray -td, FORMATS<<<"${FLAGS_image_compression_formats}," + # unset the last element + unset 'FORMATS[-1]' + + # An associative array we set an element on whenever we process a format. + # This way we don't process the same format twice. A unique for array elements. + declare -A processed_format + for format in "${FORMATS[@]}";do + if [ -z "${processed_format[${format}]}" ]; then + info "Compressing ${filename##*/} to ${format}" + COMPRESSED_FILENAME=$(compress_file "${filename}" "${format}") + local_resulting_archives+=( "$COMPRESSED_FILENAME" ) + processed_format["${format}"]=1 + fi + done + else + local_extra_files+=( "${filename}" ) + fi + done +} + +upload_legacy_digests() { + [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0 + + local local_digest_file="$1" + local -n local_compressed_files="$2" + + [[ "${#local_compressed_files[@]}" -gt 0 ]] || return 0 + + # Upload legacy digests + declare -a digests_to_upload + for file in "${local_compressed_files[@]}";do + legacy_digest_file="${file}.DIGESTS" + cp "${local_digest_file}" "${legacy_digest_file}" + digests_to_upload+=( "${legacy_digest_file}" ) + done + local def_upload_path="${UPLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}" + upload_files "digests" "${def_upload_path}" "" "${digests_to_upload[@]}" } check_gsutil_opts() { @@ -243,15 +308,30 @@ upload_image() { if [[ ! -f "${filename}" ]]; then die "File '${filename}' does not exist!" fi - + uploads+=( "${filename}" ) # Compress disk images - if [[ "${filename}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then - info "Compressing ${filename##*/}" - COMPRESSED_FILENAME=$(compress_file "${filename}") - uploads+=( "$COMPRESSED_FILENAME" ) - else - uploads+=( "${filename}" ) - fi + #if [[ "${filename}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then + # # Parse the formats as an array. This will yield an extra empty + # # array element at the end. + # readarray -td, FORMATS <<<"${FLAGS_image_compression_formats}," + # # unset the last element + # unset 'FORMATS[-1]' + # + # # An associative array we set an element on whenever we process a format. + # # This way we don't process the same format twice. A unique for array elements. + # declare -A processed_format + # for format in ${FORMATS[@]} + # do + # if [ ! -z ${processed_format[${format}]} ]; then + # info "Compressing ${filename##*/} to ${format}" + # COMPRESSED_FILENAME=$(compress_file "${filename}" "${format}") + # uploads+=( "$COMPRESSED_FILENAME" ) + # processed_format[${format}]=1 + # fi + # done + #else + # uploads+=( "${filename}" ) + #fi done if [[ -z "${digests}" ]]; then diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index f0d76eed91..9017e466bf 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -1175,47 +1175,46 @@ vm_cleanup() { } vm_upload() { + + declare -a legacy_uploads + declare -a uploadable_files + declare -a compressed_images + declare -a image_files + declare -a digest_uploads + + compress_disk_images VM_GENERATED_FILES compressed_images uploadable_files + + if [ "${#compressed_images[@]}" -gt 0 ]; then + uploadable_files+=( "${compressed_images[@]}" ) + legacy_uploads+=( "${compressed_images[@]}" ) + fi + local digests="$(_dst_dir)/$(_dst_name .DIGESTS)" - upload_image -d "${digests}" "${VM_GENERATED_FILES[@]}" + upload_image -d "${digests}" "${uploadable_files[@]}" [[ -e "${digests}" ]] || return 0 - # FIXME(marineam): Temporary alternate name for .DIGESTS - # This used to be derived from the first file listed in - # ${VM_GENERATED_FILES[@]}", usually $VM_DST_IMG or similar. - # Since not everything actually uploads $VM_DST_IMG this was not very - # consistent and relying on ordering was breakable. - # Now the common prefix, output by $(_dst_name) is used above. - # Some download/install scripts may still refer to the old name. - local uploaded legacy_uploaded - for uploaded in "${VM_GENERATED_FILES[@]}"; do - if [[ "${uploaded}" == "${VM_DST_IMG}" ]]; then - legacy_uploaded="$(_dst_dir)/$(basename ${VM_DST_IMG})" - break + # Since depending on the ordering of $VM_GENERATED_FILES is brittle only + # use it if $VM_DST_IMG isn't included in the uploaded files. + if [ "${#legacy_uploads[@]}" -eq 0 ];then + legacy_uploads+=( "${VM_GENERATED_FILES[0]}" ) + fi + + for legacy_upload in "${legacy_uploads[@]}";do + local legacy_digest_file="${legacy_upload}.DIGESTS" + [[ "${legacy_digest_file}" == "${digests}" ]] && continue + + cp "${digests}" "${legacy_digest_file}" + digest_uploads+=( "${legacy_digest_file}" ) + + if [[ -e "${digests}.asc" ]]; then + digest_uploads+=( "${legacy_digest_file}.asc" ) + cp "${digests}.asc" "${legacy_digest_file}.asc" fi done - # Since depending on the ordering of $VM_GENERATED_FILES is brittle only - # use it if $VM_DST_IMG isn't included in the uploaded files. - if [[ -z "${legacy_uploaded}" ]]; then - legacy_uploaded="${VM_GENERATED_FILES[0]}" - fi - - # If upload_images compressed $legacy_uploaded be sure to add .bz2 - if [[ "${legacy_uploaded}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then - if ! [[ "${FLAGS_image_compression_format}" =~ ^(""|none)$ ]]; then - legacy_uploaded+=".${FLAGS_image_compression_format}" - fi - fi - - local legacy_digests="${legacy_uploaded}.DIGESTS" - [[ "${legacy_digests}" != "${digests}" ]] || return 0 - - local legacy_uploads=( "${legacy_digests}" ) - cp "${digests}" "${legacy_digests}" - if [[ -e "${digests}.asc" ]]; then - legacy_uploads+=( "${legacy_digests}.asc" ) - cp "${digests}.asc" "${legacy_digests}.asc" + if [ "${#digest_uploads[@]}" -gt 0 ];then + legacy_uploads+=( "${digest_uploads[@]}" ) fi local def_upload_path="${UPLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}" diff --git a/changelog/changes/2022-02-22-configurable-image-compression.md b/changelog/changes/2022-02-22-configurable-image-compression.md index a06c7a31eb..a0a19f82fc 100644 --- a/changelog/changes/2022-02-22-configurable-image-compression.md +++ b/changelog/changes/2022-02-22-configurable-image-compression.md @@ -1,2 +1,2 @@ -- Image compression format is now configurable. Supported formats are: bz2, gz, zip, none. Selecting the image format can now be done by passing the ```--image_compression_format``` option to ```image_to_vm.sh``` -- The Jenkins ```vms.sh``` script sets the image compression format to ```gz``` for OpenStack images, which allows Glance to directly consume the images by simply passing in the URL of the image. +- OpenStack: In addition to the `bz2` image, a `gz` compressed image is published. This allows Glance to directly consume the images by simply passing in the URL of the image. +- SDK: The image compression format is now configurable. Supported formats are: `bz2`, `gz`, `zip`, `none`, `zstd`. Selecting the image format can now be done by passing the `--image_compression_formats` option. This flag gets a comma separated list of formats. diff --git a/jenkins/vms.sh b/jenkins/vms.sh index 1ebcd1970b..e4fe10d701 100755 --- a/jenkins/vms.sh +++ b/jenkins/vms.sh @@ -130,7 +130,7 @@ for FORMAT in ${FORMATS}; do --sign_digests="${SIGNING_USER}" \ --download_root="${DOWNLOAD_ROOT}" \ --upload_root="${UPLOAD_ROOT}" \ - --image_compression_format="${COMPRESSION_FORMAT}" + --image_compression_format="${COMPRESSION_FORMAT}" \ --upload \ ${PRIVATE_UPLOAD_OPT} done From 4c86576bc475447eebebeb32b9141ed4b42ddcad Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Tue, 8 Mar 2022 08:07:47 +0000 Subject: [PATCH 4/7] Remove commented code --- build_library/release_util.sh | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/build_library/release_util.sh b/build_library/release_util.sh index 56c6ca0d71..9f3e766798 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -309,29 +309,6 @@ upload_image() { die "File '${filename}' does not exist!" fi uploads+=( "${filename}" ) - # Compress disk images - #if [[ "${filename}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then - # # Parse the formats as an array. This will yield an extra empty - # # array element at the end. - # readarray -td, FORMATS <<<"${FLAGS_image_compression_formats}," - # # unset the last element - # unset 'FORMATS[-1]' - # - # # An associative array we set an element on whenever we process a format. - # # This way we don't process the same format twice. A unique for array elements. - # declare -A processed_format - # for format in ${FORMATS[@]} - # do - # if [ ! -z ${processed_format[${format}]} ]; then - # info "Compressing ${filename##*/} to ${format}" - # COMPRESSED_FILENAME=$(compress_file "${filename}" "${format}") - # uploads+=( "$COMPRESSED_FILENAME" ) - # processed_format[${format}]=1 - # fi - # done - #else - # uploads+=( "${filename}" ) - #fi done if [[ -z "${digests}" ]]; then From 716c15a729dfbbfaf979e3960efe8118dc27363c Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Tue, 8 Mar 2022 12:27:18 +0000 Subject: [PATCH 5/7] Rename zstd to zst Rename sztd to zst and amend the changelog. The zstd binary generates a compressed file with the .zst extension by default. Signed-off-by: Gabriel Adrian Samfira --- build_library/release_util.sh | 6 +++--- .../changes/2022-02-22-configurable-image-compression.md | 2 +- jenkins/vms.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/build_library/release_util.sh b/build_library/release_util.sh index 9f3e766798..1c9cb23323 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -41,7 +41,7 @@ DEFINE_string sign "" \ DEFINE_string sign_digests "" \ "Sign image DIGESTS files with the given GPG key." DEFINE_string image_compression_formats "${DEFAULT_IMAGE_COMPRESSION_FORMAT}" \ - "Compress the resulting images using thise formats. This option acceps a list of comma separated values. Options are: none, bz2, gz, zip, zstd" + "Compress the resulting images using thise formats. This option acceps a list of comma separated values. Options are: none, bz2, gz, zip, zst" compress_file() { @@ -65,8 +65,8 @@ compress_file() { "zip") IMAGE_ZIPPER="pigz --keep --zip" ;; - "zstd") - IMAGE_ZIPPER="zstd --format=zstd -k -q -f --no-progress -o ${filepath}.${compression_format}" + "zst") + IMAGE_ZIPPER="zstd --format=zstd -k -q --no-progress" ;; *) die "Unsupported compression format ${compression_format}" diff --git a/changelog/changes/2022-02-22-configurable-image-compression.md b/changelog/changes/2022-02-22-configurable-image-compression.md index a0a19f82fc..3112faf1a1 100644 --- a/changelog/changes/2022-02-22-configurable-image-compression.md +++ b/changelog/changes/2022-02-22-configurable-image-compression.md @@ -1,2 +1,2 @@ - OpenStack: In addition to the `bz2` image, a `gz` compressed image is published. This allows Glance to directly consume the images by simply passing in the URL of the image. -- SDK: The image compression format is now configurable. Supported formats are: `bz2`, `gz`, `zip`, `none`, `zstd`. Selecting the image format can now be done by passing the `--image_compression_formats` option. This flag gets a comma separated list of formats. +- SDK: The image compression format is now configurable. Supported formats are: `bz2`, `gz`, `zip`, `none`, `zst`. Selecting the image format can now be done by passing the `--image_compression_formats` option. This flag gets a comma separated list of formats. diff --git a/jenkins/vms.sh b/jenkins/vms.sh index e4fe10d701..ac9a8fd7c2 100755 --- a/jenkins/vms.sh +++ b/jenkins/vms.sh @@ -116,7 +116,7 @@ for FORMAT in ${FORMATS}; do COMPRESSION_FORMAT="bz2" if [[ "${FORMAT}" =~ ^(openstack|openstack_mini)$ ]];then - COMPRESSION_FORMAT="gz" + COMPRESSION_FORMAT="gz,bz2" fi script image_to_vm.sh \ @@ -130,7 +130,7 @@ for FORMAT in ${FORMATS}; do --sign_digests="${SIGNING_USER}" \ --download_root="${DOWNLOAD_ROOT}" \ --upload_root="${UPLOAD_ROOT}" \ - --image_compression_format="${COMPRESSION_FORMAT}" \ + --image_compression_formats="${COMPRESSION_FORMAT}" \ --upload \ ${PRIVATE_UPLOAD_OPT} done From 94aa4a6c14c558d8b392517e77ff2528e59b0acf Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 9 Mar 2022 17:19:07 +0200 Subject: [PATCH 6/7] Update jenkins/vms.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Kai Lüke --- jenkins/vms.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/vms.sh b/jenkins/vms.sh index ac9a8fd7c2..132f9a08f1 100755 --- a/jenkins/vms.sh +++ b/jenkins/vms.sh @@ -115,7 +115,7 @@ for FORMAT in ${FORMATS}; do COMPRESSION_FORMAT="bz2" - if [[ "${FORMAT}" =~ ^(openstack|openstack_mini)$ ]];then + if [[ "${FORMAT}" =~ ^(openstack|openstack_mini|digitalocean)$ ]];then COMPRESSION_FORMAT="gz,bz2" fi From dec0d3142082053011e9391281d30df0c5454148 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 9 Mar 2022 17:26:05 +0200 Subject: [PATCH 7/7] Update changelog/changes/2022-02-22-configurable-image-compression.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Kai Lüke --- changelog/changes/2022-02-22-configurable-image-compression.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog/changes/2022-02-22-configurable-image-compression.md b/changelog/changes/2022-02-22-configurable-image-compression.md index 3112faf1a1..a4982bd916 100644 --- a/changelog/changes/2022-02-22-configurable-image-compression.md +++ b/changelog/changes/2022-02-22-configurable-image-compression.md @@ -1,2 +1,3 @@ - OpenStack: In addition to the `bz2` image, a `gz` compressed image is published. This allows Glance to directly consume the images by simply passing in the URL of the image. +- DigitalOcean: In addition to the `bz2` image, a `gz` compressed image is published. This helps against hitting the compression timeout that sometimes lets the image import fail. - SDK: The image compression format is now configurable. Supported formats are: `bz2`, `gz`, `zip`, `none`, `zst`. Selecting the image format can now be done by passing the `--image_compression_formats` option. This flag gets a comma separated list of formats.