diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index a31931cb82..e99d266b05 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -58,10 +58,22 @@ extract_update() { local image_name="$1" local disk_layout="$2" local update_path="${BUILD_DIR}/${image_name%_image.bin}_update.bin" + local digest_path="${update_path}.DIGESTS" "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout="${disk_layout}" \ extract "${BUILD_DIR}/${image_name}" "USR-A" "${update_path}" - upload_image "${update_path}" + + # Compress image + files_to_evaluate+=( "${update_path}" ) + declare -a compressed_images + declare -a extra_files + compress_disk_images files_to_evaluate compressed_images extra_files + + # Upload compressed image + upload_image -d "${digest_path}" "${compressed_images[@]}" "${extra_files[@]}" + + # Upload legacy digests + upload_legacy_digests "${digest_path}" compressed_images # For production as well as dev builds we generate a dev-key-signed update # payload for running tests (the signature won't be accepted by production systems). @@ -105,7 +117,18 @@ generate_update() { -new_kernel "${image_kernel}" \ -out_file "${update}.gz" - upload_image -d "${update}.DIGESTS" "${update}".{bin,gz,zip} + # Compress image + declare -a files_to_evaluate + declare -a compressed_images + declare -a extra_files + files_to_evaluate+=( "${update}.bin" ) + compress_disk_images files_to_evaluate compressed_images extra_files + + # Upload images + upload_image -d "${update}.DIGESTS" "${update}".{gz,zip} "${compressed_images[@]}" "${extra_files[@]}" + + # Upload legacy digests + upload_legacy_digests "${update}.DIGESTS" compressed_images } # ldconfig cannot generate caches for non-native arches. diff --git a/build_library/dev_container_util.sh b/build_library/dev_container_util.sh index 9e8f9d8346..34a028baef 100755 --- a/build_library/dev_container_util.sh +++ b/build_library/dev_container_util.sh @@ -108,9 +108,21 @@ create_dev_container() { systemd_enable "${root_fs_dir}" "multi-user.target" "remount-usr.service" finish_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${image_contents}" - upload_image -d "${BUILD_DIR}/${image_name}.bz2.DIGESTS" \ + + declare -a files_to_evaluate + declare -a compressed_images + declare -a extra_files + + files_to_evaluate+=( "${BUILD_DIR}/${image_name}" ) + compress_disk_images files_to_evaluate compressed_images extra_files + + upload_image -d "${BUILD_DIR}/${image_name}.DIGESTS" \ "${BUILD_DIR}/${image_contents}" \ "${BUILD_DIR}/${image_packages}" \ "${BUILD_DIR}/${image_licenses}" \ - "${BUILD_DIR}/${image_name}" + "${compressed_images[@]}" \ + "${extra_files[@]}" + + # Upload legacy digests + upload_legacy_digests "${BUILD_DIR}/${image_name}.DIGESTS" compressed_images } diff --git a/build_library/modify_image_util.sh b/build_library/modify_image_util.sh index f8ed535762..53fa0cc81e 100755 --- a/build_library/modify_image_util.sh +++ b/build_library/modify_image_util.sh @@ -83,7 +83,21 @@ finish_modify_image() { cleanup_mounts "${ROOT_FS_DIR}" trap - EXIT - upload_image "${DST_IMAGE}" + + declare -a files_to_evaluate + declare -a compressed_images + declare -a extra_files + + files_to_evaluate+=( "${DST_IMAGE}" ) + compress_disk_images files_to_evaluate compressed_images extra_files + + upload_image -d "${DST_IMAGE}.DIGESTS" \ + "${compressed_images[@]}" \ + "${extra_files[@]}" + + # Upload legacy digests + upload_legacy_digests "${DST_IMAGE}.DIGESTS" compressed_images + for filename in "${EXTRA_FILES[@]}"; do if [[ -e "${BUILD_DIR}/${filename}" ]]; then upload_image "${BUILD_DIR}/${filename}" diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 509e350aa4..935c9aaf49 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -138,17 +138,27 @@ EOF "${BUILD_DIR}/${image_contents}" "${BUILD_DIR}/${image_packages}" "${BUILD_DIR}/${image_licenses}" - "${BUILD_DIR}/${image_name}" "${BUILD_DIR}/${image_kernel}" "${BUILD_DIR}/${image_pcr_policy}" "${BUILD_DIR}/${image_grub}" "${BUILD_DIR}/${image_kconfig}" ) + + local files_to_evaluate=( "${BUILD_DIR}/${image_name}" ) + declare -a compressed_images + declare -a extra_files + compress_disk_images files_to_evaluate compressed_images extra_files + to_upload+=( "${compressed_images[@]}" ) + to_upload+=( "${extra_files[@]}" ) + # FIXME(bgilbert): no shim on arm64 if [[ -f "${BUILD_DIR}/${image_shim}" ]]; then to_upload+=("${BUILD_DIR}/${image_shim}") fi - upload_image -d "${BUILD_DIR}/${image_name}.bz2.DIGESTS" "${to_upload[@]}" + upload_image -d "${BUILD_DIR}/${image_name}.DIGESTS" "${to_upload[@]}" + + # Upload legacy digests + upload_legacy_digests "${BUILD_DIR}/${image_name}.DIGESTS" compressed_images } create_prod_tar() { diff --git a/build_library/release_util.sh b/build_library/release_util.sh index 9905645c33..56c6ca0d71 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -40,16 +40,18 @@ DEFINE_string sign "" \ "Sign all files to be uploaded with the given GPG key." DEFINE_string sign_digests "" \ "Sign image DIGESTS files with the given GPG key." -DEFINE_string image_compression_format "${DEFAULT_IMAGE_COMPRESSION_FORMAT}" \ - "Compress the resulting images using this format. Options are: none, bz2, gz, zip" +DEFINE_string image_compression_formats "${DEFAULT_IMAGE_COMPRESSION_FORMAT}" \ + "Compress the resulting images using thise formats. This option acceps a list of comma separated values. Options are: none, bz2, gz, zip, zstd" compress_file() { local filepath="$1" + local compression_format="$2" [ ! -f "${filepath}" ] && die "Image file ${filepath} does not exist" + [ -z "${compression_format}" ] && die "compression format parameter is mandatory" - case "${FLAGS_image_compression_format}" in + case "${compression_format}" in "none"|"") echo -n "${filepath}" return 0 @@ -63,14 +65,77 @@ compress_file() { "zip") IMAGE_ZIPPER="pigz --keep --zip" ;; + "zstd") + IMAGE_ZIPPER="zstd --format=zstd -k -q -f --no-progress -o ${filepath}.${compression_format}" + ;; *) - die "Unsupported compression format ${FLAGS_image_compression_format}" + die "Unsupported compression format ${compression_format}" ;; esac ${IMAGE_ZIPPER} -f "${filepath}" 2>&1 >/dev/null || die "failed to compress ${filepath}" - echo -n "${filepath}.${FLAGS_image_compression_format}" + echo -n "${filepath}.${compression_format}" +} + +compress_disk_images() { + # An array of files that are to be evaluated and possibly compressed if images are + # among them. + local -n local_files_to_evaluate="$1" + + # An array that will hold the path on disk to the resulting disk image archives. + # Multiple compression formats may be requested, so this array may hold + # multiple archives for the same image. + local -n local_resulting_archives="$2" + + # Files that did not match the filter for disk images. + local -n local_extra_files="$3" + + info "Compressing images" + # We want to compress images, but we also want to remove the uncompressed files + # from the list of uploadable files. + for filename in "${local_files_to_evaluate[@]}"; do + if [[ "${filename}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then + # Parse the formats as an array. This will yield an extra empty + # array element at the end. + readarray -td, FORMATS<<<"${FLAGS_image_compression_formats}," + # unset the last element + unset 'FORMATS[-1]' + + # An associative array we set an element on whenever we process a format. + # This way we don't process the same format twice. A unique for array elements. + declare -A processed_format + for format in "${FORMATS[@]}";do + if [ -z "${processed_format[${format}]}" ]; then + info "Compressing ${filename##*/} to ${format}" + COMPRESSED_FILENAME=$(compress_file "${filename}" "${format}") + local_resulting_archives+=( "$COMPRESSED_FILENAME" ) + processed_format["${format}"]=1 + fi + done + else + local_extra_files+=( "${filename}" ) + fi + done +} + +upload_legacy_digests() { + [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0 + + local local_digest_file="$1" + local -n local_compressed_files="$2" + + [[ "${#local_compressed_files[@]}" -gt 0 ]] || return 0 + + # Upload legacy digests + declare -a digests_to_upload + for file in "${local_compressed_files[@]}";do + legacy_digest_file="${file}.DIGESTS" + cp "${local_digest_file}" "${legacy_digest_file}" + digests_to_upload+=( "${legacy_digest_file}" ) + done + local def_upload_path="${UPLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}" + upload_files "digests" "${def_upload_path}" "" "${digests_to_upload[@]}" } check_gsutil_opts() { @@ -243,15 +308,30 @@ upload_image() { if [[ ! -f "${filename}" ]]; then die "File '${filename}' does not exist!" fi - + uploads+=( "${filename}" ) # Compress disk images - if [[ "${filename}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then - info "Compressing ${filename##*/}" - COMPRESSED_FILENAME=$(compress_file "${filename}") - uploads+=( "$COMPRESSED_FILENAME" ) - else - uploads+=( "${filename}" ) - fi + #if [[ "${filename}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then + # # Parse the formats as an array. This will yield an extra empty + # # array element at the end. + # readarray -td, FORMATS <<<"${FLAGS_image_compression_formats}," + # # unset the last element + # unset 'FORMATS[-1]' + # + # # An associative array we set an element on whenever we process a format. + # # This way we don't process the same format twice. A unique for array elements. + # declare -A processed_format + # for format in ${FORMATS[@]} + # do + # if [ ! -z ${processed_format[${format}]} ]; then + # info "Compressing ${filename##*/} to ${format}" + # COMPRESSED_FILENAME=$(compress_file "${filename}" "${format}") + # uploads+=( "$COMPRESSED_FILENAME" ) + # processed_format[${format}]=1 + # fi + # done + #else + # uploads+=( "${filename}" ) + #fi done if [[ -z "${digests}" ]]; then diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index a2798c8d76..27e23b41fc 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -1199,47 +1199,46 @@ vm_cleanup() { } vm_upload() { + + declare -a legacy_uploads + declare -a uploadable_files + declare -a compressed_images + declare -a image_files + declare -a digest_uploads + + compress_disk_images VM_GENERATED_FILES compressed_images uploadable_files + + if [ "${#compressed_images[@]}" -gt 0 ]; then + uploadable_files+=( "${compressed_images[@]}" ) + legacy_uploads+=( "${compressed_images[@]}" ) + fi + local digests="$(_dst_dir)/$(_dst_name .DIGESTS)" - upload_image -d "${digests}" "${VM_GENERATED_FILES[@]}" + upload_image -d "${digests}" "${uploadable_files[@]}" [[ -e "${digests}" ]] || return 0 - # FIXME(marineam): Temporary alternate name for .DIGESTS - # This used to be derived from the first file listed in - # ${VM_GENERATED_FILES[@]}", usually $VM_DST_IMG or similar. - # Since not everything actually uploads $VM_DST_IMG this was not very - # consistent and relying on ordering was breakable. - # Now the common prefix, output by $(_dst_name) is used above. - # Some download/install scripts may still refer to the old name. - local uploaded legacy_uploaded - for uploaded in "${VM_GENERATED_FILES[@]}"; do - if [[ "${uploaded}" == "${VM_DST_IMG}" ]]; then - legacy_uploaded="$(_dst_dir)/$(basename ${VM_DST_IMG})" - break + # Since depending on the ordering of $VM_GENERATED_FILES is brittle only + # use it if $VM_DST_IMG isn't included in the uploaded files. + if [ "${#legacy_uploads[@]}" -eq 0 ];then + legacy_uploads+=( "${VM_GENERATED_FILES[0]}" ) + fi + + for legacy_upload in "${legacy_uploads[@]}";do + local legacy_digest_file="${legacy_upload}.DIGESTS" + [[ "${legacy_digest_file}" == "${digests}" ]] && continue + + cp "${digests}" "${legacy_digest_file}" + digest_uploads+=( "${legacy_digest_file}" ) + + if [[ -e "${digests}.asc" ]]; then + digest_uploads+=( "${legacy_digest_file}.asc" ) + cp "${digests}.asc" "${legacy_digest_file}.asc" fi done - # Since depending on the ordering of $VM_GENERATED_FILES is brittle only - # use it if $VM_DST_IMG isn't included in the uploaded files. - if [[ -z "${legacy_uploaded}" ]]; then - legacy_uploaded="${VM_GENERATED_FILES[0]}" - fi - - # If upload_images compressed $legacy_uploaded be sure to add .bz2 - if [[ "${legacy_uploaded}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then - if ! [[ "${FLAGS_image_compression_format}" =~ ^(""|none)$ ]]; then - legacy_uploaded+=".${FLAGS_image_compression_format}" - fi - fi - - local legacy_digests="${legacy_uploaded}.DIGESTS" - [[ "${legacy_digests}" != "${digests}" ]] || return 0 - - local legacy_uploads=( "${legacy_digests}" ) - cp "${digests}" "${legacy_digests}" - if [[ -e "${digests}.asc" ]]; then - legacy_uploads+=( "${legacy_digests}.asc" ) - cp "${digests}.asc" "${legacy_digests}.asc" + if [ "${#digest_uploads[@]}" -gt 0 ];then + legacy_uploads+=( "${digest_uploads[@]}" ) fi local def_upload_path="${UPLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}" diff --git a/changelog/changes/2022-02-22-configurable-image-compression.md b/changelog/changes/2022-02-22-configurable-image-compression.md index a06c7a31eb..a0a19f82fc 100644 --- a/changelog/changes/2022-02-22-configurable-image-compression.md +++ b/changelog/changes/2022-02-22-configurable-image-compression.md @@ -1,2 +1,2 @@ -- Image compression format is now configurable. Supported formats are: bz2, gz, zip, none. Selecting the image format can now be done by passing the ```--image_compression_format``` option to ```image_to_vm.sh``` -- The Jenkins ```vms.sh``` script sets the image compression format to ```gz``` for OpenStack images, which allows Glance to directly consume the images by simply passing in the URL of the image. +- OpenStack: In addition to the `bz2` image, a `gz` compressed image is published. This allows Glance to directly consume the images by simply passing in the URL of the image. +- SDK: The image compression format is now configurable. Supported formats are: `bz2`, `gz`, `zip`, `none`, `zstd`. Selecting the image format can now be done by passing the `--image_compression_formats` option. This flag gets a comma separated list of formats. diff --git a/jenkins/vms.sh b/jenkins/vms.sh index 06e3972eb2..9b6aad48f8 100755 --- a/jenkins/vms.sh +++ b/jenkins/vms.sh @@ -130,7 +130,7 @@ for FORMAT in ${FORMATS}; do --sign_digests="${SIGNING_USER}" \ --download_root="${DOWNLOAD_ROOT}" \ --upload_root="${UPLOAD_ROOT}" \ - --image_compression_format="${COMPRESSION_FORMAT}" + --image_compression_format="${COMPRESSION_FORMAT}" \ --upload \ ${PRIVATE_UPLOAD_OPT} done