From de4eb8f75561d09e7e530db56aa4282de392d7d8 Mon Sep 17 00:00:00 2001 From: Kai Lueke Date: Mon, 8 Apr 2024 11:55:26 +0900 Subject: [PATCH] Set up symlinks for same image artifacts to remove qemu/qemu_uefi_secure The qemu and qemu_uefi_secure images have the same contents as the qemu_uefi image which wastes space on the release server. A similar case is the PXE vmlinuz which is the same as the regular one, too. Set up symlinks for same images, and also detect this when compressing to set up symlinks there as well. To reduce complexity, the qemu and qemu_uefi_secure images are not supported anymore and the Jenkins or GitHub CI will skip over them if specified. Users that build their own images need to adapt, though. --- .github/workflows/ci.yaml | 10 +++++ build_library/release_util.sh | 24 ++++++++++-- build_library/vm_image_util.sh | 39 ++++++++++--------- .../changes/2024-04-08-unify-qemu-images.md | 1 + ci-automation/vms.sh | 3 ++ 5 files changed, 56 insertions(+), 21 deletions(-) create mode 100644 changelog/changes/2024-04-08-unify-qemu-images.md diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b1eb6d61ee..6e07d6d897 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -218,6 +218,9 @@ jobs: formats=$(echo "$formats" | tr ' ' '\n' | sed 's/equinix_metal/packet/g') for format in ${formats}; do + if [ "${format}" = qemu ] || [ "${format}" = qemu_uefi_secure ]; then + continue + fi echo " ################### VENDOR '${format}' ################### " ./run_sdk_container -n "${container_name}" \ ./image_to_vm.sh --format "${format}" --board="${arch}-usr" \ @@ -225,6 +228,13 @@ jobs: --image_compression_formats=bz2 done + # Zip doesn't handle symlinks well, remove them + rm -f artifacts/${arch}-usr/latest/flatcar_production_{qemu,qemu_uefi_secure}_image.img* + # or create an explicit copy: + if [ -e artifacts/${arch}-usr/latest/flatcar_production_pxe.vmlinuz ]; then + rm -f artifacts/${arch}-usr/latest/flatcar_production_pxe.vmlinuz + cp artifacts/${arch}-usr/latest/flatcar_production_{image,pxe}.vmlinuz + fi # upload-artifacts cannot handle artifact uploads from sym-linked directories (no, really) # so we move things around. mkdir -p artifacts/images diff --git a/build_library/release_util.sh b/build_library/release_util.sh index ff7be260b0..c1e518c7cd 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -67,9 +67,21 @@ compress_file() { ;; esac - ${IMAGE_ZIPPER} -f "${filepath}" 2>&1 >/dev/null || die "failed to compress ${filepath}" + # Check if symlink in which case we set up a "compressed" symlink + local compressed_name="${filepath}.${compression_format}" + if [ -L "${filepath}" ]; then + # We could also test if the target exists and otherwise do the compression + # but we might then end up with two different compressed artifacts + local link_target + link_target=$(readlink -f "${filepath}") + local target_basename + target_basename=$(basename "${link_target}") + ln -fs "${target_basename}.${compression_format}" "${compressed_name}" + else + ${IMAGE_ZIPPER} -f "${filepath}" 2>&1 >/dev/null || die "failed to compress ${filepath}" + fi - echo -n "${filepath}.${compression_format}" + echo -n "${compressed_name}" } compress_disk_images() { @@ -85,7 +97,7 @@ compress_disk_images() { # Files that did not match the filter for disk images. local -n local_extra_files="$3" - info "Compressing images" + info "Compressing ${#local_files_to_evaluate[@]} images" # We want to compress images, but we also want to remove the uncompressed files # from the list of uploadable files. for filename in "${local_files_to_evaluate[@]}"; do @@ -98,6 +110,9 @@ compress_disk_images() { # An associative array we set an element on whenever we process a format. # This way we don't process the same format twice. A unique for array elements. + # (But first we need to unset the previous loop or we can only compress a single + # file per list of files). + unset processed_format declare -A processed_format for format in "${FORMATS[@]}";do if [ -z "${processed_format[${format}]}" ]; then @@ -113,7 +128,10 @@ compress_disk_images() { [ "${filename##*/}" != "flatcar_production_image.bin" ] && [ "${filename##*/}" != "flatcar_production_update.bin" ] && ! echo "${FORMATS[@]}" | grep -q "none"; then + info "Removing ${filename}" rm "${filename}" + else + info "Keeping ${filename}" fi else local_extra_files+=( "${filename}" ) diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index fbbe3e197b..d768d922c4 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -23,9 +23,7 @@ VALID_IMG_TYPES=( packet parallels pxe - qemu qemu_uefi - qemu_uefi_secure rackspace rackspace_onmetal rackspace_vhd @@ -126,13 +124,6 @@ IMG_DEFAULT_MEM=2048 IMG_DEFAULT_CPUS=2 ## qemu -IMG_qemu_DISK_FORMAT=qcow2 -IMG_qemu_DISK_LAYOUT=vm -IMG_qemu_CONF_FORMAT=qemu -IMG_qemu_OEM_USE=qemu -IMG_qemu_OEM_PACKAGE=common-oem-files -IMG_qemu_OEM_SYSEXT=oem-qemu - IMG_qemu_uefi_DISK_FORMAT=qcow2 IMG_qemu_uefi_DISK_LAYOUT=vm IMG_qemu_uefi_CONF_FORMAT=qemu_uefi @@ -140,13 +131,6 @@ IMG_qemu_uefi_OEM_USE=qemu IMG_qemu_uefi_OEM_PACKAGE=common-oem-files IMG_qemu_uefi_OEM_SYSEXT=oem-qemu -IMG_qemu_uefi_secure_DISK_FORMAT=qcow2 -IMG_qemu_uefi_secure_DISK_LAYOUT=vm -IMG_qemu_uefi_secure_CONF_FORMAT=qemu_uefi_secure -IMG_qemu_uefi_secure_OEM_USE=qemu -IMG_qemu_uefi_secure_OEM_PACKAGE=common-oem-files -IMG_qemu_uefi_secure_OEM_SYSEXT=oem-qemu - ## xen IMG_xen_CONF_FORMAT=xl @@ -326,7 +310,7 @@ get_default_vm_type() { local board="$1" case "$board" in amd64-usr) - echo "qemu" + echo "qemu_uefi" ;; arm64-usr) echo "qemu_uefi" @@ -603,6 +587,18 @@ write_vm_disk() { info "Writing $disk_format image $(basename "${VM_DST_IMG}")" _write_${disk_format}_disk "${VM_TMP_IMG}" "${VM_DST_IMG}" + # We now only support building qemu_uefi and set up symlinks + # for the qemu and qemu_uefi_secure images + if [ "${VM_IMG_TYPE}" = qemu_uefi ]; then + local qemu="${VM_DST_IMG/qemu_uefi/qemu}" + local qemu_uefi_secure="${VM_DST_IMG/qemu_uefi/qemu_uefi_secure}" + local target_basename + target_basename=$(basename "${VM_DST_IMG}") + ln -fs "${target_basename}" "${qemu}" + ln -fs "${target_basename}" "${qemu_uefi_secure}" + VM_GENERATED_FILES+=( "${qemu}" "${qemu_uefi_secure}" ) + fi + # Add disk image to final file list if it isn't going to be bundled if [[ -z "$(_get_vm_opt BUNDLE_FORMAT)" ]]; then VM_GENERATED_FILES+=( "${VM_DST_IMG}" ) @@ -693,7 +689,7 @@ _write_cpio_disk() { local grub_name="$(_dst_name "_grub.efi")" _write_cpio_common $@ # Pull the kernel and loader out of the filesystem - cp "${base_dir}"/boot/flatcar/vmlinuz-a "${dst_dir}/${vmlinuz_name}" + ln -fs flatcar_production_image.vmlinuz "${dst_dir}/${vmlinuz_name}" local grub_arch case $BOARD in @@ -809,6 +805,13 @@ _write_qemu_uefi_conf() { sed -e "s%^VM_PFLASH_RO=.*%VM_PFLASH_RO='${flash_ro}'%" \ -e "s%^VM_PFLASH_RW=.*%VM_PFLASH_RW='${flash_rw}'%" -i "${script}" VM_GENERATED_FILES+=( "$(_dst_dir)/${flash_ro}" "$(_dst_dir)/${flash_rw}" ) + + # We now only support building qemu_uefi and generate the + # other artifacts from here + if [ "${VM_IMG_TYPE}" = qemu_uefi ]; then + VM_IMG_TYPE=qemu _write_qemu_conf + VM_IMG_TYPE=qemu_uefi_secure _write_qemu_uefi_secure_conf + fi } _write_qemu_uefi_secure_conf() { diff --git a/changelog/changes/2024-04-08-unify-qemu-images.md b/changelog/changes/2024-04-08-unify-qemu-images.md new file mode 100644 index 0000000000..6f7125d556 --- /dev/null +++ b/changelog/changes/2024-04-08-unify-qemu-images.md @@ -0,0 +1 @@ +- SDK: Unified qemu image formats, so that the `qemu_uefi` build target provides the regular `qemu` and the `qemu_uefi_secure` artifacts ([scripts#1847](https://github.com/flatcar/scripts/pull/1847)) diff --git a/ci-automation/vms.sh b/ci-automation/vms.sh index e35bc577a3..7d4d9d3198 100644 --- a/ci-automation/vms.sh +++ b/ci-automation/vms.sh @@ -125,6 +125,9 @@ function _vm_build_impl() { mv "${images_in}" "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest-input" for format in ${formats}; do + if [ "${format}" = qemu ] || [ "${format}" = qemu_uefi_secure ]; then + continue + fi echo " ################### VENDOR '${format}' ################### " COMPRESSION_FORMAT="bz2" if [[ "${format}" =~ ^(openstack_mini|digitalocean)$ ]];then