From 8bfad0d8f7c7a8be4527e2e1d1e4b9ec4582d40a Mon Sep 17 00:00:00 2001 From: Kai Lueke Date: Mon, 8 Apr 2024 11:55:26 +0900 Subject: [PATCH] Set up symlinks for same image artifacts to remove qemu/qemu_uefi_secure The qemu and qemu_uefi_secure images have the same contents as the qemu_uefi image which wastes space on the release server. A similar case is the PXE vmlinuz which is the same as the regular one, too. Set up symlinks for same images, and also detect this when compressing to set up symlinks there as well. To reduce complexity, the qemu and qemu_uefi_secure images are not supported anymore and the Jenkins or GitHub CI will skip over them if specified. Users that build their own images need to adapt, though. --- .github/workflows/ci.yaml | 10 ++++++ build_library/release_util.sh | 24 +++++++++++-- build_library/vm_image_util.sh | 35 +++++++++++-------- .../changes/2024-04-08-unify-qemu-images.md | 1 + ci-automation/vms.sh | 3 ++ 5 files changed, 56 insertions(+), 17 deletions(-) create mode 100644 changelog/changes/2024-04-08-unify-qemu-images.md diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 349c1bb06e..0609a676b2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -204,6 +204,9 @@ jobs: formats=$(echo "$formats" | tr ' ' '\n' | sed 's/equinix_metal/packet/g') for format in ${formats}; do + if [ "${format}" = qemu ] || [ "${format}" = qemu_uefi_secure ]; then + continue + fi echo " ################### VENDOR '${format}' ################### " ./run_sdk_container -n "${container_name}" \ ./image_to_vm.sh --format "${format}" --board="${arch}-usr" \ @@ -211,6 +214,13 @@ jobs: --image_compression_formats=bz2 done + # Zip doesn't handle symlinks well, remove them + rm -f artifacts/${arch}-usr/latest/flatcar_production_{qemu,qemu_uefi_secure}_image.img* + # or create an explicit copy: + if [ -e artifacts/${arch}-usr/latest/flatcar_production_pxe.vmlinuz ]; then + rm -f artifacts/${arch}-usr/latest/flatcar_production_pxe.vmlinuz + cp artifacts/${arch}-usr/latest/flatcar_production_{image,pxe}.vmlinuz + fi # upload-artifacts cannot handle artifact uploads from sym-linked directories (no, really) # so we move things around. mkdir -p artifacts/images diff --git a/build_library/release_util.sh b/build_library/release_util.sh index 16d33f2c93..659ab3136f 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -75,9 +75,21 @@ compress_file() { ;; esac - ${IMAGE_ZIPPER} -f "${filepath}" 2>&1 >/dev/null || die "failed to compress ${filepath}" + # Check if symlink in which case we set up a "compressed" symlink + local compressed_name="${filepath}.${compression_format}" + if [ -L "${filepath}" ]; then + # We could also test if the target exists and otherwise do the compression + # but we might then end up with two different compressed artifacts + local link_target + link_target=$(readlink -f "${filepath}") + local target_basename + target_basename=$(basename "${link_target}") + ln -fs "${target_basename}.${compression_format}" "${compressed_name}" + else + ${IMAGE_ZIPPER} -f "${filepath}" 2>&1 >/dev/null || die "failed to compress ${filepath}" + fi - echo -n "${filepath}.${compression_format}" + echo -n "${compressed_name}" } compress_disk_images() { @@ -93,7 +105,7 @@ compress_disk_images() { # Files that did not match the filter for disk images. local -n local_extra_files="$3" - info "Compressing images" + info "Compressing ${#local_files_to_evaluate[@]} images" # We want to compress images, but we also want to remove the uncompressed files # from the list of uploadable files. for filename in "${local_files_to_evaluate[@]}"; do @@ -106,6 +118,9 @@ compress_disk_images() { # An associative array we set an element on whenever we process a format. # This way we don't process the same format twice. A unique for array elements. + # (But first we need to unset the previous loop or we can only compress a single + # file per list of files). + unset processed_format declare -A processed_format for format in "${FORMATS[@]}";do if [ -z "${processed_format[${format}]}" ]; then @@ -121,7 +136,10 @@ compress_disk_images() { [ "${filename##*/}" != "flatcar_production_image.bin" ] && [ "${filename##*/}" != "flatcar_production_update.bin" ] && ! echo "${FORMATS[@]}" | grep -q "none"; then + info "Removing ${filename}" rm "${filename}" + else + info "Keeping ${filename}" fi else local_extra_files+=( "${filename}" ) diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index 7d96d3c8d4..eb90e02ef3 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -24,9 +24,7 @@ VALID_IMG_TYPES=( packet parallels pxe - qemu qemu_uefi - qemu_uefi_secure rackspace rackspace_onmetal rackspace_vhd @@ -120,21 +118,11 @@ IMG_DEFAULT_MEM=1024 IMG_DEFAULT_CPUS=2 ## qemu -IMG_qemu_DISK_FORMAT=qcow2 -IMG_qemu_DISK_LAYOUT=vm -IMG_qemu_CONF_FORMAT=qemu -IMG_qemu_OEM_PACKAGE=oem-qemu - IMG_qemu_uefi_DISK_FORMAT=qcow2 IMG_qemu_uefi_DISK_LAYOUT=vm IMG_qemu_uefi_CONF_FORMAT=qemu_uefi IMG_qemu_uefi_OEM_PACKAGE=oem-qemu -IMG_qemu_uefi_secure_DISK_FORMAT=qcow2 -IMG_qemu_uefi_secure_DISK_LAYOUT=vm -IMG_qemu_uefi_secure_CONF_FORMAT=qemu_uefi_secure -IMG_qemu_uefi_secure_OEM_PACKAGE=oem-qemu - ## xen IMG_xen_CONF_FORMAT=xl @@ -298,7 +286,7 @@ get_default_vm_type() { local board="$1" case "$board" in amd64-usr) - echo "qemu" + echo "qemu_uefi" ;; arm64-usr) echo "qemu_uefi" @@ -537,6 +525,18 @@ write_vm_disk() { info "Writing $disk_format image $(basename "${VM_DST_IMG}")" _write_${disk_format}_disk "${VM_TMP_IMG}" "${VM_DST_IMG}" + # We now only support building qemu_uefi and set up symlinks + # for the qemu and qemu_uefi_secure images + if [ "${VM_IMG_TYPE}" = qemu_uefi ]; then + local qemu="${VM_DST_IMG/qemu_uefi/qemu}" + local qemu_uefi_secure="${VM_DST_IMG/qemu_uefi/qemu_uefi_secure}" + local target_basename + target_basename=$(basename "${VM_DST_IMG}") + ln -fs "${target_basename}" "${qemu}" + ln -fs "${target_basename}" "${qemu_uefi_secure}" + VM_GENERATED_FILES+=( "${qemu}" "${qemu_uefi_secure}" ) + fi + # Add disk image to final file list if it isn't going to be bundled if [[ -z "$(_get_vm_opt BUNDLE_FORMAT)" ]]; then VM_GENERATED_FILES+=( "${VM_DST_IMG}" ) @@ -622,7 +622,7 @@ _write_cpio_disk() { local grub_name="$(_dst_name "_grub.efi")" _write_cpio_common $@ # Pull the kernel and loader out of the filesystem - cp "${base_dir}"/boot/flatcar/vmlinuz-a "${dst_dir}/${vmlinuz_name}" + ln -fs flatcar_production_image.vmlinuz "${dst_dir}/${vmlinuz_name}" local grub_arch case $BOARD in @@ -742,6 +742,13 @@ _write_qemu_uefi_conf() { sed -e "s%^VM_PFLASH_RO=.*%VM_PFLASH_RO='${flash_ro}'%" \ -e "s%^VM_PFLASH_RW=.*%VM_PFLASH_RW='${flash_rw}'%" -i "${script}" VM_GENERATED_FILES+=( "$(_dst_dir)/${flash_ro}" "$(_dst_dir)/${flash_rw}" ) + + # We now only support building qemu_uefi and generate the + # other artifacts from here + if [ "${VM_IMG_TYPE}" = qemu_uefi ]; then + VM_IMG_TYPE=qemu _write_qemu_conf + VM_IMG_TYPE=qemu_uefi_secure _write_qemu_uefi_secure_conf + fi } _write_qemu_uefi_secure_conf() { diff --git a/changelog/changes/2024-04-08-unify-qemu-images.md b/changelog/changes/2024-04-08-unify-qemu-images.md new file mode 100644 index 0000000000..6f7125d556 --- /dev/null +++ b/changelog/changes/2024-04-08-unify-qemu-images.md @@ -0,0 +1 @@ +- SDK: Unified qemu image formats, so that the `qemu_uefi` build target provides the regular `qemu` and the `qemu_uefi_secure` artifacts ([scripts#1847](https://github.com/flatcar/scripts/pull/1847)) diff --git a/ci-automation/vms.sh b/ci-automation/vms.sh index 45e3698589..2594ebe83b 100644 --- a/ci-automation/vms.sh +++ b/ci-automation/vms.sh @@ -115,6 +115,9 @@ function _vm_build_impl() { mv "${images_in}" "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest-input" for format in ${formats}; do + if [ "${format}" = qemu ] || [ "${format}" = qemu_uefi_secure ]; then + continue + fi echo " ################### VENDOR '${format}' ################### " COMPRESSION_FORMAT="bz2" if [[ "${format}" =~ ^(openstack_mini|digitalocean)$ ]];then