From f4829fd86007587c5531cd90a285630d6e1d583c Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Mon, 30 Jan 2023 13:25:48 +0100 Subject: [PATCH 1/3] build_library: Add generation of image contents with different details Timestamp and user/group information are out, in are device ID and inode number. That way, the file can be used for accounting size differences of files/image. --- build_library/build_image_util.sh | 36 +++++++++++++++++++++++++---- build_library/dev_container_util.sh | 4 +++- build_library/prod_image_util.sh | 3 +++ 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 409693268b..40fb4e3eda 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -244,12 +244,36 @@ systemd_enable() { write_contents() { info "Writing ${2##*/}" pushd "$1" >/dev/null + # %M - file permissions + # %n - number of hard links to file + # %u - file's user name + # %g - file's group name + # %s - size in bytes + # %Tx - modification time (Y - year, m - month, d - day, H - hours, M - minutes) + # %P - file's path + # %l - symlink target (empty if not a symlink) sudo TZ=UTC find -printf \ '%M %2n %-7u %-7g %7s %TY-%Tm-%Td %TH:%TM ./%P -> %l\n' \ | sed -e 's/ -> $//' > "$2" popd >/dev/null } +# Generate a listing that can be used by other tools to analyze +# image/file size changes. +write_contents_with_technical_details() { + info "Writing ${2##*/}" + pushd "$1" >/dev/null + # %M - file permissions + # %D - ID of a device where file resides + # %i - inode number + # %n - number of hard links to file + # %s - size in bytes + # %P - file's path + sudo find -printf \ + '%M %D %i %n %s ./%P\n' > "$2" + popd >/dev/null +} + # "equery list" a potentially uninstalled board package query_available_package() { local pkg="$1" @@ -624,11 +648,12 @@ finish_image() { local disk_layout="$2" local root_fs_dir="$3" local image_contents="$4" - local image_kernel="$5" - local pcr_policy="$6" - local image_grub="$7" - local image_shim="$8" - local image_kconfig="$9" + local image_contents_wtd="$5" + local image_kernel="$6" + local pcr_policy="$7" + local image_grub="$8" + local image_shim="$9" + local image_kconfig="${10}" local install_grub=0 local disk_img="${BUILD_DIR}/${image_name}" @@ -731,6 +756,7 @@ EOF fi write_contents "${root_fs_dir}" "${BUILD_DIR}/${image_contents}" + write_contents_with_technical_details "${root_fs_dir}" "${BUILD_DIR}/${image_contents_wtd}" # Zero all fs free space to make it more compressible so auto-update # payloads become smaller, not fatal since it won't work on linux < 3.2 diff --git a/build_library/dev_container_util.sh b/build_library/dev_container_util.sh index 7fa195b443..8710b304fd 100755 --- a/build_library/dev_container_util.sh +++ b/build_library/dev_container_util.sh @@ -77,6 +77,7 @@ create_dev_container() { info "Building developer image ${image_name}" local root_fs_dir="${BUILD_DIR}/rootfs" local image_contents="${image_name%.bin}_contents.txt" + local image_contents_wtd="${image_name%.bin}_contents_wtd.txt" local image_packages="${image_name%.bin}_packages.txt" local image_licenses="${image_name%.bin}_licenses.json" @@ -104,7 +105,7 @@ create_dev_container() { # The remount services are provided by coreos-base/coreos-init systemd_enable "${root_fs_dir}" "multi-user.target" "remount-usr.service" - finish_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${image_contents}" + finish_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${image_contents}" "${image_contents_wtd}" declare -a files_to_evaluate declare -a compressed_images @@ -115,6 +116,7 @@ create_dev_container() { upload_image -d "${BUILD_DIR}/${image_name}.DIGESTS" \ "${BUILD_DIR}/${image_contents}" \ + "${BUILD_DIR}/${image_contents_wtd}" \ "${BUILD_DIR}/${image_packages}" \ "${BUILD_DIR}/${image_licenses}" \ "${compressed_images[@]}" \ diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 77502402bb..8f5d514f4b 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -65,6 +65,7 @@ create_prod_image() { info "Building production image ${image_name}" local root_fs_dir="${BUILD_DIR}/rootfs" local image_contents="${image_name%.bin}_contents.txt" + local image_contents_wtd="${image_name%.bin}_contents_wtd.txt" local image_packages="${image_name%.bin}_packages.txt" local image_sbom="${image_name%.bin}_sbom.json" local image_licenses="${image_name%.bin}_licenses.json" @@ -131,6 +132,7 @@ EOF "${disk_layout}" \ "${root_fs_dir}" \ "${image_contents}" \ + "${image_contents_wtd}" \ "${image_kernel}" \ "${image_pcr_policy}" \ "${image_grub}" \ @@ -140,6 +142,7 @@ EOF # Upload local to_upload=( "${BUILD_DIR}/${image_contents}" + "${BUILD_DIR}/${image_contents_wtd}" "${BUILD_DIR}/${image_packages}" "${BUILD_DIR}/${image_sbom}" "${BUILD_DIR}/${image_licenses}" From 1c1c0099c7e4f6283ba917f3af8d7bc8f2f2e7c3 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Mon, 30 Jan 2023 13:41:37 +0100 Subject: [PATCH 2/3] build_library: Generate content files for initrd too --- build_library/build_image_util.sh | 14 +++ .../extract-initramfs-from-vmlinuz.sh | 107 ++++++++++++++++++ build_library/prod_image_util.sh | 8 +- 3 files changed, 128 insertions(+), 1 deletion(-) create mode 100755 build_library/extract-initramfs-from-vmlinuz.sh diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 40fb4e3eda..b399832dcd 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -654,6 +654,8 @@ finish_image() { local image_grub="$8" local image_shim="$9" local image_kconfig="${10}" + local image_initrd_contents="${11}" + local image_initrd_contents_wtd="${12}" local install_grub=0 local disk_img="${BUILD_DIR}/${image_name}" @@ -812,6 +814,18 @@ EOF >"${BUILD_DIR}/pcrs/kernel.config" fi + if [[ -n "${image_initrd_contents}" ]] || [[ -n "${image_initrd_contents_wtd}" ]]; then + "${BUILD_LIBRARY_DIR}/extract-initramfs-from-vmlinuz.sh" "${root_fs_dir}/boot/flatcar/vmlinuz-a" "${BUILD_DIR}/tmp_initrd_contents" + if [[ -n "${image_initrd_contents}" ]]; then + write_contents "${BUILD_DIR}/tmp_initrd_contents" "${BUILD_DIR}/${image_initrd_contents}" + fi + + if [[ -n "${image_initrd_contents_wtd}" ]]; then + write_contents_with_technical_details "${BUILD_DIR}/tmp_initrd_contents" "${BUILD_DIR}/${image_initrd_contents_wtd}" + fi + rm -rf "${BUILD_DIR}/tmp_initrd_contents" + fi + rm -rf "${BUILD_DIR}"/configroot cleanup_mounts "${root_fs_dir}" trap - EXIT diff --git a/build_library/extract-initramfs-from-vmlinuz.sh b/build_library/extract-initramfs-from-vmlinuz.sh new file mode 100755 index 0000000000..9c1ffb6952 --- /dev/null +++ b/build_library/extract-initramfs-from-vmlinuz.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +# Does as it says on the tin. +# +# Example: extract-initramfs-from-vmlinuz /boot/flatcar/vmlinuz-a out-dir +# +# This will create one or more out-dir/rootfs-N directories that contain the contents of the initramfs. + +set -euo pipefail +# check for unzstd. Will abort the script with an error message if the tool is not present. +unzstd -V >/dev/null +fail() { + echo "${*}" >&2 + exit 1 +} + +# Stolen from extract-vmlinux and modified. +try_decompress() { + local header="${1}" + local no_idea="${2}" + local tool="${3}" + local image="${4}" + local tmp="${5}" + local output_basename="${6}" + + local pos + local tool_filename=$(echo "${tool}" | cut -f1 -d' ') + # The obscure use of the "tr" filter is to work around older versions of + # "grep" that report the byte offset of the line instead of the pattern. + + # Try to find the header and decompress from here. + for pos in $(tr "${header}\n${no_idea}" "\n${no_idea}=" < "${image}" | + grep --text --byte-offset --only-matching "^${no_idea}") + do + pos=${pos%%:*} + # Disable error handling, because we will be potentially + # giving the tool garbage or a valid archive with some garbage + # appended to it. So let the tool extract the valid archive + # and then complain about the garbage at the end, but don't + # fail the script because of it. + set +e; tail "-c+${pos}" "${image}" | "${tool}" >"${tmp}/out" 2>/dev/null; set -e; + if [ -s "${tmp}/out" ]; then + mv "${tmp}/out" "${output_basename}-${tool_filename}-at-${pos}" + else + rm -f "${tmp}/out" + fi + done +} + +try_unzstd_decompress() { + local image="${1}" + local tmp="${2}" + local output_basename="${3}" + try_decompress '(\265/\375' xxx unzstd "${image}" "${tmp}" "${output_basename}" +} + +me="${0##*/}" +if [[ $# -ne 2 ]]; then + fail "Usage: ${me} " +fi +image="${1}" +out="${2}" +if [[ ! -s "${image}" ]]; then + fail "The image file '${image}' either does not exist or is empty" +fi +mkdir -p "${out}" + +tmp=$(mktemp --directory /tmp/eifv-XXXXXX) +trap "rm -rf ${tmp}" EXIT + +tmp_dec="${tmp}/decompress" +mkdir "${tmp_dec}" +fr_prefix="${tmp}/first-round" + +ROOTFS_IDX=0 +perform_round() { + local image="${1}" + local tmp_dec="${2}" + local round_prefix="${3}" + try_unzstd_decompress "${image}" "${tmp_dec}" "${round_prefix}" + for rnd in "${round_prefix}"*; do + if [[ $(file --brief "${rnd}") =~ 'cpio archive' ]]; then + mkdir -p "${out}/rootfs-${ROOTFS_IDX}" + while cpio --quiet --extract --make-directories --directory="${out}/rootfs-${ROOTFS_IDX}" --nonmatching 'dev/*'; do + ROOTFS_IDX=$(( ROOTFS_IDX + 1 )) + mkdir -p "${out}/rootfs-${ROOTFS_IDX}" + done <${rnd} + rmdir "${out}/rootfs-${ROOTFS_IDX}" + fi + done +} + +shopt -s nullglob +perform_round "${image}" "${tmp_dec}" "${fr_prefix}" +for fr in "${fr_prefix}"*; do + fr_files="${fr}-files" + fr_dec="${fr_files}/decompress" + mkdir -p "${fr_dec}" + sr_prefix="${fr_files}/second-round" + perform_round "${fr}" "${fr_dec}" "${sr_prefix}" +done + +if [[ ${ROOTFS_IDX} -eq 0 ]]; then + fail "no initramfs found in ${image}" +fi + +echo "done, found ${ROOTFS_IDX} rootfs(es)" diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 8f5d514f4b..00005440e5 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -74,6 +74,8 @@ create_prod_image() { local image_pcr_policy="${image_name%.bin}_pcr_policy.zip" local image_grub="${image_name%.bin}.grub" local image_shim="${image_name%.bin}.shim" + local image_initrd_contents="${image_name%.bin}_initrd_contents.txt" + local image_initrd_contents_wtd="${image_name%.bin}_initrd_contents_wtd.txt" start_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${update_group}" @@ -137,7 +139,9 @@ EOF "${image_pcr_policy}" \ "${image_grub}" \ "${image_shim}" \ - "${image_kconfig}" + "${image_kconfig}" \ + "${image_initrd_contents}" \ + "${image_initrd_contents_wtd}" # Upload local to_upload=( @@ -150,6 +154,8 @@ EOF "${BUILD_DIR}/${image_pcr_policy}" "${BUILD_DIR}/${image_grub}" "${BUILD_DIR}/${image_kconfig}" + "${BUILD_DIR}/${image_initrd_contents}" + "${BUILD_DIR}/${image_initrd_contents_wtd}" ) local files_to_evaluate=( "${BUILD_DIR}/${image_name}" ) From edc90b4e5916d760b9343988dd5842f7e358fea0 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Wed, 1 Feb 2023 14:25:07 +0100 Subject: [PATCH 3/3] build_library: Add generation of disk space usage This could replace an ad-hoc calculations we do in package-diff. --- build_library/build_image_util.sh | 67 ++++++++++++++++++++++++------- build_library/prod_image_util.sh | 5 ++- 2 files changed, 56 insertions(+), 16 deletions(-) diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index b399832dcd..8ef0866f01 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -274,6 +274,30 @@ write_contents_with_technical_details() { popd >/dev/null } +# Generate a report like the following: +# +# File Size Used Avail Use% Type +# /boot 127M 62M 65M 50% vfat +# /usr 983M 721M 212M 78% ext2 +# / 6,0G 13M 5,6G 1% ext4 +# SUM 7,0G 796M 5,9G 12% - +write_disk_space_usage() { + info "Writing ${2##*/}" + pushd "${1}" >/dev/null + # The sed's first command turns './' into '/ ', second + # command replaces '- ' with 'SUM' for the total row. All this to + # keep the numbers neatly aligned in columns. + sudo df \ + --human-readable \ + --total \ + --output='file,size,used,avail,pcent,fstype' \ + ./boot ./usr ./ | \ + sed \ + -e 's#^\.\(/[^ ]*\)#\1 #' \ + -e 's/^- /SUM/' >"${2}" + popd >/dev/null +} + # "equery list" a potentially uninstalled board package query_available_package() { local pkg="$1" @@ -656,6 +680,7 @@ finish_image() { local image_kconfig="${10}" local image_initrd_contents="${11}" local image_initrd_contents_wtd="${12}" + local image_disk_space_usage="${13}" local install_grub=0 local disk_img="${BUILD_DIR}/${image_name}" @@ -757,9 +782,6 @@ EOF "${BUILD_DIR}/${image_kconfig}" fi - write_contents "${root_fs_dir}" "${BUILD_DIR}/${image_contents}" - write_contents_with_technical_details "${root_fs_dir}" "${BUILD_DIR}/${image_contents_wtd}" - # Zero all fs free space to make it more compressible so auto-update # payloads become smaller, not fatal since it won't work on linux < 3.2 sudo fstrim "${root_fs_dir}" || true @@ -814,18 +836,6 @@ EOF >"${BUILD_DIR}/pcrs/kernel.config" fi - if [[ -n "${image_initrd_contents}" ]] || [[ -n "${image_initrd_contents_wtd}" ]]; then - "${BUILD_LIBRARY_DIR}/extract-initramfs-from-vmlinuz.sh" "${root_fs_dir}/boot/flatcar/vmlinuz-a" "${BUILD_DIR}/tmp_initrd_contents" - if [[ -n "${image_initrd_contents}" ]]; then - write_contents "${BUILD_DIR}/tmp_initrd_contents" "${BUILD_DIR}/${image_initrd_contents}" - fi - - if [[ -n "${image_initrd_contents_wtd}" ]]; then - write_contents_with_technical_details "${BUILD_DIR}/tmp_initrd_contents" "${BUILD_DIR}/${image_initrd_contents_wtd}" - fi - rm -rf "${BUILD_DIR}/tmp_initrd_contents" - fi - rm -rf "${BUILD_DIR}"/configroot cleanup_mounts "${root_fs_dir}" trap - EXIT @@ -868,4 +878,31 @@ EOF popd >/dev/null rm -rf "${BUILD_DIR}/pcrs" fi + + # Mount the final image again, as readonly, to generate some reports. + "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout="${disk_layout}" \ + mount --read_only "${disk_img}" "${root_fs_dir}" + trap "cleanup_mounts '${root_fs_dir}'" EXIT + + write_contents "${root_fs_dir}" "${BUILD_DIR}/${image_contents}" + write_contents_with_technical_details "${root_fs_dir}" "${BUILD_DIR}/${image_contents_wtd}" + + if [[ -n "${image_initrd_contents}" ]] || [[ -n "${image_initrd_contents_wtd}" ]]; then + "${BUILD_LIBRARY_DIR}/extract-initramfs-from-vmlinuz.sh" "${root_fs_dir}/boot/flatcar/vmlinuz-a" "${BUILD_DIR}/tmp_initrd_contents" + if [[ -n "${image_initrd_contents}" ]]; then + write_contents "${BUILD_DIR}/tmp_initrd_contents" "${BUILD_DIR}/${image_initrd_contents}" + fi + + if [[ -n "${image_initrd_contents_wtd}" ]]; then + write_contents_with_technical_details "${BUILD_DIR}/tmp_initrd_contents" "${BUILD_DIR}/${image_initrd_contents_wtd}" + fi + rm -rf "${BUILD_DIR}/tmp_initrd_contents" + fi + + if [[ -n "${image_disk_space_usage}" ]]; then + write_disk_space_usage "${root_fs_dir}" "${BUILD_DIR}/${image_disk_space_usage}" + fi + + cleanup_mounts "${root_fs_dir}" + trap - EXIT } diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 00005440e5..723f09a5c8 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -76,6 +76,7 @@ create_prod_image() { local image_shim="${image_name%.bin}.shim" local image_initrd_contents="${image_name%.bin}_initrd_contents.txt" local image_initrd_contents_wtd="${image_name%.bin}_initrd_contents_wtd.txt" + local image_disk_usage="${image_name%.bin}_disk_usage.txt" start_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${update_group}" @@ -141,7 +142,8 @@ EOF "${image_shim}" \ "${image_kconfig}" \ "${image_initrd_contents}" \ - "${image_initrd_contents_wtd}" + "${image_initrd_contents_wtd}" \ + "${image_disk_usage}" # Upload local to_upload=( @@ -156,6 +158,7 @@ EOF "${BUILD_DIR}/${image_kconfig}" "${BUILD_DIR}/${image_initrd_contents}" "${BUILD_DIR}/${image_initrd_contents_wtd}" + "${BUILD_DIR}/${image_disk_usage}" ) local files_to_evaluate=( "${BUILD_DIR}/${image_name}" )