diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 409693268b..8ef0866f01 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -244,12 +244,60 @@ systemd_enable() { write_contents() { info "Writing ${2##*/}" pushd "$1" >/dev/null + # %M - file permissions + # %n - number of hard links to file + # %u - file's user name + # %g - file's group name + # %s - size in bytes + # %Tx - modification time (Y - year, m - month, d - day, H - hours, M - minutes) + # %P - file's path + # %l - symlink target (empty if not a symlink) sudo TZ=UTC find -printf \ '%M %2n %-7u %-7g %7s %TY-%Tm-%Td %TH:%TM ./%P -> %l\n' \ | sed -e 's/ -> $//' > "$2" popd >/dev/null } +# Generate a listing that can be used by other tools to analyze +# image/file size changes. +write_contents_with_technical_details() { + info "Writing ${2##*/}" + pushd "$1" >/dev/null + # %M - file permissions + # %D - ID of a device where file resides + # %i - inode number + # %n - number of hard links to file + # %s - size in bytes + # %P - file's path + sudo find -printf \ + '%M %D %i %n %s ./%P\n' > "$2" + popd >/dev/null +} + +# Generate a report like the following: +# +# File Size Used Avail Use% Type +# /boot 127M 62M 65M 50% vfat +# /usr 983M 721M 212M 78% ext2 +# / 6,0G 13M 5,6G 1% ext4 +# SUM 7,0G 796M 5,9G 12% - +write_disk_space_usage() { + info "Writing ${2##*/}" + pushd "${1}" >/dev/null + # The sed's first command turns './' into '/ ', second + # command replaces '- ' with 'SUM' for the total row. All this to + # keep the numbers neatly aligned in columns. + sudo df \ + --human-readable \ + --total \ + --output='file,size,used,avail,pcent,fstype' \ + ./boot ./usr ./ | \ + sed \ + -e 's#^\.\(/[^ ]*\)#\1 #' \ + -e 's/^- /SUM/' >"${2}" + popd >/dev/null +} + # "equery list" a potentially uninstalled board package query_available_package() { local pkg="$1" @@ -624,11 +672,15 @@ finish_image() { local disk_layout="$2" local root_fs_dir="$3" local image_contents="$4" - local image_kernel="$5" - local pcr_policy="$6" - local image_grub="$7" - local image_shim="$8" - local image_kconfig="$9" + local image_contents_wtd="$5" + local image_kernel="$6" + local pcr_policy="$7" + local image_grub="$8" + local image_shim="$9" + local image_kconfig="${10}" + local image_initrd_contents="${11}" + local image_initrd_contents_wtd="${12}" + local image_disk_space_usage="${13}" local install_grub=0 local disk_img="${BUILD_DIR}/${image_name}" @@ -730,8 +782,6 @@ EOF "${BUILD_DIR}/${image_kconfig}" fi - write_contents "${root_fs_dir}" "${BUILD_DIR}/${image_contents}" - # Zero all fs free space to make it more compressible so auto-update # payloads become smaller, not fatal since it won't work on linux < 3.2 sudo fstrim "${root_fs_dir}" || true @@ -828,4 +878,31 @@ EOF popd >/dev/null rm -rf "${BUILD_DIR}/pcrs" fi + + # Mount the final image again, as readonly, to generate some reports. + "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout="${disk_layout}" \ + mount --read_only "${disk_img}" "${root_fs_dir}" + trap "cleanup_mounts '${root_fs_dir}'" EXIT + + write_contents "${root_fs_dir}" "${BUILD_DIR}/${image_contents}" + write_contents_with_technical_details "${root_fs_dir}" "${BUILD_DIR}/${image_contents_wtd}" + + if [[ -n "${image_initrd_contents}" ]] || [[ -n "${image_initrd_contents_wtd}" ]]; then + "${BUILD_LIBRARY_DIR}/extract-initramfs-from-vmlinuz.sh" "${root_fs_dir}/boot/flatcar/vmlinuz-a" "${BUILD_DIR}/tmp_initrd_contents" + if [[ -n "${image_initrd_contents}" ]]; then + write_contents "${BUILD_DIR}/tmp_initrd_contents" "${BUILD_DIR}/${image_initrd_contents}" + fi + + if [[ -n "${image_initrd_contents_wtd}" ]]; then + write_contents_with_technical_details "${BUILD_DIR}/tmp_initrd_contents" "${BUILD_DIR}/${image_initrd_contents_wtd}" + fi + rm -rf "${BUILD_DIR}/tmp_initrd_contents" + fi + + if [[ -n "${image_disk_space_usage}" ]]; then + write_disk_space_usage "${root_fs_dir}" "${BUILD_DIR}/${image_disk_space_usage}" + fi + + cleanup_mounts "${root_fs_dir}" + trap - EXIT } diff --git a/build_library/dev_container_util.sh b/build_library/dev_container_util.sh index 7fa195b443..8710b304fd 100755 --- a/build_library/dev_container_util.sh +++ b/build_library/dev_container_util.sh @@ -77,6 +77,7 @@ create_dev_container() { info "Building developer image ${image_name}" local root_fs_dir="${BUILD_DIR}/rootfs" local image_contents="${image_name%.bin}_contents.txt" + local image_contents_wtd="${image_name%.bin}_contents_wtd.txt" local image_packages="${image_name%.bin}_packages.txt" local image_licenses="${image_name%.bin}_licenses.json" @@ -104,7 +105,7 @@ create_dev_container() { # The remount services are provided by coreos-base/coreos-init systemd_enable "${root_fs_dir}" "multi-user.target" "remount-usr.service" - finish_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${image_contents}" + finish_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${image_contents}" "${image_contents_wtd}" declare -a files_to_evaluate declare -a compressed_images @@ -115,6 +116,7 @@ create_dev_container() { upload_image -d "${BUILD_DIR}/${image_name}.DIGESTS" \ "${BUILD_DIR}/${image_contents}" \ + "${BUILD_DIR}/${image_contents_wtd}" \ "${BUILD_DIR}/${image_packages}" \ "${BUILD_DIR}/${image_licenses}" \ "${compressed_images[@]}" \ diff --git a/build_library/extract-initramfs-from-vmlinuz.sh b/build_library/extract-initramfs-from-vmlinuz.sh new file mode 100755 index 0000000000..9c1ffb6952 --- /dev/null +++ b/build_library/extract-initramfs-from-vmlinuz.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +# Does as it says on the tin. +# +# Example: extract-initramfs-from-vmlinuz /boot/flatcar/vmlinuz-a out-dir +# +# This will create one or more out-dir/rootfs-N directories that contain the contents of the initramfs. + +set -euo pipefail +# check for unzstd. Will abort the script with an error message if the tool is not present. +unzstd -V >/dev/null +fail() { + echo "${*}" >&2 + exit 1 +} + +# Stolen from extract-vmlinux and modified. +try_decompress() { + local header="${1}" + local no_idea="${2}" + local tool="${3}" + local image="${4}" + local tmp="${5}" + local output_basename="${6}" + + local pos + local tool_filename=$(echo "${tool}" | cut -f1 -d' ') + # The obscure use of the "tr" filter is to work around older versions of + # "grep" that report the byte offset of the line instead of the pattern. + + # Try to find the header and decompress from here. + for pos in $(tr "${header}\n${no_idea}" "\n${no_idea}=" < "${image}" | + grep --text --byte-offset --only-matching "^${no_idea}") + do + pos=${pos%%:*} + # Disable error handling, because we will be potentially + # giving the tool garbage or a valid archive with some garbage + # appended to it. So let the tool extract the valid archive + # and then complain about the garbage at the end, but don't + # fail the script because of it. + set +e; tail "-c+${pos}" "${image}" | "${tool}" >"${tmp}/out" 2>/dev/null; set -e; + if [ -s "${tmp}/out" ]; then + mv "${tmp}/out" "${output_basename}-${tool_filename}-at-${pos}" + else + rm -f "${tmp}/out" + fi + done +} + +try_unzstd_decompress() { + local image="${1}" + local tmp="${2}" + local output_basename="${3}" + try_decompress '(\265/\375' xxx unzstd "${image}" "${tmp}" "${output_basename}" +} + +me="${0##*/}" +if [[ $# -ne 2 ]]; then + fail "Usage: ${me} " +fi +image="${1}" +out="${2}" +if [[ ! -s "${image}" ]]; then + fail "The image file '${image}' either does not exist or is empty" +fi +mkdir -p "${out}" + +tmp=$(mktemp --directory /tmp/eifv-XXXXXX) +trap "rm -rf ${tmp}" EXIT + +tmp_dec="${tmp}/decompress" +mkdir "${tmp_dec}" +fr_prefix="${tmp}/first-round" + +ROOTFS_IDX=0 +perform_round() { + local image="${1}" + local tmp_dec="${2}" + local round_prefix="${3}" + try_unzstd_decompress "${image}" "${tmp_dec}" "${round_prefix}" + for rnd in "${round_prefix}"*; do + if [[ $(file --brief "${rnd}") =~ 'cpio archive' ]]; then + mkdir -p "${out}/rootfs-${ROOTFS_IDX}" + while cpio --quiet --extract --make-directories --directory="${out}/rootfs-${ROOTFS_IDX}" --nonmatching 'dev/*'; do + ROOTFS_IDX=$(( ROOTFS_IDX + 1 )) + mkdir -p "${out}/rootfs-${ROOTFS_IDX}" + done <${rnd} + rmdir "${out}/rootfs-${ROOTFS_IDX}" + fi + done +} + +shopt -s nullglob +perform_round "${image}" "${tmp_dec}" "${fr_prefix}" +for fr in "${fr_prefix}"*; do + fr_files="${fr}-files" + fr_dec="${fr_files}/decompress" + mkdir -p "${fr_dec}" + sr_prefix="${fr_files}/second-round" + perform_round "${fr}" "${fr_dec}" "${sr_prefix}" +done + +if [[ ${ROOTFS_IDX} -eq 0 ]]; then + fail "no initramfs found in ${image}" +fi + +echo "done, found ${ROOTFS_IDX} rootfs(es)" diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 77502402bb..723f09a5c8 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -65,6 +65,7 @@ create_prod_image() { info "Building production image ${image_name}" local root_fs_dir="${BUILD_DIR}/rootfs" local image_contents="${image_name%.bin}_contents.txt" + local image_contents_wtd="${image_name%.bin}_contents_wtd.txt" local image_packages="${image_name%.bin}_packages.txt" local image_sbom="${image_name%.bin}_sbom.json" local image_licenses="${image_name%.bin}_licenses.json" @@ -73,6 +74,9 @@ create_prod_image() { local image_pcr_policy="${image_name%.bin}_pcr_policy.zip" local image_grub="${image_name%.bin}.grub" local image_shim="${image_name%.bin}.shim" + local image_initrd_contents="${image_name%.bin}_initrd_contents.txt" + local image_initrd_contents_wtd="${image_name%.bin}_initrd_contents_wtd.txt" + local image_disk_usage="${image_name%.bin}_disk_usage.txt" start_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${update_group}" @@ -131,15 +135,20 @@ EOF "${disk_layout}" \ "${root_fs_dir}" \ "${image_contents}" \ + "${image_contents_wtd}" \ "${image_kernel}" \ "${image_pcr_policy}" \ "${image_grub}" \ "${image_shim}" \ - "${image_kconfig}" + "${image_kconfig}" \ + "${image_initrd_contents}" \ + "${image_initrd_contents_wtd}" \ + "${image_disk_usage}" # Upload local to_upload=( "${BUILD_DIR}/${image_contents}" + "${BUILD_DIR}/${image_contents_wtd}" "${BUILD_DIR}/${image_packages}" "${BUILD_DIR}/${image_sbom}" "${BUILD_DIR}/${image_licenses}" @@ -147,6 +156,9 @@ EOF "${BUILD_DIR}/${image_pcr_policy}" "${BUILD_DIR}/${image_grub}" "${BUILD_DIR}/${image_kconfig}" + "${BUILD_DIR}/${image_initrd_contents}" + "${BUILD_DIR}/${image_initrd_contents_wtd}" + "${BUILD_DIR}/${image_disk_usage}" ) local files_to_evaluate=( "${BUILD_DIR}/${image_name}" )