Merge pull request #634 from flatcar/krnowak/report-generation

Update report generation

build_library/build_image_util.sh

@@ -244,12 +244,60 @@ systemd_enable() {
 write_contents() {
     info "Writing ${2##*/}"
     pushd "$1" >/dev/null
+    # %M - file permissions
+    # %n - number of hard links to file
+    # %u - file's user name
+    # %g - file's group name
+    # %s - size in bytes
+    # %Tx - modification time (Y - year, m - month, d - day, H - hours, M - minutes)
+    # %P - file's path
+    # %l - symlink target (empty if not a symlink)
     sudo TZ=UTC find -printf \
         '%M %2n %-7u %-7g %7s %TY-%Tm-%Td %TH:%TM ./%P -> %l\n' \
         | sed -e 's/ -> $//' > "$2"
     popd >/dev/null
 }
 
+# Generate a listing that can be used by other tools to analyze
+# image/file size changes.
+write_contents_with_technical_details() {
+    info "Writing ${2##*/}"
+    pushd "$1" >/dev/null
+    # %M - file permissions
+    # %D - ID of a device where file resides
+    # %i - inode number
+    # %n - number of hard links to file
+    # %s - size in bytes
+    # %P - file's path
+    sudo find -printf \
+        '%M %D %i %n %s ./%P\n' > "$2"
+    popd >/dev/null
+}
+
+# Generate a report like the following:
+#
+# File    Size  Used Avail Use% Type
+# /boot   127M   62M   65M  50% vfat
+# /usr    983M  721M  212M  78% ext2
+# /       6,0G   13M  5,6G   1% ext4
+# SUM     7,0G  796M  5,9G  12% -
+write_disk_space_usage() {
+    info "Writing ${2##*/}"
+    pushd "${1}" >/dev/null
+    # The sed's first command turns './<path>' into '/<path> ', second
+    # command replaces '- ' with 'SUM' for the total row. All this to
+    # keep the numbers neatly aligned in columns.
+    sudo df \
+         --human-readable \
+         --total \
+         --output='file,size,used,avail,pcent,fstype' \
+         ./boot ./usr ./ | \
+        sed \
+            -e 's#^\.\(/[^ ]*\)#\1 #' \
+            -e 's/^-  /SUM/' >"${2}"
+    popd >/dev/null
+}
+
 # "equery list" a potentially uninstalled board package
 query_available_package() {
     local pkg="$1"
@@ -624,11 +672,15 @@ finish_image() {
   local disk_layout="$2"
   local root_fs_dir="$3"
   local image_contents="$4"
-  local image_kernel="$5"
+  local image_contents_wtd="$5"
-  local pcr_policy="$6"
+  local image_kernel="$6"
-  local image_grub="$7"
+  local pcr_policy="$7"
-  local image_shim="$8"
+  local image_grub="$8"
-  local image_kconfig="$9"
+  local image_shim="$9"
+  local image_kconfig="${10}"
+  local image_initrd_contents="${11}"
+  local image_initrd_contents_wtd="${12}"
+  local image_disk_space_usage="${13}"
 
   local install_grub=0
   local disk_img="${BUILD_DIR}/${image_name}"
@@ -730,8 +782,6 @@ EOF
         "${BUILD_DIR}/${image_kconfig}"
   fi
 
-  write_contents "${root_fs_dir}" "${BUILD_DIR}/${image_contents}"
-
   # Zero all fs free space to make it more compressible so auto-update
   # payloads become smaller, not fatal since it won't work on linux < 3.2
   sudo fstrim "${root_fs_dir}" || true
@@ -828,4 +878,31 @@ EOF
     popd >/dev/null
     rm -rf "${BUILD_DIR}/pcrs"
   fi
+
+  # Mount the final image again, as readonly, to generate some reports.
+  "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout="${disk_layout}" \
+      mount --read_only "${disk_img}" "${root_fs_dir}"
+  trap "cleanup_mounts '${root_fs_dir}'" EXIT
+
+  write_contents "${root_fs_dir}" "${BUILD_DIR}/${image_contents}"
+  write_contents_with_technical_details "${root_fs_dir}" "${BUILD_DIR}/${image_contents_wtd}"
+
+  if [[ -n "${image_initrd_contents}" ]] || [[ -n "${image_initrd_contents_wtd}" ]]; then
+      "${BUILD_LIBRARY_DIR}/extract-initramfs-from-vmlinuz.sh" "${root_fs_dir}/boot/flatcar/vmlinuz-a" "${BUILD_DIR}/tmp_initrd_contents"
+      if [[ -n "${image_initrd_contents}" ]]; then
+          write_contents "${BUILD_DIR}/tmp_initrd_contents" "${BUILD_DIR}/${image_initrd_contents}"
+      fi
+
+      if [[ -n "${image_initrd_contents_wtd}" ]]; then
+          write_contents_with_technical_details "${BUILD_DIR}/tmp_initrd_contents" "${BUILD_DIR}/${image_initrd_contents_wtd}"
+      fi
+      rm -rf "${BUILD_DIR}/tmp_initrd_contents"
+  fi
+
+  if [[ -n "${image_disk_space_usage}" ]]; then
+      write_disk_space_usage "${root_fs_dir}" "${BUILD_DIR}/${image_disk_space_usage}"
+  fi
+
+  cleanup_mounts "${root_fs_dir}"
+  trap - EXIT
 }

build_library/dev_container_util.sh

@@ -77,6 +77,7 @@ create_dev_container() {
   info "Building developer image ${image_name}"
   local root_fs_dir="${BUILD_DIR}/rootfs"
   local image_contents="${image_name%.bin}_contents.txt"
+  local image_contents_wtd="${image_name%.bin}_contents_wtd.txt"
   local image_packages="${image_name%.bin}_packages.txt"
   local image_licenses="${image_name%.bin}_licenses.json"
 
@@ -104,7 +105,7 @@ create_dev_container() {
   # The remount services are provided by coreos-base/coreos-init
   systemd_enable "${root_fs_dir}" "multi-user.target" "remount-usr.service"
 
-  finish_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${image_contents}"
+  finish_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${image_contents}" "${image_contents_wtd}"
 
   declare -a files_to_evaluate
   declare -a compressed_images
@@ -115,6 +116,7 @@ create_dev_container() {
 
   upload_image -d "${BUILD_DIR}/${image_name}.DIGESTS" \
       "${BUILD_DIR}/${image_contents}" \
+      "${BUILD_DIR}/${image_contents_wtd}" \
       "${BUILD_DIR}/${image_packages}" \
       "${BUILD_DIR}/${image_licenses}" \
       "${compressed_images[@]}" \

build_library/extract-initramfs-from-vmlinuz.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+
+# Does as it says on the tin.
+#
+# Example: extract-initramfs-from-vmlinuz /boot/flatcar/vmlinuz-a out-dir
+#
+# This will create one or more out-dir/rootfs-N directories that contain the contents of the initramfs.
+
+set -euo pipefail
+# check for unzstd. Will abort the script with an error message if the tool is not present.
+unzstd -V >/dev/null
+fail() {
+    echo "${*}" >&2
+    exit 1
+}
+
+# Stolen from extract-vmlinux and modified.
+try_decompress() {
+    local header="${1}"
+    local no_idea="${2}"
+    local tool="${3}"
+    local image="${4}"
+    local tmp="${5}"
+    local output_basename="${6}"
+
+    local pos
+    local tool_filename=$(echo "${tool}" | cut -f1 -d' ')
+    # The obscure use of the "tr" filter is to work around older versions of
+    # "grep" that report the byte offset of the line instead of the pattern.
+
+    # Try to find the header and decompress from here.
+    for pos in $(tr "${header}\n${no_idea}" "\n${no_idea}=" < "${image}" |
+                     grep --text --byte-offset --only-matching "^${no_idea}")
+    do
+        pos=${pos%%:*}
+        # Disable error handling, because we will be potentially
+        # giving the tool garbage or a valid archive with some garbage
+        # appended to it. So let the tool extract the valid archive
+        # and then complain about the garbage at the end, but don't
+        # fail the script because of it.
+        set +e; tail "-c+${pos}" "${image}" | "${tool}" >"${tmp}/out" 2>/dev/null; set -e;
+        if [ -s "${tmp}/out" ]; then
+            mv "${tmp}/out" "${output_basename}-${tool_filename}-at-${pos}"
+        else
+            rm -f "${tmp}/out"
+        fi
+    done
+}
+
+try_unzstd_decompress() {
+    local image="${1}"
+    local tmp="${2}"
+    local output_basename="${3}"
+    try_decompress '(\265/\375' xxx unzstd "${image}" "${tmp}" "${output_basename}"
+}
+
+me="${0##*/}"
+if [[ $# -ne 2 ]]; then
+    fail "Usage: ${me} <vmlinuz> <output_directory>"
+fi
+image="${1}"
+out="${2}"
+if [[ ! -s "${image}" ]]; then
+    fail "The image file '${image}' either does not exist or is empty"
+fi
+mkdir -p "${out}"
+
+tmp=$(mktemp --directory /tmp/eifv-XXXXXX)
+trap "rm -rf ${tmp}" EXIT
+
+tmp_dec="${tmp}/decompress"
+mkdir "${tmp_dec}"
+fr_prefix="${tmp}/first-round"
+
+ROOTFS_IDX=0
+perform_round() {
+    local image="${1}"
+    local tmp_dec="${2}"
+    local round_prefix="${3}"
+    try_unzstd_decompress "${image}" "${tmp_dec}" "${round_prefix}"
+    for rnd in "${round_prefix}"*; do
+        if [[ $(file --brief "${rnd}") =~ 'cpio archive' ]]; then
+            mkdir -p "${out}/rootfs-${ROOTFS_IDX}"
+            while cpio --quiet --extract --make-directories --directory="${out}/rootfs-${ROOTFS_IDX}" --nonmatching 'dev/*'; do
+                ROOTFS_IDX=$(( ROOTFS_IDX + 1 ))
+                mkdir -p "${out}/rootfs-${ROOTFS_IDX}"
+            done <${rnd}
+            rmdir "${out}/rootfs-${ROOTFS_IDX}"
+        fi
+    done
+}
+
+shopt -s nullglob
+perform_round "${image}" "${tmp_dec}" "${fr_prefix}"
+for fr in "${fr_prefix}"*; do
+    fr_files="${fr}-files"
+    fr_dec="${fr_files}/decompress"
+    mkdir -p "${fr_dec}"
+    sr_prefix="${fr_files}/second-round"
+    perform_round "${fr}" "${fr_dec}" "${sr_prefix}"
+done
+
+if [[ ${ROOTFS_IDX} -eq 0 ]]; then
+    fail "no initramfs found in ${image}"
+fi
+
+echo "done, found ${ROOTFS_IDX} rootfs(es)"

build_library/prod_image_util.sh

@@ -65,6 +65,7 @@ create_prod_image() {
   info "Building production image ${image_name}"
   local root_fs_dir="${BUILD_DIR}/rootfs"
   local image_contents="${image_name%.bin}_contents.txt"
+  local image_contents_wtd="${image_name%.bin}_contents_wtd.txt"
   local image_packages="${image_name%.bin}_packages.txt"
   local image_sbom="${image_name%.bin}_sbom.json"
   local image_licenses="${image_name%.bin}_licenses.json"
@@ -73,6 +74,9 @@ create_prod_image() {
   local image_pcr_policy="${image_name%.bin}_pcr_policy.zip"
   local image_grub="${image_name%.bin}.grub"
   local image_shim="${image_name%.bin}.shim"
+  local image_initrd_contents="${image_name%.bin}_initrd_contents.txt"
+  local image_initrd_contents_wtd="${image_name%.bin}_initrd_contents_wtd.txt"
+  local image_disk_usage="${image_name%.bin}_disk_usage.txt"
 
   start_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${update_group}"
 
@@ -131,15 +135,20 @@ EOF
       "${disk_layout}" \
       "${root_fs_dir}" \
       "${image_contents}" \
+      "${image_contents_wtd}" \
       "${image_kernel}" \
       "${image_pcr_policy}" \
       "${image_grub}" \
       "${image_shim}" \
-      "${image_kconfig}"
+      "${image_kconfig}" \
+      "${image_initrd_contents}" \
+      "${image_initrd_contents_wtd}" \
+      "${image_disk_usage}"
 
   # Upload
   local to_upload=(
     "${BUILD_DIR}/${image_contents}"
+    "${BUILD_DIR}/${image_contents_wtd}"
     "${BUILD_DIR}/${image_packages}"
     "${BUILD_DIR}/${image_sbom}"
     "${BUILD_DIR}/${image_licenses}"
@@ -147,6 +156,9 @@ EOF
     "${BUILD_DIR}/${image_pcr_policy}"
     "${BUILD_DIR}/${image_grub}"
     "${BUILD_DIR}/${image_kconfig}"
+    "${BUILD_DIR}/${image_initrd_contents}"
+    "${BUILD_DIR}/${image_initrd_contents_wtd}"
+    "${BUILD_DIR}/${image_disk_usage}"
   )
 
   local files_to_evaluate=( "${BUILD_DIR}/${image_name}" )