base OS sysexts: separate build script, inventory generation

This change refactors base OS sysext builds to use a separate build
script `build_library/sysext_prod_builder`, which is called from
`build_library/prod_image_util.sh` when `build_image` runs.

This allows for better separation of cleanup traps: prod image sysext
builds need its own trap / cleanup function for temporary build
directories and loopback mounts.

Prod sysext builds properly generate lincense and SBOM information, and
provide detailed file listings and disk space usage stats.

- SBOM / licenses JSON now include all packages of the
  final image, i.e. a combined list of base image and all base OS
  sysexts.
- Packages lists, files list and detailed files list include the sysext
  squashfs files for the base image, and separate sections with files /
  packages lists for each sysext.
- Disk usage contains both final disk image usage as well as usage of
  each individual sysext squashfs.

build_library/build_image_util.sh

@@ -301,11 +301,6 @@ image_packages_implicit() {
             query_available_package "${pkg}"
         done < "${profile}/package.provided"
     fi
-
-    # Include source packages of all sysext images installed on disk.
-    for docker_containerd_package in $(package_run_dependencies docker) $(package_run_dependencies containerd); do
-     query_available_package "${docker_containerd_package}" ;
-    done
 }
 
 # Generate a list of packages installed in an image.

build_library/prod_image_util.sh

@@ -52,58 +52,6 @@ extract_prod_gcc() {
     package_provided "${gcc}"
 }
 
-# Create a sysext from a package and install it to the OS image.
-# Conventions:
-# - For each <group>/<package>, <group>_<package>_pkginfo will be built. Can be used in subsequent calls
-#   to build dependent sysexts.
-# - If ${BUILD_LIBRARY_DIR}/sysext_mangle_<group>_<package> exists it will be used as FS mangle script
-#   when building the sysext.
-#
-create_prod_sysext() {
-  local install_root="$1"
-  local base_image="$2"
-  local grp_pkg="$3"
-  local pkginfo="${4:-}"
-
-  local name="${grp_pkg//\//_}" # some-group/some-package => some-group_some-package
-  local pkginfo_opt=""
-  local manglefs_opt=""
-
-  local msg="Creating sysext '${grp_pkg}' ==> ${name}.raw"
-
-  # Include previous sysexts' pkginfo if supplied
-  if [[ -n "${pkginfo}" ]] ; then
-    if [[ ! -f "${BUILD_DIR}/${pkginfo}" ]] ; then
-      die "Sysext build '${grp_pkg}': unable to find package info at '${BUILD_DIR}/${pkginfo}'."
-    fi
-    msg="${msg} w/ package info '${pkginfo}'"
-    pkginfo_opt="--base_pkginfo=${BUILD_DIR}/${pkginfo}"
-  fi
-
-  # Include FS mangle script if present
-  if [[ -x "${BUILD_LIBRARY_DIR}/sysext_mangle_${name}" ]] ; then
-    manglefs_opt="--manglefs_script=${BUILD_LIBRARY_DIR}/sysext_mangle_${name}"
-    msg="${msg}, FS mangle script 'sysext_mangle_${name}'"
-  fi
-
-  info "${msg}."
-  
-  sudo "${SCRIPTS_DIR}/build_sysext" \
-            --board="${BOARD}" \
-            --image_builddir="${BUILD_DIR}" \
-            --squashfs_base="${base_image}" \
-            --generate_pkginfo \
-            ${manglefs_opt} ${pkginfo_opt} \
-            "${name}" "${grp_pkg}"
-
-  sudo mkdir -p "${install_root}"/usr/share/flatcar/sysext
-  sudo install -m 0644 -D "${BUILD_DIR}/${name}.raw" "${install_root}"/usr/share/flatcar/sysext/
-
-  sudo mkdir -p "${install_root}"/etc/extensions/
-  sudo ln -sf "/usr/share/flatcar/sysext/${name}.raw" "${install_root}/etc/extensions/${name}.raw"
-}
-# --
-
 create_prod_image() {
   local image_name="$1"
   local disk_layout="$2"
@@ -118,6 +66,7 @@ create_prod_image() {
 
   info "Building production image ${image_name}"
   local root_fs_dir="${BUILD_DIR}/rootfs"
+  local root_fs_sysexts_output_dir="${BUILD_DIR}/rootfs-included-sysexts"
   local image_contents="${image_name%.bin}_contents.txt"
   local image_contents_wtd="${image_name%.bin}_contents_wtd.txt"
   local image_packages="${image_name%.bin}_packages.txt"
@@ -141,9 +90,31 @@ create_prod_image() {
   emerge_to_image "${root_fs_dir}" "${base_pkg}"
   run_ldconfig "${root_fs_dir}"
   run_localedef "${root_fs_dir}"
+
+  local root_with_everything="${root_fs_dir}"
+
+  # Call helper script for adding sysexts to the base OS.
+  # Helper will generate a rootfs dir with all packages (base OS and sysexts) included.
+  local root_sysext_mergedir="${BUILD_DIR}/rootfs-with-sysext-pkgs"
+  if [[ -n "${base_sysexts}" ]] ; then
+    "${BUILD_LIBRARY_DIR}/sysext_prod_builder" \
+        "${BOARD}" "${BUILD_DIR}" "${root_fs_dir}" \
+        "${root_sysext_mergedir}" \
+        "${root_fs_sysexts_output_dir}" \
+        "${base_sysexts}"
+    root_with_everything="${root_sysext_mergedir}"
+  fi
+
+
+  write_sbom "${root_with_everything}" "${BUILD_DIR}/${image_sbom}"
+  write_licenses "${root_with_everything}" "${BUILD_DIR}/${image_licenses}"
+
+  if [[ -n "${base_sysexts}" ]] ; then
+    sudo rm -rf "${root_sysext_mergedir}"
+  fi
+
   write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}"
-  write_sbom "${root_fs_dir}" "${BUILD_DIR}/${image_sbom}"
+
-  write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}"
   insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}"
   insert_extra_slsa "${root_fs_dir}"
 
@@ -190,19 +161,9 @@ EOF
   # Remove source locale data, only need to ship the compiled archive.
   sudo rm -rf ${root_fs_dir}/usr/share/i18n/
 
-  if [[ -n "${base_sysexts}" ]] ; then
-    local grp_pkg=""
-    local prev_pkginfo=""
-    for grp_pkg in ${base_sysexts//,/ }; do
-      create_prod_sysext "${root_fs_dir}"\
-                         "${BUILD_DIR}/${image_sysext_base}" \
-                         "${grp_pkg}" \
-                         "${prev_pkginfo}"
-      prev_pkginfo="${grp_pkg//\//_}_pkginfo.raw"
-    done
-  fi
-
   # Finish image will move files from /etc to /usr/share/flatcar/etc.
+  # Note that image filesystem contents generated by finish_image will not
+  # include sysext contents (only the sysext squashfs files themselves).
   finish_image \
       "${image_name}" \
       "${disk_layout}" \
@@ -218,6 +179,30 @@ EOF
       "${image_initrd_contents_wtd}" \
       "${image_disk_usage}"
 
+  # append sysext inventories to image contents files.
+  if [[ -n "${base_sysexts}" ]] ; then
+    local inventory_file="" image_basename="${image_name%.bin}"
+
+    for inventory_file in "${image_contents}" "${image_contents_wtd}" "${image_disk_usage}" "${image_packages}" ; do
+      local suffix="${inventory_file/${image_basename}/}" sysext=""
+
+      info "Processing '${inventory_file}'"
+
+      for sysext in ${base_sysexts//,/ }; do
+        local name="${sysext//\//_}"
+        local sysext_inventory="${root_fs_sysexts_output_dir}/${name}${suffix}"
+        if [[ ! -f "${sysext_inventory}" ]] ; then
+          die "Sysext inventory file '${sysext//\//_}${suffix}' for '${inventory_file}' not found in '${root_fs_sysexts_output_dir}'"
+        fi
+        info "Adding sysext inventory '${name}${suffix}' to '${inventory_file}'"
+        {
+          echo -e "\n\n### Sysext ${name}.raw\n"
+          cat "${sysext_inventory}"
+        } >> "${BUILD_DIR}/${inventory_file}"
+      done
+    done
+  fi
+
   # Upload
   local to_upload=(
     "${BUILD_DIR}/${image_contents}"

build_library/sysext_prod_builder

@@ -0,0 +1,151 @@
+#!/bin/bash
+# Copyright (c) 2023 by the Flatcar Maintainers.
+# Use of this source code is governed by the Apache 2.0 license.
+
+# Helper script for building OS images w/ sysexts included.
+# Called by build_image -> prod_image_util.sh.
+# This is a separate script mainly so we can trap EXIT and clean up our mounts
+#  without interfering with traps set by build_image.
+
+# We're in build_library/, script root is one up
+SCRIPT_ROOT="$(cd "$(dirname "$(readlink -f "$0")")/../"; pwd)"
+. "${SCRIPT_ROOT}/common.sh" || exit 1
+
+# Script must run inside the chroot
+assert_inside_chroot
+switch_to_strict_mode
+
+. "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1
+
+# Create a sysext from a package and install it to the OS image.
+# Conventions:
+# - For each <group>/<package>, <group>_<package>_pkginfo will be built. Can be used in subsequent calls
+#   to build dependent sysexts.
+# - If ${BUILD_LIBRARY_DIR}/sysext_mangle_<group>_<package> exists it will be used as FS mangle script
+#   when building the sysext.
+create_prod_sysext() {
+  local BOARD="$1"
+  local output_dir="$2"
+  local workdir="$3"
+  local base_sysext="$4"
+  local install_root="$5"
+  local grp_pkg="$6"
+  local pkginfo="${7:-}"
+
+  local name="${grp_pkg//\//_}" # some-group/some-package => some-group_some-package
+  local pkginfo_opt=""
+  local manglefs_opt=""
+
+  local msg="Creating sysext '${grp_pkg}' ==> ${name}.raw"
+
+  # Include previous sysexts' pkginfo if supplied
+  if [[ -n "${pkginfo}" ]] ; then
+    if [[ ! -f "${output_dir}/${pkginfo}" ]] ; then
+      die "Sysext build '${grp_pkg}': unable to find package info at '${output_dir}/${pkginfo}'."
+    fi
+    msg="${msg} w/ package info '${pkginfo}'"
+    pkginfo_opt="--base_pkginfo=${output_dir}/${pkginfo}"
+  fi
+
+  # Include FS mangle script if present
+  if [[ -x "${BUILD_LIBRARY_DIR}/sysext_mangle_${name}" ]] ; then
+    manglefs_opt="--manglefs_script=${BUILD_LIBRARY_DIR}/sysext_mangle_${name}"
+    msg="${msg}, FS mangle script 'sysext_mangle_${name}'"
+  fi
+
+  info "${msg}."
+  
+  sudo "${SCRIPTS_DIR}/build_sysext" \
+            --board="${BOARD}" \
+            --image_builddir="${workdir}/sysext-build" \
+            --squashfs_base="${base_sysext}" \
+            --generate_pkginfo \
+            ${manglefs_opt} ${pkginfo_opt} \
+            "${name}" "${grp_pkg}"
+
+  sudo mv "${workdir}/sysext-build/${name}.raw" "${workdir}/sysext-build/${name}_pkginfo.raw" \
+                              "${workdir}/sysext-build/${name}"_*.txt "${output_dir}"
+
+  sudo mkdir -p "${install_root}"/usr/share/flatcar/sysext
+  sudo install -m 0644 -D "${output_dir}/${name}.raw" "${install_root}"/usr/share/flatcar/sysext/
+
+  sudo mkdir -p "${install_root}"/etc/extensions/
+  sudo ln -sf "/usr/share/flatcar/sysext/${name}.raw" "${install_root}/etc/extensions/${name}.raw"
+}
+# --
+
+BOARD="$1"
+BUILD_DIR="$2"
+root_fs_dir="$3"
+
+merged_rootfs_dir="$4"
+sysext_output_dir="$5"
+
+sysexts_list="$6"
+
+grp_pkg=""
+prev_pkginfo=""
+sysext_workdir="${BUILD_DIR}/prod-sysext-work"
+sysext_mountdir="${BUILD_DIR}/prod-sysext-work/mounts"
+sysext_base="${sysext_workdir}/base-os.squashfs"
+
+function cleanup() {
+  sudo umount "${sysext_mountdir}"/* || true
+  rm -rf "${sysext_workdir}" || true
+}
+# --
+
+trap cleanup EXIT
+
+rm -rf "${sysext_workdir}" "${sysext_output_dir}"
+mkdir "${sysext_workdir}" "${sysext_output_dir}"
+
+info "creating temporary base OS squashfs"
+sudo mksquashfs "${root_fs_dir}" "${sysext_base}" -noappend
+
+# Build sysexts on top of root fs and mount sysexts' squashfs + pkginfo squashfs
+# for combined overlay later.
+prev_pkginfo=""
+sysext_lowerdirs="${sysext_mountdir}/rootfs-lower"
+for grp_pkg in ${sysexts_list//,/ }; do
+  create_prod_sysext "${BOARD}" \
+                     "${sysext_output_dir}" \
+                     "${sysext_workdir}" \
+                     "${sysext_base}" \
+                     "${root_fs_dir}"\
+                     "${grp_pkg}" \
+                     "${prev_pkginfo}"
+  name="${grp_pkg//\//_}"
+
+  mkdir -p "${sysext_mountdir}/${name}" \
+           "${sysext_mountdir}/${name}_pkginfo"
+  sudo mount -rt squashfs -o loop,nodev "${sysext_output_dir}/${name}.raw" \
+                                    "${sysext_mountdir}/${name}"
+  sudo mount -rt squashfs -o loop,nodev "${sysext_output_dir}/${name}_pkginfo.raw" \
+                                    "${sysext_mountdir}/${name}_pkginfo"
+
+  sysext_lowerdirs="${sysext_lowerdirs}:${sysext_mountdir}/${name}"
+  sysext_lowerdirs="${sysext_lowerdirs}:${sysext_mountdir}/${name}_pkginfo"
+
+  prev_pkginfo="${name}_pkginfo.raw"
+done
+
+# Mount the combined overlay (base OS, sysexts, and syset pkginfos) and copy a snapshot
+# into the designated output dir for upper layers to process.
+mkdir -p "${sysext_mountdir}/rootfs-lower" 
+sudo mount -rt squashfs -o loop,nodev "${sysext_base}" "${sysext_mountdir}/rootfs-lower"
+
+# Mount overlay for report generation
+mkdir -p "${sysext_workdir}/.work"
+mkdir -p "${sysext_mountdir}/rootfs-upper" 
+sudo mount -t overlay overlay \
+         -o lowerdir="${sysext_lowerdirs}",upperdir="${sysext_mountdir}/rootfs-upper",workdir="${sysext_workdir}/.work" \
+         "${sysext_mountdir}/rootfs-upper"
+
+
+sudo rm -rf "${merged_rootfs_dir}"
+sudo cp -a "${sysext_mountdir}/rootfs-upper" "${merged_rootfs_dir}"
+
+
+cleanup
+trap -- EXIT

build_sysext

@@ -130,10 +130,11 @@ cleanup() {
   )
   umount "${dirs[@]}" 2>/dev/null || true
   rm -rf "${dirs[@]}" || true
-  if [[ -d "${BUILD_DIR}/img-pkginfo" ]] ; then
+  if [[ -d "${BUILD_DIR}/base-pkginfo" ]] ; then
-    umount "${BUILD_DIR}/img-pkginfo"/* 2>/dev/null || true
+    umount "${BUILD_DIR}/base-pkginfo"/* 2>/dev/null || true
-    rm -rf "${BUILD_DIR}/img-pkginfo" || true
+    rm -rf "${BUILD_DIR}/base-pkginfo" || true
   fi
+  rm -rf "${BUILD_DIR}/img-pkginfo"
 }
 
 # Set up trap to execute cleanup() on script exit
@@ -143,7 +144,7 @@ ARCH=$(_get_sysext_arch "${FLAGS_board}")
 cleanup
 
 # If we need to handle pkginfo squashfs files, create mount points under
-# ${BUILD_DIR}/img-pkginfo, mount the squashfs images, and add the mount paths to
+# ${BUILD_DIR}/base-pkginfo, mount the squashfs images, and add the mount paths to
 # the list of lowerdirs.
 pkginfo_lowerdirs=""
 if [[ -n "${FLAGS_base_pkginfo}" ]] ; then
@@ -156,7 +157,7 @@ if [[ -n "${FLAGS_base_pkginfo}" ]] ; then
     fi
 
     pfile="$(basename "${ppath}")"
-    pmdir="${BUILD_DIR}/img-pkginfo/${pfile}"
+    pmdir="${BUILD_DIR}/base-pkginfo/${pfile}"
     mkdir -p "${pmdir}"
     mount -rt squashfs -o loop,nodev "${ppath}" "${pmdir}"
     pkginfo_lowerdirs="${pkginfo_lowerdirs}:${pmdir}"
@@ -169,6 +170,7 @@ mount -rt squashfs -o loop,nodev "${FLAGS_squashfs_base}" "${BUILD_DIR}/fs-root"
 mkdir "${BUILD_DIR}/install-root"
 mkdir "${BUILD_DIR}/workdir"
 mount -t overlay overlay -o lowerdir="${BUILD_DIR}/fs-root${pkginfo_lowerdirs}",upperdir="${BUILD_DIR}/install-root",workdir="${BUILD_DIR}/workdir" "${BUILD_DIR}/install-root"
+
 VERSION_BOARD=$(grep "^VERSION=" ${BUILD_DIR}/fs-root/usr/lib/os-release | cut -d = -f 2-)
 if [ "$VERSION_BOARD" != "$FLATCAR_VERSION" ]; then
   warn "Base squashfs version: $VERSION_BOARD"
@@ -223,6 +225,10 @@ if [[ "$FLAGS_generate_pkginfo" = "${FLAGS_TRUE}" ]] ; then
   mksquashfs "${BUILD_DIR}/img-pkginfo" "${BUILD_DIR}/${SYSEXTNAME}_pkginfo.raw" -noappend
 fi
 
+info "Writing ${SYSEXTNAME}_packages.txt"
+ROOT="${BUILD_DIR}/install-root" PORTAGE_CONFIGROOT="${BUILD_DIR}/install-root"\
+      equery --no-color list --format '$cpv::$repo' '*' > "${BUILD_DIR}/${SYSEXTNAME}_packages.txt"
+
 info "Removing non-/usr directories from sysext image"
 for entry in "${BUILD_DIR}/install-root"/*; do
   if [[ "${entry}" = */usr ]]; then