diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 7073da75d3..790c1a9556 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -301,11 +301,6 @@ image_packages_implicit() { query_available_package "${pkg}" done < "${profile}/package.provided" fi - - # Include source packages of all sysext images installed on disk. - for docker_containerd_package in $(package_run_dependencies docker) $(package_run_dependencies containerd); do - query_available_package "${docker_containerd_package}" ; - done } # Generate a list of packages installed in an image. diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 6259f0dbdd..09fa70ac59 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -52,58 +52,6 @@ extract_prod_gcc() { package_provided "${gcc}" } -# Create a sysext from a package and install it to the OS image. -# Conventions: -# - For each /, __pkginfo will be built. Can be used in subsequent calls -# to build dependent sysexts. -# - If ${BUILD_LIBRARY_DIR}/sysext_mangle__ exists it will be used as FS mangle script -# when building the sysext. -# -create_prod_sysext() { - local install_root="$1" - local base_image="$2" - local grp_pkg="$3" - local pkginfo="${4:-}" - - local name="${grp_pkg//\//_}" # some-group/some-package => some-group_some-package - local pkginfo_opt="" - local manglefs_opt="" - - local msg="Creating sysext '${grp_pkg}' ==> ${name}.raw" - - # Include previous sysexts' pkginfo if supplied - if [[ -n "${pkginfo}" ]] ; then - if [[ ! -f "${BUILD_DIR}/${pkginfo}" ]] ; then - die "Sysext build '${grp_pkg}': unable to find package info at '${BUILD_DIR}/${pkginfo}'." - fi - msg="${msg} w/ package info '${pkginfo}'" - pkginfo_opt="--base_pkginfo=${BUILD_DIR}/${pkginfo}" - fi - - # Include FS mangle script if present - if [[ -x "${BUILD_LIBRARY_DIR}/sysext_mangle_${name}" ]] ; then - manglefs_opt="--manglefs_script=${BUILD_LIBRARY_DIR}/sysext_mangle_${name}" - msg="${msg}, FS mangle script 'sysext_mangle_${name}'" - fi - - info "${msg}." - - sudo "${SCRIPTS_DIR}/build_sysext" \ - --board="${BOARD}" \ - --image_builddir="${BUILD_DIR}" \ - --squashfs_base="${base_image}" \ - --generate_pkginfo \ - ${manglefs_opt} ${pkginfo_opt} \ - "${name}" "${grp_pkg}" - - sudo mkdir -p "${install_root}"/usr/share/flatcar/sysext - sudo install -m 0644 -D "${BUILD_DIR}/${name}.raw" "${install_root}"/usr/share/flatcar/sysext/ - - sudo mkdir -p "${install_root}"/etc/extensions/ - sudo ln -sf "/usr/share/flatcar/sysext/${name}.raw" "${install_root}/etc/extensions/${name}.raw" -} -# -- - create_prod_image() { local image_name="$1" local disk_layout="$2" @@ -118,6 +66,7 @@ create_prod_image() { info "Building production image ${image_name}" local root_fs_dir="${BUILD_DIR}/rootfs" + local root_fs_sysexts_output_dir="${BUILD_DIR}/rootfs-included-sysexts" local image_contents="${image_name%.bin}_contents.txt" local image_contents_wtd="${image_name%.bin}_contents_wtd.txt" local image_packages="${image_name%.bin}_packages.txt" @@ -141,9 +90,31 @@ create_prod_image() { emerge_to_image "${root_fs_dir}" "${base_pkg}" run_ldconfig "${root_fs_dir}" run_localedef "${root_fs_dir}" + + local root_with_everything="${root_fs_dir}" + + # Call helper script for adding sysexts to the base OS. + # Helper will generate a rootfs dir with all packages (base OS and sysexts) included. + local root_sysext_mergedir="${BUILD_DIR}/rootfs-with-sysext-pkgs" + if [[ -n "${base_sysexts}" ]] ; then + "${BUILD_LIBRARY_DIR}/sysext_prod_builder" \ + "${BOARD}" "${BUILD_DIR}" "${root_fs_dir}" \ + "${root_sysext_mergedir}" \ + "${root_fs_sysexts_output_dir}" \ + "${base_sysexts}" + root_with_everything="${root_sysext_mergedir}" + fi + + + write_sbom "${root_with_everything}" "${BUILD_DIR}/${image_sbom}" + write_licenses "${root_with_everything}" "${BUILD_DIR}/${image_licenses}" + + if [[ -n "${base_sysexts}" ]] ; then + sudo rm -rf "${root_sysext_mergedir}" + fi + write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}" - write_sbom "${root_fs_dir}" "${BUILD_DIR}/${image_sbom}" - write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}" + insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}" insert_extra_slsa "${root_fs_dir}" @@ -190,19 +161,9 @@ EOF # Remove source locale data, only need to ship the compiled archive. sudo rm -rf ${root_fs_dir}/usr/share/i18n/ - if [[ -n "${base_sysexts}" ]] ; then - local grp_pkg="" - local prev_pkginfo="" - for grp_pkg in ${base_sysexts//,/ }; do - create_prod_sysext "${root_fs_dir}"\ - "${BUILD_DIR}/${image_sysext_base}" \ - "${grp_pkg}" \ - "${prev_pkginfo}" - prev_pkginfo="${grp_pkg//\//_}_pkginfo.raw" - done - fi - # Finish image will move files from /etc to /usr/share/flatcar/etc. + # Note that image filesystem contents generated by finish_image will not + # include sysext contents (only the sysext squashfs files themselves). finish_image \ "${image_name}" \ "${disk_layout}" \ @@ -218,6 +179,30 @@ EOF "${image_initrd_contents_wtd}" \ "${image_disk_usage}" + # append sysext inventories to image contents files. + if [[ -n "${base_sysexts}" ]] ; then + local inventory_file="" image_basename="${image_name%.bin}" + + for inventory_file in "${image_contents}" "${image_contents_wtd}" "${image_disk_usage}" "${image_packages}" ; do + local suffix="${inventory_file/${image_basename}/}" sysext="" + + info "Processing '${inventory_file}'" + + for sysext in ${base_sysexts//,/ }; do + local name="${sysext//\//_}" + local sysext_inventory="${root_fs_sysexts_output_dir}/${name}${suffix}" + if [[ ! -f "${sysext_inventory}" ]] ; then + die "Sysext inventory file '${sysext//\//_}${suffix}' for '${inventory_file}' not found in '${root_fs_sysexts_output_dir}'" + fi + info "Adding sysext inventory '${name}${suffix}' to '${inventory_file}'" + { + echo -e "\n\n### Sysext ${name}.raw\n" + cat "${sysext_inventory}" + } >> "${BUILD_DIR}/${inventory_file}" + done + done + fi + # Upload local to_upload=( "${BUILD_DIR}/${image_contents}" diff --git a/build_library/sysext_prod_builder b/build_library/sysext_prod_builder new file mode 100755 index 0000000000..4f9aacb3ad --- /dev/null +++ b/build_library/sysext_prod_builder @@ -0,0 +1,151 @@ +#!/bin/bash +# Copyright (c) 2023 by the Flatcar Maintainers. +# Use of this source code is governed by the Apache 2.0 license. + +# Helper script for building OS images w/ sysexts included. +# Called by build_image -> prod_image_util.sh. +# This is a separate script mainly so we can trap EXIT and clean up our mounts +# without interfering with traps set by build_image. + +# We're in build_library/, script root is one up +SCRIPT_ROOT="$(cd "$(dirname "$(readlink -f "$0")")/../"; pwd)" +. "${SCRIPT_ROOT}/common.sh" || exit 1 + +# Script must run inside the chroot +assert_inside_chroot +switch_to_strict_mode + +. "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1 + +# Create a sysext from a package and install it to the OS image. +# Conventions: +# - For each /, __pkginfo will be built. Can be used in subsequent calls +# to build dependent sysexts. +# - If ${BUILD_LIBRARY_DIR}/sysext_mangle__ exists it will be used as FS mangle script +# when building the sysext. +create_prod_sysext() { + local BOARD="$1" + local output_dir="$2" + local workdir="$3" + local base_sysext="$4" + local install_root="$5" + local grp_pkg="$6" + local pkginfo="${7:-}" + + local name="${grp_pkg//\//_}" # some-group/some-package => some-group_some-package + local pkginfo_opt="" + local manglefs_opt="" + + local msg="Creating sysext '${grp_pkg}' ==> ${name}.raw" + + # Include previous sysexts' pkginfo if supplied + if [[ -n "${pkginfo}" ]] ; then + if [[ ! -f "${output_dir}/${pkginfo}" ]] ; then + die "Sysext build '${grp_pkg}': unable to find package info at '${output_dir}/${pkginfo}'." + fi + msg="${msg} w/ package info '${pkginfo}'" + pkginfo_opt="--base_pkginfo=${output_dir}/${pkginfo}" + fi + + # Include FS mangle script if present + if [[ -x "${BUILD_LIBRARY_DIR}/sysext_mangle_${name}" ]] ; then + manglefs_opt="--manglefs_script=${BUILD_LIBRARY_DIR}/sysext_mangle_${name}" + msg="${msg}, FS mangle script 'sysext_mangle_${name}'" + fi + + info "${msg}." + + sudo "${SCRIPTS_DIR}/build_sysext" \ + --board="${BOARD}" \ + --image_builddir="${workdir}/sysext-build" \ + --squashfs_base="${base_sysext}" \ + --generate_pkginfo \ + ${manglefs_opt} ${pkginfo_opt} \ + "${name}" "${grp_pkg}" + + sudo mv "${workdir}/sysext-build/${name}.raw" "${workdir}/sysext-build/${name}_pkginfo.raw" \ + "${workdir}/sysext-build/${name}"_*.txt "${output_dir}" + + sudo mkdir -p "${install_root}"/usr/share/flatcar/sysext + sudo install -m 0644 -D "${output_dir}/${name}.raw" "${install_root}"/usr/share/flatcar/sysext/ + + sudo mkdir -p "${install_root}"/etc/extensions/ + sudo ln -sf "/usr/share/flatcar/sysext/${name}.raw" "${install_root}/etc/extensions/${name}.raw" +} +# -- + +BOARD="$1" +BUILD_DIR="$2" +root_fs_dir="$3" + +merged_rootfs_dir="$4" +sysext_output_dir="$5" + +sysexts_list="$6" + +grp_pkg="" +prev_pkginfo="" +sysext_workdir="${BUILD_DIR}/prod-sysext-work" +sysext_mountdir="${BUILD_DIR}/prod-sysext-work/mounts" +sysext_base="${sysext_workdir}/base-os.squashfs" + +function cleanup() { + sudo umount "${sysext_mountdir}"/* || true + rm -rf "${sysext_workdir}" || true +} +# -- + +trap cleanup EXIT + +rm -rf "${sysext_workdir}" "${sysext_output_dir}" +mkdir "${sysext_workdir}" "${sysext_output_dir}" + +info "creating temporary base OS squashfs" +sudo mksquashfs "${root_fs_dir}" "${sysext_base}" -noappend + +# Build sysexts on top of root fs and mount sysexts' squashfs + pkginfo squashfs +# for combined overlay later. +prev_pkginfo="" +sysext_lowerdirs="${sysext_mountdir}/rootfs-lower" +for grp_pkg in ${sysexts_list//,/ }; do + create_prod_sysext "${BOARD}" \ + "${sysext_output_dir}" \ + "${sysext_workdir}" \ + "${sysext_base}" \ + "${root_fs_dir}"\ + "${grp_pkg}" \ + "${prev_pkginfo}" + name="${grp_pkg//\//_}" + + mkdir -p "${sysext_mountdir}/${name}" \ + "${sysext_mountdir}/${name}_pkginfo" + sudo mount -rt squashfs -o loop,nodev "${sysext_output_dir}/${name}.raw" \ + "${sysext_mountdir}/${name}" + sudo mount -rt squashfs -o loop,nodev "${sysext_output_dir}/${name}_pkginfo.raw" \ + "${sysext_mountdir}/${name}_pkginfo" + + sysext_lowerdirs="${sysext_lowerdirs}:${sysext_mountdir}/${name}" + sysext_lowerdirs="${sysext_lowerdirs}:${sysext_mountdir}/${name}_pkginfo" + + prev_pkginfo="${name}_pkginfo.raw" +done + +# Mount the combined overlay (base OS, sysexts, and syset pkginfos) and copy a snapshot +# into the designated output dir for upper layers to process. +mkdir -p "${sysext_mountdir}/rootfs-lower" +sudo mount -rt squashfs -o loop,nodev "${sysext_base}" "${sysext_mountdir}/rootfs-lower" + +# Mount overlay for report generation +mkdir -p "${sysext_workdir}/.work" +mkdir -p "${sysext_mountdir}/rootfs-upper" +sudo mount -t overlay overlay \ + -o lowerdir="${sysext_lowerdirs}",upperdir="${sysext_mountdir}/rootfs-upper",workdir="${sysext_workdir}/.work" \ + "${sysext_mountdir}/rootfs-upper" + + +sudo rm -rf "${merged_rootfs_dir}" +sudo cp -a "${sysext_mountdir}/rootfs-upper" "${merged_rootfs_dir}" + + +cleanup +trap -- EXIT diff --git a/build_sysext b/build_sysext index bc7b0646e5..95b00d49ce 100755 --- a/build_sysext +++ b/build_sysext @@ -130,10 +130,11 @@ cleanup() { ) umount "${dirs[@]}" 2>/dev/null || true rm -rf "${dirs[@]}" || true - if [[ -d "${BUILD_DIR}/img-pkginfo" ]] ; then - umount "${BUILD_DIR}/img-pkginfo"/* 2>/dev/null || true - rm -rf "${BUILD_DIR}/img-pkginfo" || true + if [[ -d "${BUILD_DIR}/base-pkginfo" ]] ; then + umount "${BUILD_DIR}/base-pkginfo"/* 2>/dev/null || true + rm -rf "${BUILD_DIR}/base-pkginfo" || true fi + rm -rf "${BUILD_DIR}/img-pkginfo" } # Set up trap to execute cleanup() on script exit @@ -143,7 +144,7 @@ ARCH=$(_get_sysext_arch "${FLAGS_board}") cleanup # If we need to handle pkginfo squashfs files, create mount points under -# ${BUILD_DIR}/img-pkginfo, mount the squashfs images, and add the mount paths to +# ${BUILD_DIR}/base-pkginfo, mount the squashfs images, and add the mount paths to # the list of lowerdirs. pkginfo_lowerdirs="" if [[ -n "${FLAGS_base_pkginfo}" ]] ; then @@ -156,7 +157,7 @@ if [[ -n "${FLAGS_base_pkginfo}" ]] ; then fi pfile="$(basename "${ppath}")" - pmdir="${BUILD_DIR}/img-pkginfo/${pfile}" + pmdir="${BUILD_DIR}/base-pkginfo/${pfile}" mkdir -p "${pmdir}" mount -rt squashfs -o loop,nodev "${ppath}" "${pmdir}" pkginfo_lowerdirs="${pkginfo_lowerdirs}:${pmdir}" @@ -169,6 +170,7 @@ mount -rt squashfs -o loop,nodev "${FLAGS_squashfs_base}" "${BUILD_DIR}/fs-root" mkdir "${BUILD_DIR}/install-root" mkdir "${BUILD_DIR}/workdir" mount -t overlay overlay -o lowerdir="${BUILD_DIR}/fs-root${pkginfo_lowerdirs}",upperdir="${BUILD_DIR}/install-root",workdir="${BUILD_DIR}/workdir" "${BUILD_DIR}/install-root" + VERSION_BOARD=$(grep "^VERSION=" ${BUILD_DIR}/fs-root/usr/lib/os-release | cut -d = -f 2-) if [ "$VERSION_BOARD" != "$FLATCAR_VERSION" ]; then warn "Base squashfs version: $VERSION_BOARD" @@ -223,6 +225,10 @@ if [[ "$FLAGS_generate_pkginfo" = "${FLAGS_TRUE}" ]] ; then mksquashfs "${BUILD_DIR}/img-pkginfo" "${BUILD_DIR}/${SYSEXTNAME}_pkginfo.raw" -noappend fi +info "Writing ${SYSEXTNAME}_packages.txt" +ROOT="${BUILD_DIR}/install-root" PORTAGE_CONFIGROOT="${BUILD_DIR}/install-root"\ + equery --no-color list --format '$cpv::$repo' '*' > "${BUILD_DIR}/${SYSEXTNAME}_packages.txt" + info "Removing non-/usr directories from sysext image" for entry in "${BUILD_DIR}/install-root"/*; do if [[ "${entry}" = */usr ]]; then