mirror of
https://github.com/flatcar/scripts.git
synced 2025-08-07 04:56:58 +02:00
base OS sysexts: separate build script, inventory generation
This change refactors base OS sysext builds to use a separate build script `build_library/sysext_prod_builder`, which is called from `build_library/prod_image_util.sh` when `build_image` runs. This allows for better separation of cleanup traps: prod image sysext builds need its own trap / cleanup function for temporary build directories and loopback mounts. Prod sysext builds properly generate lincense and SBOM information, and provide detailed file listings and disk space usage stats. - SBOM / licenses JSON now include all packages of the final image, i.e. a combined list of base image and all base OS sysexts. - Packages lists, files list and detailed files list include the sysext squashfs files for the base image, and separate sections with files / packages lists for each sysext. - Disk usage contains both final disk image usage as well as usage of each individual sysext squashfs.
This commit is contained in:
Thilo Fromm
parent
c589fb8e56
commit
89555ed2bd
@ -301,11 +301,6 @@ image_packages_implicit() {
|
||||
query_available_package "${pkg}"
|
||||
done < "${profile}/package.provided"
|
||||
fi
|
||||
|
||||
# Include source packages of all sysext images installed on disk.
|
||||
for docker_containerd_package in $(package_run_dependencies docker) $(package_run_dependencies containerd); do
|
||||
query_available_package "${docker_containerd_package}" ;
|
||||
done
|
||||
}
|
||||
|
||||
# Generate a list of packages installed in an image.
|
||||
|
||||
@ -52,58 +52,6 @@ extract_prod_gcc() {
|
||||
package_provided "${gcc}"
|
||||
}
|
||||
|
||||
# Create a sysext from a package and install it to the OS image.
|
||||
# Conventions:
|
||||
# - For each <group>/<package>, <group>_<package>_pkginfo will be built. Can be used in subsequent calls
|
||||
# to build dependent sysexts.
|
||||
# - If ${BUILD_LIBRARY_DIR}/sysext_mangle_<group>_<package> exists it will be used as FS mangle script
|
||||
# when building the sysext.
|
||||
#
|
||||
create_prod_sysext() {
|
||||
local install_root="$1"
|
||||
local base_image="$2"
|
||||
local grp_pkg="$3"
|
||||
local pkginfo="${4:-}"
|
||||
|
||||
local name="${grp_pkg//\//_}" # some-group/some-package => some-group_some-package
|
||||
local pkginfo_opt=""
|
||||
local manglefs_opt=""
|
||||
|
||||
local msg="Creating sysext '${grp_pkg}' ==> ${name}.raw"
|
||||
|
||||
# Include previous sysexts' pkginfo if supplied
|
||||
if [[ -n "${pkginfo}" ]] ; then
|
||||
if [[ ! -f "${BUILD_DIR}/${pkginfo}" ]] ; then
|
||||
die "Sysext build '${grp_pkg}': unable to find package info at '${BUILD_DIR}/${pkginfo}'."
|
||||
fi
|
||||
msg="${msg} w/ package info '${pkginfo}'"
|
||||
pkginfo_opt="--base_pkginfo=${BUILD_DIR}/${pkginfo}"
|
||||
fi
|
||||
|
||||
# Include FS mangle script if present
|
||||
if [[ -x "${BUILD_LIBRARY_DIR}/sysext_mangle_${name}" ]] ; then
|
||||
manglefs_opt="--manglefs_script=${BUILD_LIBRARY_DIR}/sysext_mangle_${name}"
|
||||
msg="${msg}, FS mangle script 'sysext_mangle_${name}'"
|
||||
fi
|
||||
|
||||
info "${msg}."
|
||||
|
||||
sudo "${SCRIPTS_DIR}/build_sysext" \
|
||||
--board="${BOARD}" \
|
||||
--image_builddir="${BUILD_DIR}" \
|
||||
--squashfs_base="${base_image}" \
|
||||
--generate_pkginfo \
|
||||
${manglefs_opt} ${pkginfo_opt} \
|
||||
"${name}" "${grp_pkg}"
|
||||
|
||||
sudo mkdir -p "${install_root}"/usr/share/flatcar/sysext
|
||||
sudo install -m 0644 -D "${BUILD_DIR}/${name}.raw" "${install_root}"/usr/share/flatcar/sysext/
|
||||
|
||||
sudo mkdir -p "${install_root}"/etc/extensions/
|
||||
sudo ln -sf "/usr/share/flatcar/sysext/${name}.raw" "${install_root}/etc/extensions/${name}.raw"
|
||||
}
|
||||
# --
|
||||
|
||||
create_prod_image() {
|
||||
local image_name="$1"
|
||||
local disk_layout="$2"
|
||||
@ -118,6 +66,7 @@ create_prod_image() {
|
||||
|
||||
info "Building production image ${image_name}"
|
||||
local root_fs_dir="${BUILD_DIR}/rootfs"
|
||||
local root_fs_sysexts_output_dir="${BUILD_DIR}/rootfs-included-sysexts"
|
||||
local image_contents="${image_name%.bin}_contents.txt"
|
||||
local image_contents_wtd="${image_name%.bin}_contents_wtd.txt"
|
||||
local image_packages="${image_name%.bin}_packages.txt"
|
||||
@ -141,9 +90,31 @@ create_prod_image() {
|
||||
emerge_to_image "${root_fs_dir}" "${base_pkg}"
|
||||
run_ldconfig "${root_fs_dir}"
|
||||
run_localedef "${root_fs_dir}"
|
||||
|
||||
local root_with_everything="${root_fs_dir}"
|
||||
|
||||
# Call helper script for adding sysexts to the base OS.
|
||||
# Helper will generate a rootfs dir with all packages (base OS and sysexts) included.
|
||||
local root_sysext_mergedir="${BUILD_DIR}/rootfs-with-sysext-pkgs"
|
||||
if [[ -n "${base_sysexts}" ]] ; then
|
||||
"${BUILD_LIBRARY_DIR}/sysext_prod_builder" \
|
||||
"${BOARD}" "${BUILD_DIR}" "${root_fs_dir}" \
|
||||
"${root_sysext_mergedir}" \
|
||||
"${root_fs_sysexts_output_dir}" \
|
||||
"${base_sysexts}"
|
||||
root_with_everything="${root_sysext_mergedir}"
|
||||
fi
|
||||
|
||||
|
||||
write_sbom "${root_with_everything}" "${BUILD_DIR}/${image_sbom}"
|
||||
write_licenses "${root_with_everything}" "${BUILD_DIR}/${image_licenses}"
|
||||
|
||||
if [[ -n "${base_sysexts}" ]] ; then
|
||||
sudo rm -rf "${root_sysext_mergedir}"
|
||||
fi
|
||||
|
||||
write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}"
|
||||
write_sbom "${root_fs_dir}" "${BUILD_DIR}/${image_sbom}"
|
||||
write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}"
|
||||
|
||||
insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}"
|
||||
insert_extra_slsa "${root_fs_dir}"
|
||||
|
||||
@ -190,19 +161,9 @@ EOF
|
||||
# Remove source locale data, only need to ship the compiled archive.
|
||||
sudo rm -rf ${root_fs_dir}/usr/share/i18n/
|
||||
|
||||
if [[ -n "${base_sysexts}" ]] ; then
|
||||
local grp_pkg=""
|
||||
local prev_pkginfo=""
|
||||
for grp_pkg in ${base_sysexts//,/ }; do
|
||||
create_prod_sysext "${root_fs_dir}"\
|
||||
"${BUILD_DIR}/${image_sysext_base}" \
|
||||
"${grp_pkg}" \
|
||||
"${prev_pkginfo}"
|
||||
prev_pkginfo="${grp_pkg//\//_}_pkginfo.raw"
|
||||
done
|
||||
fi
|
||||
|
||||
# Finish image will move files from /etc to /usr/share/flatcar/etc.
|
||||
# Note that image filesystem contents generated by finish_image will not
|
||||
# include sysext contents (only the sysext squashfs files themselves).
|
||||
finish_image \
|
||||
"${image_name}" \
|
||||
"${disk_layout}" \
|
||||
@ -218,6 +179,30 @@ EOF
|
||||
"${image_initrd_contents_wtd}" \
|
||||
"${image_disk_usage}"
|
||||
|
||||
# append sysext inventories to image contents files.
|
||||
if [[ -n "${base_sysexts}" ]] ; then
|
||||
local inventory_file="" image_basename="${image_name%.bin}"
|
||||
|
||||
for inventory_file in "${image_contents}" "${image_contents_wtd}" "${image_disk_usage}" "${image_packages}" ; do
|
||||
local suffix="${inventory_file/${image_basename}/}" sysext=""
|
||||
|
||||
info "Processing '${inventory_file}'"
|
||||
|
||||
for sysext in ${base_sysexts//,/ }; do
|
||||
local name="${sysext//\//_}"
|
||||
local sysext_inventory="${root_fs_sysexts_output_dir}/${name}${suffix}"
|
||||
if [[ ! -f "${sysext_inventory}" ]] ; then
|
||||
die "Sysext inventory file '${sysext//\//_}${suffix}' for '${inventory_file}' not found in '${root_fs_sysexts_output_dir}'"
|
||||
fi
|
||||
info "Adding sysext inventory '${name}${suffix}' to '${inventory_file}'"
|
||||
{
|
||||
echo -e "\n\n### Sysext ${name}.raw\n"
|
||||
cat "${sysext_inventory}"
|
||||
} >> "${BUILD_DIR}/${inventory_file}"
|
||||
done
|
||||
done
|
||||
fi
|
||||
|
||||
# Upload
|
||||
local to_upload=(
|
||||
"${BUILD_DIR}/${image_contents}"
|
||||
|
||||
151
build_library/sysext_prod_builder
Executable file
151
build_library/sysext_prod_builder
Executable file
@ -0,0 +1,151 @@
|
||||
#!/bin/bash
|
||||
# Copyright (c) 2023 by the Flatcar Maintainers.
|
||||
# Use of this source code is governed by the Apache 2.0 license.
|
||||
|
||||
# Helper script for building OS images w/ sysexts included.
|
||||
# Called by build_image -> prod_image_util.sh.
|
||||
# This is a separate script mainly so we can trap EXIT and clean up our mounts
|
||||
# without interfering with traps set by build_image.
|
||||
|
||||
# We're in build_library/, script root is one up
|
||||
SCRIPT_ROOT="$(cd "$(dirname "$(readlink -f "$0")")/../"; pwd)"
|
||||
. "${SCRIPT_ROOT}/common.sh" || exit 1
|
||||
|
||||
# Script must run inside the chroot
|
||||
assert_inside_chroot
|
||||
switch_to_strict_mode
|
||||
|
||||
. "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1
|
||||
|
||||
# Create a sysext from a package and install it to the OS image.
|
||||
# Conventions:
|
||||
# - For each <group>/<package>, <group>_<package>_pkginfo will be built. Can be used in subsequent calls
|
||||
# to build dependent sysexts.
|
||||
# - If ${BUILD_LIBRARY_DIR}/sysext_mangle_<group>_<package> exists it will be used as FS mangle script
|
||||
# when building the sysext.
|
||||
create_prod_sysext() {
|
||||
local BOARD="$1"
|
||||
local output_dir="$2"
|
||||
local workdir="$3"
|
||||
local base_sysext="$4"
|
||||
local install_root="$5"
|
||||
local grp_pkg="$6"
|
||||
local pkginfo="${7:-}"
|
||||
|
||||
local name="${grp_pkg//\//_}" # some-group/some-package => some-group_some-package
|
||||
local pkginfo_opt=""
|
||||
local manglefs_opt=""
|
||||
|
||||
local msg="Creating sysext '${grp_pkg}' ==> ${name}.raw"
|
||||
|
||||
# Include previous sysexts' pkginfo if supplied
|
||||
if [[ -n "${pkginfo}" ]] ; then
|
||||
if [[ ! -f "${output_dir}/${pkginfo}" ]] ; then
|
||||
die "Sysext build '${grp_pkg}': unable to find package info at '${output_dir}/${pkginfo}'."
|
||||
fi
|
||||
msg="${msg} w/ package info '${pkginfo}'"
|
||||
pkginfo_opt="--base_pkginfo=${output_dir}/${pkginfo}"
|
||||
fi
|
||||
|
||||
# Include FS mangle script if present
|
||||
if [[ -x "${BUILD_LIBRARY_DIR}/sysext_mangle_${name}" ]] ; then
|
||||
manglefs_opt="--manglefs_script=${BUILD_LIBRARY_DIR}/sysext_mangle_${name}"
|
||||
msg="${msg}, FS mangle script 'sysext_mangle_${name}'"
|
||||
fi
|
||||
|
||||
info "${msg}."
|
||||
|
||||
sudo "${SCRIPTS_DIR}/build_sysext" \
|
||||
--board="${BOARD}" \
|
||||
--image_builddir="${workdir}/sysext-build" \
|
||||
--squashfs_base="${base_sysext}" \
|
||||
--generate_pkginfo \
|
||||
${manglefs_opt} ${pkginfo_opt} \
|
||||
"${name}" "${grp_pkg}"
|
||||
|
||||
sudo mv "${workdir}/sysext-build/${name}.raw" "${workdir}/sysext-build/${name}_pkginfo.raw" \
|
||||
"${workdir}/sysext-build/${name}"_*.txt "${output_dir}"
|
||||
|
||||
sudo mkdir -p "${install_root}"/usr/share/flatcar/sysext
|
||||
sudo install -m 0644 -D "${output_dir}/${name}.raw" "${install_root}"/usr/share/flatcar/sysext/
|
||||
|
||||
sudo mkdir -p "${install_root}"/etc/extensions/
|
||||
sudo ln -sf "/usr/share/flatcar/sysext/${name}.raw" "${install_root}/etc/extensions/${name}.raw"
|
||||
}
|
||||
# --
|
||||
|
||||
BOARD="$1"
|
||||
BUILD_DIR="$2"
|
||||
root_fs_dir="$3"
|
||||
|
||||
merged_rootfs_dir="$4"
|
||||
sysext_output_dir="$5"
|
||||
|
||||
sysexts_list="$6"
|
||||
|
||||
grp_pkg=""
|
||||
prev_pkginfo=""
|
||||
sysext_workdir="${BUILD_DIR}/prod-sysext-work"
|
||||
sysext_mountdir="${BUILD_DIR}/prod-sysext-work/mounts"
|
||||
sysext_base="${sysext_workdir}/base-os.squashfs"
|
||||
|
||||
function cleanup() {
|
||||
sudo umount "${sysext_mountdir}"/* || true
|
||||
rm -rf "${sysext_workdir}" || true
|
||||
}
|
||||
# --
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
rm -rf "${sysext_workdir}" "${sysext_output_dir}"
|
||||
mkdir "${sysext_workdir}" "${sysext_output_dir}"
|
||||
|
||||
info "creating temporary base OS squashfs"
|
||||
sudo mksquashfs "${root_fs_dir}" "${sysext_base}" -noappend
|
||||
|
||||
# Build sysexts on top of root fs and mount sysexts' squashfs + pkginfo squashfs
|
||||
# for combined overlay later.
|
||||
prev_pkginfo=""
|
||||
sysext_lowerdirs="${sysext_mountdir}/rootfs-lower"
|
||||
for grp_pkg in ${sysexts_list//,/ }; do
|
||||
create_prod_sysext "${BOARD}" \
|
||||
"${sysext_output_dir}" \
|
||||
"${sysext_workdir}" \
|
||||
"${sysext_base}" \
|
||||
"${root_fs_dir}"\
|
||||
"${grp_pkg}" \
|
||||
"${prev_pkginfo}"
|
||||
name="${grp_pkg//\//_}"
|
||||
|
||||
mkdir -p "${sysext_mountdir}/${name}" \
|
||||
"${sysext_mountdir}/${name}_pkginfo"
|
||||
sudo mount -rt squashfs -o loop,nodev "${sysext_output_dir}/${name}.raw" \
|
||||
"${sysext_mountdir}/${name}"
|
||||
sudo mount -rt squashfs -o loop,nodev "${sysext_output_dir}/${name}_pkginfo.raw" \
|
||||
"${sysext_mountdir}/${name}_pkginfo"
|
||||
|
||||
sysext_lowerdirs="${sysext_lowerdirs}:${sysext_mountdir}/${name}"
|
||||
sysext_lowerdirs="${sysext_lowerdirs}:${sysext_mountdir}/${name}_pkginfo"
|
||||
|
||||
prev_pkginfo="${name}_pkginfo.raw"
|
||||
done
|
||||
|
||||
# Mount the combined overlay (base OS, sysexts, and syset pkginfos) and copy a snapshot
|
||||
# into the designated output dir for upper layers to process.
|
||||
mkdir -p "${sysext_mountdir}/rootfs-lower"
|
||||
sudo mount -rt squashfs -o loop,nodev "${sysext_base}" "${sysext_mountdir}/rootfs-lower"
|
||||
|
||||
# Mount overlay for report generation
|
||||
mkdir -p "${sysext_workdir}/.work"
|
||||
mkdir -p "${sysext_mountdir}/rootfs-upper"
|
||||
sudo mount -t overlay overlay \
|
||||
-o lowerdir="${sysext_lowerdirs}",upperdir="${sysext_mountdir}/rootfs-upper",workdir="${sysext_workdir}/.work" \
|
||||
"${sysext_mountdir}/rootfs-upper"
|
||||
|
||||
|
||||
sudo rm -rf "${merged_rootfs_dir}"
|
||||
sudo cp -a "${sysext_mountdir}/rootfs-upper" "${merged_rootfs_dir}"
|
||||
|
||||
|
||||
cleanup
|
||||
trap -- EXIT
|
||||
16
build_sysext
16
build_sysext
@ -130,10 +130,11 @@ cleanup() {
|
||||
)
|
||||
umount "${dirs[@]}" 2>/dev/null || true
|
||||
rm -rf "${dirs[@]}" || true
|
||||
if [[ -d "${BUILD_DIR}/img-pkginfo" ]] ; then
|
||||
umount "${BUILD_DIR}/img-pkginfo"/* 2>/dev/null || true
|
||||
rm -rf "${BUILD_DIR}/img-pkginfo" || true
|
||||
if [[ -d "${BUILD_DIR}/base-pkginfo" ]] ; then
|
||||
umount "${BUILD_DIR}/base-pkginfo"/* 2>/dev/null || true
|
||||
rm -rf "${BUILD_DIR}/base-pkginfo" || true
|
||||
fi
|
||||
rm -rf "${BUILD_DIR}/img-pkginfo"
|
||||
}
|
||||
|
||||
# Set up trap to execute cleanup() on script exit
|
||||
@ -143,7 +144,7 @@ ARCH=$(_get_sysext_arch "${FLAGS_board}")
|
||||
cleanup
|
||||
|
||||
# If we need to handle pkginfo squashfs files, create mount points under
|
||||
# ${BUILD_DIR}/img-pkginfo, mount the squashfs images, and add the mount paths to
|
||||
# ${BUILD_DIR}/base-pkginfo, mount the squashfs images, and add the mount paths to
|
||||
# the list of lowerdirs.
|
||||
pkginfo_lowerdirs=""
|
||||
if [[ -n "${FLAGS_base_pkginfo}" ]] ; then
|
||||
@ -156,7 +157,7 @@ if [[ -n "${FLAGS_base_pkginfo}" ]] ; then
|
||||
fi
|
||||
|
||||
pfile="$(basename "${ppath}")"
|
||||
pmdir="${BUILD_DIR}/img-pkginfo/${pfile}"
|
||||
pmdir="${BUILD_DIR}/base-pkginfo/${pfile}"
|
||||
mkdir -p "${pmdir}"
|
||||
mount -rt squashfs -o loop,nodev "${ppath}" "${pmdir}"
|
||||
pkginfo_lowerdirs="${pkginfo_lowerdirs}:${pmdir}"
|
||||
@ -169,6 +170,7 @@ mount -rt squashfs -o loop,nodev "${FLAGS_squashfs_base}" "${BUILD_DIR}/fs-root"
|
||||
mkdir "${BUILD_DIR}/install-root"
|
||||
mkdir "${BUILD_DIR}/workdir"
|
||||
mount -t overlay overlay -o lowerdir="${BUILD_DIR}/fs-root${pkginfo_lowerdirs}",upperdir="${BUILD_DIR}/install-root",workdir="${BUILD_DIR}/workdir" "${BUILD_DIR}/install-root"
|
||||
|
||||
VERSION_BOARD=$(grep "^VERSION=" ${BUILD_DIR}/fs-root/usr/lib/os-release | cut -d = -f 2-)
|
||||
if [ "$VERSION_BOARD" != "$FLATCAR_VERSION" ]; then
|
||||
warn "Base squashfs version: $VERSION_BOARD"
|
||||
@ -223,6 +225,10 @@ if [[ "$FLAGS_generate_pkginfo" = "${FLAGS_TRUE}" ]] ; then
|
||||
mksquashfs "${BUILD_DIR}/img-pkginfo" "${BUILD_DIR}/${SYSEXTNAME}_pkginfo.raw" -noappend
|
||||
fi
|
||||
|
||||
info "Writing ${SYSEXTNAME}_packages.txt"
|
||||
ROOT="${BUILD_DIR}/install-root" PORTAGE_CONFIGROOT="${BUILD_DIR}/install-root"\
|
||||
equery --no-color list --format '$cpv::$repo' '*' > "${BUILD_DIR}/${SYSEXTNAME}_packages.txt"
|
||||
|
||||
info "Removing non-/usr directories from sysext image"
|
||||
for entry in "${BUILD_DIR}/install-root"/*; do
|
||||
if [[ "${entry}" = */usr ]]; then
|
||||
|
||||
Loading…
Reference in New Issue
Block a user