diff --git a/build_library/oem_sysext_util.sh b/build_library/oem_sysext_util.sh deleted file mode 100755 index e5e057bdcd..0000000000 --- a/build_library/oem_sysext_util.sh +++ /dev/null @@ -1,216 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2023 The Flatcar Maintainers. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -source "${BUILD_LIBRARY_DIR}/reports_util.sh" || exit 1 - -_generate_listing() { - local rootfs="${1%/}"; shift - local listing="${1}"; shift - - local slashes="${rootfs//[^\/]}" - local slash_count="${#slashes}" - - # Invoking find with sudo as it's used for traversing root-owned - # rootfs, which means that some places may be unreachable by the - # sdk user. - sudo find "${rootfs}//" | cut -d/ -f$((slash_count + 2))- | sort >"${listing}" -} - -_prepend_action () { - local -n prepend_array="${1}"; shift - - prepend_array=( "${#}" "${@}" "${prepend_array[@]}" ) -} - -_invoke_actions () { - local arg_count - local command - while [[ "${#}" -gt 0 ]]; do - arg_count="${1}" - shift - command=( "${@:1:${arg_count}}" ) - shift "${arg_count}" - "${command[@]}" || : - done -} - -# Architecture values are taken from systemd.unit(5). -declare -A SYSEXT_ARCHES -SYSEXT_ARCHES['amd64-usr']='x86-64' -SYSEXT_ARCHES['arm64-usr']='arm64' - -declare -r SYSEXT_ARCHES - -# Usage: _get_sysext_arch board [board...] -_get_sysext_arch() { - local board - for board in "$@"; do - if [[ ${#SYSEXT_ARCHES["${board}"]} -ne 0 ]]; then - echo "${SYSEXT_ARCHES["${board}"]}" - else - die "Unknown board '${board}'" - fi - done -} - -oem_sysext_create() { - local oem="${1}"; shift - local board="${1}"; shift - local version_id="${1}"; shift - local prod_image="${1}"; shift - local prod_pkgdb="${1}"; shift - local work_dir="${1}"; shift - - local base_pkg="coreos-base/${oem}" - local sysext_work_dir="${work_dir}/sysext-${oem}" - local prod_rw_image="${sysext_work_dir}/prod_for_sysext.bin" - local prod_rw_rootfs="${sysext_work_dir}/prod_rw_rootfs" - local sysext_overlay_work="${sysext_work_dir}/overlay.work" - local sysext_overlay_upper="${sysext_work_dir}/overlay.upper" - - local cleanup_actions=() - trap '_invoke_actions "${cleanup_actions[@]}"' EXIT - - _prepend_action cleanup_actions rmdir "${sysext_work_dir}" - mkdir -p "${sysext_work_dir}" - - info 'Creating a production image copy for work rootfs' - _prepend_action cleanup_actions rm -f "${prod_rw_image}" - cp --sparse=always "${prod_image}" "${prod_rw_image}" - - info 'Preparing work image for mounting' - "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout=base \ - tune --randomize_uuid "${prod_rw_image}" OEM - - info "Mounting work image to ${prod_rw_rootfs}" - _prepend_action cleanup_actions rmdir "${prod_rw_rootfs}" - _prepend_action cleanup_actions "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout=base \ - umount "${prod_rw_rootfs}" - "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout=base \ - mount "${prod_rw_image}" "${prod_rw_rootfs}" - - local initial_files="${sysext_work_dir}/initial_files" - info "Generating list of initial files in work image" - _prepend_action cleanup_actions rm -f "${initial_files}" - _generate_listing "${prod_rw_rootfs}" "${initial_files}" - - # /usr partition may be too small to fit the sysext files, so mount - # an overlay temporarily. - _prepend_action cleanup_actions sudo rm -rf "${sysext_overlay_work}" "${sysext_overlay_upper}" - mkdir -p "${sysext_overlay_work}" "${sysext_overlay_upper}" - - _prepend_action cleanup_actions sudo umount "${prod_rw_rootfs}/usr" - sudo mount -t overlay usr-overlay \ - -o lowerdir="${prod_rw_rootfs}/usr",upperdir="${sysext_overlay_upper}",workdir="${sysext_overlay_work}" \ - "${prod_rw_rootfs}/usr" - - info "Stuffing package database into into ${prod_rw_rootfs}" - sudo tar -xf "${prod_pkgdb}" -C "${prod_rw_rootfs}" - - # Split into two steps because we want to always install - # $${base_pkg} from the ebuild (build_packages doesn't handle it) - # *but* we never want to build anything else from source - # here. emerge doesn't have a way to enforce this in a single - # command. - info "Building ${base_pkg}" - "emerge-${board}" --nodeps --buildpkgonly --usepkg n --verbose "${base_pkg}" - - info "Installing ${base_pkg} to ${prod_rw_rootfs}" - sudo emerge \ - --config-root="/build/${board}" \ - --root="${prod_rw_rootfs}" \ - --sysroot="${prod_rw_rootfs}" \ - --root-deps=rdeps \ - --usepkgonly \ - --verbose \ - "${base_pkg}" - - info "Removing portage db from ${prod_rw_rootfs}" - sudo rm -rf \ - "${prod_rw_rootfs}/var/cache/edb" \ - "${prod_rw_rootfs}/var/db/pkg" - - local all_files="${sysext_work_dir}/all_files" - local sysext_files="${sysext_work_dir}/sysext_files" - - info "Generating list of files in work image after installing OEM package" - _prepend_action cleanup_actions rm -f "${all_files}" - _generate_listing "${prod_rw_rootfs}" "${all_files}" - - info "Generating list of files for sysext image" - _prepend_action cleanup_actions rm -f "${sysext_files}" - comm -1 -3 "${initial_files}" "${all_files}" >"${sysext_files}" - - info "Copying files for sysext image" - local sysext_rootfs="${sysext_work_dir}/sysext_rootfs" - _prepend_action cleanup_actions rm -rf "${sysext_rootfs}" - rsync --links --files-from="${sysext_files}" "${prod_rw_rootfs}" "${sysext_rootfs}" - - info "Mangling files for sysext image" - local overlay_path mangle_fs - overlay_path=$(portageq get_repo_path / coreos) - mangle_fs="${overlay_path}/${base_pkg}/files/manglefs.sh" - if [[ -x "${mangle_fs}" ]]; then - "${mangle_fs}" "${sysext_rootfs}" - fi - - local entry - info "Removing non-/usr directories from sysext image" - for entry in "${sysext_rootfs}"/*; do - if [[ "${entry}" = */usr ]]; then - continue - fi - info " Removing ${entry##*/}" - rm -rf "${entry}" - done - - local metadata metadata_file metadata_version_entry - info "Adding sysext metadata" - mkdir -p "${sysext_rootfs}/usr/lib/extension-release.d" - if [[ "${version_id}" = 'initial' ]]; then - metadata_version_entry="SYSEXT_LEVEL=1.0" - else - metadata_version_entry="VERSION_ID=${version_id}" - fi - metadata=( - 'ID=flatcar' - "${metadata_version_entry}" - "ARCHITECTURE=$(_get_sysext_arch "${board}")" - ) - metadata_file="${sysext_rootfs}/usr/lib/extension-release.d/extension-release.${oem}" - printf '%s\n' "${metadata[@]}" >"${metadata_file}" - - info "Generating a squashfs image" - local sysext_raw_image_filename="${oem}.raw" - local output_raw_image="${sysext_work_dir}/${sysext_raw_image_filename}" - _prepend_action cleanup_actions rm -f "${output_raw_image}" - mksquashfs "${sysext_rootfs}" "${output_raw_image}" -all-root - - info "Generating image reports" - local sysext_mounted="${sysext_work_dir}/squashfs_mounted" - _prepend_action cleanup_actions rmdir "${sysext_mounted}" - mkdir "${sysext_mounted}" - _prepend_action cleanup_actions sudo umount "${sysext_mounted}" - sudo mount -t squashfs -o loop "${output_raw_image}" "${sysext_mounted}" - local contents="${sysext_raw_image_filename%.raw}_contents.txt" - local contents_wtd="${sysext_raw_image_filename%.raw}_contents_wtd.txt" - local disk_usage="${sysext_raw_image_filename%.raw}_disk_usage.txt" - _prepend_action cleanup_actions rm -f "${sysext_work_dir}/${contents}" - write_contents "${sysext_mounted}" "${sysext_work_dir}/${contents}" - _prepend_action cleanup_actions rm -f "${sysext_work_dir}/${contents_wtd}" - write_contents_with_technical_details "${sysext_mounted}" "${sysext_work_dir}/${contents_wtd}" - _prepend_action cleanup_actions rm -f "${sysext_work_dir}/${disk_usage}" - write_disk_space_usage_in_paths "${sysext_mounted}" "${sysext_work_dir}/${disk_usage}" - - local to_move - for to_move in "${sysext_raw_image_filename}" "${contents}" "${contents_wtd}" "${disk_usage}"; do - mv "${sysext_work_dir}/${to_move}" "${work_dir}/${to_move}" - done - - info "Alles jut, cleaning up" - trap - EXIT - _invoke_actions "${cleanup_actions[@]}" -} diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index b9daa2e377..0cf4154de3 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -71,7 +71,7 @@ VM_IMG_TYPE=DEFAULT # Set at runtime to the source and destination image paths VM_SRC_IMG= -VM_SRC_PKGDB= +VM_SRC_SYSEXT_IMG= VM_TMP_IMG= VM_TMP_DIR= VM_TMP_ROOT= @@ -363,15 +363,15 @@ set_vm_paths() { local src_dir="${1}"; shift local dst_dir="${1}"; shift local src_name="${1}"; shift - local pkgdb_name="${1}"; shift + local sysext_base_name="${1}"; shift VM_SRC_IMG="${src_dir}/${src_name}" if [[ ! -f "${VM_SRC_IMG}" ]]; then die "Source image does not exist: ${VM_SRC_IMG}" fi - VM_SRC_PKGDB="${src_dir}/${pkgdb_name}" - if [[ ! -f "${VM_SRC_PKGDB}" ]]; then - die "Source package database does not exist: ${VM_SRC_PKGDB}" + VM_SRC_SYSEXT_IMG="${src_dir}/${sysext_base_name}" + if [[ ! -f "${VM_SRC_SYSEXT_IMG}" ]]; then + die "Sysext base image does not exist: ${VM_SRC_SYSEXT_IMG}" fi local dst_name="$(_src_to_dst_name "${src_name}" "_image.$(_disk_ext)")" @@ -548,19 +548,30 @@ install_oem_sysext() { local built_sysext_dir="${FLAGS_to}/${oem_sysext}-sysext" local built_sysext_filename="${oem_sysext}.raw" local built_sysext_path="${built_sysext_dir}/${built_sysext_filename}" - # TODO: Set 'version' to with "${FLATCAR_VERSION}" and - # 'version_id' to "${FLATCAR_VERSION_ID}" when we implement updating OEM sysexts + # TODO: Set 'version' to "${FLATCAR_VERSION}" and drop + # VERSION_FIELD_OVERRIDE when we implement updating OEM sysexts. local version='initial' - local version_id='initial' - local build_oem_sysext_flags=( + local build_sysext_env=( + VERSION_FIELD_OVERRIDE='SYSEXT_LEVEL=1.0' + ) + local metapkg="coreos-base/${oem_sysext}" + local build_sysext_flags=( --board="${BOARD}" --build_dir="${built_sysext_dir}" - --prod_image_path="${VM_SRC_IMG}" - --prod_pkgdb_path="${VM_SRC_PKGDB}" - --version_id="${version_id}" + --squashfs_base="${VM_SRC_SYSEXT_IMG}" + --metapkgs="${metapkg}" ) + local overlay_path mangle_fs + overlay_path=$(portageq get_repo_path / coreos) + mangle_fs="${overlay_path}/${metapkg}/files/manglefs.sh" + if [[ -x "${mangle_fs}" ]]; then + build_sysext_flags+=( + --manglefs_script="${mangle_fs}" + ) + fi - "${SCRIPT_ROOT}/build_oem_sysext" "${build_oem_sysext_flags[@]}" "${oem_sysext}" + mkdir -p "${built_sysext_dir}" + sudo "${build_sysext_env[@]}" "${SCRIPT_ROOT}/build_sysext" "${build_sysext_flags[@]}" "${oem_sysext}" local installed_sysext_oem_dir='/oem/sysext' local installed_sysext_file_prefix="${oem_sysext}-${version}" diff --git a/build_oem_sysext b/build_oem_sysext deleted file mode 100755 index ece37ebd8f..0000000000 --- a/build_oem_sysext +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2023 The Flatcar Maintainers. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -SCRIPT_ROOT=$(dirname $(readlink -f "$0")) -. "${SCRIPT_ROOT}/common.sh" || exit 1 - -# Script must run inside the chroot -assert_inside_chroot - -assert_not_root_user - -# Developer-visible flags. -DEFINE_string board "${DEFAULT_BOARD}" \ - "The board to build an image for." -DEFINE_string build_dir "" \ - "Directory in which to place image result directories (named by version)" -DEFINE_string prod_image_path "" \ - "Path to the generic production image" -DEFINE_string prod_pkgdb_path "" \ - "Path to the tarball with portage package database from generic image production image" -DEFINE_string version_id "${FLATCAR_VERSION_ID}" \ - "Version ID stored inside the sysext extension" - -FLAGS_HELP="USAGE: build_oem_sysext [flags] [oem name]. -This script is used to build a Flatcar OEM sysext images. -The built image is in /oem-.raw. - -Examples: - -build_oem_sysext \ - --board=amd64-usr \ - --build_dir= \ - --prod_image_path= \ - --prod_pkgdb_path= \ - --version_id=\"\${FLATCAR_VERSION_ID}\" \ - oem-azure -... -" -show_help_if_requested "$@" - -# Parse command line. -FLAGS "$@" || exit 1 -if [[ -z "${FLAGS_ARGV}" ]]; then - echo 'No OEM given' - exit 0 -fi - -eval set -- "${FLAGS_ARGV}" - -# Only now can we die on error. shflags functions leak non-zero error codes, -# so will die prematurely if 'switch_to_strict_mode' is specified before now. -switch_to_strict_mode - -# N.B. Ordering matters for some of the libraries below, because -# some of the files contain initialization used by later files. -. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 -. "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 -. "${BUILD_LIBRARY_DIR}/oem_sysext_util.sh" || exit 1 - -BUILD_DIR=${FLAGS_build_dir:-"${BUILD_DIR}"} - -if [[ -z "${FLAGS_prod_image_path}" ]]; then - error "--prod_image_path is required." - exit 1 -fi - -if [[ -z "${FLAGS_prod_pkgdb_path}" ]]; then - error "--prod_pkgdb_path is required." - exit 1 -fi - -for oem; do - oem_sysext_create "${oem}" "${BOARD}" "${FLAGS_version_id}" "${FLAGS_prod_image_path}" "${FLAGS_prod_pkgdb_path}" "${BUILD_DIR}" -done diff --git a/build_sysext b/build_sysext index 25fcdd2d7b..710049770c 100755 --- a/build_sysext +++ b/build_sysext @@ -28,7 +28,7 @@ DEFINE_string squashfs_base '' \ FLAGS_HELP="USAGE: build_sysext [flags] [sysext name] [binary packages to install into image]. -This script is used to build a Flatcar sysext image. +This script is used to build a Flatcar sysext image. Examples: @@ -59,7 +59,8 @@ FLAGS "$@" || exit 1 eval set -- "${FLAGS_ARGV}" -. "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 +source "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 +source "${BUILD_LIBRARY_DIR}/reports_util.sh" || exit 1 if [[ -z "${FLAGS_build_dir}" ]]; then die "Need a build directory to be specified with a --build_dir option" @@ -88,8 +89,14 @@ _get_sysext_arch() { set -euo pipefail cleanup() { - umount "${BUILD_DIR}/fs-root" "${BUILD_DIR}/install-root" "${BUILD_DIR}/workdir" 2>/dev/null || true - rm -rf "${BUILD_DIR}/fs-root" "${BUILD_DIR}/install-root" "${BUILD_DIR}/workdir" || true + local dirs=( + "${BUILD_DIR}/fs-root" + "${BUILD_DIR}/install-root" + "${BUILD_DIR}/workdir" + "${BUILD_DIR}/img-rootfs" + ) + umount "${dirs[@]}" 2>/dev/null || true + rm -rf "${dirs[@]}" || true } if [[ ${#} -lt 1 ]]; then @@ -169,3 +176,11 @@ all_fields=( printf '%s\n' "${all_fields[@]}" >"${BUILD_DIR}/install-root/usr/lib/extension-release.d/extension-release.${SYSEXTNAME}" mksquashfs "${BUILD_DIR}/install-root" "${BUILD_DIR}/${SYSEXTNAME}.raw" rm -rf "${BUILD_DIR}"/{fs-root,install-root,workdir} + +# Generate reports +mkdir "${BUILD_DIR}/img-rootfs" +mount -rt squashfs -o loop,nodev "${BUILD_DIR}/${SYSEXTNAME}.raw" "${BUILD_DIR}/img-rootfs" +write_contents "${BUILD_DIR}/img-rootfs" "${BUILD_DIR}/${SYSEXTNAME}_contents.txt" +write_contents_with_technical_details "${BUILD_DIR}/img-rootfs" "${BUILD_DIR}/${SYSEXTNAME}_contents_wtd.txt" +write_disk_space_usage_in_paths "${BUILD_DIR}/img-rootfs" "${BUILD_DIR}/${SYSEXTNAME}_disk_usage.txt" +umount "${BUILD_DIR}/img-rootfs" diff --git a/ci-automation/vms.sh b/ci-automation/vms.sh index a27cba0175..1a64b7fda8 100644 --- a/ci-automation/vms.sh +++ b/ci-automation/vms.sh @@ -105,7 +105,7 @@ function _vm_build_impl() { local images_in="images-in/" local file rm -rf "${images_in}" - for file in flatcar_production_image.bin.bz2 flatcar_production_image_pkgdb.tar.xz version.txt; do + for file in flatcar_production_image.bin.bz2 flatcar_production_image_sysext.squashfs version.txt; do copy_from_buildcache "images/${arch}/${vernum}/${file}" "${images_in}" done lbunzip2 "${images_in}/flatcar_production_image.bin.bz2" diff --git a/common.sh b/common.sh index eafbc928c1..0ca5346878 100644 --- a/common.sh +++ b/common.sh @@ -425,7 +425,7 @@ BUILD_DIR= # Standard filenames FLATCAR_DEVELOPER_CONTAINER_NAME="flatcar_developer_container.bin" FLATCAR_PRODUCTION_IMAGE_NAME="flatcar_production_image.bin" -FLATCAR_PRODUCTION_IMAGE_PKGDB_NAME="flatcar_production_image_pkgdb.tar.xz" +FLATCAR_PRODUCTION_IMAGE_SYSEXT_BASE="flatcar_production_image_sysext.squashfs" # ----------------------------------------------------------------------------- # Functions diff --git a/image_to_vm.sh b/image_to_vm.sh index 525490e313..2a6c144766 100755 --- a/image_to_vm.sh +++ b/image_to_vm.sh @@ -105,7 +105,7 @@ if [ -f "${FLAGS_from}/version.txt" ]; then FLATCAR_VERSION_STRING="${FLATCAR_VERSION}" fi -set_vm_paths "${FLAGS_from}" "${FLAGS_to}" "${FLATCAR_PRODUCTION_IMAGE_NAME}" "${FLATCAR_PRODUCTION_IMAGE_PKGDB_NAME}" +set_vm_paths "${FLAGS_from}" "${FLAGS_to}" "${FLATCAR_PRODUCTION_IMAGE_NAME}" "${FLATCAR_PRODUCTION_IMAGE_SYSEXT_BASE}" # Make sure things are cleaned up on failure trap vm_cleanup EXIT