mirrors/flatcar-scripts
1
0
Fork 0
mirror of https://github.com/flatcar/scripts.git synced 2026-05-05 04:06:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #969 from flatcar/krnowak/test-new-oem-script

Browse Source 
Use `build_sysext` for building OEM sysext images, drop old stuff
This commit is contained in:
Krzesimir Nowak 2023-07-05 12:44:04 +02:00 committed by GitHub
commit 3a879fb7c2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 46 additions and 313 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

216
build_library/oem_sysext_util.sh
View File

@ -1,216 +0,0 @@
#!/bin/bash
#
# Copyright (c) 2023 The Flatcar Maintainers.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
source "${BUILD_LIBRARY_DIR}/reports_util.sh" || exit 1
_generate_listing() {
local rootfs="${1%/}"; shift
local listing="${1}"; shift
local slashes="${rootfs//[^\/]}"
local slash_count="${#slashes}"
# Invoking find with sudo as it's used for traversing root-owned
# rootfs, which means that some places may be unreachable by the
# sdk user.
sudo find "${rootfs}//" | cut -d/ -f$((slash_count + 2))- | sort >"${listing}"
}
_prepend_action () {
local -n prepend_array="${1}"; shift
prepend_array=( "${#}" "${@}" "${prepend_array[@]}" )
}
_invoke_actions () {
local arg_count
local command
while [[ "${#}" -gt 0 ]]; do
arg_count="${1}"
shift
command=( "${@:1:${arg_count}}" )
shift "${arg_count}"
"${command[@]}" || :
done
}
# Architecture values are taken from systemd.unit(5).
declare -A SYSEXT_ARCHES
SYSEXT_ARCHES['amd64-usr']='x86-64'
SYSEXT_ARCHES['arm64-usr']='arm64'
declare -r SYSEXT_ARCHES
# Usage: _get_sysext_arch board [board...]
_get_sysext_arch() {
local board
for board in "$@"; do
if [[ ${#SYSEXT_ARCHES["${board}"]} -ne 0 ]]; then
echo "${SYSEXT_ARCHES["${board}"]}"
else
die "Unknown board '${board}'"
fi
done
}
oem_sysext_create() {
local oem="${1}"; shift
local board="${1}"; shift
local version_id="${1}"; shift
local prod_image="${1}"; shift
local prod_pkgdb="${1}"; shift
local work_dir="${1}"; shift
local base_pkg="coreos-base/${oem}"
local sysext_work_dir="${work_dir}/sysext-${oem}"
local prod_rw_image="${sysext_work_dir}/prod_for_sysext.bin"
local prod_rw_rootfs="${sysext_work_dir}/prod_rw_rootfs"
local sysext_overlay_work="${sysext_work_dir}/overlay.work"
local sysext_overlay_upper="${sysext_work_dir}/overlay.upper"
local cleanup_actions=()
trap '_invoke_actions "${cleanup_actions[@]}"' EXIT
_prepend_action cleanup_actions rmdir "${sysext_work_dir}"
mkdir -p "${sysext_work_dir}"
info 'Creating a production image copy for work rootfs'
_prepend_action cleanup_actions rm -f "${prod_rw_image}"
cp --sparse=always "${prod_image}" "${prod_rw_image}"
info 'Preparing work image for mounting'
"${BUILD_LIBRARY_DIR}/disk_util" --disk_layout=base \
tune --randomize_uuid "${prod_rw_image}" OEM
info "Mounting work image to ${prod_rw_rootfs}"
_prepend_action cleanup_actions rmdir "${prod_rw_rootfs}"
_prepend_action cleanup_actions "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout=base \
umount "${prod_rw_rootfs}"
"${BUILD_LIBRARY_DIR}/disk_util" --disk_layout=base \
mount "${prod_rw_image}" "${prod_rw_rootfs}"
local initial_files="${sysext_work_dir}/initial_files"
info "Generating list of initial files in work image"
_prepend_action cleanup_actions rm -f "${initial_files}"
_generate_listing "${prod_rw_rootfs}" "${initial_files}"
# /usr partition may be too small to fit the sysext files, so mount
# an overlay temporarily.
_prepend_action cleanup_actions sudo rm -rf "${sysext_overlay_work}" "${sysext_overlay_upper}"
mkdir -p "${sysext_overlay_work}" "${sysext_overlay_upper}"
_prepend_action cleanup_actions sudo umount "${prod_rw_rootfs}/usr"
sudo mount -t overlay usr-overlay \
-o lowerdir="${prod_rw_rootfs}/usr",upperdir="${sysext_overlay_upper}",workdir="${sysext_overlay_work}" \
"${prod_rw_rootfs}/usr"
info "Stuffing package database into into ${prod_rw_rootfs}"
sudo tar -xf "${prod_pkgdb}" -C "${prod_rw_rootfs}"
# Split into two steps because we want to always install
# $${base_pkg} from the ebuild (build_packages doesn't handle it)
# *but* we never want to build anything else from source
# here. emerge doesn't have a way to enforce this in a single
# command.
info "Building ${base_pkg}"
"emerge-${board}" --nodeps --buildpkgonly --usepkg n --verbose "${base_pkg}"
info "Installing ${base_pkg} to ${prod_rw_rootfs}"
sudo emerge \
--config-root="/build/${board}" \
--root="${prod_rw_rootfs}" \
--sysroot="${prod_rw_rootfs}" \
--root-deps=rdeps \
--usepkgonly \
--verbose \
"${base_pkg}"
info "Removing portage db from ${prod_rw_rootfs}"
sudo rm -rf \
"${prod_rw_rootfs}/var/cache/edb" \
"${prod_rw_rootfs}/var/db/pkg"
local all_files="${sysext_work_dir}/all_files"
local sysext_files="${sysext_work_dir}/sysext_files"
info "Generating list of files in work image after installing OEM package"
_prepend_action cleanup_actions rm -f "${all_files}"
_generate_listing "${prod_rw_rootfs}" "${all_files}"
info "Generating list of files for sysext image"
_prepend_action cleanup_actions rm -f "${sysext_files}"
comm -1 -3 "${initial_files}" "${all_files}" >"${sysext_files}"
info "Copying files for sysext image"
local sysext_rootfs="${sysext_work_dir}/sysext_rootfs"
_prepend_action cleanup_actions rm -rf "${sysext_rootfs}"
rsync --links --files-from="${sysext_files}" "${prod_rw_rootfs}" "${sysext_rootfs}"
info "Mangling files for sysext image"
local overlay_path mangle_fs
overlay_path=$(portageq get_repo_path / coreos)
mangle_fs="${overlay_path}/${base_pkg}/files/manglefs.sh"
if [[ -x "${mangle_fs}" ]]; then
"${mangle_fs}" "${sysext_rootfs}"
fi
local entry
info "Removing non-/usr directories from sysext image"
for entry in "${sysext_rootfs}"/*; do
if [[ "${entry}" = */usr ]]; then
continue
fi
info " Removing ${entry##*/}"
rm -rf "${entry}"
done
local metadata metadata_file metadata_version_entry
info "Adding sysext metadata"
mkdir -p "${sysext_rootfs}/usr/lib/extension-release.d"
if [[ "${version_id}" = 'initial' ]]; then
metadata_version_entry="SYSEXT_LEVEL=1.0"
else
metadata_version_entry="VERSION_ID=${version_id}"
fi
metadata=(
'ID=flatcar'
"${metadata_version_entry}"
"ARCHITECTURE=$(_get_sysext_arch "${board}")"
)
metadata_file="${sysext_rootfs}/usr/lib/extension-release.d/extension-release.${oem}"
printf '%s\n' "${metadata[@]}" >"${metadata_file}"
info "Generating a squashfs image"
local sysext_raw_image_filename="${oem}.raw"
local output_raw_image="${sysext_work_dir}/${sysext_raw_image_filename}"
_prepend_action cleanup_actions rm -f "${output_raw_image}"
mksquashfs "${sysext_rootfs}" "${output_raw_image}" -all-root
info "Generating image reports"
local sysext_mounted="${sysext_work_dir}/squashfs_mounted"
_prepend_action cleanup_actions rmdir "${sysext_mounted}"
mkdir "${sysext_mounted}"
_prepend_action cleanup_actions sudo umount "${sysext_mounted}"
sudo mount -t squashfs -o loop "${output_raw_image}" "${sysext_mounted}"
local contents="${sysext_raw_image_filename%.raw}_contents.txt"
local contents_wtd="${sysext_raw_image_filename%.raw}_contents_wtd.txt"
local disk_usage="${sysext_raw_image_filename%.raw}_disk_usage.txt"
_prepend_action cleanup_actions rm -f "${sysext_work_dir}/${contents}"
write_contents "${sysext_mounted}" "${sysext_work_dir}/${contents}"
_prepend_action cleanup_actions rm -f "${sysext_work_dir}/${contents_wtd}"
write_contents_with_technical_details "${sysext_mounted}" "${sysext_work_dir}/${contents_wtd}"
_prepend_action cleanup_actions rm -f "${sysext_work_dir}/${disk_usage}"
write_disk_space_usage_in_paths "${sysext_mounted}" "${sysext_work_dir}/${disk_usage}"
local to_move
for to_move in "${sysext_raw_image_filename}" "${contents}" "${contents_wtd}" "${disk_usage}"; do
mv "${sysext_work_dir}/${to_move}" "${work_dir}/${to_move}"
done
info "Alles jut, cleaning up"
trap - EXIT
_invoke_actions "${cleanup_actions[@]}"
}

37
build_library/vm_image_util.sh
View File

@ -71,7 +71,7 @@ VM_IMG_TYPE=DEFAULT
# Set at runtime to the source and destination image paths
VM_SRC_IMG=
VM_SRC_PKGDB=
VM_SRC_SYSEXT_IMG=
VM_TMP_IMG=
VM_TMP_DIR=
VM_TMP_ROOT=
@ -363,15 +363,15 @@ set_vm_paths() {
local src_dir="${1}"; shift
local dst_dir="${1}"; shift
local src_name="${1}"; shift
local pkgdb_name="${1}"; shift
local sysext_base_name="${1}"; shift
VM_SRC_IMG="${src_dir}/${src_name}"
if [[ ! -f "${VM_SRC_IMG}" ]]; then
die "Source image does not exist: ${VM_SRC_IMG}"
fi
VM_SRC_PKGDB="${src_dir}/${pkgdb_name}"
if [[ ! -f "${VM_SRC_PKGDB}" ]]; then
die "Source package database does not exist: ${VM_SRC_PKGDB}"
VM_SRC_SYSEXT_IMG="${src_dir}/${sysext_base_name}"
if [[ ! -f "${VM_SRC_SYSEXT_IMG}" ]]; then
die "Sysext base image does not exist: ${VM_SRC_SYSEXT_IMG}"
fi
local dst_name="$(_src_to_dst_name "${src_name}" "_image.$(_disk_ext)")"
@ -548,19 +548,30 @@ install_oem_sysext() {
local built_sysext_dir="${FLAGS_to}/${oem_sysext}-sysext"
local built_sysext_filename="${oem_sysext}.raw"
local built_sysext_path="${built_sysext_dir}/${built_sysext_filename}"
# TODO: Set 'version' to with "${FLATCAR_VERSION}" and
# 'version_id' to "${FLATCAR_VERSION_ID}" when we implement updating OEM sysexts
# TODO: Set 'version' to "${FLATCAR_VERSION}" and drop
# VERSION_FIELD_OVERRIDE when we implement updating OEM sysexts.
local version='initial'
local version_id='initial'
local build_oem_sysext_flags=(
local build_sysext_env=(
VERSION_FIELD_OVERRIDE='SYSEXT_LEVEL=1.0'
)
local metapkg="coreos-base/${oem_sysext}"
local build_sysext_flags=(
--board="${BOARD}"
--build_dir="${built_sysext_dir}"
--prod_image_path="${VM_SRC_IMG}"
--prod_pkgdb_path="${VM_SRC_PKGDB}"
--version_id="${version_id}"
--squashfs_base="${VM_SRC_SYSEXT_IMG}"
--metapkgs="${metapkg}"
)
local overlay_path mangle_fs
overlay_path=$(portageq get_repo_path / coreos)
mangle_fs="${overlay_path}/${metapkg}/files/manglefs.sh"
if [[ -x "${mangle_fs}" ]]; then
build_sysext_flags+=(
--manglefs_script="${mangle_fs}"
)
fi
"${SCRIPT_ROOT}/build_oem_sysext" "${build_oem_sysext_flags[@]}" "${oem_sysext}"
mkdir -p "${built_sysext_dir}"
sudo "${build_sysext_env[@]}" "${SCRIPT_ROOT}/build_sysext" "${build_sysext_flags[@]}" "${oem_sysext}"
local installed_sysext_oem_dir='/oem/sysext'
local installed_sysext_file_prefix="${oem_sysext}-${version}"

77
build_oem_sysext
View File

@ -1,77 +0,0 @@
#!/bin/bash
#
# Copyright (c) 2023 The Flatcar Maintainers.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
. "${SCRIPT_ROOT}/common.sh" || exit 1
# Script must run inside the chroot
assert_inside_chroot
assert_not_root_user
# Developer-visible flags.
DEFINE_string board "${DEFAULT_BOARD}" \
"The board to build an image for."
DEFINE_string build_dir "" \
"Directory in which to place image result directories (named by version)"
DEFINE_string prod_image_path "" \
"Path to the generic production image"
DEFINE_string prod_pkgdb_path "" \
"Path to the tarball with portage package database from generic image production image"
DEFINE_string version_id "${FLATCAR_VERSION_ID}" \
"Version ID stored inside the sysext extension"
FLAGS_HELP="USAGE: build_oem_sysext [flags] [oem name].
This script is used to build a Flatcar OEM sysext images.
The built image is in <build_dir>/oem-<oem>.raw.
Examples:
build_oem_sysext \
--board=amd64-usr \
--build_dir=<build_dir> \
--prod_image_path=<path_to_bin_file> \
--prod_pkgdb_path=<path_to_pkgdb_tarbal> \
--version_id=\"\${FLATCAR_VERSION_ID}\" \
oem-azure
...
"
show_help_if_requested "$@"
# Parse command line.
FLAGS "$@" || exit 1
if [[ -z "${FLAGS_ARGV}" ]]; then
echo 'No OEM given'
exit 0
fi
eval set -- "${FLAGS_ARGV}"
# Only now can we die on error. shflags functions leak non-zero error codes,
# so will die prematurely if 'switch_to_strict_mode' is specified before now.
switch_to_strict_mode
# N.B. Ordering matters for some of the libraries below, because
# some of the files contain initialization used by later files.
. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1
. "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
. "${BUILD_LIBRARY_DIR}/oem_sysext_util.sh" || exit 1
BUILD_DIR=${FLAGS_build_dir:-"${BUILD_DIR}"}
if [[ -z "${FLAGS_prod_image_path}" ]]; then
error "--prod_image_path is required."
exit 1
fi
if [[ -z "${FLAGS_prod_pkgdb_path}" ]]; then
error "--prod_pkgdb_path is required."
exit 1
fi
for oem; do
oem_sysext_create "${oem}" "${BOARD}" "${FLAGS_version_id}" "${FLAGS_prod_image_path}" "${FLAGS_prod_pkgdb_path}" "${BUILD_DIR}"
done

23
build_sysext
View File

@ -28,7 +28,7 @@ DEFINE_string squashfs_base '' \
FLAGS_HELP="USAGE: build_sysext [flags] [sysext name] [binary packages to install into image].
This script is used to build a Flatcar sysext image.
This script is used to build a Flatcar sysext image.
Examples:
@ -59,7 +59,8 @@ FLAGS "$@" || exit 1
eval set -- "${FLAGS_ARGV}"
. "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
source "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
source "${BUILD_LIBRARY_DIR}/reports_util.sh" || exit 1
if [[ -z "${FLAGS_build_dir}" ]]; then
die "Need a build directory to be specified with a --build_dir option"
@ -88,8 +89,14 @@ _get_sysext_arch() {
set -euo pipefail
cleanup() {
umount "${BUILD_DIR}/fs-root" "${BUILD_DIR}/install-root" "${BUILD_DIR}/workdir" 2>/dev/null || true
rm -rf "${BUILD_DIR}/fs-root" "${BUILD_DIR}/install-root" "${BUILD_DIR}/workdir" || true
local dirs=(
"${BUILD_DIR}/fs-root"
"${BUILD_DIR}/install-root"
"${BUILD_DIR}/workdir"
"${BUILD_DIR}/img-rootfs"
)
umount "${dirs[@]}" 2>/dev/null || true
rm -rf "${dirs[@]}" || true
}
if [[ ${#} -lt 1 ]]; then
@ -169,3 +176,11 @@ all_fields=(
printf '%s\n' "${all_fields[@]}" >"${BUILD_DIR}/install-root/usr/lib/extension-release.d/extension-release.${SYSEXTNAME}"
mksquashfs "${BUILD_DIR}/install-root" "${BUILD_DIR}/${SYSEXTNAME}.raw"
rm -rf "${BUILD_DIR}"/{fs-root,install-root,workdir}
# Generate reports
mkdir "${BUILD_DIR}/img-rootfs"
mount -rt squashfs -o loop,nodev "${BUILD_DIR}/${SYSEXTNAME}.raw" "${BUILD_DIR}/img-rootfs"
write_contents "${BUILD_DIR}/img-rootfs" "${BUILD_DIR}/${SYSEXTNAME}_contents.txt"
write_contents_with_technical_details "${BUILD_DIR}/img-rootfs" "${BUILD_DIR}/${SYSEXTNAME}_contents_wtd.txt"
write_disk_space_usage_in_paths "${BUILD_DIR}/img-rootfs" "${BUILD_DIR}/${SYSEXTNAME}_disk_usage.txt"
umount "${BUILD_DIR}/img-rootfs"

2
ci-automation/vms.sh
View File

@ -105,7 +105,7 @@ function _vm_build_impl() {
local images_in="images-in/"
local file
rm -rf "${images_in}"
for file in flatcar_production_image.bin.bz2 flatcar_production_image_pkgdb.tar.xz version.txt; do
for file in flatcar_production_image.bin.bz2 flatcar_production_image_sysext.squashfs version.txt; do
copy_from_buildcache "images/${arch}/${vernum}/${file}" "${images_in}"
done
lbunzip2 "${images_in}/flatcar_production_image.bin.bz2"

2
common.sh
View File

@ -425,7 +425,7 @@ BUILD_DIR=
# Standard filenames
FLATCAR_DEVELOPER_CONTAINER_NAME="flatcar_developer_container.bin"
FLATCAR_PRODUCTION_IMAGE_NAME="flatcar_production_image.bin"
FLATCAR_PRODUCTION_IMAGE_PKGDB_NAME="flatcar_production_image_pkgdb.tar.xz"
FLATCAR_PRODUCTION_IMAGE_SYSEXT_BASE="flatcar_production_image_sysext.squashfs"
# -----------------------------------------------------------------------------
# Functions

2
image_to_vm.sh
View File

@ -105,7 +105,7 @@ if [ -f "${FLAGS_from}/version.txt" ]; then
FLATCAR_VERSION_STRING="${FLATCAR_VERSION}"
fi
set_vm_paths "${FLAGS_from}" "${FLAGS_to}" "${FLATCAR_PRODUCTION_IMAGE_NAME}" "${FLATCAR_PRODUCTION_IMAGE_PKGDB_NAME}"
set_vm_paths "${FLAGS_from}" "${FLAGS_to}" "${FLATCAR_PRODUCTION_IMAGE_NAME}" "${FLATCAR_PRODUCTION_IMAGE_SYSEXT_BASE}"
# Make sure things are cleaned up on failure
trap vm_cleanup EXIT