mirror of
https://github.com/flatcar/scripts.git
synced 2026-02-07 16:51:41 +01:00
10b808642b
Move OEM sysext building from the vms phase to the image phase. This ensures OEM sysexts are signed with the same ephemeral key as other sysexts, which is generated during image build and discarded afterward. - Add create_oem_sysexts() to build all OEM sysexts during image build - Add oem_sysexts.sh with OEM sysext definitions - Update install_oem_sysext() to use prebuilt sysexts - Add OEM sysext download to vms.sh for CI builds Signed-off-by: Daniel Zatovic <daniel.zatovic@gmail.com>
392 lines
14 KiB
Bash
Executable File
392 lines
14 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
. "$(dirname "$0")/common.sh" || exit 1
|
|
|
|
# Script must run inside the chroot
|
|
assert_inside_chroot
|
|
|
|
assert_not_root_user
|
|
|
|
# Developer-visible flags.
|
|
DEFINE_string board "${DEFAULT_BOARD}" \
|
|
"The board to build packages for."
|
|
DEFINE_boolean usepkg "${FLAGS_TRUE}" \
|
|
"Use binary packages when possible."
|
|
DEFINE_boolean usepkgonly "${FLAGS_FALSE}" \
|
|
"Only use/download binary packages. Implies --noworkon"
|
|
DEFINE_string usepkg_exclude "" \
|
|
"Exclude these binary packages."
|
|
DEFINE_boolean getbinpkg "${FLAGS_TRUE}" \
|
|
"Download binary packages from remote repository."
|
|
DEFINE_string getbinpkgver "" \
|
|
"Use binary packages from a specific version."
|
|
DEFINE_boolean workon "${FLAGS_TRUE}" \
|
|
"Automatically rebuild updated flatcar-workon packages."
|
|
DEFINE_boolean fetchonly "${FLAGS_FALSE}" \
|
|
"Don't build anything, instead only fetch what is needed."
|
|
DEFINE_boolean rebuild "${FLAGS_FALSE}" \
|
|
"Automatically rebuild dependencies."
|
|
DEFINE_boolean skip_toolchain_update "${FLAGS_FALSE}" \
|
|
"Don't update toolchain automatically."
|
|
DEFINE_boolean skip_chroot_upgrade "${FLAGS_FALSE}" \
|
|
"Don't run the chroot upgrade automatically; use with care."
|
|
DEFINE_boolean only_resolve_circular_deps "${FLAGS_FALSE}" \
|
|
"Don't build all packages; only resolve circular dependencies, then stop."
|
|
DEFINE_boolean debug_emerge "${FLAGS_FALSE}" \
|
|
"Enable debug output for emerge."
|
|
|
|
# include upload options
|
|
. "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
|
|
|
|
FLAGS_HELP="usage: $(basename $0) [flags] [packages]
|
|
|
|
build_packages updates the set of binary packages needed by CoreOS. It will
|
|
cross compile all packages that have been updated into the given target's root
|
|
and build binary packages as a side-effect. The output packages will be picked
|
|
up by the build_image script to put together a bootable CoreOS image.
|
|
|
|
If [packages] are specified, only build those specific packages (and any
|
|
dependencies they might need).
|
|
"
|
|
|
|
# Parse command line
|
|
FLAGS "$@" || exit 1
|
|
eval set -- "${FLAGS_ARGV}"
|
|
|
|
# Die on any errors.
|
|
switch_to_strict_mode
|
|
|
|
# TODO(marineam): specify the default top-level ebuild in the portage profile.
|
|
# I could have sworn we did that or similar but maybe got lost at some point.
|
|
if [[ $# -eq 0 ]]; then
|
|
set -- @system coreos-devel/board-packages
|
|
fi
|
|
|
|
if [[ -z "${FLAGS_board}" ]]; then
|
|
echo "Error: --board is required."
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then
|
|
for flag in usepkg getbinpkg; do
|
|
fvar="FLAGS_${flag}"
|
|
if [[ "${!fvar}" -ne "${FLAGS_TRUE}" ]]; then
|
|
die_notrace "--usepkgonly is incompatible with --no${flag}"
|
|
fi
|
|
done
|
|
if [[ "${FLAGS_rebuild}" -eq "${FLAGS_TRUE}" ]]; then
|
|
die_notrace "--usepkgonly is incompatible with --rebuild"
|
|
fi
|
|
FLAGS_workon="${FLAGS_FALSE}"
|
|
fi
|
|
|
|
# Before we can run any tools, we need to update chroot or setup_board.
|
|
UPDATE_ARGS=( --regen_configs )
|
|
if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then
|
|
UPDATE_ARGS+=( --usepkg )
|
|
if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then
|
|
UPDATE_ARGS+=( --usepkgonly )
|
|
else
|
|
UPDATE_ARGS+=( --nousepkgonly )
|
|
fi
|
|
if [[ "${FLAGS_getbinpkg}" -eq "${FLAGS_TRUE}" ]]; then
|
|
UPDATE_ARGS+=( --getbinpkg )
|
|
else
|
|
UPDATE_ARGS+=( --nogetbinpkg )
|
|
fi
|
|
if [[ -n "${FLAGS_getbinpkgver}" ]]; then
|
|
UPDATE_ARGS+=( --getbinpkgver="${FLAGS_getbinpkgver}" )
|
|
fi
|
|
else
|
|
UPDATE_ARGS+=( --nousepkg )
|
|
fi
|
|
if [ "${FLAGS_skip_toolchain_update}" -eq "${FLAGS_TRUE}" ]; then
|
|
UPDATE_ARGS+=( --skip_toolchain_update )
|
|
fi
|
|
if [ "${FLAGS_skip_chroot_upgrade}" -eq "${FLAGS_TRUE}" ]; then
|
|
UPDATE_ARGS+=( --skip_chroot_upgrade )
|
|
fi
|
|
|
|
"${SCRIPTS_DIR}"/setup_board --quiet --board=${FLAGS_board} "${UPDATE_ARGS[@]}"
|
|
|
|
# set BOARD and BOARD_ROOT
|
|
. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1
|
|
. "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
|
|
. "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1
|
|
. "${BUILD_LIBRARY_DIR}/extra_sysexts.sh" || exit 1
|
|
. "${BUILD_LIBRARY_DIR}/oem_sysexts.sh" || exit 1
|
|
|
|
# Setup all the emerge command/flags.
|
|
EMERGE_FLAGS=( --update --deep --newuse --verbose --backtrack=30 --select )
|
|
REBUILD_FLAGS=( --verbose )
|
|
EMERGE_CMD=( "emerge-${FLAGS_board}" )
|
|
if [[ "${FLAGS_fetchonly}" -eq "${FLAGS_TRUE}" ]]; then
|
|
EMERGE_CMD+=( --fetchonly )
|
|
fi
|
|
|
|
EMERGE_CMD+=( ${EXTRA_BOARD_FLAGS} )
|
|
|
|
if [[ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]]; then
|
|
# Use binary packages. Include all build-time dependencies,
|
|
# so as to avoid unnecessary differences between source
|
|
# and binary builds except for --usepkgonly for speed.
|
|
if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then
|
|
EMERGE_FLAGS+=( --usepkgonly --rebuilt-binaries n )
|
|
else
|
|
EMERGE_FLAGS+=( --usepkg --with-bdeps y )
|
|
|
|
# Only update toolchain when binpkgs are available.
|
|
EMERGE_FLAGS+=( $(get_binonly_args) )
|
|
REBUILD_FLAGS+=( $(get_binonly_args) )
|
|
fi
|
|
if [[ "${FLAGS_getbinpkg}" -eq "${FLAGS_TRUE}" ]]; then
|
|
EMERGE_FLAGS+=( --getbinpkg )
|
|
fi
|
|
if [[ -n "${FLAGS_usepkg_exclude}" ]]; then
|
|
EMERGE_FLAGS+=("--usepkg-exclude=${FLAGS_usepkg_exclude}")
|
|
fi
|
|
fi
|
|
|
|
EMERGE_FLAGS+=( "--jobs=${NUM_JOBS}" )
|
|
REBUILD_FLAGS+=( "--jobs=${NUM_JOBS}" )
|
|
|
|
if [[ "${FLAGS_rebuild}" -eq "${FLAGS_TRUE}" ]]; then
|
|
EMERGE_FLAGS+=( --rebuild-if-unbuilt )
|
|
fi
|
|
|
|
if [[ "${FLAGS_debug_emerge}" -eq "${FLAGS_TRUE}" ]]; then
|
|
EMERGE_FLAGS+=( --debug )
|
|
fi
|
|
|
|
# Build flatcar_workon packages when they are changed.
|
|
WORKON_PKGS=()
|
|
if [[ ${FLAGS_workon} -eq "${FLAGS_TRUE}" ]]; then
|
|
mapfile -t WORKON_PKGS < <("${SRC_ROOT}"/scripts/flatcar_workon list --board="${FLAGS_board}")
|
|
fi
|
|
|
|
if [[ ${#WORKON_PKGS[@]} -gt 0 ]]; then
|
|
EMERGE_FLAGS+=(
|
|
--reinstall-atoms="${WORKON_PKGS[*]}"
|
|
--usepkg-exclude="${WORKON_PKGS[*]}"
|
|
)
|
|
fi
|
|
|
|
# Goo to attempt to resolve dependency loops on individual packages.
|
|
# If this becomes insufficient we will need to move to a full multi-stage
|
|
# bootstrap process like we do with the SDK via catalyst.
|
|
#
|
|
# Called like:
|
|
#
|
|
# break_dep_loop [PKG_USE_PAIR]…
|
|
#
|
|
# PKG_USE_PAIR consists of two arguments: a package name (for example:
|
|
# sys-fs/lvm2), and a comma-separated list of USE flags to clear (for
|
|
# example: udev,systemd).
|
|
break_dep_loop() {
|
|
local -a pkgs
|
|
local -a all_flags
|
|
local -a args
|
|
local flag_file="${BOARD_ROOT}/etc/portage/package.use/break_dep_loop"
|
|
local -a flag_file_entries
|
|
local -a pkg_summaries
|
|
|
|
# Be sure to clean up use flag hackery from previous failed runs
|
|
sudo rm -f "${flag_file}"
|
|
|
|
if [[ $# -eq 0 ]]; then
|
|
return 0
|
|
fi
|
|
|
|
# Temporarily compile/install packages with flags disabled. If a binary
|
|
# package is available use it regardless of its version or use flags.
|
|
local pkg
|
|
local -a flags
|
|
local disabled_flags
|
|
while [[ $# -gt 0 ]]; do
|
|
pkg="${1}"
|
|
pkgs+=("${pkg}")
|
|
flags=( ${2//,/ } )
|
|
all_flags+=("${flags[@]}")
|
|
disabled_flags="${flags[@]/#/-}"
|
|
flag_file_entries+=("${pkg} ${disabled_flags}")
|
|
args+=("--buildpkg-exclude=${pkg}")
|
|
pkg_summaries+=("${pkg}[${disabled_flags}]")
|
|
shift 2
|
|
done
|
|
|
|
# If packages are already installed we have nothing to do
|
|
local any_package_uninstalled=0
|
|
for pkg in "${pkgs[@]}"; do
|
|
if ! portageq-"${BOARD}" has_version "${BOARD_ROOT}" "${pkgs[@]}"; then
|
|
any_package_uninstalled=1
|
|
break
|
|
fi
|
|
done
|
|
if [[ ${any_package_uninstalled} -eq 0 ]]; then
|
|
return 0
|
|
fi
|
|
|
|
# Likewise, nothing to do if the flags aren't actually enabled.
|
|
local any_flag_enabled=0
|
|
for pkg in "${pkgs[@]}"; do
|
|
if pkg_use_enabled "${pkg}" "${all_flags[@]}"; then
|
|
any_flag_enabled=1
|
|
break
|
|
fi
|
|
done
|
|
if [[ ${any_flag_enabled} -eq 0 ]]; then
|
|
return 0
|
|
fi
|
|
|
|
info "Merging ${pkg_summaries[@]}"
|
|
sudo mkdir -p "${flag_file%/*}"
|
|
sudo_clobber "${flag_file}" <<<"${flag_file_entries[0]}"
|
|
local entry
|
|
for entry in "${flag_file_entries[@]:1}"; do
|
|
sudo_append "${flag_file}" <<<"${entry}"
|
|
done
|
|
# rebuild-if-unbuilt is disabled to prevent portage from needlessly
|
|
# rebuilding zlib for some unknown reason, in turn triggering more rebuilds.
|
|
sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" \
|
|
--rebuild-if-unbuilt=n \
|
|
"${args[@]}" "${pkgs[@]}"
|
|
sudo rm -f "${flag_file}"
|
|
}
|
|
|
|
if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_FALSE}" ]]; then
|
|
# Breaking the following loops here:
|
|
#
|
|
# util-linux[udev] -> virtual/udev -> systemd -> util-linux
|
|
# util-linux[systemd] -> systemd -> util-linux
|
|
# util-linux[cryptsetup] -> cryptsetup -> util-linux
|
|
# cryptsetup[udev] -> virtual/udev -> systemd[cryptsetup] -> cryptsetup
|
|
# lvm2[udev] -> virtual/udev -> systemd[cryptsetup] -> cryptsetup -> lvm2
|
|
# lvm2[systemd] -> systemd[cryptsetup] -> cryptsetup -> lvm2
|
|
# systemd[cryptsetup] -> cryptsetup[udev] -> virtual/udev -> systemd
|
|
# systemd[tpm] -> tpm2-tss -> util-linux[udev] -> virtual/udev -> systemd
|
|
# curl[http2] -> nghttp2[systemd] -> systemd[curl] -> curl
|
|
# sys-libs/pam[systemd] -> sys-apps/system[pam] -> sys-libs/pam
|
|
# not dropping pam from sys-apps/systemd, otherwise we would need
|
|
# to drop pam from sys-auth/pambase
|
|
break_dep_loop sys-apps/util-linux udev,systemd,cryptsetup \
|
|
sys-fs/cryptsetup udev \
|
|
sys-fs/lvm2 udev,systemd \
|
|
sys-apps/systemd cryptsetup,tpm \
|
|
net-misc/curl http2 \
|
|
net-libs/nghttp2 systemd \
|
|
sys-libs/pam systemd
|
|
fi
|
|
|
|
if [[ "${FLAGS_only_resolve_circular_deps}" -eq "${FLAGS_TRUE}" ]]; then
|
|
info "Circular dependencies resolved. Stopping as requested."
|
|
exit
|
|
fi
|
|
|
|
export KBUILD_BUILD_USER="${BUILD_USER:-build}"
|
|
export KBUILD_BUILD_HOST="${BUILD_HOST:-pony-truck.infra.kinvolk.io}"
|
|
|
|
# Build sysext packages from an array of sysext definitions.
|
|
# Usage: build_sysext_packages "description" "${SYSEXT_ARRAY[@]}"
|
|
# Array format: "name|packages|useflags|arches"
|
|
build_sysext_packages() {
|
|
local description="$1"
|
|
shift
|
|
local sysexts=("$@")
|
|
|
|
info "Merging ${description} packages now"
|
|
for sysext in "${sysexts[@]}"; do
|
|
local sysext_name package_atoms useflags arches
|
|
IFS="|" read -r sysext_name package_atoms useflags arches <<< "$sysext"
|
|
[[ -z ${arches} || ,${arches}, == *,"${ARCH}",* ]] || continue
|
|
|
|
info "Building packages for $sysext_name sysext with USE=$useflags"
|
|
IFS=,
|
|
for package in $package_atoms; do
|
|
# --buildpkgonly does not install dependencies, so we install them
|
|
# separately before building the binary package
|
|
sudo --preserve-env=MODULES_SIGN_KEY,MODULES_SIGN_CERT \
|
|
env USE="$useflags" FEATURES="-ebuild-locks binpkg-multi-instance" "${EMERGE_CMD[@]}" \
|
|
"${EMERGE_FLAGS[@]}" \
|
|
--quiet \
|
|
--onlydeps \
|
|
--binpkg-respect-use=y \
|
|
"${package}"
|
|
|
|
sudo --preserve-env=MODULES_SIGN_KEY,MODULES_SIGN_CERT \
|
|
env USE="$useflags" FEATURES="-ebuild-locks binpkg-multi-instance" "${EMERGE_CMD[@]}" \
|
|
"${EMERGE_FLAGS[@]}" \
|
|
--quiet \
|
|
--buildpkgonly \
|
|
--binpkg-respect-use=y \
|
|
"${package}"
|
|
done
|
|
unset IFS
|
|
done
|
|
}
|
|
|
|
info "Merging board packages now"
|
|
sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "$@"
|
|
|
|
build_sysext_packages "extra sysexts" "${EXTRA_SYSEXTS[@]}"
|
|
|
|
declare -a oem_sysexts
|
|
get_oem_sysext_matrix "${ARCH}" oem_sysexts
|
|
if [[ ${#oem_sysexts[@]} -gt 0 ]]; then
|
|
build_sysext_packages "OEM sysexts" "${oem_sysexts[@]}"
|
|
fi
|
|
|
|
info "Removing obsolete packages"
|
|
# The return value of emerge is not clearly reliable. It may fail with
|
|
# an output like following:
|
|
#
|
|
# Calculating dependencies... done!
|
|
# dev-libs/gmp-6.3.0 pulled in by:
|
|
# cross-aarch64-cros-linux-gnu/gcc-12.3.1_p20230526 requires >=dev-libs/gmp-4.3.2:0/10.4=, >=dev-libs/gmp-4.3.2:0=
|
|
# cross-aarch64-cros-linux-gnu/gdb-13.2-r1 requires dev-libs/gmp:=, dev-libs/gmp:0/10.4=
|
|
# cross-x86_64-cros-linux-gnu/gcc-12.3.1_p20230526 requires >=dev-libs/gmp-4.3.2:0/10.4=, >=dev-libs/gmp-4.3.2:0=
|
|
# cross-x86_64-cros-linux-gnu/gdb-13.2-r1 requires dev-libs/gmp:0/10.4=, dev-libs/gmp:=
|
|
# dev-libs/mpc-1.2.1 requires >=dev-libs/gmp-5.0.0:0=[abi_x86_64(-)], >=dev-libs/gmp-5.0.0:0/10.4=[abi_x86_64(-)]
|
|
# dev-libs/mpfr-4.1.0-r1 requires >=dev-libs/gmp-5.0.0:=[abi_x86_64(-)], >=dev-libs/gmp-5.0.0:0/10.4=[abi_x86_64(-)]
|
|
# dev-libs/nettle-3.9.1 requires >=dev-libs/gmp-6.1:0/10.4=[abi_x86_64(-)], >=dev-libs/gmp-6.1:=[abi_x86_64(-)]
|
|
# net-libs/gnutls-3.8.0 requires >=dev-libs/gmp-5.1.3-r1:0/10.4=[abi_x86_64(-)], >=dev-libs/gmp-5.1.3-r1:=[abi_x86_64(-)]
|
|
# sys-devel/gcc-12.3.1_p20230526 requires >=dev-libs/gmp-4.3.2:0=, >=dev-libs/gmp-4.3.2:0/10.4=
|
|
#
|
|
# >>> No packages selected for removal by depclean
|
|
#
|
|
# Which gives you completely no reason for returning with non-zero
|
|
# status. Ignore emerge failures here.
|
|
#
|
|
# Well, actually, technically the reason for failure is that we asked
|
|
# for a removal of the unavailable packages and emerge found that
|
|
# dev-libs/gmp-6.3.0 is not available but didn't remove it, because
|
|
# it's pulled as a dependency by other packages. Question is why
|
|
# emerge thinks that dev-libs/gmp-6.3.0 is not available.
|
|
sudo -E "${EMERGE_CMD[@]}" --verbose --depclean @unavailable || :
|
|
|
|
if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_FALSE}" ]]; then
|
|
if "portageq-${BOARD}" list_preserved_libs "${BOARD_ROOT}" >/dev/null; then
|
|
sudo -E "${EMERGE_CMD[@]}" "${REBUILD_FLAGS[@]}" @preserved-rebuild
|
|
fi
|
|
fi
|
|
|
|
exclusions_file=$(mktemp)
|
|
if [ ! -f "$exclusions_file" ]; then
|
|
die_notrace "Couldn't create temporary exclusions file $exclusions_file for eclean"
|
|
fi
|
|
get_unversioned_sysext_packages > "$exclusions_file"
|
|
eclean-"$BOARD" -d --exclude-file="$exclusions_file" packages
|
|
rm -f "$exclusions_file"
|
|
# run eclean again, this time without the --deep option, to clean old versions
|
|
# of sysext packages (those, for which .ebuild file no longer exists)
|
|
eclean-"$BOARD" packages
|
|
|
|
info "Checking build root"
|
|
test_image_content "${BOARD_ROOT}"
|
|
|
|
info "Builds complete"
|
|
command_completed
|