mirror of
https://github.com/flatcar/scripts.git
synced 2026-03-01 19:41:39 +01:00
build_toolchains: Break dep loop and handle more dependencies
Switching to a selinux profile caused more USE flags to be enabled (selinux, audit, caps), thus more dependencies to be pulled. More dependencies caused two things: - cyclic dependencies appeared - sys-apps/baselayout is being pulled in Cyclic dependencies need to be handled in a similar way it was done in build_packages, thus factor out the code doing it into a separate and reusable part. The dependency on baselayout needs to be handled by installing the package as a first thing in $ROOT, followed by a more careful way of copying things from $SYSROOT to $ROOT (due to split-usr differences), followed by installing the rest of the packages. Signed-off-by: Krzesimir Nowak <knowak@microsoft.com>
This commit is contained in:
Krzesimir Nowak
parent
2a225b47d9
commit
4e03c44cde
137
build_library/break_dep_loop.sh
Normal file
137
build_library/break_dep_loop.sh
Normal file
@ -0,0 +1,137 @@
|
||||
# Goo to attempt to resolve dependency loops on individual packages.
|
||||
# If this becomes insufficient we will need to move to a full multi-stage
|
||||
# bootstrap process like we do with the SDK via catalyst.
|
||||
#
|
||||
# Called like:
|
||||
#
|
||||
# break_dep_loop [-v] [PKG_USE_PAIR]…
|
||||
#
|
||||
# Pass -v for verbose output.
|
||||
#
|
||||
# PKG_USE_PAIR consists of two arguments: a package name (for example:
|
||||
# sys-fs/lvm2), and a comma-separated list of USE flags to clear (for
|
||||
# example: udev,systemd).
|
||||
#
|
||||
# Env vars:
|
||||
#
|
||||
# BDL_ROOT, BDL_PORTAGEQ, BDL_EQUERY, BDL_EMERGE, BDL_INFO
|
||||
break_dep_loop() {
|
||||
local bdl_root=${BDL_ROOT:-/}
|
||||
local bdl_portageq=${BDL_PORTAGEQ:-portageq}
|
||||
local bdl_equery=${BDL_EQUERY:-equery}
|
||||
local bdl_emerge=${BDL_EMERGE:-emerge}
|
||||
local bdl_info=${BDL_INFO:-echo}
|
||||
local conf_dir="${bdl_root%/}/etc/portage"
|
||||
local flag_file="${conf_dir}/package.use/break_dep_loop"
|
||||
local force_flag_file="${conf_dir}/profile/package.use.force/break_dep_loop"
|
||||
|
||||
local verbose=
|
||||
if [[ ${1:-} = '-v' ]]; then
|
||||
verbose=x
|
||||
shift
|
||||
fi
|
||||
|
||||
# Be sure to clean up use flag hackery from previous failed runs
|
||||
sudo rm -f "${flag_file}" "${force_flag_file}"
|
||||
|
||||
if [[ ${#} -eq 0 ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
function bdl_call() {
|
||||
local output_var_name=${1}; shift
|
||||
if [[ ${output_var_name} = '-' ]]; then
|
||||
local throw_away
|
||||
output_var_name=throw_away
|
||||
fi
|
||||
local -n output_ref=${output_var_name}
|
||||
if [[ -n ${verbose} ]]; then
|
||||
"${bdl_info}" "${*@Q}"
|
||||
fi
|
||||
local -i rv=0
|
||||
output_ref=$("${@}") || rv=${?}
|
||||
if [[ -n ${verbose} ]]; then
|
||||
"${bdl_info}" "output: ${output_ref}"
|
||||
"${bdl_info}" "exit status: ${rv}"
|
||||
fi
|
||||
return ${rv}
|
||||
}
|
||||
|
||||
# Temporarily compile/install packages with flags disabled. If a binary
|
||||
# package is available use it regardless of its version or use flags.
|
||||
local pkg use_flags disabled_flags
|
||||
local -a flags
|
||||
local -a pkgs args flag_file_entries pkg_summaries
|
||||
local -A per_pkg_flags=()
|
||||
while [[ $# -gt 1 ]]; do
|
||||
pkg=${1}
|
||||
use_flags=${2}
|
||||
shift 2
|
||||
|
||||
mapfile -t flags <<<"${use_flags//,/$'\n'}"
|
||||
disabled_flags="${flags[*]/#/-}"
|
||||
|
||||
pkgs+=( "${pkg}" )
|
||||
per_pkg_flags["${pkg}"]=${use_flags}
|
||||
flag_file_entries+=( "${pkg} ${disabled_flags}" )
|
||||
args+=( "--buildpkg-exclude=${pkg}" )
|
||||
pkg_summaries+=( "${pkg}[${disabled_flags}]" )
|
||||
done
|
||||
unset pkg use_flags disabled_flags flags
|
||||
|
||||
# If packages are already installed we have nothing to do
|
||||
local pkg any_package_uninstalled=
|
||||
for pkg in "${pkgs[@]}"; do
|
||||
if ! bdl_call - "${bdl_portageq}" has_version "${bdl_root}" "${pkg}"; then
|
||||
any_package_uninstalled=x
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ -z ${any_package_uninstalled} ]]; then
|
||||
if [[ -n ${verbose} ]]; then
|
||||
"${bdl_info}" "all packages (${pkgs[*]}) are installed already, skipping"
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
unset pkg any_package_uninstalled
|
||||
|
||||
# Likewise, nothing to do if the flags aren't actually enabled.
|
||||
local pkg any_flag_enabled= equery_output flag flags_str
|
||||
local -a flags grep_args
|
||||
for pkg in "${pkgs[@]}"; do
|
||||
bdl_call equery_output "${bdl_equery}" -q uses "${pkg}"
|
||||
flags_str=${per_pkg_flags["${pkg}"]}
|
||||
mapfile -t flags <<<"${flags_str//,/$'\n'}"
|
||||
for flag in "${flags[@]}"; do
|
||||
grep_args+=( -e "${flag/#/+}" )
|
||||
done
|
||||
if bdl_call - grep --quiet --line-regexp --fixed-strings "${grep_args[@]}" <<<"${equery_output}"; then
|
||||
any_flag_enabled=x
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ -z ${any_flag_enabled} ]]; then
|
||||
if [[ -n ${verbose} ]]; then
|
||||
"${bdl_info}" "all packages (${pkgs[*]}) has all the desired USE flags already disabled, skipping"
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
unset pkg any_flag_enabled equery_output flag flags_str flags grep_args
|
||||
|
||||
"${bdl_info}" "Merging ${pkg_summaries[*]}"
|
||||
sudo mkdir -p "${flag_file%/*}" "${force_flag_file%/*}"
|
||||
printf '%s\n' "${flag_file_entries[@]}" | sudo tee "${flag_file}" >/dev/null
|
||||
cp -a "${flag_file}" "${force_flag_file}"
|
||||
if [[ -n ${verbose} ]]; then
|
||||
"${bdl_info}" "contents of ${flag_file@Q}:"
|
||||
"${bdl_info}" "$(<"${flag_file}")"
|
||||
"${bdl_info}" "${bdl_emerge}" --rebuild-if-unbuilt=n "${args[@]}" "${pkgs[@]}"
|
||||
fi
|
||||
# rebuild-if-unbuilt is disabled to prevent portage from needlessly
|
||||
# rebuilding zlib for some unknown reason, in turn triggering more rebuilds.
|
||||
"${bdl_emerge}" \
|
||||
--rebuild-if-unbuilt=n \
|
||||
"${args[@]}" "${pkgs[@]}"
|
||||
sudo rm -f "${flag_file}" "${force_flag_file}"
|
||||
unset bdl_call
|
||||
}
|
||||
@ -3,6 +3,7 @@
|
||||
set -e
|
||||
source /tmp/chroot-functions.sh
|
||||
source /tmp/toolchain_util.sh
|
||||
source /tmp/break_dep_loop.sh
|
||||
|
||||
# A note on packages:
|
||||
# The default PKGDIR is /usr/portage/packages
|
||||
@ -57,8 +58,53 @@ build_target_toolchain() {
|
||||
cp -at "${ROOT}"/usr "${SYSROOT}"/usr/include
|
||||
)
|
||||
|
||||
btt_emerge --update "${TOOLCHAIN_PKGS[@]}"
|
||||
unset -f btt_emerge
|
||||
local -a args_for_bdl=()
|
||||
if [[ -n ${clst_VERBOSE} ]]; then
|
||||
args_for_bdl+=(-v)
|
||||
fi
|
||||
function btt_bdl_portageq() {
|
||||
ROOT=${ROOT} SYSROOT=${ROOT} PORTAGE_CONFIGROOT=${ROOT} portageq "${@}"
|
||||
}
|
||||
function btt_bdl_equery() {
|
||||
ROOT=${ROOT} SYSROOT=${ROOT} PORTAGE_CONFIGROOT=${ROOT} equery "${@}"
|
||||
}
|
||||
# Breaking the following loops here:
|
||||
#
|
||||
# glibc[nscd] -> libcap[pam] -> sys-libs/pam -> libcrypt -> libxcrypt[system] -> glibc
|
||||
# glibc[nscd] -> audit[python] -> python -> libcrypt -> libxcrypt[system] -> glibc
|
||||
# glibc[selinux] -> libselinux[python] -> python -> libcrypt -> libxcrypt[system] -> glibc
|
||||
# systemd[cryptsetup] -> cryptsetup[udev] -> libudev[systemd] -> systemd
|
||||
# systemd[cryptsetup] -> cryptsetup -> lvm2[udev] -> libudev[systemd] -> systemd
|
||||
# systemd[cryptsetup] -> cryptsetup -> lvm2[lvm,systemd] -> systemd
|
||||
# systemd[cryptsetup] -> cryptsetup -> tmpfiles[systemd] -> systemd
|
||||
# systemd[curl] -> curl -> nghttp2[systemd] -> systemd
|
||||
# importd requires curl, so needs to be disabled too
|
||||
# systemd[pam] -> pam[systemd] -> systemd
|
||||
# not dropping pam from sys-apps/systemd, otherwise we would
|
||||
# need to drop pam from sys-auth/pambase
|
||||
# systemd[tpm] -> tpm2-tss -> tmpfiles[systemd] -> systemd
|
||||
# util-linux[audit] -> audit[python] -> python -> util-linux
|
||||
# util-linux[cryptsetup] -> cryptsetup -> util-linux
|
||||
# util-linux[pam] -> sys-libs/pam[audit] -> sys-process/audit[python] -> python -> util-linux
|
||||
# su requires pam, so needs to be disabled too
|
||||
# util-linux[selinux] -> libselinux[python] -> python -> util-linux
|
||||
# util-linux[systemd] -> systemd -> util-linux
|
||||
# util-linux[udev] -> libudev[systemd] -> systemd -> util-linux
|
||||
args_for_bdl+=(
|
||||
sys-apps/systemd cryptsetup,curl,importd,tpm
|
||||
sys-apps/util-linux audit,cryptsetup,pam,selinux,su,systemd,udev
|
||||
sys-libs/glibc nscd,selinux
|
||||
sys-libs/pam systemd
|
||||
)
|
||||
BDL_ROOT=${ROOT} \
|
||||
BDL_PORTAGEQ=btt_bdl_portageq \
|
||||
BDL_EQUERY=btt_bdl_equery \
|
||||
BDL_EMERGE=btt_emerge \
|
||||
break_dep_loop "${args_for_bdl[@]}"
|
||||
unset btt_bdl_portageq btt_bdl_equery
|
||||
|
||||
btt_emerge --changed-use --update --deep "${TOOLCHAIN_PKGS[@]}"
|
||||
unset btt_emerge
|
||||
}
|
||||
|
||||
configure_crossdev_overlay / /usr/local/portage/crossdev
|
||||
|
||||
112
build_packages
112
build_packages
@ -175,89 +175,12 @@ if [[ ${#WORKON_PKGS[@]} -gt 0 ]]; then
|
||||
)
|
||||
fi
|
||||
|
||||
# Goo to attempt to resolve dependency loops on individual packages.
|
||||
# If this becomes insufficient we will need to move to a full multi-stage
|
||||
# bootstrap process like we do with the SDK via catalyst.
|
||||
#
|
||||
# Called like:
|
||||
#
|
||||
# break_dep_loop [PKG_USE_PAIR]…
|
||||
#
|
||||
# PKG_USE_PAIR consists of two arguments: a package name (for example:
|
||||
# sys-fs/lvm2), and a comma-separated list of USE flags to clear (for
|
||||
# example: udev,systemd).
|
||||
break_dep_loop() {
|
||||
local -a pkgs
|
||||
local -a all_flags
|
||||
local -a args
|
||||
local flag_file="${BOARD_ROOT}/etc/portage/package.use/break_dep_loop"
|
||||
local -a flag_file_entries
|
||||
local -a pkg_summaries
|
||||
|
||||
# Be sure to clean up use flag hackery from previous failed runs
|
||||
sudo rm -f "${flag_file}"
|
||||
|
||||
if [[ $# -eq 0 ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Temporarily compile/install packages with flags disabled. If a binary
|
||||
# package is available use it regardless of its version or use flags.
|
||||
local pkg
|
||||
local -a flags
|
||||
local disabled_flags
|
||||
while [[ $# -gt 0 ]]; do
|
||||
pkg="${1}"
|
||||
pkgs+=("${pkg}")
|
||||
flags=( ${2//,/ } )
|
||||
all_flags+=("${flags[@]}")
|
||||
disabled_flags="${flags[@]/#/-}"
|
||||
flag_file_entries+=("${pkg} ${disabled_flags}")
|
||||
args+=("--buildpkg-exclude=${pkg}")
|
||||
pkg_summaries+=("${pkg}[${disabled_flags}]")
|
||||
shift 2
|
||||
done
|
||||
|
||||
# If packages are already installed we have nothing to do
|
||||
local any_package_uninstalled=0
|
||||
for pkg in "${pkgs[@]}"; do
|
||||
if ! portageq-"${BOARD}" has_version "${BOARD_ROOT}" "${pkgs[@]}"; then
|
||||
any_package_uninstalled=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ ${any_package_uninstalled} -eq 0 ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Likewise, nothing to do if the flags aren't actually enabled.
|
||||
local any_flag_enabled=0
|
||||
for pkg in "${pkgs[@]}"; do
|
||||
if pkg_use_enabled "${pkg}" "${all_flags[@]}"; then
|
||||
any_flag_enabled=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ ${any_flag_enabled} -eq 0 ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
info "Merging ${pkg_summaries[@]}"
|
||||
sudo mkdir -p "${flag_file%/*}"
|
||||
sudo_clobber "${flag_file}" <<<"${flag_file_entries[0]}"
|
||||
local entry
|
||||
for entry in "${flag_file_entries[@]:1}"; do
|
||||
sudo_append "${flag_file}" <<<"${entry}"
|
||||
done
|
||||
# rebuild-if-unbuilt is disabled to prevent portage from needlessly
|
||||
# rebuilding zlib for some unknown reason, in turn triggering more rebuilds.
|
||||
sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" \
|
||||
--rebuild-if-unbuilt=n \
|
||||
"${args[@]}" "${pkgs[@]}"
|
||||
sudo rm -f "${flag_file}"
|
||||
}
|
||||
|
||||
if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_FALSE}" ]]; then
|
||||
. "${BUILD_LIBRARY_DIR}/break_dep_loop.sh" || exit 1
|
||||
|
||||
function bp_bdl_emerge() {
|
||||
sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "${@}"
|
||||
}
|
||||
# Breaking the following loops here:
|
||||
#
|
||||
# util-linux[udev] -> virtual/udev -> systemd -> util-linux
|
||||
@ -273,14 +196,23 @@ if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_FALSE}" ]]; then
|
||||
# dropping USE=pam from sys-apps/systemd requires dropping
|
||||
# USE=systemd from sys-auth/pambase
|
||||
# sys-auth/pambase[sssd] -> sys-auth/sssd -> sys-apps/shadow[pam] -> sys-auth/pambase
|
||||
break_dep_loop sys-apps/util-linux cryptsetup,systemd,udev \
|
||||
sys-fs/cryptsetup udev \
|
||||
sys-fs/lvm2 systemd,udev \
|
||||
sys-apps/systemd cryptsetup,pam,tpm \
|
||||
net-misc/curl http2 \
|
||||
net-libs/nghttp2 systemd \
|
||||
sys-libs/pam systemd \
|
||||
sys-auth/pambase sssd,systemd
|
||||
args_for_bdl=(
|
||||
sys-apps/util-linux cryptsetup,systemd,udev \
|
||||
sys-fs/cryptsetup udev \
|
||||
sys-fs/lvm2 systemd,udev \
|
||||
sys-apps/systemd cryptsetup,pam,tpm \
|
||||
net-misc/curl http2 \
|
||||
net-libs/nghttp2 systemd \
|
||||
sys-libs/pam systemd \
|
||||
sys-auth/pambase sssd,systemd
|
||||
)
|
||||
BDL_ROOT=${BOARD_ROOT} \
|
||||
BDL_PORTAGEQ=portageq-"${BOARD}" \
|
||||
BDL_EQUERY=equery-"${BOARD}" \
|
||||
BDL_EMERGE=bp_bdl_emerge \
|
||||
BDL_INFO=info \
|
||||
break_dep_loop "${args_for_bdl[@]}"
|
||||
unset bp_bdl_emerge args_for_bdl
|
||||
fi
|
||||
|
||||
if [[ "${FLAGS_only_resolve_circular_deps}" -eq "${FLAGS_TRUE}" ]]; then
|
||||
|
||||
@ -49,9 +49,10 @@ catalyst_init "$@"
|
||||
|
||||
ROOT_OVERLAY="${TEMPDIR}/stage4-${ARCH}-$FLAGS_version-overlay"
|
||||
|
||||
# toolchain_util.sh is required by catalyst_toolchains.sh
|
||||
# toolchain_util.sh and break_dep_loop.sh are required by
|
||||
# catalyst_toolchains.sh
|
||||
mkdir -p "${ROOT_OVERLAY}/tmp"
|
||||
cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"
|
||||
cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${BUILD_LIBRARY_DIR}/break_dep_loop.sh" "${ROOT_OVERLAY}/tmp"
|
||||
create_provenance_overlay "${ROOT_OVERLAY}"
|
||||
|
||||
catalyst_build
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user