From 4e03c44cde80c0dbbdec94272cf4f4d60280c541 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 13 May 2025 13:46:08 +0200 Subject: [PATCH] build_toolchains: Break dep loop and handle more dependencies Switching to a selinux profile caused more USE flags to be enabled (selinux, audit, caps), thus more dependencies to be pulled. More dependencies caused two things: - cyclic dependencies appeared - sys-apps/baselayout is being pulled in Cyclic dependencies need to be handled in a similar way it was done in build_packages, thus factor out the code doing it into a separate and reusable part. The dependency on baselayout needs to be handled by installing the package as a first thing in $ROOT, followed by a more careful way of copying things from $SYSROOT to $ROOT (due to split-usr differences), followed by installing the rest of the packages. Signed-off-by: Krzesimir Nowak --- build_library/break_dep_loop.sh | 137 +++++++++++++++++++++++++++ build_library/catalyst_toolchains.sh | 50 +++++++++- build_packages | 112 +++++----------------- build_toolchains | 5 +- 4 files changed, 210 insertions(+), 94 deletions(-) create mode 100644 build_library/break_dep_loop.sh diff --git a/build_library/break_dep_loop.sh b/build_library/break_dep_loop.sh new file mode 100644 index 0000000000..599aa4cc9e --- /dev/null +++ b/build_library/break_dep_loop.sh @@ -0,0 +1,137 @@ +# Goo to attempt to resolve dependency loops on individual packages. +# If this becomes insufficient we will need to move to a full multi-stage +# bootstrap process like we do with the SDK via catalyst. +# +# Called like: +# +# break_dep_loop [-v] [PKG_USE_PAIR]… +# +# Pass -v for verbose output. +# +# PKG_USE_PAIR consists of two arguments: a package name (for example: +# sys-fs/lvm2), and a comma-separated list of USE flags to clear (for +# example: udev,systemd). +# +# Env vars: +# +# BDL_ROOT, BDL_PORTAGEQ, BDL_EQUERY, BDL_EMERGE, BDL_INFO +break_dep_loop() { + local bdl_root=${BDL_ROOT:-/} + local bdl_portageq=${BDL_PORTAGEQ:-portageq} + local bdl_equery=${BDL_EQUERY:-equery} + local bdl_emerge=${BDL_EMERGE:-emerge} + local bdl_info=${BDL_INFO:-echo} + local conf_dir="${bdl_root%/}/etc/portage" + local flag_file="${conf_dir}/package.use/break_dep_loop" + local force_flag_file="${conf_dir}/profile/package.use.force/break_dep_loop" + + local verbose= + if [[ ${1:-} = '-v' ]]; then + verbose=x + shift + fi + + # Be sure to clean up use flag hackery from previous failed runs + sudo rm -f "${flag_file}" "${force_flag_file}" + + if [[ ${#} -eq 0 ]]; then + return 0 + fi + + function bdl_call() { + local output_var_name=${1}; shift + if [[ ${output_var_name} = '-' ]]; then + local throw_away + output_var_name=throw_away + fi + local -n output_ref=${output_var_name} + if [[ -n ${verbose} ]]; then + "${bdl_info}" "${*@Q}" + fi + local -i rv=0 + output_ref=$("${@}") || rv=${?} + if [[ -n ${verbose} ]]; then + "${bdl_info}" "output: ${output_ref}" + "${bdl_info}" "exit status: ${rv}" + fi + return ${rv} + } + + # Temporarily compile/install packages with flags disabled. If a binary + # package is available use it regardless of its version or use flags. + local pkg use_flags disabled_flags + local -a flags + local -a pkgs args flag_file_entries pkg_summaries + local -A per_pkg_flags=() + while [[ $# -gt 1 ]]; do + pkg=${1} + use_flags=${2} + shift 2 + + mapfile -t flags <<<"${use_flags//,/$'\n'}" + disabled_flags="${flags[*]/#/-}" + + pkgs+=( "${pkg}" ) + per_pkg_flags["${pkg}"]=${use_flags} + flag_file_entries+=( "${pkg} ${disabled_flags}" ) + args+=( "--buildpkg-exclude=${pkg}" ) + pkg_summaries+=( "${pkg}[${disabled_flags}]" ) + done + unset pkg use_flags disabled_flags flags + + # If packages are already installed we have nothing to do + local pkg any_package_uninstalled= + for pkg in "${pkgs[@]}"; do + if ! bdl_call - "${bdl_portageq}" has_version "${bdl_root}" "${pkg}"; then + any_package_uninstalled=x + break + fi + done + if [[ -z ${any_package_uninstalled} ]]; then + if [[ -n ${verbose} ]]; then + "${bdl_info}" "all packages (${pkgs[*]}) are installed already, skipping" + fi + return 0 + fi + unset pkg any_package_uninstalled + + # Likewise, nothing to do if the flags aren't actually enabled. + local pkg any_flag_enabled= equery_output flag flags_str + local -a flags grep_args + for pkg in "${pkgs[@]}"; do + bdl_call equery_output "${bdl_equery}" -q uses "${pkg}" + flags_str=${per_pkg_flags["${pkg}"]} + mapfile -t flags <<<"${flags_str//,/$'\n'}" + for flag in "${flags[@]}"; do + grep_args+=( -e "${flag/#/+}" ) + done + if bdl_call - grep --quiet --line-regexp --fixed-strings "${grep_args[@]}" <<<"${equery_output}"; then + any_flag_enabled=x + break + fi + done + if [[ -z ${any_flag_enabled} ]]; then + if [[ -n ${verbose} ]]; then + "${bdl_info}" "all packages (${pkgs[*]}) has all the desired USE flags already disabled, skipping" + fi + return 0 + fi + unset pkg any_flag_enabled equery_output flag flags_str flags grep_args + + "${bdl_info}" "Merging ${pkg_summaries[*]}" + sudo mkdir -p "${flag_file%/*}" "${force_flag_file%/*}" + printf '%s\n' "${flag_file_entries[@]}" | sudo tee "${flag_file}" >/dev/null + cp -a "${flag_file}" "${force_flag_file}" + if [[ -n ${verbose} ]]; then + "${bdl_info}" "contents of ${flag_file@Q}:" + "${bdl_info}" "$(<"${flag_file}")" + "${bdl_info}" "${bdl_emerge}" --rebuild-if-unbuilt=n "${args[@]}" "${pkgs[@]}" + fi + # rebuild-if-unbuilt is disabled to prevent portage from needlessly + # rebuilding zlib for some unknown reason, in turn triggering more rebuilds. + "${bdl_emerge}" \ + --rebuild-if-unbuilt=n \ + "${args[@]}" "${pkgs[@]}" + sudo rm -f "${flag_file}" "${force_flag_file}" + unset bdl_call +} diff --git a/build_library/catalyst_toolchains.sh b/build_library/catalyst_toolchains.sh index d1b032f79b..7257f17cc2 100644 --- a/build_library/catalyst_toolchains.sh +++ b/build_library/catalyst_toolchains.sh @@ -3,6 +3,7 @@ set -e source /tmp/chroot-functions.sh source /tmp/toolchain_util.sh +source /tmp/break_dep_loop.sh # A note on packages: # The default PKGDIR is /usr/portage/packages @@ -57,8 +58,53 @@ build_target_toolchain() { cp -at "${ROOT}"/usr "${SYSROOT}"/usr/include ) - btt_emerge --update "${TOOLCHAIN_PKGS[@]}" - unset -f btt_emerge + local -a args_for_bdl=() + if [[ -n ${clst_VERBOSE} ]]; then + args_for_bdl+=(-v) + fi + function btt_bdl_portageq() { + ROOT=${ROOT} SYSROOT=${ROOT} PORTAGE_CONFIGROOT=${ROOT} portageq "${@}" + } + function btt_bdl_equery() { + ROOT=${ROOT} SYSROOT=${ROOT} PORTAGE_CONFIGROOT=${ROOT} equery "${@}" + } + # Breaking the following loops here: + # + # glibc[nscd] -> libcap[pam] -> sys-libs/pam -> libcrypt -> libxcrypt[system] -> glibc + # glibc[nscd] -> audit[python] -> python -> libcrypt -> libxcrypt[system] -> glibc + # glibc[selinux] -> libselinux[python] -> python -> libcrypt -> libxcrypt[system] -> glibc + # systemd[cryptsetup] -> cryptsetup[udev] -> libudev[systemd] -> systemd + # systemd[cryptsetup] -> cryptsetup -> lvm2[udev] -> libudev[systemd] -> systemd + # systemd[cryptsetup] -> cryptsetup -> lvm2[lvm,systemd] -> systemd + # systemd[cryptsetup] -> cryptsetup -> tmpfiles[systemd] -> systemd + # systemd[curl] -> curl -> nghttp2[systemd] -> systemd + # importd requires curl, so needs to be disabled too + # systemd[pam] -> pam[systemd] -> systemd + # not dropping pam from sys-apps/systemd, otherwise we would + # need to drop pam from sys-auth/pambase + # systemd[tpm] -> tpm2-tss -> tmpfiles[systemd] -> systemd + # util-linux[audit] -> audit[python] -> python -> util-linux + # util-linux[cryptsetup] -> cryptsetup -> util-linux + # util-linux[pam] -> sys-libs/pam[audit] -> sys-process/audit[python] -> python -> util-linux + # su requires pam, so needs to be disabled too + # util-linux[selinux] -> libselinux[python] -> python -> util-linux + # util-linux[systemd] -> systemd -> util-linux + # util-linux[udev] -> libudev[systemd] -> systemd -> util-linux + args_for_bdl+=( + sys-apps/systemd cryptsetup,curl,importd,tpm + sys-apps/util-linux audit,cryptsetup,pam,selinux,su,systemd,udev + sys-libs/glibc nscd,selinux + sys-libs/pam systemd + ) + BDL_ROOT=${ROOT} \ + BDL_PORTAGEQ=btt_bdl_portageq \ + BDL_EQUERY=btt_bdl_equery \ + BDL_EMERGE=btt_emerge \ + break_dep_loop "${args_for_bdl[@]}" + unset btt_bdl_portageq btt_bdl_equery + + btt_emerge --changed-use --update --deep "${TOOLCHAIN_PKGS[@]}" + unset btt_emerge } configure_crossdev_overlay / /usr/local/portage/crossdev diff --git a/build_packages b/build_packages index 6f69b75d30..5978370006 100755 --- a/build_packages +++ b/build_packages @@ -175,89 +175,12 @@ if [[ ${#WORKON_PKGS[@]} -gt 0 ]]; then ) fi -# Goo to attempt to resolve dependency loops on individual packages. -# If this becomes insufficient we will need to move to a full multi-stage -# bootstrap process like we do with the SDK via catalyst. -# -# Called like: -# -# break_dep_loop [PKG_USE_PAIR]… -# -# PKG_USE_PAIR consists of two arguments: a package name (for example: -# sys-fs/lvm2), and a comma-separated list of USE flags to clear (for -# example: udev,systemd). -break_dep_loop() { - local -a pkgs - local -a all_flags - local -a args - local flag_file="${BOARD_ROOT}/etc/portage/package.use/break_dep_loop" - local -a flag_file_entries - local -a pkg_summaries - - # Be sure to clean up use flag hackery from previous failed runs - sudo rm -f "${flag_file}" - - if [[ $# -eq 0 ]]; then - return 0 - fi - - # Temporarily compile/install packages with flags disabled. If a binary - # package is available use it regardless of its version or use flags. - local pkg - local -a flags - local disabled_flags - while [[ $# -gt 0 ]]; do - pkg="${1}" - pkgs+=("${pkg}") - flags=( ${2//,/ } ) - all_flags+=("${flags[@]}") - disabled_flags="${flags[@]/#/-}" - flag_file_entries+=("${pkg} ${disabled_flags}") - args+=("--buildpkg-exclude=${pkg}") - pkg_summaries+=("${pkg}[${disabled_flags}]") - shift 2 - done - - # If packages are already installed we have nothing to do - local any_package_uninstalled=0 - for pkg in "${pkgs[@]}"; do - if ! portageq-"${BOARD}" has_version "${BOARD_ROOT}" "${pkgs[@]}"; then - any_package_uninstalled=1 - break - fi - done - if [[ ${any_package_uninstalled} -eq 0 ]]; then - return 0 - fi - - # Likewise, nothing to do if the flags aren't actually enabled. - local any_flag_enabled=0 - for pkg in "${pkgs[@]}"; do - if pkg_use_enabled "${pkg}" "${all_flags[@]}"; then - any_flag_enabled=1 - break - fi - done - if [[ ${any_flag_enabled} -eq 0 ]]; then - return 0 - fi - - info "Merging ${pkg_summaries[@]}" - sudo mkdir -p "${flag_file%/*}" - sudo_clobber "${flag_file}" <<<"${flag_file_entries[0]}" - local entry - for entry in "${flag_file_entries[@]:1}"; do - sudo_append "${flag_file}" <<<"${entry}" - done - # rebuild-if-unbuilt is disabled to prevent portage from needlessly - # rebuilding zlib for some unknown reason, in turn triggering more rebuilds. - sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" \ - --rebuild-if-unbuilt=n \ - "${args[@]}" "${pkgs[@]}" - sudo rm -f "${flag_file}" -} - if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_FALSE}" ]]; then + . "${BUILD_LIBRARY_DIR}/break_dep_loop.sh" || exit 1 + + function bp_bdl_emerge() { + sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "${@}" + } # Breaking the following loops here: # # util-linux[udev] -> virtual/udev -> systemd -> util-linux @@ -273,14 +196,23 @@ if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_FALSE}" ]]; then # dropping USE=pam from sys-apps/systemd requires dropping # USE=systemd from sys-auth/pambase # sys-auth/pambase[sssd] -> sys-auth/sssd -> sys-apps/shadow[pam] -> sys-auth/pambase - break_dep_loop sys-apps/util-linux cryptsetup,systemd,udev \ - sys-fs/cryptsetup udev \ - sys-fs/lvm2 systemd,udev \ - sys-apps/systemd cryptsetup,pam,tpm \ - net-misc/curl http2 \ - net-libs/nghttp2 systemd \ - sys-libs/pam systemd \ - sys-auth/pambase sssd,systemd + args_for_bdl=( + sys-apps/util-linux cryptsetup,systemd,udev \ + sys-fs/cryptsetup udev \ + sys-fs/lvm2 systemd,udev \ + sys-apps/systemd cryptsetup,pam,tpm \ + net-misc/curl http2 \ + net-libs/nghttp2 systemd \ + sys-libs/pam systemd \ + sys-auth/pambase sssd,systemd + ) + BDL_ROOT=${BOARD_ROOT} \ + BDL_PORTAGEQ=portageq-"${BOARD}" \ + BDL_EQUERY=equery-"${BOARD}" \ + BDL_EMERGE=bp_bdl_emerge \ + BDL_INFO=info \ + break_dep_loop "${args_for_bdl[@]}" + unset bp_bdl_emerge args_for_bdl fi if [[ "${FLAGS_only_resolve_circular_deps}" -eq "${FLAGS_TRUE}" ]]; then diff --git a/build_toolchains b/build_toolchains index 9fe99c1947..b5076d3f7c 100755 --- a/build_toolchains +++ b/build_toolchains @@ -49,9 +49,10 @@ catalyst_init "$@" ROOT_OVERLAY="${TEMPDIR}/stage4-${ARCH}-$FLAGS_version-overlay" -# toolchain_util.sh is required by catalyst_toolchains.sh +# toolchain_util.sh and break_dep_loop.sh are required by +# catalyst_toolchains.sh mkdir -p "${ROOT_OVERLAY}/tmp" -cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp" +cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${BUILD_LIBRARY_DIR}/break_dep_loop.sh" "${ROOT_OVERLAY}/tmp" create_provenance_overlay "${ROOT_OVERLAY}" catalyst_build