From d575ef441af5b6682b4ae566796f654f3122d805 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Wed, 3 Feb 2021 08:45:09 +0100 Subject: [PATCH 1/2] build_packages: Build multiple packages in one go when breaking loops Instead of rebuilding just one package and maybe rebuilding others as a fallout, force rebuilding all the mentioned packages. This makes the build process a bit more robust against package build ordering changes. May be useful when breaking multiple dep loops that have some common packages, so we build them all once. --- build_packages | 86 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 62 insertions(+), 24 deletions(-) diff --git a/build_packages b/build_packages index df822efbd3..4729f44ea8 100755 --- a/build_packages +++ b/build_packages @@ -186,44 +186,82 @@ fi # Goo to attempt to resolve dependency loops on individual packages. # If this becomes insufficient we will need to move to a full multi-stage # bootstrap process like we do with the SDK via catalyst. +# +# Called like: +# +# break_dep_loop [PKG_USE_PAIR]… +# +# PKG_USE_PAIR consists of two arguments: a package name (for example: +# sys-fs/lvm2), and a comma-separated list of USE flags to clear (for +# example: udev,systemd). break_dep_loop() { - local pkg="$1" - local flags=( ${2//,/ } ) - shift 2 + local -a pkgs + local -a all_flags + local -a args local flag_file="${BOARD_ROOT}/etc/portage/package.use/break_dep_loop" + local -a flag_file_entries + local -a pkg_summaries # Be sure to clean up use flag hackery from previous failed runs sudo rm -f "${flag_file}" - # If the package is already installed we have nothing to do - if portageq-"${BOARD}" has_version "${BOARD_ROOT}" "${pkg}"; then - return 0 + if [[ $# -eq 0 ]]; then + return 0 fi - # Likewise, nothing to do if the flag isn't actually enabled. - if ! pkg_use_enabled "${pkg}" "${flags[@]}"; then - return 0 - fi - - # Temporarily compile/install package with flag disabled. If a binary + # Temporarily compile/install packages with flags disabled. If a binary # package is available use it regardless of its version or use flags. - local disabled_flags="${flags[@]/#/-}" - info "Merging ${pkg} with USE=${disabled_flags}" - sudo mkdir -p "${flag_file%/*}" - sudo_clobber "${flag_file}" <<<"${pkg} ${disabled_flags}" - # Disable any other problematic flags - extra_args="" + local pkg + local -a flags + local disabled_flags while [[ $# -gt 0 ]]; do - sudo_append "${flag_file}" <<<"$1 -$2" - extra_args+=" --buildpkg-exclude=$1 --useoldpkg-atoms=$1" - shift 2 + pkg="${1}" + pkgs+=("${pkg}") + flags=( ${2//,/ } ) + all_flags+=("${flags[@]}") + disabled_flags="${flags[@]/#/-}" + flag_file_entries+=("${pkg} ${disabled_flags}") + args+=("--buildpkg-exclude=${pkg}") + pkg_summaries+=("${pkg}[${disabled_flags}]") + shift 2 + done + + # If packages are already installed we have nothing to do + local any_package_uninstalled=0 + for pkg in "${pkgs[@]}"; do + if ! portageq-"${BOARD}" has_version "${BOARD_ROOT}" "${pkgs[@]}"; then + any_package_uninstalled=1 + break + fi + done + if [[ ${any_package_uninstalled} -eq 0 ]]; then + return 0 + fi + + # Likewise, nothing to do if the flags aren't actually enabled. + local any_flag_enabled=0 + for pkg in "${pkgs[@]}"; do + if pkg_use_enabled "${pkg}" "${all_flags[@]}"; then + any_flag_enabled=1 + break + fi + done + if [[ ${any_flag_enabled} -eq 0 ]]; then + return 0 + fi + + info "Merging ${pkg_summaries[@]}" + sudo mkdir -p "${flag_file%/*}" + sudo_clobber "${flag_file}" <<<"${flag_file_entries[0]}" + local entry + for entry in "${flag_file_entries[@]:1}"; do + sudo_append "${flag_file}" <<<"${entry}" done # rebuild-if-unbuilt is disabled to prevent portage from needlessly # rebuilding zlib for some unknown reason, in turn triggering more rebuilds. sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" \ - --rebuild-if-unbuilt=n \ - --buildpkg-exclude="${pkg}" \ - ${extra_args} "${pkg}" + --rebuild-if-unbuilt=n \ + "${args[@]}" "${pkgs[@]}" sudo rm -f "${flag_file}" } From 7f2a437f0a4d5b79a644710e47425c2afe5c0040 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Wed, 3 Feb 2021 08:55:30 +0100 Subject: [PATCH 2/2] build_packages: Build all packages in the broken cycle Previously we broke the cycle caused by sys-apps/util-linux only, while disabling cryptsetup USE flag in systemd to avoid another cycle. That worked before, because the follow-up merge of the rest of packages built sys-fs/cryptsetup before sys-apps/systemd. After an update, the new portage is ordering the builds in different way and sys-apps/systemd ended up being built before sys-fs/cryptsetup and that failed during the configure phase because of unmet dependencies. Better build all the packages taking part in the loop (not counting the virtual packages), so we become less reliant on the package build ordering. It is going to take slightly more time as we build a couple of packages more. --- build_packages | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/build_packages b/build_packages index 4729f44ea8..c81eaccb8c 100755 --- a/build_packages +++ b/build_packages @@ -266,11 +266,18 @@ break_dep_loop() { } if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_FALSE}" ]]; then + # Breaking the following loops here: + # # util-linux[udev] -> virtual->udev -> systemd -> util-linux - break_dep_loop sys-apps/util-linux udev,systemd sys-apps/systemd cryptsetup - - # systemd[cryptsetup] -> cryptsetup -> lvm2 -> virtual/udev -> systemd - break_dep_loop sys-apps/systemd cryptsetup + # util-linux[systemd] -> systemd -> util-linux + # cryptsetup[udev] -> virtual/udev -> systemd[cryptsetup] -> cryptsetup + # lvm2[udev] -> virtual/udev -> systemd[cryptsetup] -> cryptsetup -> lvm2 + # lvm2[systemd] -> systemd[cryptsetup] -> cryptsetup -> lvm2 + # systemd[cryptsetup] -> cryptsetup[udev] -> virtual/udev -> systemd + break_dep_loop sys-apps/util-linux udev,systemd \ + sys-fs/cryptsetup udev \ + sys-fs/lvm2 udev,systemd \ + sys-apps/systemd cryptsetup fi export KBUILD_BUILD_USER="${BUILD_USER:-build}"