flatcar-scripts/build_packages

#!/bin/bash

# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

. "$(dirname "$0")/common.sh" || exit 1

# Script must run inside the chroot
assert_inside_chroot

assert_not_root_user

# Developer-visible flags.
DEFINE_string board "${DEFAULT_BOARD}" \
  "The board to build packages for."
DEFINE_boolean usepkg "${FLAGS_TRUE}" \
  "Use binary packages when possible."
DEFINE_boolean usepkgonly "${FLAGS_FALSE}" \
  "Only use/download binary packages. Implies --noworkon"
DEFINE_string usepkg_exclude "" \
  "Exclude these binary packages."
DEFINE_boolean getbinpkg "${FLAGS_TRUE}" \
  "Download binary packages from remote repository."
DEFINE_string getbinpkgver "" \
  "Use binary packages from a specific version."
DEFINE_boolean workon "${FLAGS_TRUE}" \
  "Automatically rebuild updated flatcar-workon packages."
DEFINE_boolean fetchonly "${FLAGS_FALSE}" \
  "Don't build anything, instead only fetch what is needed."
DEFINE_boolean rebuild "${FLAGS_FALSE}" \
  "Automatically rebuild dependencies."
DEFINE_boolean skip_toolchain_update "${FLAGS_FALSE}" \
  "Don't update toolchain automatically."
DEFINE_boolean skip_chroot_upgrade "${FLAGS_FALSE}" \
  "Don't run the chroot upgrade automatically; use with care."
DEFINE_boolean only_resolve_circular_deps "${FLAGS_FALSE}" \
  "Don't build all packages; only resolve circular dependencies, then stop."
DEFINE_boolean debug_emerge "${FLAGS_FALSE}" \
  "Enable debug output for emerge."

# include upload options
. "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1

FLAGS_HELP="usage: $(basename $0) [flags] [packages]

build_packages updates the set of binary packages needed by CoreOS. It will
cross compile all packages that have been updated into the given target's root
and build binary packages as a side-effect. The output packages will be picked
up by the build_image script to put together a bootable CoreOS image.

If [packages] are specified, only build those specific packages (and any
dependencies they might need).
"

# Parse command line
FLAGS "$@" || exit 1
eval set -- "${FLAGS_ARGV}"

# Die on any errors.
switch_to_strict_mode

# TODO(marineam): specify the default top-level ebuild in the portage profile.
# I could have sworn we did that or similar but maybe got lost at some point.
if [[ $# -eq 0 ]]; then
  set -- @system coreos-devel/board-packages
fi

if [[ -z "${FLAGS_board}" ]]; then
  echo "Error: --board is required."
  exit 1
fi

if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then
  for flag in usepkg getbinpkg; do
    fvar="FLAGS_${flag}"
    if [[ "${!fvar}" -ne "${FLAGS_TRUE}" ]]; then
      die_notrace "--usepkgonly is incompatible with --no${flag}"
    fi
  done
  if [[ "${FLAGS_rebuild}" -eq "${FLAGS_TRUE}" ]]; then
    die_notrace "--usepkgonly is incompatible with --rebuild"
  fi
  FLAGS_workon="${FLAGS_FALSE}"
fi

# Before we can run any tools, we need to update chroot or setup_board.
UPDATE_ARGS=( --regen_configs )
if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then
  UPDATE_ARGS+=( --usepkg )
  if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then
    UPDATE_ARGS+=( --usepkgonly )
  else
    UPDATE_ARGS+=( --nousepkgonly )
  fi
  if [[ "${FLAGS_getbinpkg}" -eq "${FLAGS_TRUE}" ]]; then
    UPDATE_ARGS+=( --getbinpkg )
  else
    UPDATE_ARGS+=( --nogetbinpkg )
  fi
  if [[ -n "${FLAGS_getbinpkgver}" ]]; then
    UPDATE_ARGS+=( --getbinpkgver="${FLAGS_getbinpkgver}" )
  fi
else
  UPDATE_ARGS+=( --nousepkg )
fi
if [ "${FLAGS_skip_toolchain_update}" -eq "${FLAGS_TRUE}" ]; then
  UPDATE_ARGS+=( --skip_toolchain_update )
fi
if [ "${FLAGS_skip_chroot_upgrade}" -eq "${FLAGS_TRUE}" ]; then
  UPDATE_ARGS+=( --skip_chroot_upgrade )
fi

"${SCRIPTS_DIR}"/setup_board --quiet --board=${FLAGS_board} "${UPDATE_ARGS[@]}"

# set BOARD and BOARD_ROOT
. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1
. "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
. "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1
. "${BUILD_LIBRARY_DIR}/extra_sysexts.sh" || exit 1

# Setup all the emerge command/flags.
EMERGE_FLAGS=( --update --deep --newuse --verbose --backtrack=30 --select )
REBUILD_FLAGS=( --verbose )
EMERGE_CMD=( "emerge-${FLAGS_board}" )
if [[ "${FLAGS_fetchonly}" -eq "${FLAGS_TRUE}" ]]; then
  EMERGE_CMD+=( --fetchonly )
fi

EMERGE_CMD+=( ${EXTRA_BOARD_FLAGS} )

if [[ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]]; then
  # Use binary packages. Include all build-time dependencies,
  # so as to avoid unnecessary differences between source
  # and binary builds except for --usepkgonly for speed.
  if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then
    EMERGE_FLAGS+=( --usepkgonly --rebuilt-binaries n )
  else
    EMERGE_FLAGS+=( --usepkg --with-bdeps y )

    # Only update toolchain when binpkgs are available.
    EMERGE_FLAGS+=( $(get_binonly_args) )
    REBUILD_FLAGS+=( $(get_binonly_args) )
  fi
  if [[ "${FLAGS_getbinpkg}" -eq "${FLAGS_TRUE}" ]]; then
      EMERGE_FLAGS+=( --getbinpkg )
  fi
  if [[ -n "${FLAGS_usepkg_exclude}"  ]]; then
    EMERGE_FLAGS+=("--usepkg-exclude=${FLAGS_usepkg_exclude}")
  fi
fi

EMERGE_FLAGS+=( "--jobs=${NUM_JOBS}" )
REBUILD_FLAGS+=( "--jobs=${NUM_JOBS}" )

if [[ "${FLAGS_rebuild}" -eq "${FLAGS_TRUE}" ]]; then
  EMERGE_FLAGS+=( --rebuild-if-unbuilt )
fi

if [[ "${FLAGS_debug_emerge}" -eq "${FLAGS_TRUE}" ]]; then
  EMERGE_FLAGS+=( --debug )
fi

# Build flatcar_workon packages when they are changed.
WORKON_PKGS=()
if [[ ${FLAGS_workon} -eq "${FLAGS_TRUE}" ]]; then
  mapfile -t WORKON_PKGS < <("${SRC_ROOT}"/scripts/flatcar_workon list --board="${FLAGS_board}")
fi

if [[ ${#WORKON_PKGS[@]} -gt 0 ]]; then
  EMERGE_FLAGS+=(
    --reinstall-atoms="${WORKON_PKGS[*]}"
    --usepkg-exclude="${WORKON_PKGS[*]}"
  )
fi

# Goo to attempt to resolve dependency loops on individual packages.
# If this becomes insufficient we will need to move to a full multi-stage
# bootstrap process like we do with the SDK via catalyst.
#
# Called like:
#
#     break_dep_loop [PKG_USE_PAIR]…
#
# PKG_USE_PAIR consists of two arguments: a package name (for example:
# sys-fs/lvm2), and a comma-separated list of USE flags to clear (for
# example: udev,systemd).
break_dep_loop() {
  local -a pkgs
  local -a all_flags
  local -a args
  local flag_file="${BOARD_ROOT}/etc/portage/package.use/break_dep_loop"
  local -a flag_file_entries
  local -a pkg_summaries

  # Be sure to clean up use flag hackery from previous failed runs
  sudo rm -f "${flag_file}"

  if [[ $# -eq 0 ]]; then
      return 0
  fi

  # Temporarily compile/install packages with flags disabled. If a binary
  # package is available use it regardless of its version or use flags.
  local pkg
  local -a flags
  local disabled_flags
  while [[ $# -gt 0 ]]; do
    pkg="${1}"
    pkgs+=("${pkg}")
    flags=( ${2//,/ } )
    all_flags+=("${flags[@]}")
    disabled_flags="${flags[@]/#/-}"
    flag_file_entries+=("${pkg} ${disabled_flags}")
    args+=("--buildpkg-exclude=${pkg}")
    pkg_summaries+=("${pkg}[${disabled_flags}]")
    shift 2
  done

  # If packages are already installed we have nothing to do
  local any_package_uninstalled=0
  for pkg in "${pkgs[@]}"; do
    if ! portageq-"${BOARD}" has_version "${BOARD_ROOT}" "${pkgs[@]}"; then
      any_package_uninstalled=1
      break
    fi
  done
  if [[ ${any_package_uninstalled} -eq 0 ]]; then
    return 0
  fi

  # Likewise, nothing to do if the flags aren't actually enabled.
  local any_flag_enabled=0
  for pkg in "${pkgs[@]}"; do
    if pkg_use_enabled "${pkg}" "${all_flags[@]}"; then
      any_flag_enabled=1
      break
    fi
  done
  if [[ ${any_flag_enabled} -eq 0 ]]; then
    return 0
  fi

  info "Merging ${pkg_summaries[@]}"
  sudo mkdir -p "${flag_file%/*}"
  sudo_clobber "${flag_file}" <<<"${flag_file_entries[0]}"
  local entry
  for entry in "${flag_file_entries[@]:1}"; do
    sudo_append "${flag_file}" <<<"${entry}"
  done
  # rebuild-if-unbuilt is disabled to prevent portage from needlessly
  # rebuilding zlib for some unknown reason, in turn triggering more rebuilds.
  sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" \
    --rebuild-if-unbuilt=n \
    "${args[@]}" "${pkgs[@]}"
  sudo rm -f "${flag_file}"
}

if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_FALSE}" ]]; then
  # Breaking the following loops here:
  #
  # util-linux[udev] -> virtual/udev -> systemd -> util-linux
  # util-linux[systemd] -> systemd -> util-linux
  # util-linux[cryptsetup] -> cryptsetup -> util-linux
  # cryptsetup[udev] -> virtual/udev -> systemd[cryptsetup] -> cryptsetup
  # lvm2[udev] -> virtual/udev -> systemd[cryptsetup] -> cryptsetup -> lvm2
  # lvm2[systemd] -> systemd[cryptsetup] -> cryptsetup -> lvm2
  # systemd[cryptsetup] -> cryptsetup[udev] -> virtual/udev -> systemd
  # curl[http2] -> nghttp2[systemd] -> systemd[curl] -> curl
  break_dep_loop sys-apps/util-linux udev,systemd,cryptsetup \
                 sys-fs/cryptsetup udev \
                 sys-fs/lvm2 udev,systemd \
                 sys-apps/systemd cryptsetup,tpm \
                 net-misc/curl http2 \
                 net-libs/nghttp2 systemd
fi

if [[ "${FLAGS_only_resolve_circular_deps}" -eq "${FLAGS_TRUE}" ]]; then
  info "Circular dependencies resolved. Stopping as requested."
  exit
fi

export KBUILD_BUILD_USER="${BUILD_USER:-build}"
export KBUILD_BUILD_HOST="${BUILD_HOST:-pony-truck.infra.kinvolk.io}"

info "Merging board packages now"
sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "$@"

info "Merging sysext packages now"
for sysext in "${EXTRA_SYSEXTS[@]}"; do
  IFS="|" read -r SYSEXT_NAME PACKAGE_ATOMS USEFLAGS ARCHES <<< "$sysext"

  arch_array=("${ARCHES//,/ }")
  if [[ -n $ARCHES ]]; then
    should_skip=1
    for arch in "${arch_array[@]}"; do
      if [[ $arch == "$ARCH" ]]; then
        should_skip=0
      fi
    done
    if [[ $should_skip -eq 1 ]]; then
      continue
    fi
  fi


  info "Building packages for $SYSEXT_NAME sysext with USE=$USEFLAGS"
  IFS=,
  for package in $PACKAGE_ATOMS; do
     # --buildpkgonly does not install dependencies, so we install them
     # separately before building the binary package
     sudo --preserve-env=MODULES_SIGN_KEY,MODULES_SIGN_CERT \
       env USE="$USEFLAGS" FEATURES="-ebuild-locks binpkg-multi-instance" "${EMERGE_CMD[@]}" \
       "${EMERGE_FLAGS[@]}" \
       --quiet \
       --onlydeps \
       --binpkg-respect-use=y \
       "${package}"

     sudo --preserve-env=MODULES_SIGN_KEY,MODULES_SIGN_CERT \
       env USE="$USEFLAGS" FEATURES="-ebuild-locks binpkg-multi-instance" "${EMERGE_CMD[@]}" \
       "${EMERGE_FLAGS[@]}" \
       --quiet \
       --buildpkgonly \
       --binpkg-respect-use=y \
       "${package}"
  done
  unset IFS
done

info "Removing obsolete packages"
# The return value of emerge is not clearly reliable. It may fail with
# an output like following:
#
# Calculating dependencies... done!
#   dev-libs/gmp-6.3.0 pulled in by:
#     cross-aarch64-cros-linux-gnu/gcc-12.3.1_p20230526 requires >=dev-libs/gmp-4.3.2:0/10.4=, >=dev-libs/gmp-4.3.2:0=
#     cross-aarch64-cros-linux-gnu/gdb-13.2-r1 requires dev-libs/gmp:=, dev-libs/gmp:0/10.4=
#     cross-x86_64-cros-linux-gnu/gcc-12.3.1_p20230526 requires >=dev-libs/gmp-4.3.2:0/10.4=, >=dev-libs/gmp-4.3.2:0=
#     cross-x86_64-cros-linux-gnu/gdb-13.2-r1 requires dev-libs/gmp:0/10.4=, dev-libs/gmp:=
#     dev-libs/mpc-1.2.1 requires >=dev-libs/gmp-5.0.0:0=[abi_x86_64(-)], >=dev-libs/gmp-5.0.0:0/10.4=[abi_x86_64(-)]
#     dev-libs/mpfr-4.1.0-r1 requires >=dev-libs/gmp-5.0.0:=[abi_x86_64(-)], >=dev-libs/gmp-5.0.0:0/10.4=[abi_x86_64(-)]
#     dev-libs/nettle-3.9.1 requires >=dev-libs/gmp-6.1:0/10.4=[abi_x86_64(-)], >=dev-libs/gmp-6.1:=[abi_x86_64(-)]
#     net-libs/gnutls-3.8.0 requires >=dev-libs/gmp-5.1.3-r1:0/10.4=[abi_x86_64(-)], >=dev-libs/gmp-5.1.3-r1:=[abi_x86_64(-)]
#     sys-devel/gcc-12.3.1_p20230526 requires >=dev-libs/gmp-4.3.2:0=, >=dev-libs/gmp-4.3.2:0/10.4=
#
# >>> No packages selected for removal by depclean
#
# Which gives you completely no reason for returning with non-zero
# status. Ignore emerge failures here.
#
# Well, actually, technically the reason for failure is that we asked
# for a removal of the unavailable packages and emerge found that
# dev-libs/gmp-6.3.0 is not available but didn't remove it, because
# it's pulled as a dependency by other packages. Question is why
# emerge thinks that dev-libs/gmp-6.3.0 is not available.
sudo -E "${EMERGE_CMD[@]}" --verbose --depclean @unavailable || :

if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_FALSE}" ]]; then
  if "portageq-${BOARD}" list_preserved_libs "${BOARD_ROOT}" >/dev/null; then
    sudo -E "${EMERGE_CMD[@]}" "${REBUILD_FLAGS[@]}" @preserved-rebuild
  fi
fi

exclusions_file=$(mktemp)
if [ ! -f "$exclusions_file" ]; then
    die_notrace "Couldn't create temporary exclusions file $exclusions_file for eclean"
fi
get_unversioned_sysext_packages > "$exclusions_file"
eclean-"$BOARD" -d --exclude-file="$exclusions_file" packages
rm -f "$exclusions_file"
# run eclean again, this time without the --deep option, to clean old versions
# of sysext packages (those, for which .ebuild file no  longer exists)
eclean-"$BOARD" packages

info "Checking build root"
test_image_content "${BOARD_ROOT}"

info "Builds complete"
command_completed