flatcar-scripts/update_chroot

#!/bin/bash

# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.


. "$(dirname "$0")/common.sh" || exit 1
. "${BUILD_LIBRARY_DIR}/toolchain_util.sh"
. "${BUILD_LIBRARY_DIR}/update_chroot_util.sh"

# Script must run inside the chroot
assert_inside_chroot "$@"

# Do not run as root
assert_not_root_user

DEFINE_boolean usepkg "${FLAGS_TRUE}" \
  "Use binary packages when possible."
DEFINE_boolean getbinpkg "${FLAGS_TRUE}" \
  "Download binary packages from remote repository."
DEFINE_boolean usepkgonly "${FLAGS_FALSE}" \
  "Only use/download binary packages. Implies --noworkon"
DEFINE_boolean workon "${FLAGS_TRUE}" \
  "Automatically rebuild updated cros-workon packages."
DEFINE_boolean skip_toolchain_update "${FLAGS_FALSE}" \
  "Don't update the toolchains."
DEFINE_boolean setuponly "${FLAGS_FALSE}" \
  "Only configure portage, without updating packages. Useful when only boostrap_sdk/build_toolchains will be called"
DEFINE_string toolchain_boards "" \
  "Extra toolchains to setup for the specified boards."
DEFINE_string dev_builds_sdk "" \
  "Set FLATCAR_DEV_BUILDS_SDK which defaults to FLATCAR_DEV_BUILDS/sdk"
DEFINE_string binhost "" \
  "Use binary packages from a specific location (like $(get_sdk_binhost | tr '\n' ' '}))"

FLAGS_HELP="usage: $(basename $0) [flags]
Performs an update of the chroot. This script is called as part of
build_packages, so there is typically no need to call this script directly.
"

# Parse command line flags
FLAGS "$@" || exit 1
eval set -- "${FLAGS_ARGV}"

# Only now can we die on error.  shflags functions leak non-zero error codes,
# so will die prematurely if 'switch_to_strict_mode' is specified before now.
switch_to_strict_mode

if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then
  for flag in usepkg getbinpkg; do
    fvar="FLAGS_${flag}"
    if [[ "${!fvar}" -ne "${FLAGS_TRUE}" ]]; then
      die_notrace "--usepkgonly is incompatible with --no${flag}"
    fi
  done
  FLAGS_workon="${FLAGS_FALSE}"
fi

if [[ -n "${FLAGS_dev_builds_sdk}" ]]; then
  FLATCAR_DEV_BUILDS_SDK="${FLAGS_dev_builds_sdk}"
fi

PORTAGE_STABLE_OVERLAY="${REPO_ROOT}/src/third_party/portage-stable"
CROSSDEV_OVERLAY="/usr/local/portage/crossdev"
COREOS_OVERLAY="${REPO_ROOT}/src/third_party/coreos-overlay"
COREOS_CONFIG="${COREOS_OVERLAY}/coreos/config"
PORTAGE_BINHOST="$FLAGS_binhost $(get_sdk_binhost)"

# PORTAGE_USERNAME may already be in the env but set just to be safe.
: ${PORTAGE_USERNAME:=${USER}}

# Clean up old distfiles cache. It used to be split for 'host' and
# 'target' but that just duplicates files.
if [[ -d "${REPO_CACHE_DIR}/distfiles/host" ]]; then
  info "Cleaning up old distfiles cache..."
  sudo mv "${REPO_CACHE_DIR}"/{distfiles/host,distfiles.host}
  sudo rm -rf "${REPO_CACHE_DIR}/distfiles"
  sudo mv "${REPO_CACHE_DIR}"/{distfiles.host,distfiles}
fi

info "Setting up portage..."
sudo mkdir -p "${REPO_CACHE_DIR}/distfiles"
sudo chown "${PORTAGE_USERNAME}:portage" "${REPO_CACHE_DIR}/distfiles"
sudo mkdir -p /etc/portage/repos.conf /var/lib/portage/pkgs
sudo ln -sfT "${COREOS_OVERLAY}/coreos/user-patches" '/etc/portage/patches'
sudo touch /etc/portage/make.conf.user

sudo_clobber "/etc/portage/make.conf" <<EOF
# Created by update_chroot
DISTDIR="${REPO_CACHE_DIR}/distfiles"
PKGDIR="/var/lib/portage/pkgs"
PORT_LOGDIR="/var/log/portage"
PORTAGE_BINHOST="$PORTAGE_BINHOST"
PORTAGE_USERNAME="${PORTAGE_USERNAME}"
MAKEOPTS="--jobs=${NUM_JOBS} --load-average=$((NUM_JOBS * 2))"

# Generally there isn't any need to add packages to @world by default.
# You can use --select to override this.
EMERGE_DEFAULT_OPTS="--verbose --oneshot"

# Allow the user to override or define additional settings.
source "/etc/portage/make.conf.user"
EOF

sudo_clobber "/etc/portage/repos.conf/coreos.conf" <<EOF
[DEFAULT]
main-repo = portage-stable

[coreos]
location = ${COREOS_OVERLAY}

[portage-stable]
location = ${PORTAGE_STABLE_OVERLAY}
EOF

# Automatically symlink any provided portage repo configurations into
# the appropriate destination directory from "config/portage/repos".
# CAUTON: Any repo configuration files with the same name, will be silently
# relinked to the last file found.
# Automatic file renaming to avoid this is challenging as it has to deal with
# the case where we actually do want the files overwritten and so is outside
# the scope of this function for now.
# If your taking advantage of this functionality in the build toolchain
# you probably found out about it from these code comments so we can assume
# for now at least, that you have been somewhat warned.

# If any directories match config/portage/repos/*/repos.conf
if [[ -n $(sudo find /mnt/host/source/config/portage/repos \
  -maxdepth 2 -mindepth 2 -type d -name "repos.conf") ]]; then
  # If any files match config/portage/repos/*/repos.conf/*
  if [[ -n $(sudo find /mnt/host/source/config/portage/repos \
    -maxdepth 3 -mindepth 3 -path "*/repos.conf/*" -type f) ]]; then
    info "Symlinking portage repo configuration files into repos.conf dir..."
    # Do not name any of your repo configs "coreos.conf" bad things will happen.
    # We abort if any file matches config/portage/repos/*/repos.conf/coreos.conf
    if [[ -n $(sudo find /mnt/host/source/config/portage/repos \
      -maxdepth 3 -mindepth 3 -path "*/repos.conf/*" -name "coreos.conf") ]]; then
      die_notrace "One of the portage repo configuration files to be symlinked is " \
        "named \"coreos.conf\" which will conflict with the main configuation file " \
        "called \"coreos.conf\" which is created during the chroot build process."
    fi
    # We run through all the other files matching config/portage/repos/*/repos.conf/*
    # and symlink them to the right place for portage.
    sudo find /mnt/host/source/config/portage/repos \
      -maxdepth 3 -mindepth 3 -path "*/repos.conf/*" -type f \
      -exec bash -c 'ln -sf {} /etc/portage/repos.conf/$(basename {})' \;
  fi
fi

sudo eselect profile set --force "$(get_sdk_profile)"

# Create crossdev repo_name and metadata
info "Setting up crossdev..."
configure_crossdev_overlay "${FLAGS_chroot}" "${CROSSDEV_OVERLAY}"

# Run version hooks as pre-update
if [[ -f /etc/os-release ]]; then
  OLDVER=$(grep "^VERSION=" /etc/os-release | cut -d = -f 2-)
else
  OLDVER="0.0.0"
fi

info "Updating chroot:"
info " chroot version: $OLDVER"
info " Flatcar version: $FLATCAR_VERSION"

# Updates should be of the form 1.2.3_desc.sh
for update_script in ${SCRIPTS_DIR}/sdk_lib/updates/*.sh; do
  update_name="${update_script##*/}"
  update_ver="${update_name%%_*}"
  # Run newer updates but don't pass our current version
  if ! cmp_ver le "$update_ver" "$FLATCAR_VERSION"; then
      warn "Skipping update from the future: $update_name"
      warn "Perhaps it is time to run a repo sync?"
  elif ! cmp_ver ge "$OLDVER" "$update_ver"; then
      info "Running chroot update $update_name"
      bash -e "$update_script" || die "Update failed: $update_name"
  fi
done

"${BUILD_LIBRARY_DIR}/set_lsb_release" --root /

EMERGE_FLAGS=( --update --newuse --verbose --with-bdeps=y --select )
REBUILD_FLAGS=( --verbose )
if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then
  EMERGE_FLAGS+=( --usepkg )
  if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then
    EMERGE_FLAGS+=( --usepkgonly --rebuilt-binaries n )
  fi
  if [ "${FLAGS_getbinpkg}" -eq "${FLAGS_TRUE}" ] && [ -n "${PORTAGE_BINHOST// }" ]; then
    EMERGE_FLAGS+=( --getbinpkg )
  fi

  # Only update toolchain when binpkgs are available.
  EMERGE_FLAGS+=( $(get_binonly_args $(get_chost_list)) )
  REBUILD_FLAGS+=( $(get_binonly_args $(get_chost_list)) )
fi

EMERGE_FLAGS+=( "--jobs=${NUM_JOBS}" )
REBUILD_FLAGS+=( "--jobs=${NUM_JOBS}" )

# Perform an update of coreos-devel/sdk-depends and world in the chroot.
EMERGE_CMD="emerge"

# In first pass, update portage and toolchains. Lagged updates of both
# can cause serious issues later.
info "Updating basic system packages"
sudo -E ${EMERGE_CMD} "${EMERGE_FLAGS[@]}" \
    sys-apps/portage \
    sys-devel/crossdev \
    sys-devel/sysroot-wrappers \
    sys-libs/nss-usrfiles \
    "${TOOLCHAIN_PKGS[@]}"

gcc_set_latest_profile "$(portageq envvar CHOST)"

if [[ "${FLAGS_setuponly}" -eq "${FLAGS_TRUE}" ]]; then
  command_completed
  exit 0
fi

if [[ "${FLAGS_skip_toolchain_update}" -eq "${FLAGS_FALSE}" && \
      -n "${FLAGS_toolchain_boards}" ]]; then
  CROSS_CHOSTS=( $(get_board_chost ${FLAGS_toolchain_boards} | sort -u) )
  # double check get_board_chost actually returned results, pipefail isn't set.
  [[ -n "${CROSS_CHOSTS[*]}" ]] || exit 1

  for cross_chost in "${CROSS_CHOSTS[@]}"; do
    info "Updating cross ${cross_chost} toolchain"
    install_cross_toolchain "${cross_chost}" "${EMERGE_FLAGS[@]}"
  done
fi

# Build cros_workon packages when they are changed.
if [ "${FLAGS_workon}" -eq "${FLAGS_TRUE}" ]; then
  for pkg in $("${SRC_ROOT}/scripts/cros_workon" list --host); do
    EMERGE_FLAGS+=( "--reinstall-atoms=${pkg}" "--usepkg-exclude=${pkg}" )
  done
fi

sudo_e_emerge() {
    sudo -E "${EMERGE_CMD}" "${@}"
}

info "Maybe removing some hard blocks"
# dev-python/setuptools_scm: blocks the update of the package (newer
# versions have !!<dev-python/setuptools_scm-2 in BDEPEND).
remove_hard_blocks \
    sudo_e_emerge equery \
    dev-python/setuptools_scm:2 \
    dev-python/setuptools-scm:2

# Second pass, update everything else.
EMERGE_FLAGS+=( --deep )
info "Updating all SDK packages"
sudo -E ${EMERGE_CMD} "${EMERGE_FLAGS[@]}" \
    coreos-devel/sdk-depends world

info "Removing obsolete packages"
# The return value of emerge is not clearly reliable. It may fail with
# an output like following:
#
# Calculating dependencies... done!
#   dev-libs/gmp-6.3.0 pulled in by:
#     cross-aarch64-cros-linux-gnu/gcc-12.3.1_p20230526 requires >=dev-libs/gmp-4.3.2:0/10.4=, >=dev-libs/gmp-4.3.2:0=
#     cross-aarch64-cros-linux-gnu/gdb-13.2-r1 requires dev-libs/gmp:=, dev-libs/gmp:0/10.4=
#     cross-x86_64-cros-linux-gnu/gcc-12.3.1_p20230526 requires >=dev-libs/gmp-4.3.2:0/10.4=, >=dev-libs/gmp-4.3.2:0=
#     cross-x86_64-cros-linux-gnu/gdb-13.2-r1 requires dev-libs/gmp:0/10.4=, dev-libs/gmp:=
#     dev-libs/mpc-1.2.1 requires >=dev-libs/gmp-5.0.0:0=[abi_x86_64(-)], >=dev-libs/gmp-5.0.0:0/10.4=[abi_x86_64(-)]
#     dev-libs/mpfr-4.1.0-r1 requires >=dev-libs/gmp-5.0.0:=[abi_x86_64(-)], >=dev-libs/gmp-5.0.0:0/10.4=[abi_x86_64(-)]
#     dev-libs/nettle-3.9.1 requires >=dev-libs/gmp-6.1:0/10.4=[abi_x86_64(-)], >=dev-libs/gmp-6.1:=[abi_x86_64(-)]
#     net-libs/gnutls-3.8.0 requires >=dev-libs/gmp-5.1.3-r1:0/10.4=[abi_x86_64(-)], >=dev-libs/gmp-5.1.3-r1:=[abi_x86_64(-)]
#     sys-devel/gcc-12.3.1_p20230526 requires >=dev-libs/gmp-4.3.2:0=, >=dev-libs/gmp-4.3.2:0/10.4=
#
# >>> No packages selected for removal by depclean
#
# Which gives you completely no reason for returning with non-zero
# status. Ignore emerge failures here.
#
# Well, actually, technically the reason for failure is that we asked
# for a removal of the unavailable packages and emerge found that
# dev-libs/gmp-6.3.0 is not available but didn't remove it, because
# it's pulled as a dependency by other packages. Question is why
# emerge thinks that dev-libs/gmp-6.3.0 is not available.
sudo -E ${EMERGE_CMD} --verbose --depclean @unavailable || :

if portageq list_preserved_libs / >/dev/null; then
  info "Rebuilding packages linked against old libraries"
  sudo -E ${EMERGE_CMD} "${REBUILD_FLAGS[@]}" @preserved-rebuild
fi

# Automatically discard all CONFIG_PROTECT'ed files. Those that are
# protected should not be overwritten until the variable is changed.
# Autodiscard is option "-9" followed by the "YES" confirmation.
printf '%s\nYES\n' -9 | sudo etc-update

# If the user still has old perl modules installed, update them.
PERL_VERSIONS=$(find /usr/lib*/perl5/vendor_perl/ -maxdepth 1 -mindepth 1 \
  -type d -printf '%P\n' | sort -u | wc -w)
if [ "$PERL_VERSIONS" -gt 1 ] ; then
  sudo perl-cleaner --all -- --quiet
fi

# Old $PS1 customization that doesn't work any more
if [[ -e /etc/profile.d/coreos-niceties.sh ]]; then
  sudo rm -f /etc/profile.d/coreos-niceties.sh
fi

command_completed