From 2fbd5af6a808a8ab4f2ece5371c5255dc7cd7c2d Mon Sep 17 00:00:00 2001
From: Zdenek Behan <zbehan@chromium.org>
Date: Mon, 12 Mar 2012 19:38:50 +0100
Subject: [PATCH] Make use of cros_setup_toolchains, take 2

Resubmit of If62b4f3973f02fd8e1deed35864c824a02ab0c22
This will be safe to land after
I2c4e21ec7e8c0c0cf58947e2b0a3a9edf7617a09
The breakage was a timing issue paired with people not always syncing
the complete tree. No changes to the CL are needed.

It is now used for:
- make_chroot (cros_sdk --bootstrap)
- update_chroot

setup_board is stripped of redundant code which was deprecated by this.

Also stripped is some usepkg logic in make_chroot, as that is now
exclusively source-only.

BUG=chromium-os:23032
TEST=trybot chromiumos-sdk

Change-Id: Ib888cf2886218622d9cfeebb17b9cd4462d06c89
Reviewed-on: https://gerrit.chromium.org/gerrit/22578
Tested-by: Zdenek Behan <zbehan@chromium.org>
Reviewed-by: Mike Frysinger <vapier@chromium.org>
Reviewed-by: asharif <asharif@chromium.org>
Commit-Ready: Zdenek Behan <zbehan@chromium.org>
---
 sdk_lib/make_chroot.sh |  33 ++++----
 setup_board            | 188 +++--------------------------------------
 update_chroot          |  22 +++--
 3 files changed, 41 insertions(+), 202 deletions(-)

diff --git a/sdk_lib/make_chroot.sh b/sdk_lib/make_chroot.sh
index d99eca9d26..c0512bcfe6 100755
--- a/sdk_lib/make_chroot.sh
+++ b/sdk_lib/make_chroot.sh
@@ -396,19 +396,15 @@ sudo sh -c "echo STAGE3=$STAGE3 > $CHROOT_STATE"
 info "Updating portage"
 early_enter_chroot emerge -uNv --quiet portage
 
-info "Updating toolchain"
-early_enter_chroot emerge -uNv --quiet $USEPKG '>=sys-devel/gcc-4.4' \
-  sys-libs/glibc sys-devel/binutils sys-kernel/linux-headers
-
-# HACK: Select the latest toolchain. We're assuming that when this is
-# ran, the chroot has no experimental versions of new toolchains, just
-# one that is very old, and one that was just emerged.
-GCC_ATOM="$(early_enter_chroot portageq best_version / sys-devel/gcc)"
-early_enter_chroot emerge --unmerge "<${GCC_ATOM}"
-CHOST="$(early_enter_chroot portageq envvar CHOST)"
-LATEST="$(early_enter_chroot gcc-config -l | grep "${CHOST}" | tail -n1 | \
-          cut -f3 -d' ')"
-early_enter_chroot gcc-config "${LATEST}"
+info "Updating host toolchain"
+early_enter_chroot emerge -uNv --quiet crossdev
+TOOLCHAIN_ARGS=( --deleteold )
+if [[ ${FLAGS_usepkg} -eq ${FLAGS_FALSE} ]]; then
+  TOOLCHAIN_ARGS+=( --nousepkg )
+fi
+# Note: early_enter_chroot executes as root.
+early_enter_chroot "${CHROOT_TRUNK}/chromite/bin/cros_setup_toolchains" \
+    --hostonly "${TOOLCHAIN_ARGS[@]}"
 
 # dhcpcd is included in 'world' by the stage3 that we pull in for some reason.
 # We have no need to install it in our host environment, so pull it out here.
@@ -427,8 +423,10 @@ if [ -n "${INITIALIZE_CHROOT}" ]; then
 fi
 
 # Update chroot.
-UPDATE_ARGS=()
-if [[ $FLAGS_usepkg -eq $FLAGS_TRUE ]]; then
+# Skip toolchain update because it already happened above, and the chroot is
+# not ready to emerge all cross toolchains.
+UPDATE_ARGS=( --skip_toolchain_update )
+if [[ ${FLAGS_usepkg} -eq ${FLAGS_TRUE} ]]; then
   UPDATE_ARGS+=( --usepkg )
 else
   UPDATE_ARGS+=( --nousepkg )
@@ -448,6 +446,11 @@ if [[ "$FLAGS_chroot" != "$DEFAULT_CHROOT_DIR" ]]; then
   CHROOT_EXAMPLE_OPT="--chroot=$FLAGS_chroot"
 fi
 
+# As a final pass, build all desired cross-toolchains.
+info "Updating toolchains"
+enter_chroot sudo "${CHROOT_TRUNK}/chromite/bin/cros_setup_toolchains" \
+    "${TOOLCHAIN_ARGS[@]}"
+
 print_time_elapsed
 
 cat <<EOF
diff --git a/setup_board b/setup_board
index bd7631f50a..0d7d2cc6d4 100755
--- a/setup_board
+++ b/setup_board
@@ -131,139 +131,6 @@ board_needs_libc_update() {
   return 0
 }
 
-# Check whether any new toolchain packages are available.
-# Returns true if new toolchain packages are available and false otherwise.
-toolchain_needs_update() {
-  # Skip toolchain updates if requested.
-  if [ $FLAGS_skip_toolchain_update -eq $FLAGS_TRUE ]; then
-    return 1
-  fi
-
-  local toolchain=$1 crossdev_cfg category arch
-
-  # Query crossdev for how it will be laying things out.
-  crossdev_cfg=$(crossdev --show-target-cfg "${toolchain}" --ex-gdb)
-  category=$(eval "${crossdev_cfg}"; echo "${category}")
-  arch=$(eval "${crossdev_cfg}"; echo "${arch}")
-
-  # If toolchain symlinks weren't created yet, we definitely need to update.
-  if [ ! -d ${CROSSDEV_OVERLAY}/${category} ]; then
-    # NOTE: In this branch, the versions have not been resolved, and will
-    # be passed directly to crossdev. That works because crossdev understands
-    # '[stable]'.
-    # We cannot resolve the versions because the cross toolchain category is
-    # not yet set up.
-    return 0
-  fi
-
-  # Unmask any ebuilds previously [un]masked by crossdev. crossdev will
-  # re-setup its masks appropriately the next time we run it.
-  local d
-  for d in package.{mask,keywords} ; do
-    d="/etc/portage/${d}"
-    if [[ -e ${d}/${category} ]] ; then
-      sudo mv -f ${d}/{,.bak.}${category}
-    fi
-  done
-
-  # Now get the atoms out of crossdev for the emerge below.
-  local pfx
-  if [[ ${FLAGS_latest_toolchain} -eq ${FLAGS_TRUE} ]] ; then
-    pfx=""
-  elif [[ ${FLAGS_usepkg} -eq ${FLAGS_TRUE} ]] ; then
-    pfx="<="
-  else
-    pfx="="
-  fi
-
-  local crosspkgs=$(eval "${crossdev_cfg}"; echo "${crosspkgs}")
-  local pkgs=()
-  for pkg in ${crosspkgs} ; do
-    local pn=$(eval "${crossdev_cfg}" \; echo \${${pkg}_pn})
-    local atom="${category}/${pn}"
-
-    if [[ -z ${pn} ]] ; then
-      # Not all toolchains provide all packages
-      # e.g. arm-elf does not have "kernel" headers.
-      continue
-    fi
-
-    if [[ ${FLAGS_latest_toolchain} -ne ${FLAGS_TRUE} ]] ; then
-      local flagname="FLAGS_${pkg}_version"
-
-      # Some packages (like gdb) don't have pinned versions.
-      if [[ -n ${!flagname} ]] ; then
-        if [[ ${!flagname} == "[stable]" ]] ; then
-          # This will return the full cat/pn-ver for us.
-          atom="${pfx}"$(ACCEPT_KEYWORDS="${arch}" \
-                         portageq best_visible / ebuild "${atom}")
-        else
-          atom="${pfx}${atom}-${!flagname}"
-        fi
-      fi
-    fi
-
-    pkgs+=( "${atom}" )
-  done
-
-  local flags=( --pretend --quiet --update --nodeps )
-  if [[ ${FLAGS_usepkg} -eq ${FLAGS_TRUE} ]] ; then
-    flags+=( --getbinpkg --usepkgonly )
-  fi
-
-  ACCEPT_KEYWORDS="~* *" emerge "${flags[@]}" "${pkgs[@]}" | grep ${category}/
-  local ret=$?
-
-  # Restore the masks in case we don't end up running crossdev.
-  for d in package.{mask,keywords} ; do
-    d="/etc/portage/${d}"
-    if [[ -e ${d}/.bak.${category} ]] ; then
-      sudo mv -f ${d}/{.bak.,}${category}
-    fi
-  done
-
-  return ${ret}
-}
-
-uninstall_toolchain() {
-  local toolchain=$1
-  info "Uninstalling the toolchain."
-  # Even if the uninstall fails, keep going. It's likely
-  # that it didn't exist in the first place.
-  sudo crossdev -v --force -C "${toolchain}" || true
-}
-
-# Build the toolchain with crossdev.
-build_toolchain() {
-  local toolchain=$1
-  info "Building the toolchain."
-  local CROSS_ARGS=( --show-fail-log --target "${toolchain}" -P --oneshot )
-  if [ $FLAGS_usepkg -eq $FLAGS_TRUE ]; then
-    # Grab the latest packages from the prebuilt server.
-    #  --getbinpkg: Use packages from the prebuilt server.
-    #  --usepkgonly: Always use prebuilts. Don't build from source.
-    #  --without-headers: Don't build headers-only versions of packages for
-    #                     bootstrapping. Because we use binary packages, this
-    #                     isn't necessary.
-    CROSS_ARGS+=( -P --getbinpkg -P --usepkgonly --without-headers )
-  fi
-  if [ $FLAGS_latest_toolchain -ne $FLAGS_TRUE ]; then
-    CROSS_ARGS+=(
-      --binutils "${FLAGS_binutils_version}"
-      --gcc "${FLAGS_gcc_version}"
-      --kernel "${FLAGS_kernel_version}"
-      --libc "${FLAGS_libc_version}"
-    )
-  fi
-  CROSS_ARGS+=(
-    --overlays "${CHROMIUMOS_OVERLAY} /usr/local/portage/stable"
-    --ov-output "${CROSSDEV_OVERLAY}"
-    --ex-gdb
-  )
-
-  sudo -E FEATURES="splitdebug ${FEATURES}" crossdev "${CROSS_ARGS[@]}"
-}
-
 # Get the version number of a toolchain package.
 cross_get_version() {
   local pkg="$1"
@@ -543,62 +410,33 @@ BOARD_OVERLAY_LIST=$(cros_overlay_list \
 
 eval $(portageq envvar -v CHOST PKGDIR)
 
-# Update all the toolchains that this board wants.
-for toolchain in ${all_toolchains[@]} ; do
-  [[ "${CHOST}" == "${toolchain}" ]] && continue
-  info "Checking for updates to ${toolchain} ..."
-
-  toolchain_updated=${FLAGS_FALSE}
-  if toolchain_needs_update ${toolchain} ; then
-    toolchain_updated=${FLAGS_TRUE}
-
-    warn "Toolchain needs to be updated! Updating ${toolchain} ..."
-    uninstall_toolchain ${toolchain}
-    build_toolchain ${toolchain}
-
-    if [[ "${toolchain}" != arm* ]] ; then
-      info "Switching on gold as the default linker."
-      BINUTILS_VERSION=$(cross_get_version binutils ${toolchain} | \
-                         sed 's/-r[0-9]\+//g')
-      sudo binutils-config "${toolchain}-${BINUTILS_VERSION}-gold"
-    fi
-  fi
-
-  if [[ "${toolchain}" == "${FLAGS_toolchain}" ]] &&
-     [ -d "$BOARD_ROOT" ] && [ "$FLAGS_force" == "$FLAGS_FALSE" ] ; then
-    if [[ "${toolchain_updated}" == "${FLAGS_TRUE}" ]] ; then
-      # If the board root already exists, re-install the toolchain there.
-      install_toolchain_in_board
-    elif board_needs_libc_update; then
-      # Update the users libc in their board if needed.
-      echo "Updating libc in the board."
-      install_toolchain_in_board
-    fi
-  fi
-done
-
-if [ -d "$BOARD_ROOT" ] ; then
-  if [[ $FLAGS_force -eq $FLAGS_TRUE ]]; then
-    echo "--force set.  Re-creating $BOARD_ROOT..."
+if [ -d "${BOARD_ROOT}" ] ; then
+  if [[ ${FLAGS_force} -eq ${FLAGS_TRUE} ]]; then
+    echo "--force set.  Re-creating ${BOARD_ROOT}..."
     # Removal takes long. Make it asynchronous.
     TEMP_DIR=`mktemp -d`
-    sudo mv $BOARD_ROOT $TEMP_DIR
-    sudo rm -rf $TEMP_DIR &
+    sudo mv "${BOARD_ROOT}" "${TEMP_DIR}"
+    sudo rm -rf "${TEMP_DIR}" &
   else
-    if ! $HOST_BOARD; then
+    if ! ${HOST_BOARD}; then
       { print_board_make_conf; print_board_binhost_config; } | \
         sudo_clobber $BOARD_ETC/make.conf.board
     fi
-    if [[ $FLAGS_quiet -eq $FLAGS_FALSE ]]; then
+    if [[ ${FLAGS_quiet} -eq ${FLAGS_FALSE} ]]; then
       warn "Board output directory '$BOARD_ROOT' already exists."
       warn "Not setting up board root. "
       warn "Use --force to clobber the board root and start again."
     fi
+    if board_needs_libc_update; then
+      # Update the users libc in their board if needed.
+      echo "Updating libc in the board."
+      install_toolchain_in_board
+    fi
     exit 0
   fi
 fi
 
-sudo mkdir -p "$BOARD_ROOT" "${BOARD_ETC}" "${BOARD_PROFILE}"
+sudo mkdir -p "${BOARD_ROOT}" "${BOARD_ETC}" "${BOARD_PROFILE}"
 
 if [ "${CHOST}" != "$FLAGS_toolchain" ] ; then
   # TODO(cmasone): Do this more cleanly, if we figure out what "cleanly" means.
diff --git a/update_chroot b/update_chroot
index f2b40c7d52..bbfaff310c 100755
--- a/update_chroot
+++ b/update_chroot
@@ -29,6 +29,8 @@ show_help_if_requested "$@"
 DEFINE_boolean fast ${DEFAULT_FAST} "Call many emerges in parallel"
 DEFINE_integer jobs -1 \
   "How many packages to build in parallel at maximum."
+DEFINE_boolean skip_toolchain_update $FLAGS_FALSE \
+  "Don't update the toolchains."
 
 # Parse command line flags
 FLAGS "$@" || exit 1
@@ -80,18 +82,14 @@ fi
 
 # In first pass, update portage and toolchains. Lagged updates of both
 # can cause serious issues later.
-export CHOST="$(portageq envvar CHOST)"
-LATEST=$(gcc-config -l | awk -v chost="${CHOST}" '$2 ~ chost { print $2 }' | \
-         sort -V | tail -n 1)
-CURRENT="$(gcc-config -c)" || true # This fails if current profile is invalid.
-sudo -E ${EMERGE_CMD} ${EMERGE_FLAGS} \
-    sys-devel/gcc sys-devel/binutils sys-libs/glibc sys-apps/portage
-# If the latest toolchain wasn't already selected before we updated, do nothing,
-# otherwise autoselect the latest. Also fix if the current profile is invalid.
-if [ "${LATEST}" != "${CURRENT}" ] || ! gcc-config -c &> /dev/null; then
-  LATEST=$(gcc-config -l | awk -v chost="${CHOST}" '$2 ~ chost { print $2 }' | \
-           sort -V | tail -n 1 )
-  sudo -E gcc-config "${LATEST}"
+if [ "${FLAGS_skip_toolchain_update}" -eq "${FLAGS_FALSE}" ]; then
+  TOOLCHAIN_FLAGS=""
+  # This should really only be skipped while bootstrapping.
+  if [ "${FLAGS_usepkg}" -eq "${FLAGS_FALSE}" ]; then
+    TOOLCHAIN_FLAGS="--nousepkg"
+  fi
+  # Expand the path before sudo, as root doesn't have the same path magic.
+  sudo $(type -p cros_setup_toolchains) ${TOOLCHAIN_FLAGS}
 fi
 
 # Second pass, update everything else.