Merge pull request #138 from marineam/turtle-chain

Turtle chain

build_image

@@ -49,7 +49,9 @@ show_help_if_requested "$@"
 # not needed for the typical developer workflow.
 DEFINE_integer build_attempt 1 \
   "The build attempt for this image build."
-DEFINE_integer jobs -1 \
+DEFINE_boolean fast "${DEFAULT_FAST}" \
+  "Use the parallel_emerge wrapper script."
+DEFINE_integer jobs "${NUM_JOBS}" \
   "How many packages to build in parallel at maximum."
 DEFINE_boolean replace ${FLAGS_FALSE} \
   "Overwrite existing output, if any."

build_library/base_image_util.sh

@@ -143,56 +143,23 @@ create_base_image() {
 
   # First thing first, install baselayout with USE=build to create a
   # working directory tree. Don't use binpkgs due to the use flag change.
-  sudo -E USE=build ${EMERGE_BOARD_CMD} --root="${root_fs_dir}" \
+  sudo -E USE=build "emerge-${BOARD}" --root="${root_fs_dir}" \
       --usepkg=n --buildpkg=n --oneshot --quiet --nodeps sys-apps/baselayout
 
-  # We need to install libc manually from the cross toolchain.
+  # FIXME(marineam): Work around glibc setting EROOT=$ROOT
-  # TODO: Improve this? It would be ideal to use emerge to do this.
+  # https://bugs.gentoo.org/show_bug.cgi?id=473728#c12
-  PKGDIR="/var/lib/portage/pkgs"
+  sudo mkdir -p "${root_fs_dir}/etc/ld.so.conf.d"
-  LIBC_TAR="glibc-${LIBC_VERSION}.tbz2"
-  LIBC_PATH="${PKGDIR}/cross-${CHOST}/${LIBC_TAR}"
-
-  if ! [[ -e ${LIBC_PATH} ]]; then
-  die_notrace \
-    "${LIBC_PATH} does not exist. Try running ./setup_board" \
-    "--board=${BOARD} to update the version of libc installed on that board."
-  fi
-
-  # Strip out files we don't need in the final image at runtime.
-  local libc_excludes=(
-    # Compile-time headers.
-    'usr/include' 'sys-include'
-    # Link-time objects.
-    '*.[ao]'
-    # Empty lib dirs, replaced by symlinks
-    'lib'
-    # Locales and info pages
-    usr/share/{i18n,info,locale}
-  )
-
-  mkdir -p "${root_fs_dir}"/sbin "${root_fs_dir}"/lib64
-  lbzip2 -dc "${LIBC_PATH}" | \
-    sudo tar xpf - -C "${root_fs_dir}" ./usr/${CHOST} \
-      --strip-components=3 "${libc_excludes[@]/#/--exclude=}" \
-      --exclude=${CHOST}/sbin --exclude=${CHOST}/lib64
-  lbzip2 -dc "${LIBC_PATH}" | \
-    sudo tar xpf - -C "${root_fs_dir}"/lib64 ./usr/${CHOST}/lib64 \
-      --strip-components=4 "${libc_excludes[@]/#/--exclude=}"
-  lbzip2 -dc "${LIBC_PATH}" | \
-    sudo tar xpf - -C "${root_fs_dir}"/sbin ./usr/${CHOST}/sbin \
-      --strip-components=4 "${libc_excludes[@]/#/--exclude=}"
-
-  board_ctarget=$(get_ctarget_from_board "${BOARD}")
-  for atom in $(portageq match / cross-$board_ctarget/gcc); do
-    copy_gcc_libs "${root_fs_dir}" $atom
-  done
 
   # We "emerge --root=${root_fs_dir} --root-deps=rdeps --usepkgonly" all of the
   # runtime packages for chrome os. This builds up a chrome os image from
   # binary packages with runtime dependencies only.  We use INSTALL_MASK to
   # trim the image size as much as possible.
+  emerge_prod_gcc --root="${root_fs_dir}"
   emerge_to_image --root="${root_fs_dir}" ${BASE_PACKAGE}
 
+  # Make sure profile.env and ld.so.cache has been generated
+  sudo ROOT="${root_fs_dir}" env-update
+
   # Record directories installed to the state partition.
   # Ignore /var/tmp, systemd covers this entry.
   sudo "${BUILD_LIBRARY_DIR}/gen_tmpfiles.py" --root="${root_fs_dir}" \

build_library/build_image_util.sh

@@ -20,15 +20,6 @@ BUILD_DIR="${FLAGS_output_root}/${BOARD}/${IMAGE_SUBDIR}"
 OUTSIDE_OUTPUT_DIR="../build/images/${BOARD}/${IMAGE_SUBDIR}"
 IMAGES_TO_BUILD=
 
-EMERGE_BOARD_CMD="$GCLIENT_ROOT/chromite/bin/parallel_emerge"
-EMERGE_BOARD_CMD="$EMERGE_BOARD_CMD --board=$BOARD"
-
-export INSTALL_MASK="${DEFAULT_INSTALL_MASK}"
-
-if [[ $FLAGS_jobs -ne -1 ]]; then
-  EMERGE_JOBS="--jobs=$FLAGS_jobs"
-fi
-
 # Populates list of IMAGES_TO_BUILD from args passed in.
 # Arguments should be the shortnames of images we want to build.
 get_images_to_build() {
@@ -140,6 +131,41 @@ generate_au_zip () {
 # Arguments to this command are passed as addition options/arguments
 # to the basic emerge command.
 emerge_to_image() {
-  sudo -E ${EMERGE_BOARD_CMD} --root-deps=rdeps --usepkgonly -v \
+  local mask="${INSTALL_MASK:-$(portageq-$BOARD envvar PROD_INSTALL_MASK)}"
-    "$@" ${EMERGE_JOBS}
+  test -n "$mask" || die "PROD_INSTALL_MASK not defined"
+
+  local emerge_cmd
+  if [[ "${FLAGS_fast}" -eq "${FLAGS_TRUE}" ]]; then
+    emerge_cmd="$GCLIENT_ROOT/chromite/bin/parallel_emerge --board=$BOARD"
+  else
+    emerge_cmd="emerge-$BOARD"
+  fi
+  emerge_cmd+=" --root-deps=rdeps --usepkgonly -v"
+
+  if [[ $FLAGS_jobs -ne -1 ]]; then
+    emerge_cmd+=" --jobs=$FLAGS_jobs"
+  fi
+
+  sudo -E INSTALL_MASK="$mask" ${emerge_cmd} "$@"
+}
+
+# The GCC package includes both its libraries and the compiler.
+# In prod images we only need the shared libraries.
+emerge_prod_gcc() {
+    local mask="${INSTALL_MASK:-$(portageq-$BOARD envvar PROD_INSTALL_MASK)}"
+    test -n "$mask" || die "PROD_INSTALL_MASK not defined"
+
+    mask="${mask}
+        /usr/bin
+        /usr/*/gcc-bin
+        /usr/lib/gcc/*/*/*.o
+        /usr/lib/gcc/*/*/include
+        /usr/lib/gcc/*/*/include-fixed
+        /usr/lib/gcc/*/*/plugin
+        /usr/libexec
+        /usr/share/gcc-data/*/*/c89
+        /usr/share/gcc-data/*/*/c99
+        /usr/share/gcc-data/*/*/python"
+
+    INSTALL_MASK="${mask}" emerge_to_image --nodeps sys-devel/gcc "$@"
 }

build_library/catalyst_toolchains.sh

@@ -11,33 +11,6 @@ source /tmp/toolchain_util.sh
 # build deps in crossdev's sysroot use ${PKGDIR}/cross/${CHOST} (no upload)
 # native toolchains use ${PKGDIR}/target/${BOARD} (uploaded to board location)
 
-build_cross_libs() {
-    local cross_chost="$1"
-    local ROOT="/usr/${cross_chost}"
-
-    CHOST="${cross_chost}" ROOT="$ROOT" SYSROOT="$ROOT" \
-        _configure_sysroot "${CROSS_PROFILE[${cross_chost}]}"
-
-    # In order to get a dependency list we must calculate it before
-    # updating package.provided. Otherwise portage will no-op.
-    ROOT="$ROOT" _get_dependency_list \
-        ${clst_myemergeopts} "${TOOLCHAIN_PKGS[@]}" > \
-        "$ROOT/etc/portage/cross-${cross_chost}-depends"
-
-    # Add toolchain to packages.provided since they are on the host system
-    mkdir -p "$ROOT/etc/portage/profile/package.provided"
-    local native_pkg cross_pkg cross_pkg_version
-    for native_pkg in "${TOOLCHAIN_PKGS[@]}"; do
-        cross_pkg="${native_pkg/*\//cross-${cross_chost}/}"
-        cross_pkg_version=$(portageq match / "${cross_pkg}")
-        echo "${native_pkg%/*}/${cross_pkg_version#*/}"
-    done > "$ROOT/etc/portage/profile/package.provided/cross-${cross_chost}"
-    local cross_deps=$(<"$ROOT/etc/portage/cross-${cross_chost}-depends")
-
-    # --root is required because run_merge overrides ROOT=
-    PORTAGE_CONFIGROOT="$ROOT" run_merge -u --root="$ROOT" $cross_deps
-}
-
 configure_target_root() {
     local board="$1"
     local cross_chost=$(get_board_chost "$1")
@@ -65,9 +38,8 @@ for cross_chost in $(get_chost_list); do
     echo "Building cross toolchain for ${cross_chost}"
     PKGDIR="$(portageq envvar PKGDIR)/crossdev" \
         install_cross_toolchain "${cross_chost}" ${clst_myemergeopts}
-
+    PKGDIR="$(portageq envvar PKGDIR)/cross/${cross_chost}" \
-    sysroot_pkgdir="$(portageq envvar PKGDIR)/cross/${cross_chost}"
+        install_cross_libs "${cross_chost}" ${clst_myemergeopts}
-    PKGDIR="${sysroot_pkgdir}" build_cross_libs "${cross_chost}"
 done
 
 for board in $(get_board_list); do

build_library/dev_image_util.sh

@@ -22,12 +22,8 @@ install_dev_packages() {
   # Install developer packages described in coreos-dev.
   emerge_to_image --root="${root_fs_dir}" coreos-base/coreos-dev
 
-  # Copy over the libc debug info so that gdb
+  # Make sure profile.env and ld.so.cache has been generated
-  # works with threads and also for a better debugging experience.
+  sudo ROOT="${root_fs_dir}" env-update
-  sudo mkdir -p "${root_fs_dir}/usr/lib/debug"
-  lbzip2 -dc "${LIBC_PATH}" | \
-    sudo tar xpf - -C "${root_fs_dir}/usr/lib/debug" \
-      ./usr/lib/debug/usr/${CHOST} --strip-components=6
 
   # Install the bare necessary files so that the "emerge" command works
   sudo mkdir -p ${root_fs_dir}/etc/make.profile

build_library/toolchain_util.sh

@@ -10,23 +10,28 @@ TOOLCHAIN_PKGS=(
     sys-libs/glibc
 )
 
-# Portage arguments to enforce the toolchain to only use binpkgs.
-TOOLCHAIN_BINONLY=( "${TOOLCHAIN_PKGS[@]/#/--useoldpkg-atoms=}"
-                    "${TOOLCHAIN_PKGS[@]/#/--rebuild-exclude=}" )
-
 # Portage profile to use for building out the cross compiler's SYSROOT.
 # This is only used as an intermediate step to be able to use the cross
 # compiler to build a full native toolchain. Packages are not uploaded.
-CROSS_PROFILE["x86_64-cros-linux-gnu"]="coreos:coreos/amd64/generic"
+declare -A CROSS_PROFILES
+CROSS_PROFILES["x86_64-cros-linux-gnu"]="coreos:coreos/amd64/generic"
 
 # Map board names to CHOSTs and portage profiles. This is the
 # definitive list, there is assorted code new and old that either
 # guesses or hard-code these. All that should migrate to this list.
-declare -A BOARD_CHOST BOARD_PROFILE
+declare -A BOARD_CHOSTS BOARD_PROFILES
-BOARD_CHOST["amd64-generic"]="x86_64-cros-linux-gnu"
+BOARD_CHOSTS["amd64-generic"]="x86_64-cros-linux-gnu"
-BOARD_PROFILE["amd64-generic"]="coreos:coreos/amd64/generic"
+BOARD_PROFILES["amd64-generic"]="coreos:coreos/amd64/generic"
 BOARD_NAMES=( "${!BOARD_CHOST[@]}" )
 
+# Declare the above globals as read-only to avoid accidental conflicts.
+declare -r \
+    TOOLCHAIN_PKGS \
+    CROSS_PROFILES \
+    BOARD_CHOSTS \
+    BOARD_NAMES \
+    BOARD_PROFILES
+
 ### Generic metadata fetching functions ###
 
 # map CHOST to portage ARCH, list came from crossdev
@@ -58,12 +63,12 @@ get_board_list() {
 
 get_chost_list() {
     local IFS=$'\n\t '
-    sort -u <<<"${BOARD_CHOST[*]}"
+    sort -u <<<"${BOARD_CHOSTS[*]}"
 }
 
 get_profile_list() {
     local IFS=$'\n\t '
-    sort -u <<<"${BOARD_PROFILE[*]}"
+    sort -u <<<"${BOARD_PROFILES[*]}"
 }
 
 # Usage: get_board_arch board [board...]
@@ -78,8 +83,8 @@ get_board_arch() {
 get_board_chost() {
     local board
     for board in "$@"; do
-        if [[ ${#BOARD_CHOST["$board"]} -ne 0 ]]; then
+        if [[ ${#BOARD_CHOSTS["$board"]} -ne 0 ]]; then
-            echo "${BOARD_CHOST["$board"]}"
+            echo "${BOARD_CHOSTS["$board"]}"
         else
             die "Unknown board '$board'"
         fi
@@ -90,8 +95,8 @@ get_board_chost() {
 get_board_profile() {
     local board
     for board in "$@"; do
-        if [[ ${#BOARD_PROFILE["$board"]} -ne 0 ]]; then
+        if [[ ${#BOARD_PROFILES["$board"]} -ne 0 ]]; then
-            echo "${BOARD_PROFILE["$board"]}"
+            echo "${BOARD_PROFILES["$board"]}"
         else
             die "Unknown board '$board'"
         fi
@@ -108,6 +113,12 @@ get_cross_pkgs() {
     done
 }
 
+# Get portage arguments restricting toolchains to binary packages only.
+get_binonly_args() {
+    local pkgs=( "${TOOLCHAIN_PKGS[@]}" $(get_cross_pkgs "$@") )
+    echo "${pkgs[@]/#/--useoldpkg-atoms=}" "${pkgs[@]/#/--rebuild-exclude=}"
+}
+
 ### Toolchain building utilities ###
 
 # Ugly hack to get a dependency list of a set of packages.
@@ -129,11 +140,17 @@ _get_dependency_list() {
 _configure_sysroot() {
     local profile="$1"
 
-    mkdir -p "${ROOT}/etc/portage"
+    # may be called from either catalyst (root) or setup_board (user)
-    echo "eselect will report '!!! Warning: Strange path.' but that's OK"
+    local sudo=
-    eselect profile set --force "$profile"
+    if [[ $(id -u) -ne 0 ]]; then
+        sudo="sudo -E"
+    fi
 
-    cat >"${ROOT}/etc/portage/make.conf" <<EOF
+    $sudo mkdir -p "${ROOT}/etc/portage"
+    echo "eselect will report '!!! Warning: Strange path.' but that's OK"
+    $sudo eselect profile set --force "$profile"
+
+    $sudo tee "${ROOT}/etc/portage/make.conf" >/dev/null <<EOF
 $(portageq envvar -v CHOST CBUILD ROOT SYSROOT \
     PORTDIR PORTDIR_OVERLAY DISTDIR PKGDIR)
 HOSTCC=\${CBUILD}-gcc
@@ -171,16 +188,16 @@ install_cross_toolchain() {
         $sudo tee "${cross_cfg}" <<<"${cross_cfg_data}" >/dev/null
     fi
 
-    # If binary packages are enabled try to just emerge them instead of
+    # Check if any packages need to be built from source. If so do a full
-    # doing a full bootstrap which speeds things up greatly. :)
+    # bootstrap via crossdev, otherwise just install the binaries (if enabled).
-    if [[ "$*" == *--usepkg* ]] && \
+    if emerge "$@" --binpkg-respect-use=y --update --newuse \
-        emerge "$@" --usepkgonly --binpkg-respect-use=y \
+        --pretend "${cross_pkgs[@]}" | grep -q '^\[ebuild'
-            --pretend "${cross_pkgs[@]}" &>/dev/null
     then
-        $sudo emerge "$@" --binpkg-respect-use=y -u "${cross_pkgs[@]}"
-    else
         $sudo crossdev --stable --portage "$*" \
             --stage4 --target "${cross_chost}"
+    else
+        $sudo emerge "$@" --binpkg-respect-use=y --update --newuse \
+            "${cross_pkgs[@]}"
     fi
 
     # Setup wrappers for our shiny new toolchain
@@ -191,3 +208,41 @@ install_cross_toolchain() {
         $sudo sysroot-config --install-links "${cross_chost}"
     fi
 }
+
+# Build/install toolchain dependencies into the cross sysroot for a
+# given CHOST. This is required to build target board toolchains since
+# the target sysroot under /build/$BOARD is incomplete at this stage.
+# Usage: build_cross_toolchain chost [--portage-opts....]
+install_cross_libs() {
+    local cross_chost="$1"; shift
+    local ROOT="/usr/${cross_chost}"
+    local package_provided="$ROOT/etc/portage/profile/package.provided"
+
+    # may be called from either catalyst (root) or setup_board (user)
+    local sudo=
+    if [[ $(id -u) -ne 0 ]]; then
+        sudo="sudo -E"
+    fi
+
+    CHOST="${cross_chost}" ROOT="$ROOT" SYSROOT="$ROOT" \
+        _configure_sysroot "${CROSS_PROFILES[${cross_chost}]}"
+
+    # In order to get a dependency list we must calculate it before
+    # updating package.provided. Otherwise portage will no-op.
+    $sudo rm -f "${package_provided}/cross-${cross_chost}"
+    local cross_deps=$(ROOT="$ROOT" _get_dependency_list \
+        "$@" "${TOOLCHAIN_PKGS[@]}" | $sudo tee \
+        "$ROOT/etc/portage/cross-${cross_chost}-depends")
+
+    # Add toolchain to packages.provided since they are on the host system
+    $sudo mkdir -p "${package_provided}"
+    local native_pkg cross_pkg cross_pkg_version
+    for native_pkg in "${TOOLCHAIN_PKGS[@]}"; do
+        cross_pkg="${native_pkg/*\//cross-${cross_chost}/}"
+        cross_pkg_version=$(portageq match / "${cross_pkg}")
+        echo "${native_pkg%/*}/${cross_pkg_version#*/}"
+    done | $sudo tee "${package_provided}/cross-${cross_chost}" >/dev/null
+
+    # OK, clear as mud? Install those dependencies now!
+    PORTAGE_CONFIGROOT="$ROOT" ROOT="$ROOT" $sudo emerge "$@" -u $cross_deps
+}

build_packages

@@ -50,8 +50,8 @@ show_help_if_requested "$@"
 DEFINE_string accept_licenses "" \
   "Licenses to append to the accept list."
 DEFINE_boolean fast "${DEFAULT_FAST}" \
-  "Call many emerges in parallel."
+  "Use the parallel_emerge wrapper script."
-DEFINE_integer jobs -1 \
+DEFINE_integer jobs "${NUM_JOBS}" \
   "How many packages to build in parallel at maximum."
 DEFINE_boolean norebuild "${FLAGS_FALSE}" \
   "Don't automatically rebuild dependencies."
@@ -156,6 +156,9 @@ if [[ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ||
   if [[ "${FLAGS_getbinpkg}" -eq "${FLAGS_TRUE}" ]]; then
       EMERGE_FLAGS+=( --getbinpkg )
   fi
+
+  # Only update toolchain when binpkgs are available.
+  EMERGE_FLAGS+=( $(get_binonly_args) )
 fi
 
 if [[ "${FLAGS_jobs}" -ne -1 ]]; then

common.sh

@@ -413,49 +413,6 @@ COREOS_DEVELOPER_IMAGE_NAME=${CHROMEOS_DEVELOPER_IMAGE_NAME}
 COREOS_PRODUCTION_IMAGE_NAME="coreos_production_image.bin"
 COREOS_RECOVERY_IMAGE_NAME=${CHROMEOS_RECOVERY_IMAGE_NAME}
 
-# Install make for portage ebuilds.  Used by build_image and gmergefs.
-# TODO: Is /usr/local/autotest-chrome still used by anyone?
-COMMON_INSTALL_MASK="
-  *.a
-  *.la
-  *.h
-  *.hpp
-  /etc/init.d
-  /etc/runlevels
-  /firmware
-  /lib/rc
-  /usr/bin/Xnest
-  /usr/bin/Xvfb
-  /usr/lib/debug
-  /usr/lib/gcc
-  /usr/lib*/pkgconfig
-  /usr/lib/systemd/system/local-fs.target.wants
-  /usr/local/autotest-chrome
-  /usr/man
-  /usr/share/aclocal
-  /usr/share/doc
-  /usr/share/gettext
-  /usr/share/gtk-2.0
-  /usr/share/gtk-doc
-  /usr/share/info
-  /usr/share/man
-  /usr/share/openrc
-  /usr/share/pkgconfig
-  /usr/share/profiling
-  /usr/share/readline
-  /usr/src
-  "
-
-# Mask for base, dev, and test images (build_image, build_image --test)
-DEFAULT_INSTALL_MASK="
-  ${COMMON_INSTALL_MASK}
-  /usr/local/autotest
-  /lib/modules/*/kernel/drivers/input/misc/uinput.ko
-  /lib/modules/*/build
-  /lib/modules/*/source
-  test_*.ko
-  "
-
 # -----------------------------------------------------------------------------
 # Functions
 
@@ -927,44 +884,6 @@ relpath() {
   python2 -c "${py}" "${1}" "${2:-.}"
 }
 
-emerge_custom_kernel() {
-  local install_root=$1
-  local root=/build/${FLAGS_board}
-  local tmp_pkgdir=${root}/custom-packages
-
-  # Clean up any leftover state in custom directories.
-  sudo rm -rf "${tmp_pkgdir}"
-
-  # Update chromeos-initramfs to contain the latest binaries from the build
-  # tree. This is basically just packaging up already-built binaries from
-  # ${root}. We are careful not to muck with the existing prebuilts so that
-  # prebuilts can be uploaded in parallel.
-  # TODO(davidjames): Implement ABI deps so that chromeos-initramfs will be
-  # rebuilt automatically when its dependencies change.
-  sudo -E PKGDIR="${tmp_pkgdir}" ${EMERGE_BOARD_CMD} -1 \
-    chromeos-base/chromeos-initramfs || die "Cannot emerge chromeos-initramfs"
-
-  # Verify all dependencies of the kernel are installed. This should be a
-  # no-op, but it's good to check in case a developer didn't run
-  # build_packages.  We need the expand_virtual call to workaround a bug
-  # in portage where it only installs the virtual pkg.
-  local kernel=$(portageq-${FLAGS_board} expand_virtual ${root} \
-                 virtual/linux-sources)
-  sudo -E PKGDIR="${tmp_pkgdir}" ${EMERGE_BOARD_CMD} --onlydeps \
-    ${kernel} || die "Cannot emerge kernel dependencies"
-
-  # Build the kernel. This uses the standard root so that we can pick up the
-  # initramfs from there. But we don't actually install the kernel to the
-  # standard root, because that'll muck up the kernel debug symbols there,
-  # which we want to upload in parallel.
-  sudo -E PKGDIR="${tmp_pkgdir}" ${EMERGE_BOARD_CMD} --buildpkgonly \
-    ${kernel} || die "Cannot emerge kernel"
-
-  # Install the custom kernel to the provided install root.
-  sudo -E PKGDIR="${tmp_pkgdir}" ${EMERGE_BOARD_CMD} --usepkgonly \
-    --root=${install_root} ${kernel} || die "Cannot emerge kernel to root"
-}
-
 enable_strict_sudo() {
   if [[ -z ${CROS_SUDO_KEEP_ALIVE} ]]; then
     echo "$0 was somehow invoked in a way that the sudo keep alive could"

install_toolchain

@@ -1,208 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-. "$(dirname "$0")/common.sh" || exit 1
-. "${SRC_ROOT}/platform/dev/toolchain_utils.sh" || exit 1
-
-# Script must run inside the chroot
-restart_in_chroot_if_needed "$@"
-
-assert_not_root_user
-
-# Developer-visible flags.
-DEFINE_string board "$DEFAULT_BOARD" \
-  "The name of the board to set up."
-DEFINE_string board_root "" \
-  "Path of the board root to install into.  Defaults to /build/<board>."
-DEFINE_boolean configure $FLAGS_TRUE \
-  "Update config files in <board_root>/etc after installation."
-DEFINE_boolean force $FLAGS_FALSE \
-  "Install toolchain even if already up to date."
-DEFINE_string toolchain "" \
-  "Toolchain.  For example: i686-pc-linux-gnu, armv7a-softfloat-linux-gnueabi"
-
-
-FLAGS_HELP="usage: $(basename $0) [flags]
-
-Installs cross toolchain libraries into a board_root.  This script is not
-meant to be used by developers directly.  Run at your own risk.
-"
-
-show_help_if_requested "$@"
-
-# Parse command line flags
-FLAGS "$@" || exit 1
-eval set -- "${FLAGS_ARGV}"
-
-# Only now can we die on error.  shflags functions leak non-zero error codes,
-# so will die prematurely if 'switch_to_strict_mode' is specified before now.
-switch_to_strict_mode
-
-# Get the version number of a toolchain package.
-cross_get_version() {
-  local pkg=$1
-  local toolchain=${2:-${FLAGS_toolchain}}
-  local cpv
-  if [[ "$CHOST" != "${toolchain}" ]]; then
-    if [[  "$pkg" = "gcc" ]]; then
-      # Users can install multiple versions of gcc at once, so we need to call
-      # gcc-config to find out which installed version is selected.
-      local path=$(CTARGET="${toolchain}" gcc-config -B || true)
-      cpv=$(portageq owners / "$path" | sed -e '/^\t/d')
-    else
-      cpv=$(portageq match / "cross-${toolchain}/${pkg}" || true)
-    fi
-  else
-    if [[ "$pkg" = glibc ]] ; then
-      cpv=$(portageq match / sys-libs/glibc || true)
-    elif [[ "$pkg" = gcc ]] ; then
-      cpv=$(portageq match / sys-devel/gcc || true)
-    else
-      die "Unknown pkg ${pkg}"
-    fi
-  fi
-  local cp=$(echo $cpv | sed -e 's/-r[0-9]*$//; s/-[^-]*$//')
-  local result="${cpv#$cp-}"
-  local count="$(echo $result | wc -w)"
-  if [ "$count" -gt "1" ]; then
-    die "Multiple versions of $pkg installed"
-  elif [ "$count" -lt "1" ]; then
-    die "Cannot find $pkg"
-  fi
-  echo $result
-}
-
-# Checks whether the libc version installed in the board
-# matches the one installed by the toolchain.
-board_needs_libc_update() {
-  if [[ ! -e "${BOARD_SETUP}" ]]; then
-    return 0
-  fi
-
-  local board_version=$(. "${BOARD_SETUP}"; echo "${LIBC_VERSION}")
-  local toolchain_version=$(cross_get_version glibc)
-  if [[ "${board_version}" = "${toolchain_version}" ]]; then
-    return 1
-  fi
-  return 0
-}
-
-install_toolchain_in_provided() {
-  local gcc_ver="$1"
-  local glibc_ver="$2"
-  # Tell portage that toolchain packages are already installed in the sysroot.
-  sudo_clobber "$BOARD_PROFILE/package.provided" << EOF
-sys-devel/gcc-$gcc_ver
-sys-libs/glibc-$glibc_ver
-EOF
-}
-
-# Install all of the stuff that depends on the toolchain versions
-# into the board root.
-install_toolchain_in_board() {
-  local cmds
-  local gcc_ver=$(cross_get_version gcc)
-  local libc_ver=$(cross_get_version glibc)
-  if [[ -z ${gcc_ver} || -z ${libc_ver} ]]; then
-    die "Cannot find toolchain to install into board root"
-  fi
-  echo "Installing the toolchain into the board root."
-  # Untar glibc to get most of the headers required to build.
-  local libc_tar="glibc-${libc_ver}.tbz2"
-
-  # Install libc libraries.
-  if [ "${CHOST}" != "$FLAGS_toolchain" ] ; then
-    local libc_path="${PKGDIR}/cross-${FLAGS_toolchain}/${libc_tar}"
-    local libc_excludes=(
-      # Empty lib dirs, replaced by symlinks
-      'lib'
-      # Locales and info pages
-      usr/share/{i18n,info,locale}
-    )
-    cmds=(
-      "mkdir -p '${BOARD_ROOT}/lib64' '${BOARD_ROOT}/sbin'"
-      "tar jxpf '${libc_path}' -C '${BOARD_ROOT}' \
-          --exclude=${FLAGS_toolchain}/lib64 --exclude=${FLAGS_toolchain}/sbin \
-          ${libc_excludes[@]/#/--exclude=} \
-          './usr/${FLAGS_toolchain}' --strip-components=3"
-      "tar jxpf '${libc_path}' -C '${BOARD_ROOT}/lib64' \
-          ${libc_excludes[@]/#/--exclude=} \
-          './usr/${FLAGS_toolchain}/lib64' --strip-components=4"
-      "tar jxpf '${libc_path}' -C '${BOARD_ROOT}/sbin' \
-          ${libc_excludes[@]/#/--exclude=} \
-          './usr/${FLAGS_toolchain}/sbin' --strip-components=4"
-      "mkdir -p '${BOARD_ROOT}/usr/lib/debug'"
-      "tar jxpf '${libc_path}' -C '${BOARD_ROOT}/usr/lib/debug' \
-          ${libc_excludes[@]/#/--exclude=} \
-          './usr/lib/debug/usr/${FLAGS_toolchain}' --strip-components=6 \
-          || warn 'libc debug info not copied.'"
-    )
-    # TODO(asharif): Remove this hack after a while.
-    local board_gcc_dir="${BOARD_ROOT}/usr/lib/gcc"
-    if [[ -L "${board_gcc_dir}" ]] ; then
-      cmd+=("rm -f '${board_gcc_dir}'")
-    fi
-    sudo_multi "${cmds[@]}"
-    copy_gcc_libs "${BOARD_ROOT}" "cross-$FLAGS_toolchain/gcc-$gcc_ver"
-  else
-    cmds=(
-      "mkdir -p '${BOARD_ROOT}'{/usr,}/lib64 '${BOARD_ROOT}/usr/lib/debug'"
-      "ln -sfT '${BOARD_ROOT}/usr/lib64' '${BOARD_ROOT}/usr/lib'"
-      "ln -sfT '${BOARD_ROOT}/lib64' '${BOARD_ROOT}/lib'"
-      "emerge --oneshot --nodeps -k --root='${BOARD_ROOT}' \
-          =sys-libs/glibc-${libc_ver}"
-    )
-    sudo_multi "${cmds[@]}"
-  fi
-
-  # Some header files are needed also for rpcbind (NFS support)
-  # TODO: Figure out a better way of doing this too?
-  cmds=(
-    "cp -a /usr/include/rpcsvc/mount.h '${BOARD_ROOT}/usr/include/rpcsvc'"
-    "cp -a /usr/include/rpcsvc/rquota.h '${BOARD_ROOT}/usr/include/rpcsvc'"
-    "cp -a /usr/include/rpcsvc/nfs_prot.h '${BOARD_ROOT}/usr/include/rpcsvc'"
-    "cp -a /usr/include/rpcsvc/yppasswd.h '${BOARD_ROOT}/usr/include/rpcsvc'"
-  )
-  sudo_multi "${cmds[@]}"
-
-  if [[ ${FLAGS_configure} -eq ${FLAGS_TRUE} ]]; then
-    install_toolchain_in_provided "$gcc_ver" "$libc_ver"
-
-    # Configure new libc version in make.conf.board_setup.
-    sudo sed -i -e "/^LIBC_VERSION=/d" "$BOARD_SETUP"
-    echo "LIBC_VERSION=\"$libc_ver\"" | sudo_append "$BOARD_SETUP"
-  fi
-}
-
-set -u
-
-if [ -z "$FLAGS_board" ] ; then
-  die "--board required."
-fi
-
-get_board_and_variant $FLAGS_board ""
-
-BOARD_ROOT="${FLAGS_board_root:-/build/${BOARD_VARIANT}}"
-BOARD_ETC="${BOARD_ROOT}/etc"
-BOARD_SETUP="${BOARD_ETC}/make.conf.board_setup"
-BOARD_PROFILE="${BOARD_ETC}/portage/profile"
-
-all_toolchains=( $(get_all_board_toolchains "${BOARD}") )
-: ${FLAGS_toolchain:=${all_toolchains[0]}}
-
-if [ -z "${FLAGS_toolchain}" ]; then
-  die "No toolchain specified in board overlay or on command line."
-fi
-
-eval "$(portageq envvar -v CHOST PKGDIR)"
-
-if [[ ${FLAGS_force} -eq ${FLAGS_TRUE} ]] || \
-    board_needs_libc_update; then
-  info "Updating libc in the board."
-  install_toolchain_in_board
-else
-  info "Cross toolchain already up to date.  Nothing to do."
-fi

sdk_lib/make_chroot.sh

@@ -40,8 +40,10 @@ DEFINE_boolean getbinpkg $FLAGS_TRUE \
   "Download binary packages from remote repository."
 DEFINE_boolean delete $FLAGS_FALSE "Delete an existing chroot."
 DEFINE_boolean replace $FLAGS_FALSE "Overwrite existing chroot, if any."
-DEFINE_integer jobs -1 "How many packages to build in parallel at maximum."
+DEFINE_boolean fast "${DEFAULT_FAST}" \
-DEFINE_boolean fast ${DEFAULT_FAST} "Call many emerges in parallel"
+  "Use the parallel_emerge wrapper script."
+DEFINE_integer jobs "${NUM_JOBS}" \
+  "How many packages to build in parallel at maximum."
 DEFINE_string stage3_date "20130130" \
   "Use the stage3 with the given date."
 DEFINE_string stage3_path "" \

setup_board

@@ -21,8 +21,6 @@ DEFINE_boolean force $FLAGS_FALSE \
   "Force re-creating board root."
 DEFINE_boolean usepkg $FLAGS_TRUE \
   "Use binary packages to bootstrap."
-DEFINE_boolean getbinpkg $FLAGS_TRUE \
-  "Download binary packages from remote repository."
 
 FLAGS_HELP="usage: $(basename $0) [flags]
 
@@ -39,8 +37,9 @@ DEFINE_string accept_licenses "" \
   "Licenses to append to the accept list."
 DEFINE_string board_overlay "" \
   "Location of the board overlay."
-DEFINE_boolean fast ${DEFAULT_FAST} "Call many emerges in parallel"
+DEFINE_boolean fast "${DEFAULT_FAST}" \
-DEFINE_integer jobs -1 \
+  "Use the parallel_emerge wrapper script."
+DEFINE_integer jobs "${NUM_JOBS}" \
   "How many packages to build in parallel at maximum."
 DEFINE_boolean latest_toolchain $FLAGS_FALSE \
   "Use the latest ebuild for all toolchain packages (gcc, binutils, libc, \
@@ -57,6 +56,8 @@ DEFINE_string variant "" \
   "Board variant."
 DEFINE_boolean regen_configs ${FLAGS_FALSE} \
   "Regenerate all config files (useful for modifying profiles w/out rebuild)."
+DEFINE_boolean getbinpkg $FLAGS_TRUE \
+  "Passed to update_chroot, ignored by setup_board itself."
 
 
 # builds wrappers like equery-arm-generic.
@@ -125,10 +126,6 @@ EOF
   sudo_multi "${cmds[@]}"
 }
 
-install_toolchain() {
-  "${GCLIENT_ROOT}/src/scripts/install_toolchain" --board="${BOARD_VARIANT}"
-}
-
 # Parse command line flags
 FLAGS "$@" || exit 1
 eval set -- "${FLAGS_ARGV}"
@@ -183,6 +180,7 @@ BOARD_ETC="${BOARD_ROOT}/etc"
 BOARD_SETUP="${BOARD_ETC}/make.conf.board_setup"
 BOARD_PROFILE="${BOARD_ETC}/portage/profile"
 BOARD_ARCH=$(get_board_arch "$BOARD")
+BOARD_CHOST=$(get_board_chost ${BOARD})
 PORTAGE_PROFILE=$(get_board_profile "$BOARD")
 
 if [ -d "${BOARD_ROOT}" ]; then
@@ -198,8 +196,6 @@ if [ -d "${BOARD_ROOT}" ]; then
       warn "Not setting up board root. "
       warn "Use --force to clobber the board root and start again."
     fi
-    # Update the users libc in their board if needed.
-    install_toolchain
     exit 0
   fi
 else
@@ -231,11 +227,6 @@ for d in "${SCRIPTS_DIR}"/hooks/*; do
 done
 sudo_multi "${cmds[@]}"
 
-SAVED_VERSION=
-if [[ ${FLAGS_regen_configs} -eq ${FLAGS_TRUE} ]]; then
-  SAVED_VERSION=$(grep -s ^LIBC_VERSION= ${BOARD_SETUP} || true)
-fi
-
 ACCEPT_LICENSE=
 if [[ -n ${FLAGS_accept_licenses} ]]; then
   ACCEPT_LICENSE="ACCEPT_LICENSE='${FLAGS_accept_licenses}'"
@@ -243,12 +234,11 @@ fi
 
 sudo_clobber "${BOARD_SETUP}" <<EOF
 # Created by setup_board
-CHOST="$(get_board_chost ${BOARD})"
+CHOST="${BOARD_CHOST}"
 ROOT="${BOARD_ROOT}/"
 MAKEOPTS="--jobs=${NUM_JOBS} --load-average=${NUM_JOBS}"
 PKG_CONFIG="pkg-config-${BOARD_VARIANT}"
 BOARD_USE="${BOARD_VARIANT}"
-${SAVED_VERSION}
 ${ACCEPT_LICENSE}
 EOF
 
@@ -263,22 +253,23 @@ if [[ ${FLAGS_regen_configs} -eq ${FLAGS_FALSE} ]]; then
       --oneshot --quiet --nodeps sys-apps/baselayout
 
   # Now time for tool chain happy fun time
-  install_toolchain
+  EMERGE_FLAGS="--select --quiet --root-deps=rdeps"
-
+  if [[ "${FLAGS_jobs}" -ne -1 ]]; then
-  # Emerge the kernel headers into the board build root.  Use rdeps to
+    EMERGE_FLAGS+=" --jobs=${FLAGS_jobs}"
-  # avoid pulling any spurious DEPEND things in that we don't care about.
-  KERNEL_EMERGE_FLAGS="--select --quiet --root-deps=rdeps"
-  if [[ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]]; then
-    KERNEL_EMERGE_FLAGS+=" --usepkg"
-    if [[ "${FLAGS_getbinpkg}" -eq "${FLAGS_TRUE}" ]]; then
-      KERNEL_EMERGE_FLAGS+=" --getbinpkg "
-    fi
   fi
 
-  sudo -E "${EMERGE_WRAPPER}" ${KERNEL_EMERGE_FLAGS} \
+  if [[ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]]; then
-    sys-kernel/linux-headers
+    EMERGE_FLAGS+=" --usepkgonly --getbinpkg"
+  else
+    # When binary packages are disabled we need to make sure the cross
+    # sysroot includes any build dependencies for the toolchain.
+    info "Installing toolchain build dependencies"
+    install_cross_libs "${BOARD_CHOST}" ${EMERGE_FLAGS} --buildpkg=n
+  fi
 
-  unset KERNEL_EMERGE_FLAGS
+  info "Installing toolchain"
+  SYSROOT="/usr/${BOARD_CHOST}" "${EMERGE_WRAPPER}" \
+      ${EMERGE_FLAGS} "${TOOLCHAIN_PKGS[@]}"
 fi
 
 if [ $FLAGS_default -eq $FLAGS_TRUE ] ; then

update_chroot

@@ -28,8 +28,9 @@ show_help_if_requested "$@"
 # The following options are advanced options, only available to those willing
 # to read the source code. They are not shown in help output, since they are
 # not needed for the typical developer workflow.
-DEFINE_boolean fast ${DEFAULT_FAST} "Call many emerges in parallel"
+DEFINE_boolean fast ${DEFAULT_FAST} \
-DEFINE_integer jobs -1 \
+  "Use the parallel_emerge wrapper script."
+DEFINE_integer jobs "${NUM_JOBS}" \
   "How many packages to build in parallel at maximum."
 DEFINE_boolean skip_toolchain_update $FLAGS_FALSE \
   "Don't update the toolchains."
@@ -95,7 +96,7 @@ if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then
   fi
 
   # Only update toolchain when binpkgs are available.
-  EMERGE_FLAGS+=" ${TOOLCHAIN_BINONLY[*]}"
+  EMERGE_FLAGS+=" $(get_binonly_args $(get_chost_list))"
 fi
 
 if [[ "${FLAGS_jobs}" -ne -1 ]]; then