Merge pull request #251 from marineam/refactor

Refactor sdk setup

build_library/toolchain_util.sh

@@ -127,6 +127,28 @@ get_board_binhost() {
     done
 }
 
+get_sdk_arch() {
+    get_portage_arch $(uname -m)
+}
+
+get_sdk_profile() {
+    echo "coreos:coreos/$(get_sdk_arch)/sdk"
+}
+
+# Usage: get_sdk_binhost [version...]
+# If no versions are specified the current and SDK versions are used.
+get_sdk_binhost() {
+    local arch=$(get_sdk_arch) ver
+    if [[ $# -eq 0 ]]; then
+        set -- "${COREOS_SDK_VERSION}" "${COREOS_VERSION_ID}"
+    fi
+
+    for ver in "$@"; do
+        echo "${COREOS_DOWNLOAD_ROOT}/sdk/${arch}/${ver}/pkgs/"
+        echo "${COREOS_DOWNLOAD_ROOT}/sdk/${arch}/${ver}/toolchain/"
+    done
+}
+
 # Usage: get_cross_pkgs chost [chost2...]
 get_cross_pkgs() {
     local cross_chost native_pkg

sdk_lib/make_chroot.sh

@@ -44,8 +44,6 @@ DEFINE_boolean fast "${DEFAULT_FAST}" \
   "Use the parallel_emerge wrapper script."
 DEFINE_integer jobs "${NUM_JOBS}" \
   "How many packages to build in parallel at maximum."
-DEFINE_string stage3_date "20130130" \
-  "Use the stage3 with the given date."
 DEFINE_string stage3_path "" \
   "Use the stage3 located on this path."
 DEFINE_string cache_dir "" "Directory to store caches within."
@@ -71,30 +69,9 @@ switch_to_strict_mode
   [[ -z "${FLAGS_cache_dir}" ]] && \
   die "--cache_dir is required"
 
-. "${SCRIPT_ROOT}"/sdk_lib/make_conf_util.sh
-
-USEPKG=""
-if [[ $FLAGS_usepkg -eq $FLAGS_TRUE ]]; then
-  # Use binary packages. Include all build-time dependencies,
-  # so as to avoid unnecessary differences between source
-  # and binary builds.
-  USEPKG="--usepkg --with-bdeps y"
-  if [[ $FLAGS_getbinpkg -eq $FLAGS_TRUE ]]; then
-    USEPKG="$USEPKG --getbinpkg"
-  fi
-fi
-
-# Support faster build if necessary.
-EMERGE_CMD="emerge"
-if [ "$FLAGS_fast" -eq "${FLAGS_TRUE}" ]; then
-  CHROOT_CHROMITE_DIR="${CHROOT_TRUNK_DIR}/chromite"
-  EMERGE_CMD="${CHROOT_CHROMITE_DIR}/bin/parallel_emerge"
-fi
-
 ENTER_CHROOT_ARGS=(
   CROS_WORKON_SRCROOT="$CHROOT_TRUNK"
   PORTAGE_USERNAME="${SUDO_USER}"
-  IGNORE_PREFLIGHT_BINHOST="$IGNORE_PREFLIGHT_BINHOST"
 )
 
 # Invoke enter_chroot.  This can only be used after sudo has been installed.
@@ -172,8 +149,6 @@ init_setup () {
    mkdir -p -m 755 "${FLAGS_chroot}/usr" \
      "${FLAGS_chroot}/usr/local/portage" \
      "${FLAGS_chroot}"/"${CROSSDEV_OVERLAY}"
-   ln -sf "${CHROOT_TRUNK_DIR}/src/third_party/portage" \
-     "${FLAGS_chroot}/usr/portage"
    ln -sf "${CHROOT_TRUNK_DIR}/src/third_party/coreos-overlay" \
      "${FLAGS_chroot}"/"${CHROOT_OVERLAY}"
    ln -sf "${CHROOT_TRUNK_DIR}/src/third_party/portage-stable" \
@@ -220,19 +195,6 @@ EOF
    cp /etc/{hosts,resolv.conf} "$FLAGS_chroot/etc/"
    chmod 0644 "$FLAGS_chroot"/etc/{hosts,resolv.conf}
 
-   # Setup host make.conf. This includes any overlay that we may be using
-   # and a pointer to pre-built packages.
-   # TODO: This should really be part of a profile in the portage.
-   info "Setting up /etc/make.*..."
-   ln -sf "${CHROOT_CONFIG}/make.conf.amd64-host" \
-     "${FLAGS_chroot}/etc/portage/make.conf"
-   ln -sf "${CHROOT_OVERLAY}/profiles/default/linux/amd64/10.0" \
-     "${FLAGS_chroot}/etc/portage/make.profile"
-
-   # Create make.conf.user .
-   touch "${FLAGS_chroot}"/etc/portage/make.conf.user
-   chmod 0644 "${FLAGS_chroot}"/etc/portage/make.conf.user
-
    # Create directories referred to by our conf files.
    mkdir -p -m 775 "${FLAGS_chroot}/var/lib/portage/pkgs" \
      "${FLAGS_chroot}/var/cache/"chromeos-{cache,chrome} \
@@ -265,6 +227,7 @@ PATH=${CHROOT_TRUNK_DIR}/chromite/bin:${DEPOT_TOOLS_DIR}
 CROS_WORKON_SRCROOT="${CHROOT_TRUNK_DIR}"
 PORTAGE_USERNAME=${SUDO_USER}
 EOF
+   early_enter_chroot env-update
 
    # Add chromite into python path.
    for python_path in "${FLAGS_chroot}/usr/lib/"python2.*; do
@@ -272,19 +235,6 @@ EOF
      sudo ln -s "${CHROOT_TRUNK_DIR}"/chromite "${python_path}"
    done
 
-   # TODO(zbehan): Configure stuff that is usually configured in postinst's,
-   # but wasn't. Fix the postinst's.
-   info "Running post-inst configuration hacks"
-   early_enter_chroot env-update
-
-
-   # This is basically a sanity check of our chroot.  If any of these
-   # don't exist, then either bind mounts have failed, an invocation
-   # from above is broke, or some assumption about the stage3 is no longer
-   # true.
-   early_enter_chroot ls -l /etc/portage/make.{conf,profile} \
-     /usr/local/portage/coreos/profiles/default/linux/amd64/10.0
-
    target="${FLAGS_chroot}/etc/profile.d"
    mkdir -p "${target}"
    cat << EOF > "${target}/coreos-niceties.sh"
@@ -356,10 +306,6 @@ if [[ $FLAGS_delete  -eq $FLAGS_TRUE || \
 fi
 
 CHROOT_TRUNK="${CHROOT_TRUNK_DIR}"
-PORTAGE="${SRC_ROOT}/third_party/portage"
-OVERLAY="${SRC_ROOT}/third_party/coreos-overlay"
-CONFIG_DIR="${OVERLAY}/coreos/config"
-CHROOT_CONFIG="${CHROOT_TRUNK_DIR}/src/third_party/coreos-overlay/coreos/config"
 PORTAGE_STABLE_OVERLAY="/usr/local/portage/stable"
 CROSSDEV_OVERLAY="/usr/local/portage/crossdev"
 CHROOT_OVERLAY="/usr/local/portage/coreos"
@@ -373,22 +319,11 @@ for type in http ftp all; do
    fi
 done
 
-# Create the base Gentoo stage3 based on last version put in chroot.
-STAGE3="${OVERLAY}/coreos/stage3/stage3-amd64-${FLAGS_stage3_date}.tar.bz2"
-if [ -f $CHROOT_STATE ] && \
-  ! egrep -q "^STAGE3=$STAGE3" $CHROOT_STATE >/dev/null 2>&1
-then
-  info "STAGE3 version has changed."
-  delete_existing
-fi
-
-if [ -n "${FLAGS_stage3_path}" ]; then
 if [ ! -f "${FLAGS_stage3_path}" ]; then
   error "Invalid stage3!"
   exit 1;
 fi
 STAGE3="${FLAGS_stage3_path}"
-fi
 
 # Create the destination directory.
 mkdir -p "$FLAGS_chroot"
@@ -407,39 +342,21 @@ else
   ${DECOMPRESS} -dc "${STAGE3}" | \
     tar -xp -C "${FLAGS_chroot}"
   rm -f "$FLAGS_chroot/etc/"make.{globals,conf.user}
-fi
 
   # Set up users, if needed, before mkdir/mounts below.
-[ -f $CHROOT_STATE ] || init_users
+  init_users
 
   # Reset internal vars to force them to the 'inside the chroot' value;
   # since user directories now exist, this can do the upgrade in place.
   set_chroot_trunk_dir "${FLAGS_chroot}" poppycock
-
-echo
-info "Setting up mounts..."
-# Set up necessary mounts and make sure we clean them up on exit.
   mkdir -p "${FLAGS_chroot}/${CHROOT_TRUNK_DIR}" \
       "${FLAGS_chroot}/${DEPOT_TOOLS_DIR}" "${FLAGS_chroot}/run"
 
-# Create a special /etc/make.conf.host_setup that we use to bootstrap
-# the chroot.  The regular content for the file will be generated the
-# first time we invoke update_chroot (further down in this script).
-create_bootstrap_host_setup "${FLAGS_chroot}"
-
-if ! [ -f "$CHROOT_STATE" ];then
-  INITIALIZE_CHROOT=1
-fi
-
-if [ -z "${INITIALIZE_CHROOT}" ];then
-  info "chroot already initialized.  Skipping..."
-else
   # Run all the init stuff to setup the env.
   init_setup
 fi
 
 # Add file to indicate that it is a chroot.
-# Add version of $STAGE3 for update checks.
 echo STAGE3=$STAGE3 > $CHROOT_STATE
 
 # Update chroot.

sdk_lib/make_conf_util.sh

@@ -1,110 +0,0 @@
-# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# When bootstrapping the chroot, only wget is available, and we must
-# disable certificate checking.  Once the chroot is fully
-# initialized, we can switch to curl, and re-enable the certificate
-# checks.  See http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=409938
-#
-# Usage:
-# $1 - 'wget' requests the bootstrap special content; otherwise
-#      uses 'curl'.
-_make_conf_fetchcommand() {
-  local cmd options output_opt resume_opt
-  local fileref='\"\${DISTDIR}/\${FILE}\"'
-  local uri_ref='\"\${URI}\"'
-
-  if [ "$1" = "wget" ] ; then
-    cmd=/usr/bin/wget
-    options="-t 5 -T 60 --no-check-certificate --passive-ftp"
-    resume_opt="-c"
-    output_opt="-O"
-  else
-    cmd=curl
-    options="-f -y 30 --retry 9 -L"
-    resume_opt="-C -"
-    output_opt="--output"
-  fi
-
-  local args="$options $output_opt $fileref $uri_ref"
-  echo FETCHCOMMAND=\"$cmd $args\"
-  echo RESUMECOMMAND=\"$cmd $resume_opt $args\"
-  echo
-}
-
-# The default PORTAGE_BINHOST setting selects the preflight
-# binhosts.  We override the setting if the build environment
-# requests it.
-_make_conf_prebuilt() {
-  if [[ -n "$IGNORE_PREFLIGHT_BINHOST" ]]; then
-    echo 'PORTAGE_BINHOST="$FULL_BINHOST"'
-    echo
-  fi
-}
-
-# Include configuration settings for building private overlay
-# packages, if the overlay is present.
-#
-# N.B.  We explicitly disallow creating content for the private
-# overlay during bootstrapping, as it's not currently required,
-# and at least a minor nuisance to implement.  Note also that the
-# use of an inside-the-chroot path is based on the (currently true)
-# assumption that bootstrapping use is outside the chroot, and
-# non-bootstrapping use is inside the chroot.
-_make_conf_private() {
-  if [ "$1" = "wget" ] ; then
-    return
-  fi
-  local chromeos_overlay="src/private-overlays/coreos-overlay"
-  chromeos_overlay="$CHROOT_TRUNK_DIR/$chromeos_overlay"
-  if [ -d "$chromeos_overlay" ]; then
-    local boto_config="$chromeos_overlay/googlestorage_account.boto"
-    local gsutil_cmd='gsutil cp \"${URI}\" \"${DISTDIR}/${FILE}\"'
-    cat <<EOF
-source $chromeos_overlay/make.conf
-
-FETCHCOMMAND_GS="bash -c 'BOTO_CONFIG=$boto_config $gsutil_cmd'"
-RESUMECOMMAND_GS="$FETCHCOMMAND_GS"
-
-PORTDIR_OVERLAY="\$PORTDIR_OVERLAY $chromeos_overlay"
-
-EOF
-  fi
-}
-
-# Create /etc/make.conf.host_setup according to parameters.
-#
-# Usage:
-# $1 - 'wget' for bootstrapping; 'curl' otherwise.
-# $2 - When outside the chroot, path to the chroot.  Empty when
-#      inside the chroot.
-_create_host_setup() {
-  local fetchtype="$1"
-  local host_setup="$2/etc/portage/make.conf.host_setup"
-  ( echo "# Automatically generated.  EDIT THIS AND BE SORRY."
-    echo
-    _make_conf_fetchcommand "$fetchtype"
-    _make_conf_private "$fetchtype"
-    _make_conf_prebuilt
-    echo "MAKEOPTS='--jobs=${NUM_JOBS} --load-average=${NUM_JOBS}'"
-  ) | sudo_clobber "$host_setup"
-  sudo chmod 644 "$host_setup"
-}
-
-
-# Create /etc/make.conf.host_setup for early bootstrapping of the
-# chroot.  This is done early in make_chroot, and the results are
-# overwritten later in the process.
-#
-# Usage:
-#   $1 - Path to chroot as seen from outside
-create_bootstrap_host_setup() {
-  _create_host_setup wget "$@"
-}
-
-
-# Create /etc/make.conf.host_setup for normal usage.
-create_host_setup() {
-  _create_host_setup curl ''
-}

setup_board

@@ -37,8 +37,6 @@ DEFINE_string getbinpkgver "" \
   "Use binary packages from a specific version."
 DEFINE_boolean toolchainpkgonly $FLAGS_FALSE \
   "Use binary packages only for the board toolchain."
-DEFINE_string accept_licenses "" \
-  "Licenses to append to the accept list."
 DEFINE_boolean fast "${DEFAULT_FAST}" \
   "Use the parallel_emerge wrapper script."
 DEFINE_integer jobs "${NUM_JOBS}" \
@@ -238,11 +236,6 @@ for d in "${SCRIPTS_DIR}"/hooks/*; do
 done
 sudo_multi "${cmds[@]}"
 
-ACCEPT_LICENSE=
-if [[ -n ${FLAGS_accept_licenses} ]]; then
-  ACCEPT_LICENSE="ACCEPT_LICENSE='${FLAGS_accept_licenses}'"
-fi
-
 sudo_clobber "${BOARD_SETUP}" <<EOF
 # Created by setup_board
 CHOST="${BOARD_CHOST}"
@@ -251,7 +244,6 @@ MAKEOPTS="--jobs=${NUM_JOBS} --load-average=$((NUM_JOBS * 2))"
 PKG_CONFIG="pkg-config-${BOARD_VARIANT}"
 BOARD_USE="${BOARD_VARIANT}"
 PORTAGE_BINHOST="${BOARD_BINHOST}"
-${ACCEPT_LICENSE}
 EOF
 
 generate_all_wrappers

update_chroot

@@ -46,27 +46,33 @@ eval set -- "${FLAGS_ARGV}"
 switch_to_strict_mode
 
 . "${BUILD_LIBRARY_DIR}/toolchain_util.sh"
-. "${SCRIPTS_DIR}/sdk_lib/make_conf_util.sh"
-
-# Create /etc/make.conf.host_setup.  The file content is regenerated
-# from scratch every update.  There are various reasons to do this:
-#  + It's cheap, so this is an easy way to guarantee correct content
-#    after an upgrade.
-#  + Inside make_chroot.sh, we use a temporary version of the file
-#    which must be updated before the script completes; that final
-#    update happens here.
-#  + If the repositories change to add or remove the private
-#    overlay, the file may need to be regenerated.
-create_host_setup
 
 PORTAGE_STABLE_OVERLAY="/usr/local/portage/stable"
 CROSSDEV_OVERLAY="/usr/local/portage/crossdev"
-CHROOT_OVERLAY="/usr/local/portage/coreos"
+COREOS_OVERLAY="/usr/local/portage/coreos"
+COREOS_CONFIG="${COREOS_OVERLAY}/coreos/config"
 
-# Create repos.conf configs.
+info "Setting up portage..."
-info "Setting up repos.conf..."
 sudo mkdir -p "/etc/portage/repos.conf/"
-sudo tee "/etc/portage/repos.conf/coreos.conf" > /dev/null <<EOF
+sudo ln -sf "${COREOS_CONFIG}/make.conf.amd64-host" "/etc/portage/make.conf"
+sudo touch "/etc/portage/make.conf.user"
+
+sudo_clobber "/etc/portage/make.conf.host_setup" <<EOF
+# Created by update_chroot
+PORTDIR="${PORTAGE_STABLE_OVERLAY}"
+PORTDIR_OVERLAY="${CROSSDEV_OVERLAY} ${COREOS_OVERLAY}"
+DISTDIR="/var/lib/portage/distfiles"
+PKGDIR="/var/lib/portage/pkgs"
+PORT_LOGDIR="/var/log/portage"
+PORTAGE_BINHOST="$(get_sdk_binhost)"
+MAKEOPTS="--jobs=${NUM_JOBS} --load-average=$((NUM_JOBS * 2))"
+
+# Generally there isn't any need to add packages to @world by default.
+# You can use --select to override this.
+EMERGE_DEFAULT_OPTS="--oneshot"
+EOF
+
+sudo_clobber "/etc/portage/repos.conf/coreos.conf" <<EOF
 [DEFAULT]
 main-repo = portage-stable
 
@@ -74,12 +80,14 @@ main-repo = portage-stable
 disabled = true
 
 [coreos]
-location = ${CHROOT_OVERLAY}
+location = ${COREOS_OVERLAY}
 
 [portage-stable]
 location = ${PORTAGE_STABLE_OVERLAY}
 EOF
 
+sudo eselect profile set --force "$(get_sdk_profile)"
+
 # Create crossdev repo_name and metadata
 info "Setting up crossdev..."
 sudo mkdir -p "${FLAGS_chroot}/${CROSSDEV_OVERLAY}/profiles"