From 7d6619df8ab390720bf6b8a90a911789b75377ee Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Tue, 13 May 2014 14:09:30 -0700 Subject: [PATCH 1/4] fix(setup_board): Remove unused option to accept restrictive licenses --- setup_board | 8 -------- 1 file changed, 8 deletions(-) diff --git a/setup_board b/setup_board index 0a368294df..001c4f5ad3 100755 --- a/setup_board +++ b/setup_board @@ -37,8 +37,6 @@ DEFINE_string getbinpkgver "" \ "Use binary packages from a specific version." DEFINE_boolean toolchainpkgonly $FLAGS_FALSE \ "Use binary packages only for the board toolchain." -DEFINE_string accept_licenses "" \ - "Licenses to append to the accept list." DEFINE_boolean fast "${DEFAULT_FAST}" \ "Use the parallel_emerge wrapper script." DEFINE_integer jobs "${NUM_JOBS}" \ @@ -238,11 +236,6 @@ for d in "${SCRIPTS_DIR}"/hooks/*; do done sudo_multi "${cmds[@]}" -ACCEPT_LICENSE= -if [[ -n ${FLAGS_accept_licenses} ]]; then - ACCEPT_LICENSE="ACCEPT_LICENSE='${FLAGS_accept_licenses}'" -fi - sudo_clobber "${BOARD_SETUP}" < Date: Tue, 13 May 2014 15:40:43 -0700 Subject: [PATCH 2/4] fix(sdk_lib): Delete a pile of dead/pointless code. For the most part this doesn't influence anything. The one exception is the custom configuration for using curl is dropped, just rely on the portage defaults. It appears curl was only used to work around a wget issue with Google's internal SSL certificates. We care not. :) --- sdk_lib/make_chroot.sh | 74 +++++----------------------- sdk_lib/make_conf_util.sh | 101 +------------------------------------- 2 files changed, 12 insertions(+), 163 deletions(-) diff --git a/sdk_lib/make_chroot.sh b/sdk_lib/make_chroot.sh index fa56a00138..bed87463d6 100755 --- a/sdk_lib/make_chroot.sh +++ b/sdk_lib/make_chroot.sh @@ -44,8 +44,6 @@ DEFINE_boolean fast "${DEFAULT_FAST}" \ "Use the parallel_emerge wrapper script." DEFINE_integer jobs "${NUM_JOBS}" \ "How many packages to build in parallel at maximum." -DEFINE_string stage3_date "20130130" \ - "Use the stage3 with the given date." DEFINE_string stage3_path "" \ "Use the stage3 located on this path." DEFINE_string cache_dir "" "Directory to store caches within." @@ -71,30 +69,9 @@ switch_to_strict_mode [[ -z "${FLAGS_cache_dir}" ]] && \ die "--cache_dir is required" -. "${SCRIPT_ROOT}"/sdk_lib/make_conf_util.sh - -USEPKG="" -if [[ $FLAGS_usepkg -eq $FLAGS_TRUE ]]; then - # Use binary packages. Include all build-time dependencies, - # so as to avoid unnecessary differences between source - # and binary builds. - USEPKG="--usepkg --with-bdeps y" - if [[ $FLAGS_getbinpkg -eq $FLAGS_TRUE ]]; then - USEPKG="$USEPKG --getbinpkg" - fi -fi - -# Support faster build if necessary. -EMERGE_CMD="emerge" -if [ "$FLAGS_fast" -eq "${FLAGS_TRUE}" ]; then - CHROOT_CHROMITE_DIR="${CHROOT_TRUNK_DIR}/chromite" - EMERGE_CMD="${CHROOT_CHROMITE_DIR}/bin/parallel_emerge" -fi - ENTER_CHROOT_ARGS=( CROS_WORKON_SRCROOT="$CHROOT_TRUNK" PORTAGE_USERNAME="${SUDO_USER}" - IGNORE_PREFLIGHT_BINHOST="$IGNORE_PREFLIGHT_BINHOST" ) # Invoke enter_chroot. This can only be used after sudo has been installed. @@ -373,22 +350,11 @@ for type in http ftp all; do fi done -# Create the base Gentoo stage3 based on last version put in chroot. -STAGE3="${OVERLAY}/coreos/stage3/stage3-amd64-${FLAGS_stage3_date}.tar.bz2" -if [ -f $CHROOT_STATE ] && \ - ! egrep -q "^STAGE3=$STAGE3" $CHROOT_STATE >/dev/null 2>&1 -then - info "STAGE3 version has changed." - delete_existing -fi - -if [ -n "${FLAGS_stage3_path}" ]; then - if [ ! -f "${FLAGS_stage3_path}" ]; then - error "Invalid stage3!" - exit 1; - fi - STAGE3="${FLAGS_stage3_path}" +if [ ! -f "${FLAGS_stage3_path}" ]; then + error "Invalid stage3!" + exit 1; fi +STAGE3="${FLAGS_stage3_path}" # Create the destination directory. mkdir -p "$FLAGS_chroot" @@ -407,39 +373,21 @@ else ${DECOMPRESS} -dc "${STAGE3}" | \ tar -xp -C "${FLAGS_chroot}" rm -f "$FLAGS_chroot/etc/"make.{globals,conf.user} -fi -# Set up users, if needed, before mkdir/mounts below. -[ -f $CHROOT_STATE ] || init_users + # Set up users, if needed, before mkdir/mounts below. + init_users -# Reset internal vars to force them to the 'inside the chroot' value; -# since user directories now exist, this can do the upgrade in place. -set_chroot_trunk_dir "${FLAGS_chroot}" poppycock + # Reset internal vars to force them to the 'inside the chroot' value; + # since user directories now exist, this can do the upgrade in place. + set_chroot_trunk_dir "${FLAGS_chroot}" poppycock + mkdir -p "${FLAGS_chroot}/${CHROOT_TRUNK_DIR}" \ + "${FLAGS_chroot}/${DEPOT_TOOLS_DIR}" "${FLAGS_chroot}/run" -echo -info "Setting up mounts..." -# Set up necessary mounts and make sure we clean them up on exit. -mkdir -p "${FLAGS_chroot}/${CHROOT_TRUNK_DIR}" \ - "${FLAGS_chroot}/${DEPOT_TOOLS_DIR}" "${FLAGS_chroot}/run" - -# Create a special /etc/make.conf.host_setup that we use to bootstrap -# the chroot. The regular content for the file will be generated the -# first time we invoke update_chroot (further down in this script). -create_bootstrap_host_setup "${FLAGS_chroot}" - -if ! [ -f "$CHROOT_STATE" ];then - INITIALIZE_CHROOT=1 -fi - -if [ -z "${INITIALIZE_CHROOT}" ];then - info "chroot already initialized. Skipping..." -else # Run all the init stuff to setup the env. init_setup fi # Add file to indicate that it is a chroot. -# Add version of $STAGE3 for update checks. echo STAGE3=$STAGE3 > $CHROOT_STATE # Update chroot. diff --git a/sdk_lib/make_conf_util.sh b/sdk_lib/make_conf_util.sh index 914b67eeb0..fe614b1c25 100644 --- a/sdk_lib/make_conf_util.sh +++ b/sdk_lib/make_conf_util.sh @@ -2,109 +2,10 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -# When bootstrapping the chroot, only wget is available, and we must -# disable certificate checking. Once the chroot is fully -# initialized, we can switch to curl, and re-enable the certificate -# checks. See http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=409938 -# -# Usage: -# $1 - 'wget' requests the bootstrap special content; otherwise -# uses 'curl'. -_make_conf_fetchcommand() { - local cmd options output_opt resume_opt - local fileref='\"\${DISTDIR}/\${FILE}\"' - local uri_ref='\"\${URI}\"' - - if [ "$1" = "wget" ] ; then - cmd=/usr/bin/wget - options="-t 5 -T 60 --no-check-certificate --passive-ftp" - resume_opt="-c" - output_opt="-O" - else - cmd=curl - options="-f -y 30 --retry 9 -L" - resume_opt="-C -" - output_opt="--output" - fi - - local args="$options $output_opt $fileref $uri_ref" - echo FETCHCOMMAND=\"$cmd $args\" - echo RESUMECOMMAND=\"$cmd $resume_opt $args\" - echo -} - -# The default PORTAGE_BINHOST setting selects the preflight -# binhosts. We override the setting if the build environment -# requests it. -_make_conf_prebuilt() { - if [[ -n "$IGNORE_PREFLIGHT_BINHOST" ]]; then - echo 'PORTAGE_BINHOST="$FULL_BINHOST"' - echo - fi -} - -# Include configuration settings for building private overlay -# packages, if the overlay is present. -# -# N.B. We explicitly disallow creating content for the private -# overlay during bootstrapping, as it's not currently required, -# and at least a minor nuisance to implement. Note also that the -# use of an inside-the-chroot path is based on the (currently true) -# assumption that bootstrapping use is outside the chroot, and -# non-bootstrapping use is inside the chroot. -_make_conf_private() { - if [ "$1" = "wget" ] ; then - return - fi - local chromeos_overlay="src/private-overlays/coreos-overlay" - chromeos_overlay="$CHROOT_TRUNK_DIR/$chromeos_overlay" - if [ -d "$chromeos_overlay" ]; then - local boto_config="$chromeos_overlay/googlestorage_account.boto" - local gsutil_cmd='gsutil cp \"${URI}\" \"${DISTDIR}/${FILE}\"' - cat < Date: Tue, 13 May 2014 15:51:38 -0700 Subject: [PATCH 3/4] refactor(update_chroot): Write make.conf.host_setup There is no need for this to be in sdk_lib any more, remove what little remains of make_conf_util.sh. --- sdk_lib/make_conf_util.sh | 11 ----------- update_chroot | 22 +++++++--------------- 2 files changed, 7 insertions(+), 26 deletions(-) delete mode 100644 sdk_lib/make_conf_util.sh diff --git a/sdk_lib/make_conf_util.sh b/sdk_lib/make_conf_util.sh deleted file mode 100644 index fe614b1c25..0000000000 --- a/sdk_lib/make_conf_util.sh +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -create_host_setup() { - local host_setup="$2/etc/portage/make.conf.host_setup" - ( echo "# Automatically generated. EDIT THIS AND BE SORRY." - echo "MAKEOPTS='--jobs=${NUM_JOBS} --load-average=${NUM_JOBS}'" - ) | sudo_clobber "$host_setup" - sudo chmod 644 "$host_setup" -} diff --git a/update_chroot b/update_chroot index 764048f068..a42096aa17 100755 --- a/update_chroot +++ b/update_chroot @@ -46,27 +46,19 @@ eval set -- "${FLAGS_ARGV}" switch_to_strict_mode . "${BUILD_LIBRARY_DIR}/toolchain_util.sh" -. "${SCRIPTS_DIR}/sdk_lib/make_conf_util.sh" - -# Create /etc/make.conf.host_setup. The file content is regenerated -# from scratch every update. There are various reasons to do this: -# + It's cheap, so this is an easy way to guarantee correct content -# after an upgrade. -# + Inside make_chroot.sh, we use a temporary version of the file -# which must be updated before the script completes; that final -# update happens here. -# + If the repositories change to add or remove the private -# overlay, the file may need to be regenerated. -create_host_setup PORTAGE_STABLE_OVERLAY="/usr/local/portage/stable" CROSSDEV_OVERLAY="/usr/local/portage/crossdev" CHROOT_OVERLAY="/usr/local/portage/coreos" -# Create repos.conf configs. -info "Setting up repos.conf..." +info "Setting up portage..." sudo mkdir -p "/etc/portage/repos.conf/" -sudo tee "/etc/portage/repos.conf/coreos.conf" > /dev/null < Date: Tue, 13 May 2014 17:12:11 -0700 Subject: [PATCH 4/4] fix(update_chroot): Consolidate more portage configuration To behave more like setup_board/build_packages update_chroot should fully configure portage to make sure everything is accurate. Now binhosts are defined in make.conf.host_setup so the static config in coreos-overlays doesn't need to refer to version.txt. setup_board already made this change in 7a43a07f. Define path locations to reduce dependency between static configs in coreos-overlays and the behavior of the scripts repo. Spreading configuration across two repos makes everything harder to understand. Eventually everything should either be defined in profiles in coreos-overlays or minimal auto-generated config files here in scripts. --- build_library/toolchain_util.sh | 22 ++++++++++++++++++++++ sdk_lib/make_chroot.sh | 33 +-------------------------------- update_chroot | 20 ++++++++++++++++++-- 3 files changed, 41 insertions(+), 34 deletions(-) diff --git a/build_library/toolchain_util.sh b/build_library/toolchain_util.sh index 81db20157f..49eb872400 100644 --- a/build_library/toolchain_util.sh +++ b/build_library/toolchain_util.sh @@ -127,6 +127,28 @@ get_board_binhost() { done } +get_sdk_arch() { + get_portage_arch $(uname -m) +} + +get_sdk_profile() { + echo "coreos:coreos/$(get_sdk_arch)/sdk" +} + +# Usage: get_sdk_binhost [version...] +# If no versions are specified the current and SDK versions are used. +get_sdk_binhost() { + local arch=$(get_sdk_arch) ver + if [[ $# -eq 0 ]]; then + set -- "${COREOS_SDK_VERSION}" "${COREOS_VERSION_ID}" + fi + + for ver in "$@"; do + echo "${COREOS_DOWNLOAD_ROOT}/sdk/${arch}/${ver}/pkgs/" + echo "${COREOS_DOWNLOAD_ROOT}/sdk/${arch}/${ver}/toolchain/" + done +} + # Usage: get_cross_pkgs chost [chost2...] get_cross_pkgs() { local cross_chost native_pkg diff --git a/sdk_lib/make_chroot.sh b/sdk_lib/make_chroot.sh index bed87463d6..f518877d0a 100755 --- a/sdk_lib/make_chroot.sh +++ b/sdk_lib/make_chroot.sh @@ -149,8 +149,6 @@ init_setup () { mkdir -p -m 755 "${FLAGS_chroot}/usr" \ "${FLAGS_chroot}/usr/local/portage" \ "${FLAGS_chroot}"/"${CROSSDEV_OVERLAY}" - ln -sf "${CHROOT_TRUNK_DIR}/src/third_party/portage" \ - "${FLAGS_chroot}/usr/portage" ln -sf "${CHROOT_TRUNK_DIR}/src/third_party/coreos-overlay" \ "${FLAGS_chroot}"/"${CHROOT_OVERLAY}" ln -sf "${CHROOT_TRUNK_DIR}/src/third_party/portage-stable" \ @@ -197,19 +195,6 @@ EOF cp /etc/{hosts,resolv.conf} "$FLAGS_chroot/etc/" chmod 0644 "$FLAGS_chroot"/etc/{hosts,resolv.conf} - # Setup host make.conf. This includes any overlay that we may be using - # and a pointer to pre-built packages. - # TODO: This should really be part of a profile in the portage. - info "Setting up /etc/make.*..." - ln -sf "${CHROOT_CONFIG}/make.conf.amd64-host" \ - "${FLAGS_chroot}/etc/portage/make.conf" - ln -sf "${CHROOT_OVERLAY}/profiles/default/linux/amd64/10.0" \ - "${FLAGS_chroot}/etc/portage/make.profile" - - # Create make.conf.user . - touch "${FLAGS_chroot}"/etc/portage/make.conf.user - chmod 0644 "${FLAGS_chroot}"/etc/portage/make.conf.user - # Create directories referred to by our conf files. mkdir -p -m 775 "${FLAGS_chroot}/var/lib/portage/pkgs" \ "${FLAGS_chroot}/var/cache/"chromeos-{cache,chrome} \ @@ -242,6 +227,7 @@ PATH=${CHROOT_TRUNK_DIR}/chromite/bin:${DEPOT_TOOLS_DIR} CROS_WORKON_SRCROOT="${CHROOT_TRUNK_DIR}" PORTAGE_USERNAME=${SUDO_USER} EOF + early_enter_chroot env-update # Add chromite into python path. for python_path in "${FLAGS_chroot}/usr/lib/"python2.*; do @@ -249,19 +235,6 @@ EOF sudo ln -s "${CHROOT_TRUNK_DIR}"/chromite "${python_path}" done - # TODO(zbehan): Configure stuff that is usually configured in postinst's, - # but wasn't. Fix the postinst's. - info "Running post-inst configuration hacks" - early_enter_chroot env-update - - - # This is basically a sanity check of our chroot. If any of these - # don't exist, then either bind mounts have failed, an invocation - # from above is broke, or some assumption about the stage3 is no longer - # true. - early_enter_chroot ls -l /etc/portage/make.{conf,profile} \ - /usr/local/portage/coreos/profiles/default/linux/amd64/10.0 - target="${FLAGS_chroot}/etc/profile.d" mkdir -p "${target}" cat << EOF > "${target}/coreos-niceties.sh" @@ -333,10 +306,6 @@ if [[ $FLAGS_delete -eq $FLAGS_TRUE || \ fi CHROOT_TRUNK="${CHROOT_TRUNK_DIR}" -PORTAGE="${SRC_ROOT}/third_party/portage" -OVERLAY="${SRC_ROOT}/third_party/coreos-overlay" -CONFIG_DIR="${OVERLAY}/coreos/config" -CHROOT_CONFIG="${CHROOT_TRUNK_DIR}/src/third_party/coreos-overlay/coreos/config" PORTAGE_STABLE_OVERLAY="/usr/local/portage/stable" CROSSDEV_OVERLAY="/usr/local/portage/crossdev" CHROOT_OVERLAY="/usr/local/portage/coreos" diff --git a/update_chroot b/update_chroot index a42096aa17..2b57b36a4c 100755 --- a/update_chroot +++ b/update_chroot @@ -49,13 +49,27 @@ switch_to_strict_mode PORTAGE_STABLE_OVERLAY="/usr/local/portage/stable" CROSSDEV_OVERLAY="/usr/local/portage/crossdev" -CHROOT_OVERLAY="/usr/local/portage/coreos" +COREOS_OVERLAY="/usr/local/portage/coreos" +COREOS_CONFIG="${COREOS_OVERLAY}/coreos/config" info "Setting up portage..." sudo mkdir -p "/etc/portage/repos.conf/" +sudo ln -sf "${COREOS_CONFIG}/make.conf.amd64-host" "/etc/portage/make.conf" +sudo touch "/etc/portage/make.conf.user" + sudo_clobber "/etc/portage/make.conf.host_setup" <