From 174a847e362f8a1d05210750eafb6ccb4735f59c Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sun, 5 Jul 2015 15:29:21 -0700 Subject: [PATCH 1/5] update_chroot: remove dependence on /usr/local/portage/* symlinks A step in reducing the amount of initialization code required: drop needless symlinks under /usr/local/portage to the portage trees. Just configure portage to point directly at the source instead. Only crossdev remains in that location because it is a locally managed overlay. --- sdk_lib/make_chroot.sh | 8 -------- setup_board | 3 ++- update_chroot | 4 ++-- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/sdk_lib/make_chroot.sh b/sdk_lib/make_chroot.sh index 5057aedfd2..9773349425 100755 --- a/sdk_lib/make_chroot.sh +++ b/sdk_lib/make_chroot.sh @@ -146,14 +146,6 @@ init_users () { init_setup () { info "Running init_setup()..." - mkdir -p -m 755 "${FLAGS_chroot}/usr" \ - "${FLAGS_chroot}/usr/local/portage" \ - "${FLAGS_chroot}"/"${CROSSDEV_OVERLAY}" - ln -sf "${CHROOT_TRUNK_DIR}/src/third_party/coreos-overlay" \ - "${FLAGS_chroot}"/"${CHROOT_OVERLAY}" - ln -sf "${CHROOT_TRUNK_DIR}/src/third_party/portage-stable" \ - "${FLAGS_chroot}"/"${PORTAGE_STABLE_OVERLAY}" - # Set up sudoers. Inside the chroot, the user can sudo without a password. # (Safe enough, since the only way into the chroot is to 'sudo chroot', so # the user's already typed in one sudo password...) diff --git a/setup_board b/setup_board index 4f65ddfc38..5f948fbe84 100755 --- a/setup_board +++ b/setup_board @@ -168,7 +168,8 @@ case "$BOARD" in ;; esac # Locations we will need -COREOS_CONFIG="/usr/local/portage/coreos/coreos/config" +COREOS_OVERLAY="${REPO_ROOT}/src/third_party/coreos-overlay" +COREOS_CONFIG="${COREOS_OVERLAY}/coreos/config" BOARD_ROOT="/build/${BOARD_VARIANT}" BOARD_ETC="${BOARD_ROOT}/etc" BOARD_ARCH=$(get_board_arch "$BOARD") diff --git a/update_chroot b/update_chroot index 2d5c474db9..89abd6eb94 100755 --- a/update_chroot +++ b/update_chroot @@ -45,9 +45,9 @@ switch_to_strict_mode . "${BUILD_LIBRARY_DIR}/toolchain_util.sh" -PORTAGE_STABLE_OVERLAY="/usr/local/portage/stable" +PORTAGE_STABLE_OVERLAY="${REPO_ROOT}/src/third_party/portage-stable" CROSSDEV_OVERLAY="/usr/local/portage/crossdev" -COREOS_OVERLAY="/usr/local/portage/coreos" +COREOS_OVERLAY="${REPO_ROOT}/src/third_party/coreos-overlay" COREOS_CONFIG="${COREOS_OVERLAY}/coreos/config" info "Setting up portage..." From db734ea417b8769586910712ca37c4fec3cd91bc Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sun, 5 Jul 2015 15:59:13 -0700 Subject: [PATCH 2/5] bootstrap_sdk: do not leak catalyst config into sdk tarballs The version of repos.conf/coreos.conf that catalyst needs isn't valid for normal SDK chroots and causes env-update to spew errors when it is run prior to update_chroot which configures portage properly. --- bootstrap_sdk | 4 ++-- sdk_lib/make_chroot.sh | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/bootstrap_sdk b/bootstrap_sdk index 805c983a15..b6a0da1a79 100755 --- a/bootstrap_sdk +++ b/bootstrap_sdk @@ -48,8 +48,8 @@ pkgcache_path: $BINPKGS stage4/packages: coreos-devel/sdk-depends stage4/fsscript: ${BUILD_LIBRARY_DIR}/catalyst_default_stage4.sh stage4/root_overlay: ${TEMPDIR}/stage4_overlay -stage4/empty: /root /usr/portage /var/cache/edb -stage4/rm: /etc/machine-id /etc/resolv.conf +stage4/empty: /etc/portage/repos.conf /root /usr/portage /var/cache/edb +stage4/rm: /etc/machine-id /etc/resolv.conf /etc/portage/make.conf EOF catalyst_stage_default } diff --git a/sdk_lib/make_chroot.sh b/sdk_lib/make_chroot.sh index 9773349425..ef2123e384 100755 --- a/sdk_lib/make_chroot.sh +++ b/sdk_lib/make_chroot.sh @@ -146,6 +146,11 @@ init_users () { init_setup () { info "Running init_setup()..." + # clean up old catalyst configs to avoid error from env-update + # TODO(marineam): remove in a week or so + rm -f "${FLAGS_chroot}/etc/portage/make.conf" \ + "${FLAGS_chroot}/etc/portage/repos.conf/coreos.conf" + # Set up sudoers. Inside the chroot, the user can sudo without a password. # (Safe enough, since the only way into the chroot is to 'sudo chroot', so # the user's already typed in one sudo password...) From 611ed7698e157aa96f558d57298005030a034b70 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sun, 5 Jul 2015 17:14:53 -0700 Subject: [PATCH 3/5] enter_chroot: remove subversion config goo We do not have any svn based live ebuilds that require credentials which I presume is the only reason for shuffling svn configs like this. --- sdk_lib/enter_chroot.sh | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/sdk_lib/enter_chroot.sh b/sdk_lib/enter_chroot.sh index 036d6d3255..ca3c20d84d 100755 --- a/sdk_lib/enter_chroot.sh +++ b/sdk_lib/enter_chroot.sh @@ -316,21 +316,6 @@ setup_env() { fi fi - if [[ -d "$SUDO_HOME/.subversion" ]]; then - TARGET="/home/${SUDO_USER}/.subversion" - setup_mount "${SUDO_HOME}/.subversion" "--bind" "${TARGET}" - # Symbolic-link the .subversion directory so sandboxed subversion.class - # clients can use it. - for d in \ - "${FLAGS_cache_dir}"/distfiles/{host,target}/svn-src/"${SUDO_USER}"; do - if [[ ! -L "${d}/.subversion" ]]; then - rm -rf "${d}/.subversion" - user_mkdir "${d}" - user_symlink /home/${SUDO_USER}/.subversion "${d}/.subversion" - fi - done - fi - # Mount GnuPG's data directory for signing uploads if [[ -d "$SUDO_HOME/.gnupg" ]]; then debug "Mounting GnuPG" From 1fa7ef236b8b1c7735561a613825b0ee456d2e87 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sun, 5 Jul 2015 16:54:48 -0700 Subject: [PATCH 4/5] update_chroot: remove dependency on complex 'chromeos-cache' links The distfiles cache is always under .cache in the repo tree but there is a lot of extra logic to make that configurable along with compatibility symlinks for previous locations. Just yank it all out. --- sdk_lib/enter_chroot.sh | 37 +------------------------------------ sdk_lib/make_chroot.sh | 33 ++------------------------------- update_chroot | 22 ++++++++++++++++++---- update_distfiles | 2 +- 4 files changed, 22 insertions(+), 72 deletions(-) diff --git a/sdk_lib/enter_chroot.sh b/sdk_lib/enter_chroot.sh index ca3c20d84d..236a0beb86 100755 --- a/sdk_lib/enter_chroot.sh +++ b/sdk_lib/enter_chroot.sh @@ -26,7 +26,7 @@ DEFINE_string chrome_root "" \ "The root of your chrome browser source. Should contain a 'src' subdir." DEFINE_string chrome_root_mount "/home/${SUDO_USER}/chrome_root" \ "The mount point of the chrome broswer source in the chroot." -DEFINE_string cache_dir "" "Directory to use for caching." +DEFINE_string cache_dir "" "unused" DEFINE_boolean official_build $FLAGS_FALSE \ "Set COREOS_OFFICIAL=1 for release builds." @@ -70,9 +70,6 @@ if [ $FLAGS_official_build -eq $FLAGS_TRUE ]; then COREOS_OFFICIAL=1 fi -[ -z "${FLAGS_cache_dir}" ] && \ - die "--cache_dir is required" - # Only now can we die on error. shflags functions leak non-zero error codes, # so will die prematurely if 'switch_to_strict_mode' is specified before now. # TODO: replace shflags with something less error-prone, or contribute a fix. @@ -281,27 +278,6 @@ setup_env() { fi unset REFERENCE_DIR - chroot_cache='/var/cache/chromeos-cache' - debug "Setting up shared cache dir directory." - user_mkdir "${FLAGS_cache_dir}"/distfiles/{target,host} - user_mkdir "${FLAGS_chroot}/${chroot_cache}" - setup_mount "${FLAGS_cache_dir}" "--bind" "${chroot_cache}" - # TODO(build): remove this as of 12/01/12. - # Because of how distfiles -> cache_dir was deployed, if this isn't - # a symlink, we *know* the ondisk pathways aren't compatible- thus - # fix it now. - distfiles_path="${FLAGS_chroot}/var/cache/distfiles" - if [ ! -L "${distfiles_path}" ]; then - # While we're at it, ensure the var is exported w/in the chroot; it - # won't exist if distfiles isn't a symlink. - p="${FLAGS_chroot}/etc/profile.d/chromeos-cachedir.sh" - rm -rf "${distfiles_path}" - ln -s chromeos-cache/distfiles "${distfiles_path}" - mkdir -p -m 775 "${p%/*}" - echo 'export CHROMEOS_CACHEDIR=${chroot_cache}' > "${p}" - chmod 0644 "${p}" - fi - user_mkdir "${FLAGS_chroot}/home/${SUDO_USER}/.ssh" if [ $FLAGS_ssh_agent -eq $FLAGS_TRUE ]; then # Clean up previous ssh agents. @@ -376,17 +352,6 @@ setup_env() { warn "-- Note: modprobe fuse failed. gmergefs will not work" fi - # Fix permissions on ccache tree. If this is a fresh chroot, then they - # might not be set up yet. Or if the user manually `rm -rf`-ed things, - # we need to reset it. Otherwise, gcc itself takes care of fixing things - # on demand, but only when it updates. - ccache_dir="${FLAGS_chroot}/var/cache/distfiles/ccache" - if [[ ! -d ${ccache_dir} ]]; then - mkdir -p -m 2775 "${ccache_dir}" - fi - find -H "${ccache_dir}" -type d -exec chmod 2775 {} + & - find -H "${ccache_dir}" -gid 0 -exec chgrp 250 {} + & - # Certain files get copied into the chroot when entering. for fn in "${FILES_TO_COPY_TO_CHROOT[@]}"; do copy_into_chroot_if_exists "${SUDO_HOME}/${fn}" "/home/${SUDO_USER}/${fn}" diff --git a/sdk_lib/make_chroot.sh b/sdk_lib/make_chroot.sh index ef2123e384..914e8351f1 100755 --- a/sdk_lib/make_chroot.sh +++ b/sdk_lib/make_chroot.sh @@ -45,7 +45,7 @@ DEFINE_integer jobs "${NUM_JOBS}" \ "How many packages to build in parallel at maximum." DEFINE_string stage3_path "" \ "Use the stage3 located on this path." -DEFINE_string cache_dir "" "Directory to store caches within." +DEFINE_string cache_dir "" "unused" # Parse command line flags. FLAGS_HELP="usage: $SCRIPT_NAME [flags]" @@ -64,10 +64,6 @@ umask 022 # TODO: replace shflags with something less error-prone, or contribute a fix. switch_to_strict_mode -[[ "${FLAGS_delete}" == "${FLAGS_FALSE}" ]] && \ - [[ -z "${FLAGS_cache_dir}" ]] && \ - die "--cache_dir is required" - ENTER_CHROOT_ARGS=( CROS_WORKON_SRCROOT="$CHROOT_TRUNK" PORTAGE_USERNAME="${SUDO_USER}" @@ -75,15 +71,13 @@ ENTER_CHROOT_ARGS=( # Invoke enter_chroot. This can only be used after sudo has been installed. enter_chroot() { - "$ENTER_CHROOT" --cache_dir "${FLAGS_cache_dir}" --chroot "$FLAGS_chroot" \ - -- "${ENTER_CHROOT_ARGS[@]}" "$@" + "$ENTER_CHROOT" --chroot "$FLAGS_chroot" -- "${ENTER_CHROOT_ARGS[@]}" "$@" } # Invoke enter_chroot running the command as root, and w/out sudo. # This should be used prior to sudo being merged. early_enter_chroot() { "$ENTER_CHROOT" --chroot "$FLAGS_chroot" --early_make_chroot \ - --cache_dir "${FLAGS_cache_dir}" \ -- "${ENTER_CHROOT_ARGS[@]}" "$@" } @@ -169,29 +163,6 @@ EOF chmod 0440 "${FLAGS_chroot}/etc/sudoers.d/90_cros" - # Create directories referred to by our conf files. - mkdir -p -m 775 "${FLAGS_chroot}/var/lib/portage/pkgs" \ - "${FLAGS_chroot}/var/cache/"chromeos-{cache,chrome} \ - "${FLAGS_chroot}/etc/profile.d" - - echo "export CHROMEOS_CACHEDIR=/var/cache/chromeos-cache" > \ - "${FLAGS_chroot}/etc/profile.d/chromeos-cachedir.sh" - chmod 0644 "${FLAGS_chroot}/etc/profile.d/chromeos-cachedir.sh" - rm -rf "${FLAGS_chroot}/var/cache/distfiles" - ln -s chromeos-cache/distfiles "${FLAGS_chroot}/var/cache/distfiles" - - # Run this from w/in the chroot so we use whatever uid/gid - # these are defined as w/in the chroot. - bare_chroot chown "${SUDO_USER}:portage" /var/cache/chromeos-chrome - - # These are created for compatibility while transitioning - # make.conf and friends over to the new location. - # TODO(ferringb): remove this 01/13 or so. - ln -s ../../cache/chromeos-cache/distfiles/host \ - "${FLAGS_chroot}/var/lib/portage/distfiles" - ln -s ../../cache/chromeos-cache/distfiles/target \ - "${FLAGS_chroot}/var/lib/portage/distfiles-target" - # Add chromite/bin into the path globally # We rely on 'env-update' getting called below. target="${FLAGS_chroot}/etc/env.d/99coreos" diff --git a/update_chroot b/update_chroot index 89abd6eb94..594eef2e01 100755 --- a/update_chroot +++ b/update_chroot @@ -50,19 +50,33 @@ CROSSDEV_OVERLAY="/usr/local/portage/crossdev" COREOS_OVERLAY="${REPO_ROOT}/src/third_party/coreos-overlay" COREOS_CONFIG="${COREOS_OVERLAY}/coreos/config" +# PORTAGE_USERNAME may already be in the env but set just to be safe. +: ${PORTAGE_USERNAME:=${USER}} + +# Clean up old distfiles cache. It used to be split for 'host' and +# 'target' but that just duplicates files. Also a ccache was in there. +if [[ -d "${REPO_CACHE_DIR}/distfiles/host" ]]; then + info "Cleaning up old distfiles cache..." + sudo mv "${REPO_CACHE_DIR}"/{distfiles/host,distfiles.host} + sudo rm -rf "${REPO_CACHE_DIR}/distfiles" + sudo mv "${REPO_CACHE_DIR}"/{distfiles.host,distfiles} +fi + info "Setting up portage..." -sudo mkdir -p "/etc/portage/repos.conf/" -sudo rm -f "/etc/portage/make.conf" -sudo touch "/etc/portage/make.conf.user" +sudo mkdir -p "${REPO_CACHE_DIR}/distfiles" +sudo chown "${PORTAGE_USERNAME}:portage" "${REPO_CACHE_DIR}/distfiles" +sudo mkdir -p /etc/portage/repos.conf /var/lib/portage/pkgs +sudo touch /etc/portage/make.conf.user sudo_clobber "/etc/portage/make.conf" < Date: Sun, 5 Jul 2015 18:01:50 -0700 Subject: [PATCH 5/5] setup_board: simplify PORTAGE_USERNAME handling --- setup_board | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup_board b/setup_board index 5f948fbe84..7937d82e39 100755 --- a/setup_board +++ b/setup_board @@ -71,9 +71,6 @@ _generate_wrapper() { export PORTAGE_CONFIGROOT="$BOARD_ROOT" export SYSROOT="\${SYSROOT:-$BOARD_ROOT}" -if [ -z "\$PORTAGE_USERNAME" ]; then - export PORTAGE_USERNAME=\$(basename \$HOME) -fi export ROOT="$BOARD_ROOT" exec sudo -E ${command} ${extra_args} "\$@" EOF @@ -237,6 +234,7 @@ PORTDIR="$(portageq envvar PORTDIR)" PORTDIR_OVERLAY="$(portageq envvar PORTDIR_OVERLAY)" DISTDIR="$(portageq envvar DISTDIR)" MAKEOPTS="$(portageq envvar MAKEOPTS)" +PORTAGE_USERNAME="$(portageq envvar PORTAGE_USERNAME)" # Board specific settings CHOST="${BOARD_CHOST}"