From 7ee892d95a71310ffbaabb69654b862d696e49f1 Mon Sep 17 00:00:00 2001
From: Brian Harring <ferringb@chromium.org>
Date: Thu, 2 Feb 2012 09:33:10 -0800
Subject: [PATCH] Mount bind external distfiles into the chroot.

Purpose of this is to allow us to avoid re-downloading everything
every time we rebuild the chroot.

This maintains two directories; host and target.  Future enhancement
involves collapsing this into one- this requires some host work however,
and has some potential gotchas in doing so.

Meanwhile, we now store distfiles in repo/distfiles/{host,target},
and mount bind repo/distfiles into /var/cache/distfiles.

An upgrade script in turn optimistically tries to move the content
into the new location; if it can't complete the move, it wipes the
content and the user has to redownload it (acceptable, if annoying).

BUG=chromium-os:13115
TEST=cbuildbot x86-generic-full

Change-Id: Iea96429df0e1fdc4ac0860fbce0daabc90c4c2a3
Reviewed-on: https://gerrit.chromium.org/gerrit/15189
Reviewed-by: Brian Harring <ferringb@chromium.org>
Tested-by: Brian Harring <ferringb@chromium.org>
Commit-Ready: Brian Harring <ferringb@chromium.org>
---
 chroot_version_hooks.d/24_distfiles_migration | 33 +++++++++++++++++++
 sdk_lib/enter_chroot.sh                       | 10 ++++++
 sdk_lib/make_chroot.sh                        | 12 +++++--
 3 files changed, 52 insertions(+), 3 deletions(-)
 create mode 100644 chroot_version_hooks.d/24_distfiles_migration

diff --git a/chroot_version_hooks.d/24_distfiles_migration b/chroot_version_hooks.d/24_distfiles_migration
new file mode 100644
index 0000000000..fcd16660c7
--- /dev/null
+++ b/chroot_version_hooks.d/24_distfiles_migration
@@ -0,0 +1,33 @@
+# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Previously distfiles were stored entirely in the chroot, forcing us
+# to download it everytime.  This content is now stored externally, exposed
+# to the chroot via mount binds.
+#
+# What we're doing here is moving any existing content from the old locations,
+# into the new locations- which will already have a mount bind in place.
+# If it *doesn't*, meaning parallel cros_sdk usage, we just wipe the content
+# (user can redownload it after all).
+#
+# Once that's done, upgrade the chroot directory structure installing
+# syms pointing back to the new location.
+
+upgrade_path() {
+  local src="/var/lib/portage/$1"
+  local dest="/var/cache/distfiles/$2"
+  if [ -L "$src" ]; then
+    # Already upgraded- skip this one.
+    return 0
+  fi
+  # Suppress failures.  User just has to download in that case.
+  sudo find "$src"/ -maxdepth 1 -mindepth 1 -exec sudo mv -t "$dest/" {} + || :
+  sudo rm -rf "$src"
+  sudo ln -s ../../cache/distfiles/"$2" "$src"
+}
+
+upgrade_path distfiles host
+upgrade_path distfiles-target target
+
+exit 0
diff --git a/sdk_lib/enter_chroot.sh b/sdk_lib/enter_chroot.sh
index e6acc97b96..0339a761fe 100755
--- a/sdk_lib/enter_chroot.sh
+++ b/sdk_lib/enter_chroot.sh
@@ -25,6 +25,8 @@ DEFINE_string chrome_root "" \
   "The root of your chrome browser source. Should contain a 'src' subdir."
 DEFINE_string chrome_root_mount "/home/$USER/chrome_root" \
   "The mount point of the chrome broswer source in the chroot."
+DEFINE_string distfiles "" \
+  "Override the destination dir used for distfiles."
 
 DEFINE_boolean official_build $FLAGS_FALSE \
   "Set CHROMEOS_OFFICIAL=1 for release builds."
@@ -67,6 +69,9 @@ if [ $FLAGS_official_build -eq $FLAGS_TRUE ]; then
    CHROMEOS_OFFICIAL=1
 fi
 
+[ -z "${FLAGS_distfiles}" ] && \
+  FLAGS_distfiles="${FLAGS_trunk}/distfiles"
+
 # Only now can we die on error.  shflags functions leak non-zero error codes,
 # so will die prematurely if 'set -e' is specified before now.
 # TODO: replace shflags with something less error-prone, or contribute a fix.
@@ -290,6 +295,11 @@ function setup_env {
     fi
     unset REFERENCE_DIR
 
+    debug "Setting up shared distfiles directory."
+    mkdir -p "${FLAGS_distfiles}"/{target,host}
+    sudo mkdir -p "${FLAGS_chroot}/var/cache/distfiles/"
+    queue_mount "${FLAGS_distfiles}" "--bind" "/var/cache/distfiles"
+
     if [ $FLAGS_ssh_agent -eq $FLAGS_TRUE ]; then
       if [ -n "${SSH_AUTH_SOCK}" -a -d "${HOME}/.ssh" ]; then
         TARGET_DIR="${FLAGS_chroot}/home/${USER}/.ssh"
diff --git a/sdk_lib/make_chroot.sh b/sdk_lib/make_chroot.sh
index f26dbab1b0..2ac5767657 100755
--- a/sdk_lib/make_chroot.sh
+++ b/sdk_lib/make_chroot.sh
@@ -198,9 +198,15 @@ EOF
    sudo chmod 0644 "${FLAGS_chroot}"/etc/make.conf.user
 
    # Create directories referred to by our conf files.
-   sudo mkdir -p -m 775 "${FLAGS_chroot}/var/lib/portage/distfiles" \
-     "${FLAGS_chroot}/var/lib/portage/distfiles-target" \
-     "${FLAGS_chroot}/var/lib/portage/pkgs"
+   sudo mkdir -p -m 775 "${FLAGS_chroot}/var/lib/portage/pkgs"
+
+   # These are created for compatibility while transitioning
+   # make.conf and friends over to the new location.
+   # TODO(ferringb): remove this 03/12 or so.
+   sudo ln -s ../../cache/distfiles/host \
+     "${FLAGS_chroot}/var/lib/portage/distfiles"
+   sudo ln -s ../../cache/distfiles/target \
+     "${FLAGS_chroot}/var/lib/portage/distfiles-target"
 
    if [[ $FLAGS_jobs -ne -1 ]]; then
      EMERGE_JOBS="--jobs=$FLAGS_jobs"