Moved debootstrap phase from build_image.sh to install_packages.sh.

This puts all package installation for the rootfs in one place. It
also makes it so that you can run install_packages.sh --root="foo"
outside of the build_image.sh script if you want to test something
without going through the full image build. With this change
build_image basically:
 - Sets up a loopback mounted rootfs
 - Calls install_packages.sh on rootfs
 - Calls customize_rootfs on rootfs
 - Makes the rootfs bootable

Review URL: http://codereview.chromium.org/520004

build_image.sh

@@ -38,10 +38,6 @@ DEFINE_string suite "$DEFAULT_IMG_SUITE" "Repository suite to base image on."
 DEFINE_string pkglist "$DEFAULT_PKGLIST" \
   "Name of file listing packages to install from repository."
 
-KERNEL_DEB_PATH=$(find "${FLAGS_build_root}/x86/local_packages" -name "linux-image-*.deb")
-KERNEL_DEB=$(basename "${KERNEL_DEB_PATH}" .deb | sed -e 's/linux-image-//' -e 's/_.*//')
-KERNEL_VERSION=${KERNEL_VERSION:-${KERNEL_DEB}}
-
 # Parse command line
 FLAGS "$@" || exit 1
 eval set -- "${FLAGS_ARGV}"
@@ -60,7 +56,6 @@ ROOT_FS_DIR="${OUTPUT_DIR}/rootfs"
 ROOT_FS_IMG="${OUTPUT_DIR}/rootfs.image"
 MBR_IMG="${OUTPUT_DIR}/mbr.image"
 OUTPUT_IMG="${OUTPUT_DIR}/usb.img"
-SETUP_DIR="${OUTPUT_DIR}/tmp"
 
 LOOP_DEV=
 
@@ -81,37 +76,16 @@ fi
 # create the output directory
 mkdir -p "$OUTPUT_DIR"
 
-# Make sure anything mounted in the rootfs is cleaned up ok on exit.
-cleanup_rootfs_mounts() {
-  # Occasionally there are some daemons left hanging around that have our
-  # root image file system open. We do a best effort attempt to kill them.
-  PIDS=`sudo lsof -t "$ROOT_FS_DIR" | sort | uniq`
-  for pid in $PIDS
-  do
-    local cmdline=`cat /proc/$pid/cmdline`
-    echo "Killing process that has open file on our rootfs: $cmdline"
-    ! sudo kill $pid  # Preceded by ! to disable ERR trap.
-  done
-
-  # Sometimes the volatile directory is left mounted and sometimes it is not,
-  # so we precede by '!' to disable the ERR trap.
-  ! sudo umount "$ROOT_FS_DIR"/lib/modules/2.6.*/volatile/
-
-  sudo umount "${ROOT_FS_DIR}/proc"
-  sudo umount "${ROOT_FS_DIR}/sys"
-}
-
 cleanup_rootfs_loop() {
   sudo umount "$LOOP_DEV"
   sleep 1  # in case $LOOP_DEV is in use
   sudo losetup -d "$LOOP_DEV"
+  LOOP_DEV=""
 }
 
 cleanup() {
   # Disable die on error.
   set +e
-
-  cleanup_rootfs_mounts
   if [ -n "$LOOP_DEV" ]
   then
     cleanup_rootfs_loop
@@ -142,54 +116,18 @@ sudo mkfs.ext3 "$LOOP_DEV"
 sudo tune2fs -L "$DISK_LABEL" -U "$UUID" -c 0 -i 0 "$LOOP_DEV"
 sudo mount "$LOOP_DEV" "$ROOT_FS_DIR"
 
-# Add debootstrap link for the suite, if it doesn't exist.
+# -- Install packages and customize root file system. --
-if [ ! -e "/usr/share/debootstrap/scripts/$FLAGS_suite" ]
-then
-  sudo ln -s /usr/share/debootstrap/scripts/jaunty \
-    "/usr/share/debootstrap/scripts/$FLAGS_suite"
-fi
 
-# Bootstrap the base debian file system
-# TODO: Switch to --variant=minbase
-sudo debootstrap --arch=i386 $FLAGS_suite "$ROOT_FS_DIR" "${FLAGS_mirror}"
-
-# -- Customize the root file system --
-
-# Set up mounts for working within the chroot. We copy some basic
-# network information from the host so that the chroot can access
-# repositories on the network as needed.
-sudo mount -t proc proc "${ROOT_FS_DIR}/proc"
-sudo mount -t sysfs sysfs "${ROOT_FS_DIR}/sys" # TODO: Do we need sysfs?
-sudo cp /etc/hosts "${ROOT_FS_DIR}/etc"
-
-# Create setup directory and copy over scripts, config files, and locally
-# built packages.
-mkdir -p "$SETUP_DIR"
-mkdir -p "${SETUP_DIR}/local_packages"
-cp "${FLAGS_build_root}/x86/local_packages"/* "${SETUP_DIR}/local_packages"
-
-# Set up repository for local packages to install in the rootfs via apt-get.
-cd "$SETUP_DIR"
-dpkg-scanpackages local_packages/ /dev/null | \
-   gzip > local_packages/Packages.gz
-cd -
-
-# Run the package install script
 "${SCRIPTS_DIR}/install_packages.sh"  \
+  --build_root="${FLAGS_build_root}"  \
   --root="$ROOT_FS_DIR"               \
   --output_dir="${OUTPUT_DIR}"        \
-  --setup_dir="${SETUP_DIR}"          \
   --package_list="$FLAGS_pkglist"     \
   --server="$FLAGS_mirror"            \
-  --suite="$FLAGS_suite"              \
+  --suite="$FLAGS_suite"
-  --kernel_version="$KERNEL_VERSION"
 
-# Run the script to customize the resulting root file system.
 "${SCRIPTS_DIR}/customize_rootfs.sh" --root="${ROOT_FS_DIR}"
 
-# Unmount mounts within the rootfs so it is ready to be imaged.
-cleanup_rootfs_mounts
-
 # -- Turn root file system into bootable image --
 
 # Setup extlinux configuration.

install_packages.sh

@@ -15,24 +15,25 @@
 
 # Script must be run inside the chroot
 assert_inside_chroot
+assert_not_root_user
+
+DEFAULT_PKGLIST="${SRC_ROOT}/package_repo/package-list-prod.txt"
 
 # Flags
-DEFINE_string target "x86" \
+DEFINE_string output_dir "" \
-  "The target architecture to build for. One of { x86, arm }."
+  "The location of the output directory to use [REQUIRED]."
 DEFINE_string root ""      \
   "The root file system to install packages in."
-DEFINE_string output_dir "" \
+DEFINE_string target "x86" \
-  "The location of the output directory to use."
+  "The target architecture to build for. One of { x86, arm }."
-DEFINE_string package_list "" \
+DEFINE_string build_root "$DEFAULT_BUILD_ROOT"                \
+  "Root of build output"
+DEFINE_string package_list "$DEFAULT_PKGLIST" \
   "The package list file to use."
-DEFINE_string setup_dir "/tmp" \
+DEFINE_string server "$DEFAULT_IMG_MIRROR" \
-  "The staging directory to use."
-DEFINE_string server "" \
   "The package server to use."
-DEFINE_string suite "" \
+DEFINE_string suite "$DEFAULT_IMG_SUITE" \
   "The package suite to use."
-DEFINE_string kernel_version "" \
-  "The kernel version to use."
 
 # Parse command line
 FLAGS "$@" || exit 1
@@ -41,31 +42,71 @@ eval set -- "${FLAGS_ARGV}"
 # Die on any errors.
 set -e
 
-ROOT_FS_DIR="$FLAGS_root"
+KERNEL_DEB_PATH=$(find "${FLAGS_build_root}/${FLAGS_target}/local_packages" \
-if [[ -z "$ROOT_FS_DIR" ]]; then
+  -name "linux-image-*.deb")
-  echo "Error: --root is required."
+KERNEL_DEB=$(basename "${KERNEL_DEB_PATH}" .deb | sed -e 's/linux-image-//' \
+  -e 's/_.*//')
+KERNEL_VERSION=${KERNEL_VERSION:-${KERNEL_DEB}}
+
+if [[ -z "$FLAGS_output_dir" ]]; then
+  echo "Error: --output_dir is required."
   exit 1
 fi
-if [[ ! -d "$ROOT_FS_DIR" ]]; then
+OUTPUT_DIR=$(readlink -f "$FLAGS_output_dir")
-  echo "Error: Root FS does not exist? ($ROOT_FS_DIR)"
+SETUP_DIR="${OUTPUT_DIR}/local_repo"
-  exit 1
+ROOT_FS_DIR="${OUTPUT_DIR}/rootfs"
+if [[ -n "$FLAGS_root" ]]; then
+  ROOT_FS_DIR=$(readlink -f "$FLAGS_root")
 fi
+mkdir -p "$OUTPUT_DIR" "$SETUP_DIR" "$ROOT_FS_DIR"
+
+# Make sure anything mounted in the rootfs is cleaned up ok on exit.
+cleanup_rootfs_mounts() {
+  # Occasionally there are some daemons left hanging around that have our
+  # root image file system open. We do a best effort attempt to kill them.
+  PIDS=`sudo lsof -t "$ROOT_FS_DIR" | sort | uniq`
+  for pid in $PIDS
+  do
+    local cmdline=`cat /proc/$pid/cmdline`
+    echo "Killing process that has open file on our rootfs: $cmdline"
+    ! sudo kill $pid  # Preceded by ! to disable ERR trap.
+  done
+
+  # Sometimes the volatile directory is left mounted and sometimes it is not,
+  # so we precede by '!' to disable the ERR trap.
+  ! sudo umount "$ROOT_FS_DIR"/lib/modules/2.6.*/volatile/ > /dev/null 2>&1
+
+  sudo umount "${ROOT_FS_DIR}/proc"
+  sudo umount "${ROOT_FS_DIR}/sys"
+}
+
+# Create setup directory and copy over scripts, config files, and locally
+# built packages.
+mkdir -p "${SETUP_DIR}/local_packages"
+cp "${FLAGS_build_root}/${FLAGS_target}/local_packages"/* \
+  "${SETUP_DIR}/local_packages"
+
+# Set up repository for local packages to install in the rootfs via apt-get.
+cd "$SETUP_DIR"
+dpkg-scanpackages local_packages/ /dev/null | \
+   gzip > local_packages/Packages.gz
+cd -
 
 # Create the temporary apt source.list used to install packages.
-APT_SOURCE="${FLAGS_output_dir}/sources.list"
+APT_SOURCE="${OUTPUT_DIR}/sources.list"
 cat <<EOF > "$APT_SOURCE"
-deb file:"$FLAGS_setup_dir" local_packages/
+deb file:"$SETUP_DIR" local_packages/
 deb $FLAGS_server $FLAGS_suite main restricted multiverse universe
 EOF
 
 # Cache directory for APT to use.
-APT_CACHE_DIR="${FLAGS_output_dir}/tmp/cache/"
+APT_CACHE_DIR="${OUTPUT_DIR}/tmp/cache/"
 mkdir -p "${APT_CACHE_DIR}/archives/partial"
 
 # Create the apt configuration file. See "man apt.conf"
-APT_PARTS="${FLAGS_output_dir}/apt.conf.d"
+APT_PARTS="${OUTPUT_DIR}/apt.conf.d"
 mkdir -p "$APT_PARTS"  # An empty apt.conf.d to avoid other configs.
-export APT_CONFIG="${FLAGS_output_dir}/apt.conf"
+export APT_CONFIG="${OUTPUT_DIR}/apt.conf"
 cat <<EOF > "$APT_CONFIG"
 APT
 {
@@ -100,7 +141,28 @@ DPkg
 EOF
 
 # TODO: Full audit of the apt conf dump to make sure things are ok.
-apt-config dump > "${FLAGS_output_dir}/apt.conf.dump"
+apt-config dump > "${OUTPUT_DIR}/apt.conf.dump"
+
+# Add debootstrap link for the suite, if it doesn't exist.
+if [ ! -e "/usr/share/debootstrap/scripts/$FLAGS_suite" ]
+then
+  sudo ln -s /usr/share/debootstrap/scripts/jaunty \
+    "/usr/share/debootstrap/scripts/$FLAGS_suite"
+fi
+
+# Bootstrap the base debian file system
+# TODO: Switch to --variant=minbase
+sudo debootstrap --arch=i386 $FLAGS_suite "$ROOT_FS_DIR" "${FLAGS_server}"
+
+# Set up mounts for working within the rootfs. We copy some basic
+# network information from the host so that maintainer scripts can
+# access the network as needed.
+# TODO: All of this rootfs mount stuff can be removed as soon as we stop
+# running the maintainer scripts on install.
+sudo mount -t proc proc "${ROOT_FS_DIR}/proc"
+sudo mount -t sysfs sysfs "${ROOT_FS_DIR}/sys" # TODO: Do we need sysfs?
+sudo cp /etc/hosts "${ROOT_FS_DIR}/etc"
+trap cleanup_rootfs_mounts EXIT
 
 # Install prod packages
 COMPONENTS=`cat $FLAGS_package_list | grep -v ' *#' | grep -v '^ *$' | sed '/$/{N;s/\n/ /;}'`
@@ -123,13 +185,7 @@ EOF
 
 # Install the kernel.
 sudo APT_CONFIG="$APT_CONFIG" apt-get --force-yes \
-  install "linux-image-${FLAGS_kernel_version}"
+  install "linux-image-${KERNEL_VERSION}"
-
-# Setup bootchart.
-# TODO: Move this and other developer oriented "components" into an optional
-# package-list-prod-dev.txt (ideally with a better name).
-sudo APT_CONFIG="$APT_CONFIG" apt-get --force-yes \
-  install bootchart
 
 # Clean up the apt cache.
 # TODO: The cache was populated by debootstrap, not these installs. Remove
@@ -140,4 +196,7 @@ sudo rm -f "${ROOT_FS_DIR}"/var/cache/apt/archives/*.deb
 # repository needs to contain.
 # TODO: Replace with list_installed_packages.sh when it is fixed up.
 dpkg --root="${ROOT_FS_DIR}" -l > \
-  "${FLAGS_output_dir}/package_list_installed.txt"
+  "${OUTPUT_DIR}/package_list_installed.txt"
+
+cleanup_rootfs_mounts
+trap - EXIT