From 06c4dcf4c915c5d3ef057df008f14b9dab8b54f4 Mon Sep 17 00:00:00 2001 From: tedbo Date: Tue, 22 Dec 2009 21:20:24 -0800 Subject: [PATCH] Change install_packages.sh to run from outside of the target rootfs and install debs into the target rootfs. This will still run the maintainer scripts, but it is a step in the right direction to being able to create a cross-compiled image. This will allow us to experiment with disabling the maintainer scripts and attempt to get that working. It also makes it easier to modify the install_packages script since it now has access to shflags and the source tree properly. This CL also cleans up stuff doing in build_image.sh to bind mount the trunk into the target rootfs, set up packages for install there, etc. The next step is to experiment with disabling maintainer scripts. One possible way to do that would be to put a shim in place of dpkg so that when apt calls dpkg it install a package we call dpkg-deb with a simple unpack instead. This would allow us to continue to use the good nice features that apt provides. Review URL: http://codereview.chromium.org/517003 --- build_image.sh | 70 ++++------------------ install_packages.sh | 140 +++++++++++++++++++++++++++++--------------- 2 files changed, 107 insertions(+), 103 deletions(-) diff --git a/build_image.sh b/build_image.sh index 0e4ada5cdd..223567d746 100755 --- a/build_image.sh +++ b/build_image.sh @@ -38,11 +38,6 @@ DEFINE_string suite "$DEFAULT_IMG_SUITE" "Repository suite to base image on." DEFINE_string pkglist "$DEFAULT_PKGLIST" \ "Name of file listing packages to install from repository." -DEFINE_string mirror2 "$DEFAULT_EXT_MIRROR" "Additional mirror to use." -DEFINE_string suite2 "$DEFAULT_EXT_SUITE" "Suite to use in additional mirror." -DEFINE_string pkglist2 "" \ - "Name of file listing packages to install from additional mirror." - KERNEL_DEB_PATH=$(find "${FLAGS_build_root}/x86/local_packages" -name "linux-image-*.deb") KERNEL_DEB=$(basename "${KERNEL_DEB_PATH}" .deb | sed -e 's/linux-image-//' -e 's/_.*//') KERNEL_VERSION=${KERNEL_VERSION:-${KERNEL_DEB}} @@ -65,14 +60,7 @@ ROOT_FS_DIR="${OUTPUT_DIR}/rootfs" ROOT_FS_IMG="${OUTPUT_DIR}/rootfs.image" MBR_IMG="${OUTPUT_DIR}/mbr.image" OUTPUT_IMG="${OUTPUT_DIR}/usb.img" - -# These paths are relative to SCRIPTS_DIR. -ROOTFS_PACKAGE_INSTALL_SCRIPT="install_packages.sh" -ROOTFS_CUSTOMIZE_SCRIPT="customize_rootfs.sh" - -ROOTFS_STATIC_DATA="${SRC_ROOT}/rootfs_static_data" -ROOTFS_SETUP_DIR="/tmp/chromeos_setup" -SETUP_DIR="${ROOT_FS_DIR}/${ROOTFS_SETUP_DIR}" +SETUP_DIR="${OUTPUT_DIR}/tmp" LOOP_DEV= @@ -111,7 +99,6 @@ cleanup_rootfs_mounts() { sudo umount "${ROOT_FS_DIR}/proc" sudo umount "${ROOT_FS_DIR}/sys" - sudo umount "${ROOT_FS_DIR}/trunk" } cleanup_rootfs_loop() { @@ -138,8 +125,8 @@ trap cleanup EXIT mkdir -p "$ROOT_FS_DIR" # Create root file system disk image to fit on a 1GB memory stick. -# 1 GB in hard-drive-manufacturer-speak is 10^9, not 2^30. 950MB < 10^9 bytes. -ROOT_SIZE_BYTES=$((1024 * 1024 * 950)) +# 1 GB in hard-drive-manufacturer-speak is 10^9, not 2^30. 700MB < 10^9 bytes. +ROOT_SIZE_BYTES=$((1024 * 1024 * 700)) dd if=/dev/zero of="$ROOT_FS_IMG" bs=1 count=1 seek=$((ROOT_SIZE_BYTES - 1)) # Format, tune, and mount the rootfs. @@ -163,6 +150,7 @@ then fi # Bootstrap the base debian file system +# TODO: Switch to --variant=minbase sudo debootstrap --arch=i386 $FLAGS_suite "$ROOT_FS_DIR" "${FLAGS_mirror}" # -- Customize the root file system -- @@ -174,62 +162,30 @@ sudo mount -t proc proc "${ROOT_FS_DIR}/proc" sudo mount -t sysfs sysfs "${ROOT_FS_DIR}/sys" # TODO: Do we need sysfs? sudo cp /etc/hosts "${ROOT_FS_DIR}/etc" -# Set up bind mount for trunk, so we can get to package repository -# TODO: also use this instead of SETUP_DIR for other things below? -sudo mkdir -p "$ROOT_FS_DIR/trunk" -sudo mount --bind "$GCLIENT_ROOT" "$ROOT_FS_DIR/trunk" - # Create setup directory and copy over scripts, config files, and locally # built packages. mkdir -p "$SETUP_DIR" mkdir -p "${SETUP_DIR}/local_packages" -cp "${SCRIPTS_DIR}/${ROOTFS_PACKAGE_INSTALL_SCRIPT}" "$SETUP_DIR" -cp -r "$ROOTFS_STATIC_DATA" "$SETUP_DIR" -cp "$FLAGS_pkglist" "${SETUP_DIR}/package-list-prod.txt" cp "${FLAGS_build_root}/x86/local_packages"/* "${SETUP_DIR}/local_packages" -if [ -n "$FLAGS_pkglist2" ] -then - cp "$FLAGS_pkglist2" "${SETUP_DIR}/package-list-2.txt" -fi - # Set up repository for local packages to install in the rootfs via apt-get. cd "$SETUP_DIR" dpkg-scanpackages local_packages/ /dev/null | \ gzip > local_packages/Packages.gz cd - -# File-type mirrors have a different path when bind-mounted inside the chroot -# ${FOO/bar/baz} replaces bar with baz when evaluating $FOO. -MIRROR_INSIDE="${FLAGS_mirror/$GCLIENT_ROOT//trunk}" -MIRROR2_INSIDE="${FLAGS_mirror2/$GCLIENT_ROOT//trunk}" - -# Write options for customize script into the chroot -CUST_OPTS="${SETUP_DIR}/customize_opts.sh" -cat < $CUST_OPTS -REAL_USER=$USER -SETUP_DIR="$ROOTFS_SETUP_DIR" -KERNEL_VERSION="$KERNEL_VERSION" -SERVER="$MIRROR_INSIDE" -SUITE="$FLAGS_suite" -SERVER2="$MIRROR2_INSIDE" -SUITE2="$FLAGS_suite2" -EOF -# ...and all CHROMEOS_ vars -set | egrep "^CHROMEOS_|^BUILDBOT_" >> $CUST_OPTS - # Run the package install script -sudo chroot "$ROOT_FS_DIR" \ - "${ROOTFS_SETUP_DIR}/${ROOTFS_PACKAGE_INSTALL_SCRIPT}" +"${SCRIPTS_DIR}/install_packages.sh" \ + --root="$ROOT_FS_DIR" \ + --output_dir="${OUTPUT_DIR}" \ + --setup_dir="${SETUP_DIR}" \ + --package_list="$FLAGS_pkglist" \ + --server="$FLAGS_mirror" \ + --suite="$FLAGS_suite" \ + --kernel_version="$KERNEL_VERSION" # Run the script to customize the resulting root file system. -"${SCRIPTS_DIR}/${ROOTFS_CUSTOMIZE_SCRIPT}" --root="${ROOT_FS_DIR}" - -# No longer need the setup directory in the rootfs. -rm -rf "$SETUP_DIR" - -# Move package lists from the image into the output dir -sudo mv "$ROOT_FS_DIR"/etc/package_list_*.txt "$OUTPUT_DIR" +"${SCRIPTS_DIR}/customize_rootfs.sh" --root="${ROOT_FS_DIR}" # Unmount mounts within the rootfs so it is ready to be imaged. cleanup_rootfs_mounts diff --git a/install_packages.sh b/install_packages.sh index 86bcfebea2..79917b634b 100755 --- a/install_packages.sh +++ b/install_packages.sh @@ -1,37 +1,103 @@ -#!/bin/sh +#!/bin/bash # Copyright (c) 2009 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -# Sets up the chromium-based os from inside a chroot of the root fs. +# Script to install packages into the target root file system. +# # NOTE: This script should be called by build_image.sh. Do not run this # on your own unless you know what you are doing. +# Load common constants. This should be the first executable line. +# The path to common.sh should be relative to your script's location. +. "$(dirname "$0")/common.sh" + +# Script must be run inside the chroot +assert_inside_chroot + +# Flags +DEFINE_string target "x86" \ + "The target architecture to build for. One of { x86, arm }." +DEFINE_string root "" \ + "The root file system to install packages in." +DEFINE_string output_dir "" \ + "The location of the output directory to use." +DEFINE_string package_list "" \ + "The package list file to use." +DEFINE_string setup_dir "/tmp" \ + "The staging directory to use." +DEFINE_string server "" \ + "The package server to use." +DEFINE_string suite "" \ + "The package suite to use." +DEFINE_string kernel_version "" \ + "The kernel version to use." + +# Parse command line +FLAGS "$@" || exit 1 +eval set -- "${FLAGS_ARGV}" + +# Die on any errors. set -e -# Read options from the config file created by build_image.sh. -echo "Reading options..." -cat "$(dirname $0)/customize_opts.sh" -. "$(dirname $0)/customize_opts.sh" - -PACKAGE_LIST_FILE="${SETUP_DIR}/package-list-prod.txt" -PACKAGE_LIST_FILE2="${SETUP_DIR}/package-list-2.txt" -COMPONENTS=`cat $PACKAGE_LIST_FILE | grep -v ' *#' | grep -v '^ *$' | sed '/$/{N;s/\n/ /;}'` +ROOT_FS_DIR="$FLAGS_root" +if [[ -z "$ROOT_FS_DIR" ]]; then + echo "Error: --root is required." + exit 1 +fi +if [[ ! -d "$ROOT_FS_DIR" ]]; then + echo "Error: Root FS does not exist? ($ROOT_FS_DIR)" + exit 1 +fi # Create the temporary apt source.list used to install packages. -cat < /etc/apt/sources.list -deb file:"$SETUP_DIR" local_packages/ -deb $SERVER $SUITE main restricted multiverse universe +APT_SOURCE="${ROOT_FS_DIR}/../sources.list" +cat < "$APT_SOURCE" +deb file:"$FLAGS_setup_dir" local_packages/ +deb $FLAGS_server $FLAGS_suite main restricted multiverse universe EOF +# Cache directory for APT to use. +APT_CACHE_DIR="${FLAGS_output_dir}/tmp/cache/" +mkdir -p "${APT_CACHE_DIR}/archives/partial" + +# Create the apt configuration file. See "man apt.conf" +APT_CONFIG="${ROOT_FS_DIR}/../apt.conf" +cat < "$APT_CONFIG" +Dir +{ + Cache "$APT_CACHE_DIR"; # TODO: Empty string to disable? + Cache { + archives "${APT_CACHE_DIR}/archives"; # TODO: Why do we need this? + }; + Etc + { + sourcelist "$APT_SOURCE" + }; + State "${ROOT_FS_DIR}/var/lib/apt/"; + State + { + status "${ROOT_FS_DIR}/var/lib/dpkg/status"; + }; +}; +DPkg +{ + options {"--root=${ROOT_FS_DIR}";}; +}; +EOF + +# TODO: Full audit of the apt conf dump to make sure things are ok. +apt-config -c="$APT_CONFIG" dump > "${ROOT_FS_DIR}/../apt.conf.dump" + # Install prod packages -apt-get update -apt-get --yes --force-yes install $COMPONENTS +COMPONENTS=`cat $FLAGS_package_list | grep -v ' *#' | grep -v '^ *$' | sed '/$/{N;s/\n/ /;}'` +sudo apt-get -c="$APT_CONFIG" update +sudo apt-get -c="$APT_CONFIG" --yes --force-yes install $COMPONENTS # Create kernel installation configuration to suppress warnings, # install the kernel in /boot, and manage symlinks. -cat < /etc/kernel-img.conf +cat <> /etc/apt/sources.list - apt-get update - apt-get --yes --force-yes --no-install-recommends \ - install $COMPONENTS2 -fi +# Clean up the apt cache. +# TODO: The cache was populated by debootstrap, not these installs. Remove +# this line when we can get debootstrap to stop doing this. +sudo rm -f "${ROOT_FS_DIR}"/var/cache/apt/archives/*.deb # List all packages installed so far, since these are what the local # repository needs to contain. -# TODO: better place to put the list. Must still exist after the chroot -# is dismounted, so build_image.sh can get it. That rules out /tmp and -# $SETUP_DIR (which is under /tmp). -sudo sh -c "/trunk/src/scripts/list_installed_packages.sh \ - > /etc/package_list_installed.txt" - -# Clean up other useless stuff created as part of the install process. -rm -f /var/cache/apt/archives/*.deb - -# List all packages still installed post-pruning -sudo sh -c "/trunk/src/scripts/list_installed_packages.sh \ - > /etc/package_list_pruned.txt" +# TODO: Replace with list_installed_packages.sh when it is fixed up. +dpkg --root="${ROOT_FS_DIR}" -l > \ + "${FLAGS_output_dir}/package_list_installed.txt"