mirror of
https://github.com/flatcar/scripts.git
synced 2025-08-11 23:16:58 +02:00
Use local apt cache instead of local_repo when mastering an image.
This change switches to mastering an image without using debootstrap. We turn on the previously experimental bits that install a small set of packages manually before handing things over to apt. In both cases we use apt to download the packages so that it can populate it's local package cache. With this change we will no longer depend on the local_repo when mastering an image. Developers will not have to rebuild their local repo when repo_list_image.txt changes. Instead we will use and lazy-fill the local apt-cache. The first time you build_image.sh it will be slow since it needs to download the packages. Subsequent runs should be as before since it will use the local cache. If packages are added they should be lazily fetched in the next image build. Until we have a fully populated external mirror, developers will still have to add packages to repo_list_image.txt. Also, until make_chroot is switched over to use the external repo then developers will have to redo their local repo when the repo_list_dev.txt changes. Review URL: http://codereview.chromium.org/521073
This commit is contained in:
tedbo
parent
58e02408a6
commit
4f44d9e104
@ -33,7 +33,7 @@ DEFINE_boolean replace $FLAGS_FALSE "Overwrite existing output, if any."
|
||||
DEFINE_boolean increment $FLAGS_FALSE \
|
||||
"Picks the latest build and increments the minor version by one."
|
||||
|
||||
DEFINE_string mirror "$DEFAULT_IMG_MIRROR" "Repository mirror to use."
|
||||
DEFINE_string mirror "$DEFAULT_EXT_MIRROR" "Repository mirror to use."
|
||||
DEFINE_string suite "$DEFAULT_IMG_SUITE" "Repository suite to base image on."
|
||||
DEFINE_string pkglist "$DEFAULT_PKGLIST" \
|
||||
"Name of file listing packages to install from repository."
|
||||
|
||||
10
common.sh
10
common.sh
@ -31,7 +31,7 @@ then
|
||||
# Using bash, so we can find ourselves
|
||||
GCLIENT_ROOT="$(dirname "$BASH_SOURCE")/../.."
|
||||
else
|
||||
# Using dash or sh, we don't know where we are. $0 refers to the calling
|
||||
# Using dash or sh, we don't know where we are. $0 refers to the calling
|
||||
# script, not ourselves, so that doesn't help us.
|
||||
echo "Unable to determine location for common.sh. If you are sourcing"
|
||||
echo "common.sh from a script run via dash or sh, you must do it in the"
|
||||
@ -45,9 +45,9 @@ fi
|
||||
|
||||
# Canonicalize the directories for the root dir and the calling script.
|
||||
# readlink is part of coreutils and should be present even in a bare chroot.
|
||||
# This is better than just using
|
||||
# This is better than just using
|
||||
# FOO = "$(cd $FOO ; pwd)"
|
||||
# since that leaves symbolic links intact.
|
||||
# since that leaves symbolic links intact.
|
||||
# Note that 'realpath' is equivalent to 'readlink -f'.
|
||||
TOP_SCRIPT_DIR=`readlink -f $TOP_SCRIPT_DIR`
|
||||
GCLIENT_ROOT=`readlink -f $GCLIENT_ROOT`
|
||||
@ -90,7 +90,7 @@ DEFAULT_IMG_SUITE=${CHROMEOS_IMG_SUITE:-"chromeos"}
|
||||
# Package repositories (mirrors)
|
||||
DEFAULT_EXT_MIRROR=${CHROMEOS_EXT_MIRROR:-"http://build.chromium.org/buildbot/packages"}
|
||||
DEFAULT_DEV_MIRROR=${CHROMEOS_DEV_MIRROR:-"file://$GCLIENT_ROOT/repo/apt"}
|
||||
DEFAULT_IMG_MIRROR=${CHROMEOS_IMG_MIRROR:-"file:///home/$USER/trunk/repo/apt"}
|
||||
DEFAULT_IMG_MIRROR=${CHROMEOS_IMG_MIRROR:-"copy:///home/$USER/trunk/repo/apt"}
|
||||
|
||||
# Default location for chroot
|
||||
DEFAULT_CHROOT_DIR=${CHROMEOS_CHROOT_DIR:-"$GCLIENT_ROOT/chroot"}
|
||||
@ -120,7 +120,7 @@ function make_pkg_common {
|
||||
PKG_BASE=${1:?}
|
||||
shift
|
||||
set +e
|
||||
|
||||
|
||||
# All packages are built in the chroot
|
||||
assert_inside_chroot
|
||||
|
||||
|
||||
@ -30,7 +30,7 @@ DEFINE_string build_root "$DEFAULT_BUILD_ROOT" \
|
||||
"Root of build output"
|
||||
DEFINE_string package_list "$DEFAULT_PKGLIST" \
|
||||
"The package list file to use."
|
||||
DEFINE_string server "$DEFAULT_IMG_MIRROR" \
|
||||
DEFINE_string server "$DEFAULT_EXT_MIRROR" \
|
||||
"The package server to use."
|
||||
DEFINE_string suite "$DEFAULT_IMG_SUITE" \
|
||||
"The package suite to use."
|
||||
@ -146,113 +146,97 @@ EOF
|
||||
# TODO: Full audit of the apt conf dump to make sure things are ok.
|
||||
apt-config dump > "${OUTPUT_DIR}/apt.conf.dump"
|
||||
|
||||
# Add debootstrap link for the suite, if it doesn't exist.
|
||||
if [ ! -e "/usr/share/debootstrap/scripts/$FLAGS_suite" ]
|
||||
then
|
||||
sudo ln -s /usr/share/debootstrap/scripts/jaunty \
|
||||
"/usr/share/debootstrap/scripts/$FLAGS_suite"
|
||||
fi
|
||||
# We do a rough equivalent to debootstrap that installs the minimal
|
||||
# packages needed to be able to run apt to install the rest. We don't
|
||||
# use debootstrap since it is geared toward having a second stage that
|
||||
# needs to run package maintainer scripts. This is also simpler.
|
||||
|
||||
if [ -z "$EXPERIMENTAL_NO_DEBOOTSTRAP" -a \
|
||||
-z "$EXPERIMENTAL_NO_MAINTAINER_SCRIPTS" ]; then
|
||||
# Use debootstrap, which runs maintainer scripts.
|
||||
sudo debootstrap --arch=i386 $FLAGS_suite "$ROOT_FS_DIR" "${FLAGS_server}"
|
||||
sudo APT_CONFIG="$APT_CONFIG" DEBIAN_FRONTEND=noninteractive apt-get update
|
||||
else
|
||||
# We do a rough equivalent to debootstrap that installs the minimal
|
||||
# packages needed to be able to run apt to install the rest. We don't
|
||||
# use debootstrap since it is geared toward having a second stage that
|
||||
# needs to run package maintainer scripts. This is also simpler.
|
||||
# The set of required packages before apt can take over.
|
||||
PACKAGES="debconf libacl1 libattr1 libc6 libgcc1 libselinux1"
|
||||
|
||||
# The set of required packages before apt can take over.
|
||||
PACKAGES="debconf libacl1 libattr1 libc6 libgcc1 libselinux1"
|
||||
# Set of packages that we need to install early so that other packages
|
||||
# maintainer scripts can still basically run.
|
||||
#
|
||||
# login - So that groupadd will work
|
||||
# base-passwd/passwd - So that chmod and useradd/groupadd will work
|
||||
# bash - So that scripts can run
|
||||
# libpam-runtime/libuuid1 - Not exactly sure why
|
||||
# sysv-rc - So that we can overwrite invoke-rc.d, update-rc.d
|
||||
EXTRA_PACKAGES="base-files base-passwd bash libpam-runtime libuuid1 login passwd sysv-rc"
|
||||
|
||||
# Set of packages that we need to install early so that other packages
|
||||
# maintainer scripts can still basically run.
|
||||
#
|
||||
# login - So that groupadd will work
|
||||
# base-passwd/passwd - So that chmod and useradd/groupadd will work
|
||||
# bash - So that scripts can run
|
||||
# libpam-runtim/libuuid1 - Not exactly sure why
|
||||
# sysv-rc - So that we can overwrite invoke-rc.d, update-rc.d
|
||||
EXTRA_PACKAGES="base-files base-passwd bash libpam-runtime libuuid1 login passwd sysv-rc"
|
||||
# Prep the rootfs to work with dpgk and apt
|
||||
sudo mkdir -p "${ROOT_FS_DIR}/var/lib/dpkg/info"
|
||||
sudo touch "${ROOT_FS_DIR}/var/lib/dpkg/available" \
|
||||
"${ROOT_FS_DIR}/var/lib/dpkg/diversions" \
|
||||
"${ROOT_FS_DIR}/var/lib/dpkg/status"
|
||||
sudo mkdir -p "${ROOT_FS_DIR}/var/lib/apt/lists/partial" \
|
||||
"${ROOT_FS_DIR}/var/lib/dpkg/updates"
|
||||
|
||||
# Prep the rootfs to work with dpgk and apt
|
||||
sudo mkdir -p "${ROOT_FS_DIR}/var/lib/dpkg/info"
|
||||
sudo touch "${ROOT_FS_DIR}/var/lib/dpkg/available" \
|
||||
"${ROOT_FS_DIR}/var/lib/dpkg/diversions" \
|
||||
"${ROOT_FS_DIR}/var/lib/dpkg/status"
|
||||
sudo mkdir -p "${ROOT_FS_DIR}/var/lib/apt/lists/partial" \
|
||||
"${ROOT_FS_DIR}/var/lib/dpkg/updates"
|
||||
# Download the initial packages into the apt cache if necessary.
|
||||
REPO="${APT_CACHE_DIR}/archives"
|
||||
sudo APT_CONFIG="$APT_CONFIG" DEBIAN_FRONTEND=noninteractive apt-get update
|
||||
sudo APT_CONFIG="$APT_CONFIG" DEBIAN_FRONTEND=noninteractive \
|
||||
apt-get --download-only install $PACKAGES $EXTRA_PACKAGES
|
||||
|
||||
# Download the initial packages into the apt cache if necessary.
|
||||
REPO="${APT_CACHE_DIR}/archives"
|
||||
sudo APT_CONFIG="$APT_CONFIG" DEBIAN_FRONTEND=noninteractive apt-get update
|
||||
sudo APT_CONFIG="$APT_CONFIG" DEBIAN_FRONTEND=noninteractive \
|
||||
apt-get --download-only install $PACKAGES $EXTRA_PACKAGES
|
||||
# Install initial packages directly with dpkg_no_scripts.sh
|
||||
for p in $PACKAGES $EXTRA_PACKAGES; do
|
||||
PKG=$(ls "${REPO}"/${p}_*_i386.deb || /bin/true)
|
||||
if [ -z "$PKG" ]; then
|
||||
PKG=$(ls "${REPO}"/${p}_*_all.deb)
|
||||
fi
|
||||
sudo "${SCRIPTS_DIR}"/dpkg_no_scripts.sh \
|
||||
--root="$ROOT_FS_DIR" --unpack "$PKG"
|
||||
done
|
||||
|
||||
i=0
|
||||
for p in $PACKAGES $EXTRA_PACKAGES; do
|
||||
set +e
|
||||
PKG=$(ls "${REPO}"/${p}_*_i386.deb)
|
||||
set -e
|
||||
if [ -z "$PKG" ]; then
|
||||
PKG=$(ls "${REPO}"/${p}_*_all.deb)
|
||||
fi
|
||||
echo "Installing package: $PKG [$i]"
|
||||
sudo "${SCRIPTS_DIR}"/dpkg_no_scripts.sh \
|
||||
--root="$ROOT_FS_DIR" --unpack "$PKG"
|
||||
i=$((i + 1))
|
||||
done
|
||||
# Make sure that apt is ready to work. We use --fix-broken to trigger apt
|
||||
# to install additional critical packages. If there are any of these, we
|
||||
# disable the maintainer scripts so they install ok.
|
||||
TMP_FORCE_NO_SCRIPTS="-o=Dir::Bin::dpkg=${SCRIPTS_DIR}/dpkg_no_scripts.sh"
|
||||
sudo APT_CONFIG="$APT_CONFIG" DEBIAN_FRONTEND=noninteractive \
|
||||
apt-get $TMP_FORCE_NO_SCRIPTS --force-yes --fix-broken install
|
||||
|
||||
# Make sure that apt is ready to work. We use --fix-broken to trigger apt
|
||||
# to install additional critical packages. If there are any of these, we
|
||||
# disable the maintainer scripts so they install ok.
|
||||
TMP_FORCE_NO_SCRIPTS="-o=Dir::Bin::dpkg=${SCRIPTS_DIR}/dpkg_no_scripts.sh"
|
||||
sudo APT_CONFIG="$APT_CONFIG" DEBIAN_FRONTEND=noninteractive \
|
||||
apt-get $TMP_FORCE_NO_SCRIPTS --force-yes --fix-broken install
|
||||
# ----- MAINTAINER SCRIPT FIXUPS -----
|
||||
|
||||
# ----- MAINTAINER SCRIPT FIXUPS -----
|
||||
# TODO: Remove when we stop having maintainer scripts altogether.
|
||||
sudo cp -a /dev/* "${ROOT_FS_DIR}/dev"
|
||||
sudo cp -a /etc/resolv.conf "${ROOT_FS_DIR}/etc/resolv.conf"
|
||||
sudo ln -sf /bin/true "${ROOT_FS_DIR}/usr/sbin/invoke-rc.d"
|
||||
sudo ln -sf /bin/true "${ROOT_FS_DIR}/usr/sbin/update-rc.d"
|
||||
|
||||
# TODO: Remove when we stop having maintainer scripts altogether.
|
||||
sudo cp -a /dev/* "${ROOT_FS_DIR}/dev"
|
||||
sudo cp -a /etc/resolv.conf "${ROOT_FS_DIR}/etc/resolv.conf"
|
||||
sudo ln -sf /bin/true "${ROOT_FS_DIR}/usr/sbin/invoke-rc.d"
|
||||
sudo ln -sf /bin/true "${ROOT_FS_DIR}/usr/sbin/update-rc.d"
|
||||
# base-files
|
||||
# TODO: Careful audit of the postinst; this isn't all that is there.
|
||||
sudo cp -a "${ROOT_FS_DIR}/usr/share/base-files/networks" \
|
||||
"${ROOT_FS_DIR}/usr/share/base-files/nsswitch.conf" \
|
||||
"${ROOT_FS_DIR}/usr/share/base-files/profile" \
|
||||
"${ROOT_FS_DIR}/etc/"
|
||||
|
||||
# base-files
|
||||
# TODO: Careful audit of the postinst; this isn't all that is there.
|
||||
sudo cp -a "${ROOT_FS_DIR}/usr/share/base-files/networks" \
|
||||
"${ROOT_FS_DIR}/usr/share/base-files/nsswitch.conf" \
|
||||
"${ROOT_FS_DIR}/usr/share/base-files/profile" \
|
||||
"${ROOT_FS_DIR}/etc/"
|
||||
# base-passwd
|
||||
sudo cp "${ROOT_FS_DIR}/usr/share/base-passwd/passwd.master" \
|
||||
"${ROOT_FS_DIR}/etc/passwd"
|
||||
sudo cp "${ROOT_FS_DIR}/usr/share/base-passwd/group.master" \
|
||||
"${ROOT_FS_DIR}/etc/group"
|
||||
|
||||
# base-passwd
|
||||
sudo cp "${ROOT_FS_DIR}/usr/share/base-passwd/passwd.master" \
|
||||
"${ROOT_FS_DIR}/etc/passwd"
|
||||
sudo cp "${ROOT_FS_DIR}/usr/share/base-passwd/group.master" \
|
||||
"${ROOT_FS_DIR}/etc/group"
|
||||
# libpam-runtime
|
||||
# The postinst script calls pam-auth-update, which is a perl script that
|
||||
# expects to run within the targetfs. Until we fix this, we just copy
|
||||
# from the build chroot.
|
||||
sudo cp -a /etc/pam.d/common-* \
|
||||
/etc/pam.d/login \
|
||||
/etc/pam.d/newusers \
|
||||
/etc/pam.d/su \
|
||||
/etc/pam.d/sudo \
|
||||
"${ROOT_FS_DIR}/etc/pam.d/"
|
||||
|
||||
# libpam-runtime
|
||||
# The postinst script calls pam-auth-update, which is a perl script that
|
||||
# expects to run within the targetfs. Until we fix this, we just copy
|
||||
# from the build chroot.
|
||||
sudo cp -a /etc/pam.d/common-* \
|
||||
/etc/pam.d/login \
|
||||
/etc/pam.d/newusers \
|
||||
/etc/pam.d/su \
|
||||
/etc/pam.d/sudo \
|
||||
"${ROOT_FS_DIR}/etc/pam.d/"
|
||||
# mawk
|
||||
sudo ln -s mawk "${ROOT_FS_DIR}/usr/bin/awk"
|
||||
|
||||
# mawk
|
||||
sudo ln -s mawk "${ROOT_FS_DIR}/usr/bin/awk"
|
||||
# base-files?
|
||||
sudo touch "${ROOT_FS_DIR}/etc/fstab"
|
||||
|
||||
# base-files?
|
||||
sudo touch "${ROOT_FS_DIR}/etc/fstab"
|
||||
# sysv-rc needs this
|
||||
sudo mkdir -p "${ROOT_FS_DIR}/etc/init.d"
|
||||
|
||||
# sysv-rc needs this
|
||||
sudo mkdir -p "${ROOT_FS_DIR}/etc/init.d"
|
||||
fi # EXPERIMENTAL_NO_DEBOOTSTRAP
|
||||
# ----- END MAINTAINER SCRIPT FIXUPS -----
|
||||
|
||||
# Set up mounts for working within the rootfs. We copy some basic
|
||||
# network information from the host so that maintainer scripts can
|
||||
|
||||
Loading…
Reference in New Issue
Block a user