flatcar-scripts/install_packages.sh

#!/bin/bash

# Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Script to install packages into the target root file system.
#
# NOTE: This script should be called by build_image.sh. Do not run this
# on your own unless you know what you are doing.

# Load common constants.  This should be the first executable line.
# The path to common.sh should be relative to your script's location.
. "$(dirname "$0")/common.sh"

# Script must be run inside the chroot
assert_inside_chroot
assert_not_root_user

DEFAULT_PKGLIST="${SRC_ROOT}/package_repo/package-list-prod.txt"

# Flags
DEFINE_string output_dir "" \
  "The location of the output directory to use [REQUIRED]."
DEFINE_string root ""      \
  "The root file system to install packages in."
DEFINE_string arch "x86" \
  "The target architecture to build for. One of { x86, armel }."
DEFINE_string build_root "$DEFAULT_BUILD_ROOT"                \
  "Root of build output"
DEFINE_string package_list "$DEFAULT_PKGLIST" \
  "Comma separated set of package-list files to use."
DEFINE_string mirror "$DEFAULT_IMG_MIRROR" \
  "The upstream package mirror to use."
DEFINE_string suite "$DEFAULT_IMG_SUITE" \
  "The upstream package suite to use."
DEFINE_string mirror2 "" "Additional package mirror to use (URL only)."
DEFINE_string suite2 "" "Package suite for additional mirror."

# Parse command line
FLAGS "$@" || exit 1
eval set -- "${FLAGS_ARGV}"

# Die on any errors.
set -e

KERNEL_DEB_PATH=$(find "${FLAGS_build_root}/${FLAGS_arch}/local_packages" \
  -name "linux-image-*.deb")
KERNEL_DEB=$(basename "${KERNEL_DEB_PATH}" .deb | sed -e 's/linux-image-//' \
  -e 's/_.*//')
KERNEL_VERSION=${KERNEL_VERSION:-${KERNEL_DEB}}

if [[ -z "$FLAGS_output_dir" ]]; then
  echo "Error: --output_dir is required."
  exit 1
fi
OUTPUT_DIR=$(readlink -f "$FLAGS_output_dir")
SETUP_DIR="${OUTPUT_DIR}/local_repo"
ROOT_FS_DIR="${OUTPUT_DIR}/rootfs"
if [[ -n "$FLAGS_root" ]]; then
  ROOT_FS_DIR=$(readlink -f "$FLAGS_root")
fi
mkdir -p "$OUTPUT_DIR" "$SETUP_DIR" "$ROOT_FS_DIR"

# Make sure anything mounted in the rootfs is cleaned up ok on exit.
cleanup_rootfs_mounts() {
  # Occasionally there are some daemons left hanging around that have our
  # root image file system open. We do a best effort attempt to kill them.
  PIDS=`sudo lsof -t "$ROOT_FS_DIR" | sort | uniq`
  for pid in $PIDS
  do
    local cmdline=`cat /proc/$pid/cmdline`
    echo "Killing process that has open file on our rootfs: $cmdline"
    ! sudo kill $pid  # Preceded by ! to disable ERR trap.
  done

  # Sometimes the volatile directory is left mounted and sometimes it is not,
  # so we precede by '!' to disable the ERR trap.
  ! sudo umount "$ROOT_FS_DIR"/lib/modules/2.6.*/volatile/ > /dev/null 2>&1

  sudo umount "${ROOT_FS_DIR}/proc"
}

# Set up repository for locally built packages; these take highest precedence.
mkdir -p "${SETUP_DIR}/local_packages"
cp "${FLAGS_build_root}/${FLAGS_arch}/local_packages"/*.deb \
  "${SETUP_DIR}/local_packages"
cd "$SETUP_DIR"
dpkg-scanpackages local_packages/ /dev/null | \
   gzip > local_packages/Packages.gz
cd -

# Create the temporary apt source.list used to install packages.
APT_SOURCE="${OUTPUT_DIR}/sources.list"
cat <<EOF > "$APT_SOURCE"
deb copy:"$SETUP_DIR" local_packages/
deb $FLAGS_mirror $FLAGS_suite main restricted multiverse universe
EOF
if [ -n "$FLAGS_mirror2" ] && [ -n "$FLAGS_suite2" ]; then
  cat <<EOF >> "$APT_SOURCE"
deb $FLAGS_mirror2 $FLAGS_suite2 main restricted multiverse universe
EOF
fi
# Look for official file and use it if it exists
if [ -f ${SRC_ROOT}/package_repo/sources-official.list ]; then
  cat ${SRC_ROOT}/package_repo/sources-official.list  >> "$APT_SOURCE"
fi

# Cache directory for APT to use. This cache is re-used across builds. We
# rely on the cache to reduce traffic to the hosted repositories.
APT_CACHE_DIR="${FLAGS_build_root}/apt_cache-${FLAGS_arch}/"
mkdir -p "${APT_CACHE_DIR}/archives/partial"

if [ "${FLAGS_arch}" = x86 ]; then
  APT_ARCH=i386
else
  APT_ARCH="${FLAGS_arch}"
fi

# Create the apt configuration file. See "man apt.conf"
APT_PARTS="${OUTPUT_DIR}/apt.conf.d"
mkdir -p "$APT_PARTS"  # An empty apt.conf.d to avoid other configs.
export APT_CONFIG="${OUTPUT_DIR}/apt.conf"
cat <<EOF > "$APT_CONFIG"
APT
{
  Install-Recommends "0";
  Install-Suggests "0";
  Get
  {
    Assume-Yes "1";
    AllowUnauthenticated "1";
  };
  Architecture "${APT_ARCH}";
};
Dir
{
  Bin {
    dpkg "${SCRIPTS_DIR}/dpkg_no_scripts.sh";
  };
  Cache "$APT_CACHE_DIR";
  Cache {
    archives "${APT_CACHE_DIR}/archives";
  };
  Etc
  {
    sourcelist "$APT_SOURCE";
    parts "$APT_PARTS";
  };
  State "${ROOT_FS_DIR}/var/lib/apt/";
  State
  {
    status "${ROOT_FS_DIR}/var/lib/dpkg/status";
  };
};
DPkg
{
  options {"--root=${ROOT_FS_DIR}";};
};
EOF

# TODO: Full audit of the apt conf dump to make sure things are ok.
apt-config dump > "${OUTPUT_DIR}/apt.conf.dump"

# We do a rough equivalent to debootstrap that installs the minimal
# packages needed to be able to run apt to install the rest. We don't
# use debootstrap since it is geared toward having a second stage that
# needs to run package maintainer scripts. This is also simpler.

# The set of required packages before apt can take over.
PACKAGES="debconf libacl1 libattr1 libc6 libgcc1 libselinux1"

# Set of packages that we need to install early so that other packages
# maintainer scripts can still basically run.
#
# login  - So that groupadd will work
# base-passwd/passwd - So that chmod and useradd/groupadd will work
# bash - So that scripts can run
# libpam-runtime/libuuid1 - Not exactly sure why
# sysv-rc - So that we can overwrite invoke-rc.d, update-rc.d
EXTRA_PACKAGES="base-files base-passwd bash libpam-runtime libuuid1 login passwd sysv-rc mawk"

# Prep the rootfs to work with dpgk and apt
sudo mkdir -p "${ROOT_FS_DIR}/var/lib/dpkg/info"
sudo touch "${ROOT_FS_DIR}/var/lib/dpkg/available"   \
  "${ROOT_FS_DIR}/var/lib/dpkg/diversions"           \
  "${ROOT_FS_DIR}/var/lib/dpkg/status"
sudo mkdir -p "${ROOT_FS_DIR}/var/lib/apt/lists/partial"  \
  "${ROOT_FS_DIR}/var/lib/dpkg/updates"

# Download the initial packages into the apt cache if necessary.
REPO="${APT_CACHE_DIR}/archives"
sudo APT_CONFIG="$APT_CONFIG" DEBIAN_FRONTEND=noninteractive apt-get update
sudo APT_CONFIG="$APT_CONFIG" DEBIAN_FRONTEND=noninteractive \
  apt-get --download-only install $PACKAGES $EXTRA_PACKAGES

# Install initial packages directly with dpkg_no_scripts.sh
ARCH="$FLAGS_arch"
if [ "$ARCH" = "x86" ]; then
  ARCH="i?86"  # Match i386 | i686
fi
for p in $PACKAGES $EXTRA_PACKAGES; do
  PKG=$(ls "${REPO}"/${p}_*_$ARCH.deb || /bin/true)
  if [ -z "$PKG" ]; then
    PKG=$(ls "${REPO}"/${p}_*_all.deb)
  fi
  sudo ARCH="$FLAGS_arch" "${SCRIPTS_DIR}"/dpkg_no_scripts.sh \
    --root="$ROOT_FS_DIR" --nodpkg_fallback --unpack "$PKG"
  sudo ARCH="$FLAGS_arch" "${SCRIPTS_DIR}"/dpkg_no_scripts.sh \
    --root="$ROOT_FS_DIR" --nodpkg_fallback --configure "$p"
done

# Make sure that apt is ready to work. We use --fix-broken to trigger apt
# to install additional critical packages. If there are any of these, we
# disable the maintainer scripts so they install ok.
TMP_FORCE_NO_SCRIPTS="-o=DPkg::options::=--nodpkg_fallback"
sudo APT_CONFIG="$APT_CONFIG" DEBIAN_FRONTEND=noninteractive ARCH="$FLAGS_arch"\
  apt-get $TMP_FORCE_NO_SCRIPTS --force-yes --fix-broken install

# TODO: Remove these hacks when we stop having maintainer scripts altogether.
sudo cp -a /dev/* "${ROOT_FS_DIR}/dev"
sudo cp -a /etc/resolv.conf "${ROOT_FS_DIR}/etc/resolv.conf"
sudo ln -sf /bin/true "${ROOT_FS_DIR}/usr/sbin/invoke-rc.d"
sudo ln -sf /bin/true "${ROOT_FS_DIR}/usr/sbin/update-rc.d"

# Set up mounts for working within the rootfs. We copy some basic
# network information from the host so that maintainer scripts can
# access the network as needed.
# TODO: All of this rootfs mount stuff can be removed as soon as we stop
# running the maintainer scripts on install.
sudo mount -t proc proc "${ROOT_FS_DIR}/proc"
trap cleanup_rootfs_mounts EXIT

filter_pkgs() {
  pkglist="$1"
  arch="$2"

  # to read list of package + version skipping empty lines and comments, and
  # convert "foo 1.2-3" to "foo=1.2-3", use:
  #pkgs="$(grep '^[^#]' "$pkglist" | cut -d ' ' -f 1-2 | sed 's/ /=/')"

  # read list of "package [optional arch list]" skipping empty lines and
  # comments
  grep '^[^#]' "$pkglist" | while read pkg archspec; do
    case "$archspec" in
      ""|"[$arch "*|"[$arch]"|*" $arch]")
        echo "$pkg"
      ;;
      *"!$arch "*|*"!$arch]")
        :
      ;;
      "["*"!"*"]")
        echo "$pkg"
      ;;
    esac
  done
}

# Install packages from the given package-lists
PACKAGE_LISTS=$(echo "$FLAGS_package_list" | sed -e 's/,/ /g')
PKG_LIST_ARCH="$FLAGS_arch"
if [ "$PKG_LIST_ARCH" = "x86" ]; then
  PKG_LIST_ARCH="i386"
  FORCE_NO_SCRIPTS=""
else
  # For armel we forcefully disable all maintainer scripts.
  # TODO: Remove this when everything is whitelisted for all build variants.
  FORCE_NO_SCRIPTS="-o=DPkg::options::=--nodpkg_fallback"
fi
for p in $PACKAGE_LISTS; do
  COMPONENTS=$(filter_pkgs "$p" "$PKG_LIST_ARCH")
  sudo APT_CONFIG="$APT_CONFIG" DEBIAN_FRONTEND=noninteractive \
    ARCH="$FLAGS_arch" \
    apt-get $FORCE_NO_SCRIPTS --force-yes install $COMPONENTS
done

# Create kernel installation configuration to suppress warnings,
# install the kernel in /boot, and manage symlinks.
cat <<EOF | sudo dd of="${ROOT_FS_DIR}/etc/kernel-img.conf"
link_in_boot = yes
do_symlinks = yes
minimal_swap = yes
clobber_modules = yes
warn_reboot = no
do_bootloader = no
do_initrd = yes
warn_initrd = no
EOF

# Install the kernel.
# TODO: Support for armel kernels.
if [ "$FLAGS_arch" = "x86" ]; then
  sudo APT_CONFIG="$APT_CONFIG" DEBIAN_FRONTEND=noninteractive ARCH="$FLAGS_arch"\
    apt-get --force-yes install "linux-image-${KERNEL_VERSION}"
fi

# Install optionally present rootfs static data. This can be used to blast
# custom firmware, kernel modules, etc. onto the image.
# TODO: Remove this hack at some point.
LOCAL_ASSETS="${FLAGS_build_root}/${FLAGS_arch}/local_assets"
OPTIONAL_ROOTFS_DATA="${LOCAL_ASSETS}/rootfs_data.tgz"
if [ -f "${OPTIONAL_ROOTFS_DATA}" ]; then
  sudo tar -zxvf "${OPTIONAL_ROOTFS_DATA}" -C "${ROOT_FS_DIR}"
fi

# List all packages installed so far, since these are what the local
# repository needs to contain.
# TODO: Replace with list_installed_packages.sh when it is fixed up.
dpkg --root="${ROOT_FS_DIR}" -l > \
  "${OUTPUT_DIR}/package_list_installed.txt"

cleanup_rootfs_mounts
trap - EXIT