flatcar-scripts/build_image

#!/bin/bash

# Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Script to build a bootable keyfob-based chromeos system image from within
# a chromiumos setup. This assumes that all needed packages have been built into
# the given target's root with binary packages turned on. This script will
# build the Chrome OS image using only pre-built binary packages.

# Load common constants.  This should be the first executable line.
# The path to common.sh should be relative to your script's location.
. "$(dirname "$0")/common.sh"

# Script must be run inside the chroot.
restart_in_chroot_if_needed $*

get_default_board

# Flags.
DEFINE_string board "$DEFAULT_BOARD" \
  "The board to build an image for."
DEFINE_string build_root "/build" \
  "The root location for board sysroots."
DEFINE_integer build_attempt 1 \
  "The build attempt for this image build."
DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \
  "Directory in which to place image result directories (named by version)"
DEFINE_boolean replace $FLAGS_FALSE \
  "Overwrite existing output, if any."
DEFINE_boolean withdev $FLAGS_TRUE \
  "Include useful developer friendly utilities in the image."
DEFINE_boolean installmask $FLAGS_TRUE \
  "Use INSTALL_MASK to shrink the resulting image."
DEFINE_integer jobs -1 \
  "How many packages to build in parallel at maximum."
DEFINE_boolean statefuldev $FLAGS_TRUE \
  "Install development packages on stateful partition rather than the rootfs"
DEFINE_string to "" \
  "The target image file or device"
DEFINE_boolean factory_install $FLAGS_FALSE \
  "Build a smaller image to overlay the factory install shim on; this argument \
is also required in image_to_usb."
DEFINE_string arm_extra_bootargs "" \
  "Additional command line options to pass to the ARM kernel."
DEFINE_boolean recovery $FLAGS_FALSE \
  "Build a recovery image. Default: False."

# Parse command line.
FLAGS "$@" || exit 1
eval set -- "${FLAGS_ARGV}"

# Only now can we die on error.  shflags functions leak non-zero error codes,
# so will die prematurely if 'set -e' is specified before now.
set -e

if [ -z "$FLAGS_board" ] ; then
  error "--board is required."
  exit 1
fi

# Determine build version.
. "${SCRIPTS_DIR}/chromeos_version.sh"

# Use canonical path since some tools (e.g. mount) do not like symlinks.
# Append build attempt to output directory.
IMAGE_SUBDIR="${CHROMEOS_VERSION_STRING}-a${FLAGS_build_attempt}"
OUTPUT_DIR="${FLAGS_output_root}/${FLAGS_board}/${IMAGE_SUBDIR}"
ROOT_FS_DIR="${OUTPUT_DIR}/rootfs"
ROOT_FS_IMG="${OUTPUT_DIR}/rootfs.image"

# If we are creating a developer image, also create a pristine image with a
# different name.
DEVELOPER_IMAGE_NAME=
PRISTINE_IMAGE_NAME=chromiumos_image.bin
if [ "$FLAGS_withdev" -eq "$FLAGS_TRUE" ]; then
  PRISTINE_IMAGE_NAME=chromiumos_base_image.bin
  DEVELOPER_IMAGE_NAME=chromiumos_image.bin
fi
OUTPUT_IMG=${FLAGS_to:-${OUTPUT_DIR}/${PRISTINE_IMAGE_NAME}}

BOARD="${FLAGS_board}"
BOARD_ROOT="${FLAGS_build_root}/${BOARD}"

LOOP_DEV=
STATEFUL_LOOP_DEV=
ESP_LOOP_DEV=

# What cross-build are we targeting?
. "${BOARD_ROOT}/etc/make.conf.board_setup"
LIBC_VERSION=${LIBC_VERSION:-"2.10.1-r1"}

# Figure out ARCH from the given toolchain.
# TODO: Move to common.sh as a function after scripts are switched over.
TC_ARCH=$(echo "$CHOST" | awk -F'-' '{ print $1 }')
case "$TC_ARCH" in
  arm*)
    ARCH="arm"
    ;;
  *86)
    ARCH="x86"
    ;;
  *)
    error "Unable to determine ARCH from toolchain: $CHOST"
    exit 1
esac

# Hack to fix bug where x86_64 CHOST line gets incorrectly added.
# ToDo(msb): remove this hack.
PACKAGES_FILE="${BOARD_ROOT}/packages/Packages"
sudo sed -e "s/CHOST: x86_64-pc-linux-gnu//" -i "${PACKAGES_FILE}"

# Handle existing directory.
if [[ -e "$OUTPUT_DIR" ]]; then
  if [[ $FLAGS_replace -eq $FLAGS_TRUE ]]; then
    sudo rm -rf "$OUTPUT_DIR"
  else
    echo "Directory $OUTPUT_DIR already exists."
    echo "Use --build_attempt option to specify an unused attempt."
    echo "Or use --replace if you want to overwrite this directory."
    exit 1
  fi
fi

# Create the output directory.
mkdir -p "$OUTPUT_DIR"

cleanup_rootfs_loop() {
  sudo umount -d "$ROOT_FS_DIR"
}

cleanup_stateful_fs_loop() {
  sudo umount "${ROOT_FS_DIR}/usr/local"
  sudo umount "${ROOT_FS_DIR}/var"
  sudo umount -d "${STATEFUL_DIR}"
}

cleanup_esp_loop() {
  sudo umount -d "$ESP_DIR"
}

cleanup() {
  # Disable die on error.
  set +e

  if [[ -n "$STATEFUL_LOOP_DEV" ]]; then
    cleanup_stateful_fs_loop
  fi

  if [[ -n "$LOOP_DEV" ]]; then
    cleanup_rootfs_loop
  fi

  if [[ -n "$ESP_LOOP_DEV" ]]; then
    cleanup_esp_loop
  fi

  # Turn die on error back on.
  set -e
}

delete_prompt() {
  echo "An error occurred in your build so your latest output directory" \
    "is invalid."
  read -p "Would you like to delete the output directory (y/N)? " SURE
  SURE="${SURE:0:1}" # Get just the first character
  if [ "${SURE}" == "y" ] ; then
    sudo rm -rf "$OUTPUT_DIR"
    echo "Deleted $OUTPUT_DIR"
  else
    echo "Not deleting $OUTPUT_DIR. Note dev server updates will not work" \
      "until you successfully build another image or delete this directory"
  fi
}

# $1 - Directory where developer rootfs is mounted.
# $2 - Directory where developer stateful_partition is mounted.
developer_cleanup() {
  "$SCRIPTS_DIR/mount_gpt_image.sh" -u -r "$1" -s "$2"
  delete_prompt
}

# Creates a new image based on $OUTPUT_IMG with additional developer packages.
create_developer_image() {
  local root_fs_dir="${OUTPUT_DIR}/rootfs_dev"
  local root_fs_img="${OUTPUT_DIR}/rootfs_dev.image"
  local output_img="${OUTPUT_DIR}/${DEVELOPER_IMAGE_NAME}"

  # Create stateful partition of the same size as the rootfs.
  local stateful_img="$OUTPUT_DIR/stateful_partition_dev.image"
  local stateful_dir="$OUTPUT_DIR/stateful_partition_dev"

  trap "developer_cleanup \"$root_fs_dir\" \"$stateful_dir\"" EXIT

  # Mount a new copy of the base image.
  echo "Creating developer image from base image $OUTPUT_IMG"
  cp "$OUTPUT_IMG" "$output_img"
  $SCRIPTS_DIR/mount_gpt_image.sh --from "$OUTPUT_DIR" \
    --image "$DEVELOPER_IMAGE_NAME" -r "$root_fs_dir" -s "$stateful_dir"

  # Determine the root dir for developer packages.
  local root_dev_dir="$root_fs_dir"
  [ $FLAGS_statefuldev -eq $FLAGS_TRUE ] && \
    root_dev_dir="$root_fs_dir/usr/local"

  # Install developer packages described in chromeos-dev.
  sudo INSTALL_MASK="$INSTALL_MASK" emerge-${BOARD} \
      --root="$root_dev_dir" --root-deps=rdeps \
      --usepkgonly chromeos-dev $EMERGE_JOBS

  # Re-run ldconfig to fix /etc/ldconfig.so.cache.
  sudo /sbin/ldconfig -r "$root_fs_dir"

  # Mark the image as a developer image (input to chromeos_startup).
  sudo mkdir -p "$root_fs_dir/root"
  sudo touch "$root_fs_dir/root/.dev_mode"

  # Additional changes to developer image.

  # The ldd tool is a useful shell script but lives in glibc; just copy it.
  sudo cp -a "$(which ldd)" "${root_dev_dir}/usr/bin"

  # TODO: Temporarily create fake xterm symlink until we do proper xinitrc
  local aterm="$root_fs_dir/usr/local/bin/aterm"
  if [[ -f "${aterm}" ]]; then
    sudo chmod 0755 "$aterm"
    sudo ln -s aterm "${root_fs_dir}/usr/local/bin/xterm"
  fi

  # If vim is installed, then a vi symlink would probably help.
  if [[ -x "${root_fs_dir}/usr/local/bin/vim" ]]; then
    sudo ln -sf vim "${root_fs_dir}/usr/local/bin/vi"
  fi

  # Check that the image has been correctly created.  Only do it if not
  # building a factory install image, as the INSTALL_MASK for it will
  # make test_image fail.
  if [[ $FLAGS_factory_install -eq ${FLAGS_FALSE} ]] ; then
    "${SCRIPTS_DIR}/test_image" \
      --root="$root_fs_dir" \
      --target="$ARCH"
  fi

  trap - EXIT
  $SCRIPTS_DIR/mount_gpt_image.sh -u -r "$root_fs_dir" -s "$stateful_dir"
  sudo rm -rf "$root_fs_dir" "$stateful_dir"
  echo "Developer image built and stored at $output_img"
}

# ${DEV_IMAGE_ROOT} specifies the location of where developer packages will
# be installed on the stateful dir.  On a Chromium OS system, this will
# translate to /usr/local
DEV_IMAGE_ROOT=

trap "cleanup && delete_prompt" EXIT

mkdir -p "$ROOT_FS_DIR"

# Create and format the root file system.

# Check for loop device before creating image.
LOOP_DEV=$(sudo losetup -f)
if [ -z "$LOOP_DEV" ] ; then
  echo "No free loop device.  Free up a loop device or reboot.  exiting. "
  exit 1
fi

# Create root file system disk image to fit on a 1GB memory stick.
# 1 GB in hard-drive-manufacturer-speak is 10^9, not 2^30.  950MB < 10^9 bytes.
if [[ $FLAGS_factory_install -eq ${FLAGS_TRUE} ]] ; then
  ROOT_SIZE_BYTES=$((1024 * 1024 * 300))
else
  ROOT_SIZE_BYTES=$((1024 * 1024 * 720))
fi

dd if=/dev/zero of="$ROOT_FS_IMG" bs=1 count=1 seek=$((ROOT_SIZE_BYTES - 1))
sudo losetup "$LOOP_DEV" "$ROOT_FS_IMG"
sudo mkfs.ext3 "$LOOP_DEV"

# Tune and mount rootfs.
UUID=$(uuidgen)
DISK_LABEL="C-KEYFOB"
sudo tune2fs -L "$DISK_LABEL" -U "$UUID" -c 0 -i 0 "$LOOP_DEV"
sudo mount "$LOOP_DEV" "$ROOT_FS_DIR"

# Create stateful partition of the same size as the rootfs.
STATEFUL_IMG="$OUTPUT_DIR/stateful_partition.image"
STATEFUL_DIR="$OUTPUT_DIR/stateful_partition"
STATEFUL_LOOP_DEV=$(sudo losetup -f)
if [ -z "$STATEFUL_LOOP_DEV" ] ; then
  echo "No free loop device.  Free up a loop device or reboot.  exiting. "
  exit 1
fi
dd if=/dev/zero of="$STATEFUL_IMG" bs=1 count=1 seek=$((ROOT_SIZE_BYTES - 1))
sudo losetup "$STATEFUL_LOOP_DEV" "$STATEFUL_IMG"
sudo mkfs.ext3 "$STATEFUL_LOOP_DEV"
sudo tune2fs -L "C-STATE" -U "$UUID" -c 0 -i 0 \
    "$STATEFUL_LOOP_DEV"

# Mount the stateful partition.
mkdir -p "$STATEFUL_DIR"
sudo mount "$STATEFUL_LOOP_DEV" "$STATEFUL_DIR"

# Set dev image root now that we have mounted the stateful partition we created
DEV_IMAGE_ROOT="$STATEFUL_DIR/dev_image"

# Turn root file system into bootable image.
if [[ "$ARCH" = "x86" ]]; then
  # Setup extlinux configuration.
  # TODO: For some reason the /dev/disk/by-uuid is not being generated by udev
  # in the initramfs. When we figure that out, switch to root=UUID=$UUID.
  sudo mkdir -p "$ROOT_FS_DIR"/boot
  # TODO(adlr): use initramfs for booting.
  cat <<EOF | sudo dd of="$ROOT_FS_DIR"/boot/extlinux.conf
DEFAULT chromeos-usb
PROMPT 0
TIMEOUT 0

label chromeos-usb
  menu label chromeos-usb
  kernel vmlinuz
  append quiet console=tty2 init=/sbin/init boot=local rootwait root=/dev/sdb3 ro noresume noswap i915.modeset=1 loglevel=1

label chromeos-hd
  menu label chromeos-hd
  kernel vmlinuz
  append quiet console=tty2 init=/sbin/init boot=local rootwait root=HDROOT ro noresume noswap i915.modeset=1 loglevel=1
EOF

  # Make partition bootable and label it.
  sudo extlinux -z --install "${ROOT_FS_DIR}/boot"
fi

# -- Install packages into the root file system --

# We need to install libc manually from the cross toolchain.
# TODO: Improve this? We only want libc and not the whole toolchain.
PKGDIR="/var/lib/portage/pkgs/cross/"
sudo tar jxvpf \
  "${PKGDIR}/${CHOST}/cross-${CHOST}"/glibc-${LIBC_VERSION}.tbz2 \
  -C "$ROOT_FS_DIR" --strip-components=3 \
  --exclude=usr/include --exclude=sys-include --exclude=*.a --exclude=*.o

# We need to install libstdc++ manually from the cross toolchain.
# TODO: Figure out a better way of doing this?
sudo cp -a "${BOARD_ROOT}"/lib/libgcc_s.so* "${ROOT_FS_DIR}/lib"
sudo cp -a "${BOARD_ROOT}"/usr/lib/libstdc++.so* "${ROOT_FS_DIR}/usr/lib"

INSTALL_MASK=""
if [[ $FLAGS_installmask -eq ${FLAGS_TRUE} ]] ; then
  INSTALL_MASK="$DEFAULT_INSTALL_MASK"
fi

# Reduce the size of factory install shim.
# TODO: Build a separated ebuild for the factory install shim to reduce size.
if [[ $FLAGS_factory_install -eq ${FLAGS_TRUE} ]] ; then
  INSTALL_MASK="$INSTALL_MASK $FACTORY_INSTALL_MASK"
fi

if [[ $FLAGS_jobs -ne -1 ]]; then
  EMERGE_JOBS="--jobs=$FLAGS_jobs"
fi

# Prepare stateful partition with some pre-created directories
sudo mkdir -p "${DEV_IMAGE_ROOT}"
sudo mkdir -p "${STATEFUL_DIR}/var"

# Create symlinks so that /usr/local/usr based directories are symlinked to
# /usr/local/ directories e.g. /usr/local/usr/bin -> /usr/local/bin, etc.
setup_symlinks_on_root "${DEV_IMAGE_ROOT}" "${STATEFUL_DIR}/var" \
  "${STATEFUL_DIR}"

# Perform binding rather than symlinking because directories must exist
# on rootfs so that we can bind at run-time since rootfs is read-only
echo "Binding directories from stateful partition onto the rootfs"
sudo mkdir -p "${ROOT_FS_DIR}/usr/local"
sudo mount --bind "${DEV_IMAGE_ROOT}" "${ROOT_FS_DIR}/usr/local"
sudo mkdir -p "${ROOT_FS_DIR}/var"
sudo mount --bind "${STATEFUL_DIR}/var" "${ROOT_FS_DIR}/var"

# We "emerge --root=$ROOT_FS_DIR --root-deps=rdeps --usepkgonly" all of the
# runtime packages for chrome os. This builds up a chrome os image from binary
# packages with runtime dependencies only.  We use INSTALL_MASK to trim the
# image size as much as possible.
sudo INSTALL_MASK="$INSTALL_MASK" emerge-${BOARD} \
  --root="$ROOT_FS_DIR" --root-deps=rdeps \
  --usepkgonly chromeos $EMERGE_JOBS

# Extract the kernel from the root filesystem for use by the GPT image. Legacy
# BIOS will use the kernel in the rootfs (via syslinux), Chrome OS BIOS will
# use the kernel partition.
sudo cp -f "${ROOT_FS_DIR}/boot/vmlinuz" "${OUTPUT_DIR}/vmlinuz.image"

# Create EFI System Partition to boot stock EFI BIOS (but not ChromeOS EFI
# BIOS). We only need this for x86, but it's simpler and safer to keep the disk
# images the same for both x86 and ARM.
ESP_IMG=${OUTPUT_DIR}/esp.image
# NOTE: The size argument for mkfs.vfat is in 1024-byte blocks. We'll hard-code
# it to 16M for now.
ESP_BLOCKS=16384
/usr/sbin/mkfs.vfat -C ${OUTPUT_DIR}/esp.image ${ESP_BLOCKS}
ESP_DIR=${OUTPUT_DIR}/esp
ESP_LOOP_DEV=$(sudo losetup -f)
if [ -z "$ESP_LOOP_DEV" ] ; then
  echo "No free loop device.  Free up a loop device or reboot.  exiting. "
  exit 1
fi
mkdir -p "${ESP_DIR}"
sudo losetup "${ESP_LOOP_DEV}" "${ESP_IMG}"
sudo mount "${ESP_LOOP_DEV}" "${ESP_DIR}"
sudo mkdir -p "${ESP_DIR}/efi/boot"
sudo grub-mkimage -p /efi/boot -o "${ESP_DIR}/efi/boot/bootx64.efi" \
  part_gpt fat ext2 normal boot sh chain configfile linux
cat <<'EOF' | sudo dd of="${ESP_DIR}/efi/boot/grub.cfg"
set default=0
set timeout=2

# NOTE: These magic grub variables are a Chrome OS hack. They are not portable.

menuentry "local image A" {
  linux $grubpartA/boot/vmlinuz quiet console=tty2 init=/sbin/init boot=local rootwait root=/dev/$linuxpartA ro noresume noswap i915.modeset=1 loglevel=1
}

menuentry "local image B" {
  linux $grubpartB/boot/vmlinuz quiet console=tty2 init=/sbin/init boot=local rootwait root=/dev/$linuxpartB ro noresume noswap i915.modeset=1 loglevel=1
}

EOF

# Perform any customizations on the root file system that are needed.
"${SCRIPTS_DIR}/customize_rootfs" \
  --root="$ROOT_FS_DIR" \
  --target="$ARCH" \
  --board="$BOARD"

# Don't test the factory install shim.
if [[ $FLAGS_factory_install -eq ${FLAGS_FALSE} ]] ; then
  # Check that the image has been correctly created.
  "${SCRIPTS_DIR}/test_image" \
    --root="$ROOT_FS_DIR" \
    --target="$ARCH"
fi

# Clean up symlinks so they work on a running target rooted at "/".
# Here development packages are rooted at /usr/local.  However, do not
# create /usr/local or /var on host (already exist on target).
setup_symlinks_on_root "/usr/local" "/var" "${STATEFUL_DIR}"

# Cleanup loop devices.
cleanup

trap delete_prompt EXIT

RECOVERY="--norecovery"
if [[ ${FLAGS_recovery} -eq $FLAGS_TRUE ]]; then
  RECOVERY="--recovery"
fi

# Create the GPT-formatted image
${SCRIPTS_DIR}/build_gpt.sh \
  --arch=${ARCH} \
  --board=${FLAGS_board} \
  --arm_extra_bootargs="${FLAGS_arm_extra_bootargs}" \
  ${RECOVERY} \
  "${OUTPUT_DIR}" \
  "${OUTPUT_IMG}"

# Clean up temporary files.
rm -f "${ROOT_FS_IMG}" "${STATEFUL_IMG}" "${OUTPUT_DIR}/vmlinuz.image" \
  "${ESP_IMG}"
rmdir "${ROOT_FS_DIR}" "${STATEFUL_DIR}" "${ESP_DIR}"

OUTSIDE_OUTPUT_DIR="../build/images/${FLAGS_board}/${IMAGE_SUBDIR}"

# Create a developer image based on the chromium os base image
[ "$FLAGS_withdev" -eq "$FLAGS_TRUE" ] && create_developer_image
trap - EXIT

echo "Done.  Image created in ${OUTPUT_DIR}"
echo "Chromium OS image created as $PRISTINE_IMAGE_NAME"
if [ "$FLAGS_withdev" -eq "$FLAGS_TRUE" ]; then
  echo "Developer image created as $DEVELOPER_IMAGE_NAME"
fi
echo "To copy to USB keyfob, OUTSIDE the chroot, do something like:"
echo "  ./image_to_usb.sh --from=${OUTSIDE_OUTPUT_DIR} --to=/dev/sdX"
echo "To convert to VMWare image, OUTSIDE the chroot, do something like:"
echo "  ./image_to_vmware.sh --from=${OUTSIDE_OUTPUT_DIR}"
echo "from the scripts directory where you entered the chroot."