flatcar-scripts/build_image

#!/bin/bash

# Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Script to build a bootable keyfob-based chromeos system image from within
# a chromiumos setup. This assumes that all needed packages have been built into
# the given target's root with binary packages turned on. This script will
# build the Chrome OS image using only pre-built binary packages.

# Load common constants.  This should be the first executable line.
# The path to common.sh should be relative to your script's location.
. "$(dirname "$0")/common.sh"

# Script must be run inside the chroot.
assert_inside_chroot

get_default_board

# Flags.
DEFINE_string board "$DEFAULT_BOARD" \
  "The board to build an image for."
DEFINE_string build_root "/build" \
  "The root location for board sysroots."
DEFINE_integer build_attempt 1 \
  "The build attempt for this image build."
DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \
  "Directory in which to place image result directories (named by version)"
DEFINE_boolean replace $FLAGS_FALSE \
  "Overwrite existing output, if any."
DEFINE_boolean withdev $FLAGS_TRUE \
  "Include useful developer friendly utilities in the image."
DEFINE_boolean installmask $FLAGS_TRUE \
  "Use INSTALL_MASK to shrink the resulting image."
DEFINE_integer jobs -1 \
  "How many packages to build in parallel at maximum."
DEFINE_boolean statefuldev $FLAGS_FALSE \
  "Install development packages on stateful partition -- still experimental"
DEFINE_boolean withtest $FLAGS_FALSE \
  "Include packages required for testing and prepare image for testing"

# Parse command line.
FLAGS "$@" || exit 1
eval set -- "${FLAGS_ARGV}"

# Only now can we die on error.  shflags functions leak non-zero error codes,
# so will die prematurely if 'set -e' is specified before now.
set -e

if [ -z "$FLAGS_board" ] ; then
  error "--board is required."
  exit 1
fi

# Sanity check: statefuldev cannot be true if withdev is false.
if [ $FLAGS_statefuldev -eq $FLAGS_TRUE ] && 
    [ $FLAGS_withdev -eq $FLAGS_FALSE ] ; then
  echo "ERROR: statefuldev flag cannot be set to true without withdev"
  exit 1
fi

# Determine build version.
. "${SCRIPTS_DIR}/chromeos_version.sh"

# Use canonical path since some tools (e.g. mount) do not like symlinks.
# Append build attempt to output directory.
IMAGE_SUBDIR="${CHROMEOS_VERSION_STRING}-a${FLAGS_build_attempt}"
OUTPUT_DIR="${FLAGS_output_root}/${FLAGS_board}/${IMAGE_SUBDIR}"
ROOT_FS_DIR="${OUTPUT_DIR}/rootfs"
ROOT_FS_IMG="${OUTPUT_DIR}/rootfs.image"
MBR_IMG="${OUTPUT_DIR}/mbr.image"
OUTPUT_IMG="${OUTPUT_DIR}/usb.img"

BOARD="${FLAGS_board}"
BOARD_ROOT="${FLAGS_build_root}/${BOARD}"

LOOP_DEV=

# What cross-build are we targeting?
. "${BOARD_ROOT}/etc/make.conf.board_setup"
LIBC_VERSION=${LIBC_VERSION:-"2.10.1-r1"}

# Figure out ARCH from the given toolchain.
# TODO: Move to common.sh as a function after scripts are switched over.
TC_ARCH=$(echo "$CHOST" | awk -F'-' '{ print $1 }')
case "$TC_ARCH" in
  arm*)
    ARCH="arm"
    ;;
  *86)
    ARCH="x86"
    ;;
  *)
    error "Unable to determine ARCH from toolchain: $CHOST"
    exit 1
esac

# Hack to fix bug where x86_64 CHOST line gets incorrectly added.
# ToDo(msb): remove this hack.
PACKAGES_FILE="${BOARD_ROOT}/packages/Packages"
sudo sed -e "s/CHOST: x86_64-pc-linux-gnu//" -i "${PACKAGES_FILE}"

# Handle existing directory.
if [[ -e "$OUTPUT_DIR" ]]; then
  if [[ $FLAGS_replace -eq $FLAGS_TRUE ]]; then
    sudo rm -rf "$OUTPUT_DIR"
  else
    echo "Directory $OUTPUT_DIR already exists."
    echo "Use --build_attempt option to specify an unused attempt."
    echo "Or use --replace if you want to overwrite this directory."
    exit 1
  fi
fi

# Create the output directory.
mkdir -p "$OUTPUT_DIR"

cleanup_rootfs_loop() {
  sudo umount "$LOOP_DEV"
  sleep 1  # in case $LOOP_DEV is in use (TODO: Try umount -l?).
  sudo losetup -d "$LOOP_DEV"
}

cleanup_stateful_fs_loop() {
  sudo umount "$STATEFUL_LOOP_DEV"
  sleep 1  # follows from cleanup_root_fs_loop.
  sudo losetup -d "$STATEFUL_LOOP_DEV"
}

cleanup() {
  # Disable die on error.
  set +e

  # Unmount stateful partition from usr/local if bound.
  if [ -s "$ROOT_FS_DIR/usr/local/bin" ] ; then
    sudo umount $ROOT_FS_DIR/usr/local
  fi

  if [[ -n "$STATEFUL_LOOP_DEV" ]]; then
    cleanup_stateful_fs_loop
  fi

  if [[ -n "$LOOP_DEV" ]]; then
    cleanup_rootfs_loop
  fi

  # Turn die on error back on.
  set -e
}

trap cleanup EXIT

mkdir -p "$ROOT_FS_DIR"

# Create and format the root file system.

# Check for loop device before creating image.
LOOP_DEV=$(sudo losetup -f)
if [ -z "$LOOP_DEV" ] ; then
  echo "No free loop device.  Free up a loop device or reboot.  exiting. "
  exit 1
fi

# Create root file system disk image to fit on a 1GB memory stick.
# 1 GB in hard-drive-manufacturer-speak is 10^9, not 2^30.  950MB < 10^9 bytes.
ROOT_SIZE_BYTES=$((1024 * 1024 * 640))
dd if=/dev/zero of="$ROOT_FS_IMG" bs=1 count=1 seek=$((ROOT_SIZE_BYTES - 1))
sudo losetup "$LOOP_DEV" "$ROOT_FS_IMG"
sudo mkfs.ext3 "$LOOP_DEV"

# Tune and mount rootfs.
UUID=$(uuidgen)
DISK_LABEL="C-KEYFOB"
sudo tune2fs -L "$DISK_LABEL" -U "$UUID" -c 0 -i 0 "$LOOP_DEV"
sudo mount "$LOOP_DEV" "$ROOT_FS_DIR"

# Create stateful partition of the same size as the rootfs.
STATEFUL_IMG="$OUTPUT_DIR/stateful_partition.image"
STATEFUL_DIR="$OUTPUT_DIR/stateful_partition"
STATEFUL_LOOP_DEV=$(sudo losetup -f)
if [ -z "$STATEFUL_LOOP_DEV" ] ; then
  echo "No free loop device.  Free up a loop device or reboot.  exiting. "
  exit 1
fi
dd if=/dev/zero of="$STATEFUL_IMG" bs=1 count=1 seek=$((ROOT_SIZE_BYTES - 1))
sudo losetup "$STATEFUL_LOOP_DEV" "$STATEFUL_IMG"
sudo mkfs.ext3 "$STATEFUL_LOOP_DEV"
sudo tune2fs -L "C-STATE" -U "$UUID" -c 0 -i 0 \
    "$STATEFUL_LOOP_DEV"

# Mount the stateful partition.
mkdir -p "$STATEFUL_DIR"
sudo mount "$STATEFUL_LOOP_DEV" "$STATEFUL_DIR"

# Turn root file system into bootable image.
if [[ "$ARCH" = "x86" ]]; then
  # Setup extlinux configuration.
  # TODO: For some reason the /dev/disk/by-uuid is not being generated by udev
  # in the initramfs. When we figure that out, switch to root=UUID=$UUID.
  sudo mkdir -p "$ROOT_FS_DIR"/boot
  # TODO(adlr): use initramfs for booting.
  cat <<EOF | sudo dd of="$ROOT_FS_DIR"/boot/extlinux.conf
DEFAULT chromeos-usb
PROMPT 0
TIMEOUT 0

label chromeos-usb
  menu label chromeos-usb
  kernel vmlinuz
  append quiet console=tty2 init=/sbin/init boot=local rootwait root=/dev/sdb3 ro noresume noswap i915.modeset=1 loglevel=1

label chromeos-hd
  menu label chromeos-hd
  kernel vmlinuz
  append quiet console=tty2 init=/sbin/init boot=local rootwait root=HDROOT ro noresume noswap i915.modeset=1 loglevel=1
EOF

  # Make partition bootable and label it.
  sudo extlinux -z --install "${ROOT_FS_DIR}/boot"
fi

# -- Install packages into the root file system --

# We need to install libc manually from the cross toolchain.
# TODO: Improve this? We only want libc and not the whole toolchain.
PKGDIR="/var/lib/portage/pkgs/cross/"
sudo tar jxvpf \
  "${PKGDIR}/${CHOST}/cross-${CHOST}"/glibc-${LIBC_VERSION}.tbz2 \
  -C "$ROOT_FS_DIR" --strip-components=3 \
  --exclude=usr/include --exclude=sys-include --exclude=*.a --exclude=*.o

# We need to install libstdc++ manually from the cross toolchain.
# TODO: Figure out a better way of doing this?
sudo cp -a "${BOARD_ROOT}"/lib/libgcc_s.so* "${ROOT_FS_DIR}/lib"
sudo cp -a "${BOARD_ROOT}"/usr/lib/libstdc++.so* "${ROOT_FS_DIR}/usr/lib"

INSTALL_MASK=""
if [[ $FLAGS_installmask -eq $FLAGS_TRUE ]] ; then
  INSTALL_MASK="/usr/include/ /usr/man /usr/share/man /usr/share/doc /usr/share/gtk-doc /usr/share/gtk-2.0 /usr/lib/gtk-2.0/include /usr/share/info /usr/share/aclocal /usr/lib/gcc /usr/lib/pkgconfig /usr/share/pkgconfig /usr/share/gettext /usr/share/readline /etc/runlevels /usr/share/openrc /lib/rc *.a *.la"
fi

if [[ $FLAGS_jobs -ne -1 ]]; then
  EMERGE_JOBS="--jobs=$FLAGS_jobs"
fi

if [ $FLAGS_statefuldev -eq $FLAGS_TRUE ] ; then
  # Creating stateful partition.
  echo "Setting up symlinks for stateful partition install"
  DEV_IMAGE_ROOT="$STATEFUL_DIR/dev_image"
  sudo mkdir -p "$DEV_IMAGE_ROOT/usr"
  
  # Setup symlinks in stateful partition.
  for path in bin include lib libexec sbin share; do
    sudo mkdir "$DEV_IMAGE_ROOT/$path"
    sudo ln -s "$DEV_IMAGE_ROOT/$path" "$DEV_IMAGE_ROOT/usr/$path"
  done
   
  # Setup symlinks that don't conform to above model.
  sudo ln -s "$DEV_IMAGE_ROOT/lib" "$DEV_IMAGE_ROOT/usr/lib64"
  sudo ln -s "$DEV_IMAGE_ROOT" "$DEV_IMAGE_ROOT/usr/local"

  # Bind to rootfs.
  echo "Binding stateful partition to rootfs's /usr/local"
  sudo mkdir -p "$ROOT_FS_DIR/usr/local"
  sudo mount -n --bind "$DEV_IMAGE_ROOT" "$ROOT_FS_DIR/usr/local"
fi

# We "emerge --root=$ROOT_FS_DIR --root-deps=rdeps --usepkgonly" all of the
# runtime packages for chrome os. This builds up a chrome os image from binary
# packages with runtime dependencies only.  We use INSTALL_MASK to trim the
# image size as much as possible.
sudo INSTALL_MASK="$INSTALL_MASK" emerge-${BOARD} \
  --root="$ROOT_FS_DIR" --root-deps=rdeps \
  --usepkgonly chromeos $EMERGE_JOBS

# Determine the root dir for development packages.
ROOT_DEV_DIR="$ROOT_FS_DIR"
[ $FLAGS_statefuldev -eq $FLAGS_TRUE ] && ROOT_DEV_DIR="$ROOT_FS_DIR/usr/local"

# Install development packages.
if [[ $FLAGS_withdev -eq $FLAGS_TRUE ]] ; then
  sudo INSTALL_MASK="$INSTALL_MASK" emerge-${BOARD} \
      --root="$ROOT_DEV_DIR" --root-deps=rdeps \
      --usepkgonly chromeos-dev $EMERGE_JOBS

  # The ldd tool is a useful shell script but lives in glibc; just copy it.
  sudo cp -a "$(which ldd)" "${ROOT_DEV_DIR}/usr/bin"
fi

# Install packages required for testing.
if [[ $FLAGS_withtest -eq $FLAGS_TRUE ]] ; then
  sudo INSTALL_MASK="$INSTALL_MASK" emerge-${BOARD} \
      --root="$ROOT_DEV_DIR" --root-deps=rdeps \
      --usepkgonly chromeos-test $EMERGE_JOBS
fi

# Clean up links setup for stateful install of extra packages. 
if [ $FLAGS_statefuldev -eq $FLAGS_TRUE ] ; then
  # Fix symlinks so they work on live system.
  for path in bin include lib libexec sbin share; do
    sudo unlink $DEV_IMAGE_ROOT/usr/$path
    sudo ln -s /usr/local/$path $DEV_IMAGE_ROOT/usr/$path
  done

  # Fix exceptions.
  sudo unlink "$DEV_IMAGE_ROOT/usr/lib64"
  sudo unlink "$DEV_IMAGE_ROOT/usr/local"

  sudo ln -s "/usr/local/lib" "$DEV_IMAGE_ROOT/usr/lib64"
  sudo ln -s "/usr/local" "$DEV_IMAGE_ROOT/usr/local"

  #TODO(sosa@chromium.org) - /usr/bin/xterm symlink not created in stateful.
  sudo ln -sf "/usr/local/bin/aterm" "/usr/bin/xterm" 
fi

# Perform any customizations on the root file system that are needed.
WITH_DEV=""
if [[ $FLAGS_withdev -eq $FLAGS_TRUE ]]; then
  WITH_DEV="--withdev"
fi

#TODO(sosa@chromium.org) - Does it make sense to leave /usr/local bound here?
"${SCRIPTS_DIR}/customize_rootfs" \
  --root="$ROOT_FS_DIR" \
  --target="$ARCH" \
  --board="$BOARD" \
  $WITH_DEV

# Check that the image has been correctly created.
"${SCRIPTS_DIR}/test_image" \
  --root="$ROOT_FS_DIR" \
  --target="$ARCH"

# Set dev_mode flag and update library cache.
if [ $FLAGS_statefuldev -eq $FLAGS_TRUE ] ; then
  sudo touch "$ROOT_FS_DIR/root/.dev_mode"
  sudo /sbin/ldconfig -r "$ROOT_FS_DIR"
fi

# Only bound if installing dev to stateful.
if [ $FLAGS_statefuldev -eq $FLAGS_TRUE ] ; then
  sudo umount "$ROOT_FS_DIR/usr/local"
fi

# Cleanup loop devices.
cleanup_stateful_fs_loop
cleanup_rootfs_loop

# Create a master boot record.
# Start with the syslinux master boot record. We need to zero-pad to
# fill out a 512-byte sector size.
SYSLINUX_MBR="/usr/share/syslinux/mbr.bin"
dd if="$SYSLINUX_MBR" of="$MBR_IMG" bs=512 count=1 conv=sync
# Create a partition table in the MBR.
NUM_SECTORS=$((`stat --format="%s" "$ROOT_FS_IMG"` / 512))
KERNEL_SECTORS=8192
sudo sfdisk -H64 -S32 -uS -f "$MBR_IMG" <<EOF
,$NUM_SECTORS,L,-,
,$NUM_SECTORS,S,-,
,$NUM_SECTORS,L,*,
,$KERNEL_SECTORS,L,-,
;
EOF
if [[ "$ARCH" = "arm" ]]; then
  # Write u-boot script into MBR code block that boots 4th partition kernel.
  KERNEL_OFFSET=`printf "%08x" $(((3 * $NUM_SECTORS) + 1))`
  KERNEL_SECS_HEX=`printf "%08x" $KERNEL_SECTORS`
  MBR_SCRIPT="${OUTPUT_DIR}/mbr_script"
  echo -e "echo\necho ---- ChromeOS Boot ----\necho\n" \
          "mmc read 1 C0008000 0x$KERNEL_OFFSET 0x$KERNEL_SECS_HEX\n" \
          "bootm C0008000" > ${MBR_SCRIPT}
  MKIMAGE="${BOARD_ROOT}/u-boot/mkimage"
  if [[ -f "$MKIMAGE".gz ]]; then
    sudo gunzip "$MKIMAGE".gz
  fi
  if [[ -x "$MKIMAGE" ]]; then
    MBR_SCRIPT_UIMG="${MBR_SCRIPT}.uimg"
    "$MKIMAGE" -A "${ARCH}" -O linux  -T script -a 0 -e 0 -n "COS boot" \
               -d ${MBR_SCRIPT} ${MBR_SCRIPT_UIMG}
    dd bs=1 count=`stat --printf="%s" ${MBR_SCRIPT_UIMG}` \
       if="$MBR_SCRIPT_UIMG" of="$MBR_IMG" conv=notrunc
    hexdump -v -C "$MBR_IMG"
  else
    echo "Error: u-boot mkimage not found or not executable."
  fi
fi

OUTSIDE_OUTPUT_DIR="../build/images/${FLAGS_board}/${IMAGE_SUBDIR}"
echo "Done.  Image created in ${OUTPUT_DIR}"
echo "To copy to USB keyfob, OUTSIDE the chroot, do something like:"
echo "  ./image_to_usb.sh --from=${OUTSIDE_OUTPUT_DIR} --to=/dev/sdb"
echo "To convert to VMWare image, OUTSIDE the chroot, do something like:"
echo "  ./image_to_vmware.sh --from=${OUTSIDE_OUTPUT_DIR}"
echo "from the scripts directory where you entered the chroot."

trap - EXIT