Simplify and add flexibility to image creation process

This change adds support for building the disk layout from a
configuration file. It also cleans up much of the image creation
code.

install_gpt no longer exists, and has been replaced by cgpt.py's
write action. This spits out a file that has two functions that can
be called to write a partition layout to a disk/file. This gets rid
of the gigantic nest of calculations that built the layout previously.

All instances of partition/filesystem sizes in build scripts should now
be gone in favour of calls to the cgpt.py tool.

create_boot_desc has moved inside the base image creation, in an effort
to simplify build_image.

load_kernel_test is gone since it's apparently not supposed to be called
here anyway (asked wfrichar/rspangler about this one).

Base image creation now uses files rather than loop devices when
building an image. This means we can simply umount them once we're
done and not worry about cleaning up the loop device, since it's
been done for us.

Hash pad calculation has been removed. This is now set manually inside
the partition config file.

Hybrid MBR creation is gone, since it's now possible to do that in a board
specific hook (see overlay-beaglebone/scripts/board_specific_setup.sh).

OEM partition now has a filesystem, which is mounted at /usr/share/oem
during emerge so that packages can stash files here.

root_fs_dir and friends are still globals, but the long-term idea
is to make this not the case.

BUG=chromium-os:33817
TEST=All types of images and their respective flows
  (VM, recovery, test, factory etc)

Change-Id: I8a596728a4d1845c930e837bea627f5b6a11c098
Reviewed-on: https://gerrit.chromium.org/gerrit/29931
Commit-Ready: Liam McLoughlin <lmcloughlin@chromium.org>
Reviewed-by: Liam McLoughlin <lmcloughlin@chromium.org>
Tested-by: Liam McLoughlin <lmcloughlin@chromium.org>

bin/cros_make_image_bootable

@@ -33,7 +33,8 @@ assert_inside_chroot
 
 # Load functions and constants for chromeos-install
 . /usr/lib/installer/chromeos-common.sh || exit 1
-. "${SCRIPTS_DIR}/build_library/build_image_util.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/disk_layout_util.sh" || exit 1
 
 switch_to_strict_mode
 
@@ -70,7 +71,10 @@ locate_gpt
 set +e
 
 # Now parse the build settings from ${OUTPUT_DIR}/boot.desc
-
+DEFINE_string board "${DEFAULT_BOARD}" \
+  "Board we're building for."
+DEFINE_string image_type "base" \
+  "Type of image we're building for (base/factory_install)."
 DEFINE_string output_dir "/tmp" \
   "Directory to place output in."
 DEFINE_string image "chromiumos_base.img" \
@@ -141,6 +145,8 @@ FLAGS "${@}" || exit 1
 
 [ -z "${FLAGS_verity_salt}" ] && FLAGS_verity_salt=$(make_salt)
 
+. "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
+
 # Only now can we die on error.  shflags functions leak non-zero error codes,
 # so will die prematurely if 'switch_to_strict_mode' is specified before now.
 switch_to_strict_mode -u
@@ -238,10 +244,15 @@ make_image_bootable() {
   fi
 
   local rootfs_hash_size=$(stat -c '%s' ${FLAGS_rootfs_hash})
+  local rootfs_fs_size=$(get_filesystem_size ${FLAGS_image_type} 3)
+  local rootfs_partition_size=$(get_partition_size ${FLAGS_image_type} 3)
+  local rootfs_hash_pad=$(( rootfs_partition_size - rootfs_fs_size ))
   info "Appending rootfs.hash (${rootfs_hash_size} bytes) to the root fs"
-  if [[ ${rootfs_hash_size} -gt $((FLAGS_rootfs_hash_pad * 1024 * 1024)) ]]
+  if [[ ${rootfs_hash_size} -gt ${rootfs_hash_pad} ]]
   then
-    die "--rootfs_hash_pad reserves less than the needed ${rootfs_hash_size}"
+    die "rootfs_partition_size - rootfs_fs_size is less than the needed " \
+        "rootfs_hash_size (${rootfs_hash_size}), update your disk layout " \
+        "configuration"
   fi
   # Unfortunately, mount_gpt_image uses mount and not losetup to create the
   # loop devices.  This means that they are not the correct size.  We have to
@@ -251,7 +262,7 @@ make_image_bootable() {
     rootfs_file_size=$(stat -c '%s' ${root_dev})
     hash_offset=$((hash_offset + (${rootfs_file_size} / 512)))
   else
-    hash_offset=$((hash_offset + ((1024 * 1024 * ${FLAGS_rootfs_size}) / 512)))
+    hash_offset=$((hash_offset + (${rootfs_fs_size} / 512)))
   fi
   sudo dd bs=512 \
           seek=${hash_offset} \
@@ -267,7 +278,6 @@ make_image_bootable() {
   sudo cp "${FLAGS_output_dir}/vmlinuz_hd.vblock" \
           "${FLAGS_statefulfs_mountpoint}"
 
-  # START_KERN_A is set by the first call to install the gpt.
   local koffset="$(partoffset ${image} 2)"
   sudo dd if="${FLAGS_output_dir}/vmlinuz.image" of="${image}" \
     conv=notrunc bs=512 seek=${koffset}

build_image

@@ -12,7 +12,6 @@
 SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
 . "${SCRIPT_ROOT}/build_library/build_common.sh" || exit 1
 
-
 # Developer-visible flags.
 DEFINE_string board "${DEFAULT_BOARD}" \
   "The board to build an image for."
@@ -20,22 +19,12 @@ DEFINE_string boot_args "noinitrd" \
   "Additional boot arguments to pass to the commandline"
 DEFINE_boolean enable_rootfs_verification ${FLAGS_TRUE} \
   "Default all bootloaders to use kernel-based root fs integrity checking."
-DEFINE_boolean full "${FLAGS_FALSE}" "Build full image with all partitions."
 DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \
   "Directory in which to place image result directories (named by version)"
-DEFINE_integer rootfs_hash_pad 8 \
+DEFINE_string disk_layout "default" \
-  "MiBs reserved at the end of the rootfs image. \
+  "The board to build an image for."
- ceil(0.01 * rootfs_size) is a good minimum."
-DEFINE_integer rootfs_partition_size 1024 \
-  "rootfs partition size in MiBs."
-DEFINE_integer rootfs_size 850 \
-  "rootfs filesystem size in MiBs."
-DEFINE_integer rootfs_boost_size 0 \
-  "MiBs by which to increase the rootfs allocations."
 DEFINE_boolean standard_backdoor ${FLAGS_TRUE} \
   "Install standard backdoor credentials for testing"
-DEFINE_integer statefulfs_size 1024 \
-  "stateful filesystem size in MiBs."
 DEFINE_string usb_disk /dev/sdb3 \
   "Path syslinux should use to do a usb boot. Default: /dev/sdb3"
 
@@ -65,23 +54,12 @@ show_help_if_requested "$@"
 # not needed for the typical developer workflow.
 DEFINE_integer build_attempt 1 \
   "The build attempt for this image build."
-DEFINE_boolean hybrid_mbr ${FLAGS_FALSE} \
-  "Creates a hybrid MBR rather than a protective one"
 DEFINE_integer jobs -1 \
   "How many packages to build in parallel at maximum."
 DEFINE_boolean replace ${FLAGS_FALSE} \
   "Overwrite existing output, if any."
 DEFINE_string symlink "latest" \
   "Symlink name to use for this image."
-DEFINE_integer verity_error_behavior 3 \
-  "Kernel verified boot error behavior (0: I/O errors, 1: panic, 2: nothing, \
-3: cros) Default: 3"
-DEFINE_integer verity_max_ios -1 \
-  "Number of outstanding I/O operations dm-verity caps at. Default: -1"
-DEFINE_string verity_algorithm "sha1" \
-  "Cryptographic hash algorithm used for kernel vboot. Default : sha1"
-DEFINE_string verity_salt "" \
-  "Root filesystem salt. Default: randomly generated."
 DEFINE_string version "" \
   "Overrides version number in name to this version."
 
@@ -99,7 +77,7 @@ OVERLAY_CHROMEOS_DIR="${SRC_ROOT}/third_party/chromiumos-overlay/chromeos"
 # N.B.  Ordering matters for some of the libraries below, because
 # some of the files contain initialization used by later files.
 . "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
-. "${BUILD_LIBRARY_DIR}/build_gpt.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/disk_layout_util.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/mount_gpt_util.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/base_image_util.sh" || exit 1
@@ -116,25 +94,7 @@ for overlay in $(cros_overlay_list --board "$BOARD"); do
   fi
 done
 
-# Tweak sizes defaulted or given based on any boost given (or defaulted).
+# TODO: <prebuild hook>
-# This is syntactic sugar to separate concerns in callers, and unify
-# default values here.
-if [[ $FLAGS_rootfs_boost_size -gt 0 ]]; then
-  max() {
-    echo $(( $1 > $2 ? $1 : $2 ))
-  }
-  root_boost() {
-    echo $(( $1 + ( $1 * $FLAGS_rootfs_boost_size + $FLAGS_rootfs_size - 1 ) \
-        / $FLAGS_rootfs_size ))
-  }
-  FLAGS_rootfs_hash_pad=$(root_boost $FLAGS_rootfs_hash_pad)
-  FLAGS_rootfs_partition_size=$(root_boost $FLAGS_rootfs_partition_size)
-  FLAGS_rootfs_size=$(root_boost $FLAGS_rootfs_size)
-  FLAGS_rootfs_boost_size=0
-  # Push up to the recommended minimum if we did not get there.
-  FLAGS_rootfs_hash_pad=$(max $FLAGS_rootfs_hash_pad \
-      $(( ( $FLAGS_rootfs_size + 99 ) / 100 )))
-fi
 
 # Tweak flags, configure extra USE flags, and add packages for the factory
 # install shim.
@@ -143,16 +103,6 @@ if should_build_image ${CHROMEOS_FACTORY_INSTALL_SHIM_NAME}; then
   # TODO: Build a separated ebuild for the install shim to reduce size.
   INSTALL_MASK="${FACTORY_SHIM_INSTALL_MASK}"
 
-  # Reduce the size of factory install shim. Note that 400M is too much, it
-  # should be smaller, see http://crosbug.com/34167
-  FLAGS_rootfs_size=400
-  FLAGS_rootfs_partition_size=420
-  FLAGS_statefulfs_size=140
-  info "Fixing the rootfs size at ${FLAGS_rootfs_partition_size} MiB " \
-    "for install shim"
-  info "Fixing the statefulfs size at ${FLAGS_statefulfs_size} MiB " \
-    "for install shim"
-
   # Add the cros_factory_install boot arg.
   FLAGS_boot_args="${FLAGS_boot_args} cros_factory_install"
 
@@ -172,11 +122,7 @@ if should_build_image ${CHROMEOS_FACTORY_INSTALL_SHIM_NAME}; then
   fi
 fi
 
-if [ $((FLAGS_rootfs_size + FLAGS_rootfs_hash_pad)) -gt \
+# TODO: </prebuild hook>
-     ${FLAGS_rootfs_partition_size} ] ; then
-  die_notrace "rootfs ($((FLAGS_rootfs_size + FLAGS_rootfs_hash_pad)) MiB) is" \
-    "bigger than partition (${FLAGS_rootfs_partition_size} MiB)."
-fi
 
 # If we are creating a developer image, also create a pristine image with a
 # different name.
@@ -187,17 +133,8 @@ else
   PRISTINE_IMAGE_NAME=${CHROMEOS_BASE_IMAGE_NAME}
 fi
 
-ROOT_FS_DIR="${BUILD_DIR}/rootfs"
-STATEFUL_FS_DIR="${BUILD_DIR}/stateful_partition"
-ESP_FS_DIR=${BUILD_DIR}/esp
-
 DEVKEYSDIR="/usr/share/vboot/devkeys"
 
-# ${DEV_IMAGE_ROOT} specifies the location of where developer packages will
-# be installed on the stateful dir.  On a Chromium OS system, this will
-# translate to /usr/local.
-DEV_IMAGE_ROOT="${STATEFUL_FS_DIR}/dev_image"
-
 eclean-$BOARD -d packages
 
 if [[ ${skip_blacklist_check} -ne 1 ]]; then
@@ -228,29 +165,10 @@ fi
 
 # Create the output directory and temporary mount points.
 mkdir -p "${BUILD_DIR}"
-mkdir -p "${ROOT_FS_DIR}" "${STATEFUL_FS_DIR}" "${ESP_FS_DIR}"
-
-# Create the boot.desc file which stores the build-time configuration
-# information needed for making the image bootable after creation with
-# cros_make_image_bootable.
-create_boot_desc
 
+# Create the base image.
 create_base_image ${PRISTINE_IMAGE_NAME}
 
-
-BOOT_FLAG=
-if should_build_image ${CHROMEOS_FACTORY_INSTALL_SHIM_NAME}; then
-  BOOT_FLAG="-b 1"  # BOOT_FLAG_DEVELOPER value defined in load_kernel_fw.h
-  info "--factory_install set, pass BOOT_FLAG_DEVELOPER flag to" \
-    "load_kernel_test"
-fi
-
-# Verify pristine image if we built it.
-if should_build_image "${PRISTINE_IMAGE_NAME}"; then
-  load_kernel_test "${BUILD_DIR}/${PRISTINE_IMAGE_NAME}" \
-    "${DEVKEYSDIR}/recovery_key.vbpubk" ${BOOT_FLAG}
-fi
-
 # Running board-specific setup if any exists.
 if type board_setup &>/dev/null; then
   board_setup "${BUILD_DIR}/${PRISTINE_IMAGE_NAME}"
@@ -274,8 +192,6 @@ if should_build_image ${CHROMEOS_FACTORY_TEST_IMAGE_NAME}; then
   mod_image_for_test  ${CHROMEOS_FACTORY_TEST_IMAGE_NAME}
 fi
 
-rmdir "${ROOT_FS_DIR}" "${STATEFUL_FS_DIR}" "${ESP_FS_DIR}"
-
 # Generating AU generator zip file to run outside chroot
 generate_au_zip || echo "Failed generating AU zip file - ignoring Error..."
 

build_library/base_image_util.sh

@@ -2,52 +2,27 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-# Shell function library and global variable initialization for
+. "${SRC_ROOT}/platform/dev/toolchain_utils.sh" || exit 1
-# creating an initial base image.  The main function for export in
-# this library is 'create_base_image'; the remainder of the code is
-# not used outside this file.
 
+cleanup_mounts() {
+  # Disable die on error.
+  set +e
 
-ROOT_LOOP_DEV=
-STATEFUL_LOOP_DEV=
-
-ROOT_FS_IMG="${BUILD_DIR}/rootfs.image"
-STATEFUL_FS_IMG="${BUILD_DIR}/stateful_partition.image"
-ESP_FS_IMG=${BUILD_DIR}/esp.image
-
-cleanup_rootfs_loop() {
   # See if we ran out of space.
-  local df=$(df -B 1M "${ROOT_FS_DIR}")
+  local df=$(df -B 1M "${root_fs_dir}")
   if [[ ${df} == *100%* ]]; then
     error "Here are the biggest files (by disk usage):"
     # Send final output to stderr to match `error` behavior.
-    sudo find "${ROOT_FS_DIR}" -xdev -type f -printf '%b %P\n' | \
+    sudo find "${root_fs_dir}" -xdev -type f -printf '%b %P\n' | \
       awk '$1 > 16 { $1 = $1 * 512; print }' | sort -n | tail -100 1>&2
     error "Target image has run out of space:"
     error "${df}"
   fi
-  sudo umount -d "${ROOT_FS_DIR}"
-}
 
-cleanup_stateful_fs_loop() {
+  echo "Cleaning up mounts"
-  sudo umount "${ROOT_FS_DIR}/usr/local"
+  safe_umount_tree "${root_fs_dir}"
-  sudo umount "${ROOT_FS_DIR}/var"
+  safe_umount_tree "${stateful_fs_dir}"
-  sudo umount -d "${STATEFUL_FS_DIR}"
+  safe_umount_tree "${esp_fs_dir}"
-}
-
-loopback_cleanup() {
-  # Disable die on error.
-  set +e
-
-  if [[ -n "${STATEFUL_LOOP_DEV}" ]]; then
-    cleanup_stateful_fs_loop
-    STATEFUL_LOOP_DEV=
-  fi
-
-  if [[ -n "${ROOT_LOOP_DEV}" ]]; then
-    cleanup_rootfs_loop
-    ROOT_LOOP_DEV=
-  fi
 
    # Turn die on error back on.
   set -e
@@ -58,43 +33,62 @@ zero_free_space() {
   info "Zeroing freespace in ${fs_mount_point}"
   # dd is a silly thing and will produce a "No space left on device" message
   # that cannot be turned off and is confusing to unsuspecting victims.
+  info "${fs_mount_point}/filler"
   ( sudo dd if=/dev/zero of="${fs_mount_point}/filler" bs=4096 conv=fdatasync \
       status=noxfer || true ) 2>&1 | grep -v "No space left on device"
   sudo rm "${fs_mount_point}/filler"
 }
 
-# Takes as an arg the name of the image to be created.
 create_base_image() {
   local image_name=$1
+  local rootfs_verification_enabled=$2
+  local image_type="base"
 
-  trap "loopback_cleanup && delete_prompt" EXIT
+  if [[ "${FLAGS_disk_layout}" != "default" ]]; then
-
+      image_type="${FLAGS_disk_layout}"
-  # Create and format the root file system.
+  else
-
+    if should_build_image ${CHROMEOS_FACTORY_INSTALL_SHIM_NAME}; then
-  # Create root file system disk image.
+      image_type="factory_install"
-  ROOT_SIZE_BYTES=$((1024 * 1024 * ${FLAGS_rootfs_size}))
+    fi
-
-  # Pad out for the hash tree.
-  ROOT_HASH_PAD=$((FLAGS_rootfs_hash_pad * 1024 * 1024))
-  info "Padding the rootfs image by ${ROOT_HASH_PAD} bytes for hash data"
-
-  dd if=/dev/zero of="${ROOT_FS_IMG}" bs=1 count=1 \
-     seek=$((ROOT_SIZE_BYTES + ROOT_HASH_PAD - 1)) status=noxfer
-
-  ROOT_LOOP_DEV=$(sudo losetup --show -f "${ROOT_FS_IMG}")
-  if [ -z "${ROOT_LOOP_DEV}" ] ; then
-    die_notrace \
-        "No free loop device.  Free up a loop device or reboot.  exiting. "
   fi
 
-  # Specify a block size and block count to avoid using the hash pad.
+  info "Using image type ${image_type}"
-  sudo mkfs.ext2 -q -b 4096 "${ROOT_LOOP_DEV}" "$((ROOT_SIZE_BYTES / 4096))"
 
-  # Tune and mount rootfs.
+  root_fs_dir="${BUILD_DIR}/rootfs"
-  DISK_LABEL="C-ROOT"
+  stateful_fs_dir="${BUILD_DIR}/stateful"
-  # Disable checking and minimize metadata differences across builds
+  esp_fs_dir="${BUILD_DIR}/esp"
-  # and wasted reserved space.
+  oem_fs_dir="${BUILD_DIR}/oem"
-  sudo tune2fs -L "${DISK_LABEL}" \
+
+  trap "cleanup_mounts && delete_prompt" EXIT
+  cleanup_mounts &> /dev/null
+
+  local root_fs_img="${BUILD_DIR}/rootfs.image"
+  local root_fs_bytes=$(get_filesystem_size ${image_type} 3)
+  local root_fs_label=$(get_label ${image_type} 3)
+
+  local stateful_fs_img="${BUILD_DIR}/stateful.image"
+  local stateful_fs_bytes=$(get_filesystem_size ${image_type} 1)
+  local stateful_fs_label=$(get_label ${image_type} 1)
+  local stateful_fs_uuid=$(uuidgen)
+
+  local esp_fs_img="${BUILD_DIR}/esp.image"
+  local esp_fs_bytes=$(get_filesystem_size ${image_type} 12)
+  local esp_fs_label=$(get_label ${image_type} 12)
+
+  local oem_fs_img="${BUILD_DIR}/oem.image"
+  local oem_fs_bytes=$(get_filesystem_size ${image_type} 8)
+  local oem_fs_label=$(get_label ${image_type} 8)
+  local oem_fs_uuid=$(uuidgen)
+
+  local block_size=$(get_fs_block_size)
+
+  # Build root FS image.
+  info "Building ${root_fs_img}"
+  dd if=/dev/zero of="${root_fs_img}" bs=1 count=1 \
+    seek=$((root_fs_bytes - 1)) status=noxfer
+  sudo mkfs.ext2 -F -q -b $block_size "${root_fs_img}" \
+    "$((root_fs_bytes / block_size))"
+  sudo tune2fs -L "${root_fs_label}" \
                -U clear \
                -T 20091119110000 \
                -c 0 \
@@ -102,46 +96,59 @@ create_base_image() {
                -m 0 \
                -r 0 \
                -e remount-ro \
-                "${ROOT_LOOP_DEV}"
+                "${root_fs_img}"
-  # TODO(wad) call tune2fs prior to finalization to set the mount count to 0.
+  mkdir -p "${root_fs_dir}"
-  sudo mount -t ext2 "${ROOT_LOOP_DEV}" "${ROOT_FS_DIR}"
+  sudo mount -o loop "${root_fs_img}" "${root_fs_dir}"
 
-  # Create stateful partition of the same size as the rootfs.
+  df -h "${root_fs_dir}"
-  STATEFUL_SIZE_BYTES=$((1024 * 1024 * ${FLAGS_statefulfs_size}))
-  dd if=/dev/zero of="${STATEFUL_FS_IMG}" bs=1 count=1 \
-      seek=$((STATEFUL_SIZE_BYTES - 1)) status=noxfer
 
-  # Tune and mount the stateful partition.
+  # Build stateful FS disk image.
-  UUID=$(uuidgen)
+  info "Building ${stateful_fs_img}"
-  DISK_LABEL="C-STATE"
+  dd if=/dev/zero of="${stateful_fs_img}" bs=1 count=1 \
-  STATEFUL_LOOP_DEV=$(sudo losetup --show -f "${STATEFUL_FS_IMG}")
+    seek=$((stateful_fs_bytes - 1)) status=noxfer
-  if [ -z "${STATEFUL_LOOP_DEV}" ] ; then
+  sudo mkfs.ext4 -F -q "${stateful_fs_img}"
-    die_notrace \
+  sudo tune2fs -L "${stateful_fs_label}" -U "${stateful_fs_uuid}" \
-        "No free loop device.  Free up a loop device or reboot.  exiting. "
+               -c 0 -i 0 "${stateful_fs_img}"
-  fi
+  mkdir -p "${stateful_fs_dir}"
-  sudo mkfs.ext4 -q "${STATEFUL_LOOP_DEV}"
+  sudo mount -o loop "${stateful_fs_img}" "${stateful_fs_dir}"
-  sudo tune2fs -L "${DISK_LABEL}" -U "${UUID}" -c 0 -i 0 "${STATEFUL_LOOP_DEV}"
-  sudo mount -t ext4 "${STATEFUL_LOOP_DEV}" "${STATEFUL_FS_DIR}"
 
-  # -- Install packages into the root file system --
+  # Build ESP disk image.
+  info "Building ${esp_fs_img}"
+  dd if=/dev/zero of="${esp_fs_img}" bs=1 count=1 \
+    seek=$((esp_fs_bytes - 1)) status=noxfer
+  sudo mkfs.vfat "${esp_fs_img}"
+
+  # Build OEM FS disk image.
+  info "Building ${oem_fs_img}"
+  dd if=/dev/zero of="${oem_fs_img}" bs=1 count=1 \
+    seek=$((oem_fs_bytes - 1)) status=noxfer
+  sudo mkfs.ext4 -F -q "${oem_fs_img}"
+  sudo tune2fs -L "${oem_fs_label}" -U "${oem_fs_uuid}" \
+               -c 0 -i 0 "${oem_fs_img}"
+  mkdir -p "${oem_fs_dir}"
+  sudo mount -o loop "${oem_fs_img}" "${oem_fs_dir}"
 
   # Prepare stateful partition with some pre-created directories.
-  sudo mkdir -p "${DEV_IMAGE_ROOT}"
+  sudo mkdir "${stateful_fs_dir}/dev_image"
-  sudo mkdir -p "${STATEFUL_FS_DIR}/var_overlay"
+  sudo mkdir "${stateful_fs_dir}/var_overlay"
 
   # Create symlinks so that /usr/local/usr based directories are symlinked to
   # /usr/local/ directories e.g. /usr/local/usr/bin -> /usr/local/bin, etc.
-  setup_symlinks_on_root "${DEV_IMAGE_ROOT}" "${STATEFUL_FS_DIR}/var_overlay" \
+  setup_symlinks_on_root "${stateful_fs_dir}/dev_image" \
-    "${STATEFUL_FS_DIR}"
+    "${stateful_fs_dir}/var_overlay" "${stateful_fs_dir}"
 
   # Perform binding rather than symlinking because directories must exist
   # on rootfs so that we can bind at run-time since rootfs is read-only.
   info "Binding directories from stateful partition onto the rootfs"
-  sudo mkdir -p "${ROOT_FS_DIR}/usr/local"
+  sudo mkdir -p "${root_fs_dir}/usr/local"
-  sudo mount --bind "${DEV_IMAGE_ROOT}" "${ROOT_FS_DIR}/usr/local"
+  sudo mount --bind "${stateful_fs_dir}/dev_image" "${root_fs_dir}/usr/local"
-  sudo mkdir -p "${ROOT_FS_DIR}/var"
+  sudo mkdir -p "${root_fs_dir}/var"
-  sudo mount --bind "${STATEFUL_FS_DIR}/var_overlay" "${ROOT_FS_DIR}/var"
+  sudo mount --bind "${stateful_fs_dir}/var_overlay" "${root_fs_dir}/var"
-  sudo mkdir -p "${ROOT_FS_DIR}/dev"
+  sudo mkdir -p "${root_fs_dir}/dev"
+
+  info "Binding directories from OEM partition onto the rootfs"
+  sudo mkdir -p "${root_fs_dir}/usr/share/oem"
+  sudo mount --bind "${oem_fs_dir}" "${root_fs_dir}/usr/share/oem"
 
   # We need to install libc manually from the cross toolchain.
   # TODO: Improve this? It would be ideal to use emerge to do this.
@@ -155,33 +162,43 @@ create_base_image() {
     "--board=${BOARD} to update the version of libc installed on that board."
   fi
 
-  sudo tar jxpf "${LIBC_PATH}" -C "${ROOT_FS_DIR}" ./usr/${CHOST} \
+  sudo tar jxpf "${LIBC_PATH}" -C "${root_fs_dir}" ./usr/${CHOST} \
   --strip-components=3 --exclude=usr/include --exclude=sys-include \
   --exclude=*.a --exclude=*.o
 
-  . "${SRC_ROOT}/platform/dev/toolchain_utils.sh"
   board_ctarget=$(get_ctarget_from_board "${BOARD}")
   for atom in $(portageq match / cross-$board_ctarget/gcc); do
-    copy_gcc_libs "${ROOT_FS_DIR}" $atom
+    copy_gcc_libs "${root_fs_dir}" $atom
   done
 
   if should_build_image ${CHROMEOS_FACTORY_INSTALL_SHIM_NAME}; then
     # Install our custom factory install kernel with the appropriate use flags
     # to the image.
-    emerge_custom_kernel "${ROOT_FS_DIR}"
+    emerge_custom_kernel "${root_fs_dir}"
   fi
 
-  # We "emerge --root=${ROOT_FS_DIR} --root-deps=rdeps --usepkgonly" all of the
+  # We "emerge --root=${root_fs_dir} --root-deps=rdeps --usepkgonly" all of the
   # runtime packages for chrome os. This builds up a chrome os image from
   # binary packages with runtime dependencies only.  We use INSTALL_MASK to
   # trim the image size as much as possible.
-  emerge_to_image --root="${ROOT_FS_DIR}" chromeos ${EXTRA_PACKAGES}
+  emerge_to_image --root="${root_fs_dir}" chromeos ${EXTRA_PACKAGES}
 
   # Set /etc/lsb-release on the image.
   "${OVERLAY_CHROMEOS_DIR}/scripts/cros_set_lsb_release" \
-    --root="${ROOT_FS_DIR}" \
+  --root="${root_fs_dir}" \
   --board="${BOARD}"
 
+  # Create the boot.desc file which stores the build-time configuration
+  # information needed for making the image bootable after creation with
+  # cros_make_image_bootable.
+  create_boot_desc "${image_type}"
+
+  # Write out the GPT creation script.
+  # This MUST be done before writing bootloader templates else we'll break
+  # the hash on the root FS.
+  write_partition_script "${image_type}" \
+    "${root_fs_dir}/${PARTITION_SCRIPT_PATH}"
+
   # Populates the root filesystem with legacy bootloader templates
   # appropriate for the platform.  The autoupdater and installer will
   # use those templates to update the legacy boot partition (12/ESP)
@@ -191,79 +208,53 @@ create_base_image() {
   # not support verified boot yet (see create_legacy_bootloader_templates.sh)
   # so rootfs verification is disabled if we are building with --factory_install
   local enable_rootfs_verification=
-  if [[ ${FLAGS_enable_rootfs_verification} -eq ${FLAGS_TRUE} ]]; then
+  if [[ ${rootfs_verification_enabled} -eq 1 ]]; then
   enable_rootfs_verification="--enable_rootfs_verification"
   fi
 
   ${BUILD_LIBRARY_DIR}/create_legacy_bootloader_templates.sh \
     --arch=${ARCH} \
-    --to="${ROOT_FS_DIR}"/boot \
+    --to="${root_fs_dir}"/boot \
-    --boot_args="${FLAGS_boot_args}" \
       ${enable_rootfs_verification}
 
   # Don't test the factory install shim
   if ! should_build_image ${CHROMEOS_FACTORY_INSTALL_SHIM_NAME}; then
       if [[ ${skip_test_image_content} -ne 1 ]]; then
       # Check that the image has been correctly created.
-      test_image_content "$ROOT_FS_DIR"
+      test_image_content "$root_fs_dir"
     fi
   fi
 
   # Clean up symlinks so they work on a running target rooted at "/".
   # Here development packages are rooted at /usr/local.  However, do not
   # create /usr/local or /var on host (already exist on target).
-  setup_symlinks_on_root "/usr/local" "/var" "${STATEFUL_FS_DIR}"
+  setup_symlinks_on_root "/usr/local" "/var" "${stateful_fs_dir}"
-
-  # Create EFI System Partition to boot stock EFI BIOS (but not
-  # ChromeOS EFI BIOS).  ARM uses this space to determine which
-  # partition is bootable.  NOTE: The size argument for mkfs.vfat is
-  # in 1024-byte blocks.  We'll hard-code it to 16M for now, unless
-  # we are building a factory shim, in which case a larger room is
-  # needed to allow two kernel blobs (each including initramfs) in
-  # one EFI partition.
-  if should_build_image ${CHROMEOS_FACTORY_INSTALL_SHIM_NAME}; then
-    /usr/sbin/mkfs.vfat -C "${ESP_FS_IMG}" 32768
-  else
-    /usr/sbin/mkfs.vfat -C "${ESP_FS_IMG}" 16384
-  fi
 
   # Zero rootfs free space to make it more compressible so auto-update
   # payloads become smaller
-  zero_free_space "${ROOT_FS_DIR}"
+  zero_free_space "${root_fs_dir}"
-  loopback_cleanup
-  trap delete_prompt EXIT
 
-  if [[ ${FLAGS_full} -eq ${FLAGS_TRUE} ]]; then
+  cleanup_mounts
-    dd if=/dev/zero of="${BUILD_DIR}/${image_name}" bs=1M count=3584
-  fi
 
   # Create the GPT-formatted image.
   build_gpt "${BUILD_DIR}/${image_name}" \
-            "${ROOT_FS_IMG}" \
+          "${root_fs_img}" \
-            "${STATEFUL_FS_IMG}" \
+          "${stateful_fs_img}" \
-            "${ESP_FS_IMG}"
+          "${esp_fs_img}"
+
   # Clean up temporary files.
-  rm -f "${ROOT_FS_IMG}" "${STATEFUL_FS_IMG}" "${ESP_FS_IMG}"
+  rm -f "${root_fs_img}" "${stateful_fs_img}" "${esp_fs_img}" "{oem_fs_img}"
 
   # Emit helpful scripts for testers, etc.
   emit_gpt_scripts "${BUILD_DIR}/${image_name}" "${BUILD_DIR}"
 
-  trap - EXIT
-
   USE_DEV_KEYS=
   if should_build_image ${CHROMEOS_FACTORY_INSTALL_SHIM_NAME}; then
     USE_DEV_KEYS="--use_dev_keys"
   fi
 
   # Place flags before positional args.
-  if should_build_image ${image_name}; then
   ${SCRIPTS_DIR}/bin/cros_make_image_bootable "${BUILD_DIR}" \
                                               ${image_name} \
                                               ${USE_DEV_KEYS}
-  fi
-
-  # Setup hybrid MBR if it was enabled
-  if [[ ${FLAGS_hybrid_mbr} -eq ${FLAGS_TRUE} ]]; then
-    install_hybrid_mbr "${BUILD_DIR}/${image_name}"
-  fi
 }

build_library/build_common.sh

@@ -17,5 +17,33 @@ restart_in_chroot_if_needed "$@"
 INSTALLER_ROOT=/usr/lib/installer
 . "${INSTALLER_ROOT}/chromeos-common.sh" || exit 1
 
-BUILD_LIBRARY_DIR=${SCRIPTS_DIR}/build_library
 locate_gpt
+
+should_build_image() {
+  # Fast pass back if we should build all incremental images.
+  local image_name
+  local image_to_build
+
+  for image_name in "$@"; do
+    for image_to_build in ${IMAGES_TO_BUILD}; do
+      [ "${image_to_build}" = "${image_name}" ] && return 0
+    done
+  done
+
+  return 1
+}
+
+# Utility function for creating a copy of an image prior to
+# modification from the BUILD_DIR:
+#  $1: source filename
+#  $2: destination filename
+copy_image() {
+  local src="${BUILD_DIR}/$1"
+  local dst="${BUILD_DIR}/$2"
+  if should_build_image $1; then
+    echo "Creating $2 from $1..."
+    $COMMON_PV_CAT "${src}" >"${dst}" || die "Cannot copy $1 to $2"
+  else
+    mv "${src}" "${dst}" || die "Cannot move $1 to $2"
+  fi
+}

build_library/build_gpt.sh

@@ -1,88 +0,0 @@
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-emit_gpt_scripts() {
-  local image="$1"
-  local dir="$2"
-
-  local pack="$dir/pack_partitions.sh"
-  local unpack="$dir/unpack_partitions.sh"
-
-  cat >"$unpack" <<HEADER
-#!/bin/bash -eu
-# File automatically generated. Do not edit.
-TARGET=\${1:-}
-if [[ -z "\$TARGET" ]]; then
-  echo "Usage: \$0 DEVICE" 1>&2
-  exit 1
-fi
-set -x
-HEADER
-
-  $GPT show "$image" | sed -e 's/^/# /' >>"$unpack"
-  cp "$unpack" "$pack"
-
-  $GPT show -q "$image" |
-    while read start size part x; do
-      local file="part_$part"
-      local target="\"\$TARGET\""
-      local dd_args="bs=512 count=$size"
-      echo "dd if=$target of=$file $dd_args skip=$start" >>"$unpack"
-      echo "dd if=$file of=$target $dd_args seek=$start conv=notrunc" \
-        >>"$pack"
-    done
-
-  chmod +x "$unpack" "$pack"
-}
-
-
-build_gpt() {
-  local outdev="$1"
-  local rootfs_img="$2"
-  local stateful_img="$3"
-  local esp_img="$4"
-
-  # We'll need some code to put in the PMBR, for booting on legacy BIOS.
-  local pmbr_img
-  if [ "$ARCH" = "arm" ]; then
-    pmbr_img=/dev/zero
-  elif [[ "$ARCH" = "x86" || "$ARCH" = "amd64" ]]; then
-    pmbr_img=$(readlink -f /usr/share/syslinux/gptmbr.bin)
-  else
-    error "Unknown architecture: $ARCH"
-    return 1
-  fi
-
-  GPT_FULL="false"
-  [ "${FLAGS_full}" -eq "${FLAGS_TRUE}" ] && GPT_FULL="true"
-
-  # Create the GPT. This has the side-effect of setting some global vars
-  # describing the partition table entries (see the comments in the source).
-  install_gpt "$outdev" $(numsectors "$rootfs_img") \
-    $(numsectors "$stateful_img") $pmbr_img $(numsectors "$esp_img") \
-    $GPT_FULL $FLAGS_rootfs_partition_size
-
-  local sudo=
-  if [ ! -w "$outdev" ] ; then
-    # use sudo when writing to a block device.
-    sudo=sudo
-  fi
-
-  # Now populate the partitions.
-  info "Copying stateful partition..."
-  $sudo dd if="$stateful_img" of="$outdev" conv=notrunc bs=512 \
-      seek=$START_STATEFUL
-
-  info "Copying rootfs..."
-  $sudo dd if="$rootfs_img" of="$outdev" conv=notrunc bs=512 \
-      seek=$START_ROOTFS_A
-
-  info "Copying EFI system partition..."
-  $sudo dd if="$esp_img" of="$outdev" conv=notrunc bs=512 \
-      seek=$START_ESP
-
-  # Pre-set "sucessful" bit in gpt, so we will never mark-for-death
-  # a partition on an SDCard/USB stick.
-  $GPT add -i 2 -S 1 "$outdev"
-}

build_library/build_image_util.sh

@@ -22,6 +22,14 @@ BUILD_DIR="${FLAGS_output_root}/${BOARD}/${IMAGE_SUBDIR}"
 OUTSIDE_OUTPUT_DIR="../build/images/${BOARD}/${IMAGE_SUBDIR}"
 IMAGES_TO_BUILD=
 
+EMERGE_BOARD_CMD="$GCLIENT_ROOT/chromite/bin/parallel_emerge"
+EMERGE_BOARD_CMD="$EMERGE_BOARD_CMD --board=$BOARD"
+
+export INSTALL_MASK="${DEFAULT_INSTALL_MASK}"
+
+if [[ $FLAGS_jobs -ne -1 ]]; then
+  EMERGE_JOBS="--jobs=$FLAGS_jobs"
+fi
 
 # Populates list of IMAGES_TO_BUILD from args passed in.
 # Arguments should be the shortnames of images we want to build.
@@ -106,9 +114,9 @@ make_salt() {
   xxd -l 32 -p -c 32 /dev/urandom
 }
 
-# Takes no arguments and populates the configuration for
-# cros_make_image_bootable.
 create_boot_desc() {
+  local image_type=$1
+
   local enable_rootfs_verification_flag=""
   if [[ ${FLAGS_enable_rootfs_verification} -eq ${FLAGS_TRUE} ]]; then
     enable_rootfs_verification_flag="--enable_rootfs_verification"
@@ -116,17 +124,13 @@ create_boot_desc() {
 
   [ -z "${FLAGS_verity_salt}" ] && FLAGS_verity_salt=$(make_salt)
   cat <<EOF > ${BUILD_DIR}/boot.desc
+  --board=${BOARD}
+  --image_type=${image_type}
   --arch="${ARCH}"
-  --boot_args="${FLAGS_boot_args}"
-  --rootfs_size="${FLAGS_rootfs_size}"
-  --rootfs_hash_pad="${FLAGS_rootfs_hash_pad}"
-  --verity_error_behavior="${FLAGS_verity_error_behavior}"
-  --verity_max_ios="${FLAGS_verity_max_ios}"
-  --verity_algorithm="${FLAGS_verity_algorithm}"
-  --verity_salt="${FLAGS_verity_salt}"
   --keys_dir="${DEVKEYSDIR}"
   --usb_disk="${FLAGS_usb_disk}"
   --nocleanup_dirs
+  --verity_algorithm=sha1
   ${enable_rootfs_verification_flag}
 EOF
 }
@@ -159,3 +163,11 @@ generate_au_zip () {
   info "Running ${lgenerateauzip} ${largs} for generating AU updater zip file"
   $lgenerateauzip $largs
 }
+
+# Basic command to emerge binary packages into the target image.
+# Arguments to this command are passed as addition options/arguments
+# to the basic emerge command.
+emerge_to_image() {
+  sudo -E ${EMERGE_BOARD_CMD} --root-deps=rdeps --usepkgonly -v \
+    "$@" ${EMERGE_JOBS}
+}

build_library/cgpt.py

@@ -0,0 +1,407 @@
+#!/usr/bin/python
+# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import copy
+import json
+import os
+import sys
+
+# First sector we can use.
+START_SECTOR = 64
+
+class ConfigNotFound(Exception):
+  pass
+class PartitionNotFound(Exception):
+  pass
+class InvalidLayout(Exception):
+  pass
+
+
+def LoadPartitionConfig(filename):
+  """Loads a partition tables configuration file into a Python object.
+
+  Args:
+    filename: Filename to load into object
+  Returns:
+    Object containing disk layout configuration
+  """
+
+  if not os.path.exists(filename):
+    raise ConfigNotFound("Partition config %s was not found!" % filename)
+  with open(filename) as f:
+    config = json.load(f)
+    f.close()
+
+  metadata = config["metadata"]
+  metadata["block_size"] = int(metadata["block_size"])
+
+  for layout_name, layout in config["layouts"].items():
+    for part in layout:
+      part["blocks"] = int(part["blocks"])
+      part["bytes"] = part["blocks"] * metadata["block_size"]
+
+      if "fs_blocks" in part:
+        part["fs_blocks"] = int(part["fs_blocks"])
+        part["fs_bytes"] = part["fs_blocks"] * metadata["fs_block_size"]
+
+        if part["fs_bytes"] > part["bytes"]:
+          raise InvalidLayout("Filesystem may not be larger than partition")
+
+  return config
+
+
+def GetTableTotals(config, partitions):
+  """Calculates total sizes/counts for a partition table.
+
+  Args:
+    config: Partition configuration file object
+    partitions: List of partitions to process
+  Returns:
+    Dict containing totals data
+  """
+
+  ret = {
+    "expand_count": 0,
+    "expand_min": 0,
+    "block_count": START_SECTOR * config["metadata"]["block_size"]
+  }
+
+  # Total up the size of all non-expanding partitions to get the minimum
+  # required disk size.
+  for partition in partitions:
+    if "features" in partition and "expand" in partition["features"]:
+      ret["expand_count"] += 1
+      ret["expand_min"] += partition["blocks"]
+    else:
+      ret["block_count"] += partition["blocks"]
+
+  # At present, only one expanding partition is permitted.
+  # Whilst it'd be possible to have two, we don't need this yet
+  # and it complicates things, so it's been left out for now.
+  if ret["expand_count"] > 1:
+    raise InvalidLayout("1 expand partition allowed, %d requested"
+                        % ret["expand_count"])
+
+  ret["min_disk_size"] = ret["block_count"] + ret["expand_min"]
+
+  return ret
+
+
+def GetPartitionTable(config, image_type):
+  """Generates requested image_type layout from a layout configuration.
+
+  This loads the base table and then overlays the requested layout over
+  the base layout.
+
+  Args:
+    config: Partition configuration file object
+    image_type: Type of image eg base/test/dev/factory_install
+  Returns:
+    Object representing a selected partition table
+  """
+
+  partitions = config["layouts"]["base"]
+
+  if image_type != "base":
+    for partition_t in config["layouts"][image_type]:
+      for partition in partitions:
+        if partition["type"] == "blank" or partition_t["type"] == "blank":
+          continue
+        if partition_t["num"] == partition["num"]:
+          for k, v in partition_t.items():
+            partition[k] = v
+
+  return partitions
+
+
+def GetScriptShell():
+  """Loads and returns the skeleton script for our output script.
+
+  Returns:
+    A string containg the skeleton script
+  """
+
+  script_shell_path = os.path.join(os.path.dirname(__file__), "cgpt_shell.sh")
+  with open(script_shell_path, "r") as f:
+    script_shell = "".join(f.readlines())
+    f.close()
+
+  # Before we return, insert the path to this tool so somebody reading the
+  # script later can tell where it was generated.
+  script_shell = script_shell.replace("@SCRIPT_GENERATOR@", script_shell_path)
+
+  return script_shell
+
+
+def WriteLayoutFunction(sfile, func_name, image_type, config):
+  """Writes a shell script function to write out a given partition table.
+
+  Args:
+    sfile: File handle we're writing to
+    func_name: Function name to write out for specified layout
+    image_type: Type of image eg base/test/dev/factory_install
+    config: Partition configuration file object
+  """
+
+  partitions = GetPartitionTable(config, image_type)
+  partition_totals = GetTableTotals(config, partitions)
+
+  sfile.write("%s() {\ncreate_image $1 %d %s\n" % (
+  func_name, partition_totals["min_disk_size"],
+  config["metadata"]["block_size"]))
+
+  sfile.write("CURR=%d\n" % START_SECTOR)
+  sfile.write("$GPT create $1\n")
+
+  # Pass 1: Set up the expanding partition size.
+  for partition in partitions:
+    if partition["type"] != "blank":
+      partition["var"] = partition["blocks"]
+
+      if partition["num"] == 1:
+        if "features" in partition and "expand" in partition["features"]:
+          sfile.write("if [ -b $1 ]; then\n")
+          sfile.write("STATEFUL_SIZE=$(( $(numsectors $1) - %d))\n" %
+            partition_totals["block_count"])
+          sfile.write("else\n")
+          sfile.write("STATEFUL_SIZE=%s\n" % partition["blocks"])
+          sfile.write("fi\n")
+          partition["var"] = "$STATEFUL_SIZE"
+
+  # Pass 2: Write out all the cgpt add commands.
+  for partition in partitions:
+    if partition["type"] != "blank":
+      sfile.write("$GPT add -i %d -b $CURR -s %s -t %s -l %s $1 && " % (
+        partition["num"], str(partition["var"]), partition["type"],
+        partition["label"]))
+
+    # Increment the CURR counter ready for the next partition.
+    sfile.write("CURR=$(( $CURR + %s ))\n" % partition["blocks"])
+
+  # Set default priorities on kernel partitions
+  sfile.write("$GPT add -i 2 -S 0 -T 15 -P 15 $1\n")
+  sfile.write("$GPT add -i 4 -S 0 -T 15 -P 0 $1\n")
+  sfile.write("$GPT add -i 6 -S 0 -T 15 -P 0 $1\n")
+
+  sfile.write("$GPT boot -p -b $2 -i 12 $1\n")
+  sfile.write("$GPT show $1\n")
+  sfile.write("}\n")
+
+
+def GetPartitionByNumber(partitions, num):
+  """Given a partition table and number returns the partition object.
+
+  Args:
+    partitions: List of partitions to search in
+    num: Number of partition to find
+  Returns:
+    An object for the selected partition
+  """
+  for partition in partitions:
+    if partition["type"] == "blank":
+      continue
+    if partition["num"] == int(num):
+      return partition
+
+  raise PartitionNotFound("Partition not found")
+
+
+def WritePartitionScript(image_type, layout_filename, sfilename):
+  """Writes a shell script with functions for the base and requested layouts.
+
+  Args:
+    image_type: Type of image eg base/test/dev/factory_install
+    layout_filename: Path to partition configuration file
+    sfilename: Filename to write the finished script to
+  """
+
+  config = LoadPartitionConfig(layout_filename)
+
+  sfile = open(sfilename, "w")
+  script_shell = GetScriptShell()
+  sfile.write(script_shell)
+
+  WriteLayoutFunction(sfile, "write_base_table", "base", config)
+  WriteLayoutFunction(sfile, "write_partition_table", image_type, config)
+
+  sfile.close()
+
+
+def GetBlockSize(layout_filename):
+  """Returns the partition table block size.
+
+  Args:
+    layout_filename: Path to partition configuration file
+  Returns:
+    Block size of all partitions in the layout
+  """
+
+  config = LoadPartitionConfig(layout_filename)
+  return config["metadata"]["block_size"]
+
+
+def GetFilesystemBlockSize(layout_filename):
+  """Returns the filesystem block size.
+
+  This is used for all partitions in the table that have filesystems.
+
+  Args:
+    layout_filename: Path to partition configuration file
+  Returns:
+    Block size of all filesystems in the layout
+  """
+
+  config = LoadPartitionConfig(layout_filename)
+  return config["metadata"]["fs_block_size"]
+
+
+def GetPartitionSize(image_type, layout_filename, num):
+  """Returns the partition size of a given partition for a given layout type.
+
+  Args:
+    image_type: Type of image eg base/test/dev/factory_install
+    layout_filename: Path to partition configuration file
+    num: Number of the partition you want to read from
+  Returns:
+    Size of selected partition in bytes
+  """
+
+  config = LoadPartitionConfig(layout_filename)
+  partitions = GetPartitionTable(config, image_type)
+  partition = GetPartitionByNumber(partitions, num)
+
+  return partition["bytes"]
+
+
+def GetFilesystemSize(image_type, layout_filename, num):
+  """Returns the filesystem size of a given partition for a given layout type.
+
+  If no filesystem size is specified, returns the partition size.
+
+  Args:
+    image_type: Type of image eg base/test/dev/factory_install
+    layout_filename: Path to partition configuration file
+    num: Number of the partition you want to read from
+  Returns:
+    Size of selected partition filesystem in bytes
+  """
+
+  config = LoadPartitionConfig(layout_filename)
+  partitions = GetPartitionTable(config, image_type)
+  partition = GetPartitionByNumber(partitions, num)
+
+  if "fs_bytes" in partition:
+    return partition["fs_bytes"]
+  else:
+    return partition["bytes"]
+
+
+def GetLabel(image_type, layout_filename, num):
+  """Returns the label for a given partition.
+
+  Args:
+    image_type: Type of image eg base/test/dev/factory_install
+    layout_filename: Path to partition configuration file
+    num: Number of the partition you want to read from
+  Returns:
+    Label of selected partition, or "UNTITLED" if none specified
+  """
+
+  config = LoadPartitionConfig(layout_filename)
+  partitions = GetPartitionTable(config, image_type)
+  partition = GetPartitionByNumber(partitions, num)
+
+  if "label" in partition:
+    return partition["label"]
+  else:
+    return "UNTITLED"
+
+
+def DoDebugOutput(image_type, layout_filename):
+  """Prints out a human readable disk layout in on-disk order.
+
+  This will round values larger than 1MB, it's exists to quickly
+  visually verify a layout looks correct.
+
+  Args:
+    image_type: Type of image eg base/test/dev/factory_install
+    layout_filename: Path to partition configuration file
+  """
+  config = LoadPartitionConfig(layout_filename)
+  partitions = GetPartitionTable(config, image_type)
+
+  for partition in partitions:
+    if partition["bytes"] < 1024 * 1024:
+      size = "%d bytes" % partition["bytes"]
+    else:
+      size = "%d MB" % (partition["bytes"] / 1024 / 1024)
+    if "label" in partition:
+      if "fs_bytes" in partition:
+        if partition["fs_bytes"] < 1024 * 1024:
+          fs_size = "%d bytes" % partition["fs_bytes"]
+        else:
+          fs_size = "%d MB" % (partition["fs_bytes"] / 1024 / 1024)
+        print "%s - %s/%s" % (partition["label"], fs_size, size)
+      else:
+        print "%s - %s" % (partition["label"], size)
+    else:
+      print "blank - %s" %  size
+
+
+def main(argv):
+  action_map = {
+    "write": {
+      "argc": 4,
+      "usage": "<image_type> <partition_config_file> <script_file>",
+      "func": WritePartitionScript
+    },
+    "readblocksize": {
+      "argc": 2,
+      "usage": "<partition_config_file>",
+      "func": GetBlockSize
+    },
+    "readfsblocksize": {
+      "argc": 2,
+      "usage": "<partition_config_file>",
+      "func": GetFilesystemBlockSize
+    },
+    "readpartsize": {
+      "argc": 4,
+      "usage": "<image_type> <partition_config_file> <partition_num>",
+      "func": GetPartitionSize
+    },
+    "readfssize": {
+      "argc": 4,
+      "usage": "<image_type> <partition_config_file> <partition_num>",
+      "func": GetFilesystemSize
+    },
+    "readlabel": {
+      "argc": 4,
+      "usage": "<image_type> <partition_config_file> <partition_num>",
+      "func": GetLabel
+    },
+    "debug": {
+      "argc": 3,
+      "usage": "<image_type> <partition_config_file>",
+      "func": DoDebugOutput
+    }
+  }
+
+  if len(sys.argv) < 2 or sys.argv[1] not in action_map:
+    print "Usage: %s <action>\n" % sys.argv[0]
+    print "Valid actions are:"
+    for action in action_map:
+      print "  %s %s" % (action, action_map[action]["usage"])
+    sys.exit(1)
+  else:
+    action_name = sys.argv[1]
+    action = action_map[action_name]
+    if action["argc"] == len(sys.argv) - 1:
+      print action["func"](*sys.argv[2:])
+    else:
+      sys.exit("Usage: %s %s %s" % (sys.argv[0], sys.argv[1], action["usage"]))
+
+if __name__ == "__main__":
+  main(sys.argv)

build_library/cgpt_shell.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This script is automatically generated by @SCRIPT_GENERATOR@
+# Do not edit!
+
+if ! type numsectors &>/dev/null; then
+  if [[ -f "/usr/sbin/chromeos-common.sh" ]]; then
+      . "/usr/sbin/chromeos-common.sh"
+  else
+    die "Can't load chromeos-common.sh, dying!"
+  fi
+fi
+locate_gpt
+
+# Usage: create_image <device> <min_disk_size> <block_size>
+# If <device> is a block device, wipes out the GPT
+# If it's not, it creates a new file of the requested size
+create_image() {
+  local dev=$1
+  local min_disk_size=$2
+  local block_size=$3
+  if [[ -b "${dev}" ]]; then
+    # Zap any old partitions (otherwise gpt complains).
+    dd if=/dev/zero of="${dev}" conv=notrunc bs=512 count=32
+    dd if=/dev/zero of="${dev}" conv=notrunc bs=512 \
+      seek=$((${min_disk_size} - 1 - 33)) count=33
+  else
+    if [[ ! -e "${dev}" ]]; then
+      dd if=/dev/zero of="${dev}" bs=${block_size} count=1 \
+        seek=$((${min_disk_size} - 1))
+    fi
+  fi
+}
+

build_library/dev_image_util.sh

@@ -16,73 +16,73 @@ install_dev_packages() {
 
   trap "unmount_image ; delete_prompt" EXIT
 
-  mount_image "${BUILD_DIR}/${image_name}" "${ROOT_FS_DIR}" \
+  mount_image "${BUILD_DIR}/${image_name}" "${root_fs_dir}" \
-    "${STATEFUL_FS_DIR}" "${ESP_FS_DIR}"
+    "${stateful_fs_dir}" "${esp_fs_dir}"
 
   # Determine the root dir for developer packages.
-  local root_dev_dir="${ROOT_FS_DIR}/usr/local"
+  local root_dev_dir="${root_fs_dir}/usr/local"
 
   # Install dev-specific init scripts into / from chromeos-dev-init.
-  emerge_to_image --root="${ROOT_FS_DIR}" chromeos-dev-init
+  emerge_to_image --root="${root_fs_dir}" chromeos-dev-init
 
   # Install developer packages described in chromeos-dev.
   emerge_to_image --root="${root_dev_dir}" chromeos-dev
 
   # Copy over the libc debug info so that gdb
   # works with threads and also for a better debugging experience.
-  sudo mkdir -p "${ROOT_FS_DIR}/usr/local/usr/lib/debug"
+  sudo mkdir -p "${root_fs_dir}/usr/local/usr/lib/debug"
   pbzip2 -dc --ignore-trailing-garbage=1 "${LIBC_PATH}" | \
-    sudo tar xpf - -C "${ROOT_FS_DIR}/usr/local/usr/lib/debug" \
+    sudo tar xpf - -C "${root_fs_dir}/usr/local/usr/lib/debug" \
       ./usr/lib/debug/usr/${CHOST} --strip-components=6
   # Since gdb only looks in /usr/lib/debug, symlink the /usr/local
   # path so that it is found automatically.
-  sudo ln -s /usr/local/usr/lib/debug "${ROOT_FS_DIR}/usr/lib/debug"
+  sudo ln -s /usr/local/usr/lib/debug "${root_fs_dir}/usr/lib/debug"
 
   # Install the bare necessary files so that the "emerge" command works
-  sudo cp -a ${root_dev_dir}/share/portage ${ROOT_FS_DIR}/usr/share
+  sudo cp -a ${root_dev_dir}/share/portage ${root_fs_dir}/usr/share
   sudo sed -i s,/usr/bin/wget,wget, \
-    ${ROOT_FS_DIR}/usr/share/portage/config/make.globals
+    ${root_fs_dir}/usr/share/portage/config/make.globals
 
-  sudo mkdir -p ${ROOT_FS_DIR}/etc/make.profile
+  sudo mkdir -p ${root_fs_dir}/etc/make.profile
 
   # Re-run ldconfig to fix /etc/ldconfig.so.cache.
-  sudo /sbin/ldconfig -r "${ROOT_FS_DIR}"
+  sudo /sbin/ldconfig -r "${root_fs_dir}"
 
   # Mark the image as a developer image (input to chromeos_startup).
   # TODO(arkaitzr): Remove this file when applications no longer rely on it
   # (crosbug.com/16648). The preferred way of determining developer mode status
   # is via crossystem cros_debug?1 (checks boot args for "cros_debug").
-  sudo mkdir -p "${ROOT_FS_DIR}/root"
+  sudo mkdir -p "${root_fs_dir}/root"
-  sudo touch "${ROOT_FS_DIR}/root/.dev_mode"
+  sudo touch "${root_fs_dir}/root/.dev_mode"
 
   # Additional changes to developer image.
 
   # Leave core files for developers to inspect.
-  sudo touch "${ROOT_FS_DIR}/root/.leave_core"
+  sudo touch "${root_fs_dir}/root/.leave_core"
 
   # This hack is only needed for devs who have old versions of glibc, which
   # filtered out ldd when cross-compiling.  TODO(davidjames): Remove this hack
   # once everybody has upgraded to a new version of glibc.
-  if [[ ! -x "${ROOT_FS_DIR}/usr/bin/ldd" ]]; then
+  if [[ ! -x "${root_fs_dir}/usr/bin/ldd" ]]; then
-    sudo cp -a "$(which ldd)" "${ROOT_FS_DIR}/usr/bin"
+    sudo cp -a "$(which ldd)" "${root_fs_dir}/usr/bin"
   fi
 
   # If vim is installed, then a vi symlink would probably help.
-  if [[ -x "${ROOT_FS_DIR}/usr/local/bin/vim" ]]; then
+  if [[ -x "${root_fs_dir}/usr/local/bin/vim" ]]; then
-    sudo ln -sf vim "${ROOT_FS_DIR}/usr/local/bin/vi"
+    sudo ln -sf vim "${root_fs_dir}/usr/local/bin/vi"
   fi
 
   # If pygtk is installed in stateful-dev, then install a path.
   if [[ -d \
-      "${ROOT_FS_DIR}/usr/local/lib/python2.6/site-packages/gtk-2.0" ]]; then
+      "${root_fs_dir}/usr/local/lib/python2.6/site-packages/gtk-2.0" ]]; then
     sudo bash -c "\
         echo gtk-2.0 > \
-        ${ROOT_FS_DIR}/usr/local/lib/python2.6/site-packages/pygtk.pth"
+        ${root_fs_dir}/usr/local/lib/python2.6/site-packages/pygtk.pth"
   fi
 
   # File searches /usr/share by default, so add a wrapper script so it
   # can find the right path in /usr/local.
-  local path="${ROOT_FS_DIR}/usr/local/bin/file"
+  local path="${root_fs_dir}/usr/local/bin/file"
   if [[ -x ${path} ]]; then
     sudo mv "${path}" "${path}.bin"
     sudo_clobber "${path}" <<EOF
@@ -94,19 +94,19 @@ EOF
 
   # If python is installed on stateful-dev, fix python symlinks.
   local python_path="/usr/local/bin/python2.6"
-  if [ -e "${ROOT_FS_DIR}${python_path}" ]; then
+  if [ -e "${root_fs_dir}${python_path}" ]; then
     info "Fixing python symlinks for developer and test images."
     local python_paths="/usr/bin/python /usr/local/bin/python \
         /usr/bin/python2 /usr/local/bin/python2"
     for path in ${python_paths}; do
-      sudo rm -f "${ROOT_FS_DIR}${path}"
+      sudo rm -f "${root_fs_dir}${path}"
-      sudo ln -s  ${python_path} "${ROOT_FS_DIR}${path}"
+      sudo ln -s  ${python_path} "${root_fs_dir}${path}"
     done
   fi
 
   info "Developer image built and stored at ${image_name}"
 
-  unmount_image
+  cleanup_mounts
   trap - EXIT
 
   if should_build_image ${image_name}; then

build_library/disk_layout_util.sh

@@ -0,0 +1,201 @@
+# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+CGPT_PY="${BUILD_LIBRARY_DIR}/cgpt.py"
+PARTITION_SCRIPT_PATH="usr/sbin/write_gpt.sh"
+
+get_disk_layout_path() {
+  DISK_LAYOUT_PATH="${BUILD_LIBRARY_DIR}/legacy_disk_layout.json"
+  local partition_script_path=$(tempfile)
+  for overlay in $(cros_overlay_list --board "$BOARD"); do
+    local disk_layout="${overlay}/scripts/disk_layout.json"
+    if [[ -e ${disk_layout} ]]; then
+      DISK_LAYOUT_PATH=${disk_layout}
+    fi
+  done
+}
+
+emit_gpt_scripts() {
+  local image="$1"
+  local dir="$2"
+
+  local pack="${dir}/pack_partitions.sh"
+  local unpack="${dir}/unpack_partitions.sh"
+
+  cat >"$unpack" <<HEADER
+#!/bin/bash -eu
+HEADER
+
+  echo "# File automatically generated by $(basename $0)." >> "$unpack"
+
+  cat >>"${unpack}" <<HEADER
+# Do not edit.
+TARGET=\${1:-}
+if [[ -z "\$TARGET" ]]; then
+  echo "Usage: \$0 DEVICE" 1>&2
+  exit 1
+fi
+set -x
+HEADER
+
+  $GPT show "${image}" | sed -e 's/^/# /' >>"${unpack}"
+  cp "${unpack}" "${pack}"
+
+  $GPT show -q "${image}" |
+    while read start size part x; do
+      local file="part_${part}"
+      local target="\"\${TARGET}\""
+      local dd_args="bs=512 count=${size}"
+      echo "dd if=${target} of=${file} ${dd_args} skip=${start}" >>"${unpack}"
+      echo "dd if=${file} of=${target} ${dd_args} seek=${start} conv=notrunc" \
+        >>"${pack}"
+    done
+
+  chmod +x "${unpack}" "${pack}"
+}
+
+write_partition_script() {
+  local image_type=$1
+  local partition_script_path=$2
+  get_disk_layout_path
+
+  sudo mkdir -p "$(dirname "${partition_script_path}")"
+
+  sudo "${BUILD_LIBRARY_DIR}/cgpt.py" "write" \
+    "${image_type}" "${DISK_LAYOUT_PATH}" "${partition_script_path}"
+}
+
+run_partition_script() {
+  local outdev=$1
+  local root_fs_img=$2
+
+  local pmbr_img
+  case ${ARCH} in
+  arm)
+    pmbr_img=/dev/zero
+    ;;
+  amd64|x86)
+    pmbr_img=$(readlink -f /usr/share/syslinux/gptmbr.bin)
+    ;;
+  *)
+    error "Unknown architecture: $ARCH"
+    return 1
+    ;;
+  esac
+
+  sudo mount -o loop "${root_fs_img}" "${root_fs_dir}"
+  . "${root_fs_dir}/${PARTITION_SCRIPT_PATH}"
+  write_partition_table "${outdev}" "${pmbr_img}"
+  sudo umount "${root_fs_dir}"
+}
+
+get_fs_block_size() {
+  get_disk_layout_path
+
+  echo $(${CGPT_PY} readfsblocksize ${DISK_LAYOUT_PATH})
+}
+
+get_block_size() {
+  get_disk_layout_path
+
+  echo $(${CGPT_PY} readblocksize ${DISK_LAYOUT_PATH})
+}
+
+get_partition_size() {
+  local image_type=$1
+  local part_id=$2
+  get_disk_layout_path
+
+  echo $(${CGPT_PY} readpartsize ${image_type} ${DISK_LAYOUT_PATH} ${part_id})
+}
+
+get_filesystem_size() {
+  local image_type=$1
+  local part_id=$2
+  get_disk_layout_path
+
+  echo $(${CGPT_PY} readfssize ${image_type} ${DISK_LAYOUT_PATH} ${part_id})
+}
+
+get_label() {
+  local image_type=$1
+  local part_id=$2
+  get_disk_layout_path
+
+  echo $(${CGPT_PY} readlabel ${image_type} ${DISK_LAYOUT_PATH} ${part_id})
+}
+
+get_disk_layout_type() {
+  DISK_LAYOUT_TYPE="base"
+  if should_build_image ${CHROMEOS_FACTORY_INSTALL_SHIM_NAME}; then
+    DISK_LAYOUT_TYPE="factory_install"
+  fi
+}
+
+emit_gpt_scripts() {
+  local image="$1"
+  local dir="$2"
+
+  local pack="${dir}/pack_partitions.sh"
+  local unpack="${dir}/unpack_partitions.sh"
+
+  cat >"${unpack}" <<HEADER
+#!/bin/bash -eu
+# File automatically generated. Do not edit.
+TARGET=\${1:-}
+if [[ -z "\$TARGET" ]]; then
+  echo "Usage: \$0 DEVICE" 1>&2
+  exit 1
+fi
+set -x
+HEADER
+
+  $GPT show "${image}" | sed -e 's/^/# /' >>"${unpack}"
+  cp "${unpack}" "${pack}"
+
+  $GPT show -q "${image}" |
+    while read start size part x; do
+      local file="part_${part}"
+      local target="\"\$TARGET\""
+      local dd_args="bs=512 count=${size}"
+      echo "dd if=${target} of=${file} ${dd_args} skip=${start}" >>"${unpack}"
+      echo "dd if=${file} of=${target} ${dd_args} seek=${start} conv=notrunc" \
+        >>"${pack}"
+    done
+
+  chmod +x "${unpack}" "${pack}"
+}
+
+build_gpt() {
+  local outdev="$1"
+  local rootfs_img="$2"
+  local stateful_img="$3"
+  local esp_img="$4"
+
+  get_disk_layout_type
+  run_partition_script "${outdev}" "${rootfs_img}"
+
+  local sudo=
+  if [ ! -w "$outdev" ] ; then
+    # use sudo when writing to a block device.
+    sudo=sudo
+  fi
+
+  # Now populate the partitions.
+  info "Copying stateful partition..."
+  $sudo dd if="$stateful_img" of="$outdev" conv=notrunc bs=512 \
+      seek=$(partoffset ${outdev} 1)
+
+  info "Copying rootfs..."
+  $sudo dd if="$rootfs_img" of="$outdev" conv=notrunc bs=512 \
+      seek=$(partoffset ${outdev} 3)
+
+  info "Copying EFI system partition..."
+  $sudo dd if="$esp_img" of="$outdev" conv=notrunc bs=512 \
+      seek=$(partoffset ${outdev} 12)
+
+  # Pre-set "sucessful" bit in gpt, so we will never mark-for-death
+  # a partition on an SDCard/USB stick.
+  cgpt add -i 2 -S 1 "$outdev"
+}

build_library/legacy_disk_layout.json

@@ -0,0 +1,152 @@
+{
+  "metadata":{
+    "block_size": 512,
+    "fs_block_size": 4096
+  },
+  "layouts":{
+    "base":[
+      {
+        "num": 11,
+        "label":"RWFW",
+        "type":"firmware",
+        "blocks":"16384"
+      },
+      {
+        "num": 6,
+        "label":"KERN-C",
+        "type":"kernel",
+        "blocks":"1"
+      },
+      {
+        "num": 7,
+        "label":"ROOT-C",
+        "type":"rootfs",
+        "blocks":"1"
+      },
+      {
+        "num": 9,
+        "type":"reserved",
+        "label":"reserved",
+        "blocks":"1"
+      },
+      {
+        "num": 10,
+        "type":"reserved",
+        "label":"reserved",
+        "blocks":"1"
+      },
+      {
+        "type":"blank",
+        "blocks":"4028"
+      },
+      {
+        "num": 2,
+        "label":"KERN-A",
+        "type":"kernel",
+        "blocks":"32768"
+      },
+      {
+        "num": 4,
+        "label":"KERN-B",
+        "type":"kernel",
+        "blocks":"32768"
+      },
+      {
+        "num": 8,
+        "label":"OEM",
+        "type":"data",
+        "blocks":"32768"
+      },
+      {
+        "type":"blank",
+        "blocks":"131072"
+      },
+      {
+        "num": 12,
+        "label":"EFI-SYSTEM",
+        "type":"efi",
+        "blocks":"32768"
+      },
+      {
+        "num": 1,
+        "label":"STATE",
+        "type":"data",
+        "blocks":"2097152",
+        "features":["expand"]
+      },
+      {
+        "num": 5,
+        "label":"ROOT-B",
+        "type":"rootfs",
+        "blocks":"1"
+      },
+      {
+        "num": 3,
+        "label":"ROOT-A",
+        "type":"rootfs",
+        "blocks":"1757184",
+        "fs_blocks":"217600"
+      }
+    ],
+    "factory_install": [
+      {
+        "num": 1,
+        "label":"STATE",
+        "type":"data",
+        "blocks":"286720"
+      },
+      {
+        "num": 3,
+        "label":"ROOT-A",
+        "type":"rootfs",
+        "blocks":"860160",
+        "fs_blocks":"102400"
+      },
+      {
+        "num": 12,
+        "label":"EFI-SYSTEM",
+        "type":"efi",
+        "blocks":"65536"
+      }
+    ],
+    "vm": [
+      {
+        "num": 1,
+        "label":"STATE",
+        "type":"data",
+        "blocks":"6291456"
+      },
+      {
+        "num": 3,
+        "label":"ROOT-A",
+        "type":"rootfs",
+        "blocks":"2097152",
+        "fs_blocks":"217600"
+      },
+      {
+        "num": 5,
+        "label":"ROOT-B",
+        "type":"rootfs",
+        "blocks":"2097152",
+        "fs_blocks":"217600"
+      }
+    ],
+    "recovery": [
+      {
+        "num": 1,
+        "label":"STATE",
+        "type":"data",
+        "blocks":"4096"
+      }
+    ],
+    "pgo": [
+      {
+        "num": 3,
+        "label":"ROOT-A",
+        "type":"rootfs",
+        "blocks":"2662400",
+        "fs_blocks":"320000"
+      }
+    ]
+  }
+}

build_library/test_image_util.sh

@@ -7,77 +7,13 @@
 # functions and initialization shared between build_image and
 # mod_image_for_test.sh.
 #
-# TODO(jrbarnette): The two halves of this file aren't particularly
-# related; they're together merely to consolidate the shared code in
-# one file.  Arguably, they should be broken up.
-
-
-# ----
-# The initialization and functions below are shared between
-# build_image and mod_image_for_test.sh.  The code is not used
-# by the mod_image_for_test function.
-
-EMERGE_BOARD_CMD="$GCLIENT_ROOT/chromite/bin/parallel_emerge"
-EMERGE_BOARD_CMD="$EMERGE_BOARD_CMD --board=$BOARD"
-
-if [ $FLAGS_jobs -ne -1 ]; then
-  EMERGE_JOBS="--jobs=$FLAGS_jobs"
-fi
-
-export INSTALL_MASK="${DEFAULT_INSTALL_MASK}"
-
-
-# Utility function for creating a copy of an image prior to
-# modification from the BUILD_DIR:
-#  $1: source filename
-#  $2: destination filename
-copy_image() {
-  local src="${BUILD_DIR}/$1"
-  local dst="${BUILD_DIR}/$2"
-  if should_build_image $1; then
-    echo "Creating $2 from $1..."
-    $COMMON_PV_CAT "$src" >"$dst" || die "Cannot copy $1 to $2"
-  else
-    mv "${src}" "${dst}" || die "Cannot move $1 to $2"
-  fi
-}
-
-# Basic command to emerge binary packages into the target image.
-# Arguments to this command are passed as addition options/arguments
-# to the basic emerge command.
-emerge_to_image() {
-  sudo -E ${EMERGE_BOARD_CMD} --root-deps=rdeps --usepkgonly -v \
-    "$@" ${EMERGE_JOBS}
-}
-
-# Returns 0 if any of the images was requested to be built, 1 otherwise.
-# $@ The name(s) of the images to check.
-should_build_image() {
-  # Fast pass back if we should build all incremental images.
-  local images="$@"
-  local image_name
-  local image_to_build
-
-  for image_name in ${images}; do
-    for image_to_build in ${IMAGES_TO_BUILD}; do
-      [ ${image_to_build} = ${image_name} ] && return 0
-    done
-  done
-
-  return 1
-}
-
-# ----
-# From here down, the main exported function is
-# 'mod_image_for_test'.  The remainder of the code is not used
-# outside this file.
 
 # Emerges chromeos-test onto the image.
 emerge_chromeos_test() {
   # Determine the root dir for test packages.
-  local root_dev_dir="${ROOT_FS_DIR}/usr/local"
+  local root_dev_dir="${root_fs_dir}/usr/local"
 
-  emerge_to_image --root="${ROOT_FS_DIR}" chromeos-test-init
+  emerge_to_image --root="${root_fs_dir}" chromeos-test-init
   emerge_to_image --root="${root_dev_dir}" chromeos-test
 }
 
@@ -104,7 +40,7 @@ mod_image_for_test () {
 
   trap unmount_image EXIT
   mount_image "${BUILD_DIR}/${image_name}" \
-    "${ROOT_FS_DIR}" "${STATEFUL_FS_DIR}"
+    "${root_fs_dir}" "${stateful_fs_dir}"
 
   emerge_chromeos_test
 
@@ -115,8 +51,8 @@ mod_image_for_test () {
 
   local mod_test_script="${SCRIPTS_DIR}/mod_for_test_scripts/test_setup.sh"
   # Run test setup script to modify the image
-  sudo -E GCLIENT_ROOT="${GCLIENT_ROOT}" ROOT_FS_DIR="${ROOT_FS_DIR}" \
+  sudo -E GCLIENT_ROOT="${GCLIENT_ROOT}" ROOT_FS_DIR="${root_fs_dir}" \
-    STATEFUL_DIR="${STATEFUL_FS_DIR}" ARCH="${ARCH}" BACKDOOR="${BACKDOOR}" \
+    STATEFUL_DIR="${stateful_fs_dir}" ARCH="${ARCH}" BACKDOOR="${BACKDOOR}" \
     BOARD_ROOT="${BOARD_ROOT}" \
     "${mod_test_script}"
 
@@ -125,9 +61,9 @@ mod_image_for_test () {
 
   if [ ${FLAGS_factory} -eq ${FLAGS_TRUE} ] ||
       should_build_image "${CHROMEOS_FACTORY_TEST_IMAGE_NAME}"; then
-    emerge_to_image --root="${ROOT_FS_DIR}" factorytest-init
+    emerge_to_image --root="${root_fs_dir}" factorytest-init
     INSTALL_MASK="${FACTORY_TEST_INSTALL_MASK}"
-    emerge_to_image --root="${ROOT_FS_DIR}/usr/local" \
+    emerge_to_image --root="${root_fs_dir}/usr/local" \
       chromeos-base/autotest chromeos-base/autotest-all \
       chromeos-base/chromeos-factory
     prepare_hwid_for_factory "${BUILD_DIR}"
@@ -135,14 +71,14 @@ mod_image_for_test () {
     local mod_factory_script
     mod_factory_script="${SCRIPTS_DIR}/mod_for_factory_scripts/factory_setup.sh"
     # Run factory setup script to modify the image
-    sudo -E GCLIENT_ROOT="${GCLIENT_ROOT}" ROOT_FS_DIR="${ROOT_FS_DIR}" \
+    sudo -E GCLIENT_ROOT="${GCLIENT_ROOT}" ROOT_FS_DIR="${root_fs_dir}" \
             BOARD="${BOARD}" "${mod_factory_script}"
   fi
 
   # Re-run ldconfig to fix /etc/ldconfig.so.cache.
-  sudo ldconfig -r "${ROOT_FS_DIR}"
+  sudo ldconfig -r "${root_fs_dir}"
 
-  unmount_image
+  cleanup_mounts
   trap - EXIT
 
   # Now make it bootable with the flags from build_image.

common.sh

@@ -225,6 +225,7 @@ GCLIENT_ROOT=$(readlink -f "$GCLIENT_ROOT")
 SRC_ROOT="$GCLIENT_ROOT/src"
 SRC_INTERNAL="$GCLIENT_ROOT/src-internal"
 SCRIPTS_DIR="$SRC_ROOT/scripts"
+BUILD_LIBRARY_DIR=${SCRIPTS_DIR}/build_library
 
 # Load developer's custom settings.  Default location is in scripts dir,
 # since that's available both inside and outside the chroot.  By convention,

image_to_vm.sh

@@ -10,6 +10,8 @@
 # Helper scripts should be run from the same location as this script.
 SCRIPT_ROOT=$(dirname "$(readlink -f "$0")")
 . "${SCRIPT_ROOT}/common.sh" || exit 1
+. "${SCRIPT_ROOT}/build_library/disk_layout_util.sh" || exit 1
+. "${SCRIPT_ROOT}/build_library/build_common.sh" || exit 1
 
 # Need to be inside the chroot to load chromeos-common.sh
 assert_inside_chroot
@@ -33,30 +35,34 @@ DEFINE_string format "qemu" \
   "Output format, either qemu, vmware or virtualbox"
 DEFINE_string from "" \
   "Directory containing rootfs.image and mbr.image"
-DEFINE_boolean full "${FLAGS_FALSE}" "Build full image with all partitions."
 DEFINE_boolean make_vmx ${FLAGS_TRUE} \
   "Create a vmx file for use with vmplayer (vmware only)."
 DEFINE_integer mem "${DEFAULT_MEM}" \
   "Memory size for the vm config in MBs (vmware only)."
-DEFINE_integer rootfs_partition_size 1024 \
-  "rootfs parition size in MBs."
 DEFINE_string state_image "" \
   "Stateful partition image (defaults to creating new statful partition)"
-DEFINE_integer statefulfs_size 2048 \
-  "Stateful partition size in MBs."
 DEFINE_boolean test_image "${FLAGS_FALSE}" \
   "Copies normal image to ${CHROMEOS_TEST_IMAGE_NAME}, modifies it for test."
 DEFINE_string to "" \
   "Destination folder for VM output file(s)"
 DEFINE_string vbox_disk "${DEFAULT_VBOX_DISK}" \
   "Filename for the output disk (virtualbox only)."
-DEFINE_integer vdisk_size 3072 \
-  "virtual disk size in MBs."
 DEFINE_string vmdk "${DEFAULT_VMDK}" \
   "Filename for the vmware disk image (vmware only)."
 DEFINE_string vmx "${DEFAULT_VMX}" \
   "Filename for the vmware config (vmware only)."
 
+# The following arguments are ignored.
+# They are here as part of a transition for CL #29931 beacuse the buildbots
+# specify these arguments.
+DEFINE_integer vdisk_size 3072 \
+  "virtual disk size in MBs."
+DEFINE_boolean full "${FLAGS_FALSE}" "Build full image with all partitions."
+DEFINE_integer rootfs_partition_size 1024 \
+  "rootfs parition size in MBs."
+DEFINE_integer statefulfs_size 2048 \
+  "Stateful partition size in MBs."
+
 # Parse command line
 FLAGS "$@" || exit 1
 eval set -- "${FLAGS_ARGV}"
@@ -68,15 +74,7 @@ if [ -z "${FLAGS_board}" ] ; then
   die_notrace "--board is required."
 fi
 
-if [ "${FLAGS_full}" -eq "${FLAGS_TRUE}" ] && \
+BOARD="$FLAGS_board"
-   [[ ${FLAGS_vdisk_size} < ${MIN_VDISK_SIZE_FULL} || \
-      ${FLAGS_statefulfs_size} < ${MIN_STATEFUL_FS_SIZE_FULL} ]]; then
-  warn "Disk is too small for full, using minimum:  vdisk size equal to \
-${MIN_VDISK_SIZE_FULL} and statefulfs size equal to \
-${MIN_STATEFUL_FS_SIZE_FULL}."
-  FLAGS_vdisk_size=${MIN_VDISK_SIZE_FULL}
-  FLAGS_statefulfs_size=${MIN_STATEFUL_FS_SIZE_FULL}
-fi
 
 IMAGES_DIR="${DEFAULT_BUILD_ROOT}/images/${FLAGS_board}"
 # Default to the most recent image
@@ -125,20 +123,17 @@ TEMP_KERN="${TEMP_DIR}"/part_2
 if [ -n "${FLAGS_state_image}" ]; then
   TEMP_STATE="${FLAGS_state_image}"
 else
-  # If we have a stateful fs size specified create a new state partition
+  STATEFUL_SIZE_BYTES=$(get_filesystem_size vm 1)
-  # of the specified size.
+  STATEFUL_SIZE_MEGABYTES=$(( STATEFUL_SIZE_BYTES / 1024 / 1024 ))
-  if [ "${FLAGS_statefulfs_size}" -ne -1 ]; then
-    STATEFUL_SIZE_BYTES=$((1024 * 1024 * ${FLAGS_statefulfs_size}))
   original_image_size=$(stat -c%s "${TEMP_STATE}")
   if [ "${original_image_size}" -gt "${STATEFUL_SIZE_BYTES}" ]; then
     die "Cannot resize stateful image to smaller than original. Exiting."
   fi
 
-    echo "Resizing stateful partition to ${FLAGS_statefulfs_size}MB"
+  echo "Resizing stateful partition to ${STATEFUL_SIZE_MEGABYTES}MB"
   # Extend the original file size to the new size.
   sudo e2fsck -pf "${TEMP_STATE}"
-    sudo resize2fs "${TEMP_STATE}" ${FLAGS_statefulfs_size}M
+  sudo resize2fs "${TEMP_STATE}" ${STATEFUL_SIZE_MEGABYTES}M
-  fi
 fi
 TEMP_PMBR="${TEMP_DIR}"/pmbr
 dd if="${SRC_IMAGE}" of="${TEMP_PMBR}" bs=512 count=1
@@ -164,26 +159,22 @@ sudo sed -i -e 's/sdb3/sda3/g' "${TEMP_MNT}/boot/syslinux/usb.A.cfg"
 trap - INT TERM EXIT
 cleanup
 
-# TOOD(adlr): pick a size that will for sure accomodate the partitions.
+# Set up a new partition table
-dd if=/dev/zero of="${TEMP_IMG}" bs=1 count=1 \
+PARTITION_SCRIPT_PATH=$( tempfile )
-  seek=$((${FLAGS_vdisk_size} * 1024 * 1024 - 1))
+write_partition_script "vm" "${PARTITION_SCRIPT_PATH}"
+. "${PARTITION_SCRIPT_PATH}"
+write_partition_table "${TEMP_IMG}" "${TEMP_PMBR}"
+rm "${PARTITION_SCRIPT_PATH}"
 
-GPT_FULL="false"
-[ "${FLAGS_full}" -eq "${FLAGS_TRUE}" ] && GPT_FULL="true"
-
-# Set up the partition table
-install_gpt "${TEMP_IMG}" "$(numsectors $TEMP_ROOTFS)" \
-  "$(numsectors $TEMP_STATE)" "${TEMP_PMBR}" "$(numsectors $TEMP_ESP)" \
-  "${GPT_FULL}" ${FLAGS_rootfs_partition_size}
 # Copy into the partition parts of the file
 dd if="${TEMP_ROOTFS}" of="${TEMP_IMG}" conv=notrunc bs=512 \
-  seek="${START_ROOTFS_A}"
+  seek=$(partoffset ${TEMP_IMG} 3)
 dd if="${TEMP_STATE}"  of="${TEMP_IMG}" conv=notrunc bs=512 \
-  seek="${START_STATEFUL}"
+  seek=$(partoffset ${TEMP_IMG} 1)
 dd if="${TEMP_KERN}"   of="${TEMP_IMG}" conv=notrunc bs=512 \
-  seek="${START_KERN_A}"
+  seek=$(partoffset ${TEMP_IMG} 2)
 dd if="${TEMP_ESP}"    of="${TEMP_IMG}" conv=notrunc bs=512 \
-  seek="${START_ESP}"
+  seek=$(partoffset ${TEMP_IMG} 12)
 
 # Make the built-image bootable and ensure that the legacy default usb boot
 # uses /dev/sda instead of /dev/sdb3.

mod_image_for_recovery.sh

@@ -11,6 +11,7 @@
 
 SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
 . "${SCRIPT_ROOT}/build_library/build_common.sh" || exit 1
+. "${SCRIPT_ROOT}/build_library/disk_layout_util.sh" || exit 1
 
 # Default recovery kernel name.
 RECOVERY_KERNEL_NAME=recovery_vmlinuz.image
@@ -126,8 +127,8 @@ BOAT
 create_recovery_kernel_image() {
   local sysroot="$FACTORY_ROOT"
   local vmlinuz="$sysroot/boot/vmlinuz"
-  local root_offset=$(partoffset "$FLAGS_image" 3)
+  local root_offset=$(partoffset "${RECOVERY_IMAGE}" 3)
-  local root_size=$(partsize "$FLAGS_image" 3)
+  local root_size=$(partsize "${RECOVERY_IMAGE}" 3)
 
   local enable_rootfs_verification_flag=--noenable_rootfs_verification
   if grep -q enable_rootfs_verification "${IMAGE_DIR}/boot.desc"; then
@@ -144,12 +145,12 @@ create_recovery_kernel_image() {
   # recovery image generation.  (Alternately, it means an image can be created,
   # modified for recovery, then passed to a signer which can then sign both
   # partitions appropriately without needing any external dependencies.)
-  local kern_offset=$(partoffset "$FLAGS_image" 2)
+  local kern_offset=$(partoffset "${RECOVERY_IMAGE}" 2)
-  local kern_size=$(partsize "$FLAGS_image" 2)
+  local kern_size=$(partsize "${RECOVERY_IMAGE}" 2)
   local kern_tmp=$(mktemp)
   local kern_hash=
 
-  dd if="$FLAGS_image" bs=512 count=$kern_size \
+  dd if="${RECOVERY_IMAGE}" bs=512 count=$kern_size \
      skip=$kern_offset of="$kern_tmp" 1>&2
   # We're going to use the real signing block.
   if [ $FLAGS_sync_keys -eq $FLAGS_TRUE ]; then
@@ -170,24 +171,23 @@ create_recovery_kernel_image() {
     --keys_dir="${FLAGS_keys_dir}" \
     ${enable_rootfs_verification_flag} \
     --nouse_dev_keys 1>&2 || failboat "build_kernel_image"
-  sudo mount | sed 's/^/16651 /'
+  #sudo mount | sed 's/^/16651 /'
-  sudo losetup -a | sed 's/^/16651 /'
+  #sudo losetup -a | sed 's/^/16651 /'
   trap - RETURN
 
   # Update the EFI System Partition configuration so that the kern_hash check
   # passes.
-  local efi_offset=$(partoffset "$FLAGS_image" 12)
+  local block_size=$(get_block_size)
-  local efi_size=$(partsize "$FLAGS_image" 12)
+
+  local efi_offset=$(partoffset "${RECOVERY_IMAGE}" 12)
+  local efi_size=$(partsize "${RECOVERY_IMAGE}" 12)
+  local efi_offset_bytes=$(( $efi_offset * $block_size ))
+  local efi_size_bytes=$(( $efi_size * $block_size ))
 
-  local efi_dev=$(sudo losetup --show -f \
-                  -o $((efi_offset * 512)) \
-                  --sizelimit $((efi_size * 512)) \
-                  "$FLAGS_image")
   local efi_dir=$(mktemp -d)
-  trap "sudo losetup -d $efi_dev && rmdir \"$efi_dir\"" EXIT
+  sudo mount -o loop,offset=${efi_offset_bytes},sizelimit=${efi_size_bytes} \
-  echo "16651 mount: $efi_dev -> $efi_dir"
+    "${RECOVERY_IMAGE}" "${efi_dir}"
-  sudo mount "$efi_dev" "$efi_dir"
+
-  sudo mount | sed 's/^/16651 /'
   sudo sed  -i -e "s/cros_legacy/cros_legacy kern_b_hash=$kern_hash/g" \
     "$efi_dir/syslinux/usb.A.cfg" || true
   # This will leave the hash in the kernel for all boots, but that should be
@@ -195,8 +195,6 @@ create_recovery_kernel_image() {
   sudo sed  -i -e "s/cros_efi/cros_efi kern_b_hash=$kern_hash/g" \
     "$efi_dir/efi/boot/grub.cfg" || true
   safe_umount "$efi_dir"
-  sudo losetup -a | sed 's/^/16651 /'
-  sudo losetup -d "$efi_dev"
   rmdir "$efi_dir"
   trap - EXIT
 }
@@ -269,48 +267,59 @@ update_partition_table() {
   local resized_sectors=$3  # number of sectors in resized stateful partition
   local temp_img=$4
 
-  local kern_a_offset=$(partoffset ${src_img} 2)
+  local kern_a_dst_offset=$(partoffset ${temp_img} 2)
+  local kern_a_src_offset=$(partoffset ${src_img} 2)
   local kern_a_count=$(partsize ${src_img} 2)
-  local kern_b_offset=$(partoffset ${src_img} 4)
+
+  local kern_b_dst_offset=$(partoffset ${temp_img} 4)
+  local kern_b_src_offset=$(partoffset ${src_img} 4)
   local kern_b_count=$(partsize ${src_img} 4)
-  local rootfs_offset=$(partoffset ${src_img} 3)
+
+  local rootfs_dst_offset=$(partoffset ${temp_img} 3)
+  local rootfs_src_offset=$(partoffset ${src_img} 3)
   local rootfs_count=$(partsize ${src_img} 3)
-  local oem_offset=$(partoffset ${src_img} 8)
+
+  local oem_dst_offset=$(partoffset ${temp_img} 8)
+  local oem_src_offset=$(partoffset ${src_img} 8)
   local oem_count=$(partsize ${src_img} 8)
-  local esp_offset=$(partoffset ${src_img} 12)
+
+  local esp_dst_offset=$(partoffset ${temp_img} 12)
+  local esp_src_offset=$(partoffset ${src_img} 12)
   local esp_count=$(partsize ${src_img} 12)
 
+  local state_dst_offset=$(partoffset ${temp_img} 1)
+
   local temp_pmbr=$(mktemp "/tmp/pmbr.XXXXXX")
   dd if="${src_img}" of="${temp_pmbr}" bs=512 count=1 &>/dev/null
 
   trap "rm -rf \"${temp_pmbr}\"" EXIT
   # Set up a new partition table
-  install_gpt "${temp_img}" "${rootfs_count}" "${resized_sectors}" \
+  PARTITION_SCRIPT_PATH=$( tempfile )
-    "${temp_pmbr}" "${esp_count}" false \
+  write_partition_script "recovery" "${PARTITION_SCRIPT_PATH}"
-    $(((rootfs_count * 512)/(1024 * 1024)))
+  . "${PARTITION_SCRIPT_PATH}"
+  write_partition_table "${temp_img}" "${temp_pmbr}"
+  echo "${PARTITION_SCRIPT_PATH}"
+  #rm "${PARTITION_SCRIPT_PATH}"
 
   # Copy into the partition parts of the file
   dd if="${src_img}" of="${temp_img}" conv=notrunc bs=512 \
-    seek="${START_ROOTFS_A}" skip=${rootfs_offset} count=${rootfs_count}
+    seek=${kern_a_dst_offset} skip=${kern_a_src_offset} count=${rootfs_count}
   dd if="${temp_state}" of="${temp_img}" conv=notrunc bs=512 \
-    seek="${START_STATEFUL}"
+    seek=${state_dst_offset}
   # Copy the full kernel (i.e. with vboot sections)
   dd if="${src_img}" of="${temp_img}" conv=notrunc bs=512 \
-    seek="${START_KERN_A}" skip=${kern_a_offset} count=${kern_a_count}
+    seek=${kern_a_dst_offset} skip=${kern_a_src_offset} count=${kern_a_count}
   dd if="${src_img}" of="${temp_img}" conv=notrunc bs=512 \
-    seek="${START_KERN_B}" skip=${kern_b_offset} count=${kern_b_count}
+    seek=${kern_b_dst_offset} skip=${kern_b_src_offset} count=${kern_b_count}
   dd if="${src_img}" of="${temp_img}" conv=notrunc bs=512 \
-    seek="${START_OEM}" skip=${oem_offset} count=${oem_count}
+    seek=${oem_dst_offset} skip=${oem_src_offset} count=${oem_count}
   dd if="${src_img}" of="${temp_img}" conv=notrunc bs=512 \
-    seek="${START_ESP}" skip=${esp_offset} count=${esp_count}
+    seek=${esp_dst_offset} skip=${esp_src_offset} count=${esp_count}
 }
 
 maybe_resize_stateful() {
   # If we're not minimizing, then just copy and go.
   if [ $FLAGS_minimize_image -eq $FLAGS_FALSE ]; then
-    if [ "$FLAGS_image" != "$RECOVERY_IMAGE" ]; then
-      cp "$FLAGS_image" "$RECOVERY_IMAGE"
-    fi
     return 0
   fi
 
@@ -337,7 +346,7 @@ maybe_resize_stateful() {
   # Create a recovery image of the right size
   # TODO(wad) Make the developer script case create a custom GPT with
   # just the kernel image and stateful.
-  update_partition_table "$FLAGS_image" "$small_stateful" 4096 \
+  update_partition_table "${RECOVERY_IMAGE}" "$small_stateful" 4096 \
                          "$RECOVERY_IMAGE" 1>&2
   return $err
 }
@@ -395,6 +404,10 @@ if [ $FLAGS_modify_in_place -eq $FLAGS_TRUE ]; then
     die_notrace "Cannot use --modify_in_place and --minimize_image together."
   fi
   RECOVERY_IMAGE="${FLAGS_image}"
+else
+  if [[ ${FLAGS_modify_in_place} -eq ${FLAGS_FALSE} ]]; then
+    cp "${FLAGS_image}" "${RECOVERY_IMAGE}"
+  fi
 fi
 
 echo "Creating recovery image from ${FLAGS_image}"
@@ -422,10 +435,6 @@ if [ $FLAGS_kernel_image_only -eq $FLAGS_TRUE ]; then
   exit 0
 fi
 
-if [ $FLAGS_modify_in_place -eq $FLAGS_FALSE ]; then
-  rm "$RECOVERY_IMAGE" || true  # Start fresh :)
-fi
-
 trap cleanup EXIT
 
 maybe_resize_stateful  # Also copies the image if needed.