From 45319e9ad0d307850c20d15dce39addee3cb726b Mon Sep 17 00:00:00 2001
From: Ken Mixter <kmixter@google.com>
Date: Thu, 21 Jan 2010 14:58:40 -0800
Subject: [PATCH] Add a script to simplify some common developer workflows.

Here are some example ways to use the script (from comments)

sync_build_test.sh
  syncs, recreates local repo and chroot, builds, and masters an
  image in the checkout based on your current directory, or if you
  are not in a checkout, based on the top level directory the script
  is run from.

sync_build_test.sh --image_to_usb=/dev/sdb -i
  same as above but then images USB device /dev/sdb with the image.
  Also prompt the user in advance of the steps we'll take to make
  sure they agrees.

sync_build_test.sh --top=~/foo --nosync --remote 192.168.1.2
  builds and masters an image in ~/foo, and live updates the machine
  at 192.168.1.2 with that image.

sync_build_test.sh --top=~/newdir --test "Pam BootPerfServer" \
     --remote=192.168.1.2
  creates a new checkout in ~/newdir, builds and masters an image
  which is live updated to 192.168.1.2 and then runs
  two tests (Pam and BootPerfServer) against that machine.

sync_build_test.sh --grab_buildbot=LATEST --test Pam --remote=192.168.1.2
  grabs the latest build from the buildbot, properly modifies it,
  reimages 192.168.1.2, and runs the given test on it.

Review URL: http://codereview.chromium.org/548094
---
 sync_build_test.sh | 459 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 459 insertions(+)
 create mode 100755 sync_build_test.sh

diff --git a/sync_build_test.sh b/sync_build_test.sh
new file mode 100755
index 0000000000..763cc6a2cd
--- /dev/null
+++ b/sync_build_test.sh
@@ -0,0 +1,459 @@
+#!/bin/bash
+
+# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Script to sync your checkout, build a Chromium OS image, and test it all
+# with one command.  Can also check out a new Chromium OS checkout and
+# perform a subset of the above operations.
+#
+# Here are some example runs:
+#
+# sync_build_test.sh
+#   syncs, recreates local repo and chroot, builds, and masters an
+#   image in the checkout based on your current directory, or if you
+#   are not in a checkout, based on the top level directory the script
+#   is run from.
+# 
+# sync_build_test.sh --image_to_usb=/dev/sdb -i
+#   same as above but then images USB device /dev/sdb with the image.
+#   Also prompt the user in advance of the steps we'll take to make
+#   sure they agrees.
+#
+# sync_build_test.sh --top=~/foo --nosync --remote 192.168.1.2
+#   builds and masters an image in ~/foo, and live updates the machine
+#   at 192.168.1.2 with that image.
+#
+# sync_build_test.sh --top=~/newdir --test "Pam BootPerfServer" \
+#      --remote=192.168.1.2
+#   creates a new checkout in ~/newdir, builds and masters an image
+#   which is live updated to 192.168.1.2 and then runs
+#   two tests (Pam and BootPerfServer) against that machine.
+#
+# sync_build_test.sh --grab_buildbot=LATEST --test Pam --remote=192.168.1.2
+#   grabs the latest build from the buildbot, properly modifies it,
+#   reimages 192.168.1.2, and runs the given test on it.   
+#
+# Environment variables that may be useful:
+#   BUILDBOT_URI - default value for --buildbot_uri
+#   CHROMIUM_REPO - default value for --repo
+#   CHRONOS_PASSWD - default value for --chronos_passwd
+#
+
+# Load common constants.  This should be the first executable line.
+# The path to common.sh should be relative to your script's location.
+. "$(dirname "$0")/common.sh"
+
+
+DEFINE_string top "" \
+    "Root directory of your checkout (defaults to determining from your cwd)"
+DEFINE_string repo "${CHROMIUMOS_REPO}" "gclient repo for chromiumos"
+DEFINE_boolean sync ${FLAGS_TRUE} "Sync the checkout"
+DEFINE_boolean force_make_local_repo ${FLAGS_FALSE} \
+    "Run make_local_repo indep of sync"
+DEFINE_boolean force_make_chroot ${FLAGS_FALSE} "Run make_chroot indep of sync"
+DEFINE_boolean build ${FLAGS_TRUE} \
+    "Build all code (but not necessarily master image)"
+DEFINE_boolean master ${FLAGS_TRUE} "Master an image from built code"
+DEFINE_string grab_buildbot "" \
+    "Instead of building, grab this full image.zip URI generated by the \
+buildbot"
+DEFINE_string chronos_passwd "${CHRONOS_PASSWD}" \
+    "Use this as the chronos user passwd (defaults to \$CHRONOS_PASSWD)"
+DEFINE_boolean mod_image_for_test ${FLAGS_FALSE} "Modify the image for testing"
+DEFINE_boolean image_to_live ${FLAGS_FALSE} \
+    "Put the resulting image on live instance (requires --remote)"
+DEFINE_string remote "" \
+    "Use this hostname/IP for live updating and running tests"
+DEFINE_string image_to_usb "" \
+    "Treat this device as USB and put the image on it after build"
+DEFINE_string test "" \
+    "Test the built image with the given params to run_remote_tests"
+DEFINE_string buildbot_uri "${BUILDBOT_URI}" \
+    "Base URI to buildbot build location which contains LATEST file"
+DEFINE_boolean unittest ${FLAGS_TRUE} "Run unit tests"
+DEFINE_boolean interactive ${FLAGS_FALSE} \
+    "Tell user what we plan to do and wait for input to proceed" i
+
+
+# Returns a heuristic indicating if we believe this to be a google internal
+# development environment.
+# Returns:
+#   0 if so, 1 otherwise
+function is_google_environment() {
+  hostname | egrep -q .google.com\$
+  return $?
+}
+
+
+# Validates parameters and sets "intelligent" defaults based on other
+# parameters.
+function validate_and_set_param_defaults() {
+  if [[ -z "${FLAGS_top}" ]]; then
+    local test_dir=$(pwd)
+    while [[ "${test_dir}" != "/" ]]; do
+      if [[ -d "${test_dir}/src/platform/pam_google" ]]; then
+        FLAGS_top="${test_dir}"
+        break
+      fi
+      test_dir=$(dirname "${test_dir}")
+    done
+  fi
+
+  if [[ -z "${FLAGS_top}" ]]; then
+    # Use the top directory based on where this script runs from
+    FLAGS_top=$(dirname $(dirname $(dirname $0)))
+  fi
+
+  if [[ -z "${FLAGS_repo}" ]]; then
+    if is_google_environment; then
+      FLAGS_repo="ssh://git@chromiumos-git//chromeos"
+    else
+      FLAGS_repo="http://src.chromium.org/git/chromiumos.git"
+    fi
+  fi
+
+  if [[ -n "${FLAGS_test}" ]]; then
+    # If you specify that tests should be run, we assume the image
+    # is modified to run tests.
+    FLAGS_mod_image_for_test=${FLAGS_TRUE}
+    # If you specify that tests should be run, we assume you want
+    # to live update the image.
+    FLAGS_image_to_live=${FLAGS_TRUE}
+  fi
+
+  # If they gave us a remote host, then we assume they want us to do a live
+  # update.
+  if [[ -n "${FLAGS_remote}" ]]; then
+    FLAGS_image_to_live=${FLAGS_TRUE}
+  fi
+
+  # Grabbing a buildbot build is exclusive with building
+  if [[ -n "${FLAGS_grab_buildbot}" ]]; then
+    if [[ -z "${FLAGS_buildbot_uri}" ]]; then
+      echo "--grab_buildbot requires --buildbot_uri"
+      exit 1
+    fi
+    FLAGS_build=${FLAGS_FALSE}
+    FLAGS_master=${FLAGS_FALSE}
+  fi
+
+  if [[ ${FLAGS_image_to_live} -eq ${FLAGS_TRUE} ]]; then
+    if [[ ${FLAGS_mod_image_for_test} -eq ${FLAGS_FALSE} ]]; then
+      echo "WARNING: You have specified to live reimage a machine with"
+      echo "an image that is not modified for test (so it cannot be"
+      echo "later live reimaged)"
+    fi
+    if [[ -n "${FLAGS_image_to_usb}" ]]; then
+      echo "WARNING: You have specified to both live reimage a machine and"
+      echo "write a USB image.  Is this what you wanted?"
+    fi
+    if [[ -z "${FLAGS_remote}" ]]; then
+      echo "Please specify --remote with --image_to_live"
+      exit 1
+    fi
+  fi
+
+  if [[ ${FLAGS_mod_image_for_test} -eq ${FLAGS_TRUE} ]]; then
+    # Override any specified chronos password with the test one
+    local test_file=$(dirname $0)"/mod_for_test_scripts/test_account.passwd"
+    FLAGS_chronos_passwd=$(head -1 "${test_file}")
+  fi
+
+  if [[ -n "${FLAGS_image_to_usb}" ]]; then
+    local device=${FLAGS_image_to_usb#/dev/}
+    if [[ -z "${device}" ]]; then
+      echo "Expected --image_to_usb option of /dev/* format"
+      exit 1
+    fi
+    local is_removable=$(cat /sys/block/${device}/removable)
+    if [[ "${is_removable}" != "1" ]]; then
+      echo "Could not verify that ${device} for image_to_usb is removable"
+      exit 1
+    fi
+  fi
+}
+
+
+# Prints a description of what we are doing or did
+function describe_steps() {
+  if [[ ${FLAGS_sync} -eq ${FLAGS_true} ]]; then
+    echo " * Sync client (gclient sync)"
+    if is_google_environment; then
+      echo " * Create proper src/scripts/.chromeos_dev"
+    fi
+  fi
+  if [[ ${FLAGS_force_make_local_repo} -eq ${FLAGS_true} ]]; then
+    echo " * (Re-)create local package repository (make_local_repo.sh)"
+  fi
+  if [[ ${FLAGS_force_make_chroot} -eq ${FLAGS_true} ]]; then
+    echo " * (Re-)create development chroot (make_chroot.sh)"
+  fi
+  local set_passwd=${FLAGS_FALSE}
+  if [[ ${FLAGS_build} -eq ${FLAGS_true} ]]; then
+    echo " * Build image (build_platform_packages.sh, build_kernel.sh)"
+    set_passwd=${FLAGS_TRUE}
+  fi
+  if [[ ${FLAGS_master} -eq ${FLAGS_true} ]]; then
+    echo " * Master image (build_image.sh)"
+  fi
+  if [[ -n "${FLAGS_grab_buildbot}" ]]; then
+    if [[ "${FLAGS_grab_buildbot}" == "LATEST" ]]; then
+      echo " * Grabbing latest buildbot image under ${FLAGS_buildbot_uri}"
+    else
+      echo " * Grabbing buildbot image zip at URI ${FLAGS_grab_buildbot}"
+    fi
+  fi
+  if [[ ${FLAGS_mod_image_for_test} -eq ${FLAGS_TRUE} ]]; then
+    echo " * Make image able to run tests (mod_image_for_test)"
+    set_passwd=${FLAGS_TRUE}
+  fi
+  if [[ ${set_passwd} -eq ${FLAGS_TRUE} ]]; then
+    if [[ -n "${FLAGS_chronos_passwd}" ]]; then
+      echo " * Set chronos password to ${FLAGS_chronos_passwd}"
+    else
+      echo " * Set chronos password randomly"
+    fi
+  fi
+  if [[ -n "${FLAGS_image_to_usb}" ]]; then
+    echo " * Write the image to USB device ${FLAGS_image_to_usb}"
+  fi
+  if [[ ${FLAGS_image_to_live} -eq ${FLAGS_TRUE} ]]; then
+    echo " * Reimage live test Chromium OS instance at ${FLAGS_remote}"
+  fi
+  if [[ -n "${FLAGS_test}" ]]; then
+    echo " * Run tests (${FLAGS_test}) on machine at ${FLAGS_remote}"
+  fi
+}
+
+
+# Get user's permission on steps to take
+function interactive() {
+  echo "Planning these steps on ${FLAGS_top}:"
+  describe_steps
+  read -p "Are you sure (y/N)? " SURE
+  # Get just the first character
+  if [[ "${SURE:0:1}" != "y" ]]; then
+    echo "Ok, better safe than sorry."
+    exit 1
+  fi
+}
+
+
+# Runs gclient config on a new checkout directory.
+function config_new_checkout() {
+  # We only know how to check out to a pattern like ~/foo/chromeos so
+  # make sure that's the pattern the user has given.
+  echo "Checking out ${FLAGS_top}"
+  if [[ $(basename "${FLAGS_top}") != "chromeos" ]]; then
+    echo "The --top directory does not exist and to check it out requires"
+    echo "the name to end in chromeos (try --top=${FLAGS_top}/chromeos)"
+    exit 1
+  fi
+  local top_parent=$(dirname "${FLAGS_top}")
+  mkdir -p "${top_parent}"
+  cd "${top_parent}"
+  gclient config "${FLAGS_repo}"
+}
+
+
+# Changes to a directory relative to the top/root directory of
+# the checkout.
+# Arguments:
+#   $1 - relative path
+function chdir_relative() {
+  local dir=$1
+  echo "+ cd ${dir}"
+  # Allow use of .. before the innermost directory of FLAGS_top exists
+  if [[ "${dir}" == ".." ]]; then
+    dir=$(dirname "${FLAGS_top}")
+  else
+    dir="${FLAGS_top}/${dir}"
+  fi
+  cd "${dir}"
+}
+
+
+# Describe to the user that a phase is running (and make it obviously when
+# scrolling through lots of output).
+# Arguments:
+#   $1 - phase description
+function describe_phase() {
+  local desc="$1"
+  echo ""
+  echo "#"
+  echo "#"
+  echo "# ${desc}"
+  echo "#"
+}
+
+
+# Runs a phase, describing it first, and also updates the sudo timeout
+# afterwards.
+# Arguments:
+#   $1 - phase description
+#   $2.. - command/params to run
+function run_phase() {
+  local desc="$1"
+  shift
+  describe_phase "${desc}"
+  echo "+ $@"
+  "$@"
+  sudo -v
+}
+
+
+# Runs gclient sync, setting up .chromeos_dev and preparing for
+# local repo setup
+function sync() {
+  # cd to the directory below
+  chdir_relative ..
+  run_phase "Synchronizing client" gclient sync
+  chdir_relative .
+  git cl config "file://$(pwd)/codereview.settings"
+  if is_google_environment; then
+    local base_dir=$(dirname $(dirname "${FLAGS_top}"))
+    echo <<EOF > src/scripts/.chromeos_dev
+# Use internal chromeos-deb repository
+CHROMEOS_EXT_MIRROR="http://chromeos-deb/ubuntu"
+CHROMEOS_EXT_SUITE="karmic"
+
+# Assume Chrome is checked out nearby
+CHROMEOS_CHROME_DIR="${base_dir}/chrome"
+EOF
+  fi
+}
+
+
+# Downloads a buildbot image
+function grab_buildbot() {
+  if [[ "${FLAGS_grab_buildbot}" == "LATEST" ]]; then
+    local latest=$(curl "${FLAGS_buildbot_uri}/LATEST")
+    if [[ -z "${latest}" ]]; then
+      echo "Error finding latest."
+      exit 1
+    fi
+    FLAGS_grab_buildbot="${FLAGS_buildbot_uri}/${latest}/image.zip"
+  fi
+  local dl_dir=$(mktemp -d "/tmp/image.XXXX")
+  echo "Grabbing image from ${FLAGS_grab_buildbot} to ${dl_dir}"
+  run_phase "Downloading image" curl "${FLAGS_grab_buildbot}" \
+      -o "${dl_dir}/image.zip"
+  cd "${dl_dir}"
+  unzip image.zip
+  echo "Copying in local_repo/local_packages"
+  # TODO(kmixter): Make this architecture indep once buildbot is.
+  mv -f local_repo/local_packages/* "${FLAGS_top}/src/build/x86/local_packages"
+  local image_basename=$(basename $(dirname "${FLAGS_grab_buildbot}"))
+  local image_dir="${FLAGS_top}/src/build/images/${image_basename}"
+  echo "Copying in build image to ${image_dir}"
+  rm -rf "${image_dir}"
+  mkdir -p "${image_dir}"
+  # Note that if mbr.image does not exist, this image was not successful.
+  mv mbr.image rootfs.image "${image_dir}"
+  chdir_relative .
+  run_phase "Removing downloaded image" rm -rf "${dl_dir}"
+}
+
+
+function main() {
+  assert_outside_chroot
+  assert_not_root_user
+
+  # Parse command line
+  FLAGS "$@" || exit 1
+  eval set -- "${FLAGS_ARGV}"
+
+  # Die on any errors.
+  set -e
+
+  validate_and_set_param_defaults
+
+  # Cache up sudo status
+  sudo -v
+
+  if [[ ${FLAGS_interactive} -eq ${FLAGS_TRUE} ]]; then
+    interactive
+  fi
+
+  if [[ ! -e "${FLAGS_top}" ]]; then
+    config_new_checkout
+  fi
+
+  if [[ ${FLAGS_sync} -eq ${FLAGS_TRUE} ]]; then
+    sync
+    # The package repository is now potentially out of date, so
+    # reflect that.
+    run_phase "Removing existing package repo" sudo rm -rf repo
+    FLAGS_force_make_local_repo=${FLAGS_TRUE}
+    FLAGS_force_make_chroot=${FLAGS_TRUE}
+  fi
+
+  if [[ -n "${FLAGS_grab_buildbot}" ]]; then
+    grab_buildbot
+  fi
+
+  if [[ ${FLAGS_force_make_local_repo} -eq ${FLAGS_TRUE} ]]; then
+    chdir_relative src/scripts
+    run_phase "Refetching local repo" ./make_local_repo.sh
+  fi
+
+  if [[ ${FLAGS_force_make_chroot} -eq ${FLAGS_TRUE} ]]; then
+    chdir_relative src/scripts
+    run_phase "Replacing chroot" ./make_chroot.sh --replace
+  fi
+
+  if [[ ${FLAGS_build} -eq ${FLAGS_TRUE} ]]; then
+    chdir_relative src/scripts
+    run_phase "Building platform packages and kernel" ./enter_chroot.sh \
+        "./build_platform_packages.sh && ./build_kernel.sh"
+
+    if [[ ${FLAGS_build} -eq ${FLAGS_TRUE} ]]; then
+      run_phase "Building and running unit tests" ./enter_chroot.sh \
+        "./build_tests.sh && ./run_tests.sh"
+    fi
+  fi
+
+  if [[ ${FLAGS_master} -eq ${FLAGS_TRUE} ]]; then
+    chdir_relative src/scripts
+    if [[ -n "${FLAGS_chronos_passwd}" ]]; then
+      describe_phase "Setting default chronos password"
+      ./enter_chroot.sh "echo '${FLAGS_chronos_passwd}' | \
+                        ./set_shared_user_password.sh"
+    fi
+    run_phase "Mastering image" ./enter_chroot.sh "./build_image.sh --replace"
+  fi
+
+  if [[ ${FLAGS_mod_image_for_test} -eq ${FLAGS_TRUE} ]]; then
+    chdir_relative src/scripts
+    run_phase "Modifying image for test" ./enter_chroot.sh \
+        "./mod_image_for_test.sh"
+  fi
+
+  if [[ -n "${FLAGS_image_to_usb}" ]]; then
+    chdir_relative src/scripts
+    run_phase "Installing image to USB" \
+        ./image_to_usb.sh --yes "--to=${FLAGS_image_to_usb}"
+  fi
+
+  if [[ ${FLAGS_image_to_live} -eq ${FLAGS_TRUE} ]]; then
+    chdir_relative src/scripts
+    run_phase "Re-imaging live Chromium OS machine ${FLAGS_remote}" \
+      ./image_to_live.sh "--remote=${FLAGS_remote}" --update_known_hosts
+  fi
+
+  if [[ -n "${FLAGS_test}" ]]; then
+    chdir_relative src/scripts
+    # We purposefully do not quote FLAGS_test below as we expect it may
+    # have multiple parameters
+    run_phase "Running tests on Chromium OS machine ${FLAGS_remote}" \
+      ./run_remote_tests.sh --remote="${FLAGS_remote}" ${FLAGS_test}
+  fi
+
+  echo "Successfully used ${FLAGS_top} to:"
+  describe_steps
+}
+
+
+main $@