Merge branch 'master' of ssh://gitrw.chromium.org:9222/crosutils

bin/cbuildbot.py

@@ -232,10 +232,12 @@ def _MarkChromeAsStable(buildroot, tracking_branch, chrome_rev, board):
     return None
   else:
     chrome_atom = portage_atom_string.split('=')[1]
+    keywords_file = CHROME_KEYWORDS_FILE % {'board': board}
     # TODO(sosa): Workaround to build unstable chrome ebuild we uprevved.
-    RunCommand(['sudo', 'tee', CHROME_KEYWORDS_FILE % {'board': board}],
-               input='=%s\n' % chrome_atom, enter_chroot=True,
-               cwd=cwd)
+    RunCommand(['sudo', 'mkdir', '-p', os.path.dirname(keywords_file)],
+               enter_chroot=True, cwd=cwd)
+    RunCommand(['sudo', 'tee', keywords_file], input='=%s\n' % chrome_atom,
+               enter_chroot=True, cwd=cwd)
     return chrome_atom
 
 
@@ -415,6 +417,22 @@ def _RunSmokeSuite(buildroot, results_dir):
               ], cwd=cwd, error_ok=False)
 
 
+def _RunAUTest(buildroot, board):
+  """Runs a basic update test from the au test harness."""
+  cwd = os.path.join(buildroot, 'src', 'scripts')
+  image_path = os.path.join(buildroot, 'src', 'build', 'images', board,
+                            'latest', 'chromiumos_test_image.bin')
+  RunCommand(['bin/cros_au_test_harness',
+              '--no_graphics',
+              '--no_delta',
+              '--board=%s' % board,
+              '--test_prefix=SimpleTest',
+              '--verbose',
+              '--base_image=%s' % image_path,
+              '--target_image=%s' % image_path,
+              ], cwd=cwd, error_ok=False)
+
+
 def _UprevPackages(buildroot, tracking_branch, revisionfile, board, overlays):
   """Uprevs a package based on given revisionfile.
 
@@ -695,15 +713,16 @@ def main():
 
     _BuildImage(buildroot)
 
-    if buildconfig['smoke_bvt'] and options.tests:
+    if buildconfig['tests'] and options.tests:
       _BuildVMImageForTesting(buildroot)
       test_results_dir = '/tmp/run_remote_tests.%s' % options.buildnumber
       try:
         _RunSmokeSuite(buildroot, test_results_dir)
+        _RunAUTest(buildroot, buildconfig['board'])
       finally:
         if not options.debug:
           archive_full_path = os.path.join(options.gsutil_archive,
-                                         str(options.buildnumber))
+                                           str(options.buildnumber))
           _ArchiveTestResults(buildroot, buildconfig['board'],
                               test_results_dir=test_results_dir,
                               gsutil=options.gsutil,

bin/cbuildbot_config.py

@@ -18,7 +18,7 @@ important -- Master bot uses important bots to determine overall status.
 hostname -- Needed for 'important' slaves.  The hostname of the bot.  Should
             match hostname in slaves.cfg in buildbot checkout.
 unittests -- Runs unittests for packages.
-smoke_bvt -- Runs the test smoke suite in a qemu-based VM using KVM.
+tests -- Runs the smoke suite and au test harness in a qemu-based VM using KVM.
 rev_overlays -- Select what overlays to look at for revving. This can be
                 'public', 'private' or 'both'.
 push_overlays -- Select what overlays to push at. This should be a subset of
@@ -35,7 +35,7 @@ config['default'] = {
   'master' : False,
   'important' : False,
   'unittests' : False,
-  'smoke_bvt' : False,
+  'tests' : False,
   'rev_overlays': 'public',
   'push_overlays': None,
 }
@@ -46,7 +46,7 @@ config['x86-generic-pre-flight-queue'] = {
   'important' : False,
   'hostname' : 'chromeosbuild2',
   'unittests' : True,
-  'smoke_bvt' : True,
+  'tests' : True,
   'rev_overlays': 'public',
   'push_overlays': 'public',
 }
@@ -56,7 +56,7 @@ config['x86-mario-pre-flight-queue'] = {
   'master' : True,
   'important' : False,
   'unittests' : True,
-  'smoke_bvt' : True,
+  'tests' : True,
   'rev_overlays': 'both',
   'push_overlays': 'private',
 }
@@ -66,7 +66,7 @@ config['x86-mario-pre-flight-branch'] = {
   'master' : True,
   'important' : False,
   'unittests' : True,
-  'smoke_bvt' : True,
+  'tests' : True,
   'rev_overlays': 'both',
   'push_overlays': 'both',
 }
@@ -76,7 +76,7 @@ config['x86_agz_bin'] = {
   'master' : False,
   'important' : False,
   'unittests' : True,
-  'smoke_bvt' : True,
+  'tests' : True,
   'rev_overlays': 'both',
   'push_overlays': None,
 }
@@ -86,7 +86,7 @@ config['x86_dogfood_bin'] = {
   'master' : False,
   'important' : False,
   'unittests' : True,
-  'smoke_bvt' : True,
+  'tests' : True,
   'rev_overlays': 'both',
   'push_overlays': None,
 }

bin/cros_au_test_harness.py

@@ -4,6 +4,14 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+"""This module runs a suite of Auto Update tests.
+
+  The tests can be run on either a virtual machine or actual device depending
+  on parameters given.  Specific tests can be run by invoking --test_prefix.
+  Verbose is useful for many of the tests if you want to see individual commands
+  being run during the update process.
+"""
+
 import optparse
 import os
 import re
@@ -148,7 +156,8 @@ class AUTest(object):
 
     # This update is expected to fail...
     try:
-      self.PerformUpdate(self.target_image_path, proxy_port=proxy_port)
+      self.PerformUpdate(self.target_image_path, self.base_image_path,
+                         proxy_port=proxy_port)
     finally:
       proxy.shutdown()
 
@@ -226,7 +235,7 @@ class AUTest(object):
 
   # -------- Tests ---------
 
-  def testFullUpdateKeepStateful(self):
+  def testUpdateKeepStateful(self):
     """Tests if we can update normally.
 
     This test checks that we can update by updating the stateful partition
@@ -241,15 +250,15 @@ class AUTest(object):
 
     # Update to - all tests should pass on new image.
     Info('Updating from base image on vm to target image.')
-    self.PerformUpdate(self.base_image_path, self.target_image_path)
+    self.PerformUpdate(self.target_image_path, self.base_image_path)
     self.VerifyImage(100)
 
     # Update from - same percentage should pass that originally passed.
     Info('Updating from updated image on vm back to base image.')
-    self.PerformUpdate(self.target_image_path, self.base_image_path)
+    self.PerformUpdate(self.base_image_path, self.target_image_path)
     self.VerifyImage(percent_passed)
 
-  def testFullUpdateWipeStateful(self):
+  def testUpdateWipeStateful(self):
     """Tests if we can update after cleaning the stateful partition.
 
     This test checks that we can update successfully after wiping the
@@ -264,12 +273,12 @@ class AUTest(object):
 
     # Update to - all tests should pass on new image.
     Info('Updating from base image on vm to target image and wiping stateful.')
-    self.PerformUpdate(self.base_image_path, self.target_image_path, 'clean')
+    self.PerformUpdate(self.target_image_path, self.base_image_path, 'clean')
     self.VerifyImage(100)
 
     # Update from - same percentage should pass that originally passed.
     Info('Updating from updated image back to base image and wiping stateful.')
-    self.PerformUpdate(self.target_image_path, self.base_image_path, 'clean')
+    self.PerformUpdate(self.base_image_path, self.target_image_path, 'clean')
     self.VerifyImage(percent_passed)
 
   def testPartialUpdate(self):
@@ -372,13 +381,13 @@ class AUTest(object):
     self.AttemptUpdateWithFilter(DelayedFilter())
 
   def SimpleTest(self):
-    """A simple update  that updates the target image to itself.
+    """A simple update that updates once from a base image to a target.
 
     We explicitly don't use test prefix so that isn't run by default.  Can be
     run using test_prefix option.
     """
-    self.PrepareBase(self.target_image_path)
-    self._UpdateImage(self.target_image_path)
+    self.PrepareBase(self.base_image_path)
+    self.PerformUpdate(self.target_image_path, self.base_image_path)
     self.VerifyImage(100)
 
 
@@ -552,7 +561,7 @@ class VirtualAUTest(unittest.TestCase, AUTest):
 
   def _UpdateUsingPayload(self, update_path, stateful_change='old',
                          proxy_port=None):
-    """Updates a remote image using image_to_live.sh."""
+    """Updates a vm image using cros_run_vm_update."""
     stateful_change_flag = self.GetStatefulChangeFlag(stateful_change)
     cmd = ['%s/cros_run_vm_update' % self.crosutilsbin,
            '--payload=%s' % update_path,

bin/cros_mark_chrome_as_stable.py

@@ -60,7 +60,9 @@ def  _GetTipOfTrunkSvnRevision():
   for line in svn_info.splitlines():
     match = revision_re.search(line)
     if match:
-      return match.group(1)
+      svn_revision = match.group(1)
+      Info('Using SVN Revision %s' % svn_revision)
+      return svn_revision
 
   raise Exception('Could not find revision information from %s' % svn_url)
 

build_image

@@ -71,9 +71,9 @@ DEFINE_string usb_disk /dev/sdb3 \
 
 DEFINE_boolean enable_rootfs_verification ${FLAGS_TRUE} \
   "Default all bootloaders to use kernel-based root fs integrity checking."
-DEFINE_integer verity_error_behavior 1 \
+DEFINE_integer verity_error_behavior 3 \
   "Kernel verified boot error behavior (0: I/O errors, 1: panic, 2: nothing, \
-3: cros) Default: 1"
+3: cros) Default: 3"
 DEFINE_integer verity_depth 1 \
   "Kernel verified boot hash tree depth. Default: 1"
 DEFINE_integer verity_max_ios -1 \

image_to_live.sh

@@ -241,7 +241,6 @@ function get_update_log {
   echo "${REMOTE_OUT}" > "${FLAGS_update_log}"
 }
 
-
 # Returns ${1} reported by the update client e.g. PROGRESS, CURRENT_OP.
 function get_update_var {
   remote_sh "${UPDATER_BIN} --status 2> /dev/null |
@@ -254,21 +253,30 @@ function get_update_var {
 # This is expected to run in its own thread.
 function status_thread {
   local timeout=5
-  # Let update engine receive call to ping the dev server.
+
   info "Devserver handling ping.  Check ${FLAGS_server_log} for more info."
   sleep ${timeout}
 
-  # The devserver generates images when the update engine checks for updates.
-  while [ $(get_update_var CURRENT_OP) = ${UPDATER_UPDATE_CHECK} ]; do
-    echo -n "." && sleep ${timeout}
-  done
+  local current_state=""
+  local next_state="$(get_update_var CURRENT_OP)"
 
-  info "Update generated.  Update engine downloading update."
-  while [ $(get_update_var CURRENT_OP) = ${UPDATER_DOWNLOADING} ]; do
-    echo "Download progress $(get_update_var PROGRESS)" && sleep ${timeout}
-  done
+  # For current status, only print out status changes.
+  # For download, show progress.
+  # Finally if no status change print out .'s to keep dev informed.
+  while [ "${current_state}" != "${UPDATER_NEED_REBOOT}" ] && \
+      [ "${current_state}" != "${UPDATER_IDLE}" ]; do
+    if [ "${current_state}" != "${next_state}" ]; then
+      info "State of updater has changed to: ${next_state}"
+    elif [ "${next_state}" = "${UPDATER_DOWNLOADING}" ]; then
+      echo "Download progress $(get_update_var PROGRESS)"
+    else
+      echo -n "."
+    fi
 
-  info "Download complete."
+    sleep ${timeout}
+    current_state="${next_state}"
+    next_state="$(get_update_var CURRENT_OP)"
+  done
 }
 
 

lib/cros_image_common.sh

@@ -185,6 +185,16 @@ image_partition_copy() {
     exit 1
   fi
 
+  # Try to use larger buffer if offset/size can be re-aligned.
+  # 2M / 512 = 4096
+  local buffer_ratio=4096
+  local bs=512
+  if [ $((dstoffset % buffer_ratio)) -eq 0 -a \
+       $((length % buffer_ratio)) -eq 0 ]; then
+    dstoffset=$((dstoffset / buffer_ratio))
+    bs=$((bs * buffer_ratio))
+  fi
+
   image_dump_partition "${src}" "${srcpart}" |
-      dd of="${dst}" bs=512 seek="${dstoffset}" conv=notrunc
+      dd of="${dst}" bs="${bs}" seek="${dstoffset}" conv=notrunc oflag=dsync
 }

make_factory_package.sh

@@ -86,7 +86,7 @@ RELEASE_IMAGE="$(basename "${FLAGS_release}")"
 FACTORY_IMAGE="$(basename "${FLAGS_factory}")"
 
 prepare_img() {
-  local outdev="$FLAGS_diskimg"
+  local outdev="$(readlink -f "$FLAGS_diskimg")"
   local sectors="$FLAGS_sectors"
   local force_full="true"
 
@@ -110,9 +110,10 @@ prepare_img() {
   fi
 
   # Create GPT partition table.
+  locate_gpt
   install_gpt "${outdev}" 0 0 "${pmbrcode}" 0 "${force_full}"
   # Activate the correct partition.
-  cgpt add -i 2 -S 1 -P 1 "${outdev}"
+  sudo "${GPT}" add -i 2 -S 1 -P 1 "${outdev}"
 }
 
 prepare_omaha() {
@@ -191,7 +192,7 @@ else
 fi
 
 generate_img() {
-  local outdev="$FLAGS_diskimg"
+  local outdev="$(readlink -f "$FLAGS_diskimg")"
   local sectors="$FLAGS_sectors"
 
   prepare_img

mod_for_test_scripts/750enableExternalExtensions

@@ -8,7 +8,7 @@
 # to launch a packaged extension at login.
 
 echo "Creating external_extensions.json symlink"
-LINK_TARGET="/home/autotest/tests/external_extensions.json"
+LINK_TARGET="/usr/local/autotest/tests/external_extensions.json"
 EXT_DIR="${ROOT_FS_DIR}/opt/google/chrome/extensions"
 FILE="${EXT_DIR}/external_extensions.json"
 mkdir -p $EXT_DIR

prebuilt.py

@@ -42,7 +42,7 @@ Example of uploading prebuilt amd64 host files using rsync:
 # as per http://crosbug.com/5855 always filter the below packages
 _FILTER_PACKAGES = set()
 _RETRIES = 3
-_GSUTIL_BIN = '/b/third_party/gsutil/gsutil'
+_GSUTIL_BIN = '/b/build/third_party/gsutil/gsutil'
 _HOST_PACKAGES_PATH = 'chroot/var/lib/portage/pkgs'
 _HOST_TARGET = 'amd64'
 _BOARD_PATH = 'chroot/build/%(board)s'

remote_access.sh

@@ -68,12 +68,29 @@ function learn_board() {
   info "Target reports board is ${FLAGS_board}"
 }
 
-function remote_reboot {
-  info "Rebooting."
-  remote_sh "touch /tmp/awaiting_reboot; reboot"
-  local output_file
-  output_file="${TMP}/output"
+# Checks to see if pid $1 is running.
+function is_pid_running() {
+  ps -p ${1} 2>&1 > /dev/null
+}
 
+# Wait function given an additional timeout argument.
+# $1 - pid to wait on.
+# $2 - timeout to wait for.
+function wait_with_timeout() {
+  local pid=$1
+  local timeout=$2
+  local -r TIMEOUT_INC=1
+  local current_timeout=0
+  while is_pid_running ${pid} && [ ${current_timeout} -lt ${timeout} ]; do
+    sleep ${TIMEOUT_INC}
+    current_timeout=$((current_timeout + TIMEOUT_INC))
+  done
+  ! is_pid_running ${pid}
+}
+
+# Checks to see if a machine has rebooted using the presence of a tmp file.
+function check_if_rebooted() {
+  local output_file="${TMP}/output"
   while true; do
     REMOTE_OUT=""
     # This may fail while the machine is down so generate output and a
@@ -84,12 +101,23 @@ function remote_reboot {
     if grep -q "0" "${output_file}"; then
       if grep -q "1" "${output_file}"; then
         info "Not yet rebooted"
+        sleep .5
       else
         info "Rebooted and responding"
         break
       fi
     fi
-    sleep .5
+  done
+}
+
+function remote_reboot() {
+  info "Rebooting."
+  remote_sh "touch /tmp/awaiting_reboot; reboot"
+  while true; do
+    check_if_rebooted &
+    local pid=$!
+    wait_with_timeout ${pid} 30 && break
+    ! kill -9 ${pid} 2> /dev/null
   done
 }