From e7205e84be92f5b0fb91c259e1526b7e371c8a31 Mon Sep 17 00:00:00 2001
From: "J. Richard Barnette" <jrbarnette@chromium.org>
Date: Wed, 5 Jan 2011 18:38:49 -0800
Subject: [PATCH 01/16] Revert "Major cleanup of cros_au_test_harness code."

This reverts commit 2b2d7a1426f485bc2c6fb02399c52618db6dd0f2.

The internal x86 buildbots have been reporting failures from
cros_au_test_harness with every build, starting with the build
that included the change being reverted.

BUG=chromium-os:8901
TEST=None

Review URL: http://codereview.chromium.org/5962019

Change-Id: Iac8c0a13d659f927b3ff61a9a894c10191432e2d
---
 bin/cros_au_test_harness.py | 339 +++++++++++++++++-------------------
 1 file changed, 164 insertions(+), 175 deletions(-)

diff --git a/bin/cros_au_test_harness.py b/bin/cros_au_test_harness.py
index 5a0182c481..5d5b221760 100755
--- a/bin/cros_au_test_harness.py
+++ b/bin/cros_au_test_harness.py
@@ -1,6 +1,6 @@
 #!/usr/bin/python
 
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
+# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
@@ -23,16 +23,29 @@ from cros_build_lib import Warning
 
 import cros_test_proxy
 
+# VM Constants.
+_FULL_VDISK_SIZE = 6072
+_FULL_STATEFULFS_SIZE = 3074
+_KVM_PID_FILE = '/tmp/harness_pid'
+_VERIFY_SUITE = 'suite_Smoke'
+
+# Globals to communicate options to unit tests.
+global base_image_path
+global board
+global remote
+global target_image_path
+global vm_graphics_flag
 
 class UpdateException(Exception):
-  """Exception thrown when _UpdateImage or _UpdateUsingPayload fail"""
+  """Exception thrown when UpdateImage or UpdateUsingPayload fail"""
   def __init__(self, code, stdout):
     self.code = code
     self.stdout = stdout
 
-
 class AUTest(object):
   """Abstract interface that defines an Auto Update test."""
+  source_image = ''
+  use_delta_updates = False
   verbose = False
 
   def setUp(self):
@@ -44,8 +57,6 @@ class AUTest(object):
     if not os.path.exists(self.download_folder):
       os.makedirs(self.download_folder)
 
-  # -------- Helper functions ---------
-
   def GetStatefulChangeFlag(self, stateful_change):
     """Returns the flag to pass to image_to_vm for the stateful change."""
     stateful_change_flag = ''
@@ -54,7 +65,7 @@ class AUTest(object):
 
     return stateful_change_flag
 
-  def _ParseGenerateTestReportOutput(self, output):
+  def ParseGenerateTestReportOutput(self, output):
     """Returns the percentage of tests that passed based on output."""
     percent_passed = 0
     lines = output.split('\n')
@@ -68,60 +79,38 @@ class AUTest(object):
 
     return int(percent_passed)
 
-  def AssertEnoughTestsPassed(self, unittest, output, percent_required_to_pass):
-    """Helper function that asserts a sufficient number of tests passed.
+  # TODO(sosa) - Remove try and convert function to DeltaUpdateImage().
+  def TryDeltaAndFallbackToFull(self, src_image, image, stateful_change='old'):
+    """Tries the delta update first if set and falls back to full update."""
+    if self.use_delta_updates:
+      try:
+        self.source_image = src_image
+        self._UpdateImageReportError(image, stateful_change)
+      except:
+        Warning('Delta update failed, disabling delta updates and retrying.')
+        self.use_delta_updates = False
+        self.source_image = ''
+        self._UpdateImageReportError(image, stateful_change)
+    else:
+      self._UpdateImageReportError(image, stateful_change)
 
-    Args:
-      unittest: Handle to the unittest.
-      output: stdout from a test run.
-      percent_required_to_pass: percentage required to pass.  This should be
-        fall between 0-100.
-    Returns:
-      percent that passed.
-    """
-    Info('Output from VerifyImage():')
-    print >> sys.stderr, output
-    sys.stderr.flush()
-    percent_passed = self._ParseGenerateTestReportOutput(output)
-    Info('Percent passed: %d vs. Percent required: %d' % (
-        percent_passed, percent_required_to_pass))
-    unittest.assertTrue(percent_passed >= percent_required_to_pass)
-    return percent_passed
+  def _UpdateImageReportError(self, image_path, stateful_change='old',
+                              proxy_port=None):
+    """Calls UpdateImage and reports any error to the console.
 
-  def PerformUpdate(self, image_path, src_image_path='', stateful_change='old',
-                    proxy_port=None):
-    """Performs an update using  _UpdateImage and reports any error.
-
-    Subclasses should not override this method but override _UpdateImage
-    instead.
-
-    Args:
-      image_path:  Path to the image to update with.  This image must be a test
-        image.
-      src_image_path:  Optional.  If set, perform a delta update using the
-        image specified by the path as the source image.
-      stateful_change: How to modify the stateful partition.  Values are:
-          'old':  Don't modify stateful partition.  Just update normally.
-          'clean':  Uses clobber-state to wipe the stateful partition with the
-            exception of code needed for ssh.
-      proxy_port:  Port to have the client connect to. For use with
-        CrosTestProxy.
-    Raises an UpdateException if _UpdateImage returns an error.
+       Still throws the exception.
     """
     try:
-      if not self.use_delta_updates:
-        src_image_path = ''
-
-      self._UpdateImage(image_path, src_image_path, stateful_change, proxy_port)
+      self.UpdateImage(image_path, stateful_change, proxy_port)
     except UpdateException as err:
       # If the update fails, print it out
       Warning(err.stdout)
       raise
 
-  def AttemptUpdateWithPayloadExpectedFailure(self, payload, expected_msg):
+  def _AttemptUpdateWithPayloadExpectedFailure(self, payload, expected_msg):
     """Attempt a payload update, expect it to fail with expected log"""
     try:
-      self._UpdateUsingPayload(payload)
+      self.UpdateUsingPayload(payload)
     except UpdateException as err:
       # Will raise ValueError if expected is not found.
       if re.search(re.escape(expected_msg), err.stdout, re.MULTILINE):
@@ -131,10 +120,10 @@ class AUTest(object):
     Warning(err.stdout)
     self.fail('We managed to update when failure was expected')
 
-  def AttemptUpdateWithFilter(self, filter):
+  def _AttemptUpdateWithFilter(self, filter):
     """Update through a proxy, with a specified filter, and expect success."""
 
-    self.PrepareBase(self.target_image_path)
+    self.PrepareBase(target_image_path)
 
     # The devserver runs at port 8080 by default. We assume that here, and
     # start our proxy at 8081. We then tell our update tools to have the
@@ -148,58 +137,34 @@ class AUTest(object):
 
     # This update is expected to fail...
     try:
-      self.PerformUpdate(self.target_image_path, proxy_port=proxy_port)
+      self._UpdateImageReportError(target_image_path, proxy_port=proxy_port)
     finally:
       proxy.shutdown()
 
-  # -------- Functions that subclasses should override ---------
-
-  @classmethod
-  def ProcessOptions(cls, parser, options):
-    """Processes options.
-
-    Static method that should be called from main.  Subclasses should also
-    call their parent method if they override it.
-    """
-    cls.verbose = options.verbose
-    cls.base_image_path = options.base_image
-    cls.target_image_path = options.target_image
-    cls.use_delta_updates = options.delta
-    if options.quick_test:
-      cls.verify_suite = 'build_RootFilesystemSize'
-    else:
-      cls.verify_suite = 'suite_Smoke'
-
-    # Sanity checks.
-    if not cls.base_image_path:
-      parser.error('Need path to base image for vm.')
-    elif not os.path.exists(cls.base_image_path):
-      Die('%s does not exist' % cls.base_image_path)
-
-    if not cls.target_image_path:
-      parser.error('Need path to target image to update with.')
-    elif not os.path.exists(cls.target_image_path):
-      Die('%s does not exist' % cls.target_image_path)
-
   def PrepareBase(self, image_path):
     """Prepares target with base_image_path."""
     pass
 
-  def _UpdateImage(self, image_path, src_image_path='', stateful_change='old',
-                   proxy_port=None):
-    """Implementation of an actual update.
+  def UpdateImage(self, image_path, stateful_change='old', proxy_port=None):
+    """Updates target with the image given by the image_path.
 
-    See PerformUpdate for description of args.  Subclasses must override this
-    method with the correct update procedure for the class.
+    Args:
+      image_path:  Path to the image to update with.  This image must be a test
+        image.
+      stateful_change: How to modify the stateful partition.  Values are:
+          'old':  Don't modify stateful partition.  Just update normally.
+          'clean':  Uses clobber-state to wipe the stateful partition with the
+            exception of code needed for ssh.
+      proxy_port:  Port to have the client connect to. For use with
+        CrosTestProxy.
     """
     pass
 
-  def _UpdateUsingPayload(self, update_path, stateful_change='old',
+  def UpdateUsingPayload(self,
+                         update_path,
+                         stateful_change='old',
                          proxy_port=None):
-    """Updates target with the pre-generated update stored in update_path.
-
-    Subclasses must override this method with the correct update procedure for
-    the class.
+    """Updates target with the pre-generated update stored in update_path
 
     Args:
       update_path:  Path to the image to update with. This directory should
@@ -212,8 +177,7 @@ class AUTest(object):
   def VerifyImage(self, percent_required_to_pass):
     """Verifies the image with tests.
 
-    Verifies that the test images passes the percent required.  Subclasses must
-    override this method with the correct update procedure for the class.
+    Verifies that the test images passes the percent required.
 
     Args:
       percent_required_to_pass:  percentage required to pass.  This should be
@@ -224,7 +188,29 @@ class AUTest(object):
     """
     pass
 
-  # -------- Tests ---------
+  def CommonVerifyImage(self, unittest, output, percent_required_to_pass):
+    """Helper function for VerifyImage that returns percent of tests passed.
+
+    Takes output from a test suite, verifies the number of tests passed is
+    sufficient and outputs info.
+
+    Args:
+      unittest: Handle to the unittest.
+      output: stdout from a test run.
+      percent_required_to_pass: percentage required to pass.  This should be
+        fall between 0-100.
+    Returns:
+      percent that passed.
+    """
+    Info('Output from VerifyImage():')
+    print >> sys.stderr, output
+    sys.stderr.flush()
+    percent_passed = self.ParseGenerateTestReportOutput(output)
+    Info('Percent passed: %d vs. Percent required: %d' % (
+        percent_passed, percent_required_to_pass))
+    unittest.assertTrue(percent_passed >=
+                        percent_required_to_pass)
+    return percent_passed
 
   def testFullUpdateKeepStateful(self):
     """Tests if we can update normally.
@@ -234,19 +220,19 @@ class AUTest(object):
     """
     # Just make sure some tests pass on original image.  Some old images
     # don't pass many tests.
-    self.PrepareBase(self.base_image_path)
+    self.PrepareBase(base_image_path)
     # TODO(sosa): move to 100% once we start testing using the autotest paired
     # with the dev channel.
     percent_passed = self.VerifyImage(10)
 
     # Update to - all tests should pass on new image.
     Info('Updating from base image on vm to target image.')
-    self.PerformUpdate(self.base_image_path, self.target_image_path)
+    self.TryDeltaAndFallbackToFull(base_image_path, target_image_path)
     self.VerifyImage(100)
 
     # Update from - same percentage should pass that originally passed.
     Info('Updating from updated image on vm back to base image.')
-    self.PerformUpdate(self.target_image_path, self.base_image_path)
+    self.TryDeltaAndFallbackToFull(target_image_path, base_image_path)
     self.VerifyImage(percent_passed)
 
   def testFullUpdateWipeStateful(self):
@@ -257,25 +243,25 @@ class AUTest(object):
     """
     # Just make sure some tests pass on original image.  Some old images
     # don't pass many tests.
-    self.PrepareBase(self.base_image_path)
+    self.PrepareBase(base_image_path)
     # TODO(sosa): move to 100% once we start testing using the autotest paired
     # with the dev channel.
     percent_passed = self.VerifyImage(10)
 
     # Update to - all tests should pass on new image.
     Info('Updating from base image on vm to target image and wiping stateful.')
-    self.PerformUpdate(self.base_image_path, self.target_image_path, 'clean')
+    self.TryDeltaAndFallbackToFull(base_image_path, target_image_path, 'clean')
     self.VerifyImage(100)
 
     # Update from - same percentage should pass that originally passed.
     Info('Updating from updated image back to base image and wiping stateful.')
-    self.PerformUpdate(self.target_image_path, self.base_image_path, 'clean')
+    self.TryDeltaAndFallbackToFull(target_image_path, base_image_path, 'clean')
     self.VerifyImage(percent_passed)
 
   def testPartialUpdate(self):
     """Tests what happens if we attempt to update with a truncated payload."""
     # Preload with the version we are trying to test.
-    self.PrepareBase(self.target_image_path)
+    self.PrepareBase(target_image_path)
 
     # Image can be updated at:
     # ~chrome-eng/chromeos/localmirror/autest-images
@@ -287,12 +273,12 @@ class AUTest(object):
     urllib.urlretrieve(url, payload)
 
     expected_msg = 'download_hash_data == update_check_response_hash failed'
-    self.AttemptUpdateWithPayloadExpectedFailure(payload, expected_msg)
+    self._AttemptUpdateWithPayloadExpectedFailure(payload, expected_msg)
 
   def testCorruptedUpdate(self):
     """Tests what happens if we attempt to update with a corrupted payload."""
     # Preload with the version we are trying to test.
-    self.PrepareBase(self.target_image_path)
+    self.PrepareBase(target_image_path)
 
     # Image can be updated at:
     # ~chrome-eng/chromeos/localmirror/autest-images
@@ -305,7 +291,7 @@ class AUTest(object):
 
     # This update is expected to fail...
     expected_msg = 'zlib inflate() error:-3'
-    self.AttemptUpdateWithPayloadExpectedFailure(payload, expected_msg)
+    self._AttemptUpdateWithPayloadExpectedFailure(payload, expected_msg)
 
   def testInterruptedUpdate(self):
     """Tests what happens if we interrupt payload delivery 3 times."""
@@ -339,7 +325,7 @@ class AUTest(object):
         self.data_size += len(data)
         return data
 
-    self.AttemptUpdateWithFilter(InterruptionFilter())
+    self._AttemptUpdateWithFilter(InterruptionFilter())
 
   def testDelayedUpdate(self):
     """Tests what happens if some data is delayed during update delivery"""
@@ -369,7 +355,7 @@ class AUTest(object):
         self.data_size += len(data)
         return data
 
-    self.AttemptUpdateWithFilter(DelayedFilter())
+    self._AttemptUpdateWithFilter(DelayedFilter())
 
   def SimpleTest(self):
     """A simple update  that updates the target image to itself.
@@ -377,8 +363,8 @@ class AUTest(object):
     We explicitly don't use test prefix so that isn't run by default.  Can be
     run using test_prefix option.
     """
-    self.PrepareBase(self.target_image_path)
-    self._UpdateImage(self.target_image_path)
+    self.PrepareBase(target_image_path)
+    self.UpdateImage(target_image_path)
     self.VerifyImage(100)
 
 
@@ -388,29 +374,19 @@ class RealAUTest(unittest.TestCase, AUTest):
   def setUp(self):
     AUTest.setUp(self)
 
-  @classmethod
-  def ProcessOptions(cls, parser, options):
-    """Processes non-vm-specific options."""
-    AUTest.ProcessOptions(parser, options)
-    cls.remote = options.remote
-
-    if not cls.remote:
-      parser.error('We require a remote address for real tests.')
-
   def PrepareBase(self, image_path):
     """Auto-update to base image to prepare for test."""
-    self.PerformUpdate(image_path)
+    self._UpdateImageReportError(image_path)
 
-  def _UpdateImage(self, image_path, src_image_path='', stateful_change='old',
-                   proxy_port=None):
+  def UpdateImage(self, image_path, stateful_change='old', proxy_port=None):
     """Updates a remote image using image_to_live.sh."""
     stateful_change_flag = self.GetStatefulChangeFlag(stateful_change)
     cmd = ['%s/image_to_live.sh' % self.crosutils,
            '--image=%s' % image_path,
-           '--remote=%s' % self.remote,
+           '--remote=%s' % remote,
            stateful_change_flag,
            '--verify',
-           '--src_image=%s' % src_image_path
+           '--src_image=%s' % self.source_image
           ]
 
     if proxy_port:
@@ -426,13 +402,15 @@ class RealAUTest(unittest.TestCase, AUTest):
       if code != 0:
         raise UpdateException(code, stdout)
 
-  def _UpdateUsingPayload(self, update_path, stateful_change='old',
+  def UpdateUsingPayload(self,
+                         update_path,
+                         stateful_change='old',
                          proxy_port=None):
     """Updates a remote image using image_to_live.sh."""
     stateful_change_flag = self.GetStatefulChangeFlag(stateful_change)
     cmd = ['%s/image_to_live.sh' % self.crosutils,
            '--payload=%s' % update_path,
-           '--remote=%s' % self.remote,
+           '--remote=%s' % remote,
            stateful_change_flag,
            '--verify',
           ]
@@ -454,21 +432,16 @@ class RealAUTest(unittest.TestCase, AUTest):
     """Verifies an image using run_remote_tests.sh with verification suite."""
     output = RunCommand([
         '%s/run_remote_tests.sh' % self.crosutils,
-        '--remote=%s' % self.remote,
-        self.verify_suite,
+        '--remote=%s' % remote,
+        _VERIFY_SUITE,
        ], error_ok=True, enter_chroot=False, redirect_stdout=True)
-    return self.AssertEnoughTestsPassed(self, output, percent_required_to_pass)
+    return self.CommonVerifyImage(self, output, percent_required_to_pass)
 
 
 class VirtualAUTest(unittest.TestCase, AUTest):
   """Test harness for updating virtual machines."""
   vm_image_path = None
 
-  # VM Constants.
-  _FULL_VDISK_SIZE = 6072
-  _FULL_STATEFULFS_SIZE = 3074
-  _KVM_PID_FILE = '/tmp/harness_pid'
-
   def _KillExistingVM(self, pid_file):
     if os.path.exists(pid_file):
       Warning('Existing %s found.  Deleting and killing process' %
@@ -481,20 +454,7 @@ class VirtualAUTest(unittest.TestCase, AUTest):
   def setUp(self):
     """Unit test overriden method.  Is called before every test."""
     AUTest.setUp(self)
-    self._KillExistingVM(self._KVM_PID_FILE)
-
-  @classmethod
-  def ProcessOptions(cls, parser, options):
-    """Processes vm-specific options."""
-    AUTest.ProcessOptions(parser, options)
-    cls.board = options.board
-
-    # Communicate flags to tests.
-    cls.graphics_flag = ''
-    if options.no_graphics: cls.graphics_flag = '--no_graphics'
-
-    if not cls.board:
-      parser.error('Need board to convert base image to vm.')
+    self._KillExistingVM(_KVM_PID_FILE)
 
   def PrepareBase(self, image_path):
     """Creates an update-able VM based on base image."""
@@ -509,32 +469,33 @@ class VirtualAUTest(unittest.TestCase, AUTest):
                   '--full',
                   '--from=%s' % ReinterpretPathForChroot(
                       os.path.dirname(image_path)),
-                  '--vdisk_size=%s' % self._FULL_VDISK_SIZE,
-                  '--statefulfs_size=%s' % self._FULL_STATEFULFS_SIZE,
-                  '--board=%s' % self.board,
+                  '--vdisk_size=%s' % _FULL_VDISK_SIZE,
+                  '--statefulfs_size=%s' % _FULL_STATEFULFS_SIZE,
+                  '--board=%s' % board,
                   '--test_image'], enter_chroot=True)
     else:
       Info('Using existing VM image %s' % self.vm_image_path)
 
+
     Info('Testing for %s' % self.vm_image_path)
+
     self.assertTrue(os.path.exists(self.vm_image_path))
 
-  def _UpdateImage(self, image_path, src_image_path='', stateful_change='old',
-                   proxy_port=None):
+  def UpdateImage(self, image_path, stateful_change='old', proxy_port=None):
     """Updates VM image with image_path."""
     stateful_change_flag = self.GetStatefulChangeFlag(stateful_change)
-    if src_image_path == self.base_image_path:
-      src_image_path = self.vm_image_path
+    if self.source_image == base_image_path:
+      self.source_image = self.vm_image_path
 
     cmd = ['%s/cros_run_vm_update' % self.crosutilsbin,
            '--update_image_path=%s' % image_path,
            '--vm_image_path=%s' % self.vm_image_path,
            '--snapshot',
-           self.graphics_flag,
+           vm_graphics_flag,
            '--persist',
-           '--kvm_pid=%s' % self._KVM_PID_FILE,
+           '--kvm_pid=%s' % _KVM_PID_FILE,
            stateful_change_flag,
-           '--src_image=%s' % src_image_path,
+           '--src_image=%s' % self.source_image,
            ]
 
     if proxy_port:
@@ -550,18 +511,24 @@ class VirtualAUTest(unittest.TestCase, AUTest):
       if code != 0:
         raise UpdateException(code, stdout)
 
-  def _UpdateUsingPayload(self, update_path, stateful_change='old',
+  def UpdateUsingPayload(self,
+                         update_path,
+                         stateful_change='old',
                          proxy_port=None):
     """Updates a remote image using image_to_live.sh."""
     stateful_change_flag = self.GetStatefulChangeFlag(stateful_change)
+    if self.source_image == base_image_path:
+      self.source_image = self.vm_image_path
+
     cmd = ['%s/cros_run_vm_update' % self.crosutilsbin,
            '--payload=%s' % update_path,
            '--vm_image_path=%s' % self.vm_image_path,
            '--snapshot',
-           self.graphics_flag,
+           vm_graphics_flag,
            '--persist',
-           '--kvm_pid=%s' % self._KVM_PID_FILE,
+           '--kvm_pid=%s' % _KVM_PID_FILE,
            stateful_change_flag,
+           '--src_image=%s' % self.source_image,
            ]
 
     if proxy_port:
@@ -586,19 +553,19 @@ class VirtualAUTest(unittest.TestCase, AUTest):
                        '--image_path=%s' % self.vm_image_path,
                        '--snapshot',
                        '--persist',
-                       '--kvm_pid=%s' % self._KVM_PID_FILE,
-                       self.verify_suite,
+                       '--kvm_pid=%s' % _KVM_PID_FILE,
+                       _VERIFY_SUITE,
                        ]
 
-    if self.graphics_flag:
-      commandWithArgs.append(self.graphics_flag)
+    if vm_graphics_flag:
+      commandWithArgs.append(vm_graphics_flag)
 
     output = RunCommand(commandWithArgs, error_ok=True, enter_chroot=False,
                         redirect_stdout=True)
-    return self.AssertEnoughTestsPassed(self, output, percent_required_to_pass)
+    return self.CommonVerifyImage(self, output, percent_required_to_pass)
 
 
-def main():
+if __name__ == '__main__':
   parser = optparse.OptionParser()
   parser.add_option('-b', '--base_image',
                     help='path to the base image.')
@@ -623,25 +590,47 @@ def main():
   parser.add_option('--verbose', default=False, action='store_true',
                     help='Print out rather than capture output as much as '
                          'possible.')
-  (options, leftover_args) = parser.parse_args()
+  # Set the usage to include flags.
+  parser.set_usage(parser.format_help())
+  # Parse existing sys.argv so we can pass rest to unittest.main.
+  (options, sys.argv) = parser.parse_args(sys.argv)
 
-  if leftover_args:
-    parser.error('Found extra options we do not support: %s' % leftover_args)
+  AUTest.verbose = options.verbose
+  base_image_path = options.base_image
+  target_image_path = options.target_image
+  board = options.board
+
+  if not base_image_path:
+    parser.error('Need path to base image for vm.')
+  elif not os.path.exists(base_image_path):
+    Die('%s does not exist' % base_image_path)
+
+  if not target_image_path:
+    parser.error('Need path to target image to update with.')
+  elif not os.path.exists(target_image_path):
+    Die('%s does not exist' % target_image_path)
+
+  if not board:
+    parser.error('Need board to convert base image to vm.')
+
+  # Communicate flags to tests.
+  vm_graphics_flag = ''
+  if options.no_graphics: vm_graphics_flag = '--no_graphics'
+  if options.quick_test: _VERIFY_SUITE = 'build_RootFilesystemSize'
+  AUTest.use_delta_updates = options.delta
+
+  # Only run the test harness we care about.
+  test_loader = unittest.TestLoader()
+  test_loader.testMethodPrefix = options.test_prefix
 
   if options.type == 'vm':  test_class = VirtualAUTest
   elif options.type == 'real': test_class = RealAUTest
   else: parser.error('Could not parse harness type %s.' % options.type)
 
-  test_class.ProcessOptions(parser, options)
+  remote = options.remote
 
-  test_loader = unittest.TestLoader()
-  test_loader.testMethodPrefix = options.test_prefix
   test_suite = test_loader.loadTestsFromTestCase(test_class)
   test_result = unittest.TextTestRunner(verbosity=2).run(test_suite)
 
   if not test_result.wasSuccessful():
     Die('Test harness was not successful')
-
-
-if __name__ == '__main__':
-  main()

From 53f189e92f5f2e05086ed657ff8bc2fc39d19b9e Mon Sep 17 00:00:00 2001
From: Scott Zawalski <scottz@google.com>
Date: Thu, 6 Jan 2011 00:44:36 -0800
Subject: [PATCH 02/16] Update gsutil path as it has changed

---
 prebuilt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/prebuilt.py b/prebuilt.py
index 5330191b3a..707362b316 100755
--- a/prebuilt.py
+++ b/prebuilt.py
@@ -42,7 +42,7 @@ Example of uploading prebuilt amd64 host files using rsync:
 # as per http://crosbug.com/5855 always filter the below packages
 _FILTER_PACKAGES = set()
 _RETRIES = 3
-_GSUTIL_BIN = '/b/third_party/gsutil/gsutil'
+_GSUTIL_BIN = '/b/build/third_party/gsutil/gsutil'
 _HOST_PACKAGES_PATH = 'chroot/var/lib/portage/pkgs'
 _HOST_TARGET = 'amd64'
 _BOARD_PATH = 'chroot/build/%(board)s'

From efbce1e26194e70b3fe9049db567c4f437cf597e Mon Sep 17 00:00:00 2001
From: Chris Sosa <sosa@chromium.org>
Date: Mon, 10 Jan 2011 09:56:34 -0800
Subject: [PATCH 03/16] Add simple update test to pfq.

This CL also fixes the outstanding issue where to won't report progress in
image_to_live on updates.

Change-Id: I1efaf17f0fd5ebb367ae0872377b4d5d0bf5dbf6

BUG=chromium-os:8680,chromium-os:8901
TEST=Ran it with --debug on the x86 pfq locally.

Review URL: http://codereview.chromium.org/6135002
---
 bin/cbuildbot.py            | 21 +++++++++++++++++++--
 bin/cbuildbot_config.py     | 14 +++++++-------
 bin/cros_au_test_harness.py |  4 ++--
 image_to_live.sh            | 30 +++++++++++++++++++-----------
 4 files changed, 47 insertions(+), 22 deletions(-)

diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py
index ff3fb8a94c..bb9e0cc23d 100755
--- a/bin/cbuildbot.py
+++ b/bin/cbuildbot.py
@@ -415,6 +415,22 @@ def _RunSmokeSuite(buildroot, results_dir):
               ], cwd=cwd, error_ok=False)
 
 
+def _RunAUTest(buildroot, board):
+  """Runs a basic update test from the au test harness."""
+  cwd = os.path.join(buildroot, 'src', 'scripts')
+  image_path = os.path.join(buildroot, 'src', 'build', 'images', board,
+                            'latest', 'chromiumos_test_image.bin')
+  RunCommand(['bin/cros_au_test_harness',
+              '--no_graphics',
+              '--no_delta',
+              '--board=%s' % board,
+              '--test_prefix=SimpleTest',
+              '--verbose',
+              '--base_image=%s' % image_path,
+              '--target_image=%s' % image_path,
+              ], cwd=cwd, error_ok=False)
+
+
 def _UprevPackages(buildroot, tracking_branch, revisionfile, board, overlays):
   """Uprevs a package based on given revisionfile.
 
@@ -695,15 +711,16 @@ def main():
 
     _BuildImage(buildroot)
 
-    if buildconfig['smoke_bvt'] and options.tests:
+    if buildconfig['tests'] and options.tests:
       _BuildVMImageForTesting(buildroot)
       test_results_dir = '/tmp/run_remote_tests.%s' % options.buildnumber
       try:
         _RunSmokeSuite(buildroot, test_results_dir)
+        _RunAUTest(buildroot, buildconfig['board'])
       finally:
         if not options.debug:
           archive_full_path = os.path.join(options.gsutil_archive,
-                                         str(options.buildnumber))
+                                           str(options.buildnumber))
           _ArchiveTestResults(buildroot, buildconfig['board'],
                               test_results_dir=test_results_dir,
                               gsutil=options.gsutil,
diff --git a/bin/cbuildbot_config.py b/bin/cbuildbot_config.py
index 2f7b6361dd..c13d3626cd 100644
--- a/bin/cbuildbot_config.py
+++ b/bin/cbuildbot_config.py
@@ -18,7 +18,7 @@ important -- Master bot uses important bots to determine overall status.
 hostname -- Needed for 'important' slaves.  The hostname of the bot.  Should
             match hostname in slaves.cfg in buildbot checkout.
 unittests -- Runs unittests for packages.
-smoke_bvt -- Runs the test smoke suite in a qemu-based VM using KVM.
+tests -- Runs the smoke suite and au test harness in a qemu-based VM using KVM.
 rev_overlays -- Select what overlays to look at for revving. This can be
                 'public', 'private' or 'both'.
 push_overlays -- Select what overlays to push at. This should be a subset of
@@ -35,7 +35,7 @@ config['default'] = {
   'master' : False,
   'important' : False,
   'unittests' : False,
-  'smoke_bvt' : False,
+  'tests' : False,
   'rev_overlays': 'public',
   'push_overlays': None,
 }
@@ -46,7 +46,7 @@ config['x86-generic-pre-flight-queue'] = {
   'important' : False,
   'hostname' : 'chromeosbuild2',
   'unittests' : True,
-  'smoke_bvt' : True,
+  'tests' : True,
   'rev_overlays': 'public',
   'push_overlays': 'public',
 }
@@ -56,7 +56,7 @@ config['x86-mario-pre-flight-queue'] = {
   'master' : True,
   'important' : False,
   'unittests' : True,
-  'smoke_bvt' : True,
+  'tests' : True,
   'rev_overlays': 'both',
   'push_overlays': 'private',
 }
@@ -66,7 +66,7 @@ config['x86-mario-pre-flight-branch'] = {
   'master' : True,
   'important' : False,
   'unittests' : True,
-  'smoke_bvt' : True,
+  'tests' : True,
   'rev_overlays': 'both',
   'push_overlays': 'both',
 }
@@ -76,7 +76,7 @@ config['x86_agz_bin'] = {
   'master' : False,
   'important' : False,
   'unittests' : True,
-  'smoke_bvt' : True,
+  'tests' : True,
   'rev_overlays': 'both',
   'push_overlays': None,
 }
@@ -86,7 +86,7 @@ config['x86_dogfood_bin'] = {
   'master' : False,
   'important' : False,
   'unittests' : True,
-  'smoke_bvt' : True,
+  'tests' : True,
   'rev_overlays': 'both',
   'push_overlays': None,
 }
diff --git a/bin/cros_au_test_harness.py b/bin/cros_au_test_harness.py
index 5d5b221760..aa9f77da12 100755
--- a/bin/cros_au_test_harness.py
+++ b/bin/cros_au_test_harness.py
@@ -358,12 +358,12 @@ class AUTest(object):
     self._AttemptUpdateWithFilter(DelayedFilter())
 
   def SimpleTest(self):
-    """A simple update  that updates the target image to itself.
+    """A simple update that updates once from a base image to a target.
 
     We explicitly don't use test prefix so that isn't run by default.  Can be
     run using test_prefix option.
     """
-    self.PrepareBase(target_image_path)
+    self.PrepareBase(base_image_path)
     self.UpdateImage(target_image_path)
     self.VerifyImage(100)
 
diff --git a/image_to_live.sh b/image_to_live.sh
index 553cd4d21d..bbb5ec904d 100755
--- a/image_to_live.sh
+++ b/image_to_live.sh
@@ -241,7 +241,6 @@ function get_update_log {
   echo "${REMOTE_OUT}" > "${FLAGS_update_log}"
 }
 
-
 # Returns ${1} reported by the update client e.g. PROGRESS, CURRENT_OP.
 function get_update_var {
   remote_sh "${UPDATER_BIN} --status 2> /dev/null |
@@ -254,21 +253,30 @@ function get_update_var {
 # This is expected to run in its own thread.
 function status_thread {
   local timeout=5
-  # Let update engine receive call to ping the dev server.
+
   info "Devserver handling ping.  Check ${FLAGS_server_log} for more info."
   sleep ${timeout}
 
-  # The devserver generates images when the update engine checks for updates.
-  while [ $(get_update_var CURRENT_OP) = ${UPDATER_UPDATE_CHECK} ]; do
-    echo -n "." && sleep ${timeout}
-  done
+  local current_state=""
+  local next_state="$(get_update_var CURRENT_OP)"
 
-  info "Update generated.  Update engine downloading update."
-  while [ $(get_update_var CURRENT_OP) = ${UPDATER_DOWNLOADING} ]; do
-    echo "Download progress $(get_update_var PROGRESS)" && sleep ${timeout}
-  done
+  # For current status, only print out status changes.
+  # For download, show progress.
+  # Finally if no status change print out .'s to keep dev informed.
+  while [ "${current_state}" != "${UPDATER_NEED_REBOOT}" ] && \
+      [ "${current_state}" != "${UPDATER_IDLE}" ]; do
+    if [ "${current_state}" != "${next_state}" ]; then
+      info "State of updater has changed to: ${next_state}"
+    elif [ "${next_state}" = "${UPDATER_DOWNLOADING}" ]; then
+      echo "Download progress $(get_update_var PROGRESS)"
+    else
+      echo -n "."
+    fi
 
-  info "Download complete."
+    sleep ${timeout}
+    current_state="${next_state}"
+    next_state="$(get_update_var CURRENT_OP)"
+  done
 }
 
 

From f58bc389d9c42d1584bf6ac5a5819dbfa777942c Mon Sep 17 00:00:00 2001
From: Chris Sosa <sosa@chromium.org>
Date: Mon, 10 Jan 2011 10:58:58 -0800
Subject: [PATCH 04/16] Create package keywords directory if it doesn't exist.

Clobbering the builder uncovered a bug that was lurking.

Change-Id: Ieb487d2112a7048438943aa96fba23e9c8412a13

BUG=chromium-os:8693
TEST=Running now

Review URL: http://codereview.chromium.org/6050004
---
 bin/cbuildbot.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py
index bb9e0cc23d..168d6212df 100755
--- a/bin/cbuildbot.py
+++ b/bin/cbuildbot.py
@@ -232,10 +232,12 @@ def _MarkChromeAsStable(buildroot, tracking_branch, chrome_rev, board):
     return None
   else:
     chrome_atom = portage_atom_string.split('=')[1]
+    keywords_file = CHROME_KEYWORDS_FILE % {'board': board}
     # TODO(sosa): Workaround to build unstable chrome ebuild we uprevved.
-    RunCommand(['sudo', 'tee', CHROME_KEYWORDS_FILE % {'board': board}],
-               input='=%s\n' % chrome_atom, enter_chroot=True,
-               cwd=cwd)
+    RunCommand(['sudo', 'mkdir', '-p', os.path.dirname(keywords_file)],
+               enter_chroot=True, cwd=cwd)
+    RunCommand(['sudo', 'tee', keywords_file], input='=%s\n' % chrome_atom,
+               enter_chroot=True, cwd=cwd)
     return chrome_atom
 
 

From 86213e67952987018fea2490784aa1706d4868e0 Mon Sep 17 00:00:00 2001
From: Dale Curtis <dalecurtis@chromium.org>
Date: Mon, 10 Jan 2011 11:50:01 -0800
Subject: [PATCH 05/16] Fix path for upcoming Autotest relocation.

Additional change for the relocation of Autotest from /home/autotest to /usr/local/autotest. See http://codereview.chromium.org/6090004/

Change-Id: Id6f7ac6e93124414b0b6c6bed48efccfd982a49d

BUG=chromium-os:10799
TEST=Will test as whole with rest of /usr/local changes.

Review URL: http://codereview.chromium.org/5972012
---
 mod_for_test_scripts/750enableExternalExtensions | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mod_for_test_scripts/750enableExternalExtensions b/mod_for_test_scripts/750enableExternalExtensions
index 9d3b33827e..8ceb04c20f 100755
--- a/mod_for_test_scripts/750enableExternalExtensions
+++ b/mod_for_test_scripts/750enableExternalExtensions
@@ -8,7 +8,7 @@
 # to launch a packaged extension at login.
 
 echo "Creating external_extensions.json symlink"
-LINK_TARGET="/home/autotest/tests/external_extensions.json"
+LINK_TARGET="/usr/local/autotest/tests/external_extensions.json"
 EXT_DIR="${ROOT_FS_DIR}/opt/google/chrome/extensions"
 FILE="${EXT_DIR}/external_extensions.json"
 mkdir -p $EXT_DIR

From 8617623516c010cb2af052032dcb64bcdc99d79b Mon Sep 17 00:00:00 2001
From: Chris Sosa <sosa@chromium.org>
Date: Mon, 10 Jan 2011 13:37:49 -0800
Subject: [PATCH 06/16] Major cleanup of cros_au_test_harness code.

This cleanup code consolidates parsing of options, removes
globals, cleans up private vs. public functions, removes the backup full update option, and cleans up
docstrings.

Change-Id: I3eba26b3634fd87f4a36f195ab42ec7bd11e963f

BUG=chromium-os:8901
TEST=Ran with prefix SimpleTest and ran whole suite for a vm using my latest built image.  Also ran normal update test using latest image against latest dev channel image for mario.  Re-tested with new changes after build break.

Committed: http://chrome-svn/viewvc/chromeos?view=rev&revision=2b2d7a1

Review URL: http://codereview.chromium.org/6015013
---
 bin/cros_au_test_harness.py | 354 +++++++++++++++++++-----------------
 1 file changed, 187 insertions(+), 167 deletions(-)

diff --git a/bin/cros_au_test_harness.py b/bin/cros_au_test_harness.py
index aa9f77da12..d0bbc6c25c 100755
--- a/bin/cros_au_test_harness.py
+++ b/bin/cros_au_test_harness.py
@@ -1,9 +1,17 @@
 #!/usr/bin/python
 
-# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
+# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+"""This module runs a suite of Auto Update tests.
+
+  The tests can be run on either a virtual machine or actual device depending
+  on parameters given.  Specific tests can be run by invoking --test_prefix.
+  Verbose is useful for many of the tests if you want to see individual commands
+  being run during the update process.
+"""
+
 import optparse
 import os
 import re
@@ -23,29 +31,16 @@ from cros_build_lib import Warning
 
 import cros_test_proxy
 
-# VM Constants.
-_FULL_VDISK_SIZE = 6072
-_FULL_STATEFULFS_SIZE = 3074
-_KVM_PID_FILE = '/tmp/harness_pid'
-_VERIFY_SUITE = 'suite_Smoke'
-
-# Globals to communicate options to unit tests.
-global base_image_path
-global board
-global remote
-global target_image_path
-global vm_graphics_flag
 
 class UpdateException(Exception):
-  """Exception thrown when UpdateImage or UpdateUsingPayload fail"""
+  """Exception thrown when _UpdateImage or _UpdateUsingPayload fail"""
   def __init__(self, code, stdout):
     self.code = code
     self.stdout = stdout
 
+
 class AUTest(object):
   """Abstract interface that defines an Auto Update test."""
-  source_image = ''
-  use_delta_updates = False
   verbose = False
 
   def setUp(self):
@@ -57,6 +52,8 @@ class AUTest(object):
     if not os.path.exists(self.download_folder):
       os.makedirs(self.download_folder)
 
+  # -------- Helper functions ---------
+
   def GetStatefulChangeFlag(self, stateful_change):
     """Returns the flag to pass to image_to_vm for the stateful change."""
     stateful_change_flag = ''
@@ -65,7 +62,7 @@ class AUTest(object):
 
     return stateful_change_flag
 
-  def ParseGenerateTestReportOutput(self, output):
+  def _ParseGenerateTestReportOutput(self, output):
     """Returns the percentage of tests that passed based on output."""
     percent_passed = 0
     lines = output.split('\n')
@@ -79,38 +76,60 @@ class AUTest(object):
 
     return int(percent_passed)
 
-  # TODO(sosa) - Remove try and convert function to DeltaUpdateImage().
-  def TryDeltaAndFallbackToFull(self, src_image, image, stateful_change='old'):
-    """Tries the delta update first if set and falls back to full update."""
-    if self.use_delta_updates:
-      try:
-        self.source_image = src_image
-        self._UpdateImageReportError(image, stateful_change)
-      except:
-        Warning('Delta update failed, disabling delta updates and retrying.')
-        self.use_delta_updates = False
-        self.source_image = ''
-        self._UpdateImageReportError(image, stateful_change)
-    else:
-      self._UpdateImageReportError(image, stateful_change)
+  def AssertEnoughTestsPassed(self, unittest, output, percent_required_to_pass):
+    """Helper function that asserts a sufficient number of tests passed.
 
-  def _UpdateImageReportError(self, image_path, stateful_change='old',
-                              proxy_port=None):
-    """Calls UpdateImage and reports any error to the console.
+    Args:
+      unittest: Handle to the unittest.
+      output: stdout from a test run.
+      percent_required_to_pass: percentage required to pass.  This should be
+        fall between 0-100.
+    Returns:
+      percent that passed.
+    """
+    Info('Output from VerifyImage():')
+    print >> sys.stderr, output
+    sys.stderr.flush()
+    percent_passed = self._ParseGenerateTestReportOutput(output)
+    Info('Percent passed: %d vs. Percent required: %d' % (
+        percent_passed, percent_required_to_pass))
+    unittest.assertTrue(percent_passed >= percent_required_to_pass)
+    return percent_passed
 
-       Still throws the exception.
+  def PerformUpdate(self, image_path, src_image_path='', stateful_change='old',
+                    proxy_port=None):
+    """Performs an update using  _UpdateImage and reports any error.
+
+    Subclasses should not override this method but override _UpdateImage
+    instead.
+
+    Args:
+      image_path:  Path to the image to update with.  This image must be a test
+        image.
+      src_image_path:  Optional.  If set, perform a delta update using the
+        image specified by the path as the source image.
+      stateful_change: How to modify the stateful partition.  Values are:
+          'old':  Don't modify stateful partition.  Just update normally.
+          'clean':  Uses clobber-state to wipe the stateful partition with the
+            exception of code needed for ssh.
+      proxy_port:  Port to have the client connect to. For use with
+        CrosTestProxy.
+    Raises an UpdateException if _UpdateImage returns an error.
     """
     try:
-      self.UpdateImage(image_path, stateful_change, proxy_port)
+      if not self.use_delta_updates:
+        src_image_path = ''
+
+      self._UpdateImage(image_path, src_image_path, stateful_change, proxy_port)
     except UpdateException as err:
       # If the update fails, print it out
       Warning(err.stdout)
       raise
 
-  def _AttemptUpdateWithPayloadExpectedFailure(self, payload, expected_msg):
+  def AttemptUpdateWithPayloadExpectedFailure(self, payload, expected_msg):
     """Attempt a payload update, expect it to fail with expected log"""
     try:
-      self.UpdateUsingPayload(payload)
+      self._UpdateUsingPayload(payload)
     except UpdateException as err:
       # Will raise ValueError if expected is not found.
       if re.search(re.escape(expected_msg), err.stdout, re.MULTILINE):
@@ -120,10 +139,10 @@ class AUTest(object):
     Warning(err.stdout)
     self.fail('We managed to update when failure was expected')
 
-  def _AttemptUpdateWithFilter(self, filter):
+  def AttemptUpdateWithFilter(self, filter):
     """Update through a proxy, with a specified filter, and expect success."""
 
-    self.PrepareBase(target_image_path)
+    self.PrepareBase(self.target_image_path)
 
     # The devserver runs at port 8080 by default. We assume that here, and
     # start our proxy at 8081. We then tell our update tools to have the
@@ -137,34 +156,59 @@ class AUTest(object):
 
     # This update is expected to fail...
     try:
-      self._UpdateImageReportError(target_image_path, proxy_port=proxy_port)
+      self.PerformUpdate(self.target_image_path, self.base_image_path,
+                         proxy_port=proxy_port)
     finally:
       proxy.shutdown()
 
+  # -------- Functions that subclasses should override ---------
+
+  @classmethod
+  def ProcessOptions(cls, parser, options):
+    """Processes options.
+
+    Static method that should be called from main.  Subclasses should also
+    call their parent method if they override it.
+    """
+    cls.verbose = options.verbose
+    cls.base_image_path = options.base_image
+    cls.target_image_path = options.target_image
+    cls.use_delta_updates = options.delta
+    if options.quick_test:
+      cls.verify_suite = 'build_RootFilesystemSize'
+    else:
+      cls.verify_suite = 'suite_Smoke'
+
+    # Sanity checks.
+    if not cls.base_image_path:
+      parser.error('Need path to base image for vm.')
+    elif not os.path.exists(cls.base_image_path):
+      Die('%s does not exist' % cls.base_image_path)
+
+    if not cls.target_image_path:
+      parser.error('Need path to target image to update with.')
+    elif not os.path.exists(cls.target_image_path):
+      Die('%s does not exist' % cls.target_image_path)
+
   def PrepareBase(self, image_path):
     """Prepares target with base_image_path."""
     pass
 
-  def UpdateImage(self, image_path, stateful_change='old', proxy_port=None):
-    """Updates target with the image given by the image_path.
+  def _UpdateImage(self, image_path, src_image_path='', stateful_change='old',
+                   proxy_port=None):
+    """Implementation of an actual update.
 
-    Args:
-      image_path:  Path to the image to update with.  This image must be a test
-        image.
-      stateful_change: How to modify the stateful partition.  Values are:
-          'old':  Don't modify stateful partition.  Just update normally.
-          'clean':  Uses clobber-state to wipe the stateful partition with the
-            exception of code needed for ssh.
-      proxy_port:  Port to have the client connect to. For use with
-        CrosTestProxy.
+    See PerformUpdate for description of args.  Subclasses must override this
+    method with the correct update procedure for the class.
     """
     pass
 
-  def UpdateUsingPayload(self,
-                         update_path,
-                         stateful_change='old',
+  def _UpdateUsingPayload(self, update_path, stateful_change='old',
                          proxy_port=None):
-    """Updates target with the pre-generated update stored in update_path
+    """Updates target with the pre-generated update stored in update_path.
+
+    Subclasses must override this method with the correct update procedure for
+    the class.
 
     Args:
       update_path:  Path to the image to update with. This directory should
@@ -177,7 +221,8 @@ class AUTest(object):
   def VerifyImage(self, percent_required_to_pass):
     """Verifies the image with tests.
 
-    Verifies that the test images passes the percent required.
+    Verifies that the test images passes the percent required.  Subclasses must
+    override this method with the correct update procedure for the class.
 
     Args:
       percent_required_to_pass:  percentage required to pass.  This should be
@@ -188,31 +233,9 @@ class AUTest(object):
     """
     pass
 
-  def CommonVerifyImage(self, unittest, output, percent_required_to_pass):
-    """Helper function for VerifyImage that returns percent of tests passed.
+  # -------- Tests ---------
 
-    Takes output from a test suite, verifies the number of tests passed is
-    sufficient and outputs info.
-
-    Args:
-      unittest: Handle to the unittest.
-      output: stdout from a test run.
-      percent_required_to_pass: percentage required to pass.  This should be
-        fall between 0-100.
-    Returns:
-      percent that passed.
-    """
-    Info('Output from VerifyImage():')
-    print >> sys.stderr, output
-    sys.stderr.flush()
-    percent_passed = self.ParseGenerateTestReportOutput(output)
-    Info('Percent passed: %d vs. Percent required: %d' % (
-        percent_passed, percent_required_to_pass))
-    unittest.assertTrue(percent_passed >=
-                        percent_required_to_pass)
-    return percent_passed
-
-  def testFullUpdateKeepStateful(self):
+  def testUpdateKeepStateful(self):
     """Tests if we can update normally.
 
     This test checks that we can update by updating the stateful partition
@@ -220,22 +243,22 @@ class AUTest(object):
     """
     # Just make sure some tests pass on original image.  Some old images
     # don't pass many tests.
-    self.PrepareBase(base_image_path)
+    self.PrepareBase(self.base_image_path)
     # TODO(sosa): move to 100% once we start testing using the autotest paired
     # with the dev channel.
     percent_passed = self.VerifyImage(10)
 
     # Update to - all tests should pass on new image.
     Info('Updating from base image on vm to target image.')
-    self.TryDeltaAndFallbackToFull(base_image_path, target_image_path)
+    self.PerformUpdate(self.target_image_path, self.base_image_path)
     self.VerifyImage(100)
 
     # Update from - same percentage should pass that originally passed.
     Info('Updating from updated image on vm back to base image.')
-    self.TryDeltaAndFallbackToFull(target_image_path, base_image_path)
+    self.PerformUpdate(self.base_image_path, self.target_image_path)
     self.VerifyImage(percent_passed)
 
-  def testFullUpdateWipeStateful(self):
+  def testUpdateWipeStateful(self):
     """Tests if we can update after cleaning the stateful partition.
 
     This test checks that we can update successfully after wiping the
@@ -243,25 +266,25 @@ class AUTest(object):
     """
     # Just make sure some tests pass on original image.  Some old images
     # don't pass many tests.
-    self.PrepareBase(base_image_path)
+    self.PrepareBase(self.base_image_path)
     # TODO(sosa): move to 100% once we start testing using the autotest paired
     # with the dev channel.
     percent_passed = self.VerifyImage(10)
 
     # Update to - all tests should pass on new image.
     Info('Updating from base image on vm to target image and wiping stateful.')
-    self.TryDeltaAndFallbackToFull(base_image_path, target_image_path, 'clean')
+    self.PerformUpdate(self.target_image_path, self.base_image_path, 'clean')
     self.VerifyImage(100)
 
     # Update from - same percentage should pass that originally passed.
     Info('Updating from updated image back to base image and wiping stateful.')
-    self.TryDeltaAndFallbackToFull(target_image_path, base_image_path, 'clean')
+    self.PerformUpdate(self.base_image_path, self.target_image_path, 'clean')
     self.VerifyImage(percent_passed)
 
   def testPartialUpdate(self):
     """Tests what happens if we attempt to update with a truncated payload."""
     # Preload with the version we are trying to test.
-    self.PrepareBase(target_image_path)
+    self.PrepareBase(self.target_image_path)
 
     # Image can be updated at:
     # ~chrome-eng/chromeos/localmirror/autest-images
@@ -273,12 +296,12 @@ class AUTest(object):
     urllib.urlretrieve(url, payload)
 
     expected_msg = 'download_hash_data == update_check_response_hash failed'
-    self._AttemptUpdateWithPayloadExpectedFailure(payload, expected_msg)
+    self.AttemptUpdateWithPayloadExpectedFailure(payload, expected_msg)
 
   def testCorruptedUpdate(self):
     """Tests what happens if we attempt to update with a corrupted payload."""
     # Preload with the version we are trying to test.
-    self.PrepareBase(target_image_path)
+    self.PrepareBase(self.target_image_path)
 
     # Image can be updated at:
     # ~chrome-eng/chromeos/localmirror/autest-images
@@ -291,7 +314,7 @@ class AUTest(object):
 
     # This update is expected to fail...
     expected_msg = 'zlib inflate() error:-3'
-    self._AttemptUpdateWithPayloadExpectedFailure(payload, expected_msg)
+    self.AttemptUpdateWithPayloadExpectedFailure(payload, expected_msg)
 
   def testInterruptedUpdate(self):
     """Tests what happens if we interrupt payload delivery 3 times."""
@@ -325,7 +348,7 @@ class AUTest(object):
         self.data_size += len(data)
         return data
 
-    self._AttemptUpdateWithFilter(InterruptionFilter())
+    self.AttemptUpdateWithFilter(InterruptionFilter())
 
   def testDelayedUpdate(self):
     """Tests what happens if some data is delayed during update delivery"""
@@ -355,7 +378,7 @@ class AUTest(object):
         self.data_size += len(data)
         return data
 
-    self._AttemptUpdateWithFilter(DelayedFilter())
+    self.AttemptUpdateWithFilter(DelayedFilter())
 
   def SimpleTest(self):
     """A simple update that updates once from a base image to a target.
@@ -363,8 +386,8 @@ class AUTest(object):
     We explicitly don't use test prefix so that isn't run by default.  Can be
     run using test_prefix option.
     """
-    self.PrepareBase(base_image_path)
-    self.UpdateImage(target_image_path)
+    self.PrepareBase(self.base_image_path)
+    self.PerformUpdate(self.target_image_path, self.base_image_path)
     self.VerifyImage(100)
 
 
@@ -374,19 +397,29 @@ class RealAUTest(unittest.TestCase, AUTest):
   def setUp(self):
     AUTest.setUp(self)
 
+  @classmethod
+  def ProcessOptions(cls, parser, options):
+    """Processes non-vm-specific options."""
+    AUTest.ProcessOptions(parser, options)
+    cls.remote = options.remote
+
+    if not cls.remote:
+      parser.error('We require a remote address for real tests.')
+
   def PrepareBase(self, image_path):
     """Auto-update to base image to prepare for test."""
-    self._UpdateImageReportError(image_path)
+    self.PerformUpdate(image_path)
 
-  def UpdateImage(self, image_path, stateful_change='old', proxy_port=None):
+  def _UpdateImage(self, image_path, src_image_path='', stateful_change='old',
+                   proxy_port=None):
     """Updates a remote image using image_to_live.sh."""
     stateful_change_flag = self.GetStatefulChangeFlag(stateful_change)
     cmd = ['%s/image_to_live.sh' % self.crosutils,
            '--image=%s' % image_path,
-           '--remote=%s' % remote,
+           '--remote=%s' % self.remote,
            stateful_change_flag,
            '--verify',
-           '--src_image=%s' % self.source_image
+           '--src_image=%s' % src_image_path
           ]
 
     if proxy_port:
@@ -402,15 +435,13 @@ class RealAUTest(unittest.TestCase, AUTest):
       if code != 0:
         raise UpdateException(code, stdout)
 
-  def UpdateUsingPayload(self,
-                         update_path,
-                         stateful_change='old',
+  def _UpdateUsingPayload(self, update_path, stateful_change='old',
                          proxy_port=None):
     """Updates a remote image using image_to_live.sh."""
     stateful_change_flag = self.GetStatefulChangeFlag(stateful_change)
     cmd = ['%s/image_to_live.sh' % self.crosutils,
            '--payload=%s' % update_path,
-           '--remote=%s' % remote,
+           '--remote=%s' % self.remote,
            stateful_change_flag,
            '--verify',
           ]
@@ -432,16 +463,21 @@ class RealAUTest(unittest.TestCase, AUTest):
     """Verifies an image using run_remote_tests.sh with verification suite."""
     output = RunCommand([
         '%s/run_remote_tests.sh' % self.crosutils,
-        '--remote=%s' % remote,
-        _VERIFY_SUITE,
+        '--remote=%s' % self.remote,
+        self.verify_suite,
        ], error_ok=True, enter_chroot=False, redirect_stdout=True)
-    return self.CommonVerifyImage(self, output, percent_required_to_pass)
+    return self.AssertEnoughTestsPassed(self, output, percent_required_to_pass)
 
 
 class VirtualAUTest(unittest.TestCase, AUTest):
   """Test harness for updating virtual machines."""
   vm_image_path = None
 
+  # VM Constants.
+  _FULL_VDISK_SIZE = 6072
+  _FULL_STATEFULFS_SIZE = 3074
+  _KVM_PID_FILE = '/tmp/harness_pid'
+
   def _KillExistingVM(self, pid_file):
     if os.path.exists(pid_file):
       Warning('Existing %s found.  Deleting and killing process' %
@@ -454,7 +490,20 @@ class VirtualAUTest(unittest.TestCase, AUTest):
   def setUp(self):
     """Unit test overriden method.  Is called before every test."""
     AUTest.setUp(self)
-    self._KillExistingVM(_KVM_PID_FILE)
+    self._KillExistingVM(self._KVM_PID_FILE)
+
+  @classmethod
+  def ProcessOptions(cls, parser, options):
+    """Processes vm-specific options."""
+    AUTest.ProcessOptions(parser, options)
+    cls.board = options.board
+
+    # Communicate flags to tests.
+    cls.graphics_flag = ''
+    if options.no_graphics: cls.graphics_flag = '--no_graphics'
+
+    if not cls.board:
+      parser.error('Need board to convert base image to vm.')
 
   def PrepareBase(self, image_path):
     """Creates an update-able VM based on base image."""
@@ -469,33 +518,32 @@ class VirtualAUTest(unittest.TestCase, AUTest):
                   '--full',
                   '--from=%s' % ReinterpretPathForChroot(
                       os.path.dirname(image_path)),
-                  '--vdisk_size=%s' % _FULL_VDISK_SIZE,
-                  '--statefulfs_size=%s' % _FULL_STATEFULFS_SIZE,
-                  '--board=%s' % board,
+                  '--vdisk_size=%s' % self._FULL_VDISK_SIZE,
+                  '--statefulfs_size=%s' % self._FULL_STATEFULFS_SIZE,
+                  '--board=%s' % self.board,
                   '--test_image'], enter_chroot=True)
     else:
       Info('Using existing VM image %s' % self.vm_image_path)
 
-
     Info('Testing for %s' % self.vm_image_path)
-
     self.assertTrue(os.path.exists(self.vm_image_path))
 
-  def UpdateImage(self, image_path, stateful_change='old', proxy_port=None):
+  def _UpdateImage(self, image_path, src_image_path='', stateful_change='old',
+                   proxy_port=None):
     """Updates VM image with image_path."""
     stateful_change_flag = self.GetStatefulChangeFlag(stateful_change)
-    if self.source_image == base_image_path:
-      self.source_image = self.vm_image_path
+    if src_image_path == self.base_image_path:
+      src_image_path = self.vm_image_path
 
     cmd = ['%s/cros_run_vm_update' % self.crosutilsbin,
            '--update_image_path=%s' % image_path,
            '--vm_image_path=%s' % self.vm_image_path,
            '--snapshot',
-           vm_graphics_flag,
+           self.graphics_flag,
            '--persist',
-           '--kvm_pid=%s' % _KVM_PID_FILE,
+           '--kvm_pid=%s' % self._KVM_PID_FILE,
            stateful_change_flag,
-           '--src_image=%s' % self.source_image,
+           '--src_image=%s' % src_image_path,
            ]
 
     if proxy_port:
@@ -511,24 +559,18 @@ class VirtualAUTest(unittest.TestCase, AUTest):
       if code != 0:
         raise UpdateException(code, stdout)
 
-  def UpdateUsingPayload(self,
-                         update_path,
-                         stateful_change='old',
+  def _UpdateUsingPayload(self, update_path, stateful_change='old',
                          proxy_port=None):
-    """Updates a remote image using image_to_live.sh."""
+    """Updates a vm image using cros_run_vm_update."""
     stateful_change_flag = self.GetStatefulChangeFlag(stateful_change)
-    if self.source_image == base_image_path:
-      self.source_image = self.vm_image_path
-
     cmd = ['%s/cros_run_vm_update' % self.crosutilsbin,
            '--payload=%s' % update_path,
            '--vm_image_path=%s' % self.vm_image_path,
            '--snapshot',
-           vm_graphics_flag,
+           self.graphics_flag,
            '--persist',
-           '--kvm_pid=%s' % _KVM_PID_FILE,
+           '--kvm_pid=%s' % self._KVM_PID_FILE,
            stateful_change_flag,
-           '--src_image=%s' % self.source_image,
            ]
 
     if proxy_port:
@@ -553,19 +595,19 @@ class VirtualAUTest(unittest.TestCase, AUTest):
                        '--image_path=%s' % self.vm_image_path,
                        '--snapshot',
                        '--persist',
-                       '--kvm_pid=%s' % _KVM_PID_FILE,
-                       _VERIFY_SUITE,
+                       '--kvm_pid=%s' % self._KVM_PID_FILE,
+                       self.verify_suite,
                        ]
 
-    if vm_graphics_flag:
-      commandWithArgs.append(vm_graphics_flag)
+    if self.graphics_flag:
+      commandWithArgs.append(self.graphics_flag)
 
     output = RunCommand(commandWithArgs, error_ok=True, enter_chroot=False,
                         redirect_stdout=True)
-    return self.CommonVerifyImage(self, output, percent_required_to_pass)
+    return self.AssertEnoughTestsPassed(self, output, percent_required_to_pass)
 
 
-if __name__ == '__main__':
+def main():
   parser = optparse.OptionParser()
   parser.add_option('-b', '--base_image',
                     help='path to the base image.')
@@ -590,47 +632,25 @@ if __name__ == '__main__':
   parser.add_option('--verbose', default=False, action='store_true',
                     help='Print out rather than capture output as much as '
                          'possible.')
-  # Set the usage to include flags.
-  parser.set_usage(parser.format_help())
-  # Parse existing sys.argv so we can pass rest to unittest.main.
-  (options, sys.argv) = parser.parse_args(sys.argv)
+  (options, leftover_args) = parser.parse_args()
 
-  AUTest.verbose = options.verbose
-  base_image_path = options.base_image
-  target_image_path = options.target_image
-  board = options.board
-
-  if not base_image_path:
-    parser.error('Need path to base image for vm.')
-  elif not os.path.exists(base_image_path):
-    Die('%s does not exist' % base_image_path)
-
-  if not target_image_path:
-    parser.error('Need path to target image to update with.')
-  elif not os.path.exists(target_image_path):
-    Die('%s does not exist' % target_image_path)
-
-  if not board:
-    parser.error('Need board to convert base image to vm.')
-
-  # Communicate flags to tests.
-  vm_graphics_flag = ''
-  if options.no_graphics: vm_graphics_flag = '--no_graphics'
-  if options.quick_test: _VERIFY_SUITE = 'build_RootFilesystemSize'
-  AUTest.use_delta_updates = options.delta
-
-  # Only run the test harness we care about.
-  test_loader = unittest.TestLoader()
-  test_loader.testMethodPrefix = options.test_prefix
+  if leftover_args:
+    parser.error('Found extra options we do not support: %s' % leftover_args)
 
   if options.type == 'vm':  test_class = VirtualAUTest
   elif options.type == 'real': test_class = RealAUTest
   else: parser.error('Could not parse harness type %s.' % options.type)
 
-  remote = options.remote
+  test_class.ProcessOptions(parser, options)
 
+  test_loader = unittest.TestLoader()
+  test_loader.testMethodPrefix = options.test_prefix
   test_suite = test_loader.loadTestsFromTestCase(test_class)
   test_result = unittest.TextTestRunner(verbosity=2).run(test_suite)
 
   if not test_result.wasSuccessful():
     Die('Test harness was not successful')
+
+
+if __name__ == '__main__':
+  main()

From d07f61d8124ace34c69426d2daf2359627cdf725 Mon Sep 17 00:00:00 2001
From: Will Drewry <wad@chromium.org>
Date: Wed, 12 Jan 2011 10:39:54 -0600
Subject: [PATCH 07/16] build_image: enable Chrome OS partition/hardware error
 behavior in dm-verity

ARM (kernel-next) should now be in sync with kernel.git so that we get similar
error handling behavior on all targeted platforms (with our partition layout).

This change makes that the default.

BUG=chromium-os:9697
TEST=manual build_image with kernel-next profile; booted and validated failover error behavior

Change-Id: I281302adb3cbc96aa5770630898f103b0d3639ca

Review URL: http://codereview.chromium.org/6220002
---
 build_image | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build_image b/build_image
index 61b28215b8..9674e36ab0 100755
--- a/build_image
+++ b/build_image
@@ -71,9 +71,9 @@ DEFINE_string usb_disk /dev/sdb3 \
 
 DEFINE_boolean enable_rootfs_verification ${FLAGS_TRUE} \
   "Default all bootloaders to use kernel-based root fs integrity checking."
-DEFINE_integer verity_error_behavior 1 \
+DEFINE_integer verity_error_behavior 3 \
   "Kernel verified boot error behavior (0: I/O errors, 1: panic, 2: nothing, \
-3: cros) Default: 1"
+3: cros) Default: 3"
 DEFINE_integer verity_depth 1 \
   "Kernel verified boot hash tree depth. Default: 1"
 DEFINE_integer verity_max_ios -1 \

From 4092ff565115029559935d2d371ab6477a45ddd7 Mon Sep 17 00:00:00 2001
From: Frank Swiderski <fes@chromium.org>
Date: Wed, 12 Jan 2011 13:50:48 -0800
Subject: [PATCH 08/16] This change removes the autotest mount-specific change
 from the mod_for_test scripts as we no longer need executable /tmp or /home
 for autotest.

Change-Id: I03c563b11d5df043683e904c7794faa34748b063

BUG=chromium-os:7286
TEST=bvt autotest

Review URL: http://codereview.chromium.org/6080007
---
 mod_for_test_scripts/900mountHomeExec | 9 ---------
 1 file changed, 9 deletions(-)
 delete mode 100755 mod_for_test_scripts/900mountHomeExec

diff --git a/mod_for_test_scripts/900mountHomeExec b/mod_for_test_scripts/900mountHomeExec
deleted file mode 100755
index bc609673b8..0000000000
--- a/mod_for_test_scripts/900mountHomeExec
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# Create a marker file indicating that we are in mod_for_test
-echo "Setting /home to be mounted exec"
-sed -i -e 's/#mod_for_test#//g' ${ROOT_FS_DIR}/sbin/chromeos_startup

From 3866923b00bfe9299dbe99784d7b367386f6cc37 Mon Sep 17 00:00:00 2001
From: Antoine Labour <piman@chromium.org>
Date: Wed, 12 Jan 2011 14:36:51 -0800
Subject: [PATCH 09/16] Revert "This change removes the autotest mount-specific
 change from the mod_for_test scripts as we no longer need executable /tmp or
 /home for autotest."

This reverts commit 4092ff565115029559935d2d371ab6477a45ddd7.

TBR=fes

Change-Id: I9baa8836fce86963af97a618d75aeb90325fbc42
---
 mod_for_test_scripts/900mountHomeExec | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100755 mod_for_test_scripts/900mountHomeExec

diff --git a/mod_for_test_scripts/900mountHomeExec b/mod_for_test_scripts/900mountHomeExec
new file mode 100755
index 0000000000..bc609673b8
--- /dev/null
+++ b/mod_for_test_scripts/900mountHomeExec
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Create a marker file indicating that we are in mod_for_test
+echo "Setting /home to be mounted exec"
+sed -i -e 's/#mod_for_test#//g' ${ROOT_FS_DIR}/sbin/chromeos_startup

From 60587527c140523ec1e274a093a672991756a14f Mon Sep 17 00:00:00 2001
From: Chris Sosa <sosa@chromium.org>
Date: Wed, 12 Jan 2011 15:10:35 -0800
Subject: [PATCH 10/16] Add more robust ssh ping to be run on reboot for
 updates.

Upon investigation it seems there's a race condition when ssh
starts up the first time.  This adds a much more robust way to
ping a machine when it's up then the previous method.

Change-Id: I092744b259c169975d1c4f283f01a556fce24723

BUG=chromium-os:10867
TEST=Ran it 30 times with image_to_live --noupdate --nostateful_update.

Review URL: http://codereview.chromium.org/6115009
---
 remote_access.sh | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/remote_access.sh b/remote_access.sh
index 794deff8e7..8b8abf7a4e 100644
--- a/remote_access.sh
+++ b/remote_access.sh
@@ -34,8 +34,37 @@ function remote_sh() {
   return ${PIPESTATUS[0]}
 }
 
+# Checks to see if pid $1 is running.
+function is_pid_running() {
+  ps -p ${1} 2>&1 > /dev/null
+}
+
+# Wait function given an additional timeout argument.
+# $1 - pid to wait on.
+# $2 - timeout to wait for.
+function wait_with_timeout() {
+  local pid=$1
+  local timeout=$2
+  local -r TIMEOUT_INC=1
+  local current_timeout=0
+  while is_pid_running ${pid} && [ ${current_timeout} -lt ${timeout} ]; do
+    sleep ${TIMEOUT_INC}
+    current_timeout=$((current_timeout + TIMEOUT_INC))
+  done
+  is_pid_running ${pid}
+}
+
+# Robust ping that will monitor ssh and not hang even if ssh hangs.
+function ping_ssh() {
+  remote_sh "true" &
+  local pid=$!
+  wait_with_timeout ${pid} 5
+  ! kill -9 ${pid} 2> /dev/null
+}
+
 function remote_sh_allow_changed_host_key() {
   rm -f $TMP_KNOWN_HOSTS
+  ping_ssh
   remote_sh "$@"
 }
 

From c584cf0e957fbbecc17a772b48c0923174d309e9 Mon Sep 17 00:00:00 2001
From: Hung-Te Lin <hungte@chromium.org>
Date: Thu, 13 Jan 2011 15:34:59 +0800
Subject: [PATCH 11/16] crosutils/make_factory_package.sh: Support relative
 path

Since there's a pushd call when building images, we have to first retrieve
the complete (full/absolute) path of target disk image.

BUG=chromium-os:10706
TEST=make_factory_package.sh \
     --factory ../build/images/x86-mario/latest/factory_image.bin \
     --release ../build/images/x86-mario/latest/chromiumos_image.bin \
     --diskimg blah_relative.bin

Change-Id: I02e2edb0f76b724337a82dd400297877681bc338

Review URL: http://codereview.chromium.org/6244003
---
 make_factory_package.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/make_factory_package.sh b/make_factory_package.sh
index 71233fb09c..d89589f8a7 100755
--- a/make_factory_package.sh
+++ b/make_factory_package.sh
@@ -86,7 +86,7 @@ RELEASE_IMAGE="$(basename "${FLAGS_release}")"
 FACTORY_IMAGE="$(basename "${FLAGS_factory}")"
 
 prepare_img() {
-  local outdev="$FLAGS_diskimg"
+  local outdev="$(readlink -f "$FLAGS_diskimg")"
   local sectors="$FLAGS_sectors"
   local force_full="true"
 
@@ -191,7 +191,7 @@ else
 fi
 
 generate_img() {
-  local outdev="$FLAGS_diskimg"
+  local outdev="$(readlink -f "$FLAGS_diskimg")"
   local sectors="$FLAGS_sectors"
 
   prepare_img

From 4653cbd0e5b916451795ded8d32a2345d4518cc6 Mon Sep 17 00:00:00 2001
From: Hung-Te Lin <hungte@chromium.org>
Date: Thu, 13 Jan 2011 16:59:42 +0800
Subject: [PATCH 12/16] crosutils/make_factory_package.sh: support full SSD
 image as target outside chroot

When running outside chroot, we should use locate_cgpt + "$GPT" to invoke cgpt;
otherwise it may be not in system PATH.

For dealing with a true SSD image (over USB, for example) this CL also improved
data writing speed from <2M/s to 8M/s+ (over USB).

BUG=chromium-os:10531
TEST=./make_factory_package.sh \
  --factory=../build/images/x86-mario/latest/chromiumos_factory_image.bin \
  --release=../build/images/x86-mario/latest//chromiumos_image.bin \
  --diskimg=/dev/sdb

Also tested:
  --diskimg=blah # local file
  (chroot) --diskimg=/dev/sdd
  (chroot) --diskimg=blah

Change-Id: I647d97fe388f5b6a6223fa8cacfecdf00265f60c

Review URL: http://codereview.chromium.org/6261003
---
 lib/cros_image_common.sh | 12 +++++++++++-
 make_factory_package.sh  |  3 ++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/lib/cros_image_common.sh b/lib/cros_image_common.sh
index abfdbc57c9..b9e2802c09 100644
--- a/lib/cros_image_common.sh
+++ b/lib/cros_image_common.sh
@@ -185,6 +185,16 @@ image_partition_copy() {
     exit 1
   fi
 
+  # Try to use larger buffer if offset/size can be re-aligned.
+  # 2M / 512 = 4096
+  local buffer_ratio=4096
+  local bs=512
+  if [ $((dstoffset % buffer_ratio)) -eq 0 -a \
+       $((length % buffer_ratio)) -eq 0 ]; then
+    dstoffset=$((dstoffset / buffer_ratio))
+    bs=$((bs * buffer_ratio))
+  fi
+
   image_dump_partition "${src}" "${srcpart}" |
-      dd of="${dst}" bs=512 seek="${dstoffset}" conv=notrunc
+      dd of="${dst}" bs="${bs}" seek="${dstoffset}" conv=notrunc oflag=dsync
 }
diff --git a/make_factory_package.sh b/make_factory_package.sh
index d89589f8a7..c2575988d1 100755
--- a/make_factory_package.sh
+++ b/make_factory_package.sh
@@ -110,9 +110,10 @@ prepare_img() {
   fi
 
   # Create GPT partition table.
+  locate_gpt
   install_gpt "${outdev}" 0 0 "${pmbrcode}" 0 "${force_full}"
   # Activate the correct partition.
-  cgpt add -i 2 -S 1 -P 1 "${outdev}"
+  sudo "${GPT}" add -i 2 -S 1 -P 1 "${outdev}"
 }
 
 prepare_omaha() {

From 78476aba5f30d213eb5ec83f3a3f6f00ed83750b Mon Sep 17 00:00:00 2001
From: Mandeep Singh Baines <msb@chromium.org>
Date: Thu, 13 Jan 2011 11:16:51 -0800
Subject: [PATCH 13/16] remote_access.sh: remove dependence on ssh-agent

Change-Id: I68d02f5630bd885894ca69728254f2a5ba5cc2aa

BUG=n0ne
TEST=Verified I could successfully ssh.

Review URL: http://codereview.chromium.org/6314001
---
 remote_access.sh | 23 +++++++++--------------
 1 file changed, 9 insertions(+), 14 deletions(-)

diff --git a/remote_access.sh b/remote_access.sh
index 8b8abf7a4e..a34ed5ffaa 100644
--- a/remote_access.sh
+++ b/remote_access.sh
@@ -15,22 +15,24 @@ DEFINE_integer ssh_port 22 \
 
 # Copies $1 to $2 on remote host
 function remote_cp_to() {
-  REMOTE_OUT=$(scp -P ${FLAGS_ssh_port} -o StrictHostKeyChecking=no -o \
-    UserKnownHostsFile=$TMP_KNOWN_HOSTS $1 root@$FLAGS_remote:$2)
+  REMOTE_OUT=$(scp -P ${FLAGS_ssh_port} -o StrictHostKeyChecking=no \
+    -o UserKnownHostsFile=$TMP_KNOWN_HOSTS -i $TMP_PRIVATE_KEY $1 \
+    root@$FLAGS_remote:$2)
   return ${PIPESTATUS[0]}
 }
 
 # Copies a list of remote files specified in file $1 to local location
 # $2.  Directory paths in $1 are collapsed into $2.
 function remote_rsync_from() {
-  rsync -e "ssh -p ${FLAGS_ssh_port} -o StrictHostKeyChecking=no -o \
-            UserKnownHostsFile=$TMP_KNOWN_HOSTS" --no-R \
-    --files-from=$1 root@${FLAGS_remote}:/ $2
+  rsync -e "ssh -p ${FLAGS_ssh_port} -o StrictHostKeyChecking=no \
+             -o UserKnownHostsFile=$TMP_KNOWN_HOSTS -i $TMP_PRIVATE_KEY" \
+    --no-R --files-from=$1 root@${FLAGS_remote}:/ $2
 }
 
 function remote_sh() {
-  REMOTE_OUT=$(ssh -p ${FLAGS_ssh_port} -o StrictHostKeyChecking=no -o \
-    UserKnownHostsFile=$TMP_KNOWN_HOSTS root@$FLAGS_remote "$@")
+  REMOTE_OUT=$(ssh -p ${FLAGS_ssh_port} -o StrictHostKeyChecking=no \
+    -o UserKnownHostsFile=$TMP_KNOWN_HOSTS -i $TMP_PRIVATE_KEY \
+    root@$FLAGS_remote "$@")
   return ${PIPESTATUS[0]}
 }
 
@@ -69,15 +71,8 @@ function remote_sh_allow_changed_host_key() {
 }
 
 function set_up_remote_access() {
-  if [ -z "$SSH_AGENT_PID" ]; then
-    eval $(ssh-agent)
-    OWN_SSH_AGENT=1
-  else
-    OWN_SSH_AGENT=0
-  fi
   cp $FLAGS_private_key $TMP_PRIVATE_KEY
   chmod 0400 $TMP_PRIVATE_KEY
-  ssh-add $TMP_PRIVATE_KEY
 
   # Verify the client is reachable before continuing
   echo "Initiating first contact with remote host"

From bb4661dee724a277bbfbd093ca4563683f080d86 Mon Sep 17 00:00:00 2001
From: Chris Sosa <sosa@chromium.org>
Date: Thu, 13 Jan 2011 11:19:02 -0800
Subject: [PATCH 14/16] Put wait around remote_reboot core code itself rather
 than around the ping.

The last CL had a race condition where we could ping but not run the reboot_check code.
This CL moves to watch the reboot_check code rather than a ping.

Change-Id: I8bdda97850ac085be9cf5dfb0b80fb7e1c8dd212

BUG=chromium-os:10867
TEST=Ran it 100 times without error.

Review URL: http://codereview.chromium.org/6287001
---
 remote_access.sh | 69 ++++++++++++++++++++++++------------------------
 1 file changed, 34 insertions(+), 35 deletions(-)

diff --git a/remote_access.sh b/remote_access.sh
index a34ed5ffaa..f778fae345 100644
--- a/remote_access.sh
+++ b/remote_access.sh
@@ -36,37 +36,8 @@ function remote_sh() {
   return ${PIPESTATUS[0]}
 }
 
-# Checks to see if pid $1 is running.
-function is_pid_running() {
-  ps -p ${1} 2>&1 > /dev/null
-}
-
-# Wait function given an additional timeout argument.
-# $1 - pid to wait on.
-# $2 - timeout to wait for.
-function wait_with_timeout() {
-  local pid=$1
-  local timeout=$2
-  local -r TIMEOUT_INC=1
-  local current_timeout=0
-  while is_pid_running ${pid} && [ ${current_timeout} -lt ${timeout} ]; do
-    sleep ${TIMEOUT_INC}
-    current_timeout=$((current_timeout + TIMEOUT_INC))
-  done
-  is_pid_running ${pid}
-}
-
-# Robust ping that will monitor ssh and not hang even if ssh hangs.
-function ping_ssh() {
-  remote_sh "true" &
-  local pid=$!
-  wait_with_timeout ${pid} 5
-  ! kill -9 ${pid} 2> /dev/null
-}
-
 function remote_sh_allow_changed_host_key() {
   rm -f $TMP_KNOWN_HOSTS
-  ping_ssh
   remote_sh "$@"
 }
 
@@ -92,12 +63,29 @@ function learn_board() {
   info "Target reports board is ${FLAGS_board}"
 }
 
-function remote_reboot {
-  info "Rebooting."
-  remote_sh "touch /tmp/awaiting_reboot; reboot"
-  local output_file
-  output_file="${TMP}/output"
+# Checks to see if pid $1 is running.
+function is_pid_running() {
+  ps -p ${1} 2>&1 > /dev/null
+}
 
+# Wait function given an additional timeout argument.
+# $1 - pid to wait on.
+# $2 - timeout to wait for.
+function wait_with_timeout() {
+  local pid=$1
+  local timeout=$2
+  local -r TIMEOUT_INC=1
+  local current_timeout=0
+  while is_pid_running ${pid} && [ ${current_timeout} -lt ${timeout} ]; do
+    sleep ${TIMEOUT_INC}
+    current_timeout=$((current_timeout + TIMEOUT_INC))
+  done
+  ! is_pid_running ${pid}
+}
+
+# Checks to see if a machine has rebooted using the presence of a tmp file.
+function check_if_rebooted() {
+  local output_file="${TMP}/output"
   while true; do
     REMOTE_OUT=""
     # This may fail while the machine is down so generate output and a
@@ -108,12 +96,23 @@ function remote_reboot {
     if grep -q "0" "${output_file}"; then
       if grep -q "1" "${output_file}"; then
         info "Not yet rebooted"
+        sleep .5
       else
         info "Rebooted and responding"
         break
       fi
     fi
-    sleep .5
+  done
+}
+
+function remote_reboot() {
+  info "Rebooting."
+  remote_sh "touch /tmp/awaiting_reboot; reboot"
+  while true; do
+    check_if_rebooted &
+    local pid=$!
+    wait_with_timeout ${pid} 30 && break
+    ! kill -9 ${pid} 2> /dev/null
   done
 }
 

From 5c9571a92649788bac2b43c65e2c6c05fb54faab Mon Sep 17 00:00:00 2001
From: Chris Sosa <sosa@chromium.org>
Date: Thu, 13 Jan 2011 13:12:55 -0800
Subject: [PATCH 15/16] Revert "remote_access.sh: remove dependence on
 ssh-agent"

This reverts commit 78476aba5f30d213eb5ec83f3a3f6f00ed83750b.

BUG=
TEST=Reverted on bot and re-ran test suite

Review URL: http://codereview.chromium.org/6311002

Change-Id: Ib9405dac19b8b19d89e5d2346f5e1fd130a7280f
---
 remote_access.sh | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/remote_access.sh b/remote_access.sh
index f778fae345..45531f1552 100644
--- a/remote_access.sh
+++ b/remote_access.sh
@@ -15,24 +15,22 @@ DEFINE_integer ssh_port 22 \
 
 # Copies $1 to $2 on remote host
 function remote_cp_to() {
-  REMOTE_OUT=$(scp -P ${FLAGS_ssh_port} -o StrictHostKeyChecking=no \
-    -o UserKnownHostsFile=$TMP_KNOWN_HOSTS -i $TMP_PRIVATE_KEY $1 \
-    root@$FLAGS_remote:$2)
+  REMOTE_OUT=$(scp -P ${FLAGS_ssh_port} -o StrictHostKeyChecking=no -o \
+    UserKnownHostsFile=$TMP_KNOWN_HOSTS $1 root@$FLAGS_remote:$2)
   return ${PIPESTATUS[0]}
 }
 
 # Copies a list of remote files specified in file $1 to local location
 # $2.  Directory paths in $1 are collapsed into $2.
 function remote_rsync_from() {
-  rsync -e "ssh -p ${FLAGS_ssh_port} -o StrictHostKeyChecking=no \
-             -o UserKnownHostsFile=$TMP_KNOWN_HOSTS -i $TMP_PRIVATE_KEY" \
-    --no-R --files-from=$1 root@${FLAGS_remote}:/ $2
+  rsync -e "ssh -p ${FLAGS_ssh_port} -o StrictHostKeyChecking=no -o \
+            UserKnownHostsFile=$TMP_KNOWN_HOSTS" --no-R \
+    --files-from=$1 root@${FLAGS_remote}:/ $2
 }
 
 function remote_sh() {
-  REMOTE_OUT=$(ssh -p ${FLAGS_ssh_port} -o StrictHostKeyChecking=no \
-    -o UserKnownHostsFile=$TMP_KNOWN_HOSTS -i $TMP_PRIVATE_KEY \
-    root@$FLAGS_remote "$@")
+  REMOTE_OUT=$(ssh -p ${FLAGS_ssh_port} -o StrictHostKeyChecking=no -o \
+    UserKnownHostsFile=$TMP_KNOWN_HOSTS root@$FLAGS_remote "$@")
   return ${PIPESTATUS[0]}
 }
 
@@ -42,8 +40,15 @@ function remote_sh_allow_changed_host_key() {
 }
 
 function set_up_remote_access() {
+  if [ -z "$SSH_AGENT_PID" ]; then
+    eval $(ssh-agent)
+    OWN_SSH_AGENT=1
+  else
+    OWN_SSH_AGENT=0
+  fi
   cp $FLAGS_private_key $TMP_PRIVATE_KEY
   chmod 0400 $TMP_PRIVATE_KEY
+  ssh-add $TMP_PRIVATE_KEY
 
   # Verify the client is reachable before continuing
   echo "Initiating first contact with remote host"

From 4db7ac8f982fa7a0dbde3cc2da6423a4e404a6a5 Mon Sep 17 00:00:00 2001
From: Chris Sosa <sosa@chromium.org>
Date: Thu, 13 Jan 2011 14:11:18 -0800
Subject: [PATCH 16/16] Print out SVN revision number for TOT chrome pfq
 builds.

Change-Id: I046d49c7002586671dc7c6707afeeeb0a0360cd5

BUG=10934
TEST=Ran it with bin/cros_mark_chrome_as_stable tot. Verified output and that it still did the right thing

Review URL: http://codereview.chromium.org/6327002
---
 bin/cros_mark_chrome_as_stable.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/bin/cros_mark_chrome_as_stable.py b/bin/cros_mark_chrome_as_stable.py
index dc006a7b92..0b17218180 100755
--- a/bin/cros_mark_chrome_as_stable.py
+++ b/bin/cros_mark_chrome_as_stable.py
@@ -60,7 +60,9 @@ def  _GetTipOfTrunkSvnRevision():
   for line in svn_info.splitlines():
     match = revision_re.search(line)
     if match:
-      return match.group(1)
+      svn_revision = match.group(1)
+      Info('Using SVN Revision %s' % svn_revision)
+      return svn_revision
 
   raise Exception('Could not find revision information from %s' % svn_url)