From e7205e84be92f5b0fb91c259e1526b7e371c8a31 Mon Sep 17 00:00:00 2001 From: "J. Richard Barnette" Date: Wed, 5 Jan 2011 18:38:49 -0800 Subject: [PATCH] Revert "Major cleanup of cros_au_test_harness code." This reverts commit 2b2d7a1426f485bc2c6fb02399c52618db6dd0f2. The internal x86 buildbots have been reporting failures from cros_au_test_harness with every build, starting with the build that included the change being reverted. BUG=chromium-os:8901 TEST=None Review URL: http://codereview.chromium.org/5962019 Change-Id: Iac8c0a13d659f927b3ff61a9a894c10191432e2d --- bin/cros_au_test_harness.py | 339 +++++++++++++++++------------------- 1 file changed, 164 insertions(+), 175 deletions(-) diff --git a/bin/cros_au_test_harness.py b/bin/cros_au_test_harness.py index 5a0182c481..5d5b221760 100755 --- a/bin/cros_au_test_harness.py +++ b/bin/cros_au_test_harness.py @@ -1,6 +1,6 @@ #!/usr/bin/python -# Copyright (c) 2011 The Chromium OS Authors. All rights reserved. +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -23,16 +23,29 @@ from cros_build_lib import Warning import cros_test_proxy +# VM Constants. +_FULL_VDISK_SIZE = 6072 +_FULL_STATEFULFS_SIZE = 3074 +_KVM_PID_FILE = '/tmp/harness_pid' +_VERIFY_SUITE = 'suite_Smoke' + +# Globals to communicate options to unit tests. +global base_image_path +global board +global remote +global target_image_path +global vm_graphics_flag class UpdateException(Exception): - """Exception thrown when _UpdateImage or _UpdateUsingPayload fail""" + """Exception thrown when UpdateImage or UpdateUsingPayload fail""" def __init__(self, code, stdout): self.code = code self.stdout = stdout - class AUTest(object): """Abstract interface that defines an Auto Update test.""" + source_image = '' + use_delta_updates = False verbose = False def setUp(self): @@ -44,8 +57,6 @@ class AUTest(object): if not os.path.exists(self.download_folder): os.makedirs(self.download_folder) - # -------- Helper functions --------- - def GetStatefulChangeFlag(self, stateful_change): """Returns the flag to pass to image_to_vm for the stateful change.""" stateful_change_flag = '' @@ -54,7 +65,7 @@ class AUTest(object): return stateful_change_flag - def _ParseGenerateTestReportOutput(self, output): + def ParseGenerateTestReportOutput(self, output): """Returns the percentage of tests that passed based on output.""" percent_passed = 0 lines = output.split('\n') @@ -68,60 +79,38 @@ class AUTest(object): return int(percent_passed) - def AssertEnoughTestsPassed(self, unittest, output, percent_required_to_pass): - """Helper function that asserts a sufficient number of tests passed. + # TODO(sosa) - Remove try and convert function to DeltaUpdateImage(). + def TryDeltaAndFallbackToFull(self, src_image, image, stateful_change='old'): + """Tries the delta update first if set and falls back to full update.""" + if self.use_delta_updates: + try: + self.source_image = src_image + self._UpdateImageReportError(image, stateful_change) + except: + Warning('Delta update failed, disabling delta updates and retrying.') + self.use_delta_updates = False + self.source_image = '' + self._UpdateImageReportError(image, stateful_change) + else: + self._UpdateImageReportError(image, stateful_change) - Args: - unittest: Handle to the unittest. - output: stdout from a test run. - percent_required_to_pass: percentage required to pass. This should be - fall between 0-100. - Returns: - percent that passed. - """ - Info('Output from VerifyImage():') - print >> sys.stderr, output - sys.stderr.flush() - percent_passed = self._ParseGenerateTestReportOutput(output) - Info('Percent passed: %d vs. Percent required: %d' % ( - percent_passed, percent_required_to_pass)) - unittest.assertTrue(percent_passed >= percent_required_to_pass) - return percent_passed + def _UpdateImageReportError(self, image_path, stateful_change='old', + proxy_port=None): + """Calls UpdateImage and reports any error to the console. - def PerformUpdate(self, image_path, src_image_path='', stateful_change='old', - proxy_port=None): - """Performs an update using _UpdateImage and reports any error. - - Subclasses should not override this method but override _UpdateImage - instead. - - Args: - image_path: Path to the image to update with. This image must be a test - image. - src_image_path: Optional. If set, perform a delta update using the - image specified by the path as the source image. - stateful_change: How to modify the stateful partition. Values are: - 'old': Don't modify stateful partition. Just update normally. - 'clean': Uses clobber-state to wipe the stateful partition with the - exception of code needed for ssh. - proxy_port: Port to have the client connect to. For use with - CrosTestProxy. - Raises an UpdateException if _UpdateImage returns an error. + Still throws the exception. """ try: - if not self.use_delta_updates: - src_image_path = '' - - self._UpdateImage(image_path, src_image_path, stateful_change, proxy_port) + self.UpdateImage(image_path, stateful_change, proxy_port) except UpdateException as err: # If the update fails, print it out Warning(err.stdout) raise - def AttemptUpdateWithPayloadExpectedFailure(self, payload, expected_msg): + def _AttemptUpdateWithPayloadExpectedFailure(self, payload, expected_msg): """Attempt a payload update, expect it to fail with expected log""" try: - self._UpdateUsingPayload(payload) + self.UpdateUsingPayload(payload) except UpdateException as err: # Will raise ValueError if expected is not found. if re.search(re.escape(expected_msg), err.stdout, re.MULTILINE): @@ -131,10 +120,10 @@ class AUTest(object): Warning(err.stdout) self.fail('We managed to update when failure was expected') - def AttemptUpdateWithFilter(self, filter): + def _AttemptUpdateWithFilter(self, filter): """Update through a proxy, with a specified filter, and expect success.""" - self.PrepareBase(self.target_image_path) + self.PrepareBase(target_image_path) # The devserver runs at port 8080 by default. We assume that here, and # start our proxy at 8081. We then tell our update tools to have the @@ -148,58 +137,34 @@ class AUTest(object): # This update is expected to fail... try: - self.PerformUpdate(self.target_image_path, proxy_port=proxy_port) + self._UpdateImageReportError(target_image_path, proxy_port=proxy_port) finally: proxy.shutdown() - # -------- Functions that subclasses should override --------- - - @classmethod - def ProcessOptions(cls, parser, options): - """Processes options. - - Static method that should be called from main. Subclasses should also - call their parent method if they override it. - """ - cls.verbose = options.verbose - cls.base_image_path = options.base_image - cls.target_image_path = options.target_image - cls.use_delta_updates = options.delta - if options.quick_test: - cls.verify_suite = 'build_RootFilesystemSize' - else: - cls.verify_suite = 'suite_Smoke' - - # Sanity checks. - if not cls.base_image_path: - parser.error('Need path to base image for vm.') - elif not os.path.exists(cls.base_image_path): - Die('%s does not exist' % cls.base_image_path) - - if not cls.target_image_path: - parser.error('Need path to target image to update with.') - elif not os.path.exists(cls.target_image_path): - Die('%s does not exist' % cls.target_image_path) - def PrepareBase(self, image_path): """Prepares target with base_image_path.""" pass - def _UpdateImage(self, image_path, src_image_path='', stateful_change='old', - proxy_port=None): - """Implementation of an actual update. + def UpdateImage(self, image_path, stateful_change='old', proxy_port=None): + """Updates target with the image given by the image_path. - See PerformUpdate for description of args. Subclasses must override this - method with the correct update procedure for the class. + Args: + image_path: Path to the image to update with. This image must be a test + image. + stateful_change: How to modify the stateful partition. Values are: + 'old': Don't modify stateful partition. Just update normally. + 'clean': Uses clobber-state to wipe the stateful partition with the + exception of code needed for ssh. + proxy_port: Port to have the client connect to. For use with + CrosTestProxy. """ pass - def _UpdateUsingPayload(self, update_path, stateful_change='old', + def UpdateUsingPayload(self, + update_path, + stateful_change='old', proxy_port=None): - """Updates target with the pre-generated update stored in update_path. - - Subclasses must override this method with the correct update procedure for - the class. + """Updates target with the pre-generated update stored in update_path Args: update_path: Path to the image to update with. This directory should @@ -212,8 +177,7 @@ class AUTest(object): def VerifyImage(self, percent_required_to_pass): """Verifies the image with tests. - Verifies that the test images passes the percent required. Subclasses must - override this method with the correct update procedure for the class. + Verifies that the test images passes the percent required. Args: percent_required_to_pass: percentage required to pass. This should be @@ -224,7 +188,29 @@ class AUTest(object): """ pass - # -------- Tests --------- + def CommonVerifyImage(self, unittest, output, percent_required_to_pass): + """Helper function for VerifyImage that returns percent of tests passed. + + Takes output from a test suite, verifies the number of tests passed is + sufficient and outputs info. + + Args: + unittest: Handle to the unittest. + output: stdout from a test run. + percent_required_to_pass: percentage required to pass. This should be + fall between 0-100. + Returns: + percent that passed. + """ + Info('Output from VerifyImage():') + print >> sys.stderr, output + sys.stderr.flush() + percent_passed = self.ParseGenerateTestReportOutput(output) + Info('Percent passed: %d vs. Percent required: %d' % ( + percent_passed, percent_required_to_pass)) + unittest.assertTrue(percent_passed >= + percent_required_to_pass) + return percent_passed def testFullUpdateKeepStateful(self): """Tests if we can update normally. @@ -234,19 +220,19 @@ class AUTest(object): """ # Just make sure some tests pass on original image. Some old images # don't pass many tests. - self.PrepareBase(self.base_image_path) + self.PrepareBase(base_image_path) # TODO(sosa): move to 100% once we start testing using the autotest paired # with the dev channel. percent_passed = self.VerifyImage(10) # Update to - all tests should pass on new image. Info('Updating from base image on vm to target image.') - self.PerformUpdate(self.base_image_path, self.target_image_path) + self.TryDeltaAndFallbackToFull(base_image_path, target_image_path) self.VerifyImage(100) # Update from - same percentage should pass that originally passed. Info('Updating from updated image on vm back to base image.') - self.PerformUpdate(self.target_image_path, self.base_image_path) + self.TryDeltaAndFallbackToFull(target_image_path, base_image_path) self.VerifyImage(percent_passed) def testFullUpdateWipeStateful(self): @@ -257,25 +243,25 @@ class AUTest(object): """ # Just make sure some tests pass on original image. Some old images # don't pass many tests. - self.PrepareBase(self.base_image_path) + self.PrepareBase(base_image_path) # TODO(sosa): move to 100% once we start testing using the autotest paired # with the dev channel. percent_passed = self.VerifyImage(10) # Update to - all tests should pass on new image. Info('Updating from base image on vm to target image and wiping stateful.') - self.PerformUpdate(self.base_image_path, self.target_image_path, 'clean') + self.TryDeltaAndFallbackToFull(base_image_path, target_image_path, 'clean') self.VerifyImage(100) # Update from - same percentage should pass that originally passed. Info('Updating from updated image back to base image and wiping stateful.') - self.PerformUpdate(self.target_image_path, self.base_image_path, 'clean') + self.TryDeltaAndFallbackToFull(target_image_path, base_image_path, 'clean') self.VerifyImage(percent_passed) def testPartialUpdate(self): """Tests what happens if we attempt to update with a truncated payload.""" # Preload with the version we are trying to test. - self.PrepareBase(self.target_image_path) + self.PrepareBase(target_image_path) # Image can be updated at: # ~chrome-eng/chromeos/localmirror/autest-images @@ -287,12 +273,12 @@ class AUTest(object): urllib.urlretrieve(url, payload) expected_msg = 'download_hash_data == update_check_response_hash failed' - self.AttemptUpdateWithPayloadExpectedFailure(payload, expected_msg) + self._AttemptUpdateWithPayloadExpectedFailure(payload, expected_msg) def testCorruptedUpdate(self): """Tests what happens if we attempt to update with a corrupted payload.""" # Preload with the version we are trying to test. - self.PrepareBase(self.target_image_path) + self.PrepareBase(target_image_path) # Image can be updated at: # ~chrome-eng/chromeos/localmirror/autest-images @@ -305,7 +291,7 @@ class AUTest(object): # This update is expected to fail... expected_msg = 'zlib inflate() error:-3' - self.AttemptUpdateWithPayloadExpectedFailure(payload, expected_msg) + self._AttemptUpdateWithPayloadExpectedFailure(payload, expected_msg) def testInterruptedUpdate(self): """Tests what happens if we interrupt payload delivery 3 times.""" @@ -339,7 +325,7 @@ class AUTest(object): self.data_size += len(data) return data - self.AttemptUpdateWithFilter(InterruptionFilter()) + self._AttemptUpdateWithFilter(InterruptionFilter()) def testDelayedUpdate(self): """Tests what happens if some data is delayed during update delivery""" @@ -369,7 +355,7 @@ class AUTest(object): self.data_size += len(data) return data - self.AttemptUpdateWithFilter(DelayedFilter()) + self._AttemptUpdateWithFilter(DelayedFilter()) def SimpleTest(self): """A simple update that updates the target image to itself. @@ -377,8 +363,8 @@ class AUTest(object): We explicitly don't use test prefix so that isn't run by default. Can be run using test_prefix option. """ - self.PrepareBase(self.target_image_path) - self._UpdateImage(self.target_image_path) + self.PrepareBase(target_image_path) + self.UpdateImage(target_image_path) self.VerifyImage(100) @@ -388,29 +374,19 @@ class RealAUTest(unittest.TestCase, AUTest): def setUp(self): AUTest.setUp(self) - @classmethod - def ProcessOptions(cls, parser, options): - """Processes non-vm-specific options.""" - AUTest.ProcessOptions(parser, options) - cls.remote = options.remote - - if not cls.remote: - parser.error('We require a remote address for real tests.') - def PrepareBase(self, image_path): """Auto-update to base image to prepare for test.""" - self.PerformUpdate(image_path) + self._UpdateImageReportError(image_path) - def _UpdateImage(self, image_path, src_image_path='', stateful_change='old', - proxy_port=None): + def UpdateImage(self, image_path, stateful_change='old', proxy_port=None): """Updates a remote image using image_to_live.sh.""" stateful_change_flag = self.GetStatefulChangeFlag(stateful_change) cmd = ['%s/image_to_live.sh' % self.crosutils, '--image=%s' % image_path, - '--remote=%s' % self.remote, + '--remote=%s' % remote, stateful_change_flag, '--verify', - '--src_image=%s' % src_image_path + '--src_image=%s' % self.source_image ] if proxy_port: @@ -426,13 +402,15 @@ class RealAUTest(unittest.TestCase, AUTest): if code != 0: raise UpdateException(code, stdout) - def _UpdateUsingPayload(self, update_path, stateful_change='old', + def UpdateUsingPayload(self, + update_path, + stateful_change='old', proxy_port=None): """Updates a remote image using image_to_live.sh.""" stateful_change_flag = self.GetStatefulChangeFlag(stateful_change) cmd = ['%s/image_to_live.sh' % self.crosutils, '--payload=%s' % update_path, - '--remote=%s' % self.remote, + '--remote=%s' % remote, stateful_change_flag, '--verify', ] @@ -454,21 +432,16 @@ class RealAUTest(unittest.TestCase, AUTest): """Verifies an image using run_remote_tests.sh with verification suite.""" output = RunCommand([ '%s/run_remote_tests.sh' % self.crosutils, - '--remote=%s' % self.remote, - self.verify_suite, + '--remote=%s' % remote, + _VERIFY_SUITE, ], error_ok=True, enter_chroot=False, redirect_stdout=True) - return self.AssertEnoughTestsPassed(self, output, percent_required_to_pass) + return self.CommonVerifyImage(self, output, percent_required_to_pass) class VirtualAUTest(unittest.TestCase, AUTest): """Test harness for updating virtual machines.""" vm_image_path = None - # VM Constants. - _FULL_VDISK_SIZE = 6072 - _FULL_STATEFULFS_SIZE = 3074 - _KVM_PID_FILE = '/tmp/harness_pid' - def _KillExistingVM(self, pid_file): if os.path.exists(pid_file): Warning('Existing %s found. Deleting and killing process' % @@ -481,20 +454,7 @@ class VirtualAUTest(unittest.TestCase, AUTest): def setUp(self): """Unit test overriden method. Is called before every test.""" AUTest.setUp(self) - self._KillExistingVM(self._KVM_PID_FILE) - - @classmethod - def ProcessOptions(cls, parser, options): - """Processes vm-specific options.""" - AUTest.ProcessOptions(parser, options) - cls.board = options.board - - # Communicate flags to tests. - cls.graphics_flag = '' - if options.no_graphics: cls.graphics_flag = '--no_graphics' - - if not cls.board: - parser.error('Need board to convert base image to vm.') + self._KillExistingVM(_KVM_PID_FILE) def PrepareBase(self, image_path): """Creates an update-able VM based on base image.""" @@ -509,32 +469,33 @@ class VirtualAUTest(unittest.TestCase, AUTest): '--full', '--from=%s' % ReinterpretPathForChroot( os.path.dirname(image_path)), - '--vdisk_size=%s' % self._FULL_VDISK_SIZE, - '--statefulfs_size=%s' % self._FULL_STATEFULFS_SIZE, - '--board=%s' % self.board, + '--vdisk_size=%s' % _FULL_VDISK_SIZE, + '--statefulfs_size=%s' % _FULL_STATEFULFS_SIZE, + '--board=%s' % board, '--test_image'], enter_chroot=True) else: Info('Using existing VM image %s' % self.vm_image_path) + Info('Testing for %s' % self.vm_image_path) + self.assertTrue(os.path.exists(self.vm_image_path)) - def _UpdateImage(self, image_path, src_image_path='', stateful_change='old', - proxy_port=None): + def UpdateImage(self, image_path, stateful_change='old', proxy_port=None): """Updates VM image with image_path.""" stateful_change_flag = self.GetStatefulChangeFlag(stateful_change) - if src_image_path == self.base_image_path: - src_image_path = self.vm_image_path + if self.source_image == base_image_path: + self.source_image = self.vm_image_path cmd = ['%s/cros_run_vm_update' % self.crosutilsbin, '--update_image_path=%s' % image_path, '--vm_image_path=%s' % self.vm_image_path, '--snapshot', - self.graphics_flag, + vm_graphics_flag, '--persist', - '--kvm_pid=%s' % self._KVM_PID_FILE, + '--kvm_pid=%s' % _KVM_PID_FILE, stateful_change_flag, - '--src_image=%s' % src_image_path, + '--src_image=%s' % self.source_image, ] if proxy_port: @@ -550,18 +511,24 @@ class VirtualAUTest(unittest.TestCase, AUTest): if code != 0: raise UpdateException(code, stdout) - def _UpdateUsingPayload(self, update_path, stateful_change='old', + def UpdateUsingPayload(self, + update_path, + stateful_change='old', proxy_port=None): """Updates a remote image using image_to_live.sh.""" stateful_change_flag = self.GetStatefulChangeFlag(stateful_change) + if self.source_image == base_image_path: + self.source_image = self.vm_image_path + cmd = ['%s/cros_run_vm_update' % self.crosutilsbin, '--payload=%s' % update_path, '--vm_image_path=%s' % self.vm_image_path, '--snapshot', - self.graphics_flag, + vm_graphics_flag, '--persist', - '--kvm_pid=%s' % self._KVM_PID_FILE, + '--kvm_pid=%s' % _KVM_PID_FILE, stateful_change_flag, + '--src_image=%s' % self.source_image, ] if proxy_port: @@ -586,19 +553,19 @@ class VirtualAUTest(unittest.TestCase, AUTest): '--image_path=%s' % self.vm_image_path, '--snapshot', '--persist', - '--kvm_pid=%s' % self._KVM_PID_FILE, - self.verify_suite, + '--kvm_pid=%s' % _KVM_PID_FILE, + _VERIFY_SUITE, ] - if self.graphics_flag: - commandWithArgs.append(self.graphics_flag) + if vm_graphics_flag: + commandWithArgs.append(vm_graphics_flag) output = RunCommand(commandWithArgs, error_ok=True, enter_chroot=False, redirect_stdout=True) - return self.AssertEnoughTestsPassed(self, output, percent_required_to_pass) + return self.CommonVerifyImage(self, output, percent_required_to_pass) -def main(): +if __name__ == '__main__': parser = optparse.OptionParser() parser.add_option('-b', '--base_image', help='path to the base image.') @@ -623,25 +590,47 @@ def main(): parser.add_option('--verbose', default=False, action='store_true', help='Print out rather than capture output as much as ' 'possible.') - (options, leftover_args) = parser.parse_args() + # Set the usage to include flags. + parser.set_usage(parser.format_help()) + # Parse existing sys.argv so we can pass rest to unittest.main. + (options, sys.argv) = parser.parse_args(sys.argv) - if leftover_args: - parser.error('Found extra options we do not support: %s' % leftover_args) + AUTest.verbose = options.verbose + base_image_path = options.base_image + target_image_path = options.target_image + board = options.board + + if not base_image_path: + parser.error('Need path to base image for vm.') + elif not os.path.exists(base_image_path): + Die('%s does not exist' % base_image_path) + + if not target_image_path: + parser.error('Need path to target image to update with.') + elif not os.path.exists(target_image_path): + Die('%s does not exist' % target_image_path) + + if not board: + parser.error('Need board to convert base image to vm.') + + # Communicate flags to tests. + vm_graphics_flag = '' + if options.no_graphics: vm_graphics_flag = '--no_graphics' + if options.quick_test: _VERIFY_SUITE = 'build_RootFilesystemSize' + AUTest.use_delta_updates = options.delta + + # Only run the test harness we care about. + test_loader = unittest.TestLoader() + test_loader.testMethodPrefix = options.test_prefix if options.type == 'vm': test_class = VirtualAUTest elif options.type == 'real': test_class = RealAUTest else: parser.error('Could not parse harness type %s.' % options.type) - test_class.ProcessOptions(parser, options) + remote = options.remote - test_loader = unittest.TestLoader() - test_loader.testMethodPrefix = options.test_prefix test_suite = test_loader.loadTestsFromTestCase(test_class) test_result = unittest.TextTestRunner(verbosity=2).run(test_suite) if not test_result.wasSuccessful(): Die('Test harness was not successful') - - -if __name__ == '__main__': - main()