diff --git a/bin/au_test_harness/au_test.py b/bin/au_test_harness/au_test.py index e8e88a7a79..225a7eb42d 100644 --- a/bin/au_test_harness/au_test.py +++ b/bin/au_test_harness/au_test.py @@ -5,6 +5,7 @@ """Module containing a test suite that is run to test auto updates.""" import os +import tempfile import time import unittest @@ -23,6 +24,8 @@ class AUTest(unittest.TestCase): be created to perform and validates updates on both virtual and real devices. See documentation for au_worker for more information. """ + test_results_root = None + @classmethod def ProcessOptions(cls, options, use_dummy_worker): """Processes options for the test suite and sets up the worker class. @@ -55,6 +58,15 @@ class AUTest(unittest.TestCase): elif not os.path.exists(cls.target_image_path): cros_lib.Die('%s does not exist' % cls.target_image_path) + # Initialize test root. + if not cls.test_results_root: + if options.test_results_root: + cls.test_results_root = options.test_results_root + else: + cls.test_results_root = tempfile.mkdtemp(prefix='au_test_harness') + + cros_lib.Info('Using %s as the test results root' % cls.test_results_root) + # Cache away options to instantiate workers later. cls.options = options @@ -98,7 +110,7 @@ class AUTest(unittest.TestCase): Sets instance specific variables and initializes worker. """ unittest.TestCase.setUp(self) - self.worker = self.worker_class(self.options) + self.worker = self.worker_class(self.options, AUTest.test_results_root) self.crosutils = os.path.join(os.path.dirname(__file__), '..', '..') self.download_folder = os.path.join(self.crosutils, 'latest_download') if not os.path.exists(self.download_folder): @@ -114,6 +126,7 @@ class AUTest(unittest.TestCase): This test checks that we can update by updating the stateful partition rather than wiping it. """ + self.worker.InitializeResultsDirectory() # Just make sure some tests pass on original image. Some old images # don't pass many tests. self.worker.PrepareBase(self.base_image_path) @@ -135,6 +148,7 @@ class AUTest(unittest.TestCase): This test checks that we can update successfully after wiping the stateful partition. """ + self.worker.InitializeResultsDirectory() # Just make sure some tests pass on original image. Some old images # don't pass many tests. self.worker.PrepareBase(self.base_image_path) @@ -182,6 +196,7 @@ class AUTest(unittest.TestCase): self.data_size += len(data) return data + self.worker.InitializeResultsDirectory() self.AttemptUpdateWithFilter(InterruptionFilter(), proxy_port=8082) def testDelayedUpdate(self): @@ -212,6 +227,7 @@ class AUTest(unittest.TestCase): self.data_size += len(data) return data + self.worker.InitializeResultsDirectory() self.AttemptUpdateWithFilter(DelayedFilter(), proxy_port=8083) def SimpleTest(self): @@ -220,8 +236,9 @@ class AUTest(unittest.TestCase): We explicitly don't use test prefix so that isn't run by default. Can be run using test_prefix option. """ + self.worker.InitializeResultsDirectory() self.worker.PrepareBase(self.base_image_path) - self.worker.PerformUpdate(self.target_image_path, self.base_image_path) + #self.worker.PerformUpdate(self.target_image_path, self.base_image_path) self.worker.VerifyImage(self) # --- DISABLED TESTS --- @@ -229,6 +246,7 @@ class AUTest(unittest.TestCase): # TODO(sosa): Get test to work with verbose. def NotestPartialUpdate(self): """Tests what happens if we attempt to update with a truncated payload.""" + self.worker.InitializeResultsDirectory() # Preload with the version we are trying to test. self.worker.PrepareBase(self.target_image_path) @@ -247,6 +265,7 @@ class AUTest(unittest.TestCase): # TODO(sosa): Get test to work with verbose. def NotestCorruptedUpdate(self): """Tests what happens if we attempt to update with a corrupted payload.""" + self.worker.InitializeResultsDirectory() # Preload with the version we are trying to test. self.worker.PrepareBase(self.target_image_path) diff --git a/bin/au_test_harness/au_worker.py b/bin/au_test_harness/au_worker.py index 258cdfd4e0..20363c1d32 100644 --- a/bin/au_test_harness/au_worker.py +++ b/bin/au_test_harness/au_worker.py @@ -9,6 +9,8 @@ and validating updates on a target. This should be subclassed to handle various types of target. Types of targets include VM's, real devices, etc. """ +import inspect +import threading import os import sys @@ -20,15 +22,16 @@ import update_exception class AUWorker(object): """Interface for a worker that updates and verifies images.""" - + # Mapping between cached payloads to directory locations. update_cache = None # --- INTERFACE --- - def __init__(self, options): + def __init__(self, options, test_results_root): """Processes options for the specific-type of worker.""" self.board = options.board self.private_key = options.private_key + self.test_results_root = test_results_root self.use_delta_updates = options.delta self.verbose = options.verbose self.vm_image_path = None @@ -225,6 +228,23 @@ class AUWorker(object): unittest.assertTrue(percent_passed >= percent_required_to_pass) return percent_passed + def InitializeResultsDirectory(self): + """Called by a test to initialize a results directory for this worker.""" + # Use the name of the test. + test_name = inspect.stack()[1][3] + self.results_directory = os.path.join(self.test_results_root, test_name) + self.results_count = 0 + + def GetNextResultsPath(self, label): + """Returns a new results path based for this label. + + Prefixes directory returned for worker with time called i.e. 1_label, + 2_label, etc. + """ + self.results_count += 1 + return os.path.join(self.results_directory, '%s_%s' % (self.results_count, + label)) + # --- PRIVATE HELPER FUNCTIONS --- def _ParseGenerateTestReportOutput(self, output): diff --git a/bin/au_test_harness/cros_au_test_harness.py b/bin/au_test_harness/cros_au_test_harness.py index 76d800a117..2742a64402 100755 --- a/bin/au_test_harness/cros_au_test_harness.py +++ b/bin/au_test_harness/cros_au_test_harness.py @@ -237,6 +237,9 @@ def main(): help='Remote address for real test.') parser.add_option('-t', '--target_image', help='path to the target image.') + parser.add_option('--test_results_root', default=None, + help='Root directory to store test results. Should ' + 'be defined relative to chroot root.') parser.add_option('--test_prefix', default='test', help='Only runs tests with specific prefix i.e. ' 'testFullUpdateWipeStateful.') diff --git a/bin/au_test_harness/dummy_au_worker.py b/bin/au_test_harness/dummy_au_worker.py index d16cd690f3..fa1bee042a 100644 --- a/bin/au_test_harness/dummy_au_worker.py +++ b/bin/au_test_harness/dummy_au_worker.py @@ -18,8 +18,8 @@ class DummyAUWorker(au_worker.AUWorker): # Class variable that stores the list of payloads that would be needed. delta_list = {} - def __init__(self, options): - au_worker.AUWorker.__init__(self, options) + def __init__(self, options, test_results_root): + au_worker.AUWorker.__init__(self, options, test_results_root) self.au_type = options.type def PrepareBase(self, image_path): diff --git a/bin/au_test_harness/real_au_worker.py b/bin/au_test_harness/real_au_worker.py index 492d2e24c9..43b8c2e62c 100644 --- a/bin/au_test_harness/real_au_worker.py +++ b/bin/au_test_harness/real_au_worker.py @@ -13,9 +13,9 @@ import au_worker class RealAUWorker(au_worker.AUWorker): """Test harness for updating real images.""" - def __init__(self, options): + def __init__(self, options, test_results_root): """Processes non-vm-specific options.""" - au_worker.AUWorker.__init__(self, options) + au_worker.AUWorker.__init__(self, options, test_results_root) self.remote = options.remote if not self.remote: cros_lib.Die('We require a remote address for tests.') @@ -51,9 +51,11 @@ class RealAUWorker(au_worker.AUWorker): def VerifyImage(self, unittest, percent_required_to_pass=100): """Verifies an image using run_remote_tests.sh with verification suite.""" + test_directory = self.GetNextResultsPath('verify') output = cros_lib.RunCommand( ['%s/run_remote_tests.sh' % self.crosutils, '--remote=%s' % self.remote, + '--results_dir_root=%s' % test_directory, self.verify_suite, ], error_ok=True, enter_chroot=False, redirect_stdout=True) return self.AssertEnoughTestsPassed(unittest, output, diff --git a/bin/au_test_harness/vm_au_worker.py b/bin/au_test_harness/vm_au_worker.py index 3d11cdc436..475db58cfb 100644 --- a/bin/au_test_harness/vm_au_worker.py +++ b/bin/au_test_harness/vm_au_worker.py @@ -20,9 +20,9 @@ class VMAUWorker(au_worker.AUWorker): _vm_lock = threading.Lock() _next_port = 9222 - def __init__(self, options): + def __init__(self, options, test_results_root): """Processes vm-specific options.""" - au_worker.AUWorker.__init__(self, options) + au_worker.AUWorker.__init__(self, options, test_results_root) self.graphics_flag = '' if options.no_graphics: self.graphics_flag = '--no_graphics' if not self.board: cros_lib.Die('Need board to convert base image to vm.') @@ -95,6 +95,7 @@ class VMAUWorker(au_worker.AUWorker): def VerifyImage(self, unittest, percent_required_to_pass=100): """Runs vm smoke suite to verify image.""" + test_directory = self.GetNextResultsPath('verify') # image_to_live already verifies lsb-release matching. This is just # for additional steps. commandWithArgs = ['%s/cros_run_vm_test' % self.crosutilsbin, @@ -103,6 +104,7 @@ class VMAUWorker(au_worker.AUWorker): '--persist', '--kvm_pid=%s' % self._kvm_pid_file, '--ssh_port=%s' % self._ssh_port, + '--results_dir_root=%s' % test_directory, self.verify_suite, ] if self.graphics_flag: commandWithArgs.append(self.graphics_flag) diff --git a/bin/ctest.py b/bin/ctest.py index 95f9377e42..a6c1d916f6 100755 --- a/bin/ctest.py +++ b/bin/ctest.py @@ -226,7 +226,7 @@ def GrabZipAndExtractImage(zip_url, download_folder, image_name) : def RunAUTestHarness(board, channel, latest_url_base, zip_server_base, - no_graphics, type, remote, clean): + no_graphics, type, remote, clean, test_results_root): """Runs the auto update test harness. The auto update test harness encapsulates testing the auto-update mechanism @@ -243,6 +243,7 @@ def RunAUTestHarness(board, channel, latest_url_base, zip_server_base, type: which test harness to run. Possible values: real, vm. remote: ip address for real test harness run. clean: Clean the state of test harness before running. + test_results_root: Root directory to store au_test_harness results. """ crosutils_root = os.path.join(os.path.dirname(__file__), '..') download_folder = os.path.abspath('latest_download') @@ -270,6 +271,7 @@ def RunAUTestHarness(board, channel, latest_url_base, zip_server_base, '--public_key=%s' % os.path.join(update_engine_path, 'unittest_key.pub.pem'), ] + if test_results_root: cmd.append('--test_results_root=%s' % test_results_root) if no_graphics: cmd.append('--no_graphics') if clean: cmd.append('--clean') @@ -290,6 +292,9 @@ def main(): help='Base url for hosted images.') parser.add_option('--no_graphics', action='store_true', default=False, help='Disable graphics for the vm test.') + parser.add_option('--test_results_root', default=None, + help='Root directory to store test results. Should ' + 'be defined relative to chroot root.') parser.add_option('--type', default='vm', help='type of test to run: [vm, real]. Default: vm.') parser.add_option('--remote', default='0.0.0.0',