Add ability to pass a base test root and create results dirs relative.

Sets up dir paths as follows:

test_root/test_name i.e. SimpleUpdate/<num>_stage

For example:

test_root/testNormalUpdateKeepStateful/2_verify/...test_results...
corresponds to the second invocation of verify image.  The test
results for this stage are stored here.

This is a predecessor for storing arbitrary logs e.g. update into
this test results root.

Change-Id: I7183420b5dcb0d6971aa508a338c048c3557e359

BUG=chromium-os:12211
TEST=With simple a full vm test suite w/ w/out explicit test root
set.

Review URL: http://codereview.chromium.org/6614029

bin/au_test_harness/au_test.py

@@ -5,6 +5,7 @@
 """Module containing a test suite that is run to test auto updates."""
 
 import os
+import tempfile
 import time
 import unittest
 
@@ -23,6 +24,8 @@ class AUTest(unittest.TestCase):
   be created to perform and validates updates on both virtual and real devices.
   See documentation for au_worker for more information.
   """
+  test_results_root = None
+
   @classmethod
   def ProcessOptions(cls, options, use_dummy_worker):
     """Processes options for the test suite and sets up the worker class.
@@ -55,6 +58,15 @@ class AUTest(unittest.TestCase):
     elif not os.path.exists(cls.target_image_path):
       cros_lib.Die('%s does not exist' % cls.target_image_path)
 
+    # Initialize test root.
+    if not cls.test_results_root:
+      if options.test_results_root:
+        cls.test_results_root = options.test_results_root
+      else:
+        cls.test_results_root = tempfile.mkdtemp(prefix='au_test_harness')
+
+      cros_lib.Info('Using %s as the test results root' % cls.test_results_root)
+
     # Cache away options to instantiate workers later.
     cls.options = options
 
@@ -98,7 +110,7 @@ class AUTest(unittest.TestCase):
     Sets instance specific variables and initializes worker.
     """
     unittest.TestCase.setUp(self)
-    self.worker = self.worker_class(self.options)
+    self.worker = self.worker_class(self.options, AUTest.test_results_root)
     self.crosutils = os.path.join(os.path.dirname(__file__), '..', '..')
     self.download_folder = os.path.join(self.crosutils, 'latest_download')
     if not os.path.exists(self.download_folder):
@@ -114,6 +126,7 @@ class AUTest(unittest.TestCase):
     This test checks that we can update by updating the stateful partition
     rather than wiping it.
     """
+    self.worker.InitializeResultsDirectory()
     # Just make sure some tests pass on original image.  Some old images
     # don't pass many tests.
     self.worker.PrepareBase(self.base_image_path)
@@ -135,6 +148,7 @@ class AUTest(unittest.TestCase):
     This test checks that we can update successfully after wiping the
     stateful partition.
     """
+    self.worker.InitializeResultsDirectory()
     # Just make sure some tests pass on original image.  Some old images
     # don't pass many tests.
     self.worker.PrepareBase(self.base_image_path)
@@ -182,6 +196,7 @@ class AUTest(unittest.TestCase):
         self.data_size += len(data)
         return data
 
+    self.worker.InitializeResultsDirectory()
     self.AttemptUpdateWithFilter(InterruptionFilter(), proxy_port=8082)
 
   def testDelayedUpdate(self):
@@ -212,6 +227,7 @@ class AUTest(unittest.TestCase):
         self.data_size += len(data)
         return data
 
+    self.worker.InitializeResultsDirectory()
     self.AttemptUpdateWithFilter(DelayedFilter(), proxy_port=8083)
 
   def SimpleTest(self):
@@ -220,8 +236,9 @@ class AUTest(unittest.TestCase):
     We explicitly don't use test prefix so that isn't run by default.  Can be
     run using test_prefix option.
     """
+    self.worker.InitializeResultsDirectory()
     self.worker.PrepareBase(self.base_image_path)
-    self.worker.PerformUpdate(self.target_image_path, self.base_image_path)
+    #self.worker.PerformUpdate(self.target_image_path, self.base_image_path)
     self.worker.VerifyImage(self)
 
   # --- DISABLED TESTS ---
@@ -229,6 +246,7 @@ class AUTest(unittest.TestCase):
   # TODO(sosa): Get test to work with verbose.
   def NotestPartialUpdate(self):
     """Tests what happens if we attempt to update with a truncated payload."""
+    self.worker.InitializeResultsDirectory()
     # Preload with the version we are trying to test.
     self.worker.PrepareBase(self.target_image_path)
 
@@ -247,6 +265,7 @@ class AUTest(unittest.TestCase):
   # TODO(sosa): Get test to work with verbose.
   def NotestCorruptedUpdate(self):
     """Tests what happens if we attempt to update with a corrupted payload."""
+    self.worker.InitializeResultsDirectory()
     # Preload with the version we are trying to test.
     self.worker.PrepareBase(self.target_image_path)
 

bin/au_test_harness/au_worker.py

@@ -9,6 +9,8 @@ and validating updates on a target.  This should be subclassed to handle
 various types of target.  Types of targets include VM's, real devices, etc.
 """
 
+import inspect
+import threading
 import os
 import sys
 
@@ -20,15 +22,16 @@ import update_exception
 
 class AUWorker(object):
   """Interface for a worker that updates and verifies images."""
-
+  # Mapping between cached payloads to directory locations.
   update_cache = None
 
   # --- INTERFACE ---
 
-  def __init__(self, options):
+  def __init__(self, options, test_results_root):
     """Processes options for the specific-type of worker."""
     self.board = options.board
     self.private_key = options.private_key
+    self.test_results_root = test_results_root
     self.use_delta_updates = options.delta
     self.verbose = options.verbose
     self.vm_image_path = None
@@ -225,6 +228,23 @@ class AUWorker(object):
     unittest.assertTrue(percent_passed >= percent_required_to_pass)
     return percent_passed
 
+  def InitializeResultsDirectory(self):
+    """Called by a test to initialize a results directory for this worker."""
+    # Use the name of the test.
+    test_name = inspect.stack()[1][3]
+    self.results_directory = os.path.join(self.test_results_root, test_name)
+    self.results_count = 0
+
+  def GetNextResultsPath(self, label):
+    """Returns a new results path based for this label.
+
+    Prefixes directory returned for worker with time called i.e. 1_label,
+    2_label, etc.
+    """
+    self.results_count += 1
+    return os.path.join(self.results_directory, '%s_%s' % (self.results_count,
+                                                           label))
+
   # --- PRIVATE HELPER FUNCTIONS ---
 
   def _ParseGenerateTestReportOutput(self, output):

bin/au_test_harness/cros_au_test_harness.py

@@ -237,6 +237,9 @@ def main():
                     help='Remote address for real test.')
   parser.add_option('-t', '--target_image',
                     help='path to the target image.')
+  parser.add_option('--test_results_root', default=None,
+                    help='Root directory to store test results.  Should '
+                         'be defined relative to chroot root.')
   parser.add_option('--test_prefix', default='test',
                     help='Only runs tests with specific prefix i.e. '
                          'testFullUpdateWipeStateful.')

bin/au_test_harness/dummy_au_worker.py

@@ -18,8 +18,8 @@ class DummyAUWorker(au_worker.AUWorker):
   # Class variable that stores the list of payloads that would be needed.
   delta_list = {}
 
-  def __init__(self, options):
-    au_worker.AUWorker.__init__(self, options)
+  def __init__(self, options, test_results_root):
+    au_worker.AUWorker.__init__(self, options, test_results_root)
     self.au_type = options.type
 
   def PrepareBase(self, image_path):

bin/au_test_harness/real_au_worker.py

@@ -13,9 +13,9 @@ import au_worker
 class RealAUWorker(au_worker.AUWorker):
   """Test harness for updating real images."""
 
-  def __init__(self, options):
+  def __init__(self, options, test_results_root):
     """Processes non-vm-specific options."""
-    au_worker.AUWorker.__init__(self, options)
+    au_worker.AUWorker.__init__(self, options, test_results_root)
     self.remote = options.remote
     if not self.remote: cros_lib.Die('We require a remote address for tests.')
 
@@ -51,9 +51,11 @@ class RealAUWorker(au_worker.AUWorker):
 
   def VerifyImage(self, unittest, percent_required_to_pass=100):
     """Verifies an image using run_remote_tests.sh with verification suite."""
+    test_directory = self.GetNextResultsPath('verify')
     output = cros_lib.RunCommand(
         ['%s/run_remote_tests.sh' % self.crosutils,
          '--remote=%s' % self.remote,
+         '--results_dir_root=%s' % test_directory,
          self.verify_suite,
         ], error_ok=True, enter_chroot=False, redirect_stdout=True)
     return self.AssertEnoughTestsPassed(unittest, output,

bin/au_test_harness/vm_au_worker.py

@@ -20,9 +20,9 @@ class VMAUWorker(au_worker.AUWorker):
   _vm_lock = threading.Lock()
   _next_port = 9222
 
-  def __init__(self, options):
+  def __init__(self, options, test_results_root):
     """Processes vm-specific options."""
-    au_worker.AUWorker.__init__(self, options)
+    au_worker.AUWorker.__init__(self, options, test_results_root)
     self.graphics_flag = ''
     if options.no_graphics: self.graphics_flag = '--no_graphics'
     if not self.board: cros_lib.Die('Need board to convert base image to vm.')
@@ -95,6 +95,7 @@ class VMAUWorker(au_worker.AUWorker):
 
   def VerifyImage(self, unittest, percent_required_to_pass=100):
     """Runs vm smoke suite to verify image."""
+    test_directory = self.GetNextResultsPath('verify')
     # image_to_live already verifies lsb-release matching.  This is just
     # for additional steps.
     commandWithArgs = ['%s/cros_run_vm_test' % self.crosutilsbin,
@@ -103,6 +104,7 @@ class VMAUWorker(au_worker.AUWorker):
                        '--persist',
                        '--kvm_pid=%s' % self._kvm_pid_file,
                        '--ssh_port=%s' % self._ssh_port,
+                       '--results_dir_root=%s' % test_directory,
                        self.verify_suite,
                       ]
     if self.graphics_flag: commandWithArgs.append(self.graphics_flag)

bin/ctest.py

@@ -226,7 +226,7 @@ def GrabZipAndExtractImage(zip_url, download_folder, image_name) :
 
 
 def RunAUTestHarness(board, channel, latest_url_base, zip_server_base,
-                     no_graphics, type, remote, clean):
+                     no_graphics, type, remote, clean, test_results_root):
   """Runs the auto update test harness.
 
   The auto update test harness encapsulates testing the auto-update mechanism
@@ -243,6 +243,7 @@ def RunAUTestHarness(board, channel, latest_url_base, zip_server_base,
     type: which test harness to run.  Possible values: real, vm.
     remote: ip address for real test harness run.
     clean: Clean the state of test harness before running.
+    test_results_root: Root directory to store au_test_harness results.
   """
   crosutils_root = os.path.join(os.path.dirname(__file__), '..')
   download_folder = os.path.abspath('latest_download')
@@ -270,6 +271,7 @@ def RunAUTestHarness(board, channel, latest_url_base, zip_server_base,
          '--public_key=%s' % os.path.join(update_engine_path,
                                           'unittest_key.pub.pem'),
          ]
+  if test_results_root: cmd.append('--test_results_root=%s' % test_results_root)
   if no_graphics: cmd.append('--no_graphics')
   if clean: cmd.append('--clean')
 
@@ -290,6 +292,9 @@ def main():
                     help='Base url for hosted images.')
   parser.add_option('--no_graphics', action='store_true', default=False,
                     help='Disable graphics for the vm test.')
+  parser.add_option('--test_results_root', default=None,
+                    help='Root directory to store test results.  Should '
+                         'be defined relative to chroot root.')
   parser.add_option('--type', default='vm',
                     help='type of test to run: [vm, real]. Default: vm.')
   parser.add_option('--remote', default='0.0.0.0',