Change _ArchiveTestResults to upload to Google Storage

BUG=chromium-os:8364
TEST=

Change-Id: Icecc8268fa2c12c8413caa2879785f00e4be5a0a

Review URL: http://codereview.chromium.org/4864001

bin/cbuildbot.py

@@ -406,48 +406,40 @@ def _UprevPush(buildroot, tracking_branch, board, overlays):
              cwd=cwd)
 
 
-def _ArchiveTestResults(buildroot, board, archive_dir, test_results_dir):
+def _ArchiveTestResults(buildroot, board, test_results_dir,
-  """Archives the test results into the www dir for later use.
+                        gsutil, archive_dir, acl):
+  """Archives the test results into Google Storage
 
-  Takes the results from the test_results_dir and dumps them into the archive
+  Takes the results from the test_results_dir and the last qemu image and
-  dir specified.  This also archives the last qemu image.
+  uploads them to Google Storage.
 
-  board:  Board to find the qemu image.
+  Arguments:
-  archive_dir:  Path from ARCHIVE_BASE to store image.
+    buildroot: Root directory where build occurs
-  test_results_dir: Path from buildroot/chroot to find test results.  This must
+    board: Board to find the qemu image.
-    a subdir of /tmp.
+    test_results_dir: Path from buildroot/chroot to find test results.
+      This must a subdir of /tmp.
+    gsutil: Location of gsutil
+    archive_dir: Google Storage path to store the archive
+    acl: ACL to set on archive in Google Storage
   """
+  num_gsutil_retries = 5
   test_results_dir = test_results_dir.lstrip('/')
-  if not os.path.exists(ARCHIVE_BASE):
-    os.makedirs(ARCHIVE_BASE)
-  else:
-    dir_entries = os.listdir(ARCHIVE_BASE)
-    if len(dir_entries) >= ARCHIVE_COUNT:
-      oldest_dirs = heapq.nsmallest((len(dir_entries) - ARCHIVE_COUNT) + 1,
-          [os.path.join(ARCHIVE_BASE, filename) for filename in dir_entries],
-          key=lambda fn: os.stat(fn).st_mtime)
-      Info('Removing archive dirs %s' % oldest_dirs)
-      for oldest_dir in oldest_dirs:
-        shutil.rmtree(os.path.join(ARCHIVE_BASE, oldest_dir))
-
-  archive_target = os.path.join(ARCHIVE_BASE, str(archive_dir))
-  if os.path.exists(archive_target):
-    shutil.rmtree(archive_target)
-
   results_path = os.path.join(buildroot, 'chroot', test_results_dir)
   RunCommand(['sudo', 'chmod', '-R', '+r', results_path])
   try:
-    shutil.copytree(results_path, archive_target)
+    # gsutil has the ability to resume an upload when the command is retried
-  except:
+    RunCommand([gsutil, 'cp', '-R', results_path, archive_dir],
-    Warning('Some files could not be copied')
+               num_retries=num_gsutil_retries)
-
+    RunCommand([gsutil, 'setacl', acl, archive_dir])
-  image_name = 'chromiumos_qemu_image.bin'
-  image_path = os.path.join(buildroot, 'src', 'build', 'images', board,
-                            'latest', image_name)
-  RunCommand(['gzip', '-f', '--fast', image_path])
-  shutil.copyfile(image_path + '.gz', os.path.join(archive_target,
-                                                   image_name + '.gz'))
 
+    image_name = 'chromiumos_qemu_image.bin'
+    image_path = os.path.join(buildroot, 'src', 'build', 'images', board,
+                              'latest', image_name)
+    RunCommand(['gzip', '-f', '--fast', image_path])
+    RunCommand([gsutil, 'cp', image_path + '.gz', archive_dir],
+               num_retries=num_gsutil_retries)
+  except Exception, e:
+    Warning('Could not archive test results (error=%s)' % str(e))
 
 
 def _GetConfig(config_name):
@@ -517,6 +509,11 @@ def main():
   parser.add_option('-u', '--url', dest='url',
                     default='http://git.chromium.org/git/manifest',
                     help='Run the buildbot on internal manifest')
+  parser.add_option('-g', '--gsutil', default='', help='Location of gsutil')
+  parser.add_option('-c', '--gsutil_archive', default='',
+                    help='Datastore archive location')
+  parser.add_option('-a', '--acl', default='private',
+                    help='ACL to set on GSD archives')
 
   (options, args) = parser.parse_args()
 
@@ -572,9 +569,14 @@ def main():
       try:
         _RunSmokeSuite(buildroot, test_results_dir)
       finally:
-        _ArchiveTestResults(buildroot, buildconfig['board'],
+        if not options.debug:
-                            archive_dir=options.buildnumber,
+          archive_full_path=os.path.join(options.gsutil_archive,
-                            test_results_dir=test_results_dir)
+                                         str(options.buildnumber))
+          _ArchiveTestResults(buildroot, buildconfig['board'],
+                              test_results_dir=test_results_dir,
+                              gsutil=options.gsutil,
+                              archive_dir=archive_full_path,
+                              acl=options.acl)
 
     if buildconfig['uprev']:
       # Don't push changes for developers.

bin/cbuildbot_unittest.py

@@ -112,52 +112,32 @@ class CBuildBotTest(mox.MoxTestBase):
   #  self.mox.VerifyAll()
 
   def testArchiveTestResults(self):
-    """Test if we can archive the latest results dir as well as clean up."""
+    """Test if we can archive the latest results dir to Google Storage."""
-    self.mox.StubOutWithMock(os.path, 'exists')
-    self.mox.StubOutWithMock(os, 'listdir')
-    self.mox.StubOutWithMock(os, 'stat')
-    self.mox.StubOutWithMock(shutil, 'rmtree')
-    self.mox.StubOutWithMock(shutil, 'copytree')
-    self.mox.StubOutWithMock(shutil, 'copyfile')
-
-    # Create mock stats so that file2 is older than file1.
-    dir_listing = ['file1', 'file2']
-    stat1 = self.mox.CreateMock(posix.stat_result)
-    stat2 = self.mox.CreateMock(posix.stat_result)
-    stat1.st_mtime = 99999
-    stat2.st_mtime = 10000
-
     # Set vars for call.
     buildroot = '/fake_dir'
-    test_results_dir = 'fake_results_dir'
-    archive_dir = 1234
     board = 'fake-board'
-
+    test_results_dir = 'fake_results_dir'
-    # Expected calls.
+    gsutil_path='/fake/gsutil/path'
-    os.path.exists(cbuildbot.ARCHIVE_BASE).AndReturn(True)
+    archive_dir = 1234
-    os.listdir(os.path.join(cbuildbot.ARCHIVE_BASE)).AndReturn(dir_listing)
+    acl = 'fake_acl'
-    os.stat(os.path.join(cbuildbot.ARCHIVE_BASE, 'file1')).AndReturn(stat1)
+    num_retries = 5
-    os.stat(os.path.join(cbuildbot.ARCHIVE_BASE, 'file2')).AndReturn(stat2)
-    # Should remove the oldest path.
-    shutil.rmtree(os.path.join(cbuildbot.ARCHIVE_BASE, 'file2'))
 
     # Convenience variables to make archive easier to understand.
     path_to_results = os.path.join(buildroot, 'chroot', test_results_dir)
-    path_to_archive_dir = os.path.join(cbuildbot.ARCHIVE_BASE, str(archive_dir))
     path_to_image = os.path.join(buildroot, 'src', 'build', 'images', board,
                                  'latest', 'chromiumos_qemu_image.bin')
-    # Archive logic
+
-    os.path.exists(path_to_archive_dir).AndReturn(False)
     cbuildbot.RunCommand(['sudo', 'chmod', '-R', '+r', path_to_results])
-    shutil.copytree(path_to_results, path_to_archive_dir)
+    cbuildbot.RunCommand([gsutil_path, 'cp', '-R', path_to_results,
+                          archive_dir], num_retries=num_retries)
+    cbuildbot.RunCommand([gsutil_path, 'setacl', acl, archive_dir])
     cbuildbot.RunCommand(['gzip', '-f', '--fast', path_to_image])
-    shutil.copyfile(path_to_image + '.gz', os.path.join(
+    cbuildbot.RunCommand([gsutil_path, 'cp', path_to_image + '.gz',
-        path_to_archive_dir, 'chromiumos_qemu_image.bin.gz'))
+                          archive_dir], num_retries=num_retries)
 
     self.mox.ReplayAll()
-    cbuildbot.ARCHIVE_COUNT = 2 # Set equal to list size so we force clean up.
+    cbuildbot._ArchiveTestResults(buildroot, board, test_results_dir,
-    cbuildbot._ArchiveTestResults(buildroot, board, archive_dir,
+                                  gsutil_path, archive_dir, acl)
-                                  test_results_dir)
     self.mox.VerifyAll()
 
   # TODO(sosa):  Remove once we un-comment above.

lib/cros_build_lib.py

@@ -13,6 +13,11 @@ _STDOUT_IS_TTY = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty()
 
 # TODO(sosa):  Move logging to logging module.
 
+class RunCommandException(Exception):
+  """Raised when there is an error in RunCommand."""
+  pass
+
+
 def GetCallerName():
   """Returns the name of the calling module with __main__."""
   top_frame = inspect.stack()[-1][0]
@@ -21,24 +26,30 @@ def GetCallerName():
 
 def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None,
                exit_code=False, redirect_stdout=False, redirect_stderr=False,
-               cwd=None, input=None, enter_chroot=False):
+               cwd=None, input=None, enter_chroot=False, num_retries=0):
   """Runs a shell command.
 
-  Keyword arguments:
+  Arguments:
-    cmd - cmd to run.  Should be input to subprocess.POpen.  If a string,
+    cmd: cmd to run.  Should be input to subprocess.POpen.  If a string,
       converted to an array using split().
-    print_cmd -- prints the command before running it.
+    print_cmd: prints the command before running it.
-    error_ok -- does not raise an exception on error.
+    error_ok: does not raise an exception on error.
-    error_message -- prints out this message when an error occurrs.
+    error_message: prints out this message when an error occurrs.
-    exit_code -- returns the return code of the shell command.
+    exit_code: returns the return code of the shell command.
-    redirect_stdout -- returns the stdout.
+    redirect_stdout: returns the stdout.
-    redirect_stderr -- holds stderr output until input is communicated.
+    redirect_stderr: holds stderr output until input is communicated.
-    cwd -- the working directory to run this cmd.
+    cwd: the working directory to run this cmd.
-    input -- input to pipe into this command through stdin.
+    input: input to pipe into this command through stdin.
-    enter_chroot -- this command should be run from within the chroot.  If set,
+    enter_chroot: this command should be run from within the chroot.  If set,
       cwd must point to the scripts directory.
+    num_retries: the number of retries to perform before dying
+
+  Returns:
+    If exit_code is True, returns the return code of the shell command.
+    Else returns the output of the shell command.
+
   Raises:
-    Exception:  Raises generic exception on error with optional error_message.
+    Exception:  Raises RunCommandException on error with optional error_message.
   """
   # Set default for variables.
   stdout = None
@@ -57,21 +68,27 @@ def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None,
     Info('PROGRAM(%s) -> RunCommand: %r in dir %s' %
          (GetCallerName(), cmd, cwd))
 
-  try:
+  for retry_count in range(num_retries + 1):
-    proc = subprocess.Popen(cmd, cwd=cwd, stdin=stdin,
+    try:
-                            stdout=stdout, stderr=stderr)
+      proc = subprocess.Popen(cmd, cwd=cwd, stdin=stdin,
-    (output, error) = proc.communicate(input)
+                              stdout=stdout, stderr=stderr)
-    if exit_code:
+      (output, error) = proc.communicate(input)
-      return proc.returncode
+      if exit_code and retry_count == num_retries:
+        return proc.returncode
 
-    if not error_ok and proc.returncode:
+      if proc.returncode == 0:
-      raise Exception('Command "%r" failed.\n' % (cmd) +
+        break
-                      (error_message or error or output or ''))
+
-  except Exception, e:
+      raise RunCommandException('Command "%r" failed.\n' % (cmd) +
-    if not error_ok:
+                                (error_message or error or output or ''))
-      raise
+    except Exception, e:
-    else:
+      if not error_ok and retry_count == num_retries:
-      Warning(str(e))
+        raise RunCommandException(e)
+      else:
+        Warning(str(e))
+        if print_cmd:
+          Info('PROGRAM(%s) -> RunCommand: retrying %r in dir %s' %
+               (GetCallerName(), cmd, cwd))
 
   return output