From ac13647eda0a78c9b50c4a93d64ed68c567a41e4 Mon Sep 17 00:00:00 2001 From: Thieu Le Date: Wed, 24 Nov 2010 10:42:32 -0800 Subject: [PATCH] Change _ArchiveTestResults to upload to Google Storage BUG=chromium-os:8364 TEST= Change-Id: Icecc8268fa2c12c8413caa2879785f00e4be5a0a Review URL: http://codereview.chromium.org/4864001 --- bin/cbuildbot.py | 76 ++++++++++++++++++++------------------- bin/cbuildbot_unittest.py | 48 ++++++++----------------- lib/cros_build_lib.py | 71 ++++++++++++++++++++++-------------- 3 files changed, 97 insertions(+), 98 deletions(-) diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index aef3ec2719..6f7d0e9183 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -406,48 +406,40 @@ def _UprevPush(buildroot, tracking_branch, board, overlays): cwd=cwd) -def _ArchiveTestResults(buildroot, board, archive_dir, test_results_dir): - """Archives the test results into the www dir for later use. +def _ArchiveTestResults(buildroot, board, test_results_dir, + gsutil, archive_dir, acl): + """Archives the test results into Google Storage - Takes the results from the test_results_dir and dumps them into the archive - dir specified. This also archives the last qemu image. + Takes the results from the test_results_dir and the last qemu image and + uploads them to Google Storage. - board: Board to find the qemu image. - archive_dir: Path from ARCHIVE_BASE to store image. - test_results_dir: Path from buildroot/chroot to find test results. This must - a subdir of /tmp. + Arguments: + buildroot: Root directory where build occurs + board: Board to find the qemu image. + test_results_dir: Path from buildroot/chroot to find test results. + This must a subdir of /tmp. + gsutil: Location of gsutil + archive_dir: Google Storage path to store the archive + acl: ACL to set on archive in Google Storage """ + num_gsutil_retries = 5 test_results_dir = test_results_dir.lstrip('/') - if not os.path.exists(ARCHIVE_BASE): - os.makedirs(ARCHIVE_BASE) - else: - dir_entries = os.listdir(ARCHIVE_BASE) - if len(dir_entries) >= ARCHIVE_COUNT: - oldest_dirs = heapq.nsmallest((len(dir_entries) - ARCHIVE_COUNT) + 1, - [os.path.join(ARCHIVE_BASE, filename) for filename in dir_entries], - key=lambda fn: os.stat(fn).st_mtime) - Info('Removing archive dirs %s' % oldest_dirs) - for oldest_dir in oldest_dirs: - shutil.rmtree(os.path.join(ARCHIVE_BASE, oldest_dir)) - - archive_target = os.path.join(ARCHIVE_BASE, str(archive_dir)) - if os.path.exists(archive_target): - shutil.rmtree(archive_target) - results_path = os.path.join(buildroot, 'chroot', test_results_dir) RunCommand(['sudo', 'chmod', '-R', '+r', results_path]) try: - shutil.copytree(results_path, archive_target) - except: - Warning('Some files could not be copied') - - image_name = 'chromiumos_qemu_image.bin' - image_path = os.path.join(buildroot, 'src', 'build', 'images', board, - 'latest', image_name) - RunCommand(['gzip', '-f', '--fast', image_path]) - shutil.copyfile(image_path + '.gz', os.path.join(archive_target, - image_name + '.gz')) + # gsutil has the ability to resume an upload when the command is retried + RunCommand([gsutil, 'cp', '-R', results_path, archive_dir], + num_retries=num_gsutil_retries) + RunCommand([gsutil, 'setacl', acl, archive_dir]) + image_name = 'chromiumos_qemu_image.bin' + image_path = os.path.join(buildroot, 'src', 'build', 'images', board, + 'latest', image_name) + RunCommand(['gzip', '-f', '--fast', image_path]) + RunCommand([gsutil, 'cp', image_path + '.gz', archive_dir], + num_retries=num_gsutil_retries) + except Exception, e: + Warning('Could not archive test results (error=%s)' % str(e)) def _GetConfig(config_name): @@ -517,6 +509,11 @@ def main(): parser.add_option('-u', '--url', dest='url', default='http://git.chromium.org/git/manifest', help='Run the buildbot on internal manifest') + parser.add_option('-g', '--gsutil', default='', help='Location of gsutil') + parser.add_option('-c', '--gsutil_archive', default='', + help='Datastore archive location') + parser.add_option('-a', '--acl', default='private', + help='ACL to set on GSD archives') (options, args) = parser.parse_args() @@ -572,9 +569,14 @@ def main(): try: _RunSmokeSuite(buildroot, test_results_dir) finally: - _ArchiveTestResults(buildroot, buildconfig['board'], - archive_dir=options.buildnumber, - test_results_dir=test_results_dir) + if not options.debug: + archive_full_path=os.path.join(options.gsutil_archive, + str(options.buildnumber)) + _ArchiveTestResults(buildroot, buildconfig['board'], + test_results_dir=test_results_dir, + gsutil=options.gsutil, + archive_dir=archive_full_path, + acl=options.acl) if buildconfig['uprev']: # Don't push changes for developers. diff --git a/bin/cbuildbot_unittest.py b/bin/cbuildbot_unittest.py index 979cda55d4..ba9b545b0d 100755 --- a/bin/cbuildbot_unittest.py +++ b/bin/cbuildbot_unittest.py @@ -112,52 +112,32 @@ class CBuildBotTest(mox.MoxTestBase): # self.mox.VerifyAll() def testArchiveTestResults(self): - """Test if we can archive the latest results dir as well as clean up.""" - self.mox.StubOutWithMock(os.path, 'exists') - self.mox.StubOutWithMock(os, 'listdir') - self.mox.StubOutWithMock(os, 'stat') - self.mox.StubOutWithMock(shutil, 'rmtree') - self.mox.StubOutWithMock(shutil, 'copytree') - self.mox.StubOutWithMock(shutil, 'copyfile') - - # Create mock stats so that file2 is older than file1. - dir_listing = ['file1', 'file2'] - stat1 = self.mox.CreateMock(posix.stat_result) - stat2 = self.mox.CreateMock(posix.stat_result) - stat1.st_mtime = 99999 - stat2.st_mtime = 10000 - + """Test if we can archive the latest results dir to Google Storage.""" # Set vars for call. buildroot = '/fake_dir' - test_results_dir = 'fake_results_dir' - archive_dir = 1234 board = 'fake-board' - - # Expected calls. - os.path.exists(cbuildbot.ARCHIVE_BASE).AndReturn(True) - os.listdir(os.path.join(cbuildbot.ARCHIVE_BASE)).AndReturn(dir_listing) - os.stat(os.path.join(cbuildbot.ARCHIVE_BASE, 'file1')).AndReturn(stat1) - os.stat(os.path.join(cbuildbot.ARCHIVE_BASE, 'file2')).AndReturn(stat2) - # Should remove the oldest path. - shutil.rmtree(os.path.join(cbuildbot.ARCHIVE_BASE, 'file2')) + test_results_dir = 'fake_results_dir' + gsutil_path='/fake/gsutil/path' + archive_dir = 1234 + acl = 'fake_acl' + num_retries = 5 # Convenience variables to make archive easier to understand. path_to_results = os.path.join(buildroot, 'chroot', test_results_dir) - path_to_archive_dir = os.path.join(cbuildbot.ARCHIVE_BASE, str(archive_dir)) path_to_image = os.path.join(buildroot, 'src', 'build', 'images', board, 'latest', 'chromiumos_qemu_image.bin') - # Archive logic - os.path.exists(path_to_archive_dir).AndReturn(False) + cbuildbot.RunCommand(['sudo', 'chmod', '-R', '+r', path_to_results]) - shutil.copytree(path_to_results, path_to_archive_dir) + cbuildbot.RunCommand([gsutil_path, 'cp', '-R', path_to_results, + archive_dir], num_retries=num_retries) + cbuildbot.RunCommand([gsutil_path, 'setacl', acl, archive_dir]) cbuildbot.RunCommand(['gzip', '-f', '--fast', path_to_image]) - shutil.copyfile(path_to_image + '.gz', os.path.join( - path_to_archive_dir, 'chromiumos_qemu_image.bin.gz')) + cbuildbot.RunCommand([gsutil_path, 'cp', path_to_image + '.gz', + archive_dir], num_retries=num_retries) self.mox.ReplayAll() - cbuildbot.ARCHIVE_COUNT = 2 # Set equal to list size so we force clean up. - cbuildbot._ArchiveTestResults(buildroot, board, archive_dir, - test_results_dir) + cbuildbot._ArchiveTestResults(buildroot, board, test_results_dir, + gsutil_path, archive_dir, acl) self.mox.VerifyAll() # TODO(sosa): Remove once we un-comment above. diff --git a/lib/cros_build_lib.py b/lib/cros_build_lib.py index 4888a4eae6..0b728efd38 100644 --- a/lib/cros_build_lib.py +++ b/lib/cros_build_lib.py @@ -13,6 +13,11 @@ _STDOUT_IS_TTY = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() # TODO(sosa): Move logging to logging module. +class RunCommandException(Exception): + """Raised when there is an error in RunCommand.""" + pass + + def GetCallerName(): """Returns the name of the calling module with __main__.""" top_frame = inspect.stack()[-1][0] @@ -21,24 +26,30 @@ def GetCallerName(): def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None, exit_code=False, redirect_stdout=False, redirect_stderr=False, - cwd=None, input=None, enter_chroot=False): + cwd=None, input=None, enter_chroot=False, num_retries=0): """Runs a shell command. - Keyword arguments: - cmd - cmd to run. Should be input to subprocess.POpen. If a string, + Arguments: + cmd: cmd to run. Should be input to subprocess.POpen. If a string, converted to an array using split(). - print_cmd -- prints the command before running it. - error_ok -- does not raise an exception on error. - error_message -- prints out this message when an error occurrs. - exit_code -- returns the return code of the shell command. - redirect_stdout -- returns the stdout. - redirect_stderr -- holds stderr output until input is communicated. - cwd -- the working directory to run this cmd. - input -- input to pipe into this command through stdin. - enter_chroot -- this command should be run from within the chroot. If set, + print_cmd: prints the command before running it. + error_ok: does not raise an exception on error. + error_message: prints out this message when an error occurrs. + exit_code: returns the return code of the shell command. + redirect_stdout: returns the stdout. + redirect_stderr: holds stderr output until input is communicated. + cwd: the working directory to run this cmd. + input: input to pipe into this command through stdin. + enter_chroot: this command should be run from within the chroot. If set, cwd must point to the scripts directory. + num_retries: the number of retries to perform before dying + + Returns: + If exit_code is True, returns the return code of the shell command. + Else returns the output of the shell command. + Raises: - Exception: Raises generic exception on error with optional error_message. + Exception: Raises RunCommandException on error with optional error_message. """ # Set default for variables. stdout = None @@ -57,21 +68,27 @@ def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None, Info('PROGRAM(%s) -> RunCommand: %r in dir %s' % (GetCallerName(), cmd, cwd)) - try: - proc = subprocess.Popen(cmd, cwd=cwd, stdin=stdin, - stdout=stdout, stderr=stderr) - (output, error) = proc.communicate(input) - if exit_code: - return proc.returncode + for retry_count in range(num_retries + 1): + try: + proc = subprocess.Popen(cmd, cwd=cwd, stdin=stdin, + stdout=stdout, stderr=stderr) + (output, error) = proc.communicate(input) + if exit_code and retry_count == num_retries: + return proc.returncode - if not error_ok and proc.returncode: - raise Exception('Command "%r" failed.\n' % (cmd) + - (error_message or error or output or '')) - except Exception, e: - if not error_ok: - raise - else: - Warning(str(e)) + if proc.returncode == 0: + break + + raise RunCommandException('Command "%r" failed.\n' % (cmd) + + (error_message or error or output or '')) + except Exception, e: + if not error_ok and retry_count == num_retries: + raise RunCommandException(e) + else: + Warning(str(e)) + if print_cmd: + Info('PROGRAM(%s) -> RunCommand: retrying %r in dir %s' % + (GetCallerName(), cmd, cwd)) return output