mirrors/flatcar-scripts
1
0
Fork 0
mirror of https://github.com/flatcar/scripts.git synced 2025-09-24 15:11:19 +02:00
Code Issues Packages Projects Releases Wiki Activity

Change _ArchiveTestResults to upload to Google Storage

Browse Source 
BUG=chromium-os:8364
TEST=

Change-Id: Icecc8268fa2c12c8413caa2879785f00e4be5a0a

Review URL: http://codereview.chromium.org/4864001
This commit is contained in:
Thieu Le 2010-11-24 10:42:32 -08:00
parent 444e3a6de0
commit ac13647eda
3 changed files with 97 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
bin/cbuildbot.py
View File

@ -406,48 +406,40 @@ def _UprevPush(buildroot, tracking_branch, board, overlays):
cwd=cwd)
def _ArchiveTestResults(buildroot, board, archive_dir, test_results_dir):
"""Archives the test results into the www dir for later use.
def _ArchiveTestResults(buildroot, board, test_results_dir,
gsutil, archive_dir, acl):
"""Archives the test results into Google Storage
Takes the results from the test_results_dir and dumps them into the archive
dir specified. This also archives the last qemu image.
Takes the results from the test_results_dir and the last qemu image and
uploads them to Google Storage.
Arguments:
buildroot: Root directory where build occurs
board: Board to find the qemu image.
archive_dir: Path from ARCHIVE_BASE to store image.
test_results_dir: Path from buildroot/chroot to find test results. This must
a subdir of /tmp.
test_results_dir: Path from buildroot/chroot to find test results.
This must a subdir of /tmp.
gsutil: Location of gsutil
archive_dir: Google Storage path to store the archive
acl: ACL to set on archive in Google Storage
"""
num_gsutil_retries = 5
test_results_dir = test_results_dir.lstrip('/')
if not os.path.exists(ARCHIVE_BASE):
os.makedirs(ARCHIVE_BASE)
else:
dir_entries = os.listdir(ARCHIVE_BASE)
if len(dir_entries) >= ARCHIVE_COUNT:
oldest_dirs = heapq.nsmallest((len(dir_entries) - ARCHIVE_COUNT) + 1,
[os.path.join(ARCHIVE_BASE, filename) for filename in dir_entries],
key=lambda fn: os.stat(fn).st_mtime)
Info('Removing archive dirs %s' % oldest_dirs)
for oldest_dir in oldest_dirs:
shutil.rmtree(os.path.join(ARCHIVE_BASE, oldest_dir))
archive_target = os.path.join(ARCHIVE_BASE, str(archive_dir))
if os.path.exists(archive_target):
shutil.rmtree(archive_target)
results_path = os.path.join(buildroot, 'chroot', test_results_dir)
RunCommand(['sudo', 'chmod', '-R', '+r', results_path])
try:
shutil.copytree(results_path, archive_target)
except:
Warning('Some files could not be copied')
# gsutil has the ability to resume an upload when the command is retried
RunCommand([gsutil, 'cp', '-R', results_path, archive_dir],
num_retries=num_gsutil_retries)
RunCommand([gsutil, 'setacl', acl, archive_dir])
image_name = 'chromiumos_qemu_image.bin'
image_path = os.path.join(buildroot, 'src', 'build', 'images', board,
'latest', image_name)
RunCommand(['gzip', '-f', '--fast', image_path])
shutil.copyfile(image_path + '.gz', os.path.join(archive_target,
image_name + '.gz'))
RunCommand([gsutil, 'cp', image_path + '.gz', archive_dir],
num_retries=num_gsutil_retries)
except Exception, e:
Warning('Could not archive test results (error=%s)' % str(e))
def _GetConfig(config_name):
@ -517,6 +509,11 @@ def main():
parser.add_option('-u', '--url', dest='url',
default='http://git.chromium.org/git/manifest',
help='Run the buildbot on internal manifest')
parser.add_option('-g', '--gsutil', default='', help='Location of gsutil')
parser.add_option('-c', '--gsutil_archive', default='',
help='Datastore archive location')
parser.add_option('-a', '--acl', default='private',
help='ACL to set on GSD archives')
(options, args) = parser.parse_args()
@ -572,9 +569,14 @@ def main():
try:
_RunSmokeSuite(buildroot, test_results_dir)
finally:
if not options.debug:
archive_full_path=os.path.join(options.gsutil_archive,
str(options.buildnumber))
_ArchiveTestResults(buildroot, buildconfig['board'],
archive_dir=options.buildnumber,
test_results_dir=test_results_dir)
test_results_dir=test_results_dir,
gsutil=options.gsutil,
archive_dir=archive_full_path,
acl=options.acl)
if buildconfig['uprev']:
# Don't push changes for developers.

48
bin/cbuildbot_unittest.py
View File

@ -112,52 +112,32 @@ class CBuildBotTest(mox.MoxTestBase):
# self.mox.VerifyAll()
def testArchiveTestResults(self):
"""Test if we can archive the latest results dir as well as clean up."""
self.mox.StubOutWithMock(os.path, 'exists')
self.mox.StubOutWithMock(os, 'listdir')
self.mox.StubOutWithMock(os, 'stat')
self.mox.StubOutWithMock(shutil, 'rmtree')
self.mox.StubOutWithMock(shutil, 'copytree')
self.mox.StubOutWithMock(shutil, 'copyfile')
# Create mock stats so that file2 is older than file1.
dir_listing = ['file1', 'file2']
stat1 = self.mox.CreateMock(posix.stat_result)
stat2 = self.mox.CreateMock(posix.stat_result)
stat1.st_mtime = 99999
stat2.st_mtime = 10000
"""Test if we can archive the latest results dir to Google Storage."""
# Set vars for call.
buildroot = '/fake_dir'
test_results_dir = 'fake_results_dir'
archive_dir = 1234
board = 'fake-board'
# Expected calls.
os.path.exists(cbuildbot.ARCHIVE_BASE).AndReturn(True)
os.listdir(os.path.join(cbuildbot.ARCHIVE_BASE)).AndReturn(dir_listing)
os.stat(os.path.join(cbuildbot.ARCHIVE_BASE, 'file1')).AndReturn(stat1)
os.stat(os.path.join(cbuildbot.ARCHIVE_BASE, 'file2')).AndReturn(stat2)
# Should remove the oldest path.
shutil.rmtree(os.path.join(cbuildbot.ARCHIVE_BASE, 'file2'))
test_results_dir = 'fake_results_dir'
gsutil_path='/fake/gsutil/path'
archive_dir = 1234
acl = 'fake_acl'
num_retries = 5
# Convenience variables to make archive easier to understand.
path_to_results = os.path.join(buildroot, 'chroot', test_results_dir)
path_to_archive_dir = os.path.join(cbuildbot.ARCHIVE_BASE, str(archive_dir))
path_to_image = os.path.join(buildroot, 'src', 'build', 'images', board,
'latest', 'chromiumos_qemu_image.bin')
# Archive logic
os.path.exists(path_to_archive_dir).AndReturn(False)
cbuildbot.RunCommand(['sudo', 'chmod', '-R', '+r', path_to_results])
shutil.copytree(path_to_results, path_to_archive_dir)
cbuildbot.RunCommand([gsutil_path, 'cp', '-R', path_to_results,
archive_dir], num_retries=num_retries)
cbuildbot.RunCommand([gsutil_path, 'setacl', acl, archive_dir])
cbuildbot.RunCommand(['gzip', '-f', '--fast', path_to_image])
shutil.copyfile(path_to_image + '.gz', os.path.join(
path_to_archive_dir, 'chromiumos_qemu_image.bin.gz'))
cbuildbot.RunCommand([gsutil_path, 'cp', path_to_image + '.gz',
archive_dir], num_retries=num_retries)
self.mox.ReplayAll()
cbuildbot.ARCHIVE_COUNT = 2 # Set equal to list size so we force clean up.
cbuildbot._ArchiveTestResults(buildroot, board, archive_dir,
test_results_dir)
cbuildbot._ArchiveTestResults(buildroot, board, test_results_dir,
gsutil_path, archive_dir, acl)
self.mox.VerifyAll()
# TODO(sosa): Remove once we un-comment above.

53
lib/cros_build_lib.py
View File

@ -13,6 +13,11 @@ _STDOUT_IS_TTY = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty()
# TODO(sosa): Move logging to logging module.
class RunCommandException(Exception):
"""Raised when there is an error in RunCommand."""
pass
def GetCallerName():
"""Returns the name of the calling module with __main__."""
top_frame = inspect.stack()[-1][0]
@ -21,24 +26,30 @@ def GetCallerName():
def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None,
exit_code=False, redirect_stdout=False, redirect_stderr=False,
cwd=None, input=None, enter_chroot=False):
cwd=None, input=None, enter_chroot=False, num_retries=0):
"""Runs a shell command.
Keyword arguments:
cmd - cmd to run. Should be input to subprocess.POpen. If a string,
Arguments:
cmd: cmd to run. Should be input to subprocess.POpen. If a string,
converted to an array using split().
print_cmd -- prints the command before running it.
error_ok -- does not raise an exception on error.
error_message -- prints out this message when an error occurrs.
exit_code -- returns the return code of the shell command.
redirect_stdout -- returns the stdout.
redirect_stderr -- holds stderr output until input is communicated.
cwd -- the working directory to run this cmd.
input -- input to pipe into this command through stdin.
enter_chroot -- this command should be run from within the chroot. If set,
print_cmd: prints the command before running it.
error_ok: does not raise an exception on error.
error_message: prints out this message when an error occurrs.
exit_code: returns the return code of the shell command.
redirect_stdout: returns the stdout.
redirect_stderr: holds stderr output until input is communicated.
cwd: the working directory to run this cmd.
input: input to pipe into this command through stdin.
enter_chroot: this command should be run from within the chroot. If set,
cwd must point to the scripts directory.
num_retries: the number of retries to perform before dying
Returns:
If exit_code is True, returns the return code of the shell command.
Else returns the output of the shell command.
Raises:
Exception: Raises generic exception on error with optional error_message.
Exception: Raises RunCommandException on error with optional error_message.
"""
# Set default for variables.
stdout = None
@ -57,21 +68,27 @@ def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None,
Info('PROGRAM(%s) -> RunCommand: %r in dir %s' %
(GetCallerName(), cmd, cwd))
for retry_count in range(num_retries + 1):
try:
proc = subprocess.Popen(cmd, cwd=cwd, stdin=stdin,
stdout=stdout, stderr=stderr)
(output, error) = proc.communicate(input)
if exit_code:
if exit_code and retry_count == num_retries:
return proc.returncode
if not error_ok and proc.returncode:
raise Exception('Command "%r" failed.\n' % (cmd) +
if proc.returncode == 0:
break
raise RunCommandException('Command "%r" failed.\n' % (cmd) +
(error_message or error or output or ''))
except Exception, e:
if not error_ok:
raise
if not error_ok and retry_count == num_retries:
raise RunCommandException(e)
else:
Warning(str(e))
if print_cmd:
Info('PROGRAM(%s) -> RunCommand: retrying %r in dir %s' %
(GetCallerName(), cmd, cwd))
return output