From c663f209f730646e1e05cb5ec980deeab7ff9fd2 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Mon, 13 Dec 2010 11:11:35 -0800 Subject: [PATCH] Add ability to store the manifest in the manifest directory. Change-Id: I1ac3ed12df00424ba5ba2d1c61fd6fe9716e13b9 BUG=chromium-os:9714 TEST=Ran new unittests as well as ran cbuildbot on local b/cbuild Review URL: http://codereview.chromium.org/5689003 --- bin/cbuildbot.py | 74 ++++++++++++++++++++++++++++++++++++--- bin/cbuildbot_unittest.py | 66 +++++++++++++++++++++++++++++++++- 2 files changed, 134 insertions(+), 6 deletions(-) diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index 8b44fdf253..c332a50aae 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -25,6 +25,8 @@ _DEFAULT_RETRIES = 3 _PACKAGE_FILE = '%(buildroot)s/src/scripts/cbuildbot_package.list' ARCHIVE_BASE = '/var/www/archive' ARCHIVE_COUNT = 10 +PUBLIC_OVERLAY = '%(buildroot)s/src/third_party/chromiumos-overlay' +PRIVATE_OVERLAY = '%(buildroot)s/src/private-overlays/chromeos-overlay' # Currently, both the full buildbot and the preflight buildbot store their # data in a variable named PORTAGE_BINHOST, but they're in different files. @@ -35,6 +37,14 @@ _PREFLIGHT_BINHOST = 'PORTAGE_BINHOST' # ======================== Utility functions ================================ +def _PrintFile(path): + """Prints out the contents of a file to stderr.""" + file_handle = open(path) + print >> sys.stderr, file_handle.read() + file_handle.close() + sys.stderr.flush() + + def MakeDir(path, parents=False): """Basic wrapper around os.mkdirs. @@ -76,9 +86,6 @@ def RepoSync(buildroot, retries=_DEFAULT_RETRIES): Warning('CBUILDBOT -- Retries exhausted') raise - # Output manifest - RunCommand(['repo', 'manifest', '-r', '-o', '-'], cwd=buildroot) - # =========================== Command Helpers ================================= def _GetAllGitRepos(buildroot, debug=False): @@ -278,6 +285,31 @@ def _WipeOldOutput(buildroot): RunCommand(['rm', '-rf', 'src/build/images'], cwd=buildroot) +def _GetChromeOSVersion(buildroot): + """Returns the tuple version of the Chrome OS version of the buildroot.""" + cwd = os.path.join(buildroot, 'src', 'scripts') + version_cmd = './chromeos_version.sh' + output = RunCommand(version_cmd, cwd=cwd, redirect_stdout=True, + redirect_stderr=True) + version_re = re.compile('\s+CHROMEOS_VERSION_STRING=' + '(\d+)\.(\d+)\.(\d+)\.(\w+)') + for line in output.splitlines(): + match = version_re.match(line) + if match: + return match.group(1), match.group(2), match.group(3), match.group(4) + + raise Exception('Chrome OS version not found.') + + +def _GetManifestPath(buildroot): + """Returns the relative path that a manifest should be saved into.""" + version_tuple = _GetChromeOSVersion(buildroot) + (major, minor, branch, patch) = version_tuple + relative_path = os.path.join('.'.join([major, minor]), + '%s.xml' % '.'.join(version_tuple)) + return relative_path + + # =========================== Main Commands =================================== @@ -306,6 +338,36 @@ def _IncrementalCheckout(buildroot, retries=_DEFAULT_RETRIES): RepoSync(buildroot, retries) +def _DumpManifest(buildroot, url): + """Stores the manifest in the public | private overlay depending on url.""" + public_overlay = PUBLIC_OVERLAY % {'buildroot': buildroot} + private_overlay = PRIVATE_OVERLAY % {'buildroot': buildroot} + if url.endswith('manifest-internal'): + overlay = PRIVATE_OVERLAY % {'buildroot': buildroot} + else: + overlay = PUBLIC_OVERLAY % {'buildroot': buildroot} + + # Generate paths for manifests. + relative_path = _GetManifestPath(buildroot) + manifest_path = os.path.join(overlay, 'manifests', relative_path) + symlink_path = os.path.join(overlay, 'manifests', 'LATEST') + if not os.path.isdir(os.path.dirname(manifest_path)): + os.makedirs(os.path.dirname(manifest_path)) + + # Dump the manifest and create a symlink to it. + RunCommand(['repo', 'manifest', '-r', '-o', manifest_path], cwd=buildroot) + if os.path.exists(symlink_path): + os.unlink(symlink_path) + + os.symlink(relative_path, symlink_path) + + # Add it to git and print it to stderr. + RunCommand(['git', 'add', os.path.join('manifests', relative_path)], + cwd=overlay) + RunCommand(['git', 'add', os.path.join('manifests', 'LATEST')], cwd=overlay) + _PrintFile(manifest_path) + + def _MakeChroot(buildroot): """Wrapper around make_chroot.""" cwd = os.path.join(buildroot, 'src', 'scripts') @@ -527,8 +589,8 @@ def _ResolveOverlays(buildroot, overlays): 'public': Just the public overlay. 'both': Both the public and private overlays. """ - public_overlay = '%s/src/third_party/chromiumos-overlay' % buildroot - private_overlay = '%s/src/private-overlays/chromeos-overlay' % buildroot + public_overlay = PUBLIC_OVERLAY % {'buildroot': buildroot} + private_overlay = PRIVATE_OVERLAY % {'buildroot': buildroot} if overlays == 'private': paths = [private_overlay] elif overlays == 'public': @@ -657,6 +719,8 @@ def main(): if not os.path.isdir(path): Die('Missing overlay: %s' % path) + _DumpManifest(buildroot, options.url) + if not os.path.isdir(chroot_path): _MakeChroot(buildroot) diff --git a/bin/cbuildbot_unittest.py b/bin/cbuildbot_unittest.py index b3a4009a90..edacbe55d9 100755 --- a/bin/cbuildbot_unittest.py +++ b/bin/cbuildbot_unittest.py @@ -11,6 +11,7 @@ import mox import os import posix import shutil +import tempfile import unittest # Fixes circular dependency error. @@ -18,6 +19,7 @@ import cbuildbot_comm import cbuildbot from cros_build_lib import ReinterpretPathForChroot + class CBuildBotTest(mox.MoxTestBase): def setUp(self): @@ -117,7 +119,7 @@ class CBuildBotTest(mox.MoxTestBase): buildroot = '/fake_dir' board = 'fake-board' test_results_dir = 'fake_results_dir' - gsutil_path='/fake/gsutil/path' + gsutil_path = '/fake/gsutil/path' archive_dir = 1234 acl = 'fake_acl' num_retries = 5 @@ -230,6 +232,68 @@ class CBuildBotTest(mox.MoxTestBase): binhosts) self.mox.VerifyAll() + def testGetChromeOSVersion(self): + """Tests that we can parse the correct chromeos version using method.""" + fake_data = '\n'.join(['ChromeOS version information:', + ' SOME_OTHER_DATA=blah', + ' CHROMEOS_VERSION_STRING=9.0.232.1', + ' CHROMEOS_VERSION_CODENAME=test-bot', + ]) + cbuildbot.RunCommand('./chromeos_version.sh', + cwd='src/scripts', + redirect_stderr=True, + redirect_stdout=True).AndReturn(fake_data) + self.mox.ReplayAll() + return_tuple = cbuildbot._GetChromeOSVersion('') + self.assertEquals('.'.join(return_tuple), '9.0.232.1') + self.mox.VerifyAll() + + def testGetManifestPath(self): + """Tests whether our logic to get the manifest path is correct.""" + self.mox.StubOutWithMock(cbuildbot, '_GetChromeOSVersion') + return_tuple = cbuildbot._GetChromeOSVersion('').AndReturn( + ('9', '0', '232', '1')) + self.mox.ReplayAll() + relative_path = cbuildbot._GetManifestPath('') + self.assertEquals(relative_path, '9.0/9.0.232.1.xml') + self.mox.VerifyAll() + + def _CommonManifestTest(self, url, overlay_no_buildroot): + """Common method for dump manifest tests.""" + self.mox.StubOutWithMock(cbuildbot, '_GetManifestPath') + self.mox.StubOutWithMock(shutil, 'copy') + self.mox.StubOutWithMock(os, 'symlink') + temp_root = tempfile.mkdtemp('_unittest') + + overlay = overlay_no_buildroot % {'buildroot': temp_root} + relative_path = 'fake/manifest/path.xml' + full_path = os.path.join(overlay, 'manifests', relative_path) + + cbuildbot._GetManifestPath(temp_root).AndReturn(relative_path) + cbuildbot.RunCommand(['repo', 'manifest', '-r', '-o', full_path], + cwd=temp_root) + os.symlink(relative_path, os.path.join(overlay, 'manifests', 'LATEST')) + cbuildbot.RunCommand(['git', 'add', 'manifests/' + relative_path], + cwd=overlay) + cbuildbot.RunCommand(['git', 'add', 'manifests/LATEST'], cwd=overlay) + shutil.copy(full_path, '/dev/stderr') + + self.mox.ReplayAll() + cbuildbot._DumpManifest(temp_root, url) + self.mox.VerifyAll() + + shutil.rmtree(temp_root) + + def testDumpManifestPublic(self): + """Tests whether we push the manifest to the public overlay correctly.""" + self._CommonManifestTest('http://some_url/manifest', + cbuildbot.PUBLIC_OVERLAY) + + def testDumpManifestPrivate(self): + """Tests whether we push the manifest to the private overlay correctly.""" + self._CommonManifestTest('http://some_url/manifest-internal', + cbuildbot.PRIVATE_OVERLAY) + if __name__ == '__main__': unittest.main()