Add ability to store the manifest in the manifest directory.

Change-Id: I1ac3ed12df00424ba5ba2d1c61fd6fe9716e13b9

BUG=chromium-os:9714
TEST=Ran new unittests as well as ran cbuildbot on local b/cbuild

Review URL: http://codereview.chromium.org/5689003

bin/cbuildbot.py

@@ -25,6 +25,8 @@ _DEFAULT_RETRIES = 3
 _PACKAGE_FILE = '%(buildroot)s/src/scripts/cbuildbot_package.list'
 ARCHIVE_BASE = '/var/www/archive'
 ARCHIVE_COUNT = 10
+PUBLIC_OVERLAY = '%(buildroot)s/src/third_party/chromiumos-overlay'
+PRIVATE_OVERLAY = '%(buildroot)s/src/private-overlays/chromeos-overlay'
 
 # Currently, both the full buildbot and the preflight buildbot store their
 # data in a variable named PORTAGE_BINHOST, but they're in different files.
@@ -35,6 +37,14 @@ _PREFLIGHT_BINHOST = 'PORTAGE_BINHOST'
 
 # ======================== Utility functions ================================
 
+def _PrintFile(path):
+  """Prints out the contents of a file to stderr."""
+  file_handle = open(path)
+  print >> sys.stderr, file_handle.read()
+  file_handle.close()
+  sys.stderr.flush()
+
+
 def MakeDir(path, parents=False):
   """Basic wrapper around os.mkdirs.
 
@@ -76,9 +86,6 @@ def RepoSync(buildroot, retries=_DEFAULT_RETRIES):
         Warning('CBUILDBOT -- Retries exhausted')
         raise
 
-  # Output manifest
-  RunCommand(['repo', 'manifest', '-r', '-o', '-'], cwd=buildroot)
-
 # =========================== Command Helpers =================================
 
 def _GetAllGitRepos(buildroot, debug=False):
@@ -278,6 +285,31 @@ def _WipeOldOutput(buildroot):
   RunCommand(['rm', '-rf', 'src/build/images'], cwd=buildroot)
 
 
+def _GetChromeOSVersion(buildroot):
+  """Returns the tuple version of the Chrome OS version of the buildroot."""
+  cwd = os.path.join(buildroot, 'src', 'scripts')
+  version_cmd = './chromeos_version.sh'
+  output = RunCommand(version_cmd, cwd=cwd, redirect_stdout=True,
+                      redirect_stderr=True)
+  version_re = re.compile('\s+CHROMEOS_VERSION_STRING='
+                          '(\d+)\.(\d+)\.(\d+)\.(\w+)')
+  for line in output.splitlines():
+    match = version_re.match(line)
+    if match:
+      return match.group(1), match.group(2), match.group(3), match.group(4)
+
+  raise Exception('Chrome OS version not found.')
+
+
+def _GetManifestPath(buildroot):
+  """Returns the relative path that a manifest should be saved into."""
+  version_tuple = _GetChromeOSVersion(buildroot)
+  (major, minor, branch, patch) = version_tuple
+  relative_path = os.path.join('.'.join([major, minor]),
+                               '%s.xml' % '.'.join(version_tuple))
+  return relative_path
+
+
 # =========================== Main Commands ===================================
 
 
@@ -306,6 +338,36 @@ def _IncrementalCheckout(buildroot, retries=_DEFAULT_RETRIES):
   RepoSync(buildroot, retries)
 
 
+def _DumpManifest(buildroot, url):
+  """Stores the manifest in the public | private overlay depending on url."""
+  public_overlay = PUBLIC_OVERLAY % {'buildroot': buildroot}
+  private_overlay = PRIVATE_OVERLAY % {'buildroot': buildroot}
+  if url.endswith('manifest-internal'):
+    overlay = PRIVATE_OVERLAY % {'buildroot': buildroot}
+  else:
+    overlay = PUBLIC_OVERLAY % {'buildroot': buildroot}
+
+  # Generate paths for manifests.
+  relative_path = _GetManifestPath(buildroot)
+  manifest_path = os.path.join(overlay, 'manifests', relative_path)
+  symlink_path = os.path.join(overlay, 'manifests', 'LATEST')
+  if not os.path.isdir(os.path.dirname(manifest_path)):
+    os.makedirs(os.path.dirname(manifest_path))
+
+  # Dump the manifest and create a symlink to it.
+  RunCommand(['repo', 'manifest', '-r', '-o', manifest_path], cwd=buildroot)
+  if os.path.exists(symlink_path):
+    os.unlink(symlink_path)
+
+  os.symlink(relative_path, symlink_path)
+
+  # Add it to git and print it to stderr.
+  RunCommand(['git', 'add', os.path.join('manifests', relative_path)],
+             cwd=overlay)
+  RunCommand(['git', 'add', os.path.join('manifests', 'LATEST')], cwd=overlay)
+  _PrintFile(manifest_path)
+
+
 def _MakeChroot(buildroot):
   """Wrapper around make_chroot."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
@@ -527,8 +589,8 @@ def _ResolveOverlays(buildroot, overlays):
               'public': Just the public overlay.
               'both': Both the public and private overlays.
   """
-  public_overlay = '%s/src/third_party/chromiumos-overlay' % buildroot
-  private_overlay = '%s/src/private-overlays/chromeos-overlay' % buildroot
+  public_overlay = PUBLIC_OVERLAY % {'buildroot': buildroot}
+  private_overlay = PRIVATE_OVERLAY % {'buildroot': buildroot}
   if overlays == 'private':
     paths = [private_overlay]
   elif overlays == 'public':
@@ -657,6 +719,8 @@ def main():
       if not os.path.isdir(path):
         Die('Missing overlay: %s' % path)
 
+    _DumpManifest(buildroot, options.url)
+
     if not os.path.isdir(chroot_path):
       _MakeChroot(buildroot)
 

bin/cbuildbot_unittest.py

@@ -11,6 +11,7 @@ import mox
 import os
 import posix
 import shutil
+import tempfile
 import unittest
 
 # Fixes circular dependency error.
@@ -18,6 +19,7 @@ import cbuildbot_comm
 import cbuildbot
 from cros_build_lib import ReinterpretPathForChroot
 
+
 class CBuildBotTest(mox.MoxTestBase):
 
   def setUp(self):
@@ -117,7 +119,7 @@ class CBuildBotTest(mox.MoxTestBase):
     buildroot = '/fake_dir'
     board = 'fake-board'
     test_results_dir = 'fake_results_dir'
-    gsutil_path='/fake/gsutil/path'
+    gsutil_path = '/fake/gsutil/path'
     archive_dir = 1234
     acl = 'fake_acl'
     num_retries = 5
@@ -230,6 +232,68 @@ class CBuildBotTest(mox.MoxTestBase):
                                binhosts)
     self.mox.VerifyAll()
 
+  def testGetChromeOSVersion(self):
+    """Tests that we can parse the correct chromeos version using method."""
+    fake_data = '\n'.join(['ChromeOS version information:',
+                         '  SOME_OTHER_DATA=blah',
+                         '  CHROMEOS_VERSION_STRING=9.0.232.1',
+                         '  CHROMEOS_VERSION_CODENAME=test-bot',
+                        ])
+    cbuildbot.RunCommand('./chromeos_version.sh',
+                         cwd='src/scripts',
+                         redirect_stderr=True,
+                         redirect_stdout=True).AndReturn(fake_data)
+    self.mox.ReplayAll()
+    return_tuple = cbuildbot._GetChromeOSVersion('')
+    self.assertEquals('.'.join(return_tuple), '9.0.232.1')
+    self.mox.VerifyAll()
+
+  def testGetManifestPath(self):
+    """Tests whether our logic to get the manifest path is correct."""
+    self.mox.StubOutWithMock(cbuildbot, '_GetChromeOSVersion')
+    return_tuple = cbuildbot._GetChromeOSVersion('').AndReturn(
+        ('9', '0', '232', '1'))
+    self.mox.ReplayAll()
+    relative_path = cbuildbot._GetManifestPath('')
+    self.assertEquals(relative_path, '9.0/9.0.232.1.xml')
+    self.mox.VerifyAll()
+
+  def _CommonManifestTest(self, url, overlay_no_buildroot):
+    """Common method for dump manifest tests."""
+    self.mox.StubOutWithMock(cbuildbot, '_GetManifestPath')
+    self.mox.StubOutWithMock(shutil, 'copy')
+    self.mox.StubOutWithMock(os, 'symlink')
+    temp_root = tempfile.mkdtemp('_unittest')
+
+    overlay = overlay_no_buildroot % {'buildroot': temp_root}
+    relative_path = 'fake/manifest/path.xml'
+    full_path = os.path.join(overlay, 'manifests', relative_path)
+
+    cbuildbot._GetManifestPath(temp_root).AndReturn(relative_path)
+    cbuildbot.RunCommand(['repo', 'manifest', '-r', '-o', full_path],
+                         cwd=temp_root)
+    os.symlink(relative_path, os.path.join(overlay, 'manifests', 'LATEST'))
+    cbuildbot.RunCommand(['git', 'add', 'manifests/' + relative_path],
+                         cwd=overlay)
+    cbuildbot.RunCommand(['git', 'add', 'manifests/LATEST'], cwd=overlay)
+    shutil.copy(full_path, '/dev/stderr')
+
+    self.mox.ReplayAll()
+    cbuildbot._DumpManifest(temp_root, url)
+    self.mox.VerifyAll()
+
+    shutil.rmtree(temp_root)
+
+  def testDumpManifestPublic(self):
+    """Tests whether we push the manifest to the public overlay correctly."""
+    self._CommonManifestTest('http://some_url/manifest',
+                             cbuildbot.PUBLIC_OVERLAY)
+
+  def testDumpManifestPrivate(self):
+    """Tests whether we push the manifest to the private overlay correctly."""
+    self._CommonManifestTest('http://some_url/manifest-internal',
+                             cbuildbot.PRIVATE_OVERLAY)
+
 
 if __name__ == '__main__':
   unittest.main()