CBuildbot - Adds ability to only rev packages given by the sourcestamp.

The buildbot drops a source stamp (repo1.git@commit1 repo2.git@commit2 ...).
If CBuildbot finds the source stamp, it uses it and revs only those packages.
If this file is either missing or set to "None", it assumes a Force Build
has been pressed and keeps the old behavior (attempts to mark all).

TEST=Ran with 3 states of revisions file.
BUG=5006

Review URL: http://codereview.chromium.org/3163030

Change-Id: I226fd3bec642224b31ce51eee34d028043964943

bin/cbuildbot.py

@@ -4,9 +4,12 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+"""CBuildbot is wrapper around the build process used by the pre-flight queue"""
+
 import errno
 import optparse
 import os
+import re
 import shutil
 import subprocess
 import sys
@@ -15,26 +18,40 @@ from cbuildbot_config import config
 
 _DEFAULT_RETRIES = 3
 
-# Utility functions
+# ======================== Utility functions ================================
 
 def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None,
                exit_code=False, redirect_stdout=False, redirect_stderr=False,
-               cwd=None, input=None):
+               cwd=None, input=None, enter_chroot=False):
+  """Runs a shell command.
+
+  Keyword arguments:
+  print_cmd -- prints the command before running it.
+  error_ok -- does not raise an exception on error.
+  error_message -- prints out this message when an error occurrs.
+  exit_code -- returns the return code of the shell command.
+  redirect_stdout -- returns the stdout.
+  redirect_stderr -- holds stderr output until input is communicated.
+  cwd -- the working directory to run this cmd.
+  input -- input to pipe into this command through stdin.
+  enter_chroot -- this command should be run from within the chroot.
+
+  """
+  # Set default for variables.
+  stdout = None
+  stderr = None
+  stdin = None
+
+  # Modify defaults based on parameters.
+  if redirect_stdout:  stdout = subprocess.PIPE
+  if redirect_stderr:  stderr = subprocess.PIPE
+  if input:  stdin = subprocess.PIPE
+  if enter_chroot:  cmd = ['./enter_chroot.sh', '--'] + cmd
+
   # Print out the command before running.
   if print_cmd:
-    print >> sys.stderr, "CBUILDBOT -- RunCommand:", ' '.join(cmd)
+    print >> sys.stderr, 'CBUILDBOT -- RunCommand: ', ' '.join(cmd)
-  if redirect_stdout:
+
-    stdout = subprocess.PIPE
-  else:
-    stdout = None
-  if redirect_stderr:
-    stderr = subprocess.PIPE
-  else:
-    stderr = None
-  if input:
-    stdin = subprocess.PIPE
-  else:
-      stdin = None
   proc = subprocess.Popen(cmd, cwd=cwd, stdin=stdin,
                           stdout=stdout, stderr=stderr)
   (output, error) = proc.communicate(input)
@@ -45,7 +62,15 @@ def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None,
                     (error_message or error or output or ''))
   return output
 
+
 def MakeDir(path, parents=False):
+  """Basic wrapper around os.mkdirs.
+
+  Keyword arguments:
+  path -- Path to create.
+  parents -- Follow mkdir -p logic.
+
+  """
   try:
     os.makedirs(path)
   except OSError, e:
@@ -54,7 +79,15 @@ def MakeDir(path, parents=False):
     else:
       raise
 
-def RepoSync(buildroot, rw_checkout, retries=_DEFAULT_RETRIES):
+
+def RepoSync(buildroot, rw_checkout=False, retries=_DEFAULT_RETRIES):
+  """Uses repo to checkout the source code.
+
+  Keyword arguments:
+  rw_checkout -- Reconfigure repo after sync'ing to read-write.
+  retries -- Number of retries to try before failing on the sync.
+
+  """
   while retries > 0:
     try:
       RunCommand(['repo', 'sync'], cwd=buildroot)
@@ -73,70 +106,221 @@ def RepoSync(buildroot, rw_checkout, retries=_DEFAULT_RETRIES):
         print >> sys.stderr, 'CBUILDBOT -- Retries exhausted'
         raise
 
-# Main functions
+# =========================== Command Helpers =================================
+
+def _GetAllGitRepos(buildroot, debug=False):
+  """Returns a list of tuples containing [git_repo, src_path]."""
+  manifest_tuples = []
+  # Gets all the git repos from a full repo manifest.
+  repo_cmd = "repo manifest -o -".split()
+  output = RunCommand(repo_cmd, cwd=buildroot, redirect_stdout=True,
+                      redirect_stderr=True, print_cmd=debug)
+
+  # Extract all lines containg a project.
+  extract_cmd = ["grep", "project name="]
+  output = RunCommand(extract_cmd, cwd=buildroot, input=output,
+                      redirect_stdout=True, print_cmd=debug)
+  # Parse line using re to get tuple.
+  result_array = re.findall('.+name=\"([\w-]+)\".+path=\"(\S+)".+', output)
+
+  # Create the array.
+  for result in result_array:
+    if len(result) != 2:
+      print >> sys.stderr, 'Found in correct xml object %s', result
+    else:
+      # Remove pre-pended src directory from manifest.
+      manifest_tuples.append([result[0], result[1].replace('src/', '')])
+  return manifest_tuples
+
+
+def _GetCrosWorkOnSrcPath(buildroot, board, package, debug=False):
+  """Returns ${CROS_WORKON_SRC_PATH} for given package."""
+  cwd = os.path.join(buildroot, 'src', 'scripts')
+  equery_cmd = ('equery-%s which %s' % (board, package)).split()
+  ebuild_path = RunCommand(equery_cmd, cwd=cwd, redirect_stdout=True,
+                             redirect_stderr=True, enter_chroot=True,
+                             error_ok=True, print_cmd=debug)
+  if ebuild_path:
+    ebuild_cmd = ('ebuild-%s %s info' % (board, ebuild_path)).split()
+    cros_workon_output = RunCommand(ebuild_cmd, cwd=cwd,
+                                    redirect_stdout=True, redirect_stderr=True,
+                                    enter_chroot=True, print_cmd=debug)
+
+    temp = re.findall('CROS_WORKON_SRCDIR="(\S+)"', cros_workon_output)
+    if temp:
+      return temp[0]
+  return None
+
+
+def _CreateRepoDictionary(buildroot, board, debug=False):
+  """Returns the repo->list_of_ebuilds dictionary."""
+  repo_dictionary = {}
+  manifest_tuples = _GetAllGitRepos(buildroot)
+  print >> sys.stderr, 'Creating dictionary of git repos to portage packages ...'
+
+  cwd = os.path.join(buildroot, 'src', 'scripts')
+  get_all_workon_pkgs_cmd = './cros_workon list --all'.split()
+  packages = RunCommand(get_all_workon_pkgs_cmd, cwd=cwd,
+                        redirect_stdout=True, redirect_stderr=True,
+                        enter_chroot=True, print_cmd=debug)
+  for package in packages.split():
+    cros_workon_src_path = _GetCrosWorkOnSrcPath(buildroot, board, package)
+    if cros_workon_src_path:
+      for tuple in manifest_tuples:
+        # This path tends to have the user's home_dir prepended to it.
+        if cros_workon_src_path.endswith(tuple[1]):
+          print >> sys.stderr, ('For %s found matching package %s' %
+              (tuple[0], package))
+          if repo_dictionary.has_key(tuple[0]):
+            repo_dictionary[tuple[0]] += [package]
+          else:
+            repo_dictionary[tuple[0]] = [package]
+  return repo_dictionary
+
+
+def _ParseRevisionString(revision_string, repo_dictionary):
+  """Parses the given revision_string into a revision dictionary.
+
+  Returns a list of tuples that contain [portage_package_name, commit_id] to
+  update.
+
+  Keyword arguments:
+  revision_string -- revision_string with format
+      'repo1.git@commit_1 repo2.git@commit2 ...'.
+  repo_dictionary -- dictionary with git repository names as keys (w/out git)
+      to portage package names.
+
+  """
+  # Using a dictionary removes duplicates.
+  revisions = {}
+  for revision in revision_string.split():
+    # Format 'package@commit-id'.
+    revision_tuple = revision.split('@')
+    if len(revision_tuple) != 2:
+      print >> sys.stderr, 'Incorrectly formatted revision %s' % revision
+    repo_name = revision_tuple[0].replace('.git', '')
+    # May be many corresponding packages to a given git repo e.g. kernel)
+    for package in repo_dictionary[repo_name]:
+      revisions[package] = revision_tuple[1]
+  return revisions.items()
+
+
+def _UprevFromRevisionList(buildroot, revision_list):
+  """Uprevs based on revision list."""
+  package_str = ''
+  commit_str = ''
+  for package, revision in revision_list:
+    package_str += package + ' '
+    commit_str += revision + ' '
+  package_str = package_str.strip()
+  commit_str = commit_str.strip()
+
+  cwd = os.path.join(buildroot, 'src', 'scripts')
+  RunCommand(['./cros_mark_as_stable',
+              '--tracking_branch="cros/master"',
+              '--packages="%s"' % package_str,
+              '--commit_ids="%s"' % commit_str,
+              'commit'],
+              cwd=cwd, enter_chroot=True)
+
+
+def _UprevAllPackages(buildroot):
+  """Uprevs all packages that have been updated since last uprev."""
+  cwd = os.path.join(buildroot, 'src', 'scripts')
+  RunCommand(['./cros_mark_all_as_stable',
+              '--tracking_branch="cros/master"'],
+              cwd=cwd, enter_chroot=True)
+
+# =========================== Main Commands ===================================
 
 def _FullCheckout(buildroot, rw_checkout=True, retries=_DEFAULT_RETRIES):
+  """Performs a full checkout and clobbers any previous checkouts."""
   RunCommand(['sudo', 'rm', '-rf', buildroot])
   MakeDir(buildroot, parents=True)
   RunCommand(['repo', 'init', '-u', 'http://src.chromium.org/git/manifest'],
              cwd=buildroot, input='\n\ny\n')
   RepoSync(buildroot, rw_checkout, retries)
 
+
 def _IncrementalCheckout(buildroot, rw_checkout=True,
                          retries=_DEFAULT_RETRIES):
+  """Performs a checkout without clobbering previous checkout."""
   RepoSync(buildroot, rw_checkout, retries)
 
+
 def _MakeChroot(buildroot):
+  """Wrapper around make_chroot."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
   RunCommand(['./make_chroot', '--fast'], cwd=cwd)
 
+
 def _SetupBoard(buildroot, board='x86-generic'):
+  """Wrapper around setup_board."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
   RunCommand(['./setup_board', '--fast', '--default', '--board=%s' % board],
              cwd=cwd)
 
+
 def _Build(buildroot):
+  """Wrapper around build_packages."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
   RunCommand(['./build_packages'], cwd=cwd)
 
-def _UprevAllPackages(buildroot):
-  cwd = os.path.join(buildroot, 'src', 'scripts')
-  RunCommand(['./enter_chroot.sh', '--', './cros_mark_all_as_stable',
-              '--tracking_branch="cros/master"'],
-             cwd=cwd)
 
-def _UprevPackages(buildroot, revisionfile):
+def _UprevPackages(buildroot, revisionfile, board):
-  revisions = None
+  """Uprevs a package based on given revisionfile.
+
+  If revisionfile is set to None or does not resolve to an actual file, this
+  function will uprev all packages.
+
+  Keyword arguments:
+  revisionfile -- string specifying a file that contains a list of revisions to
+      uprev.
+  """
+  # Purposefully set to None as it means Force Build was pressed.
+  revisions = 'None'
   if (revisionfile):
     try:
       rev_file = open(revisionfile)
       revisions = rev_file.read()
       rev_file.close()
-    except:
+    except Exception, e:
-      print >> sys.stderr, 'Error reading %s' % revisionfile
+      print >> sys.stderr, 'Error reading %s, revving all' % revisionfile
-      revisions = None
+      print e
+      revisions = 'None'
 
-  # Note:  Revisions == "None" indicates a Force Build.
+  revisions = revisions.strip()
-  if revisions and revisions != 'None':
-    print 'CBUILDBOT - Revision list found %s' % revisions
-    print 'Revision list not yet propagating to build, marking all instead'
 
+  # Revisions == "None" indicates a Force Build.
+  if revisions != 'None':
+    print >> sys.stderr, 'CBUILDBOT Revision list found %s' % revisions
+    revision_list = _ParseRevisionString(revisions,
+        _CreateRepoDictionary(buildroot, board))
+    _UprevFromRevisionList(buildroot, revision_list)
+  else:
+    print >> sys.stderr, 'CBUILDBOT Revving all'
     _UprevAllPackages(buildroot)
 
+
 def _UprevCleanup(buildroot):
+  """Clean up after a previous uprev attempt."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
   RunCommand(['./cros_mark_as_stable', '--srcroot=..',
               '--tracking_branch="cros/master"', 'clean'],
              cwd=cwd)
 
+
 def _UprevPush(buildroot):
+  """Pushes uprev changes to the main line."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
   RunCommand(['./cros_mark_as_stable', '--srcroot=..',
               '--tracking_branch="cros/master"',
               '--push_options', '--bypass-hooks -f', 'push'],
              cwd=cwd)
 
+
 def _GetConfig(config_name):
+  """Gets the configuration for the build"""
   default = config['default']
   buildconfig = {}
   if config.has_key(config_name):
@@ -146,6 +330,7 @@ def _GetConfig(config_name):
       buildconfig[key] = default[key]
   return buildconfig
 
+
 def main():
   # Parse options
   usage = "usage: %prog [options] cbuildbot_config"
@@ -183,7 +368,7 @@ def main():
     if not os.path.isdir(boardpath):
       _SetupBoard(buildroot, board=buildconfig['board'])
     if buildconfig['uprev']:
-      _UprevPackages(buildroot, revisionfile)
+      _UprevPackages(buildroot, revisionfile, board=buildconfig['board'])
     _Build(buildroot)
     if buildconfig['uprev']:
       _UprevPush(buildroot)
@@ -194,5 +379,6 @@ def main():
       RunCommand(['sudo', 'rm', '-rf', buildroot], print_cmd=False)
     raise
 
+
 if __name__ == '__main__':
     main()

bin/cbuildbot_unittest.py

@@ -0,0 +1,102 @@
+#!/usr/bin/python
+
+# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Unittests for cbuildbot.  Needs to be run inside of chroot for mox."""
+
+import __builtin__
+import mox
+import unittest
+
+import cbuildbot
+
+class CBuildBotTest(mox.MoxTestBase):
+
+  def setUp(self):
+    mox.MoxTestBase.setUp(self)
+    # Always stub RunCommmand out as we use it in every method.
+    self.mox.StubOutWithMock(cbuildbot, 'RunCommand')
+    self._test_repos = [['kernel', 'third_party/kernel/files'],
+                        ['login_manager', 'platform/login_manager']
+                       ]
+    self._test_cros_workon_packages = \
+      'chromeos-base/kernel\nchromeos-base/chromeos-login\n'
+    self._test_board = 'test-board'
+    self._buildroot = '.'
+    self._test_dict = {'kernel' : ['chromos-base/kernel', 'dev-util/perf'],
+                       'cros' : ['chromos-base/libcros']
+                      }
+    self._test_string = "kernel.git@12345test cros.git@12333test"
+    self._revision_file = 'test-revisions.pfq'
+    self._test_parsed_string_array = [
+                                      ['chromeos-base/kernel', '12345test'],
+                                      ['dev-util/perf', '12345test'],
+                                      ['chromos-base/libcros', '12345test']
+                                     ]
+
+  def testParseRevisionString(self):
+    """Test whether _ParseRevisionString parses string correctly."""
+    return_array = cbuildbot._ParseRevisionString(self._test_string,
+                                                  self._test_dict)
+    self.assertEqual(len(return_array), 3)
+    self.assertTrue(
+      'chromeos-base/kernel', '12345test' in return_array)
+    self.assertTrue(
+      'dev-util/perf', '12345test' in return_array)
+    self.assertTrue(
+      'chromos-base/libcros', '12345test' in return_array)
+
+  def testCreateDictionary(self):
+    self.mox.StubOutWithMock(cbuildbot, '_GetAllGitRepos')
+    self.mox.StubOutWithMock(cbuildbot, '_GetCrosWorkOnSrcPath')
+    cbuildbot._GetAllGitRepos(mox.IgnoreArg()).AndReturn(self._test_repos)
+    cbuildbot.RunCommand(mox.IgnoreArg(),
+                         cwd='%s/src/scripts' % self._buildroot,
+                         redirect_stdout=True,
+                         redirect_stderr=True,
+                         enter_chroot=True,
+                         print_cmd=False).AndReturn(
+                             self._test_cros_workon_packages)
+    cbuildbot._GetCrosWorkOnSrcPath(self._buildroot, self._test_board,
+                                    'chromeos-base/kernel').AndReturn(
+        '/home/test/third_party/kernel/files')
+    cbuildbot._GetCrosWorkOnSrcPath(self._buildroot, self._test_board,
+                                    'chromeos-base/chromeos-login').AndReturn(
+        '/home/test/platform/login_manager')
+    self.mox.ReplayAll()
+    repo_dict = cbuildbot._CreateRepoDictionary(self._buildroot,
+                                                self._test_board)
+    self.assertEqual(repo_dict['kernel'], ['chromeos-base/kernel'])
+    self.assertEqual(repo_dict['login_manager'],
+                     ['chromeos-base/chromeos-login'])
+    self.mox.VerifyAll()
+
+  def testUprevPackages(self):
+    self.mox.StubOutWithMock(cbuildbot, '_CreateRepoDictionary')
+    self.mox.StubOutWithMock(cbuildbot, '_ParseRevisionString')
+    self.mox.StubOutWithMock(cbuildbot, '_UprevFromRevisionList')
+    self.mox.StubOutWithMock(__builtin__, 'open')
+
+    # Mock out file interaction.
+    m_file = self.mox.CreateMock(file)
+    __builtin__.open(self._revision_file).AndReturn(m_file)
+    m_file.read().AndReturn(self._test_string)
+    m_file.close()
+
+    cbuildbot._CreateRepoDictionary(self._buildroot,
+                                    self._test_board).AndReturn(self._test_dict)
+    cbuildbot._ParseRevisionString(self._test_string,
+                                   self._test_dict).AndReturn(
+                                       self._test_parsed_string_array)
+    cbuildbot._UprevFromRevisionList(self._buildroot,
+                                     self._test_parsed_string_array)
+    self.mox.ReplayAll()
+    cbuildbot._UprevPackages(self._buildroot, self._revision_file,
+                             self._test_board)
+    self.mox.VerifyAll()
+
+
+if __name__ == '__main__':
+  unittest.main()