This CL adds the ability to Rev and Build Chrome using cros_mark_chrome_as_stable.

Other things:
Removing commit|clean|push command from cros_mark_chrome because it's unnecessary ... I'm just gonna use the same stabilizing branch as the other stable marker, so I can use the same clean / push from the other script.  This makes this change fit in much more nicely to cbuildbot.

Other changes:
Add ability for cros_mark_chrome_as_stable to communicate to calling script through stdout.
Make STABLE_BRANCH in cros_mark_as_stable public for above reason.
Removed push_options from cros_mark and accompanying change to cbuildbot as it isn't used anymore.
Made it easier to debug with cbuildbot (I ran cbuildbot A LOT of times to test this change so I felt the pain)...this includes:
  - only build when syncing
  - allow one to explicitly disable tests
  - use dryrun is options.debug is set

Change-Id: I413a2e81a99cdde2e4d9139561cd518245b9b346

BUG=chromium-os:8693
TEST=Ran cbuildbot with --notest --debug --nosync x86-pre-flight and
saw it go through correctly and dryrun push.  Ran cbuildbot with same and --chrome_rev=tot.  Ran with syncing and running tests too.

Review URL: http://codereview.chromium.org/5154008

bin/cbuildbot.py

@@ -209,6 +209,20 @@ def _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board,
              cwd=cwd, enter_chroot=True)
 
 
+def _MarkChromeAsStable(buildroot, tracking_branch, chrome_rev):
+  """Returns the portage atom for the revved chrome ebuild - see man emerge."""
+  cwd = os.path.join(buildroot, 'src', 'scripts')
+  portage_atom_string = RunCommand(['bin/cros_mark_chrome_as_stable',
+                                    '--tracking_branch=%s' % tracking_branch,
+                                    chrome_rev], cwd=cwd, redirect_stdout=True,
+                                    enter_chroot=True).rstrip()
+  if not portage_atom_string:
+    Info('Found nothing to rev.')
+    return None
+  else:
+    return portage_atom_string.split('=')[1]
+
+
 def _UprevAllPackages(buildroot, tracking_branch, board, overlays):
   """Uprevs all packages that have been updated since last uprev."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
@@ -311,6 +325,13 @@ def _Build(buildroot):
   RunCommand(['./build_packages'], cwd=cwd, enter_chroot=True)
 
 
+def _BuildChrome(buildroot, board, chrome_atom_to_build):
+  """Wrapper for emerge call to build Chrome."""
+  cwd = os.path.join(buildroot, 'src', 'scripts')
+  RunCommand(['emerge-%s' % board, '=%s' % chrome_atom_to_build],
+             cwd=cwd, enter_chroot=True)
+
+
 def _EnableLocalAccount(buildroot):
   cwd = os.path.join(buildroot, 'src', 'scripts')
   # Set local account for test images.
@@ -395,15 +416,20 @@ def _UprevPackages(buildroot, tracking_branch, revisionfile, board, overlays):
   _UprevAllPackages(buildroot, tracking_branch, board, overlays)
 
 
-def _UprevPush(buildroot, tracking_branch, board, overlays):
+def _UprevPush(buildroot, tracking_branch, board, overlays, dryrun):
   """Pushes uprev changes to the main line."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
-  RunCommand(['./cros_mark_as_stable', '--srcroot=..',
+  cmd = ['./cros_mark_as_stable',
-              '--board=%s' % board,
+         '--srcroot=%s' % os.path.join(buildroot, 'src'),
-              '--overlays=%s' % ':'.join(overlays),
+         '--board=%s' % board,
-              '--tracking_branch=%s' % tracking_branch,
+         '--overlays=%s' % ':'.join(overlays),
-              '--push_options=--bypass-hooks -f', 'push'],
+         '--tracking_branch=%s' % tracking_branch
-             cwd=cwd)
+        ]
+  if dryrun:
+    cmd.append('--dryrun')
+
+  cmd.append('push')
+  RunCommand(cmd, cwd=cwd)
 
 
 def _ArchiveTestResults(buildroot, board, test_results_dir,
@@ -496,6 +522,10 @@ def main():
                     help='root directory where build occurs', default=".")
   parser.add_option('-n', '--buildnumber',
                     help='build number', type='int', default=0)
+  parser.add_option('--chrome_rev', default=None, type='string',
+                    dest='chrome_rev',
+                    help=('Chrome_rev of type [tot|latest_release|'
+                          'sticky_release]'))
   parser.add_option('-f', '--revisionfile',
                     help='file where new revisions are stored')
   parser.add_option('--clobber', action='store_true', dest='clobber',
@@ -504,6 +534,12 @@ def main():
   parser.add_option('--debug', action='store_true', dest='debug',
                     default=False,
                     help='Override some options to run as a developer.')
+  parser.add_option('--nosync', action='store_false', dest='sync',
+                    default=True,
+                    help="Don't sync before building.")
+  parser.add_option('--notests', action='store_false', dest='tests',
+                    default=True,
+                    help='Override values from buildconfig and run no tests.')
   parser.add_option('-t', '--tracking-branch', dest='tracking_branch',
                     default='cros/master', help='Run the buildbot on a branch')
   parser.add_option('-u', '--url', dest='url',
@@ -520,6 +556,7 @@ def main():
   buildroot = os.path.abspath(options.buildroot)
   revisionfile = options.revisionfile
   tracking_branch = options.tracking_branch
+  chrome_atom_to_build = None
 
   if len(args) >= 1:
     buildconfig = _GetConfig(args[-1])
@@ -533,10 +570,11 @@ def main():
 
   try:
     _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays)
-    if options.clobber or not os.path.isdir(buildroot):
+    if options.sync:
-      _FullCheckout(buildroot, tracking_branch, url=options.url)
+      if options.clobber or not os.path.isdir(buildroot):
-    else:
+        _FullCheckout(buildroot, tracking_branch, url=options.url)
-      _IncrementalCheckout(buildroot)
+      else:
+        _IncrementalCheckout(buildroot)
 
     # Check that all overlays can be found.
     for path in overlays:
@@ -552,18 +590,28 @@ def main():
     if not os.path.isdir(boardpath):
       _SetupBoard(buildroot, board=buildconfig['board'])
 
-    if buildconfig['uprev']:
+    # Perform uprev.  If chrome_uprev is set, rev Chrome ebuilds.
+    if options.chrome_rev:
+      chrome_atom_to_build = _MarkChromeAsStable(buildroot, tracking_branch,
+                                                 options.chrome_rev)
+    elif buildconfig['uprev']:
       _UprevPackages(buildroot, tracking_branch, revisionfile,
                      buildconfig['board'], overlays)
 
     _EnableLocalAccount(buildroot)
-    _Build(buildroot)
+    # Doesn't rebuild without acquiring more source.
-    if buildconfig['unittests']:
+    if options.sync:
+      _Build(buildroot)
+
+    if chrome_atom_to_build:
+      _BuildChrome(buildroot, buildconfig['board'], chrome_atom_to_build)
+
+    if buildconfig['unittests'] and options.tests:
       _RunUnitTests(buildroot)
 
     _BuildImage(buildroot)
 
-    if buildconfig['smoke_bvt']:
+    if buildconfig['smoke_bvt'] and options.tests:
       _BuildVMImageForTesting(buildroot)
       test_results_dir = '/tmp/run_remote_tests.%s' % options.buildnumber
       try:
@@ -580,19 +628,18 @@ def main():
 
     if buildconfig['uprev']:
       # Don't push changes for developers.
-      if not options.debug:
+      if buildconfig['master']:
-        if buildconfig['master']:
+        # Master bot needs to check if the other slaves completed.
-          # Master bot needs to check if the other slaves completed.
+        if cbuildbot_comm.HaveSlavesCompleted(config):
-          if cbuildbot_comm.HaveSlavesCompleted(config):
+          _UprevPush(buildroot, tracking_branch, buildconfig['board'],
-            _UprevPush(buildroot, tracking_branch, buildconfig['board'],
+                     overlays, options.debug)
-                       overlays)
-          else:
-            Die('CBUILDBOT - One of the slaves has failed!!!')
-
         else:
-          # Publish my status to the master if its expecting it.
+          Die('CBUILDBOT - One of the slaves has failed!!!')
-          if buildconfig['important']:
+
-            cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_COMPLETE)
+      else:
+        # Publish my status to the master if its expecting it.
+        if buildconfig['important'] and not options.debug:
+          cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_COMPLETE)
 
   except:
     # Send failure to master bot.

bin/cros_mark_chrome_as_stable.py

@@ -1,10 +1,20 @@
-#!/usr/bin/python2.4
+#!/usr/bin/python
 
 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-"""This module uprevs Chrome for cbuildbot."""
+"""This module uprevs Chrome for cbuildbot.
+
+After calling, it prints outs CHROME_VERSION_ATOM=(version atom string).  A
+caller could then use this atom with emerge to build the newly uprevved version
+of Chrome e.g.
+
+./cros_mark_chrome_as_stable tot
+Returns chrome-base/chromeos-chrome-8.0.552.0_alpha_r1
+
+emerge-x86-generic =chrome-base/chromeos-chrome-8.0.552.0_alpha_r1
+"""
 
 import optparse
 import os
@@ -32,9 +42,6 @@ _NON_STICKY_REGEX = '%s[(_rc.*)|(_alpha.*)]+' % _CHROME_VERSION_REGEX
 _CHROME_OVERLAY_DIR = ('%(srcroot)s/third_party/chromiumos-overlay'
                        '/chromeos-base/chromeos-chrome')
 
-# Different than cros_mark so devs don't have local collisions.
-_STABLE_BRANCH_NAME = 'chrome_stabilizing_branch'
-
 _GIT_COMMIT_MESSAGE = ('Marking %(chrome_rev)s for chrome ebuild with version '
                        '%(chrome_version)s as stable.')
 
@@ -235,6 +242,8 @@ def MarkChromeEBuildAsStable(stable_candidate, unstable_ebuild, chrome_rev,
     chrome_version:  The \d.\d.\d.\d version of Chrome.
     commit:  Used with TIP_OF_TRUNK.  The svn revision of chrome.
     overlay_dir:  Path to the chromeos-chrome package dir.
+  Returns:
+    Full portage version atom (including rc's, etc) that was revved.
   """
   base_path = os.path.join(overlay_dir, 'chromeos-chrome-%s' % chrome_version)
   # Case where we have the last stable candidate with same version just rev.
@@ -260,38 +269,26 @@ def MarkChromeEBuildAsStable(stable_candidate, unstable_ebuild, chrome_rev,
       _GIT_COMMIT_MESSAGE % {'chrome_rev': chrome_rev,
                              'chrome_version': chrome_version})
 
+  new_ebuild = ChromeEBuild(new_ebuild_path)
+  return '%s-%s' % (new_ebuild.package, new_ebuild.version)
 
-def main(argv):
+
-  usage = '%s OPTIONS commit|clean|push'
+def main():
+  usage = '%s OPTIONS [%s]' % (__file__, '|'.join(CHROME_REV))
   parser = optparse.OptionParser(usage)
-  parser.add_option('-c', '--chrome_rev', default=None,
+  parser.add_option('-s', '--srcroot', default=os.path.join(os.environ['HOME'],
-                    help='One of %s' % CHROME_REV)
+                                                            'trunk', 'src'),
-  parser.add_option('-s', '--srcroot', default='.',
                     help='Path to the src directory')
   parser.add_option('-t', '--tracking_branch', default='cros/master',
                     help='Branch we are tracking changes against')
-  (options, argv) = parser.parse_args(argv)
+  (options, args) = parser.parse_args()
 
-  if len(argv) != 2 or argv[1] not in (
+  if len(args) != 1 or args[0] not in CHROME_REV:
-      cros_mark_as_stable.COMMAND_DICTIONARY.keys()):
+    parser.error('Commit requires arg set to one of %s.' % CHROME_REV)
-    parser.error('Arguments are invalid, see usage.')
 
-  command = argv[1]
   overlay_dir = os.path.abspath(_CHROME_OVERLAY_DIR %
                                 {'srcroot': options.srcroot})
-
+  chrome_rev = args[0]
-  os.chdir(overlay_dir)
-  if command == 'clean':
-    cros_mark_as_stable.Clean(options.tracking_branch)
-    return
-  elif command == 'push':
-    cros_mark_as_stable.PushChange(_STABLE_BRANCH_NAME, options.tracking_branch)
-    return
-
-  if not options.chrome_rev or options.chrome_rev not in CHROME_REV:
-    parser.error('Commit requires type set to one of %s.' % CHROME_REV)
-
-  chrome_rev = options.chrome_rev
   version_to_uprev = None
   commit_to_use = None
 
@@ -299,6 +296,7 @@ def main(argv):
   sticky_version = _GetStickyVersion(stable_ebuilds)
   sticky_branch = sticky_version.rpartition('.')[0]
 
+
   if chrome_rev == TIP_OF_TRUNK:
     version_to_uprev = _GetTipOfTrunkVersion()
     commit_to_use = _GetTipOfTrunkSvnRevision()
@@ -315,16 +313,20 @@ def main(argv):
     Info('Found nothing to do for chrome_rev %s with version %s.' % (
         chrome_rev, version_to_uprev))
   else:
+    os.chdir(overlay_dir)
     work_branch = cros_mark_as_stable.GitBranch(
-        _STABLE_BRANCH_NAME, options.tracking_branch)
+        cros_mark_as_stable.STABLE_BRANCH_NAME, options.tracking_branch)
     work_branch.CreateBranch()
     try:
-      MarkChromeEBuildAsStable(stable_candidate, unstable_ebuild, chrome_rev,
+      chrome_version_atom = MarkChromeEBuildAsStable(
-                               version_to_uprev, commit_to_use, overlay_dir)
+          stable_candidate, unstable_ebuild, chrome_rev, version_to_uprev,
+          commit_to_use, overlay_dir)
+      # Explicit print to communicate to caller.
+      print 'CHROME_VERSION_ATOM=%s' % chrome_version_atom
     except:
       work_branch.Delete()
       raise
 
 
 if __name__ == '__main__':
-  main(sys.argv)
+  main()

cros_mark_as_stable.py

@@ -32,8 +32,6 @@ gflags.DEFINE_string('overlays', '',
 gflags.DEFINE_string('packages', '',
                      'Colon-separated list of packages to mark as stable.',
                      short_name='p')
-gflags.DEFINE_string('push_options', '',
-                     'Options to use with git-cl push using push command.')
 gflags.DEFINE_string('srcroot', '%s/trunk/src' % os.environ['HOME'],
                      'Path to root src directory.',
                      short_name='r')
@@ -59,7 +57,7 @@ COMMAND_DICTIONARY = {
                       }
 
 # Name used for stabilizing branch.
-_STABLE_BRANCH_NAME = 'stabilizing_branch'
+STABLE_BRANCH_NAME = 'stabilizing_branch'
 
 
 def BestEBuild(ebuilds):
@@ -544,9 +542,9 @@ def main(argv):
     if command == 'clean':
       Clean(gflags.FLAGS.tracking_branch)
     elif command == 'push':
-      PushChange(_STABLE_BRANCH_NAME, gflags.FLAGS.tracking_branch)
+      PushChange(STABLE_BRANCH_NAME, gflags.FLAGS.tracking_branch)
     elif command == 'commit' and ebuilds:
-      work_branch = GitBranch(_STABLE_BRANCH_NAME, gflags.FLAGS.tracking_branch)
+      work_branch = GitBranch(STABLE_BRANCH_NAME, gflags.FLAGS.tracking_branch)
       work_branch.CreateBranch()
       if not work_branch.Exists():
         Die('Unable to create stabilizing branch in %s' % overlay)