Revert "Add more error checking to preflight queue."

This reverts commit b4258c8bd48f89d87f9d1aa2fc23b0556ffc611b.

BUG=br0ken build
TBR=rspangler
TEST=

Review URL: http://codereview.chromium.org/4697007

Change-Id: I839ffd08d33fc5943696cfe31f2978fe32ce5fec

bin/cbuildbot.py

@@ -181,35 +181,37 @@ def _ParseRevisionString(revision_string, repo_dictionary):
   return revisions.items()
 
 
-def _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board,
+def _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board):
-                           overlays):
   """Uprevs based on revision list."""
   if not revision_list:
     Info('No packages found to uprev')
     return
 
-  packages = []
+  package_str = ''
   for package, revision in revision_list:
-    assert ':' not in package, 'Invalid package name: %s' % package
+    package_str += package + ' '
-    packages.append(package)
+
+  package_str = package_str.strip()
 
   cwd = os.path.join(buildroot, 'src', 'scripts')
+  # TODO(davidjames): --foo="bar baz" only works here because we're using
+  # enter_chroot.
   RunCommand(['./cros_mark_as_stable',
               '--board=%s' % board,
-              '--tracking_branch=%s' % tracking_branch,
+              '--tracking_branch="%s"' % tracking_branch,
-              '--overlays=%s' % ':'.join(overlays),
+              '--packages="%s"' % package_str,
-              '--packages=%s' % ':'.join(packages),
               'commit'],
               cwd=cwd, enter_chroot=True)
 
 
-def _UprevAllPackages(buildroot, tracking_branch, board, overlays):
+def _UprevAllPackages(buildroot, tracking_branch, board):
   """Uprevs all packages that have been updated since last uprev."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
+  # TODO(davidjames): --foo="bar baz" only works here because we're using
+  # enter_chroot.
   RunCommand(['./cros_mark_as_stable', '--all',
               '--board=%s' % board,
-              '--overlays=%s' % ':'.join(overlays),
+              '--tracking_branch="%s"' % tracking_branch, 'commit'],
-              '--tracking_branch=%s' % tracking_branch, 'commit'],
               cwd=cwd, enter_chroot=True)
 
 
@@ -226,13 +228,12 @@ def _GetVMConstants(buildroot):
   return (vdisk_size.strip(), statefulfs_size.strip())
 
 
-def _GitCleanup(buildroot, board, tracking_branch, overlays):
+def _GitCleanup(buildroot, board, tracking_branch):
   """Clean up git branch after previous uprev attempt."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
   if os.path.exists(cwd):
     RunCommand(['./cros_mark_as_stable', '--srcroot=..',
                 '--board=%s' % board,
-                '--overlays=%s' % ':'.join(overlays),
                 '--tracking_branch=%s' % tracking_branch, 'clean'],
                cwd=cwd, error_ok=True)
 
@@ -256,9 +257,9 @@ def _WipeOldOutput(buildroot):
 # =========================== Main Commands ===================================
 
 
-def _PreFlightRinse(buildroot, board, tracking_branch, overlays):
+def _PreFlightRinse(buildroot, board, tracking_branch):
   """Cleans up any leftover state from previous runs."""
-  _GitCleanup(buildroot, board, tracking_branch, overlays)
+  _GitCleanup(buildroot, board, tracking_branch)
   _CleanUpMountPoints(buildroot)
   RunCommand(['sudo', 'killall', 'kvm'], error_ok=True)
 
@@ -346,7 +347,7 @@ def _RunSmokeSuite(buildroot, results_dir):
               ], cwd=cwd, error_ok=False)
 
 
-def _UprevPackages(buildroot, tracking_branch, revisionfile, board, overlays):
+def _UprevPackages(buildroot, tracking_branch, revisionfile, board):
   """Uprevs a package based on given revisionfile.
 
   If revisionfile is set to None or does not resolve to an actual file, this
@@ -375,19 +376,26 @@ def _UprevPackages(buildroot, tracking_branch, revisionfile, board, overlays):
   #  print >> sys.stderr, 'CBUILDBOT Revision list found %s' % revisions
   #  revision_list = _ParseRevisionString(revisions,
   #      _CreateRepoDictionary(buildroot, board))
-  #  _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board,
+  #  _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board)
-  #                         overlays)
   #else:
   Info('CBUILDBOT Revving all')
-  _UprevAllPackages(buildroot, tracking_branch, board, overlays)
+  _UprevAllPackages(buildroot, tracking_branch, board)
 
 
 def _UprevPush(buildroot, tracking_branch, board, overlays):
   """Pushes uprev changes to the main line."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
+  public_overlay = '%s/src/third_party/chromiumos-overlay' % buildroot
+  private_overlay = '%s/src/private-overlays/chromeos-overlay' % buildroot
+  if overlays == 'private':
+    overlays = [private_overlay]
+  elif overlays == 'public':
+    overlays = [public_overlay]
+  else:
+    overlays = [public_overlay, private_overlay]
   RunCommand(['./cros_mark_as_stable', '--srcroot=..',
               '--board=%s' % board,
-              '--overlays=%s' % ':'.join(overlays),
+              '--overlays=%s' % " ".join(overlays),
               '--tracking_branch=%s' % tracking_branch,
               '--push_options=--bypass-hooks -f', 'push'],
              cwd=cwd)
@@ -459,32 +467,6 @@ def _GetConfig(config_name):
   return buildconfig
 
 
-def ResolveOverlays(buildroot, overlays):
-  """Return the list of overlays to use for a given buildbot.
-
-  Args:
-    overlays: A string describing which overlays you want.
-              'private': Just the private overlay.
-              'public': Just the public overlay.
-              'both': Both the public and private overlays.
-  """
-  public_overlay = '%s/src/third_party/chromiumos-overlay' % buildroot
-  private_overlay = '%s/src/private-overlays/chromeos-overlay' % buildroot
-  if overlays == 'private':
-    dirs = [private_overlay]
-  elif overlays == 'public':
-    dirs = [public_overlay]
-  elif overlays == 'both':
-    dirs = [public_overlay, private_overlay]
-  else:
-    Die('Incorrect overlay configuration: %s' % overlays)
-  for dir in dirs:
-    assert ':' not in dir, 'Overlay must not contain colons: %s' % dir
-    if not os.path.exists(dir):
-      Die('Missing overlay: %s' % dir)
-  return dirs
-
-
 def main():
   # Parse options
   usage = "usage: %prog [options] cbuildbot_config"
@@ -520,11 +502,8 @@ def main():
     parser.print_usage()
     sys.exit(1)
 
-  # Calculate list of overlay directories.
-  overlays = ResolveOverlays(buildroot, buildconfig['overlays'])
-
   try:
-    _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays)
+    _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch)
     if options.clobber or not os.path.isdir(buildroot):
       _FullCheckout(buildroot, tracking_branch, url=options.url)
     else:
@@ -540,7 +519,7 @@ def main():
 
     if buildconfig['uprev']:
       _UprevPackages(buildroot, tracking_branch, revisionfile,
-                     buildconfig['board'], overlays)
+                     board=buildconfig['board'])
 
     _EnableLocalAccount(buildroot)
     _Build(buildroot)
@@ -566,7 +545,7 @@ def main():
           # Master bot needs to check if the other slaves completed.
           if cbuildbot_comm.HaveSlavesCompleted(config):
             _UprevPush(buildroot, tracking_branch, buildconfig['board'],
-                       overlays)
+                       buildconfig['overlays'])
           else:
             Die('CBUILDBOT - One of the slaves has failed!!!')
 

bin/cbuildbot_config.py

@@ -19,9 +19,9 @@ hostname -- Needed for 'important' slaves.  The hostname of the bot.  Should
             match hostname in slaves.cfg in buildbot checkout.
 unittests -- Runs unittests for packages.
 smoke_bvt -- Runs the test smoke suite in a qemu-based VM using KVM.
-overlays -- Select what overlays to look at. This can be 'public', 'private'
+overlays -- If this bot is a master bot, select what overlays to push changes
-            or 'both'. There should only be one master bot pushing changes to
+            to. This can be 'public', 'private', or 'both'. There should only
-            each overlay per branch.
+            be one bot pushing changes to each overlay.
 """
 
 
@@ -33,7 +33,6 @@ config['default'] = {
   'important' : False,
   'unittests' : False,
   'smoke_bvt' : False,
-  'overlays': 'public',
 }
 config['x86-generic-pre-flight-queue'] = {
   'board' : 'x86-generic',
@@ -70,7 +69,6 @@ config['x86_agz_bin'] = {
   'important' : False,
   'unittests' : True,
   'smoke_bvt' : True,
-  'overlays': 'private',
 }
 config['x86_dogfood_bin'] = {
   'board' : 'x86-dogfood',
@@ -79,29 +77,52 @@ config['x86_dogfood_bin'] = {
   'important' : False,
   'unittests' : True,
   'smoke_bvt' : True,
-  'overlays': 'private',
 }
 config['x86_pineview_bin'] = {
   'board' : 'x86-pineview',
   'uprev' : True,
   'master' : False,
   'important' : False,
+  'hostname' : 'codf200.jail',
   'unittests': True,
-  'overlays': 'public',
 }
 config['arm_tegra2_bin'] = {
   'board' : 'tegra2',
   'uprev' : True,
   'master' : False,
   'important' : False,
+  'hostname' : 'codg172.jail',
+  'unittests' : False,
+}
+config['arm_voguev210_bin'] = {
+  'board' : 'voguev210',
+  'uprev' : True,
+  'master' : False,
+  'important' : False,
+  'hostname' : 'codf196.jail',
+  'unittests' : False,
+}
+config['arm_beagleboard_bin'] = {
+  'board' : 'beagleboard',
+  'master' : False,
+  'uprev' : True,
+  'important' : False,
+  'hostname' : 'codf202.jail',
+  'unittests' : False,
+}
+config['arm_st1q_bin'] = {
+  'board' : 'st1q',
+  'uprev' : True,
+  'master' : False,
+  'important' : False,
+  'hostname' : 'codg158.jail',
   'unittests' : False,
-  'overlays': 'public',
 }
 config['arm_generic_bin'] = {
   'board' : 'arm-generic',
   'uprev' : True,
   'master' : False,
   'important' : False,
+  'hostname' : 'codg175.jail',
   'unittests' : False,
-  'overlays': 'public',
 }

bin/cbuildbot_unittest.py

@@ -166,17 +166,15 @@ class CBuildBotTest(mox.MoxTestBase):
     m_file.read().AndReturn(self._test_string)
     m_file.close()
 
-    overlays = [ '%s/src/third_party/chromiumos-overlay' % self._buildroot ]
     cbuildbot.RunCommand(['./cros_mark_as_stable', '--all',
                      '--board=%s' % self._test_board,
-                     '--overlays=%s' % ':'.join(overlays),
+                     '--tracking_branch="cros/master"', 'commit'],
-                     '--tracking_branch=cros/master', 'commit'],
                      cwd='%s/src/scripts' % self._buildroot,
                      enter_chroot=True)
 
     self.mox.ReplayAll()
     cbuildbot._UprevPackages(self._buildroot, self.tracking_branch,
-                             self._revision_file, self._test_board, overlays)
+                             self._revision_file, self._test_board)
     self.mox.VerifyAll()
 
   def testUprevAllPackages(self):
@@ -189,18 +187,15 @@ class CBuildBotTest(mox.MoxTestBase):
     m_file.read().AndReturn('None')
     m_file.close()
 
-    overlays = [ '%s/src/third_party/chromiumos-overlay' % self._buildroot ]
     cbuildbot.RunCommand(['./cros_mark_as_stable', '--all',
                          '--board=%s' % self._test_board,
-                         '--overlays=%s' % ':'.join(overlays),
+                         '--tracking_branch="cros/master"', 'commit'],
-                         '--tracking_branch=cros/master', 'commit'],
                          cwd='%s/src/scripts' % self._buildroot,
                          enter_chroot=True)
 
     self.mox.ReplayAll()
-    overlays = [ '%s/src/third_party/chromiumos-overlay' % self._buildroot ]
     cbuildbot._UprevPackages(self._buildroot, self.tracking_branch,
-                             self._revision_file, self._test_board, overlays)
+                             self._revision_file, self._test_board)
     self.mox.VerifyAll()
 
 

cros_mark_as_stable.py

@@ -22,10 +22,10 @@ from cros_build_lib import Info, RunCommand, Warning, Die
 gflags.DEFINE_string('board', '',
                      'Board for which the package belongs.', short_name='b')
 gflags.DEFINE_string('overlays', '',
-                     'Colon-separated list of overlays to modify.',
+                     'Space separated list of overlays to modify.',
                      short_name='o')
 gflags.DEFINE_string('packages', '',
-                     'Colon-separated list of packages to mark as stable.',
+                     'Space separated list of packages to mark as stable.',
                      short_name='p')
 gflags.DEFINE_string('push_options', '',
                      'Options to use with git-cl push using push command.')
@@ -245,10 +245,7 @@ def _SimpleRunCommand(command):
   """Runs a shell command and returns stdout back to caller."""
   _Print('  + %s' % command)
   proc_handle = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
-  stdout = proc_handle.communicate()[0]
+  return proc_handle.communicate()[0]
-  retcode = proc_handle.wait()
-  assert retcode == 0, "Return code %s for command: %s" % (retcode, command)
-  return stdout
 
 
 # ======================= End Global Helper Functions ========================
@@ -490,16 +487,11 @@ def main(argv):
   except gflags.FlagsError, e :
     _PrintUsageAndDie(str(e))
 
-  package_list = gflags.FLAGS.packages.split(':')
+  package_list = gflags.FLAGS.packages.split()
   _CheckSaneArguments(package_list, command)
   if gflags.FLAGS.overlays:
-    overlays = {}
+    overlays = dict((path, []) for path in gflags.FLAGS.overlays.split())
-    for path in gflags.FLAGS.overlays.split(':'):
-      if not os.path.exists(path):
-        Die('Cannot find overlay: %s' % path)
-      overlays[path] = []
   else:
-    Warning('Missing --overlays argument')
     overlays = {
       '%s/private-overlays/chromeos-overlay' % gflags.FLAGS.srcroot: [],
       '%s/third_party/chromiumos-overlay' % gflags.FLAGS.srcroot: []