Update cbuildbot.py to upload prebuilts from preflight buildbot.

This change updates cbuildbot.py to upload prebuilts and reference them via PREFLIGHT_BINHOST.

Also ensure make.conf ends in a newline, as it looks nicer.

BUG=chromium-os:5311
TEST=Test that prebuilt.py updates PREFLIGHT_BINHOST in right make.conf
     file. Test that cbuildbot.py runs prebuilt.py with right
     arguments. Run unit tests for cbuildbot and prebuilt.py. Test runs of cbuildbot.py with --dryrun. Test upload of Packages file to local web server and check that emerge works with them.

Change-Id: Iad03c6c469e05b9ee1cceff69cbe2bdd51225e25

Review URL: http://codereview.chromium.org/4969003

bin/cbuildbot.py

@@ -26,6 +26,13 @@ _PACKAGE_FILE = '%(buildroot)s/src/scripts/cbuildbot_package.list'
 ARCHIVE_BASE = '/var/www/archive'
 ARCHIVE_COUNT = 10
 
+# Currently, both the full buildbot and the preflight buildbot store their
+# data in a variable named PORTAGE_BINHOST, but they're in different files.
+# We're planning on joining the two files soon and renaming the full binhost
+# to FULL_BINHOST.
+_FULL_BINHOST = 'PORTAGE_BINHOST'
+_PREFLIGHT_BINHOST = 'PORTAGE_BINHOST'
+
 # ======================== Utility functions ================================
 
 def MakeDir(path, parents=False):
@@ -305,6 +312,25 @@ def _MakeChroot(buildroot):
   RunCommand(['./make_chroot', '--fast'], cwd=cwd)
 
 
+def _GetPortageEnvVar(buildroot, board, envvar):
+  """Get a portage environment variable for the specified board, if any.
+
+  buildroot: The root directory where the build occurs. Must be an absolute
+             path.
+  board: Board type that was built on this machine. E.g. x86-generic.
+  envvar: The environment variable to get. E.g. "PORTAGE_BINHOST".
+
+  Returns:
+    The value of the environment variable, as a string. If no such variable
+    can be found, return the empty string.
+  """
+  cwd = os.path.join(buildroot, 'src', 'scripts')
+  binhost = RunCommand(['portageq-%s' % board, 'envvar', envvar],
+                       cwd=cwd, redirect_stdout=True, enter_chroot=True,
+                       error_ok=True)
+  return binhost.rstrip('\n')
+
+
 def _SetupBoard(buildroot, board='x86-generic'):
   """Wrapper around setup_board."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
@@ -507,6 +533,35 @@ def _ResolveOverlays(buildroot, overlays):
   return paths
 
 
+def _UploadPrebuilts(buildroot, board, overlay_config):
+  """Upload prebuilts.
+
+  Args:
+    buildroot: The root directory where the build occurs.
+    board: Board type that was built on this machine
+    overlay_config: A string describing which overlays you want.
+                    'private': Just the private overlay.
+                    'public': Just the public overlay.
+                    'both': Both the public and private overlays.
+  """
+
+  cwd = os.path.join(buildroot, 'src', 'scripts')
+  cmd = [os.path.join(cwd, 'prebuilt.py'),
+         '--sync-binhost-conf',
+         '--build-path', buildroot,
+         '--board', board,
+         '--prepend-version', 'preflight',
+         '--key', _PREFLIGHT_BINHOST]
+  if overlay_config == 'public':
+    cmd.extend(['--upload', 'gs://chromeos-prebuilt'])
+  else:
+    assert overlay_config in ('private', 'both')
+    cmd.extend(['--upload', 'chromeos-images:/var/www/prebuilt/',
+                '--binhost-base-url', 'http://chromeos-prebuilt'])
+
+  RunCommand(cmd, cwd=cwd)
+
+
 def main():
   # Parse options
   usage = "usage: %prog [options] cbuildbot_config"
@@ -558,16 +613,23 @@ def main():
     parser.print_usage()
     sys.exit(1)
 
-  # Calculate list of overlay directories.
-  overlays = _ResolveOverlays(buildroot, buildconfig['overlays'])
-
   try:
+    # Calculate list of overlay directories.
+    overlays = _ResolveOverlays(buildroot, buildconfig['overlays'])
+    board = buildconfig['board']
+
     _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays)
+    chroot_path = os.path.join(buildroot, 'chroot')
+    boardpath = os.path.join(chroot_path, 'build', board)
     if options.sync:
       if options.clobber or not os.path.isdir(buildroot):
         _FullCheckout(buildroot, tracking_branch, url=options.url)
       else:
+        old_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST)
         _IncrementalCheckout(buildroot)
+        new_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST)
+        if old_binhost != new_binhost:
+          RunCommand(['sudo', 'rm', '-rf', boardpath])
 
     # Check that all overlays can be found.
     for path in overlays:
@@ -575,11 +637,9 @@ def main():
       if not os.path.isdir(path):
         Die('Missing overlay: %s' % path)
 
-    chroot_path = os.path.join(buildroot, 'chroot')
     if not os.path.isdir(chroot_path):
       _MakeChroot(buildroot)
 
-    boardpath = os.path.join(chroot_path, 'build', buildconfig['board'])
     if not os.path.isdir(boardpath):
       _SetupBoard(buildroot, board=buildconfig['board'])
 
@@ -624,6 +684,7 @@ def main():
       if buildconfig['master']:
         # Master bot needs to check if the other slaves completed.
         if cbuildbot_comm.HaveSlavesCompleted(config):
+          _UploadPrebuilts(buildroot, board, buildconfig['overlays'])
           _UprevPush(buildroot, tracking_branch, buildconfig['board'],
                      overlays, options.debug)
         else:

bin/cbuildbot_unittest.py

@@ -151,10 +151,13 @@ class CBuildBotTest(mox.MoxTestBase):
     m_file.read().AndReturn(self._test_string)
     m_file.close()
 
+    drop_file = cbuildbot._PACKAGE_FILE % {'buildroot': self._buildroot}
     cbuildbot.RunCommand(['./cros_mark_as_stable', '--all',
                      '--board=%s' % self._test_board,
                      '--overlays=%s' % ':'.join(self._chroot_overlays),
-                     '--tracking_branch=cros/master', 'commit'],
+                     '--tracking_branch=cros/master',
+                     '--drop_file=%s' % ReinterpretPathForChroot(drop_file),
+                     'commit'],
                      cwd='%s/src/scripts' % self._buildroot,
                      enter_chroot=True)
 
@@ -174,10 +177,13 @@ class CBuildBotTest(mox.MoxTestBase):
     m_file.read().AndReturn('None')
     m_file.close()
 
+    drop_file = cbuildbot._PACKAGE_FILE % {'buildroot': self._buildroot}
     cbuildbot.RunCommand(['./cros_mark_as_stable', '--all',
                          '--board=%s' % self._test_board,
                          '--overlays=%s' % ':'.join(self._chroot_overlays),
-                         '--tracking_branch=cros/master', 'commit'],
+                         '--tracking_branch=cros/master',
+                         '--drop_file=%s' % ReinterpretPathForChroot(drop_file),
+                         'commit'],
                          cwd='%s/src/scripts' % self._buildroot,
                          enter_chroot=True)
 
@@ -187,6 +193,35 @@ class CBuildBotTest(mox.MoxTestBase):
                              self._overlays)
     self.mox.VerifyAll()
 
+  def testGetPortageEnvVar(self):
+    """Basic test case for _GetPortageEnvVar function."""
+    envvar = 'EXAMPLE'
+    cbuildbot.RunCommand(mox.And(mox.IsA(list), mox.In(envvar)),
+                         cwd='%s/src/scripts' % self._buildroot,
+                         redirect_stdout=True, enter_chroot=True,
+                         error_ok=True).AndReturn('RESULT\n')
+    self.mox.ReplayAll()
+    result = cbuildbot._GetPortageEnvVar(self._buildroot, self._test_board,
+                                         envvar)
+    self.mox.VerifyAll()
+    self.assertEqual(result, 'RESULT')
+
+  def testUploadPublicPrebuilts(self):
+    """Test _UploadPrebuilts with a public location."""
+    check = mox.And(mox.IsA(list), mox.In('gs://chromeos-prebuilt'))
+    cbuildbot.RunCommand(check, cwd='%s/src/scripts' % self._buildroot)
+    self.mox.ReplayAll()
+    cbuildbot._UploadPrebuilts(self._buildroot, self._test_board, 'public')
+    self.mox.VerifyAll()
+
+  def testUploadPrivatePrebuilts(self):
+    """Test _UploadPrebuilts with a private location."""
+    check = mox.And(mox.IsA(list), mox.In('chromeos-images:/var/www/prebuilt/'))
+    cbuildbot.RunCommand(check, cwd='%s/src/scripts' % self._buildroot)
+    self.mox.ReplayAll()
+    cbuildbot._UploadPrebuilts(self._buildroot, self._test_board, 'private')
+    self.mox.VerifyAll()
+
 
 if __name__ == '__main__':
   unittest.main()

prebuilt.py

@@ -59,6 +59,7 @@ _PREBUILT_BASE_DIR = 'src/third_party/chromiumos-overlay/chromeos/config/'
 # Created in the event of new host targets becoming available
 _PREBUILT_MAKE_CONF = {'amd64': os.path.join(_PREBUILT_BASE_DIR,
                                              'make.conf.amd64-host')}
+_BINHOST_CONF_DIR = 'src/third_party/chromiumos-overlay/chromeos/binhost'
 
 
 class FiltersEmpty(Exception):
@@ -92,6 +93,7 @@ def UpdateLocalFile(filename, value, key='PORTAGE_BINHOST'):
   file_fh = open(filename)
   file_lines = []
   found = False
+  keyval_str = '%(key)s=%(value)s'
   for line in file_fh:
     # Strip newlines from end of line. We already add newlines below.
     line = line.rstrip("\n")
@@ -102,7 +104,6 @@ def UpdateLocalFile(filename, value, key='PORTAGE_BINHOST'):
       continue
 
     file_var, file_val = line.split('=')
-    keyval_str = '%(key)s=%(value)s'
     if file_var == key:
       found = True
       print 'Updating %s=%s to %s="%s"' % (file_var, file_val, key, value)
@@ -117,7 +118,7 @@ def UpdateLocalFile(filename, value, key='PORTAGE_BINHOST'):
   file_fh.close()
   # write out new file
   new_file_fh = open(filename, 'w')
-  new_file_fh.write('\n'.join(file_lines))
+  new_file_fh.write('\n'.join(file_lines) + '\n')
   new_file_fh.close()
 
 
@@ -143,7 +144,7 @@ def RevGitPushWithRetry(retries=5):
     raise GitPushFailed('Failed to push change after %s retries' % retries)
 
 
-def RevGitFile(filename, value, retries=5):
+def RevGitFile(filename, value, retries=5, key='PORTAGE_BINHOST'):
   """Update and push the git file.
 
     Args:
@@ -151,6 +152,8 @@ def RevGitFile(filename, value, retries=5):
       value: string representing the version of the prebuilt that has been
         uploaded.
       retries: The number of times to retry before giving up, default: 5
+      key: The variable key to update in the git file.
+        (Default: PORTAGE_BINHOST)
   """
   prebuilt_branch = 'prebuilt_branch'
   old_cwd = os.getcwd()
@@ -162,10 +165,10 @@ def RevGitFile(filename, value, retries=5):
     'git config url.ssh://git@gitrw.chromium.org:9222.pushinsteadof '
     'http://git.chromium.org/git')
   cros_build_lib.RunCommand(git_ssh_config_cmd, shell=True)
-  description = 'Update PORTAGE_BINHOST="%s" in %s' % (value, filename)
+  description = 'Update %s="%s" in %s' % (key, value, filename)
   print description
   try:
-    UpdateLocalFile(filename, value)
+    UpdateLocalFile(filename, value, key)
     cros_build_lib.RunCommand('git config push.default tracking', shell=True)
     cros_build_lib.RunCommand('git commit -am "%s"' % description, shell=True)
     RevGitPushWithRetry(retries)
@@ -405,8 +408,32 @@ def DetermineMakeConfFile(target):
   return os.path.join(make_path)
 
 
+def UpdateBinhostConfFile(path, key, value):
+  """Update binhost config file file with key=value.
+
+  Args:
+    path: Filename to update.
+    key: Key to update.
+    value: New value for key.
+  """
+  cwd = os.path.dirname(os.path.abspath(path))
+  filename = os.path.basename(path)
+  if not os.path.isdir(cwd):
+    os.makedirs(cwd)
+  if not os.path.isfile(path):
+    config_file = file(path, 'w')
+    config_file.write('FULL_BINHOST="$PORTAGE_BINHOST"\n')
+    config_file.close()
+  UpdateLocalFile(path, value, key)
+  cros_build_lib.RunCommand('git add %s' % filename, cwd=cwd, shell=True)
+  description = 'Update %s=%s in %s' % (key, value, filename)
+  cros_build_lib.RunCommand('git commit -m "%s"' % description, cwd=cwd,
+      shell=True)
+
+
 def UploadPrebuilt(build_path, upload_location, version, binhost_base_url,
-                   board=None, git_sync=False, git_sync_retries=5):
+                   board=None, git_sync=False, git_sync_retries=5,
+                   key='PORTAGE_BINHOST', sync_binhost_conf=False):
   """Upload Host prebuilt files to Google Storage space.
 
   Args:
@@ -415,10 +442,13 @@ def UploadPrebuilt(build_path, upload_location, version, binhost_base_url,
     board: The board to upload to Google Storage, if this is None upload
       host packages.
     git_sync: If set, update make.conf of target to reference the latest
-      prebuilt packages genereated here.
+      prebuilt packages generated here.
     git_sync_retries: How many times to retry pushing when updating git files.
       This helps avoid failures when multiple bots are modifying the same Repo.
       default: 5
+    key: The variable key to update in the git file. (Default: PORTAGE_BINHOST)
+    sync_binhost_conf: If set, update binhost config file in chromiumos-overlay
+      for the current board or host.
   """
 
   if not board:
@@ -428,12 +458,16 @@ def UploadPrebuilt(build_path, upload_location, version, binhost_base_url,
     url_suffix = _REL_HOST_PATH % {'version': version, 'target': _HOST_TARGET}
     package_string = _HOST_TARGET
     git_file = os.path.join(build_path, _PREBUILT_MAKE_CONF[_HOST_TARGET])
+    binhost_conf = os.path.join(build_path, _BINHOST_CONF_DIR, 'host',
+        '%s.conf' % _HOST_TARGET)
   else:
     board_path = os.path.join(build_path, _BOARD_PATH % {'board': board})
     package_path = os.path.join(board_path, 'packages')
     package_string = board
     url_suffix = _REL_BOARD_PATH % {'board': board, 'version': version}
     git_file = os.path.join(build_path, DetermineMakeConfFile(board))
+    binhost_conf = os.path.join(build_path, _BINHOST_CONF_DIR, 'target',
+        '%s.conf' % board)
   remote_location = os.path.join(upload_location, url_suffix)
 
   if upload_location.startswith('gs://'):
@@ -452,10 +486,13 @@ def UploadPrebuilt(build_path, upload_location, version, binhost_base_url,
       if not _RetryRun(cmd, shell=True):
         raise UploadFailed('Could not run %s' % cmd)
 
-  if git_sync:
+  url_value = '%s/%s/' % (binhost_base_url, url_suffix)
-    url_value = '%s/%s/' % (binhost_base_url, url_suffix)
-    RevGitFile(git_file, url_value, retries=git_sync_retries)
 
+  if git_sync:
+    RevGitFile(git_file, url_value, retries=git_sync_retries, key=key)
+
+  if sync_binhost_conf:
+    UpdateBinhostConfFile(binhost_conf, key, url_value)
 
 def usage(parser, msg):
   """Display usage message and parser help then exit with 1."""
@@ -488,6 +525,12 @@ def main():
   parser.add_option('-f', '--filters', dest='filters', action='store_true',
                     default=False,
                     help='Turn on filtering of private ebuild packages')
+  parser.add_option('-k', '--key', dest='key',
+                    default='PORTAGE_BINHOST',
+                    help='Key to update in make.conf / binhost.conf')
+  parser.add_option('', '--sync-binhost-conf', dest='sync_binhost_conf',
+                    default=False, action='store_true',
+                    help='Update binhost.conf')
 
   options, args = parser.parse_args()
   # Setup boto environment for gsutil to use
@@ -511,12 +554,15 @@ def main():
 
   if options.sync_host:
     UploadPrebuilt(options.build_path, options.upload, version,
-                   options.binhost_base_url, git_sync=options.git_sync)
+                   options.binhost_base_url, git_sync=options.git_sync,
+                   key=options.key,
+                   sync_binhost_conf=options.sync_binhost_conf)
 
   if options.board:
     UploadPrebuilt(options.build_path, options.upload, version,
                    options.binhost_base_url, board=options.board,
-                   git_sync=options.git_sync)
+                   git_sync=options.git_sync, key=options.key,
+                   sync_binhost_conf=options.sync_binhost_conf)
 
 
 if __name__ == '__main__':