diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index f6258d12fc..2a6e22b68e 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -26,6 +26,13 @@ _PACKAGE_FILE = '%(buildroot)s/src/scripts/cbuildbot_package.list' ARCHIVE_BASE = '/var/www/archive' ARCHIVE_COUNT = 10 +# Currently, both the full buildbot and the preflight buildbot store their +# data in a variable named PORTAGE_BINHOST, but they're in different files. +# We're planning on joining the two files soon and renaming the full binhost +# to FULL_BINHOST. +_FULL_BINHOST = 'PORTAGE_BINHOST' +_PREFLIGHT_BINHOST = 'PORTAGE_BINHOST' + # ======================== Utility functions ================================ def MakeDir(path, parents=False): @@ -305,6 +312,25 @@ def _MakeChroot(buildroot): RunCommand(['./make_chroot', '--fast'], cwd=cwd) +def _GetPortageEnvVar(buildroot, board, envvar): + """Get a portage environment variable for the specified board, if any. + + buildroot: The root directory where the build occurs. Must be an absolute + path. + board: Board type that was built on this machine. E.g. x86-generic. + envvar: The environment variable to get. E.g. "PORTAGE_BINHOST". + + Returns: + The value of the environment variable, as a string. If no such variable + can be found, return the empty string. + """ + cwd = os.path.join(buildroot, 'src', 'scripts') + binhost = RunCommand(['portageq-%s' % board, 'envvar', envvar], + cwd=cwd, redirect_stdout=True, enter_chroot=True, + error_ok=True) + return binhost.rstrip('\n') + + def _SetupBoard(buildroot, board='x86-generic'): """Wrapper around setup_board.""" cwd = os.path.join(buildroot, 'src', 'scripts') @@ -507,6 +533,35 @@ def _ResolveOverlays(buildroot, overlays): return paths +def _UploadPrebuilts(buildroot, board, overlay_config): + """Upload prebuilts. + + Args: + buildroot: The root directory where the build occurs. + board: Board type that was built on this machine + overlay_config: A string describing which overlays you want. + 'private': Just the private overlay. + 'public': Just the public overlay. + 'both': Both the public and private overlays. + """ + + cwd = os.path.join(buildroot, 'src', 'scripts') + cmd = [os.path.join(cwd, 'prebuilt.py'), + '--sync-binhost-conf', + '--build-path', buildroot, + '--board', board, + '--prepend-version', 'preflight', + '--key', _PREFLIGHT_BINHOST] + if overlay_config == 'public': + cmd.extend(['--upload', 'gs://chromeos-prebuilt']) + else: + assert overlay_config in ('private', 'both') + cmd.extend(['--upload', 'chromeos-images:/var/www/prebuilt/', + '--binhost-base-url', 'http://chromeos-prebuilt']) + + RunCommand(cmd, cwd=cwd) + + def main(): # Parse options usage = "usage: %prog [options] cbuildbot_config" @@ -558,16 +613,23 @@ def main(): parser.print_usage() sys.exit(1) - # Calculate list of overlay directories. - overlays = _ResolveOverlays(buildroot, buildconfig['overlays']) - try: + # Calculate list of overlay directories. + overlays = _ResolveOverlays(buildroot, buildconfig['overlays']) + board = buildconfig['board'] + _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays) + chroot_path = os.path.join(buildroot, 'chroot') + boardpath = os.path.join(chroot_path, 'build', board) if options.sync: if options.clobber or not os.path.isdir(buildroot): _FullCheckout(buildroot, tracking_branch, url=options.url) else: + old_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST) _IncrementalCheckout(buildroot) + new_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST) + if old_binhost != new_binhost: + RunCommand(['sudo', 'rm', '-rf', boardpath]) # Check that all overlays can be found. for path in overlays: @@ -575,11 +637,9 @@ def main(): if not os.path.isdir(path): Die('Missing overlay: %s' % path) - chroot_path = os.path.join(buildroot, 'chroot') if not os.path.isdir(chroot_path): _MakeChroot(buildroot) - boardpath = os.path.join(chroot_path, 'build', buildconfig['board']) if not os.path.isdir(boardpath): _SetupBoard(buildroot, board=buildconfig['board']) @@ -624,6 +684,7 @@ def main(): if buildconfig['master']: # Master bot needs to check if the other slaves completed. if cbuildbot_comm.HaveSlavesCompleted(config): + _UploadPrebuilts(buildroot, board, buildconfig['overlays']) _UprevPush(buildroot, tracking_branch, buildconfig['board'], overlays, options.debug) else: diff --git a/bin/cbuildbot_unittest.py b/bin/cbuildbot_unittest.py index ba9b545b0d..87bfc363c2 100755 --- a/bin/cbuildbot_unittest.py +++ b/bin/cbuildbot_unittest.py @@ -151,10 +151,13 @@ class CBuildBotTest(mox.MoxTestBase): m_file.read().AndReturn(self._test_string) m_file.close() + drop_file = cbuildbot._PACKAGE_FILE % {'buildroot': self._buildroot} cbuildbot.RunCommand(['./cros_mark_as_stable', '--all', '--board=%s' % self._test_board, '--overlays=%s' % ':'.join(self._chroot_overlays), - '--tracking_branch=cros/master', 'commit'], + '--tracking_branch=cros/master', + '--drop_file=%s' % ReinterpretPathForChroot(drop_file), + 'commit'], cwd='%s/src/scripts' % self._buildroot, enter_chroot=True) @@ -174,10 +177,13 @@ class CBuildBotTest(mox.MoxTestBase): m_file.read().AndReturn('None') m_file.close() + drop_file = cbuildbot._PACKAGE_FILE % {'buildroot': self._buildroot} cbuildbot.RunCommand(['./cros_mark_as_stable', '--all', '--board=%s' % self._test_board, '--overlays=%s' % ':'.join(self._chroot_overlays), - '--tracking_branch=cros/master', 'commit'], + '--tracking_branch=cros/master', + '--drop_file=%s' % ReinterpretPathForChroot(drop_file), + 'commit'], cwd='%s/src/scripts' % self._buildroot, enter_chroot=True) @@ -187,6 +193,35 @@ class CBuildBotTest(mox.MoxTestBase): self._overlays) self.mox.VerifyAll() + def testGetPortageEnvVar(self): + """Basic test case for _GetPortageEnvVar function.""" + envvar = 'EXAMPLE' + cbuildbot.RunCommand(mox.And(mox.IsA(list), mox.In(envvar)), + cwd='%s/src/scripts' % self._buildroot, + redirect_stdout=True, enter_chroot=True, + error_ok=True).AndReturn('RESULT\n') + self.mox.ReplayAll() + result = cbuildbot._GetPortageEnvVar(self._buildroot, self._test_board, + envvar) + self.mox.VerifyAll() + self.assertEqual(result, 'RESULT') + + def testUploadPublicPrebuilts(self): + """Test _UploadPrebuilts with a public location.""" + check = mox.And(mox.IsA(list), mox.In('gs://chromeos-prebuilt')) + cbuildbot.RunCommand(check, cwd='%s/src/scripts' % self._buildroot) + self.mox.ReplayAll() + cbuildbot._UploadPrebuilts(self._buildroot, self._test_board, 'public') + self.mox.VerifyAll() + + def testUploadPrivatePrebuilts(self): + """Test _UploadPrebuilts with a private location.""" + check = mox.And(mox.IsA(list), mox.In('chromeos-images:/var/www/prebuilt/')) + cbuildbot.RunCommand(check, cwd='%s/src/scripts' % self._buildroot) + self.mox.ReplayAll() + cbuildbot._UploadPrebuilts(self._buildroot, self._test_board, 'private') + self.mox.VerifyAll() + if __name__ == '__main__': unittest.main() diff --git a/prebuilt.py b/prebuilt.py index ee9ecf1887..cb4e04f84a 100755 --- a/prebuilt.py +++ b/prebuilt.py @@ -59,6 +59,7 @@ _PREBUILT_BASE_DIR = 'src/third_party/chromiumos-overlay/chromeos/config/' # Created in the event of new host targets becoming available _PREBUILT_MAKE_CONF = {'amd64': os.path.join(_PREBUILT_BASE_DIR, 'make.conf.amd64-host')} +_BINHOST_CONF_DIR = 'src/third_party/chromiumos-overlay/chromeos/binhost' class FiltersEmpty(Exception): @@ -92,6 +93,7 @@ def UpdateLocalFile(filename, value, key='PORTAGE_BINHOST'): file_fh = open(filename) file_lines = [] found = False + keyval_str = '%(key)s=%(value)s' for line in file_fh: # Strip newlines from end of line. We already add newlines below. line = line.rstrip("\n") @@ -102,7 +104,6 @@ def UpdateLocalFile(filename, value, key='PORTAGE_BINHOST'): continue file_var, file_val = line.split('=') - keyval_str = '%(key)s=%(value)s' if file_var == key: found = True print 'Updating %s=%s to %s="%s"' % (file_var, file_val, key, value) @@ -117,7 +118,7 @@ def UpdateLocalFile(filename, value, key='PORTAGE_BINHOST'): file_fh.close() # write out new file new_file_fh = open(filename, 'w') - new_file_fh.write('\n'.join(file_lines)) + new_file_fh.write('\n'.join(file_lines) + '\n') new_file_fh.close() @@ -143,7 +144,7 @@ def RevGitPushWithRetry(retries=5): raise GitPushFailed('Failed to push change after %s retries' % retries) -def RevGitFile(filename, value, retries=5): +def RevGitFile(filename, value, retries=5, key='PORTAGE_BINHOST'): """Update and push the git file. Args: @@ -151,6 +152,8 @@ def RevGitFile(filename, value, retries=5): value: string representing the version of the prebuilt that has been uploaded. retries: The number of times to retry before giving up, default: 5 + key: The variable key to update in the git file. + (Default: PORTAGE_BINHOST) """ prebuilt_branch = 'prebuilt_branch' old_cwd = os.getcwd() @@ -162,10 +165,10 @@ def RevGitFile(filename, value, retries=5): 'git config url.ssh://git@gitrw.chromium.org:9222.pushinsteadof ' 'http://git.chromium.org/git') cros_build_lib.RunCommand(git_ssh_config_cmd, shell=True) - description = 'Update PORTAGE_BINHOST="%s" in %s' % (value, filename) + description = 'Update %s="%s" in %s' % (key, value, filename) print description try: - UpdateLocalFile(filename, value) + UpdateLocalFile(filename, value, key) cros_build_lib.RunCommand('git config push.default tracking', shell=True) cros_build_lib.RunCommand('git commit -am "%s"' % description, shell=True) RevGitPushWithRetry(retries) @@ -405,8 +408,32 @@ def DetermineMakeConfFile(target): return os.path.join(make_path) +def UpdateBinhostConfFile(path, key, value): + """Update binhost config file file with key=value. + + Args: + path: Filename to update. + key: Key to update. + value: New value for key. + """ + cwd = os.path.dirname(os.path.abspath(path)) + filename = os.path.basename(path) + if not os.path.isdir(cwd): + os.makedirs(cwd) + if not os.path.isfile(path): + config_file = file(path, 'w') + config_file.write('FULL_BINHOST="$PORTAGE_BINHOST"\n') + config_file.close() + UpdateLocalFile(path, value, key) + cros_build_lib.RunCommand('git add %s' % filename, cwd=cwd, shell=True) + description = 'Update %s=%s in %s' % (key, value, filename) + cros_build_lib.RunCommand('git commit -m "%s"' % description, cwd=cwd, + shell=True) + + def UploadPrebuilt(build_path, upload_location, version, binhost_base_url, - board=None, git_sync=False, git_sync_retries=5): + board=None, git_sync=False, git_sync_retries=5, + key='PORTAGE_BINHOST', sync_binhost_conf=False): """Upload Host prebuilt files to Google Storage space. Args: @@ -415,10 +442,13 @@ def UploadPrebuilt(build_path, upload_location, version, binhost_base_url, board: The board to upload to Google Storage, if this is None upload host packages. git_sync: If set, update make.conf of target to reference the latest - prebuilt packages genereated here. + prebuilt packages generated here. git_sync_retries: How many times to retry pushing when updating git files. This helps avoid failures when multiple bots are modifying the same Repo. default: 5 + key: The variable key to update in the git file. (Default: PORTAGE_BINHOST) + sync_binhost_conf: If set, update binhost config file in chromiumos-overlay + for the current board or host. """ if not board: @@ -428,12 +458,16 @@ def UploadPrebuilt(build_path, upload_location, version, binhost_base_url, url_suffix = _REL_HOST_PATH % {'version': version, 'target': _HOST_TARGET} package_string = _HOST_TARGET git_file = os.path.join(build_path, _PREBUILT_MAKE_CONF[_HOST_TARGET]) + binhost_conf = os.path.join(build_path, _BINHOST_CONF_DIR, 'host', + '%s.conf' % _HOST_TARGET) else: board_path = os.path.join(build_path, _BOARD_PATH % {'board': board}) package_path = os.path.join(board_path, 'packages') package_string = board url_suffix = _REL_BOARD_PATH % {'board': board, 'version': version} git_file = os.path.join(build_path, DetermineMakeConfFile(board)) + binhost_conf = os.path.join(build_path, _BINHOST_CONF_DIR, 'target', + '%s.conf' % board) remote_location = os.path.join(upload_location, url_suffix) if upload_location.startswith('gs://'): @@ -452,10 +486,13 @@ def UploadPrebuilt(build_path, upload_location, version, binhost_base_url, if not _RetryRun(cmd, shell=True): raise UploadFailed('Could not run %s' % cmd) - if git_sync: - url_value = '%s/%s/' % (binhost_base_url, url_suffix) - RevGitFile(git_file, url_value, retries=git_sync_retries) + url_value = '%s/%s/' % (binhost_base_url, url_suffix) + if git_sync: + RevGitFile(git_file, url_value, retries=git_sync_retries, key=key) + + if sync_binhost_conf: + UpdateBinhostConfFile(binhost_conf, key, url_value) def usage(parser, msg): """Display usage message and parser help then exit with 1.""" @@ -488,6 +525,12 @@ def main(): parser.add_option('-f', '--filters', dest='filters', action='store_true', default=False, help='Turn on filtering of private ebuild packages') + parser.add_option('-k', '--key', dest='key', + default='PORTAGE_BINHOST', + help='Key to update in make.conf / binhost.conf') + parser.add_option('', '--sync-binhost-conf', dest='sync_binhost_conf', + default=False, action='store_true', + help='Update binhost.conf') options, args = parser.parse_args() # Setup boto environment for gsutil to use @@ -511,12 +554,15 @@ def main(): if options.sync_host: UploadPrebuilt(options.build_path, options.upload, version, - options.binhost_base_url, git_sync=options.git_sync) + options.binhost_base_url, git_sync=options.git_sync, + key=options.key, + sync_binhost_conf=options.sync_binhost_conf) if options.board: UploadPrebuilt(options.build_path, options.upload, version, options.binhost_base_url, board=options.board, - git_sync=options.git_sync) + git_sync=options.git_sync, key=options.key, + sync_binhost_conf=options.sync_binhost_conf) if __name__ == '__main__':