mirrors/flatcar-scripts
1
0
Fork 0
mirror of https://github.com/flatcar/scripts.git synced 2025-09-24 15:11:19 +02:00
Code Issues Packages Projects Releases Wiki Activity

Update cbuildbot.py to upload prebuilts from preflight buildbot.

Browse Source 
This change updates cbuildbot.py to upload prebuilts and reference them via PREFLIGHT_BINHOST.

Also ensure make.conf ends in a newline, as it looks nicer.

BUG=chromium-os:5311
TEST=Test that prebuilt.py updates PREFLIGHT_BINHOST in right make.conf
     file. Test that cbuildbot.py runs prebuilt.py with right
     arguments. Run unit tests for cbuildbot and prebuilt.py. Test runs of cbuildbot.py with --dryrun. Test upload of Packages file to local web server and check that emerge works with them.

Change-Id: Iad03c6c469e05b9ee1cceff69cbe2bdd51225e25

Review URL: http://codereview.chromium.org/4969003
This commit is contained in:
David James 2010-11-30 11:36:57 -08:00
parent fb5ce169ee
commit c509a906da
3 changed files with 161 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
bin/cbuildbot.py
View File

@ -26,6 +26,13 @@ _PACKAGE_FILE = '%(buildroot)s/src/scripts/cbuildbot_package.list'
ARCHIVE_BASE = '/var/www/archive'
ARCHIVE_COUNT = 10
# Currently, both the full buildbot and the preflight buildbot store their
# data in a variable named PORTAGE_BINHOST, but they're in different files.
# We're planning on joining the two files soon and renaming the full binhost
# to FULL_BINHOST.
_FULL_BINHOST = 'PORTAGE_BINHOST'
_PREFLIGHT_BINHOST = 'PORTAGE_BINHOST'
# ======================== Utility functions ================================
def MakeDir(path, parents=False):
@ -305,6 +312,25 @@ def _MakeChroot(buildroot):
RunCommand(['./make_chroot', '--fast'], cwd=cwd)
def _GetPortageEnvVar(buildroot, board, envvar):
"""Get a portage environment variable for the specified board, if any.
buildroot: The root directory where the build occurs. Must be an absolute
path.
board: Board type that was built on this machine. E.g. x86-generic.
envvar: The environment variable to get. E.g. "PORTAGE_BINHOST".
Returns:
The value of the environment variable, as a string. If no such variable
can be found, return the empty string.
"""
cwd = os.path.join(buildroot, 'src', 'scripts')
binhost = RunCommand(['portageq-%s' % board, 'envvar', envvar],
cwd=cwd, redirect_stdout=True, enter_chroot=True,
error_ok=True)
return binhost.rstrip('\n')
def _SetupBoard(buildroot, board='x86-generic'):
"""Wrapper around setup_board."""
cwd = os.path.join(buildroot, 'src', 'scripts')
@ -507,6 +533,35 @@ def _ResolveOverlays(buildroot, overlays):
return paths
def _UploadPrebuilts(buildroot, board, overlay_config):
"""Upload prebuilts.
Args:
buildroot: The root directory where the build occurs.
board: Board type that was built on this machine
overlay_config: A string describing which overlays you want.
'private': Just the private overlay.
'public': Just the public overlay.
'both': Both the public and private overlays.
"""
cwd = os.path.join(buildroot, 'src', 'scripts')
cmd = [os.path.join(cwd, 'prebuilt.py'),
'--sync-binhost-conf',
'--build-path', buildroot,
'--board', board,
'--prepend-version', 'preflight',
'--key', _PREFLIGHT_BINHOST]
if overlay_config == 'public':
cmd.extend(['--upload', 'gs://chromeos-prebuilt'])
else:
assert overlay_config in ('private', 'both')
cmd.extend(['--upload', 'chromeos-images:/var/www/prebuilt/',
'--binhost-base-url', 'http://chromeos-prebuilt'])
RunCommand(cmd, cwd=cwd)
def main():
# Parse options
usage = "usage: %prog [options] cbuildbot_config"
@ -558,16 +613,23 @@ def main():
parser.print_usage()
sys.exit(1)
# Calculate list of overlay directories.
overlays = _ResolveOverlays(buildroot, buildconfig['overlays'])
try:
# Calculate list of overlay directories.
overlays = _ResolveOverlays(buildroot, buildconfig['overlays'])
board = buildconfig['board']
_PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays)
chroot_path = os.path.join(buildroot, 'chroot')
boardpath = os.path.join(chroot_path, 'build', board)
if options.sync:
if options.clobber or not os.path.isdir(buildroot):
_FullCheckout(buildroot, tracking_branch, url=options.url)
else:
old_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST)
_IncrementalCheckout(buildroot)
new_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST)
if old_binhost != new_binhost:
RunCommand(['sudo', 'rm', '-rf', boardpath])
# Check that all overlays can be found.
for path in overlays:
@ -575,11 +637,9 @@ def main():
if not os.path.isdir(path):
Die('Missing overlay: %s' % path)
chroot_path = os.path.join(buildroot, 'chroot')
if not os.path.isdir(chroot_path):
_MakeChroot(buildroot)
boardpath = os.path.join(chroot_path, 'build', buildconfig['board'])
if not os.path.isdir(boardpath):
_SetupBoard(buildroot, board=buildconfig['board'])
@ -624,6 +684,7 @@ def main():
if buildconfig['master']:
# Master bot needs to check if the other slaves completed.
if cbuildbot_comm.HaveSlavesCompleted(config):
_UploadPrebuilts(buildroot, board, buildconfig['overlays'])
_UprevPush(buildroot, tracking_branch, buildconfig['board'],
overlays, options.debug)
else:

39
bin/cbuildbot_unittest.py
View File

@ -151,10 +151,13 @@ class CBuildBotTest(mox.MoxTestBase):
m_file.read().AndReturn(self._test_string)
m_file.close()
drop_file = cbuildbot._PACKAGE_FILE % {'buildroot': self._buildroot}
cbuildbot.RunCommand(['./cros_mark_as_stable', '--all',
'--board=%s' % self._test_board,
'--overlays=%s' % ':'.join(self._chroot_overlays),
'--tracking_branch=cros/master', 'commit'],
'--tracking_branch=cros/master',
'--drop_file=%s' % ReinterpretPathForChroot(drop_file),
'commit'],
cwd='%s/src/scripts' % self._buildroot,
enter_chroot=True)
@ -174,10 +177,13 @@ class CBuildBotTest(mox.MoxTestBase):
m_file.read().AndReturn('None')
m_file.close()
drop_file = cbuildbot._PACKAGE_FILE % {'buildroot': self._buildroot}
cbuildbot.RunCommand(['./cros_mark_as_stable', '--all',
'--board=%s' % self._test_board,
'--overlays=%s' % ':'.join(self._chroot_overlays),
'--tracking_branch=cros/master', 'commit'],
'--tracking_branch=cros/master',
'--drop_file=%s' % ReinterpretPathForChroot(drop_file),
'commit'],
cwd='%s/src/scripts' % self._buildroot,
enter_chroot=True)
@ -187,6 +193,35 @@ class CBuildBotTest(mox.MoxTestBase):
self._overlays)
self.mox.VerifyAll()
def testGetPortageEnvVar(self):
"""Basic test case for _GetPortageEnvVar function."""
envvar = 'EXAMPLE'
cbuildbot.RunCommand(mox.And(mox.IsA(list), mox.In(envvar)),
cwd='%s/src/scripts' % self._buildroot,
redirect_stdout=True, enter_chroot=True,
error_ok=True).AndReturn('RESULT\n')
self.mox.ReplayAll()
result = cbuildbot._GetPortageEnvVar(self._buildroot, self._test_board,
envvar)
self.mox.VerifyAll()
self.assertEqual(result, 'RESULT')
def testUploadPublicPrebuilts(self):
"""Test _UploadPrebuilts with a public location."""
check = mox.And(mox.IsA(list), mox.In('gs://chromeos-prebuilt'))
cbuildbot.RunCommand(check, cwd='%s/src/scripts' % self._buildroot)
self.mox.ReplayAll()
cbuildbot._UploadPrebuilts(self._buildroot, self._test_board, 'public')
self.mox.VerifyAll()
def testUploadPrivatePrebuilts(self):
"""Test _UploadPrebuilts with a private location."""
check = mox.And(mox.IsA(list), mox.In('chromeos-images:/var/www/prebuilt/'))
cbuildbot.RunCommand(check, cwd='%s/src/scripts' % self._buildroot)
self.mox.ReplayAll()
cbuildbot._UploadPrebuilts(self._buildroot, self._test_board, 'private')
self.mox.VerifyAll()
if __name__ == '__main__':
unittest.main()

70
prebuilt.py
View File

@ -59,6 +59,7 @@ _PREBUILT_BASE_DIR = 'src/third_party/chromiumos-overlay/chromeos/config/'
# Created in the event of new host targets becoming available
_PREBUILT_MAKE_CONF = {'amd64': os.path.join(_PREBUILT_BASE_DIR,
'make.conf.amd64-host')}
_BINHOST_CONF_DIR = 'src/third_party/chromiumos-overlay/chromeos/binhost'
class FiltersEmpty(Exception):
@ -92,6 +93,7 @@ def UpdateLocalFile(filename, value, key='PORTAGE_BINHOST'):
file_fh = open(filename)
file_lines = []
found = False
keyval_str = '%(key)s=%(value)s'
for line in file_fh:
# Strip newlines from end of line. We already add newlines below.
line = line.rstrip("\n")
@ -102,7 +104,6 @@ def UpdateLocalFile(filename, value, key='PORTAGE_BINHOST'):
continue
file_var, file_val = line.split('=')
keyval_str = '%(key)s=%(value)s'
if file_var == key:
found = True
print 'Updating %s=%s to %s="%s"' % (file_var, file_val, key, value)
@ -117,7 +118,7 @@ def UpdateLocalFile(filename, value, key='PORTAGE_BINHOST'):
file_fh.close()
# write out new file
new_file_fh = open(filename, 'w')
new_file_fh.write('\n'.join(file_lines))
new_file_fh.write('\n'.join(file_lines) + '\n')
new_file_fh.close()
@ -143,7 +144,7 @@ def RevGitPushWithRetry(retries=5):
raise GitPushFailed('Failed to push change after %s retries' % retries)
def RevGitFile(filename, value, retries=5):
def RevGitFile(filename, value, retries=5, key='PORTAGE_BINHOST'):
"""Update and push the git file.
Args:
@ -151,6 +152,8 @@ def RevGitFile(filename, value, retries=5):
value: string representing the version of the prebuilt that has been
uploaded.
retries: The number of times to retry before giving up, default: 5
key: The variable key to update in the git file.
(Default: PORTAGE_BINHOST)
"""
prebuilt_branch = 'prebuilt_branch'
old_cwd = os.getcwd()
@ -162,10 +165,10 @@ def RevGitFile(filename, value, retries=5):
'git config url.ssh://git@gitrw.chromium.org:9222.pushinsteadof '
'http://git.chromium.org/git')
cros_build_lib.RunCommand(git_ssh_config_cmd, shell=True)
description = 'Update PORTAGE_BINHOST="%s" in %s' % (value, filename)
description = 'Update %s="%s" in %s' % (key, value, filename)
print description
try:
UpdateLocalFile(filename, value)
UpdateLocalFile(filename, value, key)
cros_build_lib.RunCommand('git config push.default tracking', shell=True)
cros_build_lib.RunCommand('git commit -am "%s"' % description, shell=True)
RevGitPushWithRetry(retries)
@ -405,8 +408,32 @@ def DetermineMakeConfFile(target):
return os.path.join(make_path)
def UpdateBinhostConfFile(path, key, value):
"""Update binhost config file file with key=value.
Args:
path: Filename to update.
key: Key to update.
value: New value for key.
"""
cwd = os.path.dirname(os.path.abspath(path))
filename = os.path.basename(path)
if not os.path.isdir(cwd):
os.makedirs(cwd)
if not os.path.isfile(path):
config_file = file(path, 'w')
config_file.write('FULL_BINHOST="$PORTAGE_BINHOST"\n')
config_file.close()
UpdateLocalFile(path, value, key)
cros_build_lib.RunCommand('git add %s' % filename, cwd=cwd, shell=True)
description = 'Update %s=%s in %s' % (key, value, filename)
cros_build_lib.RunCommand('git commit -m "%s"' % description, cwd=cwd,
shell=True)
def UploadPrebuilt(build_path, upload_location, version, binhost_base_url,
board=None, git_sync=False, git_sync_retries=5):
board=None, git_sync=False, git_sync_retries=5,
key='PORTAGE_BINHOST', sync_binhost_conf=False):
"""Upload Host prebuilt files to Google Storage space.
Args:
@ -415,10 +442,13 @@ def UploadPrebuilt(build_path, upload_location, version, binhost_base_url,
board: The board to upload to Google Storage, if this is None upload
host packages.
git_sync: If set, update make.conf of target to reference the latest
prebuilt packages genereated here.
prebuilt packages generated here.
git_sync_retries: How many times to retry pushing when updating git files.
This helps avoid failures when multiple bots are modifying the same Repo.
default: 5
key: The variable key to update in the git file. (Default: PORTAGE_BINHOST)
sync_binhost_conf: If set, update binhost config file in chromiumos-overlay
for the current board or host.
"""
if not board:
@ -428,12 +458,16 @@ def UploadPrebuilt(build_path, upload_location, version, binhost_base_url,
url_suffix = _REL_HOST_PATH % {'version': version, 'target': _HOST_TARGET}
package_string = _HOST_TARGET
git_file = os.path.join(build_path, _PREBUILT_MAKE_CONF[_HOST_TARGET])
binhost_conf = os.path.join(build_path, _BINHOST_CONF_DIR, 'host',
'%s.conf' % _HOST_TARGET)
else:
board_path = os.path.join(build_path, _BOARD_PATH % {'board': board})
package_path = os.path.join(board_path, 'packages')
package_string = board
url_suffix = _REL_BOARD_PATH % {'board': board, 'version': version}
git_file = os.path.join(build_path, DetermineMakeConfFile(board))
binhost_conf = os.path.join(build_path, _BINHOST_CONF_DIR, 'target',
'%s.conf' % board)
remote_location = os.path.join(upload_location, url_suffix)
if upload_location.startswith('gs://'):
@ -452,10 +486,13 @@ def UploadPrebuilt(build_path, upload_location, version, binhost_base_url,
if not _RetryRun(cmd, shell=True):
raise UploadFailed('Could not run %s' % cmd)
if git_sync:
url_value = '%s/%s/' % (binhost_base_url, url_suffix)
RevGitFile(git_file, url_value, retries=git_sync_retries)
url_value = '%s/%s/' % (binhost_base_url, url_suffix)
if git_sync:
RevGitFile(git_file, url_value, retries=git_sync_retries, key=key)
if sync_binhost_conf:
UpdateBinhostConfFile(binhost_conf, key, url_value)
def usage(parser, msg):
"""Display usage message and parser help then exit with 1."""
@ -488,6 +525,12 @@ def main():
parser.add_option('-f', '--filters', dest='filters', action='store_true',
default=False,
help='Turn on filtering of private ebuild packages')
parser.add_option('-k', '--key', dest='key',
default='PORTAGE_BINHOST',
help='Key to update in make.conf / binhost.conf')
parser.add_option('', '--sync-binhost-conf', dest='sync_binhost_conf',
default=False, action='store_true',
help='Update binhost.conf')
options, args = parser.parse_args()
# Setup boto environment for gsutil to use
@ -511,12 +554,15 @@ def main():
if options.sync_host:
UploadPrebuilt(options.build_path, options.upload, version,
options.binhost_base_url, git_sync=options.git_sync)
options.binhost_base_url, git_sync=options.git_sync,
key=options.key,
sync_binhost_conf=options.sync_binhost_conf)
if options.board:
UploadPrebuilt(options.build_path, options.upload, version,
options.binhost_base_url, board=options.board,
git_sync=options.git_sync)
git_sync=options.git_sync, key=options.key,
sync_binhost_conf=options.sync_binhost_conf)
if __name__ == '__main__':