From e9b94eefd0733481424fd34beeff03a1dfd42929 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Thu, 26 Aug 2010 10:55:27 -0700 Subject: [PATCH] CBuildbot - Adds ability to only rev packages given by the sourcestamp. The buildbot drops a source stamp (repo1.git@commit1 repo2.git@commit2 ...). If CBuildbot finds the source stamp, it uses it and revs only those packages. If this file is either missing or set to "None", it assumes a Force Build has been pressed and keeps the old behavior (attempts to mark all). TEST=Ran with 3 states of revisions file. BUG=5006 Review URL: http://codereview.chromium.org/3163030 Change-Id: I226fd3bec642224b31ce51eee34d028043964943 --- bin/cbuildbot.py | 252 +++++++++++++++++++++++++++++++++----- bin/cbuildbot_unittest.py | 102 +++++++++++++++ 2 files changed, 321 insertions(+), 33 deletions(-) create mode 100755 bin/cbuildbot_unittest.py diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index 972b48b539..aa3a1d851b 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -4,9 +4,12 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +"""CBuildbot is wrapper around the build process used by the pre-flight queue""" + import errno import optparse import os +import re import shutil import subprocess import sys @@ -15,26 +18,40 @@ from cbuildbot_config import config _DEFAULT_RETRIES = 3 -# Utility functions +# ======================== Utility functions ================================ def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None, exit_code=False, redirect_stdout=False, redirect_stderr=False, - cwd=None, input=None): + cwd=None, input=None, enter_chroot=False): + """Runs a shell command. + + Keyword arguments: + print_cmd -- prints the command before running it. + error_ok -- does not raise an exception on error. + error_message -- prints out this message when an error occurrs. + exit_code -- returns the return code of the shell command. + redirect_stdout -- returns the stdout. + redirect_stderr -- holds stderr output until input is communicated. + cwd -- the working directory to run this cmd. + input -- input to pipe into this command through stdin. + enter_chroot -- this command should be run from within the chroot. + + """ + # Set default for variables. + stdout = None + stderr = None + stdin = None + + # Modify defaults based on parameters. + if redirect_stdout: stdout = subprocess.PIPE + if redirect_stderr: stderr = subprocess.PIPE + if input: stdin = subprocess.PIPE + if enter_chroot: cmd = ['./enter_chroot.sh', '--'] + cmd + # Print out the command before running. if print_cmd: - print >> sys.stderr, "CBUILDBOT -- RunCommand:", ' '.join(cmd) - if redirect_stdout: - stdout = subprocess.PIPE - else: - stdout = None - if redirect_stderr: - stderr = subprocess.PIPE - else: - stderr = None - if input: - stdin = subprocess.PIPE - else: - stdin = None + print >> sys.stderr, 'CBUILDBOT -- RunCommand: ', ' '.join(cmd) + proc = subprocess.Popen(cmd, cwd=cwd, stdin=stdin, stdout=stdout, stderr=stderr) (output, error) = proc.communicate(input) @@ -45,7 +62,15 @@ def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None, (error_message or error or output or '')) return output + def MakeDir(path, parents=False): + """Basic wrapper around os.mkdirs. + + Keyword arguments: + path -- Path to create. + parents -- Follow mkdir -p logic. + + """ try: os.makedirs(path) except OSError, e: @@ -54,7 +79,15 @@ def MakeDir(path, parents=False): else: raise -def RepoSync(buildroot, rw_checkout, retries=_DEFAULT_RETRIES): + +def RepoSync(buildroot, rw_checkout=False, retries=_DEFAULT_RETRIES): + """Uses repo to checkout the source code. + + Keyword arguments: + rw_checkout -- Reconfigure repo after sync'ing to read-write. + retries -- Number of retries to try before failing on the sync. + + """ while retries > 0: try: RunCommand(['repo', 'sync'], cwd=buildroot) @@ -73,70 +106,221 @@ def RepoSync(buildroot, rw_checkout, retries=_DEFAULT_RETRIES): print >> sys.stderr, 'CBUILDBOT -- Retries exhausted' raise -# Main functions +# =========================== Command Helpers ================================= + +def _GetAllGitRepos(buildroot, debug=False): + """Returns a list of tuples containing [git_repo, src_path].""" + manifest_tuples = [] + # Gets all the git repos from a full repo manifest. + repo_cmd = "repo manifest -o -".split() + output = RunCommand(repo_cmd, cwd=buildroot, redirect_stdout=True, + redirect_stderr=True, print_cmd=debug) + + # Extract all lines containg a project. + extract_cmd = ["grep", "project name="] + output = RunCommand(extract_cmd, cwd=buildroot, input=output, + redirect_stdout=True, print_cmd=debug) + # Parse line using re to get tuple. + result_array = re.findall('.+name=\"([\w-]+)\".+path=\"(\S+)".+', output) + + # Create the array. + for result in result_array: + if len(result) != 2: + print >> sys.stderr, 'Found in correct xml object %s', result + else: + # Remove pre-pended src directory from manifest. + manifest_tuples.append([result[0], result[1].replace('src/', '')]) + return manifest_tuples + + +def _GetCrosWorkOnSrcPath(buildroot, board, package, debug=False): + """Returns ${CROS_WORKON_SRC_PATH} for given package.""" + cwd = os.path.join(buildroot, 'src', 'scripts') + equery_cmd = ('equery-%s which %s' % (board, package)).split() + ebuild_path = RunCommand(equery_cmd, cwd=cwd, redirect_stdout=True, + redirect_stderr=True, enter_chroot=True, + error_ok=True, print_cmd=debug) + if ebuild_path: + ebuild_cmd = ('ebuild-%s %s info' % (board, ebuild_path)).split() + cros_workon_output = RunCommand(ebuild_cmd, cwd=cwd, + redirect_stdout=True, redirect_stderr=True, + enter_chroot=True, print_cmd=debug) + + temp = re.findall('CROS_WORKON_SRCDIR="(\S+)"', cros_workon_output) + if temp: + return temp[0] + return None + + +def _CreateRepoDictionary(buildroot, board, debug=False): + """Returns the repo->list_of_ebuilds dictionary.""" + repo_dictionary = {} + manifest_tuples = _GetAllGitRepos(buildroot) + print >> sys.stderr, 'Creating dictionary of git repos to portage packages ...' + + cwd = os.path.join(buildroot, 'src', 'scripts') + get_all_workon_pkgs_cmd = './cros_workon list --all'.split() + packages = RunCommand(get_all_workon_pkgs_cmd, cwd=cwd, + redirect_stdout=True, redirect_stderr=True, + enter_chroot=True, print_cmd=debug) + for package in packages.split(): + cros_workon_src_path = _GetCrosWorkOnSrcPath(buildroot, board, package) + if cros_workon_src_path: + for tuple in manifest_tuples: + # This path tends to have the user's home_dir prepended to it. + if cros_workon_src_path.endswith(tuple[1]): + print >> sys.stderr, ('For %s found matching package %s' % + (tuple[0], package)) + if repo_dictionary.has_key(tuple[0]): + repo_dictionary[tuple[0]] += [package] + else: + repo_dictionary[tuple[0]] = [package] + return repo_dictionary + + +def _ParseRevisionString(revision_string, repo_dictionary): + """Parses the given revision_string into a revision dictionary. + + Returns a list of tuples that contain [portage_package_name, commit_id] to + update. + + Keyword arguments: + revision_string -- revision_string with format + 'repo1.git@commit_1 repo2.git@commit2 ...'. + repo_dictionary -- dictionary with git repository names as keys (w/out git) + to portage package names. + + """ + # Using a dictionary removes duplicates. + revisions = {} + for revision in revision_string.split(): + # Format 'package@commit-id'. + revision_tuple = revision.split('@') + if len(revision_tuple) != 2: + print >> sys.stderr, 'Incorrectly formatted revision %s' % revision + repo_name = revision_tuple[0].replace('.git', '') + # May be many corresponding packages to a given git repo e.g. kernel) + for package in repo_dictionary[repo_name]: + revisions[package] = revision_tuple[1] + return revisions.items() + + +def _UprevFromRevisionList(buildroot, revision_list): + """Uprevs based on revision list.""" + package_str = '' + commit_str = '' + for package, revision in revision_list: + package_str += package + ' ' + commit_str += revision + ' ' + package_str = package_str.strip() + commit_str = commit_str.strip() + + cwd = os.path.join(buildroot, 'src', 'scripts') + RunCommand(['./cros_mark_as_stable', + '--tracking_branch="cros/master"', + '--packages="%s"' % package_str, + '--commit_ids="%s"' % commit_str, + 'commit'], + cwd=cwd, enter_chroot=True) + + +def _UprevAllPackages(buildroot): + """Uprevs all packages that have been updated since last uprev.""" + cwd = os.path.join(buildroot, 'src', 'scripts') + RunCommand(['./cros_mark_all_as_stable', + '--tracking_branch="cros/master"'], + cwd=cwd, enter_chroot=True) + +# =========================== Main Commands =================================== def _FullCheckout(buildroot, rw_checkout=True, retries=_DEFAULT_RETRIES): + """Performs a full checkout and clobbers any previous checkouts.""" RunCommand(['sudo', 'rm', '-rf', buildroot]) MakeDir(buildroot, parents=True) RunCommand(['repo', 'init', '-u', 'http://src.chromium.org/git/manifest'], cwd=buildroot, input='\n\ny\n') RepoSync(buildroot, rw_checkout, retries) + def _IncrementalCheckout(buildroot, rw_checkout=True, retries=_DEFAULT_RETRIES): + """Performs a checkout without clobbering previous checkout.""" RepoSync(buildroot, rw_checkout, retries) + def _MakeChroot(buildroot): + """Wrapper around make_chroot.""" cwd = os.path.join(buildroot, 'src', 'scripts') RunCommand(['./make_chroot', '--fast'], cwd=cwd) + def _SetupBoard(buildroot, board='x86-generic'): + """Wrapper around setup_board.""" cwd = os.path.join(buildroot, 'src', 'scripts') RunCommand(['./setup_board', '--fast', '--default', '--board=%s' % board], cwd=cwd) + def _Build(buildroot): + """Wrapper around build_packages.""" cwd = os.path.join(buildroot, 'src', 'scripts') RunCommand(['./build_packages'], cwd=cwd) -def _UprevAllPackages(buildroot): - cwd = os.path.join(buildroot, 'src', 'scripts') - RunCommand(['./enter_chroot.sh', '--', './cros_mark_all_as_stable', - '--tracking_branch="cros/master"'], - cwd=cwd) -def _UprevPackages(buildroot, revisionfile): - revisions = None +def _UprevPackages(buildroot, revisionfile, board): + """Uprevs a package based on given revisionfile. + + If revisionfile is set to None or does not resolve to an actual file, this + function will uprev all packages. + + Keyword arguments: + revisionfile -- string specifying a file that contains a list of revisions to + uprev. + """ + # Purposefully set to None as it means Force Build was pressed. + revisions = 'None' if (revisionfile): try: rev_file = open(revisionfile) revisions = rev_file.read() rev_file.close() - except: - print >> sys.stderr, 'Error reading %s' % revisionfile - revisions = None + except Exception, e: + print >> sys.stderr, 'Error reading %s, revving all' % revisionfile + print e + revisions = 'None' - # Note: Revisions == "None" indicates a Force Build. - if revisions and revisions != 'None': - print 'CBUILDBOT - Revision list found %s' % revisions - print 'Revision list not yet propagating to build, marking all instead' + revisions = revisions.strip() + + # Revisions == "None" indicates a Force Build. + if revisions != 'None': + print >> sys.stderr, 'CBUILDBOT Revision list found %s' % revisions + revision_list = _ParseRevisionString(revisions, + _CreateRepoDictionary(buildroot, board)) + _UprevFromRevisionList(buildroot, revision_list) + else: + print >> sys.stderr, 'CBUILDBOT Revving all' + _UprevAllPackages(buildroot) - _UprevAllPackages(buildroot) def _UprevCleanup(buildroot): + """Clean up after a previous uprev attempt.""" cwd = os.path.join(buildroot, 'src', 'scripts') RunCommand(['./cros_mark_as_stable', '--srcroot=..', '--tracking_branch="cros/master"', 'clean'], cwd=cwd) + def _UprevPush(buildroot): + """Pushes uprev changes to the main line.""" cwd = os.path.join(buildroot, 'src', 'scripts') RunCommand(['./cros_mark_as_stable', '--srcroot=..', '--tracking_branch="cros/master"', '--push_options', '--bypass-hooks -f', 'push'], cwd=cwd) + def _GetConfig(config_name): + """Gets the configuration for the build""" default = config['default'] buildconfig = {} if config.has_key(config_name): @@ -146,6 +330,7 @@ def _GetConfig(config_name): buildconfig[key] = default[key] return buildconfig + def main(): # Parse options usage = "usage: %prog [options] cbuildbot_config" @@ -183,7 +368,7 @@ def main(): if not os.path.isdir(boardpath): _SetupBoard(buildroot, board=buildconfig['board']) if buildconfig['uprev']: - _UprevPackages(buildroot, revisionfile) + _UprevPackages(buildroot, revisionfile, board=buildconfig['board']) _Build(buildroot) if buildconfig['uprev']: _UprevPush(buildroot) @@ -194,5 +379,6 @@ def main(): RunCommand(['sudo', 'rm', '-rf', buildroot], print_cmd=False) raise + if __name__ == '__main__': main() diff --git a/bin/cbuildbot_unittest.py b/bin/cbuildbot_unittest.py new file mode 100755 index 0000000000..c0f3c1c7cd --- /dev/null +++ b/bin/cbuildbot_unittest.py @@ -0,0 +1,102 @@ +#!/usr/bin/python + +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Unittests for cbuildbot. Needs to be run inside of chroot for mox.""" + +import __builtin__ +import mox +import unittest + +import cbuildbot + +class CBuildBotTest(mox.MoxTestBase): + + def setUp(self): + mox.MoxTestBase.setUp(self) + # Always stub RunCommmand out as we use it in every method. + self.mox.StubOutWithMock(cbuildbot, 'RunCommand') + self._test_repos = [['kernel', 'third_party/kernel/files'], + ['login_manager', 'platform/login_manager'] + ] + self._test_cros_workon_packages = \ + 'chromeos-base/kernel\nchromeos-base/chromeos-login\n' + self._test_board = 'test-board' + self._buildroot = '.' + self._test_dict = {'kernel' : ['chromos-base/kernel', 'dev-util/perf'], + 'cros' : ['chromos-base/libcros'] + } + self._test_string = "kernel.git@12345test cros.git@12333test" + self._revision_file = 'test-revisions.pfq' + self._test_parsed_string_array = [ + ['chromeos-base/kernel', '12345test'], + ['dev-util/perf', '12345test'], + ['chromos-base/libcros', '12345test'] + ] + + def testParseRevisionString(self): + """Test whether _ParseRevisionString parses string correctly.""" + return_array = cbuildbot._ParseRevisionString(self._test_string, + self._test_dict) + self.assertEqual(len(return_array), 3) + self.assertTrue( + 'chromeos-base/kernel', '12345test' in return_array) + self.assertTrue( + 'dev-util/perf', '12345test' in return_array) + self.assertTrue( + 'chromos-base/libcros', '12345test' in return_array) + + def testCreateDictionary(self): + self.mox.StubOutWithMock(cbuildbot, '_GetAllGitRepos') + self.mox.StubOutWithMock(cbuildbot, '_GetCrosWorkOnSrcPath') + cbuildbot._GetAllGitRepos(mox.IgnoreArg()).AndReturn(self._test_repos) + cbuildbot.RunCommand(mox.IgnoreArg(), + cwd='%s/src/scripts' % self._buildroot, + redirect_stdout=True, + redirect_stderr=True, + enter_chroot=True, + print_cmd=False).AndReturn( + self._test_cros_workon_packages) + cbuildbot._GetCrosWorkOnSrcPath(self._buildroot, self._test_board, + 'chromeos-base/kernel').AndReturn( + '/home/test/third_party/kernel/files') + cbuildbot._GetCrosWorkOnSrcPath(self._buildroot, self._test_board, + 'chromeos-base/chromeos-login').AndReturn( + '/home/test/platform/login_manager') + self.mox.ReplayAll() + repo_dict = cbuildbot._CreateRepoDictionary(self._buildroot, + self._test_board) + self.assertEqual(repo_dict['kernel'], ['chromeos-base/kernel']) + self.assertEqual(repo_dict['login_manager'], + ['chromeos-base/chromeos-login']) + self.mox.VerifyAll() + + def testUprevPackages(self): + self.mox.StubOutWithMock(cbuildbot, '_CreateRepoDictionary') + self.mox.StubOutWithMock(cbuildbot, '_ParseRevisionString') + self.mox.StubOutWithMock(cbuildbot, '_UprevFromRevisionList') + self.mox.StubOutWithMock(__builtin__, 'open') + + # Mock out file interaction. + m_file = self.mox.CreateMock(file) + __builtin__.open(self._revision_file).AndReturn(m_file) + m_file.read().AndReturn(self._test_string) + m_file.close() + + cbuildbot._CreateRepoDictionary(self._buildroot, + self._test_board).AndReturn(self._test_dict) + cbuildbot._ParseRevisionString(self._test_string, + self._test_dict).AndReturn( + self._test_parsed_string_array) + cbuildbot._UprevFromRevisionList(self._buildroot, + self._test_parsed_string_array) + self.mox.ReplayAll() + cbuildbot._UprevPackages(self._buildroot, self._revision_file, + self._test_board) + self.mox.VerifyAll() + + +if __name__ == '__main__': + unittest.main()