diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index 372ddcaf5f..e723cd6439 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -18,7 +18,8 @@ import cbuildbot_comm from cbuildbot_config import config sys.path.append(os.path.join(os.path.dirname(__file__), '../lib')) -from cros_build_lib import Die, Info, RunCommand, Warning +from cros_build_lib import (Die, Info, ReinterpretPathForChroot, RunCommand, + Warning) _DEFAULT_RETRIES = 3 ARCHIVE_BASE = '/var/www/archive' @@ -181,37 +182,38 @@ def _ParseRevisionString(revision_string, repo_dictionary): return revisions.items() -def _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board): +def _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board, + overlays): """Uprevs based on revision list.""" if not revision_list: Info('No packages found to uprev') return - package_str = '' + packages = [] for package, revision in revision_list: - package_str += package + ' ' + assert ':' not in package, 'Invalid package name: %s' % package + packages.append(package) - package_str = package_str.strip() + chroot_overlays = [ReinterpretPathForChroot(path) for path in overlays] cwd = os.path.join(buildroot, 'src', 'scripts') - # TODO(davidjames): --foo="bar baz" only works here because we're using - # enter_chroot. RunCommand(['./cros_mark_as_stable', '--board=%s' % board, - '--tracking_branch="%s"' % tracking_branch, - '--packages="%s"' % package_str, + '--tracking_branch=%s' % tracking_branch, + '--overlays=%s' % ':'.join(chroot_overlays), + '--packages=%s' % ':'.join(packages), 'commit'], - cwd=cwd, enter_chroot=True) + cwd=cwd, enter_chroot=True) -def _UprevAllPackages(buildroot, tracking_branch, board): +def _UprevAllPackages(buildroot, tracking_branch, board, overlays): """Uprevs all packages that have been updated since last uprev.""" cwd = os.path.join(buildroot, 'src', 'scripts') - # TODO(davidjames): --foo="bar baz" only works here because we're using - # enter_chroot. + chroot_overlays = [ReinterpretPathForChroot(path) for path in overlays] RunCommand(['./cros_mark_as_stable', '--all', '--board=%s' % board, - '--tracking_branch="%s"' % tracking_branch, 'commit'], + '--overlays=%s' % ':'.join(chroot_overlays), + '--tracking_branch=%s' % tracking_branch, 'commit'], cwd=cwd, enter_chroot=True) @@ -228,12 +230,13 @@ def _GetVMConstants(buildroot): return (vdisk_size.strip(), statefulfs_size.strip()) -def _GitCleanup(buildroot, board, tracking_branch): +def _GitCleanup(buildroot, board, tracking_branch, overlays): """Clean up git branch after previous uprev attempt.""" cwd = os.path.join(buildroot, 'src', 'scripts') if os.path.exists(cwd): RunCommand(['./cros_mark_as_stable', '--srcroot=..', '--board=%s' % board, + '--overlays=%s' % ':'.join(overlays), '--tracking_branch=%s' % tracking_branch, 'clean'], cwd=cwd, error_ok=True) @@ -257,9 +260,9 @@ def _WipeOldOutput(buildroot): # =========================== Main Commands =================================== -def _PreFlightRinse(buildroot, board, tracking_branch): +def _PreFlightRinse(buildroot, board, tracking_branch, overlays): """Cleans up any leftover state from previous runs.""" - _GitCleanup(buildroot, board, tracking_branch) + _GitCleanup(buildroot, board, tracking_branch, overlays) _CleanUpMountPoints(buildroot) RunCommand(['sudo', 'killall', 'kvm'], error_ok=True) @@ -347,7 +350,7 @@ def _RunSmokeSuite(buildroot, results_dir): ], cwd=cwd, error_ok=False) -def _UprevPackages(buildroot, tracking_branch, revisionfile, board): +def _UprevPackages(buildroot, tracking_branch, revisionfile, board, overlays): """Uprevs a package based on given revisionfile. If revisionfile is set to None or does not resolve to an actual file, this @@ -376,26 +379,19 @@ def _UprevPackages(buildroot, tracking_branch, revisionfile, board): # print >> sys.stderr, 'CBUILDBOT Revision list found %s' % revisions # revision_list = _ParseRevisionString(revisions, # _CreateRepoDictionary(buildroot, board)) - # _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board) + # _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board, + # overlays) #else: Info('CBUILDBOT Revving all') - _UprevAllPackages(buildroot, tracking_branch, board) + _UprevAllPackages(buildroot, tracking_branch, board, overlays) def _UprevPush(buildroot, tracking_branch, board, overlays): """Pushes uprev changes to the main line.""" cwd = os.path.join(buildroot, 'src', 'scripts') - public_overlay = '%s/src/third_party/chromiumos-overlay' % buildroot - private_overlay = '%s/src/private-overlays/chromeos-overlay' % buildroot - if overlays == 'private': - overlays = [private_overlay] - elif overlays == 'public': - overlays = [public_overlay] - else: - overlays = [public_overlay, private_overlay] RunCommand(['./cros_mark_as_stable', '--srcroot=..', '--board=%s' % board, - '--overlays=%s' % " ".join(overlays), + '--overlays=%s' % ':'.join(overlays), '--tracking_branch=%s' % tracking_branch, '--push_options=--bypass-hooks -f', 'push'], cwd=cwd) @@ -467,6 +463,30 @@ def _GetConfig(config_name): return buildconfig +def _ResolveOverlays(buildroot, overlays): + """Return the list of overlays to use for a given buildbot. + + Args: + buildroot: The root directory where the build occurs. Must be an absolute + path. + overlays: A string describing which overlays you want. + 'private': Just the private overlay. + 'public': Just the public overlay. + 'both': Both the public and private overlays. + """ + public_overlay = '%s/src/third_party/chromiumos-overlay' % buildroot + private_overlay = '%s/src/private-overlays/chromeos-overlay' % buildroot + if overlays == 'private': + paths = [private_overlay] + elif overlays == 'public': + paths = [public_overlay] + elif overlays == 'both': + paths = [public_overlay, private_overlay] + else: + Die('Incorrect overlay configuration: %s' % overlays) + return paths + + def main(): # Parse options usage = "usage: %prog [options] cbuildbot_config" @@ -491,7 +511,7 @@ def main(): (options, args) = parser.parse_args() - buildroot = options.buildroot + buildroot = os.path.abspath(options.buildroot) revisionfile = options.revisionfile tracking_branch = options.tracking_branch @@ -502,13 +522,22 @@ def main(): parser.print_usage() sys.exit(1) + # Calculate list of overlay directories. + overlays = _ResolveOverlays(buildroot, buildconfig['overlays']) + try: - _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch) + _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays) if options.clobber or not os.path.isdir(buildroot): _FullCheckout(buildroot, tracking_branch, url=options.url) else: _IncrementalCheckout(buildroot) + # Check that all overlays can be found. + for path in overlays: + assert ':' not in path, 'Overlay must not contain colons: %s' % path + if not os.path.isdir(path): + Die('Missing overlay: %s' % path) + chroot_path = os.path.join(buildroot, 'chroot') if not os.path.isdir(chroot_path): _MakeChroot(buildroot) @@ -519,7 +548,7 @@ def main(): if buildconfig['uprev']: _UprevPackages(buildroot, tracking_branch, revisionfile, - board=buildconfig['board']) + buildconfig['board'], overlays) _EnableLocalAccount(buildroot) _Build(buildroot) @@ -545,7 +574,7 @@ def main(): # Master bot needs to check if the other slaves completed. if cbuildbot_comm.HaveSlavesCompleted(config): _UprevPush(buildroot, tracking_branch, buildconfig['board'], - buildconfig['overlays']) + overlays) else: Die('CBUILDBOT - One of the slaves has failed!!!') diff --git a/bin/cbuildbot_config.py b/bin/cbuildbot_config.py index 831ef06bd1..718cd36a70 100644 --- a/bin/cbuildbot_config.py +++ b/bin/cbuildbot_config.py @@ -19,9 +19,9 @@ hostname -- Needed for 'important' slaves. The hostname of the bot. Should match hostname in slaves.cfg in buildbot checkout. unittests -- Runs unittests for packages. smoke_bvt -- Runs the test smoke suite in a qemu-based VM using KVM. -overlays -- If this bot is a master bot, select what overlays to push changes - to. This can be 'public', 'private', or 'both'. There should only - be one bot pushing changes to each overlay. +overlays -- Select what overlays to look at. This can be 'public', 'private' + or 'both'. There should only be one master bot pushing changes to + each overlay per branch. """ @@ -33,6 +33,7 @@ config['default'] = { 'important' : False, 'unittests' : False, 'smoke_bvt' : False, + 'overlays': 'public', } config['x86-generic-pre-flight-queue'] = { 'board' : 'x86-generic', @@ -69,6 +70,7 @@ config['x86_agz_bin'] = { 'important' : False, 'unittests' : True, 'smoke_bvt' : True, + 'overlays': 'private', } config['x86_dogfood_bin'] = { 'board' : 'x86-dogfood', @@ -77,52 +79,29 @@ config['x86_dogfood_bin'] = { 'important' : False, 'unittests' : True, 'smoke_bvt' : True, + 'overlays': 'private', } config['x86_pineview_bin'] = { 'board' : 'x86-pineview', 'uprev' : True, 'master' : False, 'important' : False, - 'hostname' : 'codf200.jail', 'unittests': True, + 'overlays': 'public', } config['arm_tegra2_bin'] = { 'board' : 'tegra2', 'uprev' : True, 'master' : False, 'important' : False, - 'hostname' : 'codg172.jail', - 'unittests' : False, -} -config['arm_voguev210_bin'] = { - 'board' : 'voguev210', - 'uprev' : True, - 'master' : False, - 'important' : False, - 'hostname' : 'codf196.jail', - 'unittests' : False, -} -config['arm_beagleboard_bin'] = { - 'board' : 'beagleboard', - 'master' : False, - 'uprev' : True, - 'important' : False, - 'hostname' : 'codf202.jail', - 'unittests' : False, -} -config['arm_st1q_bin'] = { - 'board' : 'st1q', - 'uprev' : True, - 'master' : False, - 'important' : False, - 'hostname' : 'codg158.jail', 'unittests' : False, + 'overlays': 'public', } config['arm_generic_bin'] = { 'board' : 'arm-generic', 'uprev' : True, 'master' : False, 'important' : False, - 'hostname' : 'codg175.jail', 'unittests' : False, + 'overlays': 'public', } diff --git a/bin/cbuildbot_unittest.py b/bin/cbuildbot_unittest.py index bbbc56c6f7..979cda55d4 100755 --- a/bin/cbuildbot_unittest.py +++ b/bin/cbuildbot_unittest.py @@ -16,6 +16,7 @@ import unittest # Fixes circular dependency error. import cbuildbot_comm import cbuildbot +from cros_build_lib import ReinterpretPathForChroot class CBuildBotTest(mox.MoxTestBase): @@ -42,6 +43,10 @@ class CBuildBotTest(mox.MoxTestBase): ['dev-util/perf', '12345test'], ['chromos-base/libcros', '12345test'] ] + self._overlays = ['%s/src/third_party/chromiumos-overlay' % self._buildroot] + self._chroot_overlays = [ + ReinterpretPathForChroot(p) for p in self._overlays + ] def testParseRevisionString(self): """Test whether _ParseRevisionString parses string correctly.""" @@ -168,13 +173,15 @@ class CBuildBotTest(mox.MoxTestBase): cbuildbot.RunCommand(['./cros_mark_as_stable', '--all', '--board=%s' % self._test_board, - '--tracking_branch="cros/master"', 'commit'], + '--overlays=%s' % ':'.join(self._chroot_overlays), + '--tracking_branch=cros/master', 'commit'], cwd='%s/src/scripts' % self._buildroot, enter_chroot=True) self.mox.ReplayAll() cbuildbot._UprevPackages(self._buildroot, self.tracking_branch, - self._revision_file, self._test_board) + self._revision_file, self._test_board, + self._overlays) self.mox.VerifyAll() def testUprevAllPackages(self): @@ -189,13 +196,15 @@ class CBuildBotTest(mox.MoxTestBase): cbuildbot.RunCommand(['./cros_mark_as_stable', '--all', '--board=%s' % self._test_board, - '--tracking_branch="cros/master"', 'commit'], + '--overlays=%s' % ':'.join(self._chroot_overlays), + '--tracking_branch=cros/master', 'commit'], cwd='%s/src/scripts' % self._buildroot, enter_chroot=True) self.mox.ReplayAll() cbuildbot._UprevPackages(self._buildroot, self.tracking_branch, - self._revision_file, self._test_board) + self._revision_file, self._test_board, + self._overlays) self.mox.VerifyAll() diff --git a/bin/cros_get_chrome_version b/bin/cros_get_chrome_version new file mode 100755 index 0000000000..a29cb3275f --- /dev/null +++ b/bin/cros_get_chrome_version @@ -0,0 +1,25 @@ +#!/bin/bash + +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Returns the version of Chrome running on a remote machine. + +. "$(dirname $0)/../common.sh" +. "$(dirname $0)/../remote_access.sh" + +FLAGS "$@" || exit 1 + +set -e + +# TMP necessary for remote_access_init. +TMP=$(mktemp -d /tmp/cros_check_chrome_version.XXXX) +trap "rm -rf ${TMP}" EXIT + +remote_access_init &> /dev/null + +remote_sh "/opt/google/chrome/chrome --version" +CHROME_VERSION=$(echo ${REMOTE_OUT} | \ + sed 's/.* \([0-9]\+.[0-9]\+.[0-9]\+.[0-9]\+\).*/\1/') +echo "${CHROME_VERSION}" diff --git a/bin/cros_run_vm_test b/bin/cros_run_vm_test index fbbee23c7c..d62a2b9852 100755 --- a/bin/cros_run_vm_test +++ b/bin/cros_run_vm_test @@ -21,9 +21,17 @@ DEFINE_string results_dir_root "" "alternate root results directory" DEFINE_string test_case "" "Name of the test case to run" DEFINE_boolean use_emerged ${FLAGS_FALSE} \ "Force use of emerged autotest packages" +DEFINE_string verify_chrome_version "" \ + "Verify that this chrome version matches that on vm." set -e +# Returns normally if the given $1 is a valid chrome version. +chrome_version_is_valid() { + local chrome_version="$1" + echo ${chrome_version} | egrep '^[0-9]+.[0-9]+.[0-9]+.[0-9]+$' &> /dev/null +} + # Parse command line. FLAGS "$@" || exit 1 @@ -61,10 +69,24 @@ trap stop_kvm EXIT start_kvm "${FLAGS_image_path}" info "Checking for ssh access to virtual machine." retry_until_ssh ${MAX_RETRIES} + +if [ -n "${FLAGS_verify_chrome_version}" ]; then + info "Verifying version of Chrome matches what we expect." + if chrome_version_is_valid "${FLAGS_verify_chrome_version}"; then + chrome_version_on_vm=$("$(dirname $0)/cros_get_chrome_version" \ + --remote=127.0.0.1 \ + --ssh_port=${FLAGS_ssh_port}) + [[ ${chrome_version_on_vm} == ${FLAGS_verify_chrome_version} ]] || \ + die "Chrome version mismatch. VM reported ${chrome_version_on_vm}" + else + warn "${FLAGS_verify_chrome_version} is not a valid Chrome version" + fi +fi + "$(dirname $0)"/../run_remote_tests.sh \ - --board=${FLAGS_board} \ - --ssh_port=${FLAGS_ssh_port} \ - --remote=127.0.0.1 \ - --results_dir_root="${FLAGS_results_dir_root}" \ - ${USE_EMERGED} \ - "${tests[@]}" + --board=${FLAGS_board} \ + --ssh_port=${FLAGS_ssh_port} \ + --remote=127.0.0.1 \ + --results_dir_root="${FLAGS_results_dir_root}" \ + ${USE_EMERGED} \ + "${tests[@]}" diff --git a/chromeos_version.sh b/chromeos_version.sh index 649898719e..4499741bfe 100755 --- a/chromeos_version.sh +++ b/chromeos_version.sh @@ -26,7 +26,7 @@ export CHROMEOS_VERSION_MINOR=9 # Increment by 2 in trunk after making a release branch. # Does not reset on a major/minor change (always increases). # (Trunk is always odd; branches are always even). -export CHROMEOS_VERSION_BRANCH=113 +export CHROMEOS_VERSION_BRANCH=115 # Patch number. # Increment by 1 each release on a branch. diff --git a/cros_mark_as_stable.py b/cros_mark_as_stable.py index 56a589c682..b2f2cd4067 100755 --- a/cros_mark_as_stable.py +++ b/cros_mark_as_stable.py @@ -22,10 +22,10 @@ from cros_build_lib import Info, RunCommand, Warning, Die gflags.DEFINE_string('board', '', 'Board for which the package belongs.', short_name='b') gflags.DEFINE_string('overlays', '', - 'Space separated list of overlays to modify.', + 'Colon-separated list of overlays to modify.', short_name='o') gflags.DEFINE_string('packages', '', - 'Space separated list of packages to mark as stable.', + 'Colon-separated list of packages to mark as stable.', short_name='p') gflags.DEFINE_string('push_options', '', 'Options to use with git-cl push using push command.') @@ -245,7 +245,11 @@ def _SimpleRunCommand(command): """Runs a shell command and returns stdout back to caller.""" _Print(' + %s' % command) proc_handle = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) - return proc_handle.communicate()[0] + stdout = proc_handle.communicate()[0] + retcode = proc_handle.wait() + if retcode != 0: + raise subprocess.CalledProcessError(retcode, command, output=stdout) + return stdout # ======================= End Global Helper Functions ======================== @@ -323,7 +327,7 @@ class _EBuild(object): # Grab and evaluate CROS_WORKON variables from this ebuild. unstable_ebuild = '%s-9999.ebuild' % self.ebuild_path_no_version - cmd = ('CROS_WORKON_LOCALNAME="%s" CROS_WORKON_PROJECT="%s" ' + cmd = ('export CROS_WORKON_LOCALNAME="%s" CROS_WORKON_PROJECT="%s"; ' 'eval $(grep -E "^CROS_WORKON" %s) && ' 'echo $CROS_WORKON_PROJECT ' '$CROS_WORKON_LOCALNAME/$CROS_WORKON_SUBDIR' @@ -341,10 +345,10 @@ class _EBuild(object): # TODO(anush): This hack is only necessary because the kernel ebuild has # 'if' statements, so we can't grab the CROS_WORKON_LOCALNAME properly. # We should clean up the kernel ebuild and remove this hack. - if not os.path.exists(srcdir) and subdir == 'kernel/': + if not os.path.isdir(srcdir) and subdir == 'kernel/': srcdir = os.path.join(srcroot, 'third_party/kernel/files') - if not os.path.exists(srcdir): + if not os.path.isdir(srcdir): Die('Cannot find commit id for %s' % self.ebuild_path) # Verify that we're grabbing the commit id from the right project name. @@ -487,11 +491,16 @@ def main(argv): except gflags.FlagsError, e : _PrintUsageAndDie(str(e)) - package_list = gflags.FLAGS.packages.split() + package_list = gflags.FLAGS.packages.split(':') _CheckSaneArguments(package_list, command) if gflags.FLAGS.overlays: - overlays = dict((path, []) for path in gflags.FLAGS.overlays.split()) + overlays = {} + for path in gflags.FLAGS.overlays.split(':'): + if command != 'clean' and not os.path.isdir(path): + Die('Cannot find overlay: %s' % path) + overlays[path] = [] else: + Warning('Missing --overlays argument') overlays = { '%s/private-overlays/chromeos-overlay' % gflags.FLAGS.srcroot: [], '%s/third_party/chromiumos-overlay' % gflags.FLAGS.srcroot: [] @@ -501,9 +510,13 @@ def main(argv): _BuildEBuildDictionary(overlays, gflags.FLAGS.all, package_list) for overlay, ebuilds in overlays.items(): - if not os.path.exists(overlay): + if not os.path.isdir(overlay): Warning("Skipping %s" % overlay) continue + + # TODO(davidjames): Currently, all code that interacts with git depends on + # the cwd being set to the overlay directory. We should instead pass in + # this parameter so that we don't need to modify the cwd globally. os.chdir(overlay) if command == 'clean': diff --git a/lib/cros_build_lib.py b/lib/cros_build_lib.py index 509871c6ee..4888a4eae6 100644 --- a/lib/cros_build_lib.py +++ b/lib/cros_build_lib.py @@ -138,14 +138,20 @@ def Info(message): Color(_STDOUT_IS_TTY).Color(Color.BLUE, '\nINFO: ' + message)) -def FindRepoDir(): - """Returns the nearest higher-level repo dir from the cwd.""" - cwd = os.getcwd() - while cwd != '/': - repo_dir = os.path.join(cwd, '.repo') +def FindRepoDir(path=None): + """Returns the nearest higher-level repo dir from the specified path. + + Args: + path: The path to use. Defaults to cwd. + """ + if path is None: + path = os.getcwd() + path = os.path.abspath(path) + while path != '/': + repo_dir = os.path.join(path, '.repo') if os.path.isdir(repo_dir): return repo_dir - cwd = os.path.dirname(cwd) + path = os.path.dirname(path) return None @@ -155,7 +161,7 @@ def ReinterpretPathForChroot(path): Keyword arguments: path: The path to reinterpret. Must be in src tree. """ - root_path = os.path.join(FindRepoDir(), '..') + root_path = os.path.join(FindRepoDir(path), '..') path_abs_path = os.path.abspath(path) root_abs_path = os.path.abspath(root_path) diff --git a/make_factory_package.sh b/make_factory_package.sh index a62cfe2e3c..57c23a159c 100755 --- a/make_factory_package.sh +++ b/make_factory_package.sh @@ -218,9 +218,28 @@ fi if [ -n "${FLAGS_subfolder}" ] && \ [ -f "${OMAHA_DIR}"/miniomaha.conf"" ] ; then # Remove the ']' from the last line of the file so we can add another config. - sed -i '$d' ${OMAHA_DIR}/miniomaha.conf + while [ -s "${OMAHA_DIR}/miniomaha.conf" ]; do + # If the last line is null + if [ -z "$(tail -1 "${OMAHA_DIR}/miniomaha.conf")" ]; then + sed -i '$d' "${OMAHA_DIR}/miniomaha.conf" + elif [ "$(tail -1 "${OMAHA_DIR}/miniomaha.conf")" != ']' ]; then + sed -i '$d' "${OMAHA_DIR}/miniomaha.conf" + else + break + fi + done + + # Remove the last ] + if [ "$(tail -1 "${OMAHA_DIR}/miniomaha.conf")" = ']' ]; then + sed -i '$d' "${OMAHA_DIR}/miniomaha.conf" + fi + + # If the file is empty, create it from scratch + if [ ! -s "${OMAHA_DIR}/miniomaha.conf" ]; then + echo "config = [" > "${OMAHA_DIR}/miniomaha.conf" + fi else - echo -e "config = [" > ${OMAHA_DIR}/miniomaha.conf + echo "config = [" > "${OMAHA_DIR}/miniomaha.conf" fi if [ -n "${FLAGS_subfolder}" ] ; then diff --git a/remote_access.sh b/remote_access.sh index 092555a5bd..f3d490ab97 100644 --- a/remote_access.sh +++ b/remote_access.sh @@ -23,7 +23,7 @@ function remote_cp_to() { # Copies a list of remote files specified in file $1 to local location # $2. Directory paths in $1 are collapsed into $2. function remote_rsync_from() { - rsync -e "ssh -o StrictHostKeyChecking=no -o \ + rsync -e "ssh -p ${FLAGS_ssh_port} -o StrictHostKeyChecking=no -o \ UserKnownHostsFile=$TMP_KNOWN_HOSTS" --no-R \ --files-from=$1 root@${FLAGS_remote}:/ $2 }