Merge branch 'master' of ssh://gitrw.chromium.org:9222/crosutils

bin/cbuildbot.py

@@ -18,7 +18,8 @@ import cbuildbot_comm
 from cbuildbot_config import config
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '../lib'))
-from cros_build_lib import Die, Info, RunCommand, Warning
+from cros_build_lib import (Die, Info, ReinterpretPathForChroot, RunCommand,
+                            Warning)
 
 _DEFAULT_RETRIES = 3
 ARCHIVE_BASE = '/var/www/archive'
@@ -181,37 +182,38 @@ def _ParseRevisionString(revision_string, repo_dictionary):
   return revisions.items()
 
 
-def _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board):
+def _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board,
+                           overlays):
   """Uprevs based on revision list."""
   if not revision_list:
     Info('No packages found to uprev')
     return
 
-  package_str = ''
+  packages = []
   for package, revision in revision_list:
-    package_str += package + ' '
+    assert ':' not in package, 'Invalid package name: %s' % package
+    packages.append(package)
 
-  package_str = package_str.strip()
+  chroot_overlays = [ReinterpretPathForChroot(path) for path in overlays]
 
   cwd = os.path.join(buildroot, 'src', 'scripts')
-  # TODO(davidjames): --foo="bar baz" only works here because we're using
-  # enter_chroot.
   RunCommand(['./cros_mark_as_stable',
               '--board=%s' % board,
-              '--tracking_branch="%s"' % tracking_branch,
+              '--tracking_branch=%s' % tracking_branch,
-              '--packages="%s"' % package_str,
+              '--overlays=%s' % ':'.join(chroot_overlays),
+              '--packages=%s' % ':'.join(packages),
               'commit'],
              cwd=cwd, enter_chroot=True)
 
 
-def _UprevAllPackages(buildroot, tracking_branch, board):
+def _UprevAllPackages(buildroot, tracking_branch, board, overlays):
   """Uprevs all packages that have been updated since last uprev."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
-  # TODO(davidjames): --foo="bar baz" only works here because we're using
+  chroot_overlays = [ReinterpretPathForChroot(path) for path in overlays]
-  # enter_chroot.
   RunCommand(['./cros_mark_as_stable', '--all',
               '--board=%s' % board,
-              '--tracking_branch="%s"' % tracking_branch, 'commit'],
+              '--overlays=%s' % ':'.join(chroot_overlays),
+              '--tracking_branch=%s' % tracking_branch, 'commit'],
               cwd=cwd, enter_chroot=True)
 
 
@@ -228,12 +230,13 @@ def _GetVMConstants(buildroot):
   return (vdisk_size.strip(), statefulfs_size.strip())
 
 
-def _GitCleanup(buildroot, board, tracking_branch):
+def _GitCleanup(buildroot, board, tracking_branch, overlays):
   """Clean up git branch after previous uprev attempt."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
   if os.path.exists(cwd):
     RunCommand(['./cros_mark_as_stable', '--srcroot=..',
                 '--board=%s' % board,
+                '--overlays=%s' % ':'.join(overlays),
                 '--tracking_branch=%s' % tracking_branch, 'clean'],
                cwd=cwd, error_ok=True)
 
@@ -257,9 +260,9 @@ def _WipeOldOutput(buildroot):
 # =========================== Main Commands ===================================
 
 
-def _PreFlightRinse(buildroot, board, tracking_branch):
+def _PreFlightRinse(buildroot, board, tracking_branch, overlays):
   """Cleans up any leftover state from previous runs."""
-  _GitCleanup(buildroot, board, tracking_branch)
+  _GitCleanup(buildroot, board, tracking_branch, overlays)
   _CleanUpMountPoints(buildroot)
   RunCommand(['sudo', 'killall', 'kvm'], error_ok=True)
 
@@ -347,7 +350,7 @@ def _RunSmokeSuite(buildroot, results_dir):
               ], cwd=cwd, error_ok=False)
 
 
-def _UprevPackages(buildroot, tracking_branch, revisionfile, board):
+def _UprevPackages(buildroot, tracking_branch, revisionfile, board, overlays):
   """Uprevs a package based on given revisionfile.
 
   If revisionfile is set to None or does not resolve to an actual file, this
@@ -376,26 +379,19 @@ def _UprevPackages(buildroot, tracking_branch, revisionfile, board):
   #  print >> sys.stderr, 'CBUILDBOT Revision list found %s' % revisions
   #  revision_list = _ParseRevisionString(revisions,
   #      _CreateRepoDictionary(buildroot, board))
-  #  _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board)
+  #  _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board,
+  #                         overlays)
   #else:
   Info('CBUILDBOT Revving all')
-  _UprevAllPackages(buildroot, tracking_branch, board)
+  _UprevAllPackages(buildroot, tracking_branch, board, overlays)
 
 
 def _UprevPush(buildroot, tracking_branch, board, overlays):
   """Pushes uprev changes to the main line."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
-  public_overlay = '%s/src/third_party/chromiumos-overlay' % buildroot
-  private_overlay = '%s/src/private-overlays/chromeos-overlay' % buildroot
-  if overlays == 'private':
-    overlays = [private_overlay]
-  elif overlays == 'public':
-    overlays = [public_overlay]
-  else:
-    overlays = [public_overlay, private_overlay]
   RunCommand(['./cros_mark_as_stable', '--srcroot=..',
               '--board=%s' % board,
-              '--overlays=%s' % " ".join(overlays),
+              '--overlays=%s' % ':'.join(overlays),
               '--tracking_branch=%s' % tracking_branch,
               '--push_options=--bypass-hooks -f', 'push'],
              cwd=cwd)
@@ -467,6 +463,30 @@ def _GetConfig(config_name):
   return buildconfig
 
 
+def _ResolveOverlays(buildroot, overlays):
+  """Return the list of overlays to use for a given buildbot.
+
+  Args:
+    buildroot: The root directory where the build occurs. Must be an absolute
+               path.
+    overlays: A string describing which overlays you want.
+              'private': Just the private overlay.
+              'public': Just the public overlay.
+              'both': Both the public and private overlays.
+  """
+  public_overlay = '%s/src/third_party/chromiumos-overlay' % buildroot
+  private_overlay = '%s/src/private-overlays/chromeos-overlay' % buildroot
+  if overlays == 'private':
+    paths = [private_overlay]
+  elif overlays == 'public':
+    paths = [public_overlay]
+  elif overlays == 'both':
+    paths = [public_overlay, private_overlay]
+  else:
+    Die('Incorrect overlay configuration: %s' % overlays)
+  return paths
+
+
 def main():
   # Parse options
   usage = "usage: %prog [options] cbuildbot_config"
@@ -491,7 +511,7 @@ def main():
 
   (options, args) = parser.parse_args()
 
-  buildroot = options.buildroot
+  buildroot = os.path.abspath(options.buildroot)
   revisionfile = options.revisionfile
   tracking_branch = options.tracking_branch
 
@@ -502,13 +522,22 @@ def main():
     parser.print_usage()
     sys.exit(1)
 
+  # Calculate list of overlay directories.
+  overlays = _ResolveOverlays(buildroot, buildconfig['overlays'])
+
   try:
-    _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch)
+    _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays)
     if options.clobber or not os.path.isdir(buildroot):
       _FullCheckout(buildroot, tracking_branch, url=options.url)
     else:
       _IncrementalCheckout(buildroot)
 
+    # Check that all overlays can be found.
+    for path in overlays:
+      assert ':' not in path, 'Overlay must not contain colons: %s' % path
+      if not os.path.isdir(path):
+        Die('Missing overlay: %s' % path)
+
     chroot_path = os.path.join(buildroot, 'chroot')
     if not os.path.isdir(chroot_path):
       _MakeChroot(buildroot)
@@ -519,7 +548,7 @@ def main():
 
     if buildconfig['uprev']:
       _UprevPackages(buildroot, tracking_branch, revisionfile,
-                     board=buildconfig['board'])
+                     buildconfig['board'], overlays)
 
     _EnableLocalAccount(buildroot)
     _Build(buildroot)
@@ -545,7 +574,7 @@ def main():
           # Master bot needs to check if the other slaves completed.
           if cbuildbot_comm.HaveSlavesCompleted(config):
             _UprevPush(buildroot, tracking_branch, buildconfig['board'],
-                       buildconfig['overlays'])
+                       overlays)
           else:
             Die('CBUILDBOT - One of the slaves has failed!!!')
 

bin/cbuildbot_config.py

@@ -19,9 +19,9 @@ hostname -- Needed for 'important' slaves.  The hostname of the bot.  Should
             match hostname in slaves.cfg in buildbot checkout.
 unittests -- Runs unittests for packages.
 smoke_bvt -- Runs the test smoke suite in a qemu-based VM using KVM.
-overlays -- If this bot is a master bot, select what overlays to push changes
+overlays -- Select what overlays to look at. This can be 'public', 'private'
-            to. This can be 'public', 'private', or 'both'. There should only
+            or 'both'. There should only be one master bot pushing changes to
-            be one bot pushing changes to each overlay.
+            each overlay per branch.
 """
 
 
@@ -33,6 +33,7 @@ config['default'] = {
   'important' : False,
   'unittests' : False,
   'smoke_bvt' : False,
+  'overlays': 'public',
 }
 config['x86-generic-pre-flight-queue'] = {
   'board' : 'x86-generic',
@@ -69,6 +70,7 @@ config['x86_agz_bin'] = {
   'important' : False,
   'unittests' : True,
   'smoke_bvt' : True,
+  'overlays': 'private',
 }
 config['x86_dogfood_bin'] = {
   'board' : 'x86-dogfood',
@@ -77,52 +79,29 @@ config['x86_dogfood_bin'] = {
   'important' : False,
   'unittests' : True,
   'smoke_bvt' : True,
+  'overlays': 'private',
 }
 config['x86_pineview_bin'] = {
   'board' : 'x86-pineview',
   'uprev' : True,
   'master' : False,
   'important' : False,
-  'hostname' : 'codf200.jail',
   'unittests': True,
+  'overlays': 'public',
 }
 config['arm_tegra2_bin'] = {
   'board' : 'tegra2',
   'uprev' : True,
   'master' : False,
   'important' : False,
-  'hostname' : 'codg172.jail',
-  'unittests' : False,
-}
-config['arm_voguev210_bin'] = {
-  'board' : 'voguev210',
-  'uprev' : True,
-  'master' : False,
-  'important' : False,
-  'hostname' : 'codf196.jail',
-  'unittests' : False,
-}
-config['arm_beagleboard_bin'] = {
-  'board' : 'beagleboard',
-  'master' : False,
-  'uprev' : True,
-  'important' : False,
-  'hostname' : 'codf202.jail',
-  'unittests' : False,
-}
-config['arm_st1q_bin'] = {
-  'board' : 'st1q',
-  'uprev' : True,
-  'master' : False,
-  'important' : False,
-  'hostname' : 'codg158.jail',
   'unittests' : False,
+  'overlays': 'public',
 }
 config['arm_generic_bin'] = {
   'board' : 'arm-generic',
   'uprev' : True,
   'master' : False,
   'important' : False,
-  'hostname' : 'codg175.jail',
   'unittests' : False,
+  'overlays': 'public',
 }

bin/cbuildbot_unittest.py

@@ -16,6 +16,7 @@ import unittest
 # Fixes circular dependency error.
 import cbuildbot_comm
 import cbuildbot
+from cros_build_lib import ReinterpretPathForChroot
 
 class CBuildBotTest(mox.MoxTestBase):
 
@@ -42,6 +43,10 @@ class CBuildBotTest(mox.MoxTestBase):
                                       ['dev-util/perf', '12345test'],
                                       ['chromos-base/libcros', '12345test']
                                      ]
+    self._overlays = ['%s/src/third_party/chromiumos-overlay' % self._buildroot]
+    self._chroot_overlays = [
+        ReinterpretPathForChroot(p) for p in self._overlays
+    ]
 
   def testParseRevisionString(self):
     """Test whether _ParseRevisionString parses string correctly."""
@@ -168,13 +173,15 @@ class CBuildBotTest(mox.MoxTestBase):
 
     cbuildbot.RunCommand(['./cros_mark_as_stable', '--all',
                      '--board=%s' % self._test_board,
-                     '--tracking_branch="cros/master"', 'commit'],
+                     '--overlays=%s' % ':'.join(self._chroot_overlays),
+                     '--tracking_branch=cros/master', 'commit'],
                      cwd='%s/src/scripts' % self._buildroot,
                      enter_chroot=True)
 
     self.mox.ReplayAll()
     cbuildbot._UprevPackages(self._buildroot, self.tracking_branch,
-                             self._revision_file, self._test_board)
+                             self._revision_file, self._test_board,
+                             self._overlays)
     self.mox.VerifyAll()
 
   def testUprevAllPackages(self):
@@ -189,13 +196,15 @@ class CBuildBotTest(mox.MoxTestBase):
 
     cbuildbot.RunCommand(['./cros_mark_as_stable', '--all',
                          '--board=%s' % self._test_board,
-                         '--tracking_branch="cros/master"', 'commit'],
+                         '--overlays=%s' % ':'.join(self._chroot_overlays),
+                         '--tracking_branch=cros/master', 'commit'],
                          cwd='%s/src/scripts' % self._buildroot,
                          enter_chroot=True)
 
     self.mox.ReplayAll()
     cbuildbot._UprevPackages(self._buildroot, self.tracking_branch,
-                             self._revision_file, self._test_board)
+                             self._revision_file, self._test_board,
+                             self._overlays)
     self.mox.VerifyAll()
 
 

bin/cros_get_chrome_version

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Returns the version of Chrome running on a remote machine.
+
+. "$(dirname $0)/../common.sh"
+. "$(dirname $0)/../remote_access.sh"
+
+FLAGS "$@" || exit 1
+
+set -e
+
+# TMP necessary for remote_access_init.
+TMP=$(mktemp -d /tmp/cros_check_chrome_version.XXXX)
+trap "rm -rf ${TMP}" EXIT
+
+remote_access_init &> /dev/null
+
+remote_sh "/opt/google/chrome/chrome --version"
+CHROME_VERSION=$(echo ${REMOTE_OUT} | \
+    sed 's/.* \([0-9]\+.[0-9]\+.[0-9]\+.[0-9]\+\).*/\1/')
+echo "${CHROME_VERSION}"

bin/cros_run_vm_test

@@ -21,9 +21,17 @@ DEFINE_string results_dir_root "" "alternate root results directory"
 DEFINE_string test_case "" "Name of the test case to run"
 DEFINE_boolean use_emerged ${FLAGS_FALSE} \
     "Force use of emerged autotest packages"
+DEFINE_string verify_chrome_version "" \
+    "Verify that this chrome version matches that on vm."
 
 set -e
 
+# Returns normally if the given $1 is a valid chrome version.
+chrome_version_is_valid() {
+  local chrome_version="$1"
+  echo ${chrome_version} | egrep '^[0-9]+.[0-9]+.[0-9]+.[0-9]+$' &> /dev/null
+}
+
 # Parse command line.
 FLAGS "$@" || exit 1
 
@@ -61,6 +69,20 @@ trap stop_kvm EXIT
 start_kvm "${FLAGS_image_path}"
 info "Checking for ssh access to virtual machine."
 retry_until_ssh ${MAX_RETRIES}
+
+if [ -n "${FLAGS_verify_chrome_version}" ]; then
+  info "Verifying version of Chrome matches what we expect."
+  if chrome_version_is_valid "${FLAGS_verify_chrome_version}"; then
+    chrome_version_on_vm=$("$(dirname $0)/cros_get_chrome_version" \
+        --remote=127.0.0.1 \
+        --ssh_port=${FLAGS_ssh_port})
+    [[ ${chrome_version_on_vm} == ${FLAGS_verify_chrome_version} ]] || \
+        die "Chrome version mismatch.  VM reported ${chrome_version_on_vm}"
+  else
+    warn "${FLAGS_verify_chrome_version} is not a valid Chrome version"
+  fi
+fi
+
 "$(dirname $0)"/../run_remote_tests.sh \
     --board=${FLAGS_board} \
     --ssh_port=${FLAGS_ssh_port} \

chromeos_version.sh

@@ -26,7 +26,7 @@ export CHROMEOS_VERSION_MINOR=9
 # Increment by 2 in trunk after making a release branch.
 # Does not reset on a major/minor change (always increases).
 # (Trunk is always odd; branches are always even).
-export CHROMEOS_VERSION_BRANCH=113
+export CHROMEOS_VERSION_BRANCH=115
 
 # Patch number.
 # Increment by 1 each release on a branch.

cros_mark_as_stable.py

@@ -22,10 +22,10 @@ from cros_build_lib import Info, RunCommand, Warning, Die
 gflags.DEFINE_string('board', '',
                      'Board for which the package belongs.', short_name='b')
 gflags.DEFINE_string('overlays', '',
-                     'Space separated list of overlays to modify.',
+                     'Colon-separated list of overlays to modify.',
                      short_name='o')
 gflags.DEFINE_string('packages', '',
-                     'Space separated list of packages to mark as stable.',
+                     'Colon-separated list of packages to mark as stable.',
                      short_name='p')
 gflags.DEFINE_string('push_options', '',
                      'Options to use with git-cl push using push command.')
@@ -245,7 +245,11 @@ def _SimpleRunCommand(command):
   """Runs a shell command and returns stdout back to caller."""
   _Print('  + %s' % command)
   proc_handle = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
-  return proc_handle.communicate()[0]
+  stdout = proc_handle.communicate()[0]
+  retcode = proc_handle.wait()
+  if retcode != 0:
+    raise subprocess.CalledProcessError(retcode, command, output=stdout)
+  return stdout
 
 
 # ======================= End Global Helper Functions ========================
@@ -323,7 +327,7 @@ class _EBuild(object):
 
     # Grab and evaluate CROS_WORKON variables from this ebuild.
     unstable_ebuild = '%s-9999.ebuild' % self.ebuild_path_no_version
-    cmd = ('CROS_WORKON_LOCALNAME="%s" CROS_WORKON_PROJECT="%s" '
+    cmd = ('export CROS_WORKON_LOCALNAME="%s" CROS_WORKON_PROJECT="%s"; '
            'eval $(grep -E "^CROS_WORKON" %s) && '
            'echo $CROS_WORKON_PROJECT '
            '$CROS_WORKON_LOCALNAME/$CROS_WORKON_SUBDIR'
@@ -341,10 +345,10 @@ class _EBuild(object):
     # TODO(anush): This hack is only necessary because the kernel ebuild has
     # 'if' statements, so we can't grab the CROS_WORKON_LOCALNAME properly.
     # We should clean up the kernel ebuild and remove this hack.
-    if not os.path.exists(srcdir) and subdir == 'kernel/':
+    if not os.path.isdir(srcdir) and subdir == 'kernel/':
       srcdir = os.path.join(srcroot, 'third_party/kernel/files')
 
-    if not os.path.exists(srcdir):
+    if not os.path.isdir(srcdir):
       Die('Cannot find commit id for %s' % self.ebuild_path)
 
     # Verify that we're grabbing the commit id from the right project name.
@@ -487,11 +491,16 @@ def main(argv):
   except gflags.FlagsError, e :
     _PrintUsageAndDie(str(e))
 
-  package_list = gflags.FLAGS.packages.split()
+  package_list = gflags.FLAGS.packages.split(':')
   _CheckSaneArguments(package_list, command)
   if gflags.FLAGS.overlays:
-    overlays = dict((path, []) for path in gflags.FLAGS.overlays.split())
+    overlays = {}
+    for path in gflags.FLAGS.overlays.split(':'):
+      if command != 'clean' and not os.path.isdir(path):
+        Die('Cannot find overlay: %s' % path)
+      overlays[path] = []
   else:
+    Warning('Missing --overlays argument')
     overlays = {
       '%s/private-overlays/chromeos-overlay' % gflags.FLAGS.srcroot: [],
       '%s/third_party/chromiumos-overlay' % gflags.FLAGS.srcroot: []
@@ -501,9 +510,13 @@ def main(argv):
     _BuildEBuildDictionary(overlays, gflags.FLAGS.all, package_list)
 
   for overlay, ebuilds in overlays.items():
-    if not os.path.exists(overlay):
+    if not os.path.isdir(overlay):
       Warning("Skipping %s" % overlay)
       continue
+
+    # TODO(davidjames): Currently, all code that interacts with git depends on
+    # the cwd being set to the overlay directory. We should instead pass in
+    # this parameter so that we don't need to modify the cwd globally.
     os.chdir(overlay)
 
     if command == 'clean':

lib/cros_build_lib.py

@@ -138,14 +138,20 @@ def Info(message):
       Color(_STDOUT_IS_TTY).Color(Color.BLUE, '\nINFO: ' + message))
 
 
-def FindRepoDir():
+def FindRepoDir(path=None):
-  """Returns the nearest higher-level repo dir from the cwd."""
+  """Returns the nearest higher-level repo dir from the specified path.
-  cwd = os.getcwd()
+
-  while cwd != '/':
+  Args:
-    repo_dir = os.path.join(cwd, '.repo')
+    path: The path to use. Defaults to cwd.
+  """
+  if path is None:
+    path = os.getcwd()
+  path = os.path.abspath(path)
+  while path != '/':
+    repo_dir = os.path.join(path, '.repo')
     if os.path.isdir(repo_dir):
       return repo_dir
-    cwd = os.path.dirname(cwd)
+    path = os.path.dirname(path)
   return None
 
 
@@ -155,7 +161,7 @@ def ReinterpretPathForChroot(path):
   Keyword arguments:
     path: The path to reinterpret.  Must be in src tree.
   """
-  root_path = os.path.join(FindRepoDir(), '..')
+  root_path = os.path.join(FindRepoDir(path), '..')
 
   path_abs_path = os.path.abspath(path)
   root_abs_path = os.path.abspath(root_path)

make_factory_package.sh

@@ -218,9 +218,28 @@ fi
 if [ -n "${FLAGS_subfolder}" ] && \
    [ -f "${OMAHA_DIR}"/miniomaha.conf"" ] ; then
   # Remove the ']' from the last line of the file so we can add another config.
-  sed -i '$d' ${OMAHA_DIR}/miniomaha.conf
+  while  [ -s "${OMAHA_DIR}/miniomaha.conf" ]; do
+    # If the last line is null
+    if [ -z "$(tail -1 "${OMAHA_DIR}/miniomaha.conf")" ]; then
+      sed -i '$d' "${OMAHA_DIR}/miniomaha.conf"
+    elif [ "$(tail -1 "${OMAHA_DIR}/miniomaha.conf")" != ']' ]; then
+      sed -i '$d' "${OMAHA_DIR}/miniomaha.conf"
+    else
+      break
+    fi
+  done
+
+  # Remove the last ]
+  if [ "$(tail -1 "${OMAHA_DIR}/miniomaha.conf")" = ']' ]; then
+    sed -i '$d' "${OMAHA_DIR}/miniomaha.conf"
+  fi
+
+  # If the file is empty, create it from scratch
+  if [ ! -s "${OMAHA_DIR}/miniomaha.conf" ]; then
+    echo "config = [" > "${OMAHA_DIR}/miniomaha.conf"
+  fi
 else
-  echo -e "config = [" > ${OMAHA_DIR}/miniomaha.conf
+  echo "config = [" > "${OMAHA_DIR}/miniomaha.conf"
 fi
 
 if [ -n "${FLAGS_subfolder}" ] ; then

remote_access.sh

@@ -23,7 +23,7 @@ function remote_cp_to() {
 # Copies a list of remote files specified in file $1 to local location
 # $2.  Directory paths in $1 are collapsed into $2.
 function remote_rsync_from() {
-  rsync -e "ssh -o StrictHostKeyChecking=no -o \
+  rsync -e "ssh -p ${FLAGS_ssh_port} -o StrictHostKeyChecking=no -o \
             UserKnownHostsFile=$TMP_KNOWN_HOSTS" --no-R \
     --files-from=$1 root@${FLAGS_remote}:/ $2
 }