mirrors/flatcar-scripts
1
0
Fork 0
mirror of https://github.com/flatcar/scripts.git synced 2025-08-18 21:11:08 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of ssh://gitrw.chromium.org:9222/crosutils

Browse Source
This commit is contained in:
Scott Zawalski 2010-11-16 09:14:22 -08:00
commit 3309df01c8
10 changed files with 195 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
bin/cbuildbot.py
View File

@ -18,7 +18,8 @@ import cbuildbot_comm
from cbuildbot_config import config
sys.path.append(os.path.join(os.path.dirname(__file__), '../lib'))
from cros_build_lib import Die, Info, RunCommand, Warning
from cros_build_lib import (Die, Info, ReinterpretPathForChroot, RunCommand,
Warning)
_DEFAULT_RETRIES = 3
ARCHIVE_BASE = '/var/www/archive'
@ -181,37 +182,38 @@ def _ParseRevisionString(revision_string, repo_dictionary):
return revisions.items()
def _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board):
def _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board,
overlays):
"""Uprevs based on revision list."""
if not revision_list:
Info('No packages found to uprev')
return
package_str = ''
packages = []
for package, revision in revision_list:
package_str += package + ' '
assert ':' not in package, 'Invalid package name: %s' % package
packages.append(package)
package_str = package_str.strip()
chroot_overlays = [ReinterpretPathForChroot(path) for path in overlays]
cwd = os.path.join(buildroot, 'src', 'scripts')
# TODO(davidjames): --foo="bar baz" only works here because we're using
# enter_chroot.
RunCommand(['./cros_mark_as_stable',
'--board=%s' % board,
'--tracking_branch="%s"' % tracking_branch,
'--packages="%s"' % package_str,
'--tracking_branch=%s' % tracking_branch,
'--overlays=%s' % ':'.join(chroot_overlays),
'--packages=%s' % ':'.join(packages),
'commit'],
cwd=cwd, enter_chroot=True)
cwd=cwd, enter_chroot=True)
def _UprevAllPackages(buildroot, tracking_branch, board):
def _UprevAllPackages(buildroot, tracking_branch, board, overlays):
"""Uprevs all packages that have been updated since last uprev."""
cwd = os.path.join(buildroot, 'src', 'scripts')
# TODO(davidjames): --foo="bar baz" only works here because we're using
# enter_chroot.
chroot_overlays = [ReinterpretPathForChroot(path) for path in overlays]
RunCommand(['./cros_mark_as_stable', '--all',
'--board=%s' % board,
'--tracking_branch="%s"' % tracking_branch, 'commit'],
'--overlays=%s' % ':'.join(chroot_overlays),
'--tracking_branch=%s' % tracking_branch, 'commit'],
cwd=cwd, enter_chroot=True)
@ -228,12 +230,13 @@ def _GetVMConstants(buildroot):
return (vdisk_size.strip(), statefulfs_size.strip())
def _GitCleanup(buildroot, board, tracking_branch):
def _GitCleanup(buildroot, board, tracking_branch, overlays):
"""Clean up git branch after previous uprev attempt."""
cwd = os.path.join(buildroot, 'src', 'scripts')
if os.path.exists(cwd):
RunCommand(['./cros_mark_as_stable', '--srcroot=..',
'--board=%s' % board,
'--overlays=%s' % ':'.join(overlays),
'--tracking_branch=%s' % tracking_branch, 'clean'],
cwd=cwd, error_ok=True)
@ -257,9 +260,9 @@ def _WipeOldOutput(buildroot):
# =========================== Main Commands ===================================
def _PreFlightRinse(buildroot, board, tracking_branch):
def _PreFlightRinse(buildroot, board, tracking_branch, overlays):
"""Cleans up any leftover state from previous runs."""
_GitCleanup(buildroot, board, tracking_branch)
_GitCleanup(buildroot, board, tracking_branch, overlays)
_CleanUpMountPoints(buildroot)
RunCommand(['sudo', 'killall', 'kvm'], error_ok=True)
@ -347,7 +350,7 @@ def _RunSmokeSuite(buildroot, results_dir):
], cwd=cwd, error_ok=False)
def _UprevPackages(buildroot, tracking_branch, revisionfile, board):
def _UprevPackages(buildroot, tracking_branch, revisionfile, board, overlays):
"""Uprevs a package based on given revisionfile.
If revisionfile is set to None or does not resolve to an actual file, this
@ -376,26 +379,19 @@ def _UprevPackages(buildroot, tracking_branch, revisionfile, board):
# print >> sys.stderr, 'CBUILDBOT Revision list found %s' % revisions
# revision_list = _ParseRevisionString(revisions,
# _CreateRepoDictionary(buildroot, board))
# _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board)
# _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board,
# overlays)
#else:
Info('CBUILDBOT Revving all')
_UprevAllPackages(buildroot, tracking_branch, board)
_UprevAllPackages(buildroot, tracking_branch, board, overlays)
def _UprevPush(buildroot, tracking_branch, board, overlays):
"""Pushes uprev changes to the main line."""
cwd = os.path.join(buildroot, 'src', 'scripts')
public_overlay = '%s/src/third_party/chromiumos-overlay' % buildroot
private_overlay = '%s/src/private-overlays/chromeos-overlay' % buildroot
if overlays == 'private':
overlays = [private_overlay]
elif overlays == 'public':
overlays = [public_overlay]
else:
overlays = [public_overlay, private_overlay]
RunCommand(['./cros_mark_as_stable', '--srcroot=..',
'--board=%s' % board,
'--overlays=%s' % " ".join(overlays),
'--overlays=%s' % ':'.join(overlays),
'--tracking_branch=%s' % tracking_branch,
'--push_options=--bypass-hooks -f', 'push'],
cwd=cwd)
@ -467,6 +463,30 @@ def _GetConfig(config_name):
return buildconfig
def _ResolveOverlays(buildroot, overlays):
"""Return the list of overlays to use for a given buildbot.
Args:
buildroot: The root directory where the build occurs. Must be an absolute
path.
overlays: A string describing which overlays you want.
'private': Just the private overlay.
'public': Just the public overlay.
'both': Both the public and private overlays.
"""
public_overlay = '%s/src/third_party/chromiumos-overlay' % buildroot
private_overlay = '%s/src/private-overlays/chromeos-overlay' % buildroot
if overlays == 'private':
paths = [private_overlay]
elif overlays == 'public':
paths = [public_overlay]
elif overlays == 'both':
paths = [public_overlay, private_overlay]
else:
Die('Incorrect overlay configuration: %s' % overlays)
return paths
def main():
# Parse options
usage = "usage: %prog [options] cbuildbot_config"
@ -491,7 +511,7 @@ def main():
(options, args) = parser.parse_args()
buildroot = options.buildroot
buildroot = os.path.abspath(options.buildroot)
revisionfile = options.revisionfile
tracking_branch = options.tracking_branch
@ -502,13 +522,22 @@ def main():
parser.print_usage()
sys.exit(1)
# Calculate list of overlay directories.
overlays = _ResolveOverlays(buildroot, buildconfig['overlays'])
try:
_PreFlightRinse(buildroot, buildconfig['board'], tracking_branch)
_PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays)
if options.clobber or not os.path.isdir(buildroot):
_FullCheckout(buildroot, tracking_branch, url=options.url)
else:
_IncrementalCheckout(buildroot)
# Check that all overlays can be found.
for path in overlays:
assert ':' not in path, 'Overlay must not contain colons: %s' % path
if not os.path.isdir(path):
Die('Missing overlay: %s' % path)
chroot_path = os.path.join(buildroot, 'chroot')
if not os.path.isdir(chroot_path):
_MakeChroot(buildroot)
@ -519,7 +548,7 @@ def main():
if buildconfig['uprev']:
_UprevPackages(buildroot, tracking_branch, revisionfile,
board=buildconfig['board'])
buildconfig['board'], overlays)
_EnableLocalAccount(buildroot)
_Build(buildroot)
@ -545,7 +574,7 @@ def main():
# Master bot needs to check if the other slaves completed.
if cbuildbot_comm.HaveSlavesCompleted(config):
_UprevPush(buildroot, tracking_branch, buildconfig['board'],
buildconfig['overlays'])
overlays)
else:
Die('CBUILDBOT - One of the slaves has failed!!!')

39
bin/cbuildbot_config.py
View File

@ -19,9 +19,9 @@ hostname -- Needed for 'important' slaves. The hostname of the bot. Should
match hostname in slaves.cfg in buildbot checkout.
unittests -- Runs unittests for packages.
smoke_bvt -- Runs the test smoke suite in a qemu-based VM using KVM.
overlays -- If this bot is a master bot, select what overlays to push changes
to. This can be 'public', 'private', or 'both'. There should only
be one bot pushing changes to each overlay.
overlays -- Select what overlays to look at. This can be 'public', 'private'
or 'both'. There should only be one master bot pushing changes to
each overlay per branch.
"""
@ -33,6 +33,7 @@ config['default'] = {
'important' : False,
'unittests' : False,
'smoke_bvt' : False,
'overlays': 'public',
}
config['x86-generic-pre-flight-queue'] = {
'board' : 'x86-generic',
@ -69,6 +70,7 @@ config['x86_agz_bin'] = {
'important' : False,
'unittests' : True,
'smoke_bvt' : True,
'overlays': 'private',
}
config['x86_dogfood_bin'] = {
'board' : 'x86-dogfood',
@ -77,52 +79,29 @@ config['x86_dogfood_bin'] = {
'important' : False,
'unittests' : True,
'smoke_bvt' : True,
'overlays': 'private',
}
config['x86_pineview_bin'] = {
'board' : 'x86-pineview',
'uprev' : True,
'master' : False,
'important' : False,
'hostname' : 'codf200.jail',
'unittests': True,
'overlays': 'public',
}
config['arm_tegra2_bin'] = {
'board' : 'tegra2',
'uprev' : True,
'master' : False,
'important' : False,
'hostname' : 'codg172.jail',
'unittests' : False,
}
config['arm_voguev210_bin'] = {
'board' : 'voguev210',
'uprev' : True,
'master' : False,
'important' : False,
'hostname' : 'codf196.jail',
'unittests' : False,
}
config['arm_beagleboard_bin'] = {
'board' : 'beagleboard',
'master' : False,
'uprev' : True,
'important' : False,
'hostname' : 'codf202.jail',
'unittests' : False,
}
config['arm_st1q_bin'] = {
'board' : 'st1q',
'uprev' : True,
'master' : False,
'important' : False,
'hostname' : 'codg158.jail',
'unittests' : False,
'overlays': 'public',
}
config['arm_generic_bin'] = {
'board' : 'arm-generic',
'uprev' : True,
'master' : False,
'important' : False,
'hostname' : 'codg175.jail',
'unittests' : False,
'overlays': 'public',
}

17
bin/cbuildbot_unittest.py
View File

@ -16,6 +16,7 @@ import unittest
# Fixes circular dependency error.
import cbuildbot_comm
import cbuildbot
from cros_build_lib import ReinterpretPathForChroot
class CBuildBotTest(mox.MoxTestBase):
@ -42,6 +43,10 @@ class CBuildBotTest(mox.MoxTestBase):
['dev-util/perf', '12345test'],
['chromos-base/libcros', '12345test']
]
self._overlays = ['%s/src/third_party/chromiumos-overlay' % self._buildroot]
self._chroot_overlays = [
ReinterpretPathForChroot(p) for p in self._overlays
]
def testParseRevisionString(self):
"""Test whether _ParseRevisionString parses string correctly."""
@ -168,13 +173,15 @@ class CBuildBotTest(mox.MoxTestBase):
cbuildbot.RunCommand(['./cros_mark_as_stable', '--all',
'--board=%s' % self._test_board,
'--tracking_branch="cros/master"', 'commit'],
'--overlays=%s' % ':'.join(self._chroot_overlays),
'--tracking_branch=cros/master', 'commit'],
cwd='%s/src/scripts' % self._buildroot,
enter_chroot=True)
self.mox.ReplayAll()
cbuildbot._UprevPackages(self._buildroot, self.tracking_branch,
self._revision_file, self._test_board)
self._revision_file, self._test_board,
self._overlays)
self.mox.VerifyAll()
def testUprevAllPackages(self):
@ -189,13 +196,15 @@ class CBuildBotTest(mox.MoxTestBase):
cbuildbot.RunCommand(['./cros_mark_as_stable', '--all',
'--board=%s' % self._test_board,
'--tracking_branch="cros/master"', 'commit'],
'--overlays=%s' % ':'.join(self._chroot_overlays),
'--tracking_branch=cros/master', 'commit'],
cwd='%s/src/scripts' % self._buildroot,
enter_chroot=True)
self.mox.ReplayAll()
cbuildbot._UprevPackages(self._buildroot, self.tracking_branch,
self._revision_file, self._test_board)
self._revision_file, self._test_board,
self._overlays)
self.mox.VerifyAll()

25
bin/cros_get_chrome_version Executable file
View File

@ -0,0 +1,25 @@
#!/bin/bash
# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Returns the version of Chrome running on a remote machine.
. "$(dirname $0)/../common.sh"
. "$(dirname $0)/../remote_access.sh"
FLAGS "$@" || exit 1
set -e
# TMP necessary for remote_access_init.
TMP=$(mktemp -d /tmp/cros_check_chrome_version.XXXX)
trap "rm -rf ${TMP}" EXIT
remote_access_init &> /dev/null
remote_sh "/opt/google/chrome/chrome --version"
CHROME_VERSION=$(echo ${REMOTE_OUT} | \
sed 's/.* \([0-9]\+.[0-9]\+.[0-9]\+.[0-9]\+\).*/\1/')
echo "${CHROME_VERSION}"

34
bin/cros_run_vm_test
View File

@ -21,9 +21,17 @@ DEFINE_string results_dir_root "" "alternate root results directory"
DEFINE_string test_case "" "Name of the test case to run"
DEFINE_boolean use_emerged ${FLAGS_FALSE} \
"Force use of emerged autotest packages"
DEFINE_string verify_chrome_version "" \
"Verify that this chrome version matches that on vm."
set -e
# Returns normally if the given $1 is a valid chrome version.
chrome_version_is_valid() {
local chrome_version="$1"
echo ${chrome_version} | egrep '^[0-9]+.[0-9]+.[0-9]+.[0-9]+$' &> /dev/null
}
# Parse command line.
FLAGS "$@" || exit 1
@ -61,10 +69,24 @@ trap stop_kvm EXIT
start_kvm "${FLAGS_image_path}"
info "Checking for ssh access to virtual machine."
retry_until_ssh ${MAX_RETRIES}
if [ -n "${FLAGS_verify_chrome_version}" ]; then
info "Verifying version of Chrome matches what we expect."
if chrome_version_is_valid "${FLAGS_verify_chrome_version}"; then
chrome_version_on_vm=$("$(dirname $0)/cros_get_chrome_version" \
--remote=127.0.0.1 \
--ssh_port=${FLAGS_ssh_port})
[[ ${chrome_version_on_vm} == ${FLAGS_verify_chrome_version} ]] || \
die "Chrome version mismatch. VM reported ${chrome_version_on_vm}"
else
warn "${FLAGS_verify_chrome_version} is not a valid Chrome version"
fi
fi
"$(dirname $0)"/../run_remote_tests.sh \
--board=${FLAGS_board} \
--ssh_port=${FLAGS_ssh_port} \
--remote=127.0.0.1 \
--results_dir_root="${FLAGS_results_dir_root}" \
${USE_EMERGED} \
"${tests[@]}"
--board=${FLAGS_board} \
--ssh_port=${FLAGS_ssh_port} \
--remote=127.0.0.1 \
--results_dir_root="${FLAGS_results_dir_root}" \
${USE_EMERGED} \
"${tests[@]}"

2
chromeos_version.sh
View File

@ -26,7 +26,7 @@ export CHROMEOS_VERSION_MINOR=9
# Increment by 2 in trunk after making a release branch.
# Does not reset on a major/minor change (always increases).
# (Trunk is always odd; branches are always even).
export CHROMEOS_VERSION_BRANCH=113
export CHROMEOS_VERSION_BRANCH=115
# Patch number.
# Increment by 1 each release on a branch.

31
cros_mark_as_stable.py
View File

@ -22,10 +22,10 @@ from cros_build_lib import Info, RunCommand, Warning, Die
gflags.DEFINE_string('board', '',
'Board for which the package belongs.', short_name='b')
gflags.DEFINE_string('overlays', '',
'Space separated list of overlays to modify.',
'Colon-separated list of overlays to modify.',
short_name='o')
gflags.DEFINE_string('packages', '',
'Space separated list of packages to mark as stable.',
'Colon-separated list of packages to mark as stable.',
short_name='p')
gflags.DEFINE_string('push_options', '',
'Options to use with git-cl push using push command.')
@ -245,7 +245,11 @@ def _SimpleRunCommand(command):
"""Runs a shell command and returns stdout back to caller."""
_Print(' + %s' % command)
proc_handle = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
return proc_handle.communicate()[0]
stdout = proc_handle.communicate()[0]
retcode = proc_handle.wait()
if retcode != 0:
raise subprocess.CalledProcessError(retcode, command, output=stdout)
return stdout
# ======================= End Global Helper Functions ========================
@ -323,7 +327,7 @@ class _EBuild(object):
# Grab and evaluate CROS_WORKON variables from this ebuild.
unstable_ebuild = '%s-9999.ebuild' % self.ebuild_path_no_version
cmd = ('CROS_WORKON_LOCALNAME="%s" CROS_WORKON_PROJECT="%s" '
cmd = ('export CROS_WORKON_LOCALNAME="%s" CROS_WORKON_PROJECT="%s"; '
'eval $(grep -E "^CROS_WORKON" %s) && '
'echo $CROS_WORKON_PROJECT '
'$CROS_WORKON_LOCALNAME/$CROS_WORKON_SUBDIR'
@ -341,10 +345,10 @@ class _EBuild(object):
# TODO(anush): This hack is only necessary because the kernel ebuild has
# 'if' statements, so we can't grab the CROS_WORKON_LOCALNAME properly.
# We should clean up the kernel ebuild and remove this hack.
if not os.path.exists(srcdir) and subdir == 'kernel/':
if not os.path.isdir(srcdir) and subdir == 'kernel/':
srcdir = os.path.join(srcroot, 'third_party/kernel/files')
if not os.path.exists(srcdir):
if not os.path.isdir(srcdir):
Die('Cannot find commit id for %s' % self.ebuild_path)
# Verify that we're grabbing the commit id from the right project name.
@ -487,11 +491,16 @@ def main(argv):
except gflags.FlagsError, e :
_PrintUsageAndDie(str(e))
package_list = gflags.FLAGS.packages.split()
package_list = gflags.FLAGS.packages.split(':')
_CheckSaneArguments(package_list, command)
if gflags.FLAGS.overlays:
overlays = dict((path, []) for path in gflags.FLAGS.overlays.split())
overlays = {}
for path in gflags.FLAGS.overlays.split(':'):
if command != 'clean' and not os.path.isdir(path):
Die('Cannot find overlay: %s' % path)
overlays[path] = []
else:
Warning('Missing --overlays argument')
overlays = {
'%s/private-overlays/chromeos-overlay' % gflags.FLAGS.srcroot: [],
'%s/third_party/chromiumos-overlay' % gflags.FLAGS.srcroot: []
@ -501,9 +510,13 @@ def main(argv):
_BuildEBuildDictionary(overlays, gflags.FLAGS.all, package_list)
for overlay, ebuilds in overlays.items():
if not os.path.exists(overlay):
if not os.path.isdir(overlay):
Warning("Skipping %s" % overlay)
continue
# TODO(davidjames): Currently, all code that interacts with git depends on
# the cwd being set to the overlay directory. We should instead pass in
# this parameter so that we don't need to modify the cwd globally.
os.chdir(overlay)
if command == 'clean':

20
lib/cros_build_lib.py
View File

@ -138,14 +138,20 @@ def Info(message):
Color(_STDOUT_IS_TTY).Color(Color.BLUE, '\nINFO: ' + message))
def FindRepoDir():
"""Returns the nearest higher-level repo dir from the cwd."""
cwd = os.getcwd()
while cwd != '/':
repo_dir = os.path.join(cwd, '.repo')
def FindRepoDir(path=None):
"""Returns the nearest higher-level repo dir from the specified path.
Args:
path: The path to use. Defaults to cwd.
"""
if path is None:
path = os.getcwd()
path = os.path.abspath(path)
while path != '/':
repo_dir = os.path.join(path, '.repo')
if os.path.isdir(repo_dir):
return repo_dir
cwd = os.path.dirname(cwd)
path = os.path.dirname(path)
return None
@ -155,7 +161,7 @@ def ReinterpretPathForChroot(path):
Keyword arguments:
path: The path to reinterpret. Must be in src tree.
"""
root_path = os.path.join(FindRepoDir(), '..')
root_path = os.path.join(FindRepoDir(path), '..')
path_abs_path = os.path.abspath(path)
root_abs_path = os.path.abspath(root_path)

23
make_factory_package.sh
View File

@ -218,9 +218,28 @@ fi
if [ -n "${FLAGS_subfolder}" ] && \
[ -f "${OMAHA_DIR}"/miniomaha.conf"" ] ; then
# Remove the ']' from the last line of the file so we can add another config.
sed -i '$d' ${OMAHA_DIR}/miniomaha.conf
while [ -s "${OMAHA_DIR}/miniomaha.conf" ]; do
# If the last line is null
if [ -z "$(tail -1 "${OMAHA_DIR}/miniomaha.conf")" ]; then
sed -i '$d' "${OMAHA_DIR}/miniomaha.conf"
elif [ "$(tail -1 "${OMAHA_DIR}/miniomaha.conf")" != ']' ]; then
sed -i '$d' "${OMAHA_DIR}/miniomaha.conf"
else
break
fi
done
# Remove the last ]
if [ "$(tail -1 "${OMAHA_DIR}/miniomaha.conf")" = ']' ]; then
sed -i '$d' "${OMAHA_DIR}/miniomaha.conf"
fi
# If the file is empty, create it from scratch
if [ ! -s "${OMAHA_DIR}/miniomaha.conf" ]; then
echo "config = [" > "${OMAHA_DIR}/miniomaha.conf"
fi
else
echo -e "config = [" > ${OMAHA_DIR}/miniomaha.conf
echo "config = [" > "${OMAHA_DIR}/miniomaha.conf"
fi
if [ -n "${FLAGS_subfolder}" ] ; then

2
remote_access.sh
View File

@ -23,7 +23,7 @@ function remote_cp_to() {
# Copies a list of remote files specified in file $1 to local location
# $2. Directory paths in $1 are collapsed into $2.
function remote_rsync_from() {
rsync -e "ssh -o StrictHostKeyChecking=no -o \
rsync -e "ssh -p ${FLAGS_ssh_port} -o StrictHostKeyChecking=no -o \
UserKnownHostsFile=$TMP_KNOWN_HOSTS" --no-R \
--files-from=$1 root@${FLAGS_remote}:/ $2
}