mirrors/flatcar-scripts
1
0
Fork 0
mirror of https://github.com/flatcar/scripts.git synced 2026-05-04 19:56:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

crosutils: Make running tests easier for developers

Browse Source 
Add "fast" mode to run_remote_tests which causes the user to no longer need to emerge autotests before running them (even ones that require compilation.)

Change-Id: I60c1e8bfe562a787075b4f65b714e221e51934f7

BUG=8784
TEST=Ran bvts in and out of fast mode, ran UserCrash, SanAngeles, TPM, and backlight tests (which require compilation) in fast mode.

Review URL: http://codereview.chromium.org/4020004
This commit is contained in:
Ken Mixter 2010-11-05 10:21:08 -07:00
parent 16106bade8
commit f010d9365f
2 changed files with 134 additions and 324 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

229
autotest_workon
View File

@ -1,229 +0,0 @@
#!/usr/bin/python
# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# A python wrapper to call autotest ebuild.
import commands, logging, optparse, os, subprocess, sys
def run(cmd):
return subprocess.call(cmd, stdout=sys.stdout, stderr=sys.stderr)
class MyOptionParser(optparse.OptionParser):
"""Override python's builtin OptionParser to accept any undefined args."""
help = False
def _process_args(self, largs, rargs, values):
# see /usr/lib64/python2.6/optparse.py line 1414-1463
while rargs:
arg = rargs[0]
# We handle bare "--" explicitly, and bare "-" is handled by the
# standard arg handler since the short arg case ensures that the
# len of the opt string is greater than 1.
if arg == "--":
del rargs[0]
return
elif arg[0:2] == "--":
# process a single long option (possibly with value(s))
try:
self._process_long_opt(rargs, values)
except optparse.BadOptionError:
largs.append(arg)
elif arg[:1] == "-" and len(arg) > 1:
# process a cluster of short options (possibly with
# value(s) for the last one only)
try:
self._process_short_opts(rargs, values)
except optparse.BadOptionError:
largs.append(arg)
elif self.allow_interspersed_args:
largs.append(arg)
del rargs[0]
else:
return # stop now, leave this arg in rargs
def print_help(self, file=None):
optparse.OptionParser.print_help(self, file)
MyOptionParser.help = True
parser = MyOptionParser()
parser.allow_interspersed_args = True
DEFAULT_BOARD = os.environ.get('DEFAULT_BOARD', '')
parser.add_option('--args', dest='args', action='store',
default='',
help='The arguments to pass to the test control file.')
parser.add_option('--autox', dest='autox', action='store_true',
default=True,
help='Build autox along with autotest [default].')
parser.add_option('--noautox', dest='autox', action='store_false',
help='Don\'t build autox along with autotest.')
parser.add_option('--board', dest='board', action='store',
default=DEFAULT_BOARD,
help='The board for which you are building autotest.')
parser.add_option('--build', dest='build', action='store',
help='Only prebuild client tests, do not run tests.')
parser.add_option('--buildcheck', dest='buildcheck', action='store_true',
default=True,
help='Fail if tests fail to build [default].')
parser.add_option('--nobuildcheck', dest='buildcheck', action='store_false',
help='Ignore test build failures.')
parser.add_option('--jobs', dest='jobs', action='store', type=int,
default=-1,
help='How many packages to build in parallel at maximum.')
parser.add_option('--noprompt', dest='noprompt', action='store_true',
help='Prompt user when building all tests.')
AUTOSERV='../third_party/autotest/files/server/autoserv'
AUTOTEST_CLIENT='../third_party/autotest/files/client/bin/autotest_client'
def parse_args_and_help():
def nop(_):
pass
sys_exit = sys.exit
sys.exit = nop
options, args = parser.parse_args()
sys.exit = sys_exit
if not args and not options.build:
parser.print_help()
if MyOptionParser.help:
if options.build:
print
print 'Options inherited from autotest_client, which is used in build',
print 'only mode.'
run([AUTOTEST_CLIENT, '--help'])
else:
print
print 'Options inherited from autoserv:'
run([AUTOSERV, '--help'])
sys.exit(0)
return options, args
def assert_inside_chroot(common_sh):
status, output = commands.getstatusoutput('/bin/bash -c ". %s && '
'assert_inside_chroot"' % common_sh)
if status is not 0:
print >> sys.stderr, output
sys.exit(status)
def set_common_env(common_sh, env_var):
env_value = commands.getoutput('/bin/bash -c \'. %s && echo $%s\'' %
(common_sh, env_var))
os.environ[env_var] = env_value
def die(common_sh, msg):
output = commands.getoutput('/bin/bash -c \'. %s && die "%s"\'' %
(common_sh, msg))
print >> sys.stderr, output
sys.exit(1)
def build_autotest(options):
environ = os.environ
if options.jobs != -1:
emerge_jobs = '--jobs=%d' % options.jobs
else:
emerge_jobs = ''
# Decide on USE flags based on options
use_flag = environ.get('USE', '')
if not options.autox:
use_flag = use_flag + ' -autox'
if options.buildcheck:
use_flag = use_flag + ' buildcheck'
board_blacklist_file = ('%s/src/overlays/overlay-%s/autotest-blacklist' %
(os.environ['GCLIENT_ROOT'], options.board))
if os.path.exists(board_blacklist_file):
blacklist = [line.strip()
for line in open(board_blacklist_file).readlines()]
else:
blacklist = []
all_tests = ('compilebench,dbench,disktest,fsx,hackbench,iperf,ltp,netperf2,'
'netpipe,unixbench')
site_tests = '../third_party/autotest/files/client/site_tests'
for site_test in os.listdir(site_tests):
test_path = os.path.join(site_tests, site_test)
test_py = os.path.join(test_path, '%s.py' % site_test)
if (os.path.exists(test_path) and os.path.isdir(test_path) and
os.path.exists(test_py) and os.path.isfile(test_py) and
site_test not in blacklist):
all_tests += ',' + site_test
if 'all' == options.build.lower():
if options.noprompt is not True:
print 'You want to pre-build all client tests and it may take a long',
print 'time to finish.'
print 'Are you sure you want to continue?(N/y)',
answer = sys.stdin.readline()
if 'y' != answer[0].lower():
print 'Use --build to specify tests you like to pre-compile. '
print 'E.g.: ./autotest --build=disktest,hardware_SAT'
sys.exit(0)
test_list = all_tests
else:
test_list = options.build
environ['FEATURES'] = ('%s -buildpkg -collision-protect' %
environ.get('FEATURES', ''))
environ['TEST_LIST'] = test_list
environ['USE'] = use_flag
emerge_cmd = ['emerge-%s' % options.board,
'chromeos-base/autotest']
if emerge_jobs:
emerge_cmd.append(emerge_jobs)
return run(emerge_cmd)
def run_autoserv(options, args):
environ = os.environ
environ['AUTOSERV_TEST_ARGS'] = options.args
environ['AUTOSERV_ARGS'] = ' '.join(args)
environ['FEATURES'] = ('%s -buildpkg -digest noauto' %
environ.get('FEATURES', ''))
ebuild_cmd = [ './autotest_run.sh', '--board=%s' % options.board]
run(ebuild_cmd)
def main():
me = sys.argv[0]
common_sh = os.path.join(os.path.dirname(me), 'common.sh')
assert_inside_chroot(common_sh)
set_common_env(common_sh, 'GCLIENT_ROOT')
options, args = parse_args_and_help()
if not options.board:
die(common_sh, 'Missing --board argument.')
if options.build:
status = build_autotest(options)
if status:
die(common_sh, 'build_autotest failed.')
else:
ssh_key_file = os.path.join(os.path.dirname(me),
'mod_for_test_scripts/ssh_keys/testing_rsa')
os.chmod(ssh_key_file, 0400)
run_autoserv(options, args)
if __name__ == '__main__':
main()

229
run_remote_tests.sh
View File

@ -18,42 +18,20 @@ DEFINE_string args "" \
"Command line arguments for test. Quoted and space separated if multiple." a
DEFINE_string board "$DEFAULT_BOARD" \
"The board for which you are building autotest"
DEFINE_boolean build ${FLAGS_FALSE} "Build tests while running" b
DEFINE_string chroot "${DEFAULT_CHROOT_DIR}" "alternate chroot location" c
DEFINE_boolean cleanup ${FLAGS_FALSE} "Clean up temp directory"
DEFINE_integer iterations 1 "Iterations to run every top level test" i
DEFINE_string results_dir_root "" "alternate root results directory"
DEFINE_boolean verbose ${FLAGS_FALSE} "Show verbose autoserv output" v
DEFINE_boolean use_emerged ${FLAGS_FALSE} \
"Force use of emerged autotest packages"
RAN_ANY_TESTS=${FLAGS_FALSE}
# Check if our stdout is a tty
function is_a_tty() {
local stdout=$(readlink /proc/$$/fd/1)
[[ "${stdout#/dev/tty}" != "${stdout}" ]] && return 0
[[ "${stdout#/dev/pts}" != "${stdout}" ]] && return 0
return 1
}
# Writes out text in specified color if stdout is a tty
# Arguments:
# $1 - color
# $2 - text to color
# $3 - text following colored text (default colored)
# Returns:
# None
function echo_color() {
local color=0
[[ "$1" == "red" ]] && color=31
[[ "$1" == "green" ]] && color=32
[[ "$1" == "yellow" ]] && color=33
if is_a_tty; then
echo -e "\033[1;${color}m$2\033[0m$3"
else
echo "$2$3"
fi
}
function cleanup() {
# Always remove the build path in case it was used.
[[ -n "${BUILD_DIR}" ]] && sudo rm -rf "${BUILD_DIR}"
if [[ $FLAGS_cleanup -eq ${FLAGS_TRUE} ]] || \
[[ ${RAN_ANY_TESTS} -eq ${FLAGS_FALSE} ]]; then
rm -rf "${TMP}"
@ -63,27 +41,6 @@ function cleanup() {
cleanup_remote_access
}
# Adds attributes to all tests run
# Arguments:
# $1 - results directory
# $2 - attribute name (key)
# $3 - attribute value (value)
function add_test_attribute() {
local results_dir="$1"
local attribute_name="$2"
local attribute_value="$3"
if [[ -z "$attribute_value" ]]; then
return;
fi
for status_file in $(echo "${results_dir}"/*/status); do
local keyval_file=$(dirname $status_file)/keyval
echo "Updating ${keyval_file}"
echo "${attribute_name}=${attribute_value}" >> "${keyval_file}"
done
}
# Determine if a control is for a client or server test. Echos
# either "server" or "client".
# Arguments:
@ -94,17 +51,83 @@ function read_test_type() {
local type=$(egrep -m1 \
'^[[:space:]]*TEST_TYPE[[:space:]]*=' "${control_file}")
if [[ -z "${type}" ]]; then
echo_color "red" ">>> Unable to find TEST_TYPE line in ${control_file}"
exit 1
die "Unable to find TEST_TYPE line in ${control_file}"
fi
type=$(python -c "${type}; print TEST_TYPE.lower()")
if [[ "${type}" != "client" ]] && [[ "${type}" != "server" ]]; then
echo_color "red" ">>> Unknown type of test (${type}) in ${control_file}"
exit 1
die "Unknown type of test (${type}) in ${control_file}"
fi
echo ${type}
}
function create_tmp() {
# Set global TMP for remote_access.sh's sake
# and if --results_dir_root is specified,
# set TMP and create dir appropriately
if [[ ${INSIDE_CHROOT} -eq 0 ]]; then
if [[ -n "${FLAGS_results_dir_root}" ]]; then
TMP=${FLAGS_chroot}${FLAGS_results_dir_root}
mkdir -p -m 777 ${TMP}
else
TMP=$(mktemp -d ${FLAGS_chroot}/tmp/run_remote_tests.XXXX)
fi
TMP_INSIDE_CHROOT=$(echo ${TMP#${FLAGS_chroot}})
else
if [[ -n "${FLAGS_results_dir_root}" ]]; then
TMP=${FLAGS_results_dir_root}
mkdir -p -m 777 ${TMP}
else
TMP=$(mktemp -d /tmp/run_remote_tests.XXXX)
fi
TMP_INSIDE_CHROOT=${TMP}
fi
}
function prepare_build_dir() {
local autotest_dir="$1"
INSIDE_BUILD_DIR="${TMP_INSIDE_CHROOT}/build"
BUILD_DIR="${TMP}/build"
info "Copying autotest tree into ${BUILD_DIR}."
sudo mkdir -p "${BUILD_DIR}"
sudo rsync -rl --chmod=ugo=rwx "${autotest_dir}"/ "${BUILD_DIR}"
info "Pilfering toolchain shell environment from Portage."
local outside_ebuild_dir="${TMP}/chromeos-base/autotest-build"
local inside_ebuild_dir="${TMP_INSIDE_CHROOT}/chromeos-base/autotest-build"
mkdir -p "${outside_ebuild_dir}"
local E_only="autotest-build-9999.ebuild"
cat > "${outside_ebuild_dir}/${E_only}" <<EOF
inherit toolchain-funcs
SLOT="0"
EOF
local E="chromeos-base/autotest-build/${E_only}"
${ENTER_CHROOT} "ebuild-${FLAGS_board}" "${inside_ebuild_dir}/${E_only}" \
clean unpack 2>&1 > /dev/null
local P_tmp="${FLAGS_chroot}/build/${FLAGS_board}/tmp/portage/"
local E_dir="${E%%/*}/${E_only%.*}"
sudo cp "${P_tmp}/${E_dir}/temp/environment" "${BUILD_DIR}"
}
function autodetect_build() {
if [ ${FLAGS_use_emerged} -eq ${FLAGS_TRUE} ]; then
info \
"As requested, using emerged autotests already installed in your sysroot."
FLAGS_build=${FLAGS_FALSE}
return
fi
if ${ENTER_CHROOT} ./cros_workon --board=${FLAGS_board} list | \
grep -q autotest; then
info \
"Detected cros_workon autotests, building your sources instead of emerged \
autotest. To use installed autotest, pass --use_emerged."
FLAGS_build=${FLAGS_TRUE}
else
info \
"Using emerged autotests already installed in your sysroot. To build \
autotests directly from your source directory instead, pass --build."
FLAGS_build=${FLAGS_FALSE}
fi
}
function main() {
cd $(dirname "$0")
@ -130,26 +153,7 @@ function main() {
set -e
# Set global TMP for remote_access.sh's sake
# and if --results_dir_root is specified,
# set TMP and create dir appropriately
if [[ ${INSIDE_CHROOT} -eq 0 ]]; then
if [[ -n "${FLAGS_results_dir_root}" ]]; then
TMP=${FLAGS_chroot}${FLAGS_results_dir_root}
mkdir -p -m 777 ${TMP}
else
TMP=$(mktemp -d ${FLAGS_chroot}/tmp/run_remote_tests.XXXX)
fi
TMP_INSIDE_CHROOT=$(echo ${TMP#${FLAGS_chroot}})
else
if [[ -n "${FLAGS_results_dir_root}" ]]; then
TMP=${FLAGS_results_dir_root}
mkdir -p -m 777 ${TMP}
else
TMP=$(mktemp -d /tmp/run_remote_tests.XXXX)
fi
TMP_INSIDE_CHROOT=${TMP}
fi
create_tmp
trap cleanup EXIT
@ -158,6 +162,23 @@ function main() {
learn_board
autotest_dir="${FLAGS_chroot}/build/${FLAGS_board}/usr/local/autotest"
ENTER_CHROOT=""
if [[ ${INSIDE_CHROOT} -eq 0 ]]; then
ENTER_CHROOT="./enter_chroot.sh --chroot ${FLAGS_chroot} --"
fi
if [ ${FLAGS_build} -eq ${FLAGS_FALSE} ]; then
autodetect_build
fi
if [ ${FLAGS_build} -eq ${FLAGS_TRUE} ]; then
autotest_dir="${SRC_ROOT}/third_party/autotest/files"
else
if [ ! -d "${autotest_dir}" ]; then
die "You need to emerge autotest-tests (or use --build)"
fi
fi
local control_files_to_run=""
local chrome_autotests="${CHROME_ROOT}/src/chrome/test/chromeos/autotest/files"
# Now search for tests which unambiguously include the given identifier
@ -172,8 +193,7 @@ function main() {
! finds=$(find ${search_path} -maxdepth 2 -type f \( -name control.\* -or \
-name control \) | egrep -v "~$" | egrep "${test_request}")
if [[ -z "${finds}" ]]; then
echo_color "red" ">>> Cannot find match for \"${test_request}\""
exit 1
die "Cannot find match for \"${test_request}\""
fi
local matches=$(echo "${finds}" | wc -l)
if [[ ${matches} -gt 1 ]]; then
@ -193,13 +213,14 @@ function main() {
echo ""
if [[ -z "${control_files_to_run}" ]]; then
echo_color "red" ">>> Found no control files"
exit 1
die "Found no control files"
fi
echo_color "yellow" ">>> Running the following control files:"
[ ${FLAGS_build} -eq ${FLAGS_TRUE} ] && prepare_build_dir "${autotest_dir}"
info "Running the following control files:"
for CONTROL_FILE in ${control_files_to_run}; do
echo_color "yellow" " * " "${CONTROL_FILE}"
info " * ${CONTROL_FILE}"
done
for control_file in ${control_files_to_run}; do
@ -217,7 +238,7 @@ function main() {
option="-s"
fi
echo ""
echo_color "yellow" ">>> Running ${type} test " ${control_file}
info "Running ${type} test ${control_file}"
local control_file_name=$(basename "${control_file}")
local short_name=$(basename $(dirname "${control_file}"))
@ -243,31 +264,49 @@ function main() {
RAN_ANY_TESTS=${FLAGS_TRUE}
local enter_chroot=""
local autotest="${GCLIENT_ROOT}/src/scripts/autotest_workon"
if [[ ${INSIDE_CHROOT} -eq 0 ]]; then
enter_chroot="./enter_chroot.sh --chroot ${FLAGS_chroot} --"
autotest="./autotest_workon"
fi
# Remove chrome autotest location prefix from control_file if needed
if [[ ${control_file:0:${#chrome_autotests}} == \
"${chrome_autotests}" ]]; then
control_file="${control_file:${#chrome_autotests}+1}"
echo_color "yellow" ">>> Running chrome autotest " ${control_file}
fi
if [[ -n "${FLAGS_args}" ]]; then
passthrough_args="--args=${FLAGS_args}"
info "Running chrome autotest ${control_file}"
fi
${enter_chroot} ${autotest} --board "${FLAGS_board}" -m "${FLAGS_remote}" \
--ssh-port ${FLAGS_ssh_port} \
"${option}" "${control_file}" -r "${results_dir}" ${verbose} \
"${passthrough_args}" >&2
export AUTOSERV_TEST_ARGS="${FLAGS_args}"
export AUTOSERV_ARGS="-m ${FLAGS_remote} \
--ssh-port ${FLAGS_ssh_port} \
${option} ${control_file} -r ${results_dir} ${verbose}"
if [ ${FLAGS_build} -eq ${FLAGS_FALSE} ]; then
cat > "${TMP}/run_test.sh" <<EOF
export AUTOSERV_TEST_ARGS="${AUTOSERV_TEST_ARGS}"
export AUTOSERV_ARGS="${AUTOSERV_ARGS}"
cd /home/${USER}/trunk/src/scripts
./autotest_run.sh --board "${FLAGS_board}"
EOF
chmod a+rx "${TMP}/run_test.sh"
${ENTER_CHROOT} ${TMP_INSIDE_CHROOT}/run_test.sh >&2
else
cp "${BUILD_DIR}/environment" "${TMP}/run_test.sh"
GRAPHICS_BACKEND=${GRAPHICS_BACKEND:-OPENGL}
if [ -n "${AUTOSERV_TEST_ARGS}" ]; then
AUTOSERV_TEST_ARGS="-a \"${AUTOSERV_TEST_ARGS}\""
fi
cat >> "${TMP}/run_test.sh" <<EOF
export GCLIENT_ROOT=/home/${USER}/trunk
export GRAPHICS_BACKEND=${GRAPHICS_BACKEND}
export SSH_AUTH_SOCK=${SSH_AUTH_SOCK} TMPDIR=/tmp SSH_AGENT_PID=${SSH_AGENT_PID}
export SYSROOT=/build/${FLAGS_board}
tc-export CC CXX PKG_CONFIG
cd ${INSIDE_BUILD_DIR}
./server/autoserv ${AUTOSERV_ARGS} ${AUTOSERV_TEST_ARGS}
EOF
sudo cp "${TMP}/run_test.sh" "${BUILD_DIR}"
sudo chmod a+rx "${BUILD_DIR}/run_test.sh"
${ENTER_CHROOT} sudo bash -c "${INSIDE_BUILD_DIR}/run_test.sh" >&2
fi
done
echo ""
echo_color "yellow" ">>> Test results:"
info "Test results:"
./generate_test_report "${TMP}" --strip="${TMP}/"
print_time_elapsed