From f010d9365f05bfd4a299993617ce7093f1ebed02 Mon Sep 17 00:00:00 2001 From: Ken Mixter Date: Fri, 5 Nov 2010 10:21:08 -0700 Subject: [PATCH] crosutils: Make running tests easier for developers Add "fast" mode to run_remote_tests which causes the user to no longer need to emerge autotests before running them (even ones that require compilation.) Change-Id: I60c1e8bfe562a787075b4f65b714e221e51934f7 BUG=8784 TEST=Ran bvts in and out of fast mode, ran UserCrash, SanAngeles, TPM, and backlight tests (which require compilation) in fast mode. Review URL: http://codereview.chromium.org/4020004 --- autotest_workon | 229 -------------------------------------------- run_remote_tests.sh | 229 ++++++++++++++++++++++++++------------------ 2 files changed, 134 insertions(+), 324 deletions(-) delete mode 100755 autotest_workon diff --git a/autotest_workon b/autotest_workon deleted file mode 100755 index ee12f4ad47..0000000000 --- a/autotest_workon +++ /dev/null @@ -1,229 +0,0 @@ -#!/usr/bin/python - -# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -# -# A python wrapper to call autotest ebuild. - -import commands, logging, optparse, os, subprocess, sys - - -def run(cmd): - return subprocess.call(cmd, stdout=sys.stdout, stderr=sys.stderr) - - -class MyOptionParser(optparse.OptionParser): - """Override python's builtin OptionParser to accept any undefined args.""" - - help = False - - def _process_args(self, largs, rargs, values): - # see /usr/lib64/python2.6/optparse.py line 1414-1463 - while rargs: - arg = rargs[0] - # We handle bare "--" explicitly, and bare "-" is handled by the - # standard arg handler since the short arg case ensures that the - # len of the opt string is greater than 1. - if arg == "--": - del rargs[0] - return - elif arg[0:2] == "--": - # process a single long option (possibly with value(s)) - try: - self._process_long_opt(rargs, values) - except optparse.BadOptionError: - largs.append(arg) - elif arg[:1] == "-" and len(arg) > 1: - # process a cluster of short options (possibly with - # value(s) for the last one only) - try: - self._process_short_opts(rargs, values) - except optparse.BadOptionError: - largs.append(arg) - elif self.allow_interspersed_args: - largs.append(arg) - del rargs[0] - else: - return # stop now, leave this arg in rargs - - def print_help(self, file=None): - optparse.OptionParser.print_help(self, file) - MyOptionParser.help = True - - -parser = MyOptionParser() -parser.allow_interspersed_args = True - -DEFAULT_BOARD = os.environ.get('DEFAULT_BOARD', '') - -parser.add_option('--args', dest='args', action='store', - default='', - help='The arguments to pass to the test control file.') -parser.add_option('--autox', dest='autox', action='store_true', - default=True, - help='Build autox along with autotest [default].') -parser.add_option('--noautox', dest='autox', action='store_false', - help='Don\'t build autox along with autotest.') -parser.add_option('--board', dest='board', action='store', - default=DEFAULT_BOARD, - help='The board for which you are building autotest.') -parser.add_option('--build', dest='build', action='store', - help='Only prebuild client tests, do not run tests.') -parser.add_option('--buildcheck', dest='buildcheck', action='store_true', - default=True, - help='Fail if tests fail to build [default].') -parser.add_option('--nobuildcheck', dest='buildcheck', action='store_false', - help='Ignore test build failures.') -parser.add_option('--jobs', dest='jobs', action='store', type=int, - default=-1, - help='How many packages to build in parallel at maximum.') -parser.add_option('--noprompt', dest='noprompt', action='store_true', - help='Prompt user when building all tests.') - - -AUTOSERV='../third_party/autotest/files/server/autoserv' -AUTOTEST_CLIENT='../third_party/autotest/files/client/bin/autotest_client' - -def parse_args_and_help(): - - def nop(_): - pass - - sys_exit = sys.exit - sys.exit = nop - options, args = parser.parse_args() - sys.exit = sys_exit - - if not args and not options.build: - parser.print_help() - - if MyOptionParser.help: - if options.build: - print - print 'Options inherited from autotest_client, which is used in build', - print 'only mode.' - run([AUTOTEST_CLIENT, '--help']) - else: - print - print 'Options inherited from autoserv:' - run([AUTOSERV, '--help']) - sys.exit(0) - return options, args - - -def assert_inside_chroot(common_sh): - status, output = commands.getstatusoutput('/bin/bash -c ". %s && ' - 'assert_inside_chroot"' % common_sh) - if status is not 0: - print >> sys.stderr, output - sys.exit(status) - - -def set_common_env(common_sh, env_var): - env_value = commands.getoutput('/bin/bash -c \'. %s && echo $%s\'' % - (common_sh, env_var)) - os.environ[env_var] = env_value - - -def die(common_sh, msg): - output = commands.getoutput('/bin/bash -c \'. %s && die "%s"\'' % - (common_sh, msg)) - print >> sys.stderr, output - sys.exit(1) - - -def build_autotest(options): - environ = os.environ - if options.jobs != -1: - emerge_jobs = '--jobs=%d' % options.jobs - else: - emerge_jobs = '' - - # Decide on USE flags based on options - use_flag = environ.get('USE', '') - if not options.autox: - use_flag = use_flag + ' -autox' - if options.buildcheck: - use_flag = use_flag + ' buildcheck' - - board_blacklist_file = ('%s/src/overlays/overlay-%s/autotest-blacklist' % - (os.environ['GCLIENT_ROOT'], options.board)) - if os.path.exists(board_blacklist_file): - blacklist = [line.strip() - for line in open(board_blacklist_file).readlines()] - else: - blacklist = [] - - all_tests = ('compilebench,dbench,disktest,fsx,hackbench,iperf,ltp,netperf2,' - 'netpipe,unixbench') - site_tests = '../third_party/autotest/files/client/site_tests' - for site_test in os.listdir(site_tests): - test_path = os.path.join(site_tests, site_test) - test_py = os.path.join(test_path, '%s.py' % site_test) - if (os.path.exists(test_path) and os.path.isdir(test_path) and - os.path.exists(test_py) and os.path.isfile(test_py) and - site_test not in blacklist): - all_tests += ',' + site_test - - if 'all' == options.build.lower(): - if options.noprompt is not True: - print 'You want to pre-build all client tests and it may take a long', - print 'time to finish.' - print 'Are you sure you want to continue?(N/y)', - answer = sys.stdin.readline() - if 'y' != answer[0].lower(): - print 'Use --build to specify tests you like to pre-compile. ' - print 'E.g.: ./autotest --build=disktest,hardware_SAT' - sys.exit(0) - test_list = all_tests - else: - test_list = options.build - - environ['FEATURES'] = ('%s -buildpkg -collision-protect' % - environ.get('FEATURES', '')) - environ['TEST_LIST'] = test_list - environ['USE'] = use_flag - emerge_cmd = ['emerge-%s' % options.board, - 'chromeos-base/autotest'] - if emerge_jobs: - emerge_cmd.append(emerge_jobs) - return run(emerge_cmd) - - -def run_autoserv(options, args): - environ = os.environ - - environ['AUTOSERV_TEST_ARGS'] = options.args - environ['AUTOSERV_ARGS'] = ' '.join(args) - environ['FEATURES'] = ('%s -buildpkg -digest noauto' % - environ.get('FEATURES', '')) - ebuild_cmd = [ './autotest_run.sh', '--board=%s' % options.board] - run(ebuild_cmd) - - -def main(): - me = sys.argv[0] - common_sh = os.path.join(os.path.dirname(me), 'common.sh') - - assert_inside_chroot(common_sh) - set_common_env(common_sh, 'GCLIENT_ROOT') - - options, args = parse_args_and_help() - - if not options.board: - die(common_sh, 'Missing --board argument.') - - if options.build: - status = build_autotest(options) - if status: - die(common_sh, 'build_autotest failed.') - else: - ssh_key_file = os.path.join(os.path.dirname(me), - 'mod_for_test_scripts/ssh_keys/testing_rsa') - os.chmod(ssh_key_file, 0400) - run_autoserv(options, args) - - -if __name__ == '__main__': - main() diff --git a/run_remote_tests.sh b/run_remote_tests.sh index 864d8bb756..babbbdceb0 100755 --- a/run_remote_tests.sh +++ b/run_remote_tests.sh @@ -18,42 +18,20 @@ DEFINE_string args "" \ "Command line arguments for test. Quoted and space separated if multiple." a DEFINE_string board "$DEFAULT_BOARD" \ "The board for which you are building autotest" +DEFINE_boolean build ${FLAGS_FALSE} "Build tests while running" b DEFINE_string chroot "${DEFAULT_CHROOT_DIR}" "alternate chroot location" c DEFINE_boolean cleanup ${FLAGS_FALSE} "Clean up temp directory" DEFINE_integer iterations 1 "Iterations to run every top level test" i DEFINE_string results_dir_root "" "alternate root results directory" DEFINE_boolean verbose ${FLAGS_FALSE} "Show verbose autoserv output" v +DEFINE_boolean use_emerged ${FLAGS_FALSE} \ + "Force use of emerged autotest packages" RAN_ANY_TESTS=${FLAGS_FALSE} -# Check if our stdout is a tty -function is_a_tty() { - local stdout=$(readlink /proc/$$/fd/1) - [[ "${stdout#/dev/tty}" != "${stdout}" ]] && return 0 - [[ "${stdout#/dev/pts}" != "${stdout}" ]] && return 0 - return 1 -} - -# Writes out text in specified color if stdout is a tty -# Arguments: -# $1 - color -# $2 - text to color -# $3 - text following colored text (default colored) -# Returns: -# None -function echo_color() { - local color=0 - [[ "$1" == "red" ]] && color=31 - [[ "$1" == "green" ]] && color=32 - [[ "$1" == "yellow" ]] && color=33 - if is_a_tty; then - echo -e "\033[1;${color}m$2\033[0m$3" - else - echo "$2$3" - fi -} - function cleanup() { + # Always remove the build path in case it was used. + [[ -n "${BUILD_DIR}" ]] && sudo rm -rf "${BUILD_DIR}" if [[ $FLAGS_cleanup -eq ${FLAGS_TRUE} ]] || \ [[ ${RAN_ANY_TESTS} -eq ${FLAGS_FALSE} ]]; then rm -rf "${TMP}" @@ -63,27 +41,6 @@ function cleanup() { cleanup_remote_access } -# Adds attributes to all tests run -# Arguments: -# $1 - results directory -# $2 - attribute name (key) -# $3 - attribute value (value) -function add_test_attribute() { - local results_dir="$1" - local attribute_name="$2" - local attribute_value="$3" - if [[ -z "$attribute_value" ]]; then - return; - fi - - for status_file in $(echo "${results_dir}"/*/status); do - local keyval_file=$(dirname $status_file)/keyval - echo "Updating ${keyval_file}" - echo "${attribute_name}=${attribute_value}" >> "${keyval_file}" - done -} - - # Determine if a control is for a client or server test. Echos # either "server" or "client". # Arguments: @@ -94,17 +51,83 @@ function read_test_type() { local type=$(egrep -m1 \ '^[[:space:]]*TEST_TYPE[[:space:]]*=' "${control_file}") if [[ -z "${type}" ]]; then - echo_color "red" ">>> Unable to find TEST_TYPE line in ${control_file}" - exit 1 + die "Unable to find TEST_TYPE line in ${control_file}" fi type=$(python -c "${type}; print TEST_TYPE.lower()") if [[ "${type}" != "client" ]] && [[ "${type}" != "server" ]]; then - echo_color "red" ">>> Unknown type of test (${type}) in ${control_file}" - exit 1 + die "Unknown type of test (${type}) in ${control_file}" fi echo ${type} } +function create_tmp() { + # Set global TMP for remote_access.sh's sake + # and if --results_dir_root is specified, + # set TMP and create dir appropriately + if [[ ${INSIDE_CHROOT} -eq 0 ]]; then + if [[ -n "${FLAGS_results_dir_root}" ]]; then + TMP=${FLAGS_chroot}${FLAGS_results_dir_root} + mkdir -p -m 777 ${TMP} + else + TMP=$(mktemp -d ${FLAGS_chroot}/tmp/run_remote_tests.XXXX) + fi + TMP_INSIDE_CHROOT=$(echo ${TMP#${FLAGS_chroot}}) + else + if [[ -n "${FLAGS_results_dir_root}" ]]; then + TMP=${FLAGS_results_dir_root} + mkdir -p -m 777 ${TMP} + else + TMP=$(mktemp -d /tmp/run_remote_tests.XXXX) + fi + TMP_INSIDE_CHROOT=${TMP} + fi +} + +function prepare_build_dir() { + local autotest_dir="$1" + INSIDE_BUILD_DIR="${TMP_INSIDE_CHROOT}/build" + BUILD_DIR="${TMP}/build" + info "Copying autotest tree into ${BUILD_DIR}." + sudo mkdir -p "${BUILD_DIR}" + sudo rsync -rl --chmod=ugo=rwx "${autotest_dir}"/ "${BUILD_DIR}" + info "Pilfering toolchain shell environment from Portage." + local outside_ebuild_dir="${TMP}/chromeos-base/autotest-build" + local inside_ebuild_dir="${TMP_INSIDE_CHROOT}/chromeos-base/autotest-build" + mkdir -p "${outside_ebuild_dir}" + local E_only="autotest-build-9999.ebuild" + cat > "${outside_ebuild_dir}/${E_only}" <&1 > /dev/null + local P_tmp="${FLAGS_chroot}/build/${FLAGS_board}/tmp/portage/" + local E_dir="${E%%/*}/${E_only%.*}" + sudo cp "${P_tmp}/${E_dir}/temp/environment" "${BUILD_DIR}" +} + +function autodetect_build() { + if [ ${FLAGS_use_emerged} -eq ${FLAGS_TRUE} ]; then + info \ +"As requested, using emerged autotests already installed in your sysroot." + FLAGS_build=${FLAGS_FALSE} + return + fi + if ${ENTER_CHROOT} ./cros_workon --board=${FLAGS_board} list | \ + grep -q autotest; then + info \ +"Detected cros_workon autotests, building your sources instead of emerged \ +autotest. To use installed autotest, pass --use_emerged." + FLAGS_build=${FLAGS_TRUE} + else + info \ +"Using emerged autotests already installed in your sysroot. To build \ +autotests directly from your source directory instead, pass --build." + FLAGS_build=${FLAGS_FALSE} + fi +} + function main() { cd $(dirname "$0") @@ -130,26 +153,7 @@ function main() { set -e - # Set global TMP for remote_access.sh's sake - # and if --results_dir_root is specified, - # set TMP and create dir appropriately - if [[ ${INSIDE_CHROOT} -eq 0 ]]; then - if [[ -n "${FLAGS_results_dir_root}" ]]; then - TMP=${FLAGS_chroot}${FLAGS_results_dir_root} - mkdir -p -m 777 ${TMP} - else - TMP=$(mktemp -d ${FLAGS_chroot}/tmp/run_remote_tests.XXXX) - fi - TMP_INSIDE_CHROOT=$(echo ${TMP#${FLAGS_chroot}}) - else - if [[ -n "${FLAGS_results_dir_root}" ]]; then - TMP=${FLAGS_results_dir_root} - mkdir -p -m 777 ${TMP} - else - TMP=$(mktemp -d /tmp/run_remote_tests.XXXX) - fi - TMP_INSIDE_CHROOT=${TMP} - fi + create_tmp trap cleanup EXIT @@ -158,6 +162,23 @@ function main() { learn_board autotest_dir="${FLAGS_chroot}/build/${FLAGS_board}/usr/local/autotest" + ENTER_CHROOT="" + if [[ ${INSIDE_CHROOT} -eq 0 ]]; then + ENTER_CHROOT="./enter_chroot.sh --chroot ${FLAGS_chroot} --" + fi + + if [ ${FLAGS_build} -eq ${FLAGS_FALSE} ]; then + autodetect_build + fi + + if [ ${FLAGS_build} -eq ${FLAGS_TRUE} ]; then + autotest_dir="${SRC_ROOT}/third_party/autotest/files" + else + if [ ! -d "${autotest_dir}" ]; then + die "You need to emerge autotest-tests (or use --build)" + fi + fi + local control_files_to_run="" local chrome_autotests="${CHROME_ROOT}/src/chrome/test/chromeos/autotest/files" # Now search for tests which unambiguously include the given identifier @@ -172,8 +193,7 @@ function main() { ! finds=$(find ${search_path} -maxdepth 2 -type f \( -name control.\* -or \ -name control \) | egrep -v "~$" | egrep "${test_request}") if [[ -z "${finds}" ]]; then - echo_color "red" ">>> Cannot find match for \"${test_request}\"" - exit 1 + die "Cannot find match for \"${test_request}\"" fi local matches=$(echo "${finds}" | wc -l) if [[ ${matches} -gt 1 ]]; then @@ -193,13 +213,14 @@ function main() { echo "" if [[ -z "${control_files_to_run}" ]]; then - echo_color "red" ">>> Found no control files" - exit 1 + die "Found no control files" fi - echo_color "yellow" ">>> Running the following control files:" + [ ${FLAGS_build} -eq ${FLAGS_TRUE} ] && prepare_build_dir "${autotest_dir}" + + info "Running the following control files:" for CONTROL_FILE in ${control_files_to_run}; do - echo_color "yellow" " * " "${CONTROL_FILE}" + info " * ${CONTROL_FILE}" done for control_file in ${control_files_to_run}; do @@ -217,7 +238,7 @@ function main() { option="-s" fi echo "" - echo_color "yellow" ">>> Running ${type} test " ${control_file} + info "Running ${type} test ${control_file}" local control_file_name=$(basename "${control_file}") local short_name=$(basename $(dirname "${control_file}")) @@ -243,31 +264,49 @@ function main() { RAN_ANY_TESTS=${FLAGS_TRUE} - local enter_chroot="" - local autotest="${GCLIENT_ROOT}/src/scripts/autotest_workon" - if [[ ${INSIDE_CHROOT} -eq 0 ]]; then - enter_chroot="./enter_chroot.sh --chroot ${FLAGS_chroot} --" - autotest="./autotest_workon" - fi - # Remove chrome autotest location prefix from control_file if needed if [[ ${control_file:0:${#chrome_autotests}} == \ "${chrome_autotests}" ]]; then control_file="${control_file:${#chrome_autotests}+1}" - echo_color "yellow" ">>> Running chrome autotest " ${control_file} - fi - if [[ -n "${FLAGS_args}" ]]; then - passthrough_args="--args=${FLAGS_args}" + info "Running chrome autotest ${control_file}" fi - ${enter_chroot} ${autotest} --board "${FLAGS_board}" -m "${FLAGS_remote}" \ - --ssh-port ${FLAGS_ssh_port} \ - "${option}" "${control_file}" -r "${results_dir}" ${verbose} \ - "${passthrough_args}" >&2 + export AUTOSERV_TEST_ARGS="${FLAGS_args}" + export AUTOSERV_ARGS="-m ${FLAGS_remote} \ + --ssh-port ${FLAGS_ssh_port} \ + ${option} ${control_file} -r ${results_dir} ${verbose}" + if [ ${FLAGS_build} -eq ${FLAGS_FALSE} ]; then + cat > "${TMP}/run_test.sh" <&2 + else + cp "${BUILD_DIR}/environment" "${TMP}/run_test.sh" + GRAPHICS_BACKEND=${GRAPHICS_BACKEND:-OPENGL} + if [ -n "${AUTOSERV_TEST_ARGS}" ]; then + AUTOSERV_TEST_ARGS="-a \"${AUTOSERV_TEST_ARGS}\"" + fi + cat >> "${TMP}/run_test.sh" <&2 + fi done echo "" - echo_color "yellow" ">>> Test results:" + info "Test results:" ./generate_test_report "${TMP}" --strip="${TMP}/" print_time_elapsed