From 7333e2b4c4ba757a59db9e9bb3352405640ebaa0 Mon Sep 17 00:00:00 2001 From: Scott Zawalski Date: Tue, 2 Nov 2010 18:01:14 -0700 Subject: [PATCH 1/8] Update prebuilt.py call to use the _proper_ variable for finding the build root. BUG=8603 TEST=Tested where GCLIENT_ROOT points to on rm and x86 Review URL: http://codereview.chromium.org/4363001 --- archive_build.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/archive_build.sh b/archive_build.sh index 84f1d51d98..74d577ea84 100755 --- a/archive_build.sh +++ b/archive_build.sh @@ -273,8 +273,7 @@ then # This will upload prebuilt packages to Google Storage. prebuilt_cmd="${SCRIPTS_DIR}/prebuilt.py" prebuilt_cmd="$prebuilt_cmd -u gs://chromeos-prebuilt --git-sync -V master" - prebuilt_cmd="$prebuilt_cmd -p $(readlink -f ../../../../build/chromiumos)" - prebuilt_cmd="$prebuilt_cmd -b ${FLAGS_board}" + prebuilt_cmd="$prebuilt_cmd -p ${GCLIENT_ROOT} -b ${FLAGS_board}" if [ "${FLAGS_BOARD}" == "x86-generic" ] then From a2a597d18bd49f4cd734be10f8ee97d9658e735c Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Tue, 2 Nov 2010 18:06:34 -0700 Subject: [PATCH 2/8] Add ability to fall back to fulls for all updates. Currently works around issue till we get a good fix in for 9.x. Change-Id: I9c7225508fa8f46a5fa5e0b41abc4e373cc80085 BUG=8606 TEST=Tested by sending SEGV to a delta update during update and saw it successfully fall back to a full update and succeed. Review URL: http://codereview.chromium.org/4258002 --- bin/cros_au_test_harness.py | 51 ++++++++++++++----------------------- 1 file changed, 19 insertions(+), 32 deletions(-) diff --git a/bin/cros_au_test_harness.py b/bin/cros_au_test_harness.py index b9640b4c3f..7342310ce7 100755 --- a/bin/cros_au_test_harness.py +++ b/bin/cros_au_test_harness.py @@ -63,6 +63,21 @@ class AUTest(object): return int(percent_passed) + # TODO(sosa) - Remove try and convert function to DeltaUpdateImage(). + def TryDeltaAndFallbackToFull(self, src_image, image, stateful_change='old'): + """Tries the delta update first if set and falls back to full update.""" + if self.use_delta_updates: + try: + self.source_image = src_image + self.UpdateImage(image) + except: + Warning('Delta update failed, disabling delta updates and retrying.') + self.use_delta_updates = False + self.source_image = '' + self.UpdateImage(image) + else: + self.UpdateImage(image) + def PrepareBase(self): """Prepares target with base_image_path.""" pass @@ -130,28 +145,14 @@ class AUTest(object): # with the dev channel. percent_passed = self.VerifyImage(10) - if self.use_delta_updates: self.source_image = base_image_path - # Update to - all tests should pass on new image. Info('Updating from base image on vm to target image.') - try: - self.UpdateImage(target_image_path) - except: - if self.use_delta_updates: - Warning('Delta update failed, disabling delta updates and retrying.') - self.use_delta_updates = False - self.source_image = '' - self.UpdateImage(target_image_path) - else: - raise - + self.TryDeltaAndFallbackToFull(base_image_path, target_image_path) self.VerifyImage(100) - if self.use_delta_updates: self.source_image = target_image_path - # Update from - same percentage should pass that originally passed. Info('Updating from updated image on vm back to base image.') - self.UpdateImage(base_image_path) + self.TryDeltaAndFallbackToFull(target_image_path, base_image_path) self.VerifyImage(percent_passed) def testFullUpdateWipeStateful(self): @@ -167,28 +168,14 @@ class AUTest(object): # with the dev channel. percent_passed = self.VerifyImage(10) - if self.use_delta_updates: self.source_image = base_image_path - # Update to - all tests should pass on new image. Info('Updating from base image on vm to target image and wiping stateful.') - try: - self.UpdateImage(target_image_path, 'clean') - except: - if self.use_delta_updates: - Warning('Delta update failed, disabling delta updates and retrying.') - self.use_delta_updates = False - self.source_image = '' - self.UpdateImage(target_image_path) - else: - raise - + self.TryDeltaAndFallbackToFull(base_image_path, target_image_path, 'clean') self.VerifyImage(100) - if self.use_delta_updates: self.source_image = target_image_path - # Update from - same percentage should pass that originally passed. Info('Updating from updated image back to base image and wiping stateful.') - self.UpdateImage(base_image_path, 'clean') + self.TryDeltaAndFallbackToFull(target_image_path, base_image_path, 'clean') self.VerifyImage(percent_passed) From a424d94ab3af0502a75ac5ba25f50879655fbbdd Mon Sep 17 00:00:00 2001 From: Darin Petkov Date: Wed, 3 Nov 2010 10:09:39 -0700 Subject: [PATCH 3/8] A basic utility to allow running of tests on VMs in parallel. Current usage is simple: just specify a list of tests and each one will be run on a separate VM using the latest image on a default board. Exit code is non-zero if any of the tests fail. Output from separate tests is interleaved. More CLs to follow -- specifying a result directory, non-interleaved output, base ssh port, board, image path, etc. BUG=8585 TEST=./bin/cros_run_parallel_vm_tests suite_Smoke suite_Smoke Change-Id: Ifa3396768050905ec02e4cad8b8f065a6d129154 Review URL: http://codereview.chromium.org/4337003 --- bin/cros_run_parallel_vm_tests | 1 + bin/cros_run_parallel_vm_tests.py | 92 +++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 120000 bin/cros_run_parallel_vm_tests create mode 100755 bin/cros_run_parallel_vm_tests.py diff --git a/bin/cros_run_parallel_vm_tests b/bin/cros_run_parallel_vm_tests new file mode 120000 index 0000000000..6926c9375f --- /dev/null +++ b/bin/cros_run_parallel_vm_tests @@ -0,0 +1 @@ +cros_run_parallel_vm_tests.py \ No newline at end of file diff --git a/bin/cros_run_parallel_vm_tests.py b/bin/cros_run_parallel_vm_tests.py new file mode 100755 index 0000000000..c035785b5b --- /dev/null +++ b/bin/cros_run_parallel_vm_tests.py @@ -0,0 +1,92 @@ +#!/usr/bin/python +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Runs tests on VMs in parallel.""" + +import optparse +import os +import subprocess +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), '../lib')) +from cros_build_lib import Die +from cros_build_lib import Info + + +class ParallelTestRunner(object): + """Runs tests on VMs in parallel.""" + + _DEFAULT_START_SSH_PORT = 9222 + + def __init__(self, tests): + self._tests = tests + + def _SpawnTests(self): + """Spawns VMs and starts the test runs on them. + + Runs all tests in |self._tests|. Each test is executed on a separate VM. + + Returns: A list of test process info objects containing the following + dictionary entries: + 'test': the test name; + 'proc': the Popen process instance for this test run. + """ + ssh_port = self._DEFAULT_START_SSH_PORT + spawned_tests = [] + # Test runs shouldn't need anything from stdin. However, it seems that + # running with stdin leaves the terminal in a bad state so redirect from + # /dev/null. + dev_null = open('/dev/null') + for test in self._tests: + args = [ os.path.join(os.path.dirname(__file__), 'cros_run_vm_test'), + '--snapshot', # The image is shared so don't modify it. + '--no_graphics', + '--ssh_port=%d' % ssh_port, + '--test_case=%s' % test ] + Info('Running %r...' % args) + proc = subprocess.Popen(args, stdin=dev_null) + test_info = { 'test': test, + 'proc': proc } + spawned_tests.append(test_info) + ssh_port = ssh_port + 1 + return spawned_tests + + def _WaitForCompletion(self, spawned_tests): + """Waits for tests to complete and returns a list of failed tests. + + Arguments: + spawned_tests: A list of test info objects (see _SpawnTests). + + Returns: A list of failed test names. + """ + failed_tests = [] + for test_info in spawned_tests: + proc = test_info['proc'] + proc.wait() + if proc.returncode: failed_tests.append(test_info['test']) + return failed_tests + + def Run(self): + """Runs the tests in |self._tests| on separate VMs in parallel.""" + spawned_tests = self._SpawnTests() + failed_tests = self._WaitForCompletion(spawned_tests) + if failed_tests: Die('Tests failed: %r' % failed_tests) + + +def main(): + usage = 'Usage: %prog [options] tests...' + parser = optparse.OptionParser(usage=usage) + (options, args) = parser.parse_args() + + if not args: + parser.print_help() + Die('no tests provided') + + runner = ParallelTestRunner(args) + runner.Run() + + +if __name__ == '__main__': + main() From 1f79aafb8967bd44e35cbc5340c99d61d266a068 Mon Sep 17 00:00:00 2001 From: David James Date: Wed, 3 Nov 2010 11:15:11 -0700 Subject: [PATCH 4/8] Fix broken quoting in cbuildbot.py, allowing for correct revving of ebuilds. When you pass '--foo="bar baz"' to a script, the argument is literally passed as "bar baz", with the quotes. So we need to not add unnecessary quotes in there. When enter_chroot.sh parses arguments, it actually removes the quotes for you, and splits up arguments. As a result, it may actually be necessary to (1) use the above broken syntax when you call scripts using enter_chroot.sh; but (2) don't use that syntax when calling scripts without enter_chroot.sh. That's a broken situation, so I've added a TODO for that. I've also added more warnings to cros_mark_as_stable.py so that it's easier to debug when it doesn't work. BUG=chromium-os:8633 TEST=Ran unit tests. Planning to watch TOT buildbot post-commit to make sure behavior is right. Change-Id: Ia1a0e60153fec60cb7ed4db2da128ffd9ae81e23 Review URL: http://codereview.chromium.org/4385002 --- bin/cbuildbot.py | 12 ++++++++---- cros_mark_as_stable.py | 4 +++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index 4e6f55243b..372ddcaf5f 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -194,6 +194,8 @@ def _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board): package_str = package_str.strip() cwd = os.path.join(buildroot, 'src', 'scripts') + # TODO(davidjames): --foo="bar baz" only works here because we're using + # enter_chroot. RunCommand(['./cros_mark_as_stable', '--board=%s' % board, '--tracking_branch="%s"' % tracking_branch, @@ -205,6 +207,8 @@ def _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board): def _UprevAllPackages(buildroot, tracking_branch, board): """Uprevs all packages that have been updated since last uprev.""" cwd = os.path.join(buildroot, 'src', 'scripts') + # TODO(davidjames): --foo="bar baz" only works here because we're using + # enter_chroot. RunCommand(['./cros_mark_as_stable', '--all', '--board=%s' % board, '--tracking_branch="%s"' % tracking_branch, 'commit'], @@ -230,7 +234,7 @@ def _GitCleanup(buildroot, board, tracking_branch): if os.path.exists(cwd): RunCommand(['./cros_mark_as_stable', '--srcroot=..', '--board=%s' % board, - '--tracking_branch="%s"' % tracking_branch, 'clean'], + '--tracking_branch=%s' % tracking_branch, 'clean'], cwd=cwd, error_ok=True) @@ -391,9 +395,9 @@ def _UprevPush(buildroot, tracking_branch, board, overlays): overlays = [public_overlay, private_overlay] RunCommand(['./cros_mark_as_stable', '--srcroot=..', '--board=%s' % board, - '--overlays="%s"' % " ".join(overlays), - '--tracking_branch="%s"' % tracking_branch, - '--push_options="--bypass-hooks -f"', 'push'], + '--overlays=%s' % " ".join(overlays), + '--tracking_branch=%s' % tracking_branch, + '--push_options=--bypass-hooks -f', 'push'], cwd=cwd) diff --git a/cros_mark_as_stable.py b/cros_mark_as_stable.py index fe8706c2b1..9a3ab5221c 100755 --- a/cros_mark_as_stable.py +++ b/cros_mark_as_stable.py @@ -494,7 +494,9 @@ def main(argv): _BuildEBuildDictionary(overlays, gflags.FLAGS.all, package_list) for overlay, ebuilds in overlays.items(): - if not os.path.exists(overlay): continue + if not os.path.exists(overlay): + Warning("Skipping %s" % overlay) + continue os.chdir(overlay) if command == 'clean': From 3bab0321232e7900a9e827658ca48c5797d77023 Mon Sep 17 00:00:00 2001 From: Darin Petkov Date: Wed, 3 Nov 2010 11:20:53 -0700 Subject: [PATCH 5/8] VM: Add --results_dir_root option to cros_run_parallel_vm_tests This allows the user to specify a custom results directory root. cbuildbot can be switched to use cros_run_parallel_vm_tests if necessary without any further changes. Also create parents directories for user-specified results directory roots in run_remote_tests. BUG=8585 TEST=./bin/cros_run_parallel_vm_tests suite_Smoke suite_Smoke \ --results_dir_root=/tmp/foo Change-Id: I7314c1d9e74ca139eaa4bd95290866a43a3606ff Review URL: http://codereview.chromium.org/4297003 --- bin/cros_run_parallel_vm_tests.py | 31 +++++++++++++++++++++++-------- run_remote_tests.sh | 4 ++-- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/bin/cros_run_parallel_vm_tests.py b/bin/cros_run_parallel_vm_tests.py index c035785b5b..f4d014f74c 100755 --- a/bin/cros_run_parallel_vm_tests.py +++ b/bin/cros_run_parallel_vm_tests.py @@ -20,18 +20,28 @@ class ParallelTestRunner(object): _DEFAULT_START_SSH_PORT = 9222 - def __init__(self, tests): + def __init__(self, tests, results_dir_root=None): + """Constructs and initializes the test runner class. + + Args: + tests: A list of test names (see run_remote_tests.sh). + results_dir_root: The results directory root. If provided, the results + directory root for each test will be created under it with the SSH port + appended to the test name. + """ self._tests = tests + self._results_dir_root = results_dir_root def _SpawnTests(self): """Spawns VMs and starts the test runs on them. Runs all tests in |self._tests|. Each test is executed on a separate VM. - Returns: A list of test process info objects containing the following - dictionary entries: - 'test': the test name; - 'proc': the Popen process instance for this test run. + Returns: + A list of test process info objects containing the following dictionary + entries: + 'test': the test name; + 'proc': the Popen process instance for this test run. """ ssh_port = self._DEFAULT_START_SSH_PORT spawned_tests = [] @@ -45,6 +55,9 @@ class ParallelTestRunner(object): '--no_graphics', '--ssh_port=%d' % ssh_port, '--test_case=%s' % test ] + if self._results_dir_root: + args.append('--results_dir_root=%s/%s.%d' % + (self._results_dir_root, test, ssh_port)) Info('Running %r...' % args) proc = subprocess.Popen(args, stdin=dev_null) test_info = { 'test': test, @@ -56,10 +69,11 @@ class ParallelTestRunner(object): def _WaitForCompletion(self, spawned_tests): """Waits for tests to complete and returns a list of failed tests. - Arguments: + Args: spawned_tests: A list of test info objects (see _SpawnTests). - Returns: A list of failed test names. + Returns: + A list of failed test names. """ failed_tests = [] for test_info in spawned_tests: @@ -78,13 +92,14 @@ class ParallelTestRunner(object): def main(): usage = 'Usage: %prog [options] tests...' parser = optparse.OptionParser(usage=usage) + parser.add_option('--results_dir_root', help='Root results directory.') (options, args) = parser.parse_args() if not args: parser.print_help() Die('no tests provided') - runner = ParallelTestRunner(args) + runner = ParallelTestRunner(args, options.results_dir_root) runner.Run() diff --git a/run_remote_tests.sh b/run_remote_tests.sh index 73309be5ff..864d8bb756 100755 --- a/run_remote_tests.sh +++ b/run_remote_tests.sh @@ -136,7 +136,7 @@ function main() { if [[ ${INSIDE_CHROOT} -eq 0 ]]; then if [[ -n "${FLAGS_results_dir_root}" ]]; then TMP=${FLAGS_chroot}${FLAGS_results_dir_root} - mkdir -m 777 ${TMP} + mkdir -p -m 777 ${TMP} else TMP=$(mktemp -d ${FLAGS_chroot}/tmp/run_remote_tests.XXXX) fi @@ -144,7 +144,7 @@ function main() { else if [[ -n "${FLAGS_results_dir_root}" ]]; then TMP=${FLAGS_results_dir_root} - mkdir -m 777 ${TMP} + mkdir -p -m 777 ${TMP} else TMP=$(mktemp -d /tmp/run_remote_tests.XXXX) fi From 40508146b3557607ab05a9c471b22110d0ed4e29 Mon Sep 17 00:00:00 2001 From: Darin Petkov Date: Wed, 3 Nov 2010 12:47:15 -0700 Subject: [PATCH 6/8] VM: Add board, image_path and base_ssh_port options. BUG=8585 TEST=manual, tried different options Change-Id: I66be3930cefdf4ea382e1ed8aba54a66f4bfb36e Review URL: http://codereview.chromium.org/4403001 --- bin/cros_run_parallel_vm_tests.py | 43 ++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/bin/cros_run_parallel_vm_tests.py b/bin/cros_run_parallel_vm_tests.py index f4d014f74c..1e9ef64618 100755 --- a/bin/cros_run_parallel_vm_tests.py +++ b/bin/cros_run_parallel_vm_tests.py @@ -15,21 +15,36 @@ from cros_build_lib import Die from cros_build_lib import Info +_DEFAULT_BASE_SSH_PORT = 9222 + class ParallelTestRunner(object): - """Runs tests on VMs in parallel.""" + """Runs tests on VMs in parallel. - _DEFAULT_START_SSH_PORT = 9222 + This class is a simple wrapper around cros_run_vm_test that provides an easy + way to spawn several test instances in parallel and aggregate the results when + the tests complete. + """ - def __init__(self, tests, results_dir_root=None): + def __init__(self, tests, base_ssh_port=_DEFAULT_BASE_SSH_PORT, board=None, + image_path=None, results_dir_root=None): """Constructs and initializes the test runner class. Args: tests: A list of test names (see run_remote_tests.sh). + base_ssh_port: The base SSH port. Spawned VMs listen to localhost SSH + ports incrementally allocated starting from the base one. + board: The target board. If none, cros_run_vm_tests will use the default + board. + image_path: Full path to the VM image. If none, cros_run_vm_tests will use + the latest image. results_dir_root: The results directory root. If provided, the results directory root for each test will be created under it with the SSH port appended to the test name. """ self._tests = tests + self._base_ssh_port = base_ssh_port + self._board = board + self._image_path = image_path self._results_dir_root = results_dir_root def _SpawnTests(self): @@ -43,7 +58,7 @@ class ParallelTestRunner(object): 'test': the test name; 'proc': the Popen process instance for this test run. """ - ssh_port = self._DEFAULT_START_SSH_PORT + ssh_port = self._base_ssh_port spawned_tests = [] # Test runs shouldn't need anything from stdin. However, it seems that # running with stdin leaves the terminal in a bad state so redirect from @@ -55,6 +70,8 @@ class ParallelTestRunner(object): '--no_graphics', '--ssh_port=%d' % ssh_port, '--test_case=%s' % test ] + if self._board: args.append('--board=%s' % self._board) + if self._image_path: args.append('--image_path=%s' % self._image_path) if self._results_dir_root: args.append('--results_dir_root=%s/%s.%d' % (self._results_dir_root, test, ssh_port)) @@ -92,14 +109,28 @@ class ParallelTestRunner(object): def main(): usage = 'Usage: %prog [options] tests...' parser = optparse.OptionParser(usage=usage) - parser.add_option('--results_dir_root', help='Root results directory.') + parser.add_option('--base_ssh_port', type='int', + default=_DEFAULT_BASE_SSH_PORT, + help='Base SSH port. Spawned VMs listen to localhost SSH ' + 'ports incrementally allocated starting from the base one. ' + '[default: %default]') + parser.add_option('--board', + help='The target board. If none specified, ' + 'cros_run_vm_test will use the default board.') + parser.add_option('--image_path', + help='Full path to the VM image. If none specified, ' + 'cros_run_vm_test will use the latest image.') + parser.add_option('--results_dir_root', + help='Root results directory. If none specified, each test ' + 'will store its results in a separate /tmp directory.') (options, args) = parser.parse_args() if not args: parser.print_help() Die('no tests provided') - runner = ParallelTestRunner(args, options.results_dir_root) + runner = ParallelTestRunner(args, options.base_ssh_port, options.board, + options.image_path, options.results_dir_root) runner.Run() From cef56b7dea9d5fad82b4e2a44ef4964b14d5782b Mon Sep 17 00:00:00 2001 From: Will Drewry Date: Wed, 3 Nov 2010 16:45:54 -0500 Subject: [PATCH 7/8] make_developer_script_runner.sh: fix using full-sized kernels The developer script runner did not pad for the trailing GPT header and using a full-sized kernel image extracted from a build image resulted in failure. This change accomodates for that and warns when an error occurs. TEST=used a dd'd over kernel image and it worked BUG=chromium-os:7451 Change-Id: I8efd7fbb92fe4d5a3c715580f8c21e52c21957c9 Review URL: http://codereview.chromium.org/4418001 (cherry picked from commit 5f928bef4eb551b94fcc45821eee71aa206cef86) Review URL: http://codereview.chromium.org/4417002 --- make_developer_script_runner.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/make_developer_script_runner.sh b/make_developer_script_runner.sh index a1ef2db1e5..32d3133c1a 100755 --- a/make_developer_script_runner.sh +++ b/make_developer_script_runner.sh @@ -65,7 +65,7 @@ if [ -b "$FLAGS_image" ]; then else max_kern_size=32768 dd if=/dev/zero of="${FLAGS_image}" bs=512 count=0 \ - seek=$((1 + max_kern_size + header_offset + stateful_sectors)) + seek=$((1 + max_kern_size + (2 * header_offset) + stateful_sectors)) sudo="" fi @@ -99,7 +99,7 @@ kernel_sectors=$((kernel_bytes / 512)) kernel_sectors=$(roundup $kernel_sectors) $sudo $GPT create $FLAGS_image -trap "rm $FLAGS_image" ERR +trap "rm $FLAGS_image; echo 'An error occurred! Rerun with -v for details.'" ERR offset=$header_offset $sudo $GPT add -b $offset -s $stateful_sectors \ @@ -119,4 +119,4 @@ $sudo $GPT boot -p -b "$PMBRCODE" -i 1 $FLAGS_image 1>&2 $sudo $GPT show $FLAGS_image -echo "Done." +echo "Emitted $FLAGS_image successfully!" From c0644eb90ae0137602284a65f2e88850220ed3a0 Mon Sep 17 00:00:00 2001 From: Darin Petkov Date: Wed, 3 Nov 2010 14:53:18 -0700 Subject: [PATCH 8/8] VM: Implement --order_output option in cros_run_parallel_vm_tests. If the option is specified, the utility will pipe the test stdout/stderr to separate files and dump them at the end. BUG=8585 TEST=./bin/cros_run_parallel_vm_tests suite_Smoke suite_Smoke --order_output Change-Id: I6c4fcb02456441b4e4fc3f1717cdb5d607d733b9 Review URL: http://codereview.chromium.org/4422001 --- bin/cros_run_parallel_vm_tests.py | 33 +++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/bin/cros_run_parallel_vm_tests.py b/bin/cros_run_parallel_vm_tests.py index 1e9ef64618..b6403bfe65 100755 --- a/bin/cros_run_parallel_vm_tests.py +++ b/bin/cros_run_parallel_vm_tests.py @@ -9,6 +9,7 @@ import optparse import os import subprocess import sys +import tempfile sys.path.append(os.path.join(os.path.dirname(__file__), '../lib')) from cros_build_lib import Die @@ -26,7 +27,7 @@ class ParallelTestRunner(object): """ def __init__(self, tests, base_ssh_port=_DEFAULT_BASE_SSH_PORT, board=None, - image_path=None, results_dir_root=None): + image_path=None, order_output=False, results_dir_root=None): """Constructs and initializes the test runner class. Args: @@ -37,6 +38,8 @@ class ParallelTestRunner(object): board. image_path: Full path to the VM image. If none, cros_run_vm_tests will use the latest image. + order_output: If True, output of individual VMs will be piped to + temporary files and emitted at the end. results_dir_root: The results directory root. If provided, the results directory root for each test will be created under it with the SSH port appended to the test name. @@ -45,6 +48,7 @@ class ParallelTestRunner(object): self._base_ssh_port = base_ssh_port self._board = board self._image_path = image_path + self._order_output = order_output self._results_dir_root = results_dir_root def _SpawnTests(self): @@ -76,9 +80,15 @@ class ParallelTestRunner(object): args.append('--results_dir_root=%s/%s.%d' % (self._results_dir_root, test, ssh_port)) Info('Running %r...' % args) - proc = subprocess.Popen(args, stdin=dev_null) + output = None + if self._order_output: + output = tempfile.NamedTemporaryFile(prefix='parallel_vm_test_') + Info('Piping output to %s.' % output.name) + proc = subprocess.Popen(args, stdin=dev_null, stdout=output, + stderr=output) test_info = { 'test': test, - 'proc': proc } + 'proc': proc, + 'output': output } spawned_tests.append(test_info) ssh_port = ssh_port + 1 return spawned_tests @@ -86,6 +96,8 @@ class ParallelTestRunner(object): def _WaitForCompletion(self, spawned_tests): """Waits for tests to complete and returns a list of failed tests. + If the test output was piped to a file, dumps the file contents to stdout. + Args: spawned_tests: A list of test info objects (see _SpawnTests). @@ -97,6 +109,14 @@ class ParallelTestRunner(object): proc = test_info['proc'] proc.wait() if proc.returncode: failed_tests.append(test_info['test']) + output = test_info['output'] + if output: + test = test_info['test'] + Info('------ START %s:%s ------' % (test, output.name)) + output.seek(0) + for line in output: + print line, + Info('------ END %s:%s ------' % (test, output.name)) return failed_tests def Run(self): @@ -120,6 +140,10 @@ def main(): parser.add_option('--image_path', help='Full path to the VM image. If none specified, ' 'cros_run_vm_test will use the latest image.') + parser.add_option('--order_output', action='store_true', default=False, + help='Rather than emitting interleaved progress output ' + 'from the individual VMs, accumulate the outputs in ' + 'temporary files and dump them at the end.') parser.add_option('--results_dir_root', help='Root results directory. If none specified, each test ' 'will store its results in a separate /tmp directory.') @@ -130,7 +154,8 @@ def main(): Die('no tests provided') runner = ParallelTestRunner(args, options.base_ssh_port, options.board, - options.image_path, options.results_dir_root) + options.image_path, options.order_output, + options.results_dir_root) runner.Run()