diff --git a/bin/cros_au_test_harness.py b/bin/cros_au_test_harness.py index a495f06f9b..799a5240c8 100755 --- a/bin/cros_au_test_harness.py +++ b/bin/cros_au_test_harness.py @@ -10,7 +10,12 @@ import sys import unittest sys.path.append(os.path.join(os.path.dirname(__file__), '../lib')) -from cros_build_lib import RunCommand, Info, Warning, ReinterpretPathForChroot +from cros_build_lib import Die +from cros_build_lib import Info +from cros_build_lib import ReinterpretPathForChroot +from cros_build_lib import RunCommand +from cros_build_lib import Warning + _KVM_PID_FILE = '/tmp/harness_pid' _FULL_VDISK_SIZE = 6072 @@ -21,6 +26,8 @@ global base_image_path global board global remote global target_image_path +global vm_graphics_flag + _VERIFY_SUITE = 'suite_Smoke' @@ -41,6 +48,20 @@ class AUTest(object): return stateful_change_flag + def ParseGenerateTestReportOutput(self, output): + """Returns the percentage of tests that passed based on output.""" + percent_passed = 0 + lines = output.split('\n') + + for line in lines: + if line.startswith("Total PASS:"): + # FORMAT: ^TOTAL PASS: num_passed/num_total (percent%)$ + percent_passed = line.split()[3].strip('()%') + Info('Percent of tests passed %s' % percent_passed) + break + + return int(percent_passed) + def PrepareBase(self): """Prepares target with base_image_path.""" pass @@ -58,51 +79,90 @@ class AUTest(object): """ pass - def VerifyImage(self): - """Verifies the image is correct.""" + def VerifyImage(self, percent_required_to_pass): + """Verifies the image with tests. + + Verifies that the test images passes the percent required. + + Args: + percent_required_to_pass: percentage required to pass. This should be + fall between 0-100. + + Returns: + Returns the percent that passed. + """ pass + def CommonVerifyImage(self, unittest, output, percent_required_to_pass): + """Helper function for VerifyImage that returns percent of tests passed. + + Takes output from a test suite, verifies the number of tests passed is + sufficient and outputs info. + + Args: + unittest: Handle to the unittest. + output: stdout from a test run. + percent_required_to_pass: percentage required to pass. This should be + fall between 0-100. + Returns: + percent that passed. + """ + Info('Output from VerifyImage():') + print output + percent_passed = self.ParseGenerateTestReportOutput(output) + Info('Percent passed: %d vs. Percent required: %d' % ( + percent_passed, percent_required_to_pass)) + unittest.assertTrue(percent_passed >= + percent_required_to_pass) + return percent_passed + def testFullUpdateKeepStateful(self): """Tests if we can update normally. This test checks that we can update by updating the stateful partition rather than wiping it. """ - # Prepare and verify the base image has been prepared correctly. + # Just make sure some tests pass on original image. Some old images + # don't pass many tests. self.PrepareBase() - self.VerifyImage() + # TODO(sosa): move to 100% once we start testing using the autotest paired + # with the dev channel. + percent_passed = self.VerifyImage(42) - # Update to. + # Update to - all tests should pass on new image. Info('Updating from base image on vm to target image.') self.UpdateImage(target_image_path) - self.VerifyImage() + self.VerifyImage(100) - # Update from. + # Update from - same percentage should pass that originally passed. Info('Updating from updated image on vm back to base image.') self.UpdateImage(base_image_path) - self.VerifyImage() + self.VerifyImage(percent_passed) # TODO(sosa): Re-enable once we have a good way of checking for version - # compatability. + # compatibility. def NotestFullUpdateWipeStateful(self): """Tests if we can update after cleaning the stateful partition. This test checks that we can update successfully after wiping the stateful partition. """ - # Prepare and verify the base image has been prepared correctly. + # Just make sure some tests pass on original image. Some old images + # don't pass many tests. self.PrepareBase() - self.VerifyImage() + # TODO(sosa): move to 100% once we start testing using the autotest paired + # with the dev channel. + percent_passed = self.VerifyImage(42) - # Update to. + # Update to - all tests should pass on new image. Info('Updating from base image on vm to target image and wiping stateful.') self.UpdateImage(target_image_path, 'clean') - self.VerifyImage() + self.VerifyImage(100) - # Update from. + # Update from - same percentage should pass that originally passed. Info('Updating from updated image back to base image and wiping stateful.') self.UpdateImage(base_image_path, 'clean') - self.VerifyImage() + self.VerifyImage(percent_passed) class RealAUTest(unittest.TestCase, AUTest): @@ -111,6 +171,10 @@ class RealAUTest(unittest.TestCase, AUTest): def setUp(self): AUTest.setUp(self) + def PrepareBase(self): + """Auto-update to base image to prepare for test.""" + self.UpdateImage(base_image_path) + def UpdateImage(self, image_path, stateful_change='old'): """Updates a remote image using image_to_live.sh.""" stateful_change_flag = self.GetStatefulChangeFlag(stateful_change) @@ -124,13 +188,14 @@ class RealAUTest(unittest.TestCase, AUTest): ], enter_chroot=False) - def VerifyImage(self): + def VerifyImage(self, percent_required_to_pass): """Verifies an image using run_remote_tests.sh with verification suite.""" - RunCommand([ + output = RunCommand([ '%s/run_remote_tests.sh' % self.crosutils, '--remote=%s' % remote, _VERIFY_SUITE, - ], error_ok=False, enter_chroot=False) + ], error_ok=True, enter_chroot=False, redirect_stdout=True) + return self.CommonVerifyImage(self, output, percent_required_to_pass) class VirtualAUTest(unittest.TestCase, AUTest): @@ -181,22 +246,26 @@ class VirtualAUTest(unittest.TestCase, AUTest): '--update_image_path=%s' % image_path, '--vm_image_path=%s' % self.vm_image_path, '--snapshot', + vm_graphics_flag, '--persist', '--kvm_pid=%s' % _KVM_PID_FILE, stateful_change_flag, ], enter_chroot=False) - def VerifyImage(self): + def VerifyImage(self, percent_required_to_pass): """Runs vm smoke suite to verify image.""" # image_to_live already verifies lsb-release matching. This is just # for additional steps. - RunCommand(['%s/cros_run_vm_test' % self.crosutilsbin, - '--image_path=%s' % self.vm_image_path, - '--snapshot', - '--persist', - '--kvm_pid=%s' % _KVM_PID_FILE, - '--test_case=%s' % _VERIFY_SUITE, - ], error_ok=False, enter_chroot=False) + output = RunCommand(['%s/cros_run_vm_test' % self.crosutilsbin, + '--image_path=%s' % self.vm_image_path, + '--snapshot', + '--persist', + vm_graphics_flag, + '--kvm_pid=%s' % _KVM_PID_FILE, + '--test_case=%s' % _VERIFY_SUITE, + ], error_ok=True, enter_chroot=False, + redirect_stdout=True) + return self.CommonVerifyImage(self, output, percent_required_to_pass) if __name__ == '__main__': @@ -211,6 +280,8 @@ if __name__ == '__main__': help='type of test to run: [vm, real]. Default: vm.') parser.add_option('-m', '--remote', help='Remote address for real test.') + parser.add_option('--no_graphics', action='store_true', + help='Disable graphics for the vm test.') # Set the usage to include flags. parser.set_usage(parser.format_help()) # Parse existing sys.argv so we can pass rest to unittest.main. @@ -222,19 +293,24 @@ if __name__ == '__main__': if not base_image_path: parser.error('Need path to base image for vm.') + elif not os.path.exists(base_image_path): + Die('%s does not exist' % base_image_path) if not target_image_path: parser.error('Need path to target image to update with.') + elif not os.path.exists(target_image_path): + Die('%s does not exist' % target_image_path) if not board: parser.error('Need board to convert base image to vm.') - return_code = 0 + vm_graphics_flag = '' + if options.no_graphics: vm_graphics_flag = '--no_graphics' # Only run the test harness we care about. if options.type == 'vm': suite = unittest.TestLoader().loadTestsFromTestCase(VirtualAUTest) - return_code = unittest.TextTestRunner(verbosity=2).run(suite) + test_result = unittest.TextTestRunner(verbosity=2).run(suite) elif options.type == 'real': if not options.remote: parser.error('Real tests require a remote test machine.') @@ -242,8 +318,9 @@ if __name__ == '__main__': remote = options.remote suite = unittest.TestLoader().loadTestsFromTestCase(RealAUTest) - return_code = unittest.TextTestRunner(verbosity=2).run(suite) + test_result = unittest.TextTestRunner(verbosity=2).run(suite) else: parser.error('Could not parse harness type %s.' % options.type) - sys.exit(return_code) + if not test_result.wasSuccessful(): + Die('Test harness was not successful') diff --git a/bin/ctest.py b/bin/ctest.py index 78210b6308..418dfaba78 100755 --- a/bin/ctest.py +++ b/bin/ctest.py @@ -127,7 +127,8 @@ def GrabZipAndExtractImage(zip_url, download_folder, image_name) : fh.close() -def RunAUTestHarness(board, channel, latest_url_base, zip_server_base): +def RunAUTestHarness(board, channel, latest_url_base, zip_server_base, + no_graphics, type, remote): """Runs the auto update test harness. The auto update test harness encapsulates testing the auto-update mechanism @@ -140,22 +141,33 @@ def RunAUTestHarness(board, channel, latest_url_base, zip_server_base): channel: the channel to run the au test harness against. latest_url_base: base url for getting latest links. zip_server_base: base url for zipped images. + no_graphics: boolean - If True, disable graphics during vm test. + type: which test harness to run. Possible values: real, vm. + remote: ip address for real test harness run. """ crosutils_root = os.path.join(os.path.dirname(__file__), '..') download_folder = os.path.abspath('latest_download') zip_url = GetLatestZipUrl(board, channel, latest_url_base, zip_server_base) GrabZipAndExtractImage(zip_url, download_folder, _IMAGE_TO_EXTRACT) + no_graphics_flag = '' + if no_graphics: no_graphics_flag = '--no_graphics' + # Tests go here. latest_image = RunCommand(['./get_latest_image.sh', '--board=%s' % board], cwd=crosutils_root, redirect_stdout=True, - print_cmd=True) + print_cmd=True).strip() RunCommand(['bin/cros_au_test_harness', '--base_image=%s' % os.path.join(download_folder, _IMAGE_TO_EXTRACT), - '--target_image=%s' % latest_image, - '--board=%s' % board], cwd=crosutils_root) + '--target_image=%s' % os.path.join(latest_image, + _IMAGE_TO_EXTRACT), + no_graphics_flag, + '--board=%s' % board, + '--type=%s' % type, + '--remote=%s' % remote, + ], cwd=crosutils_root) def main(): @@ -168,6 +180,13 @@ def main(): help='Base url for latest links.') parser.add_option('-z', '--zipbase', help='Base url for hosted images.') + parser.add_option('--no_graphics', action='store_true', default=False, + help='Disable graphics for the vm test.') + parser.add_option('--type', default='vm', + help='type of test to run: [vm, real]. Default: vm.') + parser.add_option('--remote', default='0.0.0.0', + help='For real tests, ip address of the target machine.') + # Set the usage to include flags. parser.set_usage(parser.format_help()) (options, args) = parser.parse_args() @@ -188,7 +207,8 @@ def main(): parser.error('Need zip url base to get images.') RunAUTestHarness(options.board, options.channel, options.latestbase, - options.zipbase) + options.zipbase, options.no_graphics, options.type, + options.remote) if __name__ == '__main__': diff --git a/image_to_live.sh b/image_to_live.sh index fa566a6ac8..76942912f2 100755 --- a/image_to_live.sh +++ b/image_to_live.sh @@ -358,6 +358,8 @@ function main() { info "Update was successful and rebooted to $release_description" fi + print_time_elapsed + exit 0 } diff --git a/image_to_vm.sh b/image_to_vm.sh index f0c6e654fa..d6ff61fa8b 100755 --- a/image_to_vm.sh +++ b/image_to_vm.sh @@ -163,7 +163,7 @@ else seek=$((STATEFUL_SIZE_BYTES - 1)) # Resize the partition. sudo losetup "${STATEFUL_LOOP_DEV}" "${TEMP_STATE}" - sudo e2fsck -f "${STATEFUL_LOOP_DEV}" + sudo e2fsck -pf "${STATEFUL_LOOP_DEV}" sudo resize2fs "${STATEFUL_LOOP_DEV}" sudo losetup -d "${STATEFUL_LOOP_DEV}" fi diff --git a/run_remote_tests.sh b/run_remote_tests.sh index 25282088cc..36e028d2af 100755 --- a/run_remote_tests.sh +++ b/run_remote_tests.sh @@ -275,12 +275,14 @@ function main() { ${enter_chroot} ${autotest} --board "${FLAGS_board}" -m "${FLAGS_remote}" \ --ssh-port ${FLAGS_ssh_port} \ "${option}" "${control_file}" -r "${results_dir}" ${verbose} \ - "${passthrough_args}" + "${passthrough_args}" >&2 done echo "" echo_color "yellow" ">>> Test results:" ./generate_test_report "${TMP}" --strip="${TMP}/" + + print_time_elapsed } main "$@"