Fixes for cbuild to work with ctest.

1)  image_to_vm aborts because e2fschk requires a terminal to
be connected in order to check whether or not you want to repair the fs.  Since
we always want this to be true, set -p

2)  Clean up test harness to only verify the number of tests that passed on the
base image are the same on the update back to.  This is because older images
have many tests that fail.  I leave 10% as a reasonable number to pass.

3)  Redirect output from autotest in run_remote_tests to stderr so it gets
logged in a calling script that captures stdout.

Change-Id: If412274353683add20d136747113eb9c2bd41330

BUG=4690, 5533, 7287
TEST=Ran with internal tools for builders.

Review URL: http://codereview.chromium.org/3536018

bin/cros_au_test_harness.py

@@ -10,7 +10,12 @@ import sys
 import unittest
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '../lib'))
-from cros_build_lib import RunCommand, Info, Warning, ReinterpretPathForChroot
+from cros_build_lib import Die
+from cros_build_lib import Info
+from cros_build_lib import ReinterpretPathForChroot
+from cros_build_lib import RunCommand
+from cros_build_lib import Warning
+
 
 _KVM_PID_FILE = '/tmp/harness_pid'
 _FULL_VDISK_SIZE = 6072
@@ -21,6 +26,8 @@ global base_image_path
 global board
 global remote
 global target_image_path
+global vm_graphics_flag
+
 
 _VERIFY_SUITE = 'suite_Smoke'
 
@@ -41,6 +48,20 @@ class AUTest(object):
 
     return stateful_change_flag
 
+  def ParseGenerateTestReportOutput(self, output):
+    """Returns the percentage of tests that passed based on output."""
+    percent_passed = 0
+    lines = output.split('\n')
+
+    for line in lines:
+      if line.startswith("Total PASS:"):
+        # FORMAT: ^TOTAL PASS: num_passed/num_total (percent%)$
+        percent_passed = line.split()[3].strip('()%')
+        Info('Percent of tests passed %s' % percent_passed)
+        break
+
+    return int(percent_passed)
+
   def PrepareBase(self):
     """Prepares target with base_image_path."""
     pass
@@ -58,51 +79,90 @@ class AUTest(object):
     """
     pass
 
-  def VerifyImage(self):
-    """Verifies the image is correct."""
+  def VerifyImage(self, percent_required_to_pass):
+    """Verifies the image with tests.
+
+    Verifies that the test images passes the percent required.
+
+    Args:
+      percent_required_to_pass:  percentage required to pass.  This should be
+        fall between 0-100.
+
+    Returns:
+      Returns the percent that passed.
+    """
     pass
 
+  def CommonVerifyImage(self, unittest, output, percent_required_to_pass):
+    """Helper function for VerifyImage that returns percent of tests passed.
+
+    Takes output from a test suite, verifies the number of tests passed is
+    sufficient and outputs info.
+
+    Args:
+      unittest: Handle to the unittest.
+      output: stdout from a test run.
+      percent_required_to_pass: percentage required to pass.  This should be
+        fall between 0-100.
+    Returns:
+      percent that passed.
+    """
+    Info('Output from VerifyImage():')
+    print output
+    percent_passed = self.ParseGenerateTestReportOutput(output)
+    Info('Percent passed: %d vs. Percent required: %d' % (
+        percent_passed, percent_required_to_pass))
+    unittest.assertTrue(percent_passed >=
+                        percent_required_to_pass)
+    return percent_passed
+
   def testFullUpdateKeepStateful(self):
     """Tests if we can update normally.
 
     This test checks that we can update by updating the stateful partition
     rather than wiping it.
     """
-    # Prepare and verify the base image has been prepared correctly.
+    # Just make sure some tests pass on original image.  Some old images
+    # don't pass many tests.
     self.PrepareBase()
-    self.VerifyImage()
+    # TODO(sosa): move to 100% once we start testing using the autotest paired
+    # with the dev channel.
+    percent_passed = self.VerifyImage(42)
 
-    # Update to.
+    # Update to - all tests should pass on new image.
     Info('Updating from base image on vm to target image.')
     self.UpdateImage(target_image_path)
-    self.VerifyImage()
+    self.VerifyImage(100)
 
-    # Update from.
+    # Update from - same percentage should pass that originally passed.
     Info('Updating from updated image on vm back to base image.')
     self.UpdateImage(base_image_path)
-    self.VerifyImage()
+    self.VerifyImage(percent_passed)
 
   # TODO(sosa): Re-enable once we have a good way of checking for version
-  # compatability.
+  # compatibility.
   def NotestFullUpdateWipeStateful(self):
     """Tests if we can update after cleaning the stateful partition.
 
     This test checks that we can update successfully after wiping the
     stateful partition.
     """
-    # Prepare and verify the base image has been prepared correctly.
+    # Just make sure some tests pass on original image.  Some old images
+    # don't pass many tests.
     self.PrepareBase()
-    self.VerifyImage()
+    # TODO(sosa): move to 100% once we start testing using the autotest paired
+    # with the dev channel.
+    percent_passed = self.VerifyImage(42)
 
-    # Update to.
+    # Update to - all tests should pass on new image.
     Info('Updating from base image on vm to target image and wiping stateful.')
     self.UpdateImage(target_image_path, 'clean')
-    self.VerifyImage()
+    self.VerifyImage(100)
 
-    # Update from.
+    # Update from - same percentage should pass that originally passed.
     Info('Updating from updated image back to base image and wiping stateful.')
     self.UpdateImage(base_image_path, 'clean')
-    self.VerifyImage()
+    self.VerifyImage(percent_passed)
 
 
 class RealAUTest(unittest.TestCase, AUTest):
@@ -111,6 +171,10 @@ class RealAUTest(unittest.TestCase, AUTest):
   def setUp(self):
     AUTest.setUp(self)
 
+  def PrepareBase(self):
+    """Auto-update to base image to prepare for test."""
+    self.UpdateImage(base_image_path)
+
   def UpdateImage(self, image_path, stateful_change='old'):
     """Updates a remote image using image_to_live.sh."""
     stateful_change_flag = self.GetStatefulChangeFlag(stateful_change)
@@ -124,13 +188,14 @@ class RealAUTest(unittest.TestCase, AUTest):
         ], enter_chroot=False)
 
 
-  def VerifyImage(self):
+  def VerifyImage(self, percent_required_to_pass):
     """Verifies an image using run_remote_tests.sh with verification suite."""
-    RunCommand([
+    output = RunCommand([
         '%s/run_remote_tests.sh' % self.crosutils,
         '--remote=%s' % remote,
         _VERIFY_SUITE,
-       ], error_ok=False, enter_chroot=False)
+       ], error_ok=True, enter_chroot=False, redirect_stdout=True)
+    return self.CommonVerifyImage(self, output, percent_required_to_pass)
 
 
 class VirtualAUTest(unittest.TestCase, AUTest):
@@ -181,22 +246,26 @@ class VirtualAUTest(unittest.TestCase, AUTest):
                 '--update_image_path=%s' % image_path,
                 '--vm_image_path=%s' % self.vm_image_path,
                 '--snapshot',
+                vm_graphics_flag,
                 '--persist',
                 '--kvm_pid=%s' % _KVM_PID_FILE,
                 stateful_change_flag,
                ], enter_chroot=False)
 
-  def VerifyImage(self):
+  def VerifyImage(self, percent_required_to_pass):
     """Runs vm smoke suite to verify image."""
     # image_to_live already verifies lsb-release matching.  This is just
     # for additional steps.
-    RunCommand(['%s/cros_run_vm_test' % self.crosutilsbin,
-                '--image_path=%s' % self.vm_image_path,
-                '--snapshot',
-                '--persist',
-                '--kvm_pid=%s' % _KVM_PID_FILE,
-                '--test_case=%s' % _VERIFY_SUITE,
-               ], error_ok=False, enter_chroot=False)
+    output = RunCommand(['%s/cros_run_vm_test' % self.crosutilsbin,
+                         '--image_path=%s' % self.vm_image_path,
+                         '--snapshot',
+                         '--persist',
+                         vm_graphics_flag,
+                         '--kvm_pid=%s' % _KVM_PID_FILE,
+                         '--test_case=%s' % _VERIFY_SUITE,
+                         ], error_ok=True, enter_chroot=False,
+                            redirect_stdout=True)
+    return self.CommonVerifyImage(self, output, percent_required_to_pass)
 
 
 if __name__ == '__main__':
@@ -211,6 +280,8 @@ if __name__ == '__main__':
                     help='type of test to run: [vm, real]. Default: vm.')
   parser.add_option('-m', '--remote',
                     help='Remote address for real test.')
+  parser.add_option('--no_graphics', action='store_true',
+                    help='Disable graphics for the vm test.')
   # Set the usage to include flags.
   parser.set_usage(parser.format_help())
   # Parse existing sys.argv so we can pass rest to unittest.main.
@@ -222,19 +293,24 @@ if __name__ == '__main__':
 
   if not base_image_path:
     parser.error('Need path to base image for vm.')
+  elif not os.path.exists(base_image_path):
+    Die('%s does not exist' % base_image_path)
 
   if not target_image_path:
     parser.error('Need path to target image to update with.')
+  elif not os.path.exists(target_image_path):
+    Die('%s does not exist' % target_image_path)
 
   if not board:
     parser.error('Need board to convert base image to vm.')
 
-  return_code = 0
+  vm_graphics_flag = ''
+  if options.no_graphics: vm_graphics_flag = '--no_graphics'
 
   # Only run the test harness we care about.
   if options.type == 'vm':
     suite = unittest.TestLoader().loadTestsFromTestCase(VirtualAUTest)
-    return_code = unittest.TextTestRunner(verbosity=2).run(suite)
+    test_result = unittest.TextTestRunner(verbosity=2).run(suite)
   elif options.type == 'real':
     if not options.remote:
       parser.error('Real tests require a remote test machine.')
@@ -242,8 +318,9 @@ if __name__ == '__main__':
       remote = options.remote
 
     suite = unittest.TestLoader().loadTestsFromTestCase(RealAUTest)
-    return_code = unittest.TextTestRunner(verbosity=2).run(suite)
+    test_result = unittest.TextTestRunner(verbosity=2).run(suite)
   else:
     parser.error('Could not parse harness type %s.' % options.type)
 
-  sys.exit(return_code)
+  if not test_result.wasSuccessful():
+    Die('Test harness was not successful')

bin/ctest.py

@@ -127,7 +127,8 @@ def GrabZipAndExtractImage(zip_url, download_folder, image_name) :
     fh.close()
 
 
-def RunAUTestHarness(board, channel, latest_url_base, zip_server_base):
+def RunAUTestHarness(board, channel, latest_url_base, zip_server_base,
+                     no_graphics, type, remote):
   """Runs the auto update test harness.
 
   The auto update test harness encapsulates testing the auto-update mechanism
@@ -140,22 +141,33 @@ def RunAUTestHarness(board, channel, latest_url_base, zip_server_base):
     channel: the channel to run the au test harness against.
     latest_url_base: base url for getting latest links.
     zip_server_base:  base url for zipped images.
+    no_graphics: boolean - If True, disable graphics during vm test.
+    type: which test harness to run.  Possible values: real, vm.
+    remote: ip address for real test harness run.
   """
   crosutils_root = os.path.join(os.path.dirname(__file__), '..')
   download_folder = os.path.abspath('latest_download')
   zip_url = GetLatestZipUrl(board, channel, latest_url_base, zip_server_base)
   GrabZipAndExtractImage(zip_url, download_folder, _IMAGE_TO_EXTRACT)
 
+  no_graphics_flag = ''
+  if no_graphics: no_graphics_flag = '--no_graphics'
+
   # Tests go here.
   latest_image = RunCommand(['./get_latest_image.sh', '--board=%s' % board],
                             cwd=crosutils_root, redirect_stdout=True,
-                            print_cmd=True)
+                            print_cmd=True).strip()
 
   RunCommand(['bin/cros_au_test_harness',
               '--base_image=%s' % os.path.join(download_folder,
                                                  _IMAGE_TO_EXTRACT),
-              '--target_image=%s' % latest_image,
-              '--board=%s' % board], cwd=crosutils_root)
+              '--target_image=%s' % os.path.join(latest_image,
+                                                 _IMAGE_TO_EXTRACT),
+              no_graphics_flag,
+              '--board=%s' % board,
+              '--type=%s' % type,
+              '--remote=%s' % remote,
+             ], cwd=crosutils_root)
 
 
 def main():
@@ -168,6 +180,13 @@ def main():
                     help='Base url for latest links.')
   parser.add_option('-z', '--zipbase',
                     help='Base url for hosted images.')
+  parser.add_option('--no_graphics', action='store_true', default=False,
+                    help='Disable graphics for the vm test.')
+  parser.add_option('--type', default='vm',
+                    help='type of test to run: [vm, real]. Default: vm.')
+  parser.add_option('--remote', default='0.0.0.0',
+                    help='For real tests, ip address of the target machine.')
+
   # Set the usage to include flags.
   parser.set_usage(parser.format_help())
   (options, args) = parser.parse_args()
@@ -188,7 +207,8 @@ def main():
     parser.error('Need zip url base to get images.')
 
   RunAUTestHarness(options.board, options.channel, options.latestbase,
-                   options.zipbase)
+                   options.zipbase, options.no_graphics, options.type,
+                   options.remote)
 
 
 if __name__ == '__main__':

image_to_live.sh

@@ -358,6 +358,8 @@ function main() {
     info "Update was successful and rebooted to $release_description"
   fi
 
+  print_time_elapsed
+
   exit 0
 }
 

image_to_vm.sh

@@ -163,7 +163,7 @@ else
         seek=$((STATEFUL_SIZE_BYTES - 1))
     # Resize the partition.
     sudo losetup "${STATEFUL_LOOP_DEV}" "${TEMP_STATE}"
-    sudo e2fsck -f "${STATEFUL_LOOP_DEV}"
+    sudo e2fsck -pf "${STATEFUL_LOOP_DEV}"
     sudo resize2fs "${STATEFUL_LOOP_DEV}"
     sudo losetup -d "${STATEFUL_LOOP_DEV}"
   fi

run_remote_tests.sh

@@ -275,12 +275,14 @@ function main() {
     ${enter_chroot} ${autotest} --board "${FLAGS_board}" -m "${FLAGS_remote}" \
       --ssh-port ${FLAGS_ssh_port} \
       "${option}" "${control_file}" -r "${results_dir}" ${verbose} \
-      "${passthrough_args}"
+      "${passthrough_args}" >&2
   done
 
   echo ""
   echo_color "yellow" ">>> Test results:"
   ./generate_test_report "${TMP}" --strip="${TMP}/"
+
+  print_time_elapsed
 }
 
 main "$@"