diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index ff3fb8a94c..168d6212df 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -232,10 +232,12 @@ def _MarkChromeAsStable(buildroot, tracking_branch, chrome_rev, board): return None else: chrome_atom = portage_atom_string.split('=')[1] + keywords_file = CHROME_KEYWORDS_FILE % {'board': board} # TODO(sosa): Workaround to build unstable chrome ebuild we uprevved. - RunCommand(['sudo', 'tee', CHROME_KEYWORDS_FILE % {'board': board}], - input='=%s\n' % chrome_atom, enter_chroot=True, - cwd=cwd) + RunCommand(['sudo', 'mkdir', '-p', os.path.dirname(keywords_file)], + enter_chroot=True, cwd=cwd) + RunCommand(['sudo', 'tee', keywords_file], input='=%s\n' % chrome_atom, + enter_chroot=True, cwd=cwd) return chrome_atom @@ -415,6 +417,22 @@ def _RunSmokeSuite(buildroot, results_dir): ], cwd=cwd, error_ok=False) +def _RunAUTest(buildroot, board): + """Runs a basic update test from the au test harness.""" + cwd = os.path.join(buildroot, 'src', 'scripts') + image_path = os.path.join(buildroot, 'src', 'build', 'images', board, + 'latest', 'chromiumos_test_image.bin') + RunCommand(['bin/cros_au_test_harness', + '--no_graphics', + '--no_delta', + '--board=%s' % board, + '--test_prefix=SimpleTest', + '--verbose', + '--base_image=%s' % image_path, + '--target_image=%s' % image_path, + ], cwd=cwd, error_ok=False) + + def _UprevPackages(buildroot, tracking_branch, revisionfile, board, overlays): """Uprevs a package based on given revisionfile. @@ -695,15 +713,16 @@ def main(): _BuildImage(buildroot) - if buildconfig['smoke_bvt'] and options.tests: + if buildconfig['tests'] and options.tests: _BuildVMImageForTesting(buildroot) test_results_dir = '/tmp/run_remote_tests.%s' % options.buildnumber try: _RunSmokeSuite(buildroot, test_results_dir) + _RunAUTest(buildroot, buildconfig['board']) finally: if not options.debug: archive_full_path = os.path.join(options.gsutil_archive, - str(options.buildnumber)) + str(options.buildnumber)) _ArchiveTestResults(buildroot, buildconfig['board'], test_results_dir=test_results_dir, gsutil=options.gsutil, diff --git a/bin/cbuildbot_config.py b/bin/cbuildbot_config.py index 2f7b6361dd..c13d3626cd 100644 --- a/bin/cbuildbot_config.py +++ b/bin/cbuildbot_config.py @@ -18,7 +18,7 @@ important -- Master bot uses important bots to determine overall status. hostname -- Needed for 'important' slaves. The hostname of the bot. Should match hostname in slaves.cfg in buildbot checkout. unittests -- Runs unittests for packages. -smoke_bvt -- Runs the test smoke suite in a qemu-based VM using KVM. +tests -- Runs the smoke suite and au test harness in a qemu-based VM using KVM. rev_overlays -- Select what overlays to look at for revving. This can be 'public', 'private' or 'both'. push_overlays -- Select what overlays to push at. This should be a subset of @@ -35,7 +35,7 @@ config['default'] = { 'master' : False, 'important' : False, 'unittests' : False, - 'smoke_bvt' : False, + 'tests' : False, 'rev_overlays': 'public', 'push_overlays': None, } @@ -46,7 +46,7 @@ config['x86-generic-pre-flight-queue'] = { 'important' : False, 'hostname' : 'chromeosbuild2', 'unittests' : True, - 'smoke_bvt' : True, + 'tests' : True, 'rev_overlays': 'public', 'push_overlays': 'public', } @@ -56,7 +56,7 @@ config['x86-mario-pre-flight-queue'] = { 'master' : True, 'important' : False, 'unittests' : True, - 'smoke_bvt' : True, + 'tests' : True, 'rev_overlays': 'both', 'push_overlays': 'private', } @@ -66,7 +66,7 @@ config['x86-mario-pre-flight-branch'] = { 'master' : True, 'important' : False, 'unittests' : True, - 'smoke_bvt' : True, + 'tests' : True, 'rev_overlays': 'both', 'push_overlays': 'both', } @@ -76,7 +76,7 @@ config['x86_agz_bin'] = { 'master' : False, 'important' : False, 'unittests' : True, - 'smoke_bvt' : True, + 'tests' : True, 'rev_overlays': 'both', 'push_overlays': None, } @@ -86,7 +86,7 @@ config['x86_dogfood_bin'] = { 'master' : False, 'important' : False, 'unittests' : True, - 'smoke_bvt' : True, + 'tests' : True, 'rev_overlays': 'both', 'push_overlays': None, } diff --git a/bin/cros_au_test_harness.py b/bin/cros_au_test_harness.py index 5a0182c481..d0bbc6c25c 100755 --- a/bin/cros_au_test_harness.py +++ b/bin/cros_au_test_harness.py @@ -4,6 +4,14 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +"""This module runs a suite of Auto Update tests. + + The tests can be run on either a virtual machine or actual device depending + on parameters given. Specific tests can be run by invoking --test_prefix. + Verbose is useful for many of the tests if you want to see individual commands + being run during the update process. +""" + import optparse import os import re @@ -148,7 +156,8 @@ class AUTest(object): # This update is expected to fail... try: - self.PerformUpdate(self.target_image_path, proxy_port=proxy_port) + self.PerformUpdate(self.target_image_path, self.base_image_path, + proxy_port=proxy_port) finally: proxy.shutdown() @@ -226,7 +235,7 @@ class AUTest(object): # -------- Tests --------- - def testFullUpdateKeepStateful(self): + def testUpdateKeepStateful(self): """Tests if we can update normally. This test checks that we can update by updating the stateful partition @@ -241,15 +250,15 @@ class AUTest(object): # Update to - all tests should pass on new image. Info('Updating from base image on vm to target image.') - self.PerformUpdate(self.base_image_path, self.target_image_path) + self.PerformUpdate(self.target_image_path, self.base_image_path) self.VerifyImage(100) # Update from - same percentage should pass that originally passed. Info('Updating from updated image on vm back to base image.') - self.PerformUpdate(self.target_image_path, self.base_image_path) + self.PerformUpdate(self.base_image_path, self.target_image_path) self.VerifyImage(percent_passed) - def testFullUpdateWipeStateful(self): + def testUpdateWipeStateful(self): """Tests if we can update after cleaning the stateful partition. This test checks that we can update successfully after wiping the @@ -264,12 +273,12 @@ class AUTest(object): # Update to - all tests should pass on new image. Info('Updating from base image on vm to target image and wiping stateful.') - self.PerformUpdate(self.base_image_path, self.target_image_path, 'clean') + self.PerformUpdate(self.target_image_path, self.base_image_path, 'clean') self.VerifyImage(100) # Update from - same percentage should pass that originally passed. Info('Updating from updated image back to base image and wiping stateful.') - self.PerformUpdate(self.target_image_path, self.base_image_path, 'clean') + self.PerformUpdate(self.base_image_path, self.target_image_path, 'clean') self.VerifyImage(percent_passed) def testPartialUpdate(self): @@ -372,13 +381,13 @@ class AUTest(object): self.AttemptUpdateWithFilter(DelayedFilter()) def SimpleTest(self): - """A simple update that updates the target image to itself. + """A simple update that updates once from a base image to a target. We explicitly don't use test prefix so that isn't run by default. Can be run using test_prefix option. """ - self.PrepareBase(self.target_image_path) - self._UpdateImage(self.target_image_path) + self.PrepareBase(self.base_image_path) + self.PerformUpdate(self.target_image_path, self.base_image_path) self.VerifyImage(100) @@ -552,7 +561,7 @@ class VirtualAUTest(unittest.TestCase, AUTest): def _UpdateUsingPayload(self, update_path, stateful_change='old', proxy_port=None): - """Updates a remote image using image_to_live.sh.""" + """Updates a vm image using cros_run_vm_update.""" stateful_change_flag = self.GetStatefulChangeFlag(stateful_change) cmd = ['%s/cros_run_vm_update' % self.crosutilsbin, '--payload=%s' % update_path, diff --git a/bin/cros_mark_chrome_as_stable.py b/bin/cros_mark_chrome_as_stable.py index dc006a7b92..0b17218180 100755 --- a/bin/cros_mark_chrome_as_stable.py +++ b/bin/cros_mark_chrome_as_stable.py @@ -60,7 +60,9 @@ def _GetTipOfTrunkSvnRevision(): for line in svn_info.splitlines(): match = revision_re.search(line) if match: - return match.group(1) + svn_revision = match.group(1) + Info('Using SVN Revision %s' % svn_revision) + return svn_revision raise Exception('Could not find revision information from %s' % svn_url) diff --git a/build_image b/build_image index 61b28215b8..9674e36ab0 100755 --- a/build_image +++ b/build_image @@ -71,9 +71,9 @@ DEFINE_string usb_disk /dev/sdb3 \ DEFINE_boolean enable_rootfs_verification ${FLAGS_TRUE} \ "Default all bootloaders to use kernel-based root fs integrity checking." -DEFINE_integer verity_error_behavior 1 \ +DEFINE_integer verity_error_behavior 3 \ "Kernel verified boot error behavior (0: I/O errors, 1: panic, 2: nothing, \ -3: cros) Default: 1" +3: cros) Default: 3" DEFINE_integer verity_depth 1 \ "Kernel verified boot hash tree depth. Default: 1" DEFINE_integer verity_max_ios -1 \ diff --git a/image_to_live.sh b/image_to_live.sh index 553cd4d21d..bbb5ec904d 100755 --- a/image_to_live.sh +++ b/image_to_live.sh @@ -241,7 +241,6 @@ function get_update_log { echo "${REMOTE_OUT}" > "${FLAGS_update_log}" } - # Returns ${1} reported by the update client e.g. PROGRESS, CURRENT_OP. function get_update_var { remote_sh "${UPDATER_BIN} --status 2> /dev/null | @@ -254,21 +253,30 @@ function get_update_var { # This is expected to run in its own thread. function status_thread { local timeout=5 - # Let update engine receive call to ping the dev server. + info "Devserver handling ping. Check ${FLAGS_server_log} for more info." sleep ${timeout} - # The devserver generates images when the update engine checks for updates. - while [ $(get_update_var CURRENT_OP) = ${UPDATER_UPDATE_CHECK} ]; do - echo -n "." && sleep ${timeout} - done + local current_state="" + local next_state="$(get_update_var CURRENT_OP)" - info "Update generated. Update engine downloading update." - while [ $(get_update_var CURRENT_OP) = ${UPDATER_DOWNLOADING} ]; do - echo "Download progress $(get_update_var PROGRESS)" && sleep ${timeout} - done + # For current status, only print out status changes. + # For download, show progress. + # Finally if no status change print out .'s to keep dev informed. + while [ "${current_state}" != "${UPDATER_NEED_REBOOT}" ] && \ + [ "${current_state}" != "${UPDATER_IDLE}" ]; do + if [ "${current_state}" != "${next_state}" ]; then + info "State of updater has changed to: ${next_state}" + elif [ "${next_state}" = "${UPDATER_DOWNLOADING}" ]; then + echo "Download progress $(get_update_var PROGRESS)" + else + echo -n "." + fi - info "Download complete." + sleep ${timeout} + current_state="${next_state}" + next_state="$(get_update_var CURRENT_OP)" + done } diff --git a/lib/cros_image_common.sh b/lib/cros_image_common.sh index abfdbc57c9..b9e2802c09 100644 --- a/lib/cros_image_common.sh +++ b/lib/cros_image_common.sh @@ -185,6 +185,16 @@ image_partition_copy() { exit 1 fi + # Try to use larger buffer if offset/size can be re-aligned. + # 2M / 512 = 4096 + local buffer_ratio=4096 + local bs=512 + if [ $((dstoffset % buffer_ratio)) -eq 0 -a \ + $((length % buffer_ratio)) -eq 0 ]; then + dstoffset=$((dstoffset / buffer_ratio)) + bs=$((bs * buffer_ratio)) + fi + image_dump_partition "${src}" "${srcpart}" | - dd of="${dst}" bs=512 seek="${dstoffset}" conv=notrunc + dd of="${dst}" bs="${bs}" seek="${dstoffset}" conv=notrunc oflag=dsync } diff --git a/make_factory_package.sh b/make_factory_package.sh index 71233fb09c..c2575988d1 100755 --- a/make_factory_package.sh +++ b/make_factory_package.sh @@ -86,7 +86,7 @@ RELEASE_IMAGE="$(basename "${FLAGS_release}")" FACTORY_IMAGE="$(basename "${FLAGS_factory}")" prepare_img() { - local outdev="$FLAGS_diskimg" + local outdev="$(readlink -f "$FLAGS_diskimg")" local sectors="$FLAGS_sectors" local force_full="true" @@ -110,9 +110,10 @@ prepare_img() { fi # Create GPT partition table. + locate_gpt install_gpt "${outdev}" 0 0 "${pmbrcode}" 0 "${force_full}" # Activate the correct partition. - cgpt add -i 2 -S 1 -P 1 "${outdev}" + sudo "${GPT}" add -i 2 -S 1 -P 1 "${outdev}" } prepare_omaha() { @@ -191,7 +192,7 @@ else fi generate_img() { - local outdev="$FLAGS_diskimg" + local outdev="$(readlink -f "$FLAGS_diskimg")" local sectors="$FLAGS_sectors" prepare_img diff --git a/mod_for_test_scripts/750enableExternalExtensions b/mod_for_test_scripts/750enableExternalExtensions index 9d3b33827e..8ceb04c20f 100755 --- a/mod_for_test_scripts/750enableExternalExtensions +++ b/mod_for_test_scripts/750enableExternalExtensions @@ -8,7 +8,7 @@ # to launch a packaged extension at login. echo "Creating external_extensions.json symlink" -LINK_TARGET="/home/autotest/tests/external_extensions.json" +LINK_TARGET="/usr/local/autotest/tests/external_extensions.json" EXT_DIR="${ROOT_FS_DIR}/opt/google/chrome/extensions" FILE="${EXT_DIR}/external_extensions.json" mkdir -p $EXT_DIR diff --git a/prebuilt.py b/prebuilt.py index 5330191b3a..707362b316 100755 --- a/prebuilt.py +++ b/prebuilt.py @@ -42,7 +42,7 @@ Example of uploading prebuilt amd64 host files using rsync: # as per http://crosbug.com/5855 always filter the below packages _FILTER_PACKAGES = set() _RETRIES = 3 -_GSUTIL_BIN = '/b/third_party/gsutil/gsutil' +_GSUTIL_BIN = '/b/build/third_party/gsutil/gsutil' _HOST_PACKAGES_PATH = 'chroot/var/lib/portage/pkgs' _HOST_TARGET = 'amd64' _BOARD_PATH = 'chroot/build/%(board)s' diff --git a/remote_access.sh b/remote_access.sh index 794deff8e7..45531f1552 100644 --- a/remote_access.sh +++ b/remote_access.sh @@ -68,12 +68,29 @@ function learn_board() { info "Target reports board is ${FLAGS_board}" } -function remote_reboot { - info "Rebooting." - remote_sh "touch /tmp/awaiting_reboot; reboot" - local output_file - output_file="${TMP}/output" +# Checks to see if pid $1 is running. +function is_pid_running() { + ps -p ${1} 2>&1 > /dev/null +} +# Wait function given an additional timeout argument. +# $1 - pid to wait on. +# $2 - timeout to wait for. +function wait_with_timeout() { + local pid=$1 + local timeout=$2 + local -r TIMEOUT_INC=1 + local current_timeout=0 + while is_pid_running ${pid} && [ ${current_timeout} -lt ${timeout} ]; do + sleep ${TIMEOUT_INC} + current_timeout=$((current_timeout + TIMEOUT_INC)) + done + ! is_pid_running ${pid} +} + +# Checks to see if a machine has rebooted using the presence of a tmp file. +function check_if_rebooted() { + local output_file="${TMP}/output" while true; do REMOTE_OUT="" # This may fail while the machine is down so generate output and a @@ -84,12 +101,23 @@ function remote_reboot { if grep -q "0" "${output_file}"; then if grep -q "1" "${output_file}"; then info "Not yet rebooted" + sleep .5 else info "Rebooted and responding" break fi fi - sleep .5 + done +} + +function remote_reboot() { + info "Rebooting." + remote_sh "touch /tmp/awaiting_reboot; reboot" + while true; do + check_if_rebooted & + local pid=$! + wait_with_timeout ${pid} 30 && break + ! kill -9 ${pid} 2> /dev/null done }