From f485c86416129936641c08c5f3fc49d3811f0b36 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Tue, 28 Sep 2010 13:08:40 -0700 Subject: [PATCH 01/14] Fix unit tests for cbuildbot. Change-Id: I86b3eb5d7b421aaade85bd702b9089986c82cc76 BUG= TEST=Re-ran unit tests with cbuildbot TOT. Review URL: http://codereview.chromium.org/3441027 --- bin/cbuildbot_unittest.py | 63 ++++++++++++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 10 deletions(-) diff --git a/bin/cbuildbot_unittest.py b/bin/cbuildbot_unittest.py index 203f2a8c76..98cbe83baa 100755 --- a/bin/cbuildbot_unittest.py +++ b/bin/cbuildbot_unittest.py @@ -76,10 +76,35 @@ class CBuildBotTest(mox.MoxTestBase): ['chromeos-base/chromeos-login']) self.mox.VerifyAll() + # TODO(sosa): Re-add once we use cros_mark vs. cros_mark_all. + #def testUprevPackages(self): + # """Test if we get actual revisions in revisions.pfq.""" + # self.mox.StubOutWithMock(cbuildbot, '_CreateRepoDictionary') + # self.mox.StubOutWithMock(cbuildbot, '_ParseRevisionString') + # self.mox.StubOutWithMock(cbuildbot, '_UprevFromRevisionList') + # self.mox.StubOutWithMock(__builtin__, 'open') + + # # Mock out file interaction. + # m_file = self.mox.CreateMock(file) + # __builtin__.open(self._revision_file).AndReturn(m_file) + # m_file.read().AndReturn(self._test_string) + # m_file.close() + + # cbuildbot._CreateRepoDictionary(self._buildroot, + # self._test_board).AndReturn(self._test_dict) + # cbuildbot._ParseRevisionString(self._test_string, + # self._test_dict).AndReturn( + # self._test_parsed_string_array) + # cbuildbot._UprevFromRevisionList(self._buildroot, + # self._test_parsed_string_array) + # self.mox.ReplayAll() + # cbuildbot._UprevPackages(self._buildroot, self._revision_file, + # self._test_board) + # self.mox.VerifyAll() + + # TODO(sosa): Remove once we un-comment above. def testUprevPackages(self): - self.mox.StubOutWithMock(cbuildbot, '_CreateRepoDictionary') - self.mox.StubOutWithMock(cbuildbot, '_ParseRevisionString') - self.mox.StubOutWithMock(cbuildbot, '_UprevFromRevisionList') + """Test if we get actual revisions in revisions.pfq.""" self.mox.StubOutWithMock(__builtin__, 'open') # Mock out file interaction. @@ -88,13 +113,31 @@ class CBuildBotTest(mox.MoxTestBase): m_file.read().AndReturn(self._test_string) m_file.close() - cbuildbot._CreateRepoDictionary(self._buildroot, - self._test_board).AndReturn(self._test_dict) - cbuildbot._ParseRevisionString(self._test_string, - self._test_dict).AndReturn( - self._test_parsed_string_array) - cbuildbot._UprevFromRevisionList(self._buildroot, - self._test_parsed_string_array) + cbuildbot.RunCommand(['./cros_mark_all_as_stable', + '--tracking_branch="cros/master"'], + cwd='%s/src/scripts' % self._buildroot, + enter_chroot=True) + + self.mox.ReplayAll() + cbuildbot._UprevPackages(self._buildroot, self._revision_file, + self._test_board) + self.mox.VerifyAll() + + def testUprevAllPackages(self): + """Test if we get None in revisions.pfq indicating Full Builds.""" + self.mox.StubOutWithMock(__builtin__, 'open') + + # Mock out file interaction. + m_file = self.mox.CreateMock(file) + __builtin__.open(self._revision_file).AndReturn(m_file) + m_file.read().AndReturn('None') + m_file.close() + + cbuildbot.RunCommand(['./cros_mark_all_as_stable', + '--tracking_branch="cros/master"'], + cwd='%s/src/scripts' % self._buildroot, + enter_chroot=True) + self.mox.ReplayAll() cbuildbot._UprevPackages(self._buildroot, self._revision_file, self._test_board) From e86529b046a5bc86e86ae60ecef785e6f358575b Mon Sep 17 00:00:00 2001 From: David McMahon Date: Tue, 28 Sep 2010 18:23:28 -0700 Subject: [PATCH 02/14] Created branch 0.9.74.B. Update CHROMEOS_VERSION_BRANCH=75 Change-Id: Ib47af14a6a102e32cd8bdd66d6f62a8a679f7792 --- chromeos_version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chromeos_version.sh b/chromeos_version.sh index a40ec3a706..e69dd2845f 100755 --- a/chromeos_version.sh +++ b/chromeos_version.sh @@ -26,7 +26,7 @@ export CHROMEOS_VERSION_MINOR=8 # Increment by 2 in trunk after making a release branch. # Does not reset on a major/minor change (always increases). # (Trunk is always odd; branches are always even). -export CHROMEOS_VERSION_BRANCH=73 +export CHROMEOS_VERSION_BRANCH=75 # Patch number. # Increment by 1 each release on a branch. From e7f1a2b6bb270429bdb953a2e59273d434ee9b2c Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Tue, 28 Sep 2010 18:24:13 -0700 Subject: [PATCH 03/14] Add read only flag to mount_gpt_image Change-Id: I33ea8b2148a3e49a6c5de25f2957306130ff092e BUG= TEST=Ran and verified with and without flag. Checked mount and tried writing file to rootfs (to verify it was ro). Also re-ran cros_make_image_bootable to make sure it worked fine after changes. Review URL: http://codereview.chromium.org/3479007 --- mount_gpt_image.sh | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/mount_gpt_image.sh b/mount_gpt_image.sh index 247428047c..ee7b3cc6f6 100755 --- a/mount_gpt_image.sh +++ b/mount_gpt_image.sh @@ -18,6 +18,8 @@ get_default_board # Flags. DEFINE_string board "$DEFAULT_BOARD" \ "The board for which the image was built." b +DEFINE_boolean read_only $FLAGS_FALSE \ + "Mount in read only mode -- skips stateful items." DEFINE_boolean unmount $FLAGS_FALSE \ "Unmount previously mounted dir." u DEFINE_string from "/dev/sdc" \ @@ -45,9 +47,11 @@ function unmount_image() { # Don't die on error to force cleanup set +e # Reset symlinks in /usr/local. - setup_symlinks_on_root "/usr/local" "/var" \ - "${FLAGS_stateful_mountpt}" - fix_broken_symlinks "${FLAGS_rootfs_mountpt}" + if mount | grep "${FLAGS_rootfs_mountpt} (rw,bind)"; then + setup_symlinks_on_root "/usr/local" "/var" \ + "${FLAGS_stateful_mountpt}" + fix_broken_symlinks "${FLAGS_rootfs_mountpt}" + fi sudo umount "${FLAGS_rootfs_mountpt}/usr/local" sudo umount "${FLAGS_rootfs_mountpt}/var" if [[ -n "${FLAGS_esp_mountpt}" ]]; then @@ -59,10 +63,13 @@ function unmount_image() { } function get_usb_partitions() { - sudo mount "${FLAGS_from}3" "${FLAGS_rootfs_mountpt}" - sudo mount "${FLAGS_from}1" "${FLAGS_stateful_mountpt}" + local ro_flag="" + [ ${FLAGS_read_only} -eq ${FLAGS_TRUE} ] && ro_flag="-o ro" + + sudo mount ${ro_flag} "${FLAGS_from}3" "${FLAGS_rootfs_mountpt}" + sudo mount ${ro_flag} "${FLAGS_from}1" "${FLAGS_stateful_mountpt}" if [[ -n "${FLAGS_esp_mountpt}" ]]; then - sudo mount "${FLAGS_from}12" "${FLAGS_esp_mountpt}" + sudo mount ${ro_flag} "${FLAGS_from}12" "${FLAGS_esp_mountpt}" fi } @@ -71,19 +78,22 @@ function get_gpt_partitions() { # Mount the rootfs partition using a loopback device. local offset=$(partoffset "${FLAGS_from}/${filename}" 3) - sudo mount -o loop,offset=$(( offset * 512 )) "${FLAGS_from}/${filename}" \ - "${FLAGS_rootfs_mountpt}" + local ro_flag="" + [ ${FLAGS_read_only} -eq ${FLAGS_TRUE} ] && ro_flag="-o ro" + + sudo mount ${ro_flag} -o loop,offset=$(( offset * 512 )) \ + "${FLAGS_from}/${filename}" "${FLAGS_rootfs_mountpt}" # Mount the stateful partition using a loopback device. offset=$(partoffset "${FLAGS_from}/${filename}" 1) - sudo mount -o loop,offset=$(( offset * 512 )) "${FLAGS_from}/${filename}" \ - "${FLAGS_stateful_mountpt}" + sudo mount ${ro_flag} -o loop,offset=$(( offset * 512 )) \ + "${FLAGS_from}/${filename}" "${FLAGS_stateful_mountpt}" # Mount the stateful partition using a loopback device. if [[ -n "${FLAGS_esp_mountpt}" ]]; then offset=$(partoffset "${FLAGS_from}/${filename}" 12) - sudo mount -o loop,offset=$(( offset * 512 )) "${FLAGS_from}/${filename}" \ - "${FLAGS_esp_mountpt}" + sudo mount ${ro_flag} -o loop,offset=$(( offset * 512 )) \ + "${FLAGS_from}/${filename}" "${FLAGS_esp_mountpt}" fi } @@ -108,8 +118,11 @@ function mount_image() { sudo mount --bind "${FLAGS_stateful_mountpt}/dev_image" \ "${FLAGS_rootfs_mountpt}/usr/local" # Setup symlinks in /usr/local so you can emerge packages into /usr/local. - setup_symlinks_on_root "${FLAGS_stateful_mountpt}/dev_image" \ - "${FLAGS_stateful_mountpt}/var" "${FLAGS_stateful_mountpt}" + + if [ ${FLAGS_read_only} -eq ${FLAGS_FALSE} ]; then + setup_symlinks_on_root "${FLAGS_stateful_mountpt}/dev_image" \ + "${FLAGS_stateful_mountpt}/var" "${FLAGS_stateful_mountpt}" + fi echo "Image specified by ${FLAGS_from} mounted at"\ "${FLAGS_rootfs_mountpt} successfully." } From 3b65e8dd6f877a91693b7f9ce7c2efaf290de793 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Tue, 28 Sep 2010 18:41:56 -0700 Subject: [PATCH 04/14] Add verify option to image_to_live. Change-Id: I63f7ce38179b713bb26bed70d6fd392425595a81 BUG= TEST=Ran with update vm script. Review URL: http://codereview.chromium.org/3393019 --- bin/cros_run_vm_update | 1 + image_to_live.sh | 64 +++++++++++++++++++++++++++++++++++------- 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/bin/cros_run_vm_update b/bin/cros_run_vm_update index eb4da2feb7..ea1ae5e9b9 100755 --- a/bin/cros_run_vm_update +++ b/bin/cros_run_vm_update @@ -31,5 +31,6 @@ $(dirname $0)/../image_to_live.sh \ --remote=${HOSTNAME} \ --ssh_port=${FLAGS_ssh_port} \ --stateful_update_flag=${stateful_flags} \ + --verify \ --image=$(readlink -f ${FLAGS_update_image_path}) diff --git a/image_to_live.sh b/image_to_live.sh index f2b0c0556b..b08d0c9501 100755 --- a/image_to_live.sh +++ b/image_to_live.sh @@ -25,6 +25,7 @@ DEFINE_boolean update_known_hosts ${FLAGS_FALSE} \ "Update your known_hosts with the new remote instance's key." DEFINE_string update_log "update_engine.log" \ "Path to log for the update_engine." +DEFINE_boolean verify ${FLAGS_TRUE} "Verify image on device after update." # Flags for devserver. DEFINE_string archive_dir "" \ @@ -33,17 +34,21 @@ DEFINE_integer devserver_port 8080 \ "Port to use for devserver." DEFINE_string image "" \ "Update with this image path that is in this source checkout." i +DEFINE_boolean update_stateful ${FLAGS_TRUE} \ + "Perform update of stateful partition e.g. /var /usr/local." DEFINE_string update_url "" "Full url of an update image." # Flags for stateful update. DEFINE_string stateful_update_flag "" \ "Flag to pass to stateful update e.g. old, clean, etc." s -UPDATER_BIN='/usr/bin/update_engine_client' -UPDATER_IDLE='UPDATE_STATUS_IDLE' -UPDATER_NEED_REBOOT='UPDATE_STATUS_UPDATED_NEED_REBOOT' -UPDATER_UPDATE_CHECK='UPDATE_STATUS_CHECKING_FOR_UPDATE' -UPDATER_DOWNLOADING='UPDATE_STATUS_DOWNLOADING' +UPDATER_BIN="/usr/bin/update_engine_client" +UPDATER_IDLE="UPDATE_STATUS_IDLE" +UPDATER_NEED_REBOOT="UPDATE_STATUS_UPDATED_NEED_REBOOT" +UPDATER_UPDATE_CHECK="UPDATE_STATUS_CHECKING_FOR_UPDATE" +UPDATER_DOWNLOADING="UPDATE_STATUS_DOWNLOADING" + +IMAGE_PATH="" function kill_all_devservers { # Using ! here to avoid exiting with set -e is insufficient, so use @@ -89,9 +94,12 @@ function start_dev_server { if [ -n "${FLAGS_image}" ]; then devserver_flags="${devserver_flags} \ --image $(reinterpret_path_for_chroot ${FLAGS_image})" + IMAGE_PATH="${FLAGS_image}" + elif [ -n "${FLAGS_archive_dir}" ]; then devserver_flags="${devserver_flags} \ --archive_dir $(reinterpret_path_for_chroot ${FLAGS_archive_dir}) -t" + IMAGE_PATH="${FLAGS_archive_dir}/chromiumos_test_image.bin" fi info "Starting devserver with flags ${devserver_flags}" @@ -258,6 +266,31 @@ function remote_reboot { done } +function verify_image { + info "Verifying image." + "${SCRIPTS_DIR}/mount_gpt_image.sh" --from "$(dirname ${IMAGE_PATH})" \ + --image "$(basename ${IMAGE_PATH})" \ + --read_only + + local lsb_release=$(cat /tmp/m/etc/lsb-release) + info "Verifying image with release:" + echo ${lsb_release} + + "${SCRIPTS_DIR}/mount_gpt_image.sh" --unmount + + remote_sh "cat /etc/lsb-release" + info "Remote image reports:" + echo ${REMOTE_OUT} + + if [ "${lsb_release}" = "${REMOTE_OUT}" ]; then + info "Update was successful and image verified as ${lsb_release}." + return 0 + else + warn "Image verification failed." + return 1 + fi +} + function main() { assert_outside_chroot @@ -268,6 +301,12 @@ function main() { set -e + if [ ${FLAGS_verify} -eq ${FLAGS_TRUE} ] && \ + [ -n "${FLAGS_update_url}" ]; then + warn "Verify is not compatible with setting an update url." + FLAGS_verify=${FLAGS_FALSE} + fi + trap cleanup EXIT TMP=$(mktemp -d /tmp/image_to_live.XXXX) @@ -284,11 +323,12 @@ function main() { start_dev_server fi - if [ "${FLAGS_update}" -eq "${FLAGS_TRUE}" ] && ! run_auto_update; then + if [ ${FLAGS_update} -eq ${FLAGS_TRUE} ] && ! run_auto_update; then die "Update was not successful." fi - if ! run_stateful_update; then + if [ ${FLAGS_update_stateful} -eq ${FLAGS_TRUE} ] && \ + ! run_stateful_update; then warn "Stateful update was not successful." fi @@ -304,10 +344,14 @@ function main() { fi remote_sh "grep ^CHROMEOS_RELEASE_DESCRIPTION= /etc/lsb-release" - local release_description=$(echo ${REMOTE_OUT} | cut -d '=' -f 2) - info "Update was successful and rebooted to $release_description" + if [ ${FLAGS_verify} -eq ${FLAGS_TRUE} ]; then + verify_image + else + local release_description=$(echo ${REMOTE_OUT} | cut -d '=' -f 2) + info "Update was successful and rebooted to $release_description" + fi - return 0 + exit 0 } main $@ From 45bec936f65b5406c59f448d1964604926b73b97 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Wed, 29 Sep 2010 15:38:53 -0700 Subject: [PATCH 05/14] Move vm constants into their own file. This will make it easier to share these constants. Also fixed get latest image logic in image_to_vm and allowed for using the most recent image in cros_run_vm_test to follow other shell convention of using latest. Change-Id: I60ed4c03d609500da7f6ae34ef57ba2e32f4b0bb BUG= TEST=Tested by running image_to_vm with --full and cros_run_vm_test with suite_Smoke Review URL: http://codereview.chromium.org/3597001 --- bin/cros_run_vm_test | 13 +++++++++++-- image_to_vm.sh | 15 ++------------- lib/cros_vm_constants.sh | 16 ++++++++++++++++ lib/cros_vm_lib.sh | 5 +++-- 4 files changed, 32 insertions(+), 17 deletions(-) create mode 100644 lib/cros_vm_constants.sh diff --git a/bin/cros_run_vm_test b/bin/cros_run_vm_test index 6981745ac4..8036b230b1 100755 --- a/bin/cros_run_vm_test +++ b/bin/cros_run_vm_test @@ -8,6 +8,7 @@ . "$(dirname $0)/../common.sh" . "$(dirname $0)/../lib/cros_vm_lib.sh" +. "$(dirname "$0")/../lib/cros_vm_constants.sh" DEFINE_string image_path "" "Full path of the VM image" DEFINE_string test_case "" "Name of the test case to run" @@ -18,8 +19,16 @@ set -e FLAGS "$@" || exit 1 eval set -- "${FLAGS_ARGV}" -[ -n "${FLAGS_image_path}" ] || die "You must specify a path to an image" -[ -n "${FLAGS_test_case}" ] || die "You must specify a test case" +# Use latest if not specified. +if [ -z "${FLAGS_image_path}" ]; then + LATEST_IMAGE="$(${SCRIPTS_DIR}/get_latest_image.sh)/${DEFAULT_QEMU_IMAGE}" + info "Using latest vm image ${LATEST_IMAGE}" + FLAGS_image_path=${LATEST_IMAGE} +fi + +[ -e "${FLAGS_image_path}" ] || die "Image ${FLAGS_image_path} does not exist." + +[ -n "${FLAGS_test_case}" ] || die "You must specify a test case." trap stop_kvm EXIT start_kvm "${FLAGS_image_path}" diff --git a/image_to_vm.sh b/image_to_vm.sh index 3807269856..c2fb82b748 100755 --- a/image_to_vm.sh +++ b/image_to_vm.sh @@ -11,22 +11,11 @@ # The path to common.sh should be relative to your script's location. . "$(dirname "$0")/common.sh" . "$(dirname "$0")/chromeos-common.sh" +. "$(dirname "$0")/lib/cros_vm_constants.sh" get_default_board assert_inside_chroot -DEFAULT_MEM="1024" -DEFAULT_VMDK="ide.vmdk" -DEFAULT_VMX="chromiumos.vmx" -DEFAULT_VBOX_DISK="os.vdi" -DEFAULT_QEMU_IMAGE="chromiumos_qemu_image.bin" - -# Minimum sizes for full size vm images -- needed for update. -MIN_VDISK_SIZE_FULL=6072 -MIN_STATEFUL_FS_SIZE_FULL=2048 - -MOD_SCRIPTS_ROOT="${GCLIENT_ROOT}/src/scripts/mod_for_test_scripts" - # Flags DEFINE_string board "${DEFAULT_BOARD}" \ "Board for which the image was built" @@ -85,7 +74,7 @@ fi IMAGES_DIR="${DEFAULT_BUILD_ROOT}/images/${FLAGS_board}" # Default to the most recent image if [ -z "${FLAGS_from}" ] ; then - FLAGS_from="${IMAGES_DIR}/$(ls -t $IMAGES_DIR | head -1)" + FLAGS_from="$(./get_latest_image.sh)" else pushd "${FLAGS_from}" && FLAGS_from=`pwd` && popd fi diff --git a/lib/cros_vm_constants.sh b/lib/cros_vm_constants.sh new file mode 100644 index 0000000000..124d158eb5 --- /dev/null +++ b/lib/cros_vm_constants.sh @@ -0,0 +1,16 @@ +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Common constants for vm scripts. + +# Default values for creating VM's. +DEFAULT_MEM="1024" +DEFAULT_VMDK="ide.vmdk" +DEFAULT_VMX="chromiumos.vmx" +DEFAULT_VBOX_DISK="os.vdi" +DEFAULT_QEMU_IMAGE="chromiumos_qemu_image.bin" + +# Minimum sizes for full size vm images -- needed for update. +MIN_VDISK_SIZE_FULL=6072 +MIN_STATEFUL_FS_SIZE_FULL=2048 \ No newline at end of file diff --git a/lib/cros_vm_lib.sh b/lib/cros_vm_lib.sh index 61d575bfff..c47d11de33 100644 --- a/lib/cros_vm_lib.sh +++ b/lib/cros_vm_lib.sh @@ -4,6 +4,8 @@ # # Common vm functions for use in crosutils. +. "$(dirname "$0")/cros_vm_constants.sh" + DEFINE_string kvm_pid "" \ "Use this pid file. If it exists and is set, use the vm specified by pid." DEFINE_boolean no_graphics ${FLAGS_FALSE} "Runs the KVM instance silently." @@ -11,7 +13,6 @@ DEFINE_boolean persist "${FLAGS_FALSE}" "Persist vm." DEFINE_boolean snapshot ${FLAGS_FALSE} "Don't commit changes to image." DEFINE_integer ssh_port 9222 "Port to tunnel ssh traffic over." - KVM_PID_FILE=/tmp/kvm.$$.pid function get_pid() { @@ -47,7 +48,7 @@ function start_kvm() { snapshot="-snapshot" fi - sudo kvm -m 1024 \ + sudo kvm -m ${DEFAULT_MEM} \ -vga std \ -pidfile "${KVM_PID_FILE}" \ -daemonize \ From 5137a74e90da7595c0137fb3bc6d5c9633e18295 Mon Sep 17 00:00:00 2001 From: David James Date: Wed, 29 Sep 2010 15:50:08 -0700 Subject: [PATCH 06/14] Exit explicitly at end of parallel_emerge, so child processes exit as well. So, we've been seeing parallel_emerge hang after printing "Done". It's hard to know for sure why this hang is happening, but my theory right now is that the Python garbage collector doesn't always know that it needs to kill the child processes when we exit normally at the bottom of the script. Adding sys.exit(0) tells Python that yes, we really want to exit. I haven't tested my hypothesis above, because the hang we've been seeing is pretty rare and is hard to reproduce. So I'm planning on testing by just putting in this workaround and seeing if it fixes the problem. TEST=Run ./parallel_emerge world BUG=chromium-os:5976 Change-Id: I7b4f2ec4ccba9b00f22f3739dfd3eff51ceed425 Review URL: http://codereview.chromium.org/3448030 --- parallel_emerge | 1 + 1 file changed, 1 insertion(+) diff --git a/parallel_emerge b/parallel_emerge index 61e9920efb..9a77ef45c4 100755 --- a/parallel_emerge +++ b/parallel_emerge @@ -1733,6 +1733,7 @@ def main(): os.execvp(os.path.realpath(sys.argv[0]), args) print "Done" + sys.exit(0) if __name__ == "__main__": main() From 29d962035e3dda3fd76202be108d8d3e37209dfc Mon Sep 17 00:00:00 2001 From: Tan Gao Date: Thu, 30 Sep 2010 10:41:09 -0700 Subject: [PATCH 07/14] remove second call to free up loop device Change-Id: Ibf52597f01b7868f8472d2042e4d335442b82597 TEST=manually build a dev recovery image and verified no loop device warning was outputted by losetup call Review URL: http://codereview.chromium.org/3439024 --- mod_image_for_dev_recovery.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mod_image_for_dev_recovery.sh b/mod_image_for_dev_recovery.sh index 7523e59486..e94f687b6d 100755 --- a/mod_image_for_dev_recovery.sh +++ b/mod_image_for_dev_recovery.sh @@ -176,7 +176,7 @@ create_dev_recovery_image() { # Mount resized stateful FS and copy payload content to its root directory local temp_mnt=$(mktemp -d "/tmp/temp_mnt.XXXXXX") local loop_dev=$(get_loop_dev) - trap "umount_from_loop_dev ${temp_mnt} && cleanup_loop_dev ${loop_dev}" EXIT + trap "umount_from_loop_dev ${temp_mnt} && rm -f \"${temp_state}\"" EXIT mkdir -p "${temp_mnt}" sudo mount -o loop=${loop_dev} "${temp_state}" "${temp_mnt}" sudo cp -R "${FLAGS_payload_dir}" "${temp_mnt}" @@ -189,7 +189,6 @@ create_dev_recovery_image() { # TODO(tgao): handle install script (for default and custom cases) local temp_img=$(update_partition_table $temp_state $resized_sectors) - rm -f "${temp_state}" # trap handler will clean up loop device and temp mount point echo ${temp_img} } From b5ab343a0d6269f2095ffc3c3025a48402646762 Mon Sep 17 00:00:00 2001 From: Ken Mixter Date: Thu, 30 Sep 2010 13:05:56 -0700 Subject: [PATCH 08/14] Fix grab-buildbot in sync_build_test by gsdizing it Review URL: http://codereview.chromium.org/3109023 --- sync_build_test.sh | 61 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/sync_build_test.sh b/sync_build_test.sh index 9473205e67..16934419c0 100755 --- a/sync_build_test.sh +++ b/sync_build_test.sh @@ -58,10 +58,14 @@ If this is set, chrome browser will be built from source." DEFINE_string chronos_passwd "${CHRONOS_PASSWD}" \ "Use this as the chronos user passwd (defaults to \$CHRONOS_PASSWD)" DEFINE_string chroot "" "Chroot to build/use" +DEFINE_boolean enable_rootfs_verification ${FLAGS_TRUE} \ + "Enable rootfs verification when building image" DEFINE_boolean force_make_chroot ${FLAGS_FALSE} "Run make_chroot indep of sync" DEFINE_string grab_buildbot "" \ "Instead of building, grab this full image.zip URI generated by the \ buildbot" +DEFINE_boolean ignore_remote_test_failures ${FLAGS_FALSE} \ + "Ignore any remote tests that failed and don't return failure" DEFINE_boolean image_to_live ${FLAGS_FALSE} \ "Put the resulting image on live instance (requires --remote)" DEFINE_string image_to_usb "" \ @@ -183,6 +187,7 @@ function validate_and_set_param_defaults() { fi FLAGS_sync=${FLAGS_FALSE} FLAGS_build=${FLAGS_FALSE} + FLAGS_unittest=${FLAGS_FALSE} FLAGS_master=${FLAGS_FALSE} fi @@ -339,6 +344,9 @@ function describe_phase() { # Called when there is a failure and we exit early function failure() { trap - EXIT + # Clear these out just in case. + export GSDCURL_USERNAME="" + export GSDCURL_PASSWORD="" describe_phase "Failure during: ${LAST_PHASE}" show_duration } @@ -450,8 +458,13 @@ EOF # Downloads a buildbot image function grab_buildbot() { + read -p "Username [${LOGNAME}]: " GSDCURL_USERNAME + export GSDCURL_USERNAME + read -s -p "Password: " GSDCURL_PASSWORD + export GSDCURL_PASSWORD + CURL="$(dirname $0)/bin/cros_gsdcurl.py" if [[ "${FLAGS_grab_buildbot}" == "LATEST" ]]; then - local latest=$(curl "${FLAGS_buildbot_uri}/LATEST") + local latest=$(${CURL} "${FLAGS_buildbot_uri}/LATEST") if [[ -z "${latest}" ]]; then echo "Error finding latest." exit 1 @@ -459,9 +472,14 @@ function grab_buildbot() { FLAGS_grab_buildbot="${FLAGS_buildbot_uri}/${latest}/image.zip" fi local dl_dir=$(mktemp -d "/tmp/image.XXXX") + echo "Grabbing image from ${FLAGS_grab_buildbot} to ${dl_dir}" - run_phase "Downloading image" curl "${FLAGS_grab_buildbot}" \ - -o "${dl_dir}/image.zip" + run_phase "Downloading image" ${CURL} "${FLAGS_grab_buildbot}" \ + -o "${dl_dir}/image.zip" + # Clear out the credentials so they can't be used later. + export GSDCURL_USERNAME="" + export GSDCURL_PASSWORD="" + cd "${dl_dir}" unzip image.zip local image_basename=$(basename $(dirname "${FLAGS_grab_buildbot}")) @@ -474,6 +492,15 @@ function grab_buildbot() { run_phase "Installing buildbot test modified image" \ mv chromiumos_test_image.bin "${image_dir}/chromiumos_image.bin" FLAGS_mod_image_for_test=${FLAGS_FALSE} + else + run_phase "Installing buildbot base image" \ + mv chromiumos_base_image.bin "${image_dir}/chromiumos_image.bin" + fi + + if [[ -n "${FLAGS_test}" ]]; then + if [[ ! -d "${FLAGS_top}/chroot/build/${FLAGS_board}" ]]; then + die "To run tests on a buildbot image, run setup_board first." + fi if [[ -e "autotest.tgz" || -e "autotest.tar.bz2" ]]; then # pull in autotest local dir="${FLAGS_chroot}/build/${FLAGS_board}/usr/local" @@ -491,8 +518,6 @@ function grab_buildbot() { run_phase "Installing buildbot autotest cross-compiled binaries" \ sudo mv autotest ${dir} fi - else - mv chromiumos_image.bin "${image_dir}" fi chdir_relative . run_phase "Removing downloaded image" rm -rf "${dl_dir}" @@ -569,16 +594,6 @@ function main() { local build_autotest_param="" if [[ ${FLAGS_build_autotest} -eq ${FLAGS_TRUE} ]]; then build_autotest_param="--withautotest" - if [[ ${FLAGS_useworkon} -eq ${FLAGS_TRUE} ]]; then - # In workon flow, you must workon packages to run tests - run_phase_in_chroot "Setting workon for autotest" \ - ./cros_workon --board=${FLAGS_board} start autotest \ - autotest-tests autotest-deps - # In minilayout you may not yet have autotest. - if [[ ! -d "${FLAGS_top}/src/third_party/autotest/files" ]]; then - run_phase "Syncing autotest repo" repo sync autotest - fi - fi fi run_phase_in_chroot "Building packages" \ @@ -595,6 +610,7 @@ function main() { if [[ ${FLAGS_unittest} -eq ${FLAGS_TRUE} ]] && [[ "${FLAGS_board}" == \ "x86-generic" ]] ; then + chdir_relative src/scripts run_phase_in_chroot "Running unit tests" ./cros_run_unit_tests \ ${board_param} fi @@ -606,9 +622,13 @@ function main() { ./enter_chroot.sh "echo '${FLAGS_chronos_passwd}' | \ ~/trunk/src/scripts/set_shared_user_password.sh" fi + local other_params="--enable_rootfs_verification" + if [[ ${FLAGS_enable_rootfs_verification} -eq ${FLAGS_FALSE} ]]; then + other_params="--noenable_rootfs_verification" + fi run_phase_in_chroot "Mastering image" ./build_image \ "${board_param}" --replace ${withdev_param} \ - ${jobs_param} + ${jobs_param} ${other_params} fi if [[ ${FLAGS_mod_image_for_test} -eq ${FLAGS_TRUE} ]]; then @@ -633,9 +653,14 @@ function main() { chdir_relative src/scripts # We purposefully do not quote FLAGS_test below as we expect it may # have multiple parameters - run_phase "Running tests on Chromium OS machine ${FLAGS_remote}" \ + if ! run_phase "Running tests on Chromium OS machine ${FLAGS_remote}" \ ./run_remote_tests.sh "--remote=${FLAGS_remote}" ${FLAGS_test} \ - "${board_param}" + "${board_param}"; then + if [[ ${FLAGS_ignore_remote_test_failures} -eq ${FLAGS_FALSE} ]]; then + echo "Remote tests failed and --ignore_remote_test_failures not passed" + false + fi + fi fi trap - EXIT From e3049de00f257a3333928a0b8dd6585eb11cd246 Mon Sep 17 00:00:00 2001 From: Kenneth Waters Date: Thu, 30 Sep 2010 14:20:34 -0700 Subject: [PATCH 09/14] build_image: Merge x86 and ARM where possible. - I looked at all of the x86 and ARM paths through out build image scripts, these changes clean up stale comments, stale code, and unforks some small things. BUG=none TEST=Built images for x86-generic, arm-generic and tegra2-seaboard, booted tegra2-seaboard image. Review URL: http://codereview.chromium.org/3448022 Change-Id: Ibad2774ff2cbf5f15528454506542b87e43e24a2 --- bin/cros_make_image_bootable | 19 +++---------------- build_gpt.sh | 17 +---------------- build_image | 7 +------ create_esp.sh | 3 +-- image_to_usb.sh | 3 --- update_bootloaders.sh | 7 +++---- 6 files changed, 9 insertions(+), 47 deletions(-) diff --git a/bin/cros_make_image_bootable b/bin/cros_make_image_bootable index 8aacf8c913..20c21911d6 100755 --- a/bin/cros_make_image_bootable +++ b/bin/cros_make_image_bootable @@ -86,9 +86,6 @@ DEFINE_integer verity_max_ios 1024 \ DEFINE_string verity_algorithm "sha1" \ "Cryptographic hash algorithm used for kernel vboot." -DEFINE_string arm_extra_bootargs "" \ - "Additional command line options to pass to the ARM kernel." - DEFINE_string keys_dir "/usr/share/vboot/devkeys" \ "Directory containing the signing keys." @@ -199,8 +196,7 @@ make_image_bootable() { sudo dd if="${FLAGS_output_dir}/vmlinuz.image" of="${image}" \ conv=notrunc bs=512 seek=${koffset} - # Update the bootloaders. For legacy/efi x86, the EFI system partition - # will be updated and for arm, the mbr will be updated (for u-boot). + # Update the bootloaders. The EFI system partition will be updated. local kernel_part= local usb_disk="${FLAGS_usb_disk}" @@ -218,17 +214,8 @@ make_image_bootable() { # Install syslinux on the EFI System Partition. kernel_part="${kernel_part} --install_syslinux" elif [[ "${FLAGS_arch}" = "arm" ]]; then - # TODO(wad) mmcblk1p3 is hardcoded for arm for now! - usb_disk="/dev/mmcblk1p3" - # ARM doesn't support using the kernel image for kernel cmdline flags yet. - kernel_part="--kernel_cmdline=\"${FLAGS_arm_extra_bootargs}\" " - # TODO(wad) Integrate dmtable extraction into the arm build - # E.g. $(cat ${FLAGS_output_dir}/boot.config | tr -s '\n' ' ')" - local kpart_offset="--kernel_partition_offset=${koffset}" - local kpart_size="--kernel_partition_sectors=" - kpart_size="${kpart_size}$(partsize ${image} 2)" - kernel_part="${kernel_part} ${kpart_size} ${kpart_offset}" - info "Using addition bootloader arguments: ${kernel_part}" + # These flags are not used for ARM update_bootloaders.sh + kernel_part="" fi # Update partition 12 diff --git a/build_gpt.sh b/build_gpt.sh index 84888a902c..df0f4fc070 100755 --- a/build_gpt.sh +++ b/build_gpt.sh @@ -21,8 +21,6 @@ DEFINE_string arch "" \ "The target architecture (\"arm\" or \"x86\")." DEFINE_string board "$DEFAULT_BOARD" \ "The board to build an image for." -DEFINE_string arm_extra_bootargs "" \ - "Additional command line options to pass to the ARM kernel." DEFINE_integer rootfs_partition_size 1024 \ "rootfs parition size in MBs." @@ -109,8 +107,7 @@ if [ ! -s ${ESP_IMG} ]; then exit 1 fi -# We'll need some code to put in the PMBR, for booting on legacy BIOS. Some ARM -# systems will use a U-Boot script temporarily, but it goes in the same place. +# We'll need some code to put in the PMBR, for booting on legacy BIOS. if [[ "$ARCH" = "arm" ]]; then PMBRCODE=/dev/zero else @@ -122,18 +119,6 @@ fi install_gpt $OUTDEV $(numsectors $ROOTFS_IMG) $(numsectors $STATEFUL_IMG) \ $PMBRCODE $(numsectors $ESP_IMG) false $FLAGS_rootfs_partition_size -if [[ "$ARCH" = "arm" ]]; then - # assume /dev/mmcblk1. we could not get this from ${OUTDEV} - DEVICE=1 - MBR_SCRIPT_UIMG=$(make_arm_mbr \ - ${START_KERN_A} \ - ${NUM_KERN_SECTORS} \ - ${DEVICE} \ - "${FLAGS_arm_extra_bootargs}") - sudo dd bs=1 count=`stat --printf="%s" ${MBR_SCRIPT_UIMG}` \ - if="$MBR_SCRIPT_UIMG" of=${OUTDEV} conv=notrunc -fi - # Emit helpful scripts for testers, etc. ${SCRIPTS_DIR}/emit_gpt_scripts.sh "${OUTDEV}" "${IMAGEDIR}" diff --git a/build_image b/build_image index 7620d0af7f..2df9a3cc05 100755 --- a/build_image +++ b/build_image @@ -49,8 +49,6 @@ DEFINE_boolean factory_install ${FLAGS_FALSE} \ is also required in image_to_usb." DEFINE_boolean dev_install ${FLAGS_FALSE} \ "Build a smaller image to overlay the dev recovery install shim on" -DEFINE_string arm_extra_bootargs "" \ - "Additional command line options to pass to the ARM kernel." DEFINE_integer rootfs_partition_size 1024 \ "rootfs partition size in MiBs." DEFINE_integer rootfs_size 720 \ @@ -360,7 +358,6 @@ create_boot_desc() { --verity_algorithm="${FLAGS_verity_algorithm}" --keys_dir="${DEVKEYSDIR}" --usb_disk="${FLAGS_usb_disk}" - --arm_extra_bootargs="${FLAGS_arm_extra_bootargs}" --nocleanup_dirs ${enable_rootfs_verification_flag} EOF @@ -637,8 +634,7 @@ create_base_image() { # cros_make_image_bootable will clobber vmlinuz.image for x86. # Until then, just copy the kernel to vmlinuz.image. It is - # expected in build_gpt.sh and needed by ARM until it supports the - # full, signed kernel partition format. + # expected in build_gpt.sh. cp "${ROOT_FS_DIR}/boot/vmlinuz" "${OUTPUT_DIR}/vmlinuz.image" # Create an empty esp image to be updated in by update_bootloaders.sh. @@ -652,7 +648,6 @@ create_base_image() { ${SCRIPTS_DIR}/build_gpt.sh \ --arch=${ARCH} \ --board=${FLAGS_board} \ - --arm_extra_bootargs="${FLAGS_arm_extra_bootargs}" \ --rootfs_partition_size=${FLAGS_rootfs_partition_size} \ "${OUTPUT_DIR}" \ "${OUTPUT_DIR}/${image_name}" diff --git a/create_esp.sh b/create_esp.sh index 5b345e20e1..261b8de340 100755 --- a/create_esp.sh +++ b/create_esp.sh @@ -26,8 +26,7 @@ fi info "Creating a new esp image at ${FLAGS_to}" anyway. # Create EFI System Partition to boot stock EFI BIOS (but not ChromeOS EFI -# BIOS). We only need this for x86, but it's simpler and safer to keep the -# disk images the same for both x86 and ARM. +# BIOS). ARM uses this space to determine which partition is bootable. # NOTE: The size argument for mkfs.vfat is in 1024-byte blocks. # We'll hard-code it to 16M for now. ESP_BLOCKS=16384 diff --git a/image_to_usb.sh b/image_to_usb.sh index 0bd67ce5ee..f40e57a4a3 100755 --- a/image_to_usb.sh +++ b/image_to_usb.sh @@ -243,15 +243,12 @@ then echo "Installation must be done from inside the chroot." exit 1 fi - #TODO(kwaters): fix when verified root works on ARM - [ "${FLAGS_arch}" = "ARM" ] && SKIP_VBLOCK="--skip_vblock" echo "Installing ${SRC_IMAGE} to ${FLAGS_to}..." "${FLAGS_build_root}/${FLAGS_board}/usr/sbin/chromeos-install" \ --yes \ --skip_src_removable \ --skip_dst_removable \ - ${SKIP_VBLOCK:-} \ --arch="${FLAGS_arch}" \ --payload_image="${SRC_IMAGE}" \ --dst="${FLAGS_to}" diff --git a/update_bootloaders.sh b/update_bootloaders.sh index 7c6b0dd9be..63faaceec8 100755 --- a/update_bootloaders.sh +++ b/update_bootloaders.sh @@ -21,7 +21,7 @@ DEFINE_boolean install_syslinux ${FLAGS_FALSE} \ DEFINE_string from "/tmp/boot" \ "Path the legacy bootloader templates are copied from. (Default /tmp/boot)" DEFINE_string to "/tmp/esp.img" \ - "Path to esp image or ARM output MBR (Default: /tmp/esp.img)" + "Path to esp image (Default: /tmp/esp.img)" DEFINE_integer to_offset 0 \ "Offset in bytes into 'to' if it is a file (Default: 0)" DEFINE_integer to_size -1 \ @@ -39,7 +39,7 @@ DEFINE_string kernel_partition_offset "0" \ DEFINE_string kernel_partition_sectors "0" \ "Kernel partition sectors (Default: 0)" DEFINE_string usb_disk /dev/sdb3 \ - "Path syslinux should use to do a usb (or arm!) boot. Default: /dev/sdb3" + "Path syslinux should use to do a usb boot. Default: /dev/sdb3" # Parse flags FLAGS "$@" || exit 1 @@ -102,8 +102,7 @@ if [[ ! -e "${FLAGS_to}" ]]; then # This shouldn't happen. info "Creating a new esp image at ${FLAGS_to}" anyway. # Create EFI System Partition to boot stock EFI BIOS (but not ChromeOS EFI - # BIOS). We only need this for x86, but it's simpler and safer to keep the - # disk images the same for both x86 and ARM. + # BIOS). ARM uses this space to determine which partition is bootable. # NOTE: The size argument for mkfs.vfat is in 1024-byte blocks. # We'll hard-code it to 16M for now. ESP_BLOCKS=16384 From c2db26c796d6b3b5ed62c30267f561f0470a64e8 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Thu, 30 Sep 2010 14:25:25 -0700 Subject: [PATCH 10/14] Add methods to run vm tests on x86-pre-flight-queue. Right now we're leaving error_ok=True until the tests are shown to work on builder. Change-Id: I6b8c690a0da30948389fd4312032c78d87115364 BUG=6906 TEST=Ran through it locally. Review URL: http://codereview.chromium.org/3591002 --- bin/cbuildbot.py | 42 +++++++++++++++++++++++++++++++++++++++++ bin/cbuildbot_config.py | 4 ++++ lib/cros_build_lib.py | 15 +++++++++++++-- lib/cros_vm_lib.sh | 4 +--- 4 files changed, 60 insertions(+), 5 deletions(-) diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index a077c115f1..794bf15013 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -208,6 +208,20 @@ def _UprevAllPackages(buildroot): '--tracking_branch="cros/master"'], cwd=cwd, enter_chroot=True) + +def _GetVMConstants(buildroot): + """Returns minimum (vdisk_size, statefulfs_size) recommended for VM's.""" + cwd = os.path.join(buildroot, 'src', 'scripts', 'lib') + source_cmd = 'source %s/cros_vm_constants.sh' % cwd + vdisk_size = RunCommand([ + '/bin/bash', '-c', '%s && echo $MIN_VDISK_SIZE_FULL' % source_cmd], + redirect_stdout=True) + statefulfs_size = RunCommand([ + '/bin/bash', '-c', '%s && echo $MIN_STATEFUL_FS_SIZE_FULL' % source_cmd], + redirect_stdout=True) + return (vdisk_size.strip(), statefulfs_size.strip()) + + # =========================== Main Commands =================================== def _FullCheckout(buildroot, rw_checkout=True, retries=_DEFAULT_RETRIES): @@ -244,20 +258,43 @@ def _Build(buildroot): cwd = os.path.join(buildroot, 'src', 'scripts') RunCommand(['./build_packages'], cwd=cwd, enter_chroot=True) + def _WipeOldOutput(buildroot): RunCommand(['rm', '-rf', 'src/build/images'], cwd=buildroot) + def _BuildImage(buildroot): _WipeOldOutput(buildroot) cwd = os.path.join(buildroot, 'src', 'scripts') RunCommand(['./build_image', '--replace'], cwd=cwd, enter_chroot=True) + +def _BuildVMImageForTesting(buildroot): + (vdisk_size, statefulfs_size) = _GetVMConstants(buildroot) + cwd = os.path.join(buildroot, 'src', 'scripts') + RunCommand(['./image_to_vm.sh', + '--test_image', + '--full', + '--vdisk_size %s' % vdisk_size, + '--statefulfs_size %s' % statefulfs_size, + ], cwd=cwd, enter_chroot=True) + + def _RunUnitTests(buildroot): cwd = os.path.join(buildroot, 'src', 'scripts') RunCommand(['./cros_run_unit_tests'], cwd=cwd, enter_chroot=True) +def _RunSmokeSuite(buildroot): + cwd = os.path.join(buildroot, 'src', 'scripts') + RunCommand(['bin/cros_run_vm_test', + '--no_graphics', + '--test_case', + 'suite_Smoke', + ], cwd=cwd, error_ok=True) + + def _UprevPackages(buildroot, revisionfile, board): """Uprevs a package based on given revisionfile. @@ -383,6 +420,11 @@ def main(): _RunUnitTests(buildroot) _BuildImage(buildroot) + + if buildconfig['smoke_bvt']: + _BuildVMImageForTesting(buildroot) + _RunSmokeSuite(buildroot) + if buildconfig['uprev']: if buildconfig['master']: # Master bot needs to check if the other slaves completed. diff --git a/bin/cbuildbot_config.py b/bin/cbuildbot_config.py index 1ba08d51d0..dda1ef8a50 100644 --- a/bin/cbuildbot_config.py +++ b/bin/cbuildbot_config.py @@ -18,6 +18,7 @@ important -- Master bot uses important bots to determine overall status. hostname -- Needed for 'important' slaves. The hostname of the bot. Should match hostname in slaves.cfg in buildbot checkout. unittests -- Runs unittests for packages. +smoke_bvt -- Runs the test smoke suite in a qemu-based VM using KVM. """ @@ -29,6 +30,7 @@ config['default'] = { 'master' : False, 'important' : False, 'unittests' : False, + 'smoke_bvt' : False, } config['x86-generic-pre-flight-queue'] = { 'board' : 'x86-generic', @@ -37,6 +39,7 @@ config['x86-generic-pre-flight-queue'] = { 'important' : False, 'hostname' : 'chromeosbuild2', 'unittests' : True, + 'smoke_bvt' : True, } config['x86_pineview_bin'] = { 'board' : 'x86-pineview', @@ -45,6 +48,7 @@ config['x86_pineview_bin'] = { 'important' : False, 'hostname' : 'codf200.jail', 'unittests': True, + 'smoke_bvt' : True, } config['arm_tegra2_bin'] = { 'board' : 'tegra2', diff --git a/lib/cros_build_lib.py b/lib/cros_build_lib.py index a0cd73c6ed..71c908648f 100644 --- a/lib/cros_build_lib.py +++ b/lib/cros_build_lib.py @@ -4,11 +4,21 @@ """Common python commands used by various build scripts.""" +import inspect +import os import subprocess import sys _STDOUT_IS_TTY = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() +# TODO(sosa): Move logging to logging module. + +def GetCallerName(): + """Returns the name of the calling module with __main__.""" + top_frame = inspect.stack()[-1][0] + return os.path.basename(top_frame.f_code.co_filename) + + def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None, exit_code=False, redirect_stdout=False, redirect_stderr=False, cwd=None, input=None, enter_chroot=False): @@ -44,7 +54,8 @@ def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None, # Print out the command before running. if print_cmd: - Info('RunCommand: %s' % ' '.join(cmd)) + Info('PROGRAM(%s) -> RunCommand: %s in dir %s' % + (GetCallerName(), ' '.join(cmd), cwd)) try: proc = subprocess.Popen(cmd, cwd=cwd, stdin=stdin, @@ -56,7 +67,7 @@ def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None, if not error_ok and proc.returncode: raise Exception('Command "%s" failed.\n' % (' '.join(cmd)) + (error_message or error or output or '')) - except Exception,e: + except Exception, e: if not error_ok: raise else: diff --git a/lib/cros_vm_lib.sh b/lib/cros_vm_lib.sh index c47d11de33..1425f24d6c 100644 --- a/lib/cros_vm_lib.sh +++ b/lib/cros_vm_lib.sh @@ -4,8 +4,6 @@ # # Common vm functions for use in crosutils. -. "$(dirname "$0")/cros_vm_constants.sh" - DEFINE_string kvm_pid "" \ "Use this pid file. If it exists and is set, use the vm specified by pid." DEFINE_boolean no_graphics ${FLAGS_FALSE} "Runs the KVM instance silently." @@ -48,7 +46,7 @@ function start_kvm() { snapshot="-snapshot" fi - sudo kvm -m ${DEFAULT_MEM} \ + sudo kvm -m 1024 \ -vga std \ -pidfile "${KVM_PID_FILE}" \ -daemonize \ From bccc7c1d03fe3f6a8e11b65da79a48a5713df945 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Thu, 30 Sep 2010 15:06:15 -0700 Subject: [PATCH 11/14] Initial auto update engine test harness. This harness has support for test on vm machines. It takes a base image and target image path, creates an update-enabled vm image from the base image and performs updates using cros_run_vm_update (vm wrapper around image_to_live). It uses --snapshot for updating vm's and keeps them on (like a real machine would) so that the VM image is clean on the next test. The current check in has two test cases -- updates without clobbering the stateful partition and updates with clobbering the stateful partition. Change-Id: I5406eadb39fa36601d17f8dd15625bd6a8b38c15 BUG=6911 TEST=Ran through tests. Review URL: http://codereview.chromium.org/3554002 --- bin/cros_au_test_harness | 1 + bin/cros_au_test_harness.py | 170 ++++++++++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 120000 bin/cros_au_test_harness create mode 100755 bin/cros_au_test_harness.py diff --git a/bin/cros_au_test_harness b/bin/cros_au_test_harness new file mode 120000 index 0000000000..7c4176f6cb --- /dev/null +++ b/bin/cros_au_test_harness @@ -0,0 +1 @@ +cros_au_test_harness.py \ No newline at end of file diff --git a/bin/cros_au_test_harness.py b/bin/cros_au_test_harness.py new file mode 100755 index 0000000000..eaed1ef2b9 --- /dev/null +++ b/bin/cros_au_test_harness.py @@ -0,0 +1,170 @@ +#!/usr/bin/python + +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import optparse +import os +import sys +import unittest + +sys.path.append(os.path.join(os.path.dirname(__file__), '../lib')) +from cros_build_lib import RunCommand, Info, Warning + +_KVM_PID_FILE = '/tmp/harness_pid' +_SCRIPTS_DIR = os.path.join(os.path.dirname(__file__), '..') +_FULL_VDISK_SIZE = 6072 +_FULL_STATEFULFS_SIZE = 2048 + +global base_image_path +global target_image_path + +_VERIFY_SUITE = 'suite_Smoke' + +class AUTest(object): + """Abstract interface that defines an Auto Update test.""" + + def PrepareBase(self): + """Prepares target with base_image_path.""" + pass + + def UpdateImage(self, image_path, stateful_change='old'): + """Updates target with the image given by the image_path. + + Args: + image_path: Path to the image to update with. This image must be a test + image. + stateful_change: How to modify the stateful partition. Values are: + 'old': Don't modify stateful partition. Just update normally. + 'clean': Uses clobber-state to wipe the stateful partition with the + exception of code needed for ssh. + """ + pass + + def VerifyImage(self): + """Verifies the image is correct.""" + pass + + def testFullUpdateKeepStateful(self): + # Prepare and verify the base image has been prepared correctly. + self.PrepareBase() + self.VerifyImage() + + # Update to. + Info('Updating from base image on vm to target image.') + self.UpdateImage(target_image_path) + self.VerifyImage() + + # Update from. + Info('Updating from updated image on vm back to base image.') + self.UpdateImage(base_image_path) + self.VerifyImage() + + def testFullUpdateWipeStateful(self): + # Prepare and verify the base image has been prepared correctly. + self.PrepareBase() + self.VerifyImage() + + # Update to. + Info('Updating from base image on vm to target image and wiping stateful.') + self.UpdateImage(target_image_path, 'clean') + self.VerifyImage() + + # Update from. + Info('Updating from updated image back to base image and wiping stateful.') + self.UpdateImage(base_image_path, 'clean') + self.VerifyImage() + + +class VirtualAUTest(unittest.TestCase, AUTest): + """Test harness for updating virtual machines.""" + vm_image_path = None + + def _KillExistingVM(self, pid_file): + if os.path.exists(pid_file): + Warning('Existing %s found. Deleting and killing process' % + pid_file) + pid = RunCommand(['sudo', 'cat', pid_file], redirect_stdout=True, + enter_chroot=False) + if pid: + RunCommand(['sudo', 'kill', pid.strip()], error_ok=True, + enter_chroot=False) + RunCommand(['sudo', 'rm', pid_file], enter_chroot=False) + + def setUp(self): + """Unit test overriden method. Is called before every test.""" + + self._KillExistingVM(_KVM_PID_FILE) + + def PrepareBase(self): + """Creates an update-able VM based on base image.""" + + self.vm_image_path = ('%s/chromiumos_qemu_image.bin' % os.path.dirname( + base_image_path)) + if not os.path.exists(self.vm_image_path): + Info('Qemu image not found, creating one.') + RunCommand(['%s/image_to_vm.sh' % _SCRIPTS_DIR, + '--full', + '--from %s' % os.path.dirname(base_image_path), + '--vdisk_size %s' % _FULL_VDISK_SIZE, + '--statefulfs_size %s' % _FULL_STATEFULFS_SIZE, + '--test_image'], enter_chroot=True) + else: + Info('Using existing VM image') + + self.assertTrue(os.path.exists(self.vm_image_path)) + + def UpdateImage(self, image_path, stateful_change='old'): + """Updates VM image with image_path.""" + + stateful_change_flag = '' + if stateful_change: + stateful_change_flag = '--stateful_flags=%s' % stateful_change + + RunCommand(['%s/cros_run_vm_update' % os.path.dirname(__file__), + '--update_image_path=%s' % image_path, + '--vm_image_path=%s' % self.vm_image_path, + '--snapshot', + '--persist', + '--kvm_pid=%s' % _KVM_PID_FILE, + stateful_change_flag, + ], enter_chroot=False) + + def VerifyImage(self): + """Runs vm smoke suite to verify image.""" + + # image_to_live already verifies lsb-release matching. This is just + # for additional steps. + + # TODO(sosa): Compare output with results of base image. + RunCommand(['%s/cros_run_vm_test' % os.path.dirname(__file__), + '--image_path=%s' % self.vm_image_path, + '--snapshot', + '--persist', + '--kvm_pid=%s' % _KVM_PID_FILE, + '--test_case=%s' % _VERIFY_SUITE, + ], error_ok=True, enter_chroot=False) + + +if __name__ == '__main__': + parser = optparse.OptionParser() + parser.add_option('-b', '--base_image', + help='path to the base image.') + parser.add_option('-t', '--target_image', + help='path to the target image') + # Set the usage to include flags. + parser.set_usage(parser.format_help()) + # Parse existing sys.argv so we can pass rest to unittest.main. + (options, sys.argv) = parser.parse_args(sys.argv) + + base_image_path = options.base_image + target_image_path = options.target_image + + if not base_image_path: + parser.error('Need path to base image for vm.') + + if not target_image_path: + parser.error('Need path to target image to update with.') + + unittest.main() From cb9e4fe44948460346219c0873b4e9e44284d9a9 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Thu, 30 Sep 2010 18:02:09 -0700 Subject: [PATCH 12/14] Change hostname to localhost ip TBR Change-Id: I1a911da2df7cddda3eb35f6df23e203cd94c1ffe BUG= TEST=Locally. Review URL: http://codereview.chromium.org/3536003 --- bin/cros_run_vm_test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cros_run_vm_test b/bin/cros_run_vm_test index 8036b230b1..ee86879c78 100755 --- a/bin/cros_run_vm_test +++ b/bin/cros_run_vm_test @@ -34,5 +34,5 @@ trap stop_kvm EXIT start_kvm "${FLAGS_image_path}" "$(dirname $0)"/../run_remote_tests.sh \ --ssh_port=${FLAGS_ssh_port} \ - --remote="${HOSTNAME}" \ + --remote=127.0.0.1 \ "${FLAGS_test_case}" From 48d7d8bb7f55a8005c5d056eb0da515c337b455c Mon Sep 17 00:00:00 2001 From: Paul Stewart Date: Thu, 30 Sep 2010 18:16:21 -0700 Subject: [PATCH 13/14] Add sha256 to devserver clone updater output BUG=none TEST=none Review URL: http://codereview.chromium.org/3572006 --- bin/cros_image_to_target.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/bin/cros_image_to_target.py b/bin/cros_image_to_target.py index 9a4fd8ce49..dc6a0f1be5 100755 --- a/bin/cros_image_to_target.py +++ b/bin/cros_image_to_target.py @@ -219,6 +219,12 @@ class CrosEnv(object): infile=filename, capture=True, oneline=True) + def GetSha256(self, filename): + return self.cmd.RunPipe([['openssl', 'dgst', '-sha256', '-binary'], + ['openssl', 'base64']], + infile=filename, + capture=True, oneline=True) + def GetDefaultBoard(self): def_board_file = self.CrosUtilsPath('.default_board') if not os.path.exists(def_board_file): @@ -250,6 +256,7 @@ class CrosEnv(object): """Start the devserver clone.""" PingUpdateResponse.Setup(self.GetHash(update_file), + self.GetSha256(update_file), self.GetSize(update_file)) UpdateHandler.SetupUrl('/update', PingUpdateResponse()) @@ -423,6 +430,7 @@ class PingUpdateResponse(StringUpdateResponse): @@ -443,8 +451,9 @@ class PingUpdateResponse(StringUpdateResponse): self.content_type = 'text/xml' @staticmethod - def Setup(filehash, filesize): + def Setup(filehash, filesha256, filesize): PingUpdateResponse.file_hash = filehash + PingUpdateResponse.file_sha256 = filesha256 PingUpdateResponse.file_size = filesize def Reply(self, handler, send_content=True, post_data=None): @@ -462,7 +471,7 @@ class PingUpdateResponse(StringUpdateResponse): self.string = (self.payload_success_template % (self.xmlns, self.SecondsSinceMidnight(), self.app_id, 'http://%s/%s' % (host, UPDATE_FILENAME), - self.file_hash, self.file_size)) + self.file_hash, self.file_sha256, self.file_size)) else: self.string = (self.payload_failure_template % (self.xmlns, self.SecondsSinceMidnight(), self.app_id)) From 3da3113c01cba775dece1e4365549edfdfba7474 Mon Sep 17 00:00:00 2001 From: Paul Stewart Date: Fri, 1 Oct 2010 11:22:23 -0700 Subject: [PATCH 14/14] Remove cros_copy_upgrade_server.sh This script had a short painful life, constantly being brittle and broken. It is time for it to go. Please use cros_image_to_target.py instead. BUG=none TEST=none Review URL: http://codereview.chromium.org/3536002 --- bin/cros_copy_upgrade_server.sh | 223 -------------------------------- 1 file changed, 223 deletions(-) delete mode 100755 bin/cros_copy_upgrade_server.sh diff --git a/bin/cros_copy_upgrade_server.sh b/bin/cros_copy_upgrade_server.sh deleted file mode 100755 index a32056039e..0000000000 --- a/bin/cros_copy_upgrade_server.sh +++ /dev/null @@ -1,223 +0,0 @@ -#!/bin/sh - -# Copyright (c) 2009 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# Script to push the output of build_image.sh to a remote image server - -# TODO(pstew): Apparently the script files are in transition from -# src/scripts to src/scripts/bin. However this state has existed -# for months now, therefore we need to look for the common libs in -# both places -script_root=$(dirname $0) -if [ -f ${script_root}/../common.sh ] ; then - script_root=${script_root}/.. -fi - -. "${script_root}/common.sh" - -# Flags -DEFINE_string upgrade_server "" "SSH-capable host for upgrade server install" -DEFINE_string dest_path "" "Directory on host to do install" -DEFINE_string client_address "" "IP Address of netbook to update" -DEFINE_string server_address "" "IP Address of upgrade server" -DEFINE_string client_prefix "ChromeOSUpdateEngine" \ - "client_prefix arg to devserver. Old version is MementoSoftwareUpdate" -DEFINE_boolean old_prefix ${FLAGS_FALSE} "Use old MementoSoftwareUpdate" -DEFINE_boolean start_server ${FLAGS_TRUE} "Start up the server" -DEFINE_boolean stop_server ${FLAGS_FALSE} "Start up the server" -DEFINE_boolean no_copy_archive ${FLAGS_FALSE} "Skip copy of files to server" -DEFINE_string from "" "Image directory to upload to server" - -# Parse command line -FLAGS "$@" || exit 1 -eval set -- "${FLAGS_ARGV}" - -set -e - -# Make sure dev server argument has been set -require_upgrade_server () { - if [ -z "${FLAGS_upgrade_server}" ] ; then - echo "The --upgrade-server= argument is mandatory" - exit 1 - fi -} - -# Make sure a pointer to the latest image has been created -require_latest_image () { - [ -n "$latest_image" ] && return - if [ -n "${FLAGS_from}" ] ; then - latest_image=$(readlink -f ${FLAGS_from}) - else - latest_image=$(env CHROMEOS_BUILD_ROOT=${SCRIPTS_DIR}/../build \ - ${SCRIPTS_DIR}/get_latest_image.sh) - fi -} - -validate_devserver_path () { - if [ $(expr "${FLAGS_dest_path}" : '\.\.') != 0 ]; then - echo "Error: --dest_path argument (${FLAGS_dest_path}) must not be relative" - exit 1 - fi - FLAGS_dest_path=/tmp/devserver/${FLAGS_dest_path##/tmp/devserver/} -} - -# Copy the various bits of the dev server scripts over to our remote host -create_devserver () { - FLAGS_dest_path=$1 - validate_devserver_path - - echo "Creating dev server in ${FLAGS_upgrade_server}:${FLAGS_dest_path}..." - - require_upgrade_server - # Create new empty directory to hold server components - ssh "${FLAGS_upgrade_server}" rm -rf "${FLAGS_dest_path}" || true - ssh "${FLAGS_upgrade_server}" mkdir -p "${FLAGS_dest_path}/python" - - # Copy server components into place - (cd ${SCRIPTS_DIR}/../.. && \ - tar zcfh - --exclude=.git --exclude=.svn --exclude=pkgroot \ - src/scripts/lib \ - src/scripts/start_devserver \ - src/scripts/cros_generate_update_payload \ - src/scripts/chromeos-common.sh \ - src/scripts/{common,get_latest_image,mk_memento_images}.sh \ - src/platform/dev) | \ - ssh ${FLAGS_upgrade_server} "cd ${FLAGS_dest_path} && tar zxf -" - - # Copy Python web library into place out of the chroot - (cd ${SCRIPTS_DIR}/../../chroot/usr/lib/python*/site-packages && \ - tar zcf - web*) | \ - ssh ${FLAGS_upgrade_server} "cd ${FLAGS_dest_path}/python && tar zxf -" -} - -# Copy the latest image over to archive server -create_archive_dir () { - archive_dir=$1 - - echo "Creating archive dir in ${FLAGS_upgrade_server}:${archive_dir}..." - - require_upgrade_server - require_latest_image - - # Copy the latest image into the newly created archive - ssh "${FLAGS_upgrade_server}" "mkdir -p ${archive_dir}" - - image_path=${latest_image##*build/} - - (cd ${SCRIPTS_DIR}/../build && tar zcf - ${image_path}) | \ - ssh ${FLAGS_upgrade_server} "cd ${archive_dir} && tar zxf -" - - # unpack_partitions.sh lies in its hashbang. It really wants bash - unpack_script=${archive_dir}/${image_path}/unpack_partitions.sh - ssh ${FLAGS_upgrade_server} "sed -e 's/^#!\/bin\/sh/#!\/bin\/bash/' < ${unpack_script} > ${unpack_script}.new && chmod 755 ${unpack_script}.new && mv ${unpack_script}.new ${unpack_script}" - - # Since we are in static-only mode, we need to create a few links - for file in update.gz stateful.image.gz ; do - ssh ${FLAGS_upgrade_server} "cd ${archive_dir} && ln -sf ${image_path}/$file ." - ssh ${FLAGS_upgrade_server} "ln -sf ${archive_dir}/$file ${FLAGS_dest_path}/src/platform/dev/static" - done -} - -stop_server () { - require_upgrade_server - echo "Stopping remote devserver..." - echo "(Fast restart using \"$0 --upgrade_server=${FLAGS_upgrade_server} --dest_path=${FLAGS_dest_path} --no_copy_archive\")" - ssh ${FLAGS_upgrade_server} pkill -f ${archive_dir} || /bin/true -} - -# Start remote server -start_server () { - require_upgrade_server - echo "Starting remote devserver..." - server_logfile=/tmp/devserver_log.$$ - portlist=/tmp/devserver_portlist.$$ - echo "Server will be logging locally to $server_logfile" - - # Find a TCP listen socket that is not in use - ssh ${FLAGS_upgrade_server} "netstat -lnt" | awk '{ print $4 }' > $portlist - server_port=8081 - while grep -q ":${server_port}$" $portlist; do - server_port=$[server_port + 1] - done - rm -f $portlist - - if [ "${FLAGS_old_prefix}" -eq ${FLAGS_TRUE} ] ; then - FLAGS_client_prefix=MementoSoftwareUpdate - fi - - ssh ${FLAGS_upgrade_server} "cd ${FLAGS_dest_path}/src/scripts && env PYTHONPATH=${remote_root}${FLAGS_dest_path}/python CHROMEOS_BUILD_ROOT=${archive_dir} ./start_devserver --archive_dir ${archive_dir} --client_prefix ${FLAGS_client_prefix} $server_port" > $server_logfile 2>&1 & - server_pid=$! - - trap server_cleanup 2 - - # Wait for server to startup - while sleep 1; do - if fgrep -q 'Serving images from' $server_logfile; then - echo "Server is ready" - break - elif kill -0 ${server_pid}; then - continue - else - echo "Server failed to startup" - exit 1 - fi - done -} - -server_cleanup () { - trap '' 2 - stop_server - exit 0 -} - -# If destination path wasn't set on command line, create one from scratch -if [ -z "${FLAGS_dest_path}" -a ${FLAGS_stop_server} -eq ${FLAGS_FALSE} ] ; then - require_latest_image - hostname=$(uname -n) - hostname=${hostname%%.*} - image_name=${latest_image##*/} - create_devserver ${hostname}_${image_name} - FLAGS_start_server=${FLAGS_TRUE} -else - validate_devserver_path -fi - -if [ ${FLAGS_stop_server} -eq ${FLAGS_FALSE} -a \ - ${FLAGS_no_copy_archive} -eq ${FLAGS_FALSE} ] ; then - create_archive_dir "${FLAGS_dest_path}/archive" - FLAGS_start_server=${FLAGS_TRUE} -else - archive_dir="${FLAGS_dest_path}/archive" -fi - -if [ "${FLAGS_stop_server}" -eq ${FLAGS_TRUE} ] ; then - stop_server - exit 0 -fi - -# Make sure old devserver is dead, then restart it -if [ "${FLAGS_start_server}" -eq ${FLAGS_TRUE} ] ; then - stop_server - start_server - - tail -f ${server_logfile} & - - # Now tell the client to load from the server - if [ -z "${FLAGS_server_address}" ] ; then - FLAGS_server_address=${FLAGS_upgrade_server} - fi - live_args="--update_url=http://${FLAGS_server_address}:${server_port}/update \ - --remote=${FLAGS_client_address}" - if [ -n "${FLAGS_client_address}" ] ; then - echo "Running ${SCRIPTS_DIR}/image_to_live.sh $live_args" - ${SCRIPTS_DIR}/image_to_live.sh $live_args & - else - echo "Start client upgrade using:" - echo " ${SCRIPTS_DIR}/image_to_live.sh ${live_args}" - fi - - wait ${server_pid} -fi -