diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index a077c115f1..794bf15013 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -208,6 +208,20 @@ def _UprevAllPackages(buildroot): '--tracking_branch="cros/master"'], cwd=cwd, enter_chroot=True) + +def _GetVMConstants(buildroot): + """Returns minimum (vdisk_size, statefulfs_size) recommended for VM's.""" + cwd = os.path.join(buildroot, 'src', 'scripts', 'lib') + source_cmd = 'source %s/cros_vm_constants.sh' % cwd + vdisk_size = RunCommand([ + '/bin/bash', '-c', '%s && echo $MIN_VDISK_SIZE_FULL' % source_cmd], + redirect_stdout=True) + statefulfs_size = RunCommand([ + '/bin/bash', '-c', '%s && echo $MIN_STATEFUL_FS_SIZE_FULL' % source_cmd], + redirect_stdout=True) + return (vdisk_size.strip(), statefulfs_size.strip()) + + # =========================== Main Commands =================================== def _FullCheckout(buildroot, rw_checkout=True, retries=_DEFAULT_RETRIES): @@ -244,20 +258,43 @@ def _Build(buildroot): cwd = os.path.join(buildroot, 'src', 'scripts') RunCommand(['./build_packages'], cwd=cwd, enter_chroot=True) + def _WipeOldOutput(buildroot): RunCommand(['rm', '-rf', 'src/build/images'], cwd=buildroot) + def _BuildImage(buildroot): _WipeOldOutput(buildroot) cwd = os.path.join(buildroot, 'src', 'scripts') RunCommand(['./build_image', '--replace'], cwd=cwd, enter_chroot=True) + +def _BuildVMImageForTesting(buildroot): + (vdisk_size, statefulfs_size) = _GetVMConstants(buildroot) + cwd = os.path.join(buildroot, 'src', 'scripts') + RunCommand(['./image_to_vm.sh', + '--test_image', + '--full', + '--vdisk_size %s' % vdisk_size, + '--statefulfs_size %s' % statefulfs_size, + ], cwd=cwd, enter_chroot=True) + + def _RunUnitTests(buildroot): cwd = os.path.join(buildroot, 'src', 'scripts') RunCommand(['./cros_run_unit_tests'], cwd=cwd, enter_chroot=True) +def _RunSmokeSuite(buildroot): + cwd = os.path.join(buildroot, 'src', 'scripts') + RunCommand(['bin/cros_run_vm_test', + '--no_graphics', + '--test_case', + 'suite_Smoke', + ], cwd=cwd, error_ok=True) + + def _UprevPackages(buildroot, revisionfile, board): """Uprevs a package based on given revisionfile. @@ -383,6 +420,11 @@ def main(): _RunUnitTests(buildroot) _BuildImage(buildroot) + + if buildconfig['smoke_bvt']: + _BuildVMImageForTesting(buildroot) + _RunSmokeSuite(buildroot) + if buildconfig['uprev']: if buildconfig['master']: # Master bot needs to check if the other slaves completed. diff --git a/bin/cbuildbot_config.py b/bin/cbuildbot_config.py index 1ba08d51d0..dda1ef8a50 100644 --- a/bin/cbuildbot_config.py +++ b/bin/cbuildbot_config.py @@ -18,6 +18,7 @@ important -- Master bot uses important bots to determine overall status. hostname -- Needed for 'important' slaves. The hostname of the bot. Should match hostname in slaves.cfg in buildbot checkout. unittests -- Runs unittests for packages. +smoke_bvt -- Runs the test smoke suite in a qemu-based VM using KVM. """ @@ -29,6 +30,7 @@ config['default'] = { 'master' : False, 'important' : False, 'unittests' : False, + 'smoke_bvt' : False, } config['x86-generic-pre-flight-queue'] = { 'board' : 'x86-generic', @@ -37,6 +39,7 @@ config['x86-generic-pre-flight-queue'] = { 'important' : False, 'hostname' : 'chromeosbuild2', 'unittests' : True, + 'smoke_bvt' : True, } config['x86_pineview_bin'] = { 'board' : 'x86-pineview', @@ -45,6 +48,7 @@ config['x86_pineview_bin'] = { 'important' : False, 'hostname' : 'codf200.jail', 'unittests': True, + 'smoke_bvt' : True, } config['arm_tegra2_bin'] = { 'board' : 'tegra2', diff --git a/bin/cbuildbot_unittest.py b/bin/cbuildbot_unittest.py index 203f2a8c76..98cbe83baa 100755 --- a/bin/cbuildbot_unittest.py +++ b/bin/cbuildbot_unittest.py @@ -76,10 +76,35 @@ class CBuildBotTest(mox.MoxTestBase): ['chromeos-base/chromeos-login']) self.mox.VerifyAll() + # TODO(sosa): Re-add once we use cros_mark vs. cros_mark_all. + #def testUprevPackages(self): + # """Test if we get actual revisions in revisions.pfq.""" + # self.mox.StubOutWithMock(cbuildbot, '_CreateRepoDictionary') + # self.mox.StubOutWithMock(cbuildbot, '_ParseRevisionString') + # self.mox.StubOutWithMock(cbuildbot, '_UprevFromRevisionList') + # self.mox.StubOutWithMock(__builtin__, 'open') + + # # Mock out file interaction. + # m_file = self.mox.CreateMock(file) + # __builtin__.open(self._revision_file).AndReturn(m_file) + # m_file.read().AndReturn(self._test_string) + # m_file.close() + + # cbuildbot._CreateRepoDictionary(self._buildroot, + # self._test_board).AndReturn(self._test_dict) + # cbuildbot._ParseRevisionString(self._test_string, + # self._test_dict).AndReturn( + # self._test_parsed_string_array) + # cbuildbot._UprevFromRevisionList(self._buildroot, + # self._test_parsed_string_array) + # self.mox.ReplayAll() + # cbuildbot._UprevPackages(self._buildroot, self._revision_file, + # self._test_board) + # self.mox.VerifyAll() + + # TODO(sosa): Remove once we un-comment above. def testUprevPackages(self): - self.mox.StubOutWithMock(cbuildbot, '_CreateRepoDictionary') - self.mox.StubOutWithMock(cbuildbot, '_ParseRevisionString') - self.mox.StubOutWithMock(cbuildbot, '_UprevFromRevisionList') + """Test if we get actual revisions in revisions.pfq.""" self.mox.StubOutWithMock(__builtin__, 'open') # Mock out file interaction. @@ -88,13 +113,31 @@ class CBuildBotTest(mox.MoxTestBase): m_file.read().AndReturn(self._test_string) m_file.close() - cbuildbot._CreateRepoDictionary(self._buildroot, - self._test_board).AndReturn(self._test_dict) - cbuildbot._ParseRevisionString(self._test_string, - self._test_dict).AndReturn( - self._test_parsed_string_array) - cbuildbot._UprevFromRevisionList(self._buildroot, - self._test_parsed_string_array) + cbuildbot.RunCommand(['./cros_mark_all_as_stable', + '--tracking_branch="cros/master"'], + cwd='%s/src/scripts' % self._buildroot, + enter_chroot=True) + + self.mox.ReplayAll() + cbuildbot._UprevPackages(self._buildroot, self._revision_file, + self._test_board) + self.mox.VerifyAll() + + def testUprevAllPackages(self): + """Test if we get None in revisions.pfq indicating Full Builds.""" + self.mox.StubOutWithMock(__builtin__, 'open') + + # Mock out file interaction. + m_file = self.mox.CreateMock(file) + __builtin__.open(self._revision_file).AndReturn(m_file) + m_file.read().AndReturn('None') + m_file.close() + + cbuildbot.RunCommand(['./cros_mark_all_as_stable', + '--tracking_branch="cros/master"'], + cwd='%s/src/scripts' % self._buildroot, + enter_chroot=True) + self.mox.ReplayAll() cbuildbot._UprevPackages(self._buildroot, self._revision_file, self._test_board) diff --git a/bin/cros_au_test_harness b/bin/cros_au_test_harness new file mode 120000 index 0000000000..7c4176f6cb --- /dev/null +++ b/bin/cros_au_test_harness @@ -0,0 +1 @@ +cros_au_test_harness.py \ No newline at end of file diff --git a/bin/cros_au_test_harness.py b/bin/cros_au_test_harness.py new file mode 100755 index 0000000000..eaed1ef2b9 --- /dev/null +++ b/bin/cros_au_test_harness.py @@ -0,0 +1,170 @@ +#!/usr/bin/python + +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import optparse +import os +import sys +import unittest + +sys.path.append(os.path.join(os.path.dirname(__file__), '../lib')) +from cros_build_lib import RunCommand, Info, Warning + +_KVM_PID_FILE = '/tmp/harness_pid' +_SCRIPTS_DIR = os.path.join(os.path.dirname(__file__), '..') +_FULL_VDISK_SIZE = 6072 +_FULL_STATEFULFS_SIZE = 2048 + +global base_image_path +global target_image_path + +_VERIFY_SUITE = 'suite_Smoke' + +class AUTest(object): + """Abstract interface that defines an Auto Update test.""" + + def PrepareBase(self): + """Prepares target with base_image_path.""" + pass + + def UpdateImage(self, image_path, stateful_change='old'): + """Updates target with the image given by the image_path. + + Args: + image_path: Path to the image to update with. This image must be a test + image. + stateful_change: How to modify the stateful partition. Values are: + 'old': Don't modify stateful partition. Just update normally. + 'clean': Uses clobber-state to wipe the stateful partition with the + exception of code needed for ssh. + """ + pass + + def VerifyImage(self): + """Verifies the image is correct.""" + pass + + def testFullUpdateKeepStateful(self): + # Prepare and verify the base image has been prepared correctly. + self.PrepareBase() + self.VerifyImage() + + # Update to. + Info('Updating from base image on vm to target image.') + self.UpdateImage(target_image_path) + self.VerifyImage() + + # Update from. + Info('Updating from updated image on vm back to base image.') + self.UpdateImage(base_image_path) + self.VerifyImage() + + def testFullUpdateWipeStateful(self): + # Prepare and verify the base image has been prepared correctly. + self.PrepareBase() + self.VerifyImage() + + # Update to. + Info('Updating from base image on vm to target image and wiping stateful.') + self.UpdateImage(target_image_path, 'clean') + self.VerifyImage() + + # Update from. + Info('Updating from updated image back to base image and wiping stateful.') + self.UpdateImage(base_image_path, 'clean') + self.VerifyImage() + + +class VirtualAUTest(unittest.TestCase, AUTest): + """Test harness for updating virtual machines.""" + vm_image_path = None + + def _KillExistingVM(self, pid_file): + if os.path.exists(pid_file): + Warning('Existing %s found. Deleting and killing process' % + pid_file) + pid = RunCommand(['sudo', 'cat', pid_file], redirect_stdout=True, + enter_chroot=False) + if pid: + RunCommand(['sudo', 'kill', pid.strip()], error_ok=True, + enter_chroot=False) + RunCommand(['sudo', 'rm', pid_file], enter_chroot=False) + + def setUp(self): + """Unit test overriden method. Is called before every test.""" + + self._KillExistingVM(_KVM_PID_FILE) + + def PrepareBase(self): + """Creates an update-able VM based on base image.""" + + self.vm_image_path = ('%s/chromiumos_qemu_image.bin' % os.path.dirname( + base_image_path)) + if not os.path.exists(self.vm_image_path): + Info('Qemu image not found, creating one.') + RunCommand(['%s/image_to_vm.sh' % _SCRIPTS_DIR, + '--full', + '--from %s' % os.path.dirname(base_image_path), + '--vdisk_size %s' % _FULL_VDISK_SIZE, + '--statefulfs_size %s' % _FULL_STATEFULFS_SIZE, + '--test_image'], enter_chroot=True) + else: + Info('Using existing VM image') + + self.assertTrue(os.path.exists(self.vm_image_path)) + + def UpdateImage(self, image_path, stateful_change='old'): + """Updates VM image with image_path.""" + + stateful_change_flag = '' + if stateful_change: + stateful_change_flag = '--stateful_flags=%s' % stateful_change + + RunCommand(['%s/cros_run_vm_update' % os.path.dirname(__file__), + '--update_image_path=%s' % image_path, + '--vm_image_path=%s' % self.vm_image_path, + '--snapshot', + '--persist', + '--kvm_pid=%s' % _KVM_PID_FILE, + stateful_change_flag, + ], enter_chroot=False) + + def VerifyImage(self): + """Runs vm smoke suite to verify image.""" + + # image_to_live already verifies lsb-release matching. This is just + # for additional steps. + + # TODO(sosa): Compare output with results of base image. + RunCommand(['%s/cros_run_vm_test' % os.path.dirname(__file__), + '--image_path=%s' % self.vm_image_path, + '--snapshot', + '--persist', + '--kvm_pid=%s' % _KVM_PID_FILE, + '--test_case=%s' % _VERIFY_SUITE, + ], error_ok=True, enter_chroot=False) + + +if __name__ == '__main__': + parser = optparse.OptionParser() + parser.add_option('-b', '--base_image', + help='path to the base image.') + parser.add_option('-t', '--target_image', + help='path to the target image') + # Set the usage to include flags. + parser.set_usage(parser.format_help()) + # Parse existing sys.argv so we can pass rest to unittest.main. + (options, sys.argv) = parser.parse_args(sys.argv) + + base_image_path = options.base_image + target_image_path = options.target_image + + if not base_image_path: + parser.error('Need path to base image for vm.') + + if not target_image_path: + parser.error('Need path to target image to update with.') + + unittest.main() diff --git a/bin/cros_copy_upgrade_server.sh b/bin/cros_copy_upgrade_server.sh deleted file mode 100755 index a32056039e..0000000000 --- a/bin/cros_copy_upgrade_server.sh +++ /dev/null @@ -1,223 +0,0 @@ -#!/bin/sh - -# Copyright (c) 2009 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# Script to push the output of build_image.sh to a remote image server - -# TODO(pstew): Apparently the script files are in transition from -# src/scripts to src/scripts/bin. However this state has existed -# for months now, therefore we need to look for the common libs in -# both places -script_root=$(dirname $0) -if [ -f ${script_root}/../common.sh ] ; then - script_root=${script_root}/.. -fi - -. "${script_root}/common.sh" - -# Flags -DEFINE_string upgrade_server "" "SSH-capable host for upgrade server install" -DEFINE_string dest_path "" "Directory on host to do install" -DEFINE_string client_address "" "IP Address of netbook to update" -DEFINE_string server_address "" "IP Address of upgrade server" -DEFINE_string client_prefix "ChromeOSUpdateEngine" \ - "client_prefix arg to devserver. Old version is MementoSoftwareUpdate" -DEFINE_boolean old_prefix ${FLAGS_FALSE} "Use old MementoSoftwareUpdate" -DEFINE_boolean start_server ${FLAGS_TRUE} "Start up the server" -DEFINE_boolean stop_server ${FLAGS_FALSE} "Start up the server" -DEFINE_boolean no_copy_archive ${FLAGS_FALSE} "Skip copy of files to server" -DEFINE_string from "" "Image directory to upload to server" - -# Parse command line -FLAGS "$@" || exit 1 -eval set -- "${FLAGS_ARGV}" - -set -e - -# Make sure dev server argument has been set -require_upgrade_server () { - if [ -z "${FLAGS_upgrade_server}" ] ; then - echo "The --upgrade-server= argument is mandatory" - exit 1 - fi -} - -# Make sure a pointer to the latest image has been created -require_latest_image () { - [ -n "$latest_image" ] && return - if [ -n "${FLAGS_from}" ] ; then - latest_image=$(readlink -f ${FLAGS_from}) - else - latest_image=$(env CHROMEOS_BUILD_ROOT=${SCRIPTS_DIR}/../build \ - ${SCRIPTS_DIR}/get_latest_image.sh) - fi -} - -validate_devserver_path () { - if [ $(expr "${FLAGS_dest_path}" : '\.\.') != 0 ]; then - echo "Error: --dest_path argument (${FLAGS_dest_path}) must not be relative" - exit 1 - fi - FLAGS_dest_path=/tmp/devserver/${FLAGS_dest_path##/tmp/devserver/} -} - -# Copy the various bits of the dev server scripts over to our remote host -create_devserver () { - FLAGS_dest_path=$1 - validate_devserver_path - - echo "Creating dev server in ${FLAGS_upgrade_server}:${FLAGS_dest_path}..." - - require_upgrade_server - # Create new empty directory to hold server components - ssh "${FLAGS_upgrade_server}" rm -rf "${FLAGS_dest_path}" || true - ssh "${FLAGS_upgrade_server}" mkdir -p "${FLAGS_dest_path}/python" - - # Copy server components into place - (cd ${SCRIPTS_DIR}/../.. && \ - tar zcfh - --exclude=.git --exclude=.svn --exclude=pkgroot \ - src/scripts/lib \ - src/scripts/start_devserver \ - src/scripts/cros_generate_update_payload \ - src/scripts/chromeos-common.sh \ - src/scripts/{common,get_latest_image,mk_memento_images}.sh \ - src/platform/dev) | \ - ssh ${FLAGS_upgrade_server} "cd ${FLAGS_dest_path} && tar zxf -" - - # Copy Python web library into place out of the chroot - (cd ${SCRIPTS_DIR}/../../chroot/usr/lib/python*/site-packages && \ - tar zcf - web*) | \ - ssh ${FLAGS_upgrade_server} "cd ${FLAGS_dest_path}/python && tar zxf -" -} - -# Copy the latest image over to archive server -create_archive_dir () { - archive_dir=$1 - - echo "Creating archive dir in ${FLAGS_upgrade_server}:${archive_dir}..." - - require_upgrade_server - require_latest_image - - # Copy the latest image into the newly created archive - ssh "${FLAGS_upgrade_server}" "mkdir -p ${archive_dir}" - - image_path=${latest_image##*build/} - - (cd ${SCRIPTS_DIR}/../build && tar zcf - ${image_path}) | \ - ssh ${FLAGS_upgrade_server} "cd ${archive_dir} && tar zxf -" - - # unpack_partitions.sh lies in its hashbang. It really wants bash - unpack_script=${archive_dir}/${image_path}/unpack_partitions.sh - ssh ${FLAGS_upgrade_server} "sed -e 's/^#!\/bin\/sh/#!\/bin\/bash/' < ${unpack_script} > ${unpack_script}.new && chmod 755 ${unpack_script}.new && mv ${unpack_script}.new ${unpack_script}" - - # Since we are in static-only mode, we need to create a few links - for file in update.gz stateful.image.gz ; do - ssh ${FLAGS_upgrade_server} "cd ${archive_dir} && ln -sf ${image_path}/$file ." - ssh ${FLAGS_upgrade_server} "ln -sf ${archive_dir}/$file ${FLAGS_dest_path}/src/platform/dev/static" - done -} - -stop_server () { - require_upgrade_server - echo "Stopping remote devserver..." - echo "(Fast restart using \"$0 --upgrade_server=${FLAGS_upgrade_server} --dest_path=${FLAGS_dest_path} --no_copy_archive\")" - ssh ${FLAGS_upgrade_server} pkill -f ${archive_dir} || /bin/true -} - -# Start remote server -start_server () { - require_upgrade_server - echo "Starting remote devserver..." - server_logfile=/tmp/devserver_log.$$ - portlist=/tmp/devserver_portlist.$$ - echo "Server will be logging locally to $server_logfile" - - # Find a TCP listen socket that is not in use - ssh ${FLAGS_upgrade_server} "netstat -lnt" | awk '{ print $4 }' > $portlist - server_port=8081 - while grep -q ":${server_port}$" $portlist; do - server_port=$[server_port + 1] - done - rm -f $portlist - - if [ "${FLAGS_old_prefix}" -eq ${FLAGS_TRUE} ] ; then - FLAGS_client_prefix=MementoSoftwareUpdate - fi - - ssh ${FLAGS_upgrade_server} "cd ${FLAGS_dest_path}/src/scripts && env PYTHONPATH=${remote_root}${FLAGS_dest_path}/python CHROMEOS_BUILD_ROOT=${archive_dir} ./start_devserver --archive_dir ${archive_dir} --client_prefix ${FLAGS_client_prefix} $server_port" > $server_logfile 2>&1 & - server_pid=$! - - trap server_cleanup 2 - - # Wait for server to startup - while sleep 1; do - if fgrep -q 'Serving images from' $server_logfile; then - echo "Server is ready" - break - elif kill -0 ${server_pid}; then - continue - else - echo "Server failed to startup" - exit 1 - fi - done -} - -server_cleanup () { - trap '' 2 - stop_server - exit 0 -} - -# If destination path wasn't set on command line, create one from scratch -if [ -z "${FLAGS_dest_path}" -a ${FLAGS_stop_server} -eq ${FLAGS_FALSE} ] ; then - require_latest_image - hostname=$(uname -n) - hostname=${hostname%%.*} - image_name=${latest_image##*/} - create_devserver ${hostname}_${image_name} - FLAGS_start_server=${FLAGS_TRUE} -else - validate_devserver_path -fi - -if [ ${FLAGS_stop_server} -eq ${FLAGS_FALSE} -a \ - ${FLAGS_no_copy_archive} -eq ${FLAGS_FALSE} ] ; then - create_archive_dir "${FLAGS_dest_path}/archive" - FLAGS_start_server=${FLAGS_TRUE} -else - archive_dir="${FLAGS_dest_path}/archive" -fi - -if [ "${FLAGS_stop_server}" -eq ${FLAGS_TRUE} ] ; then - stop_server - exit 0 -fi - -# Make sure old devserver is dead, then restart it -if [ "${FLAGS_start_server}" -eq ${FLAGS_TRUE} ] ; then - stop_server - start_server - - tail -f ${server_logfile} & - - # Now tell the client to load from the server - if [ -z "${FLAGS_server_address}" ] ; then - FLAGS_server_address=${FLAGS_upgrade_server} - fi - live_args="--update_url=http://${FLAGS_server_address}:${server_port}/update \ - --remote=${FLAGS_client_address}" - if [ -n "${FLAGS_client_address}" ] ; then - echo "Running ${SCRIPTS_DIR}/image_to_live.sh $live_args" - ${SCRIPTS_DIR}/image_to_live.sh $live_args & - else - echo "Start client upgrade using:" - echo " ${SCRIPTS_DIR}/image_to_live.sh ${live_args}" - fi - - wait ${server_pid} -fi - diff --git a/bin/cros_image_to_target.py b/bin/cros_image_to_target.py index 9a4fd8ce49..dc6a0f1be5 100755 --- a/bin/cros_image_to_target.py +++ b/bin/cros_image_to_target.py @@ -219,6 +219,12 @@ class CrosEnv(object): infile=filename, capture=True, oneline=True) + def GetSha256(self, filename): + return self.cmd.RunPipe([['openssl', 'dgst', '-sha256', '-binary'], + ['openssl', 'base64']], + infile=filename, + capture=True, oneline=True) + def GetDefaultBoard(self): def_board_file = self.CrosUtilsPath('.default_board') if not os.path.exists(def_board_file): @@ -250,6 +256,7 @@ class CrosEnv(object): """Start the devserver clone.""" PingUpdateResponse.Setup(self.GetHash(update_file), + self.GetSha256(update_file), self.GetSize(update_file)) UpdateHandler.SetupUrl('/update', PingUpdateResponse()) @@ -423,6 +430,7 @@ class PingUpdateResponse(StringUpdateResponse): @@ -443,8 +451,9 @@ class PingUpdateResponse(StringUpdateResponse): self.content_type = 'text/xml' @staticmethod - def Setup(filehash, filesize): + def Setup(filehash, filesha256, filesize): PingUpdateResponse.file_hash = filehash + PingUpdateResponse.file_sha256 = filesha256 PingUpdateResponse.file_size = filesize def Reply(self, handler, send_content=True, post_data=None): @@ -462,7 +471,7 @@ class PingUpdateResponse(StringUpdateResponse): self.string = (self.payload_success_template % (self.xmlns, self.SecondsSinceMidnight(), self.app_id, 'http://%s/%s' % (host, UPDATE_FILENAME), - self.file_hash, self.file_size)) + self.file_hash, self.file_sha256, self.file_size)) else: self.string = (self.payload_failure_template % (self.xmlns, self.SecondsSinceMidnight(), self.app_id)) diff --git a/bin/cros_make_image_bootable b/bin/cros_make_image_bootable index 8aacf8c913..20c21911d6 100755 --- a/bin/cros_make_image_bootable +++ b/bin/cros_make_image_bootable @@ -86,9 +86,6 @@ DEFINE_integer verity_max_ios 1024 \ DEFINE_string verity_algorithm "sha1" \ "Cryptographic hash algorithm used for kernel vboot." -DEFINE_string arm_extra_bootargs "" \ - "Additional command line options to pass to the ARM kernel." - DEFINE_string keys_dir "/usr/share/vboot/devkeys" \ "Directory containing the signing keys." @@ -199,8 +196,7 @@ make_image_bootable() { sudo dd if="${FLAGS_output_dir}/vmlinuz.image" of="${image}" \ conv=notrunc bs=512 seek=${koffset} - # Update the bootloaders. For legacy/efi x86, the EFI system partition - # will be updated and for arm, the mbr will be updated (for u-boot). + # Update the bootloaders. The EFI system partition will be updated. local kernel_part= local usb_disk="${FLAGS_usb_disk}" @@ -218,17 +214,8 @@ make_image_bootable() { # Install syslinux on the EFI System Partition. kernel_part="${kernel_part} --install_syslinux" elif [[ "${FLAGS_arch}" = "arm" ]]; then - # TODO(wad) mmcblk1p3 is hardcoded for arm for now! - usb_disk="/dev/mmcblk1p3" - # ARM doesn't support using the kernel image for kernel cmdline flags yet. - kernel_part="--kernel_cmdline=\"${FLAGS_arm_extra_bootargs}\" " - # TODO(wad) Integrate dmtable extraction into the arm build - # E.g. $(cat ${FLAGS_output_dir}/boot.config | tr -s '\n' ' ')" - local kpart_offset="--kernel_partition_offset=${koffset}" - local kpart_size="--kernel_partition_sectors=" - kpart_size="${kpart_size}$(partsize ${image} 2)" - kernel_part="${kernel_part} ${kpart_size} ${kpart_offset}" - info "Using addition bootloader arguments: ${kernel_part}" + # These flags are not used for ARM update_bootloaders.sh + kernel_part="" fi # Update partition 12 diff --git a/bin/cros_run_vm_test b/bin/cros_run_vm_test index 6981745ac4..ee86879c78 100755 --- a/bin/cros_run_vm_test +++ b/bin/cros_run_vm_test @@ -8,6 +8,7 @@ . "$(dirname $0)/../common.sh" . "$(dirname $0)/../lib/cros_vm_lib.sh" +. "$(dirname "$0")/../lib/cros_vm_constants.sh" DEFINE_string image_path "" "Full path of the VM image" DEFINE_string test_case "" "Name of the test case to run" @@ -18,12 +19,20 @@ set -e FLAGS "$@" || exit 1 eval set -- "${FLAGS_ARGV}" -[ -n "${FLAGS_image_path}" ] || die "You must specify a path to an image" -[ -n "${FLAGS_test_case}" ] || die "You must specify a test case" +# Use latest if not specified. +if [ -z "${FLAGS_image_path}" ]; then + LATEST_IMAGE="$(${SCRIPTS_DIR}/get_latest_image.sh)/${DEFAULT_QEMU_IMAGE}" + info "Using latest vm image ${LATEST_IMAGE}" + FLAGS_image_path=${LATEST_IMAGE} +fi + +[ -e "${FLAGS_image_path}" ] || die "Image ${FLAGS_image_path} does not exist." + +[ -n "${FLAGS_test_case}" ] || die "You must specify a test case." trap stop_kvm EXIT start_kvm "${FLAGS_image_path}" "$(dirname $0)"/../run_remote_tests.sh \ --ssh_port=${FLAGS_ssh_port} \ - --remote="${HOSTNAME}" \ + --remote=127.0.0.1 \ "${FLAGS_test_case}" diff --git a/bin/cros_run_vm_update b/bin/cros_run_vm_update index eb4da2feb7..ea1ae5e9b9 100755 --- a/bin/cros_run_vm_update +++ b/bin/cros_run_vm_update @@ -31,5 +31,6 @@ $(dirname $0)/../image_to_live.sh \ --remote=${HOSTNAME} \ --ssh_port=${FLAGS_ssh_port} \ --stateful_update_flag=${stateful_flags} \ + --verify \ --image=$(readlink -f ${FLAGS_update_image_path}) diff --git a/build_gpt.sh b/build_gpt.sh index 84888a902c..df0f4fc070 100755 --- a/build_gpt.sh +++ b/build_gpt.sh @@ -21,8 +21,6 @@ DEFINE_string arch "" \ "The target architecture (\"arm\" or \"x86\")." DEFINE_string board "$DEFAULT_BOARD" \ "The board to build an image for." -DEFINE_string arm_extra_bootargs "" \ - "Additional command line options to pass to the ARM kernel." DEFINE_integer rootfs_partition_size 1024 \ "rootfs parition size in MBs." @@ -109,8 +107,7 @@ if [ ! -s ${ESP_IMG} ]; then exit 1 fi -# We'll need some code to put in the PMBR, for booting on legacy BIOS. Some ARM -# systems will use a U-Boot script temporarily, but it goes in the same place. +# We'll need some code to put in the PMBR, for booting on legacy BIOS. if [[ "$ARCH" = "arm" ]]; then PMBRCODE=/dev/zero else @@ -122,18 +119,6 @@ fi install_gpt $OUTDEV $(numsectors $ROOTFS_IMG) $(numsectors $STATEFUL_IMG) \ $PMBRCODE $(numsectors $ESP_IMG) false $FLAGS_rootfs_partition_size -if [[ "$ARCH" = "arm" ]]; then - # assume /dev/mmcblk1. we could not get this from ${OUTDEV} - DEVICE=1 - MBR_SCRIPT_UIMG=$(make_arm_mbr \ - ${START_KERN_A} \ - ${NUM_KERN_SECTORS} \ - ${DEVICE} \ - "${FLAGS_arm_extra_bootargs}") - sudo dd bs=1 count=`stat --printf="%s" ${MBR_SCRIPT_UIMG}` \ - if="$MBR_SCRIPT_UIMG" of=${OUTDEV} conv=notrunc -fi - # Emit helpful scripts for testers, etc. ${SCRIPTS_DIR}/emit_gpt_scripts.sh "${OUTDEV}" "${IMAGEDIR}" diff --git a/build_image b/build_image index 7620d0af7f..2df9a3cc05 100755 --- a/build_image +++ b/build_image @@ -49,8 +49,6 @@ DEFINE_boolean factory_install ${FLAGS_FALSE} \ is also required in image_to_usb." DEFINE_boolean dev_install ${FLAGS_FALSE} \ "Build a smaller image to overlay the dev recovery install shim on" -DEFINE_string arm_extra_bootargs "" \ - "Additional command line options to pass to the ARM kernel." DEFINE_integer rootfs_partition_size 1024 \ "rootfs partition size in MiBs." DEFINE_integer rootfs_size 720 \ @@ -360,7 +358,6 @@ create_boot_desc() { --verity_algorithm="${FLAGS_verity_algorithm}" --keys_dir="${DEVKEYSDIR}" --usb_disk="${FLAGS_usb_disk}" - --arm_extra_bootargs="${FLAGS_arm_extra_bootargs}" --nocleanup_dirs ${enable_rootfs_verification_flag} EOF @@ -637,8 +634,7 @@ create_base_image() { # cros_make_image_bootable will clobber vmlinuz.image for x86. # Until then, just copy the kernel to vmlinuz.image. It is - # expected in build_gpt.sh and needed by ARM until it supports the - # full, signed kernel partition format. + # expected in build_gpt.sh. cp "${ROOT_FS_DIR}/boot/vmlinuz" "${OUTPUT_DIR}/vmlinuz.image" # Create an empty esp image to be updated in by update_bootloaders.sh. @@ -652,7 +648,6 @@ create_base_image() { ${SCRIPTS_DIR}/build_gpt.sh \ --arch=${ARCH} \ --board=${FLAGS_board} \ - --arm_extra_bootargs="${FLAGS_arm_extra_bootargs}" \ --rootfs_partition_size=${FLAGS_rootfs_partition_size} \ "${OUTPUT_DIR}" \ "${OUTPUT_DIR}/${image_name}" diff --git a/chromeos_version.sh b/chromeos_version.sh index a40ec3a706..e69dd2845f 100755 --- a/chromeos_version.sh +++ b/chromeos_version.sh @@ -26,7 +26,7 @@ export CHROMEOS_VERSION_MINOR=8 # Increment by 2 in trunk after making a release branch. # Does not reset on a major/minor change (always increases). # (Trunk is always odd; branches are always even). -export CHROMEOS_VERSION_BRANCH=73 +export CHROMEOS_VERSION_BRANCH=75 # Patch number. # Increment by 1 each release on a branch. diff --git a/create_esp.sh b/create_esp.sh index 5b345e20e1..261b8de340 100755 --- a/create_esp.sh +++ b/create_esp.sh @@ -26,8 +26,7 @@ fi info "Creating a new esp image at ${FLAGS_to}" anyway. # Create EFI System Partition to boot stock EFI BIOS (but not ChromeOS EFI -# BIOS). We only need this for x86, but it's simpler and safer to keep the -# disk images the same for both x86 and ARM. +# BIOS). ARM uses this space to determine which partition is bootable. # NOTE: The size argument for mkfs.vfat is in 1024-byte blocks. # We'll hard-code it to 16M for now. ESP_BLOCKS=16384 diff --git a/image_to_live.sh b/image_to_live.sh index f2b0c0556b..b08d0c9501 100755 --- a/image_to_live.sh +++ b/image_to_live.sh @@ -25,6 +25,7 @@ DEFINE_boolean update_known_hosts ${FLAGS_FALSE} \ "Update your known_hosts with the new remote instance's key." DEFINE_string update_log "update_engine.log" \ "Path to log for the update_engine." +DEFINE_boolean verify ${FLAGS_TRUE} "Verify image on device after update." # Flags for devserver. DEFINE_string archive_dir "" \ @@ -33,17 +34,21 @@ DEFINE_integer devserver_port 8080 \ "Port to use for devserver." DEFINE_string image "" \ "Update with this image path that is in this source checkout." i +DEFINE_boolean update_stateful ${FLAGS_TRUE} \ + "Perform update of stateful partition e.g. /var /usr/local." DEFINE_string update_url "" "Full url of an update image." # Flags for stateful update. DEFINE_string stateful_update_flag "" \ "Flag to pass to stateful update e.g. old, clean, etc." s -UPDATER_BIN='/usr/bin/update_engine_client' -UPDATER_IDLE='UPDATE_STATUS_IDLE' -UPDATER_NEED_REBOOT='UPDATE_STATUS_UPDATED_NEED_REBOOT' -UPDATER_UPDATE_CHECK='UPDATE_STATUS_CHECKING_FOR_UPDATE' -UPDATER_DOWNLOADING='UPDATE_STATUS_DOWNLOADING' +UPDATER_BIN="/usr/bin/update_engine_client" +UPDATER_IDLE="UPDATE_STATUS_IDLE" +UPDATER_NEED_REBOOT="UPDATE_STATUS_UPDATED_NEED_REBOOT" +UPDATER_UPDATE_CHECK="UPDATE_STATUS_CHECKING_FOR_UPDATE" +UPDATER_DOWNLOADING="UPDATE_STATUS_DOWNLOADING" + +IMAGE_PATH="" function kill_all_devservers { # Using ! here to avoid exiting with set -e is insufficient, so use @@ -89,9 +94,12 @@ function start_dev_server { if [ -n "${FLAGS_image}" ]; then devserver_flags="${devserver_flags} \ --image $(reinterpret_path_for_chroot ${FLAGS_image})" + IMAGE_PATH="${FLAGS_image}" + elif [ -n "${FLAGS_archive_dir}" ]; then devserver_flags="${devserver_flags} \ --archive_dir $(reinterpret_path_for_chroot ${FLAGS_archive_dir}) -t" + IMAGE_PATH="${FLAGS_archive_dir}/chromiumos_test_image.bin" fi info "Starting devserver with flags ${devserver_flags}" @@ -258,6 +266,31 @@ function remote_reboot { done } +function verify_image { + info "Verifying image." + "${SCRIPTS_DIR}/mount_gpt_image.sh" --from "$(dirname ${IMAGE_PATH})" \ + --image "$(basename ${IMAGE_PATH})" \ + --read_only + + local lsb_release=$(cat /tmp/m/etc/lsb-release) + info "Verifying image with release:" + echo ${lsb_release} + + "${SCRIPTS_DIR}/mount_gpt_image.sh" --unmount + + remote_sh "cat /etc/lsb-release" + info "Remote image reports:" + echo ${REMOTE_OUT} + + if [ "${lsb_release}" = "${REMOTE_OUT}" ]; then + info "Update was successful and image verified as ${lsb_release}." + return 0 + else + warn "Image verification failed." + return 1 + fi +} + function main() { assert_outside_chroot @@ -268,6 +301,12 @@ function main() { set -e + if [ ${FLAGS_verify} -eq ${FLAGS_TRUE} ] && \ + [ -n "${FLAGS_update_url}" ]; then + warn "Verify is not compatible with setting an update url." + FLAGS_verify=${FLAGS_FALSE} + fi + trap cleanup EXIT TMP=$(mktemp -d /tmp/image_to_live.XXXX) @@ -284,11 +323,12 @@ function main() { start_dev_server fi - if [ "${FLAGS_update}" -eq "${FLAGS_TRUE}" ] && ! run_auto_update; then + if [ ${FLAGS_update} -eq ${FLAGS_TRUE} ] && ! run_auto_update; then die "Update was not successful." fi - if ! run_stateful_update; then + if [ ${FLAGS_update_stateful} -eq ${FLAGS_TRUE} ] && \ + ! run_stateful_update; then warn "Stateful update was not successful." fi @@ -304,10 +344,14 @@ function main() { fi remote_sh "grep ^CHROMEOS_RELEASE_DESCRIPTION= /etc/lsb-release" - local release_description=$(echo ${REMOTE_OUT} | cut -d '=' -f 2) - info "Update was successful and rebooted to $release_description" + if [ ${FLAGS_verify} -eq ${FLAGS_TRUE} ]; then + verify_image + else + local release_description=$(echo ${REMOTE_OUT} | cut -d '=' -f 2) + info "Update was successful and rebooted to $release_description" + fi - return 0 + exit 0 } main $@ diff --git a/image_to_usb.sh b/image_to_usb.sh index 0bd67ce5ee..f40e57a4a3 100755 --- a/image_to_usb.sh +++ b/image_to_usb.sh @@ -243,15 +243,12 @@ then echo "Installation must be done from inside the chroot." exit 1 fi - #TODO(kwaters): fix when verified root works on ARM - [ "${FLAGS_arch}" = "ARM" ] && SKIP_VBLOCK="--skip_vblock" echo "Installing ${SRC_IMAGE} to ${FLAGS_to}..." "${FLAGS_build_root}/${FLAGS_board}/usr/sbin/chromeos-install" \ --yes \ --skip_src_removable \ --skip_dst_removable \ - ${SKIP_VBLOCK:-} \ --arch="${FLAGS_arch}" \ --payload_image="${SRC_IMAGE}" \ --dst="${FLAGS_to}" diff --git a/image_to_vm.sh b/image_to_vm.sh index 3807269856..c2fb82b748 100755 --- a/image_to_vm.sh +++ b/image_to_vm.sh @@ -11,22 +11,11 @@ # The path to common.sh should be relative to your script's location. . "$(dirname "$0")/common.sh" . "$(dirname "$0")/chromeos-common.sh" +. "$(dirname "$0")/lib/cros_vm_constants.sh" get_default_board assert_inside_chroot -DEFAULT_MEM="1024" -DEFAULT_VMDK="ide.vmdk" -DEFAULT_VMX="chromiumos.vmx" -DEFAULT_VBOX_DISK="os.vdi" -DEFAULT_QEMU_IMAGE="chromiumos_qemu_image.bin" - -# Minimum sizes for full size vm images -- needed for update. -MIN_VDISK_SIZE_FULL=6072 -MIN_STATEFUL_FS_SIZE_FULL=2048 - -MOD_SCRIPTS_ROOT="${GCLIENT_ROOT}/src/scripts/mod_for_test_scripts" - # Flags DEFINE_string board "${DEFAULT_BOARD}" \ "Board for which the image was built" @@ -85,7 +74,7 @@ fi IMAGES_DIR="${DEFAULT_BUILD_ROOT}/images/${FLAGS_board}" # Default to the most recent image if [ -z "${FLAGS_from}" ] ; then - FLAGS_from="${IMAGES_DIR}/$(ls -t $IMAGES_DIR | head -1)" + FLAGS_from="$(./get_latest_image.sh)" else pushd "${FLAGS_from}" && FLAGS_from=`pwd` && popd fi diff --git a/lib/cros_build_lib.py b/lib/cros_build_lib.py index a0cd73c6ed..71c908648f 100644 --- a/lib/cros_build_lib.py +++ b/lib/cros_build_lib.py @@ -4,11 +4,21 @@ """Common python commands used by various build scripts.""" +import inspect +import os import subprocess import sys _STDOUT_IS_TTY = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() +# TODO(sosa): Move logging to logging module. + +def GetCallerName(): + """Returns the name of the calling module with __main__.""" + top_frame = inspect.stack()[-1][0] + return os.path.basename(top_frame.f_code.co_filename) + + def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None, exit_code=False, redirect_stdout=False, redirect_stderr=False, cwd=None, input=None, enter_chroot=False): @@ -44,7 +54,8 @@ def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None, # Print out the command before running. if print_cmd: - Info('RunCommand: %s' % ' '.join(cmd)) + Info('PROGRAM(%s) -> RunCommand: %s in dir %s' % + (GetCallerName(), ' '.join(cmd), cwd)) try: proc = subprocess.Popen(cmd, cwd=cwd, stdin=stdin, @@ -56,7 +67,7 @@ def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None, if not error_ok and proc.returncode: raise Exception('Command "%s" failed.\n' % (' '.join(cmd)) + (error_message or error or output or '')) - except Exception,e: + except Exception, e: if not error_ok: raise else: diff --git a/lib/cros_vm_constants.sh b/lib/cros_vm_constants.sh new file mode 100644 index 0000000000..124d158eb5 --- /dev/null +++ b/lib/cros_vm_constants.sh @@ -0,0 +1,16 @@ +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Common constants for vm scripts. + +# Default values for creating VM's. +DEFAULT_MEM="1024" +DEFAULT_VMDK="ide.vmdk" +DEFAULT_VMX="chromiumos.vmx" +DEFAULT_VBOX_DISK="os.vdi" +DEFAULT_QEMU_IMAGE="chromiumos_qemu_image.bin" + +# Minimum sizes for full size vm images -- needed for update. +MIN_VDISK_SIZE_FULL=6072 +MIN_STATEFUL_FS_SIZE_FULL=2048 \ No newline at end of file diff --git a/lib/cros_vm_lib.sh b/lib/cros_vm_lib.sh index 61d575bfff..1425f24d6c 100644 --- a/lib/cros_vm_lib.sh +++ b/lib/cros_vm_lib.sh @@ -11,7 +11,6 @@ DEFINE_boolean persist "${FLAGS_FALSE}" "Persist vm." DEFINE_boolean snapshot ${FLAGS_FALSE} "Don't commit changes to image." DEFINE_integer ssh_port 9222 "Port to tunnel ssh traffic over." - KVM_PID_FILE=/tmp/kvm.$$.pid function get_pid() { diff --git a/mod_image_for_dev_recovery.sh b/mod_image_for_dev_recovery.sh index 7523e59486..e94f687b6d 100755 --- a/mod_image_for_dev_recovery.sh +++ b/mod_image_for_dev_recovery.sh @@ -176,7 +176,7 @@ create_dev_recovery_image() { # Mount resized stateful FS and copy payload content to its root directory local temp_mnt=$(mktemp -d "/tmp/temp_mnt.XXXXXX") local loop_dev=$(get_loop_dev) - trap "umount_from_loop_dev ${temp_mnt} && cleanup_loop_dev ${loop_dev}" EXIT + trap "umount_from_loop_dev ${temp_mnt} && rm -f \"${temp_state}\"" EXIT mkdir -p "${temp_mnt}" sudo mount -o loop=${loop_dev} "${temp_state}" "${temp_mnt}" sudo cp -R "${FLAGS_payload_dir}" "${temp_mnt}" @@ -189,7 +189,6 @@ create_dev_recovery_image() { # TODO(tgao): handle install script (for default and custom cases) local temp_img=$(update_partition_table $temp_state $resized_sectors) - rm -f "${temp_state}" # trap handler will clean up loop device and temp mount point echo ${temp_img} } diff --git a/mount_gpt_image.sh b/mount_gpt_image.sh index 247428047c..ee7b3cc6f6 100755 --- a/mount_gpt_image.sh +++ b/mount_gpt_image.sh @@ -18,6 +18,8 @@ get_default_board # Flags. DEFINE_string board "$DEFAULT_BOARD" \ "The board for which the image was built." b +DEFINE_boolean read_only $FLAGS_FALSE \ + "Mount in read only mode -- skips stateful items." DEFINE_boolean unmount $FLAGS_FALSE \ "Unmount previously mounted dir." u DEFINE_string from "/dev/sdc" \ @@ -45,9 +47,11 @@ function unmount_image() { # Don't die on error to force cleanup set +e # Reset symlinks in /usr/local. - setup_symlinks_on_root "/usr/local" "/var" \ - "${FLAGS_stateful_mountpt}" - fix_broken_symlinks "${FLAGS_rootfs_mountpt}" + if mount | grep "${FLAGS_rootfs_mountpt} (rw,bind)"; then + setup_symlinks_on_root "/usr/local" "/var" \ + "${FLAGS_stateful_mountpt}" + fix_broken_symlinks "${FLAGS_rootfs_mountpt}" + fi sudo umount "${FLAGS_rootfs_mountpt}/usr/local" sudo umount "${FLAGS_rootfs_mountpt}/var" if [[ -n "${FLAGS_esp_mountpt}" ]]; then @@ -59,10 +63,13 @@ function unmount_image() { } function get_usb_partitions() { - sudo mount "${FLAGS_from}3" "${FLAGS_rootfs_mountpt}" - sudo mount "${FLAGS_from}1" "${FLAGS_stateful_mountpt}" + local ro_flag="" + [ ${FLAGS_read_only} -eq ${FLAGS_TRUE} ] && ro_flag="-o ro" + + sudo mount ${ro_flag} "${FLAGS_from}3" "${FLAGS_rootfs_mountpt}" + sudo mount ${ro_flag} "${FLAGS_from}1" "${FLAGS_stateful_mountpt}" if [[ -n "${FLAGS_esp_mountpt}" ]]; then - sudo mount "${FLAGS_from}12" "${FLAGS_esp_mountpt}" + sudo mount ${ro_flag} "${FLAGS_from}12" "${FLAGS_esp_mountpt}" fi } @@ -71,19 +78,22 @@ function get_gpt_partitions() { # Mount the rootfs partition using a loopback device. local offset=$(partoffset "${FLAGS_from}/${filename}" 3) - sudo mount -o loop,offset=$(( offset * 512 )) "${FLAGS_from}/${filename}" \ - "${FLAGS_rootfs_mountpt}" + local ro_flag="" + [ ${FLAGS_read_only} -eq ${FLAGS_TRUE} ] && ro_flag="-o ro" + + sudo mount ${ro_flag} -o loop,offset=$(( offset * 512 )) \ + "${FLAGS_from}/${filename}" "${FLAGS_rootfs_mountpt}" # Mount the stateful partition using a loopback device. offset=$(partoffset "${FLAGS_from}/${filename}" 1) - sudo mount -o loop,offset=$(( offset * 512 )) "${FLAGS_from}/${filename}" \ - "${FLAGS_stateful_mountpt}" + sudo mount ${ro_flag} -o loop,offset=$(( offset * 512 )) \ + "${FLAGS_from}/${filename}" "${FLAGS_stateful_mountpt}" # Mount the stateful partition using a loopback device. if [[ -n "${FLAGS_esp_mountpt}" ]]; then offset=$(partoffset "${FLAGS_from}/${filename}" 12) - sudo mount -o loop,offset=$(( offset * 512 )) "${FLAGS_from}/${filename}" \ - "${FLAGS_esp_mountpt}" + sudo mount ${ro_flag} -o loop,offset=$(( offset * 512 )) \ + "${FLAGS_from}/${filename}" "${FLAGS_esp_mountpt}" fi } @@ -108,8 +118,11 @@ function mount_image() { sudo mount --bind "${FLAGS_stateful_mountpt}/dev_image" \ "${FLAGS_rootfs_mountpt}/usr/local" # Setup symlinks in /usr/local so you can emerge packages into /usr/local. - setup_symlinks_on_root "${FLAGS_stateful_mountpt}/dev_image" \ - "${FLAGS_stateful_mountpt}/var" "${FLAGS_stateful_mountpt}" + + if [ ${FLAGS_read_only} -eq ${FLAGS_FALSE} ]; then + setup_symlinks_on_root "${FLAGS_stateful_mountpt}/dev_image" \ + "${FLAGS_stateful_mountpt}/var" "${FLAGS_stateful_mountpt}" + fi echo "Image specified by ${FLAGS_from} mounted at"\ "${FLAGS_rootfs_mountpt} successfully." } diff --git a/parallel_emerge b/parallel_emerge index 61e9920efb..9a77ef45c4 100755 --- a/parallel_emerge +++ b/parallel_emerge @@ -1733,6 +1733,7 @@ def main(): os.execvp(os.path.realpath(sys.argv[0]), args) print "Done" + sys.exit(0) if __name__ == "__main__": main() diff --git a/sync_build_test.sh b/sync_build_test.sh index 9473205e67..16934419c0 100755 --- a/sync_build_test.sh +++ b/sync_build_test.sh @@ -58,10 +58,14 @@ If this is set, chrome browser will be built from source." DEFINE_string chronos_passwd "${CHRONOS_PASSWD}" \ "Use this as the chronos user passwd (defaults to \$CHRONOS_PASSWD)" DEFINE_string chroot "" "Chroot to build/use" +DEFINE_boolean enable_rootfs_verification ${FLAGS_TRUE} \ + "Enable rootfs verification when building image" DEFINE_boolean force_make_chroot ${FLAGS_FALSE} "Run make_chroot indep of sync" DEFINE_string grab_buildbot "" \ "Instead of building, grab this full image.zip URI generated by the \ buildbot" +DEFINE_boolean ignore_remote_test_failures ${FLAGS_FALSE} \ + "Ignore any remote tests that failed and don't return failure" DEFINE_boolean image_to_live ${FLAGS_FALSE} \ "Put the resulting image on live instance (requires --remote)" DEFINE_string image_to_usb "" \ @@ -183,6 +187,7 @@ function validate_and_set_param_defaults() { fi FLAGS_sync=${FLAGS_FALSE} FLAGS_build=${FLAGS_FALSE} + FLAGS_unittest=${FLAGS_FALSE} FLAGS_master=${FLAGS_FALSE} fi @@ -339,6 +344,9 @@ function describe_phase() { # Called when there is a failure and we exit early function failure() { trap - EXIT + # Clear these out just in case. + export GSDCURL_USERNAME="" + export GSDCURL_PASSWORD="" describe_phase "Failure during: ${LAST_PHASE}" show_duration } @@ -450,8 +458,13 @@ EOF # Downloads a buildbot image function grab_buildbot() { + read -p "Username [${LOGNAME}]: " GSDCURL_USERNAME + export GSDCURL_USERNAME + read -s -p "Password: " GSDCURL_PASSWORD + export GSDCURL_PASSWORD + CURL="$(dirname $0)/bin/cros_gsdcurl.py" if [[ "${FLAGS_grab_buildbot}" == "LATEST" ]]; then - local latest=$(curl "${FLAGS_buildbot_uri}/LATEST") + local latest=$(${CURL} "${FLAGS_buildbot_uri}/LATEST") if [[ -z "${latest}" ]]; then echo "Error finding latest." exit 1 @@ -459,9 +472,14 @@ function grab_buildbot() { FLAGS_grab_buildbot="${FLAGS_buildbot_uri}/${latest}/image.zip" fi local dl_dir=$(mktemp -d "/tmp/image.XXXX") + echo "Grabbing image from ${FLAGS_grab_buildbot} to ${dl_dir}" - run_phase "Downloading image" curl "${FLAGS_grab_buildbot}" \ - -o "${dl_dir}/image.zip" + run_phase "Downloading image" ${CURL} "${FLAGS_grab_buildbot}" \ + -o "${dl_dir}/image.zip" + # Clear out the credentials so they can't be used later. + export GSDCURL_USERNAME="" + export GSDCURL_PASSWORD="" + cd "${dl_dir}" unzip image.zip local image_basename=$(basename $(dirname "${FLAGS_grab_buildbot}")) @@ -474,6 +492,15 @@ function grab_buildbot() { run_phase "Installing buildbot test modified image" \ mv chromiumos_test_image.bin "${image_dir}/chromiumos_image.bin" FLAGS_mod_image_for_test=${FLAGS_FALSE} + else + run_phase "Installing buildbot base image" \ + mv chromiumos_base_image.bin "${image_dir}/chromiumos_image.bin" + fi + + if [[ -n "${FLAGS_test}" ]]; then + if [[ ! -d "${FLAGS_top}/chroot/build/${FLAGS_board}" ]]; then + die "To run tests on a buildbot image, run setup_board first." + fi if [[ -e "autotest.tgz" || -e "autotest.tar.bz2" ]]; then # pull in autotest local dir="${FLAGS_chroot}/build/${FLAGS_board}/usr/local" @@ -491,8 +518,6 @@ function grab_buildbot() { run_phase "Installing buildbot autotest cross-compiled binaries" \ sudo mv autotest ${dir} fi - else - mv chromiumos_image.bin "${image_dir}" fi chdir_relative . run_phase "Removing downloaded image" rm -rf "${dl_dir}" @@ -569,16 +594,6 @@ function main() { local build_autotest_param="" if [[ ${FLAGS_build_autotest} -eq ${FLAGS_TRUE} ]]; then build_autotest_param="--withautotest" - if [[ ${FLAGS_useworkon} -eq ${FLAGS_TRUE} ]]; then - # In workon flow, you must workon packages to run tests - run_phase_in_chroot "Setting workon for autotest" \ - ./cros_workon --board=${FLAGS_board} start autotest \ - autotest-tests autotest-deps - # In minilayout you may not yet have autotest. - if [[ ! -d "${FLAGS_top}/src/third_party/autotest/files" ]]; then - run_phase "Syncing autotest repo" repo sync autotest - fi - fi fi run_phase_in_chroot "Building packages" \ @@ -595,6 +610,7 @@ function main() { if [[ ${FLAGS_unittest} -eq ${FLAGS_TRUE} ]] && [[ "${FLAGS_board}" == \ "x86-generic" ]] ; then + chdir_relative src/scripts run_phase_in_chroot "Running unit tests" ./cros_run_unit_tests \ ${board_param} fi @@ -606,9 +622,13 @@ function main() { ./enter_chroot.sh "echo '${FLAGS_chronos_passwd}' | \ ~/trunk/src/scripts/set_shared_user_password.sh" fi + local other_params="--enable_rootfs_verification" + if [[ ${FLAGS_enable_rootfs_verification} -eq ${FLAGS_FALSE} ]]; then + other_params="--noenable_rootfs_verification" + fi run_phase_in_chroot "Mastering image" ./build_image \ "${board_param}" --replace ${withdev_param} \ - ${jobs_param} + ${jobs_param} ${other_params} fi if [[ ${FLAGS_mod_image_for_test} -eq ${FLAGS_TRUE} ]]; then @@ -633,9 +653,14 @@ function main() { chdir_relative src/scripts # We purposefully do not quote FLAGS_test below as we expect it may # have multiple parameters - run_phase "Running tests on Chromium OS machine ${FLAGS_remote}" \ + if ! run_phase "Running tests on Chromium OS machine ${FLAGS_remote}" \ ./run_remote_tests.sh "--remote=${FLAGS_remote}" ${FLAGS_test} \ - "${board_param}" + "${board_param}"; then + if [[ ${FLAGS_ignore_remote_test_failures} -eq ${FLAGS_FALSE} ]]; then + echo "Remote tests failed and --ignore_remote_test_failures not passed" + false + fi + fi fi trap - EXIT diff --git a/update_bootloaders.sh b/update_bootloaders.sh index 7c6b0dd9be..63faaceec8 100755 --- a/update_bootloaders.sh +++ b/update_bootloaders.sh @@ -21,7 +21,7 @@ DEFINE_boolean install_syslinux ${FLAGS_FALSE} \ DEFINE_string from "/tmp/boot" \ "Path the legacy bootloader templates are copied from. (Default /tmp/boot)" DEFINE_string to "/tmp/esp.img" \ - "Path to esp image or ARM output MBR (Default: /tmp/esp.img)" + "Path to esp image (Default: /tmp/esp.img)" DEFINE_integer to_offset 0 \ "Offset in bytes into 'to' if it is a file (Default: 0)" DEFINE_integer to_size -1 \ @@ -39,7 +39,7 @@ DEFINE_string kernel_partition_offset "0" \ DEFINE_string kernel_partition_sectors "0" \ "Kernel partition sectors (Default: 0)" DEFINE_string usb_disk /dev/sdb3 \ - "Path syslinux should use to do a usb (or arm!) boot. Default: /dev/sdb3" + "Path syslinux should use to do a usb boot. Default: /dev/sdb3" # Parse flags FLAGS "$@" || exit 1 @@ -102,8 +102,7 @@ if [[ ! -e "${FLAGS_to}" ]]; then # This shouldn't happen. info "Creating a new esp image at ${FLAGS_to}" anyway. # Create EFI System Partition to boot stock EFI BIOS (but not ChromeOS EFI - # BIOS). We only need this for x86, but it's simpler and safer to keep the - # disk images the same for both x86 and ARM. + # BIOS). ARM uses this space to determine which partition is bootable. # NOTE: The size argument for mkfs.vfat is in 1024-byte blocks. # We'll hard-code it to 16M for now. ESP_BLOCKS=16384