diff --git a/archive_build.sh b/archive_build.sh index 74d577ea84..233540a627 100755 --- a/archive_build.sh +++ b/archive_build.sh @@ -93,7 +93,7 @@ then fi # Get version information -. "${SCRIPTS_DIR}/chromeos_version.sh" +. ${SRC_ROOT}/third_party/chromiumos-overlay/chromeos/config/chromeos_version.sh # Get git hash # Use git:8 chars of sha1 @@ -234,6 +234,7 @@ echo "$LAST_CHANGE" > "${FLAGS_to}/LATEST" # Make sure files are readable chmod 644 "$ZIPFILE" "${FLAGS_to}/LATEST" chmod 755 "$OUTDIR" +cp -f "${FLAGS_from}/au-generator.zip" "${OUTDIR}/" function gsutil_archive() { @@ -275,7 +276,7 @@ then prebuilt_cmd="$prebuilt_cmd -u gs://chromeos-prebuilt --git-sync -V master" prebuilt_cmd="$prebuilt_cmd -p ${GCLIENT_ROOT} -b ${FLAGS_board}" - if [ "${FLAGS_BOARD}" == "x86-generic" ] + if [ "${FLAGS_board}" == "x86-generic" ] then prebuilt_cmd="$prebuilt_cmd --sync-host" fi diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index 2e86ae4173..8424e9e527 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -25,6 +25,9 @@ _DEFAULT_RETRIES = 3 _PACKAGE_FILE = '%(buildroot)s/src/scripts/cbuildbot_package.list' ARCHIVE_BASE = '/var/www/archive' ARCHIVE_COUNT = 10 +PUBLIC_OVERLAY = '%(buildroot)s/src/third_party/chromiumos-overlay' +PRIVATE_OVERLAY = '%(buildroot)s/src/private-overlays/chromeos-overlay' +CHROME_KEYWORDS_FILE = ('/build/%(board)s/etc/portage/package.keywords/chrome') # Currently, both the full buildbot and the preflight buildbot store their # data in a variable named PORTAGE_BINHOST, but they're in different files. @@ -35,6 +38,14 @@ _PREFLIGHT_BINHOST = 'PORTAGE_BINHOST' # ======================== Utility functions ================================ +def _PrintFile(path): + """Prints out the contents of a file to stderr.""" + file_handle = open(path) + print >> sys.stderr, file_handle.read() + file_handle.close() + sys.stderr.flush() + + def MakeDir(path, parents=False): """Basic wrapper around os.mkdirs. @@ -76,8 +87,7 @@ def RepoSync(buildroot, retries=_DEFAULT_RETRIES): Warning('CBUILDBOT -- Retries exhausted') raise - # Output manifest - RunCommand(['repo', 'manifest', '-r', '-o', '-'], cwd=buildroot) + RunCommand(['repo', 'manifest', '-r', '-o', '/dev/stderr'], cwd=buildroot) # =========================== Command Helpers ================================= @@ -210,7 +220,7 @@ def _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board, cwd=cwd, enter_chroot=True) -def _MarkChromeAsStable(buildroot, tracking_branch, chrome_rev): +def _MarkChromeAsStable(buildroot, tracking_branch, chrome_rev, board): """Returns the portage atom for the revved chrome ebuild - see man emerge.""" cwd = os.path.join(buildroot, 'src', 'scripts') portage_atom_string = RunCommand(['bin/cros_mark_chrome_as_stable', @@ -221,7 +231,12 @@ def _MarkChromeAsStable(buildroot, tracking_branch, chrome_rev): Info('Found nothing to rev.') return None else: - return portage_atom_string.split('=')[1] + chrome_atom = portage_atom_string.split('=')[1] + # TODO(sosa): Workaround to build unstable chrome ebuild we uprevved. + RunCommand(['sudo', 'tee', CHROME_KEYWORDS_FILE % {'board': board}], + input='=%s\n' % chrome_atom, enter_chroot=True, + cwd=cwd) + return chrome_atom def _UprevAllPackages(buildroot, tracking_branch, board, overlays): @@ -341,16 +356,22 @@ def _SetupBoard(buildroot, board='x86-generic'): cwd=cwd, enter_chroot=True) -def _Build(buildroot): +def _Build(buildroot, emptytree): """Wrapper around build_packages.""" cwd = os.path.join(buildroot, 'src', 'scripts') - RunCommand(['./build_packages'], cwd=cwd, enter_chroot=True) + if emptytree: + cmd = ['sh', '-c', 'EXTRA_BOARD_FLAGS=--emptytree ./build_packages'] + else: + cmd = ['./build_packages'] + + RunCommand(cmd, cwd=cwd, enter_chroot=True) def _BuildChrome(buildroot, board, chrome_atom_to_build): """Wrapper for emerge call to build Chrome.""" cwd = os.path.join(buildroot, 'src', 'scripts') - RunCommand(['emerge-%s' % board, '=%s' % chrome_atom_to_build], + RunCommand(['emerge-%s' % board, + '=%s' % chrome_atom_to_build], cwd=cwd, enter_chroot=True) @@ -523,8 +544,8 @@ def _ResolveOverlays(buildroot, overlays): 'public': Just the public overlay. 'both': Both the public and private overlays. """ - public_overlay = '%s/src/third_party/chromiumos-overlay' % buildroot - private_overlay = '%s/src/private-overlays/chromeos-overlay' % buildroot + public_overlay = PUBLIC_OVERLAY % {'buildroot': buildroot} + private_overlay = PRIVATE_OVERLAY % {'buildroot': buildroot} if overlays == 'private': paths = [private_overlay] elif overlays == 'public': @@ -575,6 +596,8 @@ def main(): # Parse options usage = "usage: %prog [options] cbuildbot_config" parser = optparse.OptionParser(usage=usage) + parser.add_option('-a', '--acl', default='private', + help='ACL to set on GSD archives') parser.add_option('-r', '--buildroot', help='root directory where build occurs', default=".") parser.add_option('-n', '--buildnumber', @@ -583,30 +606,31 @@ def main(): dest='chrome_rev', help=('Chrome_rev of type [tot|latest_release|' 'sticky_release]')) - parser.add_option('-f', '--revisionfile', - help='file where new revisions are stored') + parser.add_option('-g', '--gsutil', default='', help='Location of gsutil') + parser.add_option('-c', '--gsutil_archive', default='', + help='Datastore archive location') parser.add_option('--clobber', action='store_true', dest='clobber', default=False, help='Clobbers an old checkout before syncing') parser.add_option('--debug', action='store_true', dest='debug', default=False, help='Override some options to run as a developer.') + parser.add_option('--noprebuilts', action='store_false', dest='prebuilts', + default=True, + help="Don't upload prebuilts.") parser.add_option('--nosync', action='store_false', dest='sync', default=True, help="Don't sync before building.") parser.add_option('--notests', action='store_false', dest='tests', default=True, help='Override values from buildconfig and run no tests.') + parser.add_option('-f', '--revisionfile', + help='file where new revisions are stored') parser.add_option('-t', '--tracking-branch', dest='tracking_branch', default='cros/master', help='Run the buildbot on a branch') parser.add_option('-u', '--url', dest='url', default='http://git.chromium.org/git/manifest', help='Run the buildbot on internal manifest') - parser.add_option('-g', '--gsutil', default='', help='Location of gsutil') - parser.add_option('-c', '--gsutil_archive', default='', - help='Datastore archive location') - parser.add_option('-a', '--acl', default='private', - help='ACL to set on GSD archives') (options, args) = parser.parse_args() @@ -646,8 +670,7 @@ def main(): _IncrementalCheckout(buildroot) new_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST) - if old_binhost and old_binhost != new_binhost: - RunCommand(['sudo', 'rm', '-rf', boardpath]) + emptytree = (old_binhost and old_binhost != new_binhost) # Check that all overlays can be found. for path in rev_overlays: @@ -663,7 +686,11 @@ def main(): # Perform uprev. If chrome_uprev is set, rev Chrome ebuilds. if options.chrome_rev: chrome_atom_to_build = _MarkChromeAsStable(buildroot, tracking_branch, - options.chrome_rev) + options.chrome_rev, board) + # If we found nothing to rev, we're done here. + if not chrome_atom_to_build: + return + elif buildconfig['uprev']: _UprevPackages(buildroot, tracking_branch, revisionfile, buildconfig['board'], rev_overlays) @@ -671,7 +698,7 @@ def main(): _EnableLocalAccount(buildroot) # Doesn't rebuild without acquiring more source. if options.sync: - _Build(buildroot) + _Build(buildroot, emptytree) if chrome_atom_to_build: _BuildChrome(buildroot, buildconfig['board'], chrome_atom_to_build) @@ -701,7 +728,7 @@ def main(): if buildconfig['master']: # Master bot needs to check if the other slaves completed. if cbuildbot_comm.HaveSlavesCompleted(config): - if not options.debug: + if not options.debug and options.prebuilts: _UploadPrebuilts(buildroot, board, buildconfig['rev_overlays'], [new_binhost]) _UprevPush(buildroot, tracking_branch, buildconfig['board'], diff --git a/bin/cbuildbot_unittest.py b/bin/cbuildbot_unittest.py index b3a4009a90..9e727423f9 100755 --- a/bin/cbuildbot_unittest.py +++ b/bin/cbuildbot_unittest.py @@ -11,6 +11,7 @@ import mox import os import posix import shutil +import tempfile import unittest # Fixes circular dependency error. @@ -18,6 +19,7 @@ import cbuildbot_comm import cbuildbot from cros_build_lib import ReinterpretPathForChroot + class CBuildBotTest(mox.MoxTestBase): def setUp(self): @@ -117,7 +119,7 @@ class CBuildBotTest(mox.MoxTestBase): buildroot = '/fake_dir' board = 'fake-board' test_results_dir = 'fake_results_dir' - gsutil_path='/fake/gsutil/path' + gsutil_path = '/fake/gsutil/path' archive_dir = 1234 acl = 'fake_acl' num_retries = 5 diff --git a/bin/cros_au_test_harness.py b/bin/cros_au_test_harness.py index 3ffdac53cc..0e484947ab 100755 --- a/bin/cros_au_test_harness.py +++ b/bin/cros_au_test_harness.py @@ -8,6 +8,8 @@ import optparse import os import re import sys +import thread +import time import unittest import urllib @@ -19,6 +21,8 @@ from cros_build_lib import RunCommand from cros_build_lib import RunCommandCaptureOutput from cros_build_lib import Warning +import cros_test_proxy + # VM Constants. _FULL_VDISK_SIZE = 6072 _FULL_STATEFULFS_SIZE = 3074 @@ -42,6 +46,7 @@ class AUTest(object): """Abstract interface that defines an Auto Update test.""" source_image = '' use_delta_updates = False + verbose = False def setUp(self): unittest.TestCase.setUp(self) @@ -80,7 +85,7 @@ class AUTest(object): if self.use_delta_updates: try: self.source_image = src_image - self.UpdateImage(image) + self._UpdateImageReportError(image) except: Warning('Delta update failed, disabling delta updates and retrying.') self.use_delta_updates = False @@ -89,20 +94,21 @@ class AUTest(object): else: self._UpdateImageReportError(image) - def _UpdateImageReportError(self, image_path, stateful_change='old'): + def _UpdateImageReportError(self, image_path, stateful_change='old', + proxy_port=None): """Calls UpdateImage and reports any error to the console. Still throws the exception. """ try: - self.UpdateImage(image_path, stateful_change) + self.UpdateImage(image_path, stateful_change, proxy_port) except UpdateException as err: # If the update fails, print it out Warning(err.stdout) raise def _AttemptUpdateWithPayloadExpectedFailure(self, payload, expected_msg): - # This update is expected to fail... + """Attempt a payload update, expect it to fail with expected log""" try: self.UpdateUsingPayload(payload) except UpdateException as err: @@ -110,15 +116,36 @@ class AUTest(object): if re.search(re.escape(expected_msg), err.stdout, re.MULTILINE): return - Warning("Didn't find '%s' in:" % expected_msg) - Warning(err.stdout) - self.fail('We managed to update when failure was expected') + Warning("Didn't find '%s' in:" % expected_msg) + Warning(err.stdout) + self.fail('We managed to update when failure was expected') + + def _AttemptUpdateWithFilter(self, filter): + """Update through a proxy, with a specified filter, and expect success.""" + + self.PrepareBase(target_image_path) + + # The devserver runs at port 8080 by default. We assume that here, and + # start our proxy at 8081. We then tell our update tools to have the + # client connect to 8081 instead of 8080. + proxy_port = 8081 + proxy = cros_test_proxy.CrosTestProxy(port_in=proxy_port, + address_out='127.0.0.1', + port_out=8080, + filter=filter) + proxy.serve_forever_in_thread() + + # This update is expected to fail... + try: + self._UpdateImageReportError(target_image_path, proxy_port=proxy_port) + finally: + proxy.shutdown() def PrepareBase(self, image_path): """Prepares target with base_image_path.""" pass - def UpdateImage(self, image_path, stateful_change='old'): + def UpdateImage(self, image_path, stateful_change='old', proxy_port=None): """Updates target with the image given by the image_path. Args: @@ -128,15 +155,22 @@ class AUTest(object): 'old': Don't modify stateful partition. Just update normally. 'clean': Uses clobber-state to wipe the stateful partition with the exception of code needed for ssh. + proxy_port: Port to have the client connect to. For use with + CrosTestProxy. """ pass - def UpdateUsingPayload(self, update_path, stateful_change='old'): + def UpdateUsingPayload(self, + update_path, + stateful_change='old', + proxy_port=None): """Updates target with the pre-generated update stored in update_path Args: update_path: Path to the image to update with. This directory should - contain both update.gz, and stateful.image.gz + contain both update.gz, and stateful.image.gz + proxy_port: Port to have the client connect to. For use with + CrosTestProxy. """ pass @@ -169,7 +203,8 @@ class AUTest(object): percent that passed. """ Info('Output from VerifyImage():') - print output + print >> sys.stderr, output + sys.stderr.flush() percent_passed = self.ParseGenerateTestReportOutput(output) Info('Percent passed: %d vs. Percent required: %d' % ( percent_passed, percent_required_to_pass)) @@ -185,7 +220,7 @@ class AUTest(object): """ # Just make sure some tests pass on original image. Some old images # don't pass many tests. - self.PrepareBase(image_path=base_image_path) + self.PrepareBase(base_image_path) # TODO(sosa): move to 100% once we start testing using the autotest paired # with the dev channel. percent_passed = self.VerifyImage(10) @@ -208,7 +243,7 @@ class AUTest(object): """ # Just make sure some tests pass on original image. Some old images # don't pass many tests. - self.PrepareBase(image_path=base_image_path) + self.PrepareBase(base_image_path) # TODO(sosa): move to 100% once we start testing using the autotest paired # with the dev channel. percent_passed = self.VerifyImage(10) @@ -226,7 +261,7 @@ class AUTest(object): def testPartialUpdate(self): """Tests what happens if we attempt to update with a truncated payload.""" # Preload with the version we are trying to test. - self.PrepareBase(image_path=target_image_path) + self.PrepareBase(target_image_path) # Image can be updated at: # ~chrome-eng/chromeos/localmirror/autest-images @@ -237,13 +272,13 @@ class AUTest(object): # Read from the URL and write to the local file urllib.urlretrieve(url, payload) - expected_msg='download_hash_data == update_check_response_hash failed' + expected_msg = 'download_hash_data == update_check_response_hash failed' self._AttemptUpdateWithPayloadExpectedFailure(payload, expected_msg) def testCorruptedUpdate(self): """Tests what happens if we attempt to update with a corrupted payload.""" # Preload with the version we are trying to test. - self.PrepareBase(image_path=target_image_path) + self.PrepareBase(target_image_path) # Image can be updated at: # ~chrome-eng/chromeos/localmirror/autest-images @@ -255,9 +290,74 @@ class AUTest(object): urllib.urlretrieve(url, payload) # This update is expected to fail... - expected_msg='zlib inflate() error:-3' + expected_msg = 'zlib inflate() error:-3' self._AttemptUpdateWithPayloadExpectedFailure(payload, expected_msg) + def testInterruptedUpdate(self): + """Tests what happens if we interrupt payload delivery 3 times.""" + + class InterruptionFilter(cros_test_proxy.Filter): + """This filter causes the proxy to interrupt the download 3 times + + It does this by closing the first three connections to transfer + 2M total in the outbound connection after they transfer the + 2M. + """ + def __init__(self): + """Defines variable shared across all connections""" + self.close_count = 0 + + def setup(self): + """Called once at the start of each connection.""" + self.data_size = 0 + + def OutBound(self, data): + """Called once per packet for outgoing data. + + The first three connections transferring more than 2M + outbound will be closed. + """ + if self.close_count < 3: + if self.data_size > (2 * 1024 * 1024): + self.close_count += 1 + return None + + self.data_size += len(data) + return data + + self._AttemptUpdateWithFilter(InterruptionFilter()) + + def testDelayedUpdate(self): + """Tests what happens if some data is delayed during update delivery""" + + class DelayedFilter(cros_test_proxy.Filter): + """Causes intermittent delays in data transmission. + + It does this by inserting 3 20 second delays when transmitting + data after 2M has been sent. + """ + def setup(self): + """Called once at the start of each connection.""" + self.data_size = 0 + self.delay_count = 0 + + def OutBound(self, data): + """Called once per packet for outgoing data. + + The first three packets after we reach 2M transferred + are delayed by 20 seconds. + """ + if self.delay_count < 3: + if self.data_size > (2 * 1024 * 1024): + self.delay_count += 1 + time.sleep(20) + + self.data_size += len(data) + return data + + + self._AttemptUpdateWithFilter(DelayedFilter()) + class RealAUTest(unittest.TestCase, AUTest): """Test harness for updating real images.""" @@ -268,36 +368,55 @@ class RealAUTest(unittest.TestCase, AUTest): """Auto-update to base image to prepare for test.""" self._UpdateImageReportError(image_path) - def UpdateImage(self, image_path, stateful_change='old'): + def UpdateImage(self, image_path, stateful_change='old', proxy_port=None): """Updates a remote image using image_to_live.sh.""" stateful_change_flag = self.GetStatefulChangeFlag(stateful_change) + cmd = ['%s/image_to_live.sh' % self.crosutils, + '--image=%s' % image_path, + '--remote=%s' % remote, + stateful_change_flag, + '--verify', + '--src_image=%s' % self.source_image + ] - (code, stdout, stderr) = RunCommandCaptureOutput([ - '%s/image_to_live.sh' % self.crosutils, - '--image=%s' % image_path, - '--remote=%s' % remote, - stateful_change_flag, - '--verify', - '--src_image=%s' % self.source_image - ]) + if proxy_port: + cmd.append('--proxy_port=%s' % proxy_port) - if code != 0: - raise UpdateException(code, stdout) + if self.verbose: + try: + RunCommand(cmd) + except Exception, e: + raise UpdateException(1, e.message) + else: + (code, stdout, stderr) = RunCommandCaptureOutput(cmd) + if code != 0: + raise UpdateException(code, stdout) - def UpdateUsingPayload(self, update_path, stateful_change='old'): + def UpdateUsingPayload(self, + update_path, + stateful_change='old', + proxy_port=None): """Updates a remote image using image_to_live.sh.""" stateful_change_flag = self.GetStatefulChangeFlag(stateful_change) + cmd = ['%s/image_to_live.sh' % self.crosutils, + '--payload=%s' % update_path, + '--remote=%s' % remote, + stateful_change_flag, + '--verify', + ] - (code, stdout, stderr) = RunCommandCaptureOutput([ - '%s/image_to_live.sh' % self.crosutils, - '--payload=%s' % update_path, - '--remote=%s' % remote, - stateful_change_flag, - '--verify', - ]) + if proxy_port: + cmd.append('--proxy_port=%s' % proxy_port) - if code != 0: - raise UpdateException(code, stdout) + if self.verbose: + try: + RunCommand(cmd) + except Exception, e: + raise UpdateException(1, e.message) + else: + (code, stdout, stderr) = RunCommandCaptureOutput(cmd) + if code != 0: + raise UpdateException(code, stdout) def VerifyImage(self, percent_required_to_pass): """Verifies an image using run_remote_tests.sh with verification suite.""" @@ -354,47 +473,68 @@ class VirtualAUTest(unittest.TestCase, AUTest): self.assertTrue(os.path.exists(self.vm_image_path)) - def UpdateImage(self, image_path, stateful_change='old'): + def UpdateImage(self, image_path, stateful_change='old', proxy_port=None): """Updates VM image with image_path.""" stateful_change_flag = self.GetStatefulChangeFlag(stateful_change) if self.source_image == base_image_path: self.source_image = self.vm_image_path - (code, stdout, stderr) = RunCommandCaptureOutput([ - '%s/cros_run_vm_update' % self.crosutilsbin, - '--update_image_path=%s' % image_path, - '--vm_image_path=%s' % self.vm_image_path, - '--snapshot', - vm_graphics_flag, - '--persist', - '--kvm_pid=%s' % _KVM_PID_FILE, - stateful_change_flag, - '--src_image=%s' % self.source_image, - ]) + cmd = ['%s/cros_run_vm_update' % self.crosutilsbin, + '--update_image_path=%s' % image_path, + '--vm_image_path=%s' % self.vm_image_path, + '--snapshot', + vm_graphics_flag, + '--persist', + '--kvm_pid=%s' % _KVM_PID_FILE, + stateful_change_flag, + '--src_image=%s' % self.source_image, + ] - if code != 0: - raise UpdateException(code, stdout) + if proxy_port: + cmd.append('--proxy_port=%s' % proxy_port) - def UpdateUsingPayload(self, update_path, stateful_change='old'): + if self.verbose: + try: + RunCommand(cmd) + except Exception, e: + raise UpdateException(1, e.message) + else: + (code, stdout, stderr) = RunCommandCaptureOutput(cmd) + if code != 0: + raise UpdateException(code, stdout) + + def UpdateUsingPayload(self, + update_path, + stateful_change='old', + proxy_port=None): """Updates a remote image using image_to_live.sh.""" stateful_change_flag = self.GetStatefulChangeFlag(stateful_change) if self.source_image == base_image_path: self.source_image = self.vm_image_path - (code, stdout, stderr) = RunCommandCaptureOutput([ - '%s/cros_run_vm_update' % self.crosutilsbin, - '--payload=%s' % update_path, - '--vm_image_path=%s' % self.vm_image_path, - '--snapshot', - vm_graphics_flag, - '--persist', - '--kvm_pid=%s' % _KVM_PID_FILE, - stateful_change_flag, - '--src_image=%s' % self.source_image, - ]) + cmd = ['%s/cros_run_vm_update' % self.crosutilsbin, + '--payload=%s' % update_path, + '--vm_image_path=%s' % self.vm_image_path, + '--snapshot', + vm_graphics_flag, + '--persist', + '--kvm_pid=%s' % _KVM_PID_FILE, + stateful_change_flag, + '--src_image=%s' % self.source_image, + ] - if code != 0: - raise UpdateException(code, stdout) + if proxy_port: + cmd.append('--proxy_port=%s' % proxy_port) + + if self.verbose: + try: + RunCommand(cmd) + except Exception, e: + raise UpdateException(1, e.message) + else: + (code, stdout, stderr) = RunCommandCaptureOutput(cmd) + if code != 0: + raise UpdateException(code, stdout) def VerifyImage(self, percent_required_to_pass): """Runs vm smoke suite to verify image.""" @@ -421,26 +561,33 @@ if __name__ == '__main__': parser = optparse.OptionParser() parser.add_option('-b', '--base_image', help='path to the base image.') - parser.add_option('-t', '--target_image', - help='path to the target image.') parser.add_option('-r', '--board', help='board for the images.') - parser.add_option('-p', '--type', default='vm', - help='type of test to run: [vm, real]. Default: vm.') - parser.add_option('-m', '--remote', - help='Remote address for real test.') - parser.add_option('--no_graphics', action='store_true', - help='Disable graphics for the vm test.') parser.add_option('--no_delta', action='store_false', default=True, dest='delta', help='Disable using delta updates.') + parser.add_option('--no_graphics', action='store_true', + help='Disable graphics for the vm test.') + parser.add_option('-m', '--remote', + help='Remote address for real test.') parser.add_option('-q', '--quick_test', default=False, action='store_true', help='Use a basic test to verify image.') + parser.add_option('-t', '--target_image', + help='path to the target image.') + parser.add_option('--test_prefix', default='test', + help='Only runs tests with specific prefix i.e. ' + 'testFullUpdateWipeStateful.') + parser.add_option('-p', '--type', default='vm', + help='type of test to run: [vm, real]. Default: vm.') + parser.add_option('--verbose', default=False, action='store_true', + help='Print out rather than capture output as much as ' + 'possible.') # Set the usage to include flags. parser.set_usage(parser.format_help()) # Parse existing sys.argv so we can pass rest to unittest.main. (options, sys.argv) = parser.parse_args(sys.argv) + AUTest.verbose = options.verbose base_image_path = options.base_image target_image_path = options.target_image board = options.board @@ -465,19 +612,17 @@ if __name__ == '__main__': AUTest.use_delta_updates = options.delta # Only run the test harness we care about. - if options.type == 'vm': - suite = unittest.TestLoader().loadTestsFromTestCase(VirtualAUTest) - test_result = unittest.TextTestRunner(verbosity=2).run(suite) - elif options.type == 'real': - if not options.remote: - parser.error('Real tests require a remote test machine.') - else: - remote = options.remote + test_loader = unittest.TestLoader() + test_loader.testMethodPrefix = options.test_prefix - suite = unittest.TestLoader().loadTestsFromTestCase(RealAUTest) - test_result = unittest.TextTestRunner(verbosity=2).run(suite) - else: - parser.error('Could not parse harness type %s.' % options.type) + if options.type == 'vm': test_class = VirtualAUTest + elif options.type == 'real': test_class = RealAUTest + else: parser.error('Could not parse harness type %s.' % options.type) + + remote = options.remote + + test_suite = test_loader.loadTestsFromTestCase(test_class) + test_result = unittest.TextTestRunner(verbosity=2).run(test_suite) if not test_result.wasSuccessful(): Die('Test harness was not successful') diff --git a/bin/cros_mark_chrome_as_stable.py b/bin/cros_mark_chrome_as_stable.py index 356b97d6de..d7a75605e6 100755 --- a/bin/cros_mark_chrome_as_stable.py +++ b/bin/cros_mark_chrome_as_stable.py @@ -101,7 +101,7 @@ def _GetLatestRelease(branch=None): if branch: chrome_version_re = re.compile('^%s\.\d+.*' % branch) else: - chrome_version_re = re.compile('^[0-9]\..*') + chrome_version_re = re.compile('^[0-9]+\..*') for chrome_version in sorted_ls.splitlines(): if chrome_version_re.match(chrome_version): current_version = chrome_version @@ -109,8 +109,8 @@ def _GetLatestRelease(branch=None): return current_version.rstrip('/') -def _GetStickyVersion(stable_ebuilds): - """Discovers the sticky version from the current stable_ebuilds.""" +def _GetStickyEBuild(stable_ebuilds): + """Returns the sticky ebuild.""" sticky_ebuilds = [] non_sticky_re = re.compile(_NON_STICKY_REGEX) for ebuild in stable_ebuilds: @@ -122,7 +122,7 @@ def _GetStickyVersion(stable_ebuilds): elif len(sticky_ebuilds) > 1: Warning('More than one sticky ebuild found') - return cros_mark_as_stable.BestEBuild(sticky_ebuilds).chrome_version + return cros_mark_as_stable.BestEBuild(sticky_ebuilds) class ChromeEBuild(cros_mark_as_stable.EBuild): @@ -146,6 +146,9 @@ class ChromeEBuild(cros_mark_as_stable.EBuild): else: return (-1) + def __str__(self): + return self.ebuild_path + def FindChromeCandidates(overlay_dir): """Return a tuple of chrome's unstable ebuild and stable ebuilds. @@ -166,7 +169,7 @@ def FindChromeCandidates(overlay_dir): if not ebuild.chrome_version: Warning('Poorly formatted ebuild found at %s' % path) else: - if not ebuild.is_stable: + if '9999' in ebuild.version: unstable_ebuilds.append(ebuild) else: stable_ebuilds.append(ebuild) @@ -203,7 +206,7 @@ def FindChromeUprevCandidate(stable_ebuilds, chrome_rev, sticky_branch): candidates.append(ebuild) elif chrome_rev == STICKY: - chrome_branch_re = re.compile('%s\.\d+.*_rc.*' % sticky_branch) + chrome_branch_re = re.compile('%s\..*' % sticky_branch) for ebuild in stable_ebuilds: if chrome_branch_re.search(ebuild.version): candidates.append(ebuild) @@ -222,7 +225,8 @@ def FindChromeUprevCandidate(stable_ebuilds, chrome_rev, sticky_branch): def MarkChromeEBuildAsStable(stable_candidate, unstable_ebuild, chrome_rev, - chrome_version, commit, overlay_dir): + chrome_version, commit, overlay_dir, + sticky_ebuild): """Uprevs the chrome ebuild specified by chrome_rev. This is the main function that uprevs the chrome_rev from a stable candidate @@ -242,6 +246,7 @@ def MarkChromeEBuildAsStable(stable_candidate, unstable_ebuild, chrome_rev, chrome_version: The \d.\d.\d.\d version of Chrome. commit: Used with TIP_OF_TRUNK. The svn revision of chrome. overlay_dir: Path to the chromeos-chrome package dir. + sticky_ebuild: EBuild class for the sticky ebuild. Returns: Full portage version atom (including rc's, etc) that was revved. """ @@ -260,9 +265,23 @@ def MarkChromeEBuildAsStable(stable_candidate, unstable_ebuild, chrome_rev, new_ebuild_path = base_path + ('%s-r1.ebuild' % portage_suffix) cros_mark_as_stable.EBuildStableMarker.MarkAsStable( - unstable_ebuild.ebuild_path, new_ebuild_path, 'CROS_SVN_COMMIT', commit) + unstable_ebuild.ebuild_path, new_ebuild_path, 'CROS_SVN_COMMIT', commit, + make_stable=False) + new_ebuild = ChromeEBuild(new_ebuild_path) + if stable_candidate and ( + stable_candidate.chrome_version == new_ebuild.chrome_version): + if 0 == RunCommand(['diff', '-Bu', stable_candidate.ebuild_path, + new_ebuild_path], + redirect_stderr=True, + redirect_stdout=True, + exit_code=True): + Info('Previous ebuild with same version found and no 9999 changes found.' + ' Nothing to do.') + os.unlink(new_ebuild_path) + return None + RunCommand(['git', 'add', new_ebuild_path]) - if stable_candidate: + if stable_candidate and stable_candidate != sticky_ebuild: RunCommand(['git', 'rm', stable_candidate.ebuild_path]) cros_mark_as_stable.EBuildStableMarker.CommitChange( @@ -293,10 +312,10 @@ def main(): commit_to_use = None (unstable_ebuild, stable_ebuilds) = FindChromeCandidates(overlay_dir) - sticky_version = _GetStickyVersion(stable_ebuilds) + sticky_ebuild = _GetStickyEBuild(stable_ebuilds) + sticky_version = sticky_ebuild.chrome_version sticky_branch = sticky_version.rpartition('.')[0] - if chrome_rev == TIP_OF_TRUNK: version_to_uprev = _GetTipOfTrunkVersion() commit_to_use = _GetTipOfTrunkSvnRevision() @@ -307,25 +326,23 @@ def main(): stable_candidate = FindChromeUprevCandidate(stable_ebuilds, chrome_rev, sticky_branch) - # There are some cases we don't need to do anything. Check for them. - if stable_candidate and (version_to_uprev == stable_candidate.chrome_version - and not commit_to_use): - Info('Found nothing to do for chrome_rev %s with version %s.' % ( - chrome_rev, version_to_uprev)) - else: - os.chdir(overlay_dir) - work_branch = cros_mark_as_stable.GitBranch( - cros_mark_as_stable.STABLE_BRANCH_NAME, options.tracking_branch) - work_branch.CreateBranch() - try: - chrome_version_atom = MarkChromeEBuildAsStable( - stable_candidate, unstable_ebuild, chrome_rev, version_to_uprev, - commit_to_use, overlay_dir) - # Explicit print to communicate to caller. + + os.chdir(overlay_dir) + work_branch = cros_mark_as_stable.GitBranch( + cros_mark_as_stable.STABLE_BRANCH_NAME, options.tracking_branch) + work_branch.CreateBranch() + try: + chrome_version_atom = MarkChromeEBuildAsStable( + stable_candidate, unstable_ebuild, chrome_rev, version_to_uprev, + commit_to_use, overlay_dir, sticky_ebuild) + # Explicit print to communicate to caller. + if chrome_version_atom: print 'CHROME_VERSION_ATOM=%s' % chrome_version_atom - except: + else: work_branch.Delete() - raise + except: + work_branch.Delete() + raise if __name__ == '__main__': diff --git a/bin/cros_mark_chrome_as_stable_unittest.py b/bin/cros_mark_chrome_as_stable_unittest.py index ab16cd68ea..dfc981de8b 100755 --- a/bin/cros_mark_chrome_as_stable_unittest.py +++ b/bin/cros_mark_chrome_as_stable_unittest.py @@ -200,12 +200,12 @@ class CrosMarkChromeAsStable(mox.MoxTestBase): self.mox.VerifyAll() self.assertEqual('8.0.224.2', release) - def testStickyVersion(self): - """Tests if we can find the sticky version from our mock directories.""" + def testStickyEBuild(self): + """Tests if we can find the sticky ebuild from our mock directories.""" stable_ebuilds = self._GetStableEBuilds() - sticky_version = cros_mark_chrome_as_stable._GetStickyVersion( + sticky_ebuild = cros_mark_chrome_as_stable._GetStickyEBuild( stable_ebuilds) - self.assertEqual(sticky_version, self.sticky_version) + self.assertEqual(sticky_ebuild.chrome_version, self.sticky_version) def testChromeEBuildInit(self): """Tests if the chrome_version is set correctly in a ChromeEBuild.""" @@ -231,6 +231,7 @@ class CrosMarkChromeAsStable(mox.MoxTestBase): 'CommitChange') stable_candidate = cros_mark_chrome_as_stable.ChromeEBuild(old_ebuild_path) unstable_ebuild = cros_mark_chrome_as_stable.ChromeEBuild(self.unstable) + sticky_ebuild = cros_mark_chrome_as_stable.ChromeEBuild(self.sticky) chrome_version = new_version commit = None overlay_dir = self.mock_chrome_dir @@ -243,14 +244,14 @@ class CrosMarkChromeAsStable(mox.MoxTestBase): self.mox.ReplayAll() cros_mark_chrome_as_stable.MarkChromeEBuildAsStable( stable_candidate, unstable_ebuild, chrome_rev, chrome_version, commit, - overlay_dir) + overlay_dir, sticky_ebuild) self.mox.VerifyAll() def testStickyMarkAsStable(self): """Tests to see if we can mark chrome as stable for a new sticky release.""" self._CommonMarkAsStableTest(cros_mark_chrome_as_stable.STICKY, self.sticky_new_rc_version, self.sticky_rc, - self.sticky_new_rc, 'sticky_release') + self.sticky_new_rc, 'stable_release') def testLatestMarkAsStable(self): """Tests to see if we can mark chrome for a latest release.""" diff --git a/bin/cros_run_vm_update b/bin/cros_run_vm_update index abd0071462..863b66dc65 100755 --- a/bin/cros_run_vm_update +++ b/bin/cros_run_vm_update @@ -10,6 +10,8 @@ . "$(dirname $0)/../lib/cros_vm_lib.sh" DEFINE_string payload "" "Full name of the payload to update with." +DEFINE_string proxy_port "" \ + "Have the client request from this proxy instead of devserver." DEFINE_string src_image "" \ "Create a delta update by passing in the image on the remote machine." DEFINE_string stateful_update_flag "" "Flags to pass to stateful update." s @@ -35,7 +37,11 @@ if [ -n "${FLAGS_update_image_path}" ]; then fi if [ -n "${FLAGS_payload}" ]; then - IMAGE_ARGS="--payload="${FLAGS_payload}"" + IMAGE_ARGS="--payload=${FLAGS_payload}" +fi + +if [ -n "${FLAGS_proxy_port}" ]; then + IMAGE_ARGS="--proxy_port=${FLAGS_proxy_port}" fi $(dirname $0)/../image_to_live.sh \ diff --git a/build_image b/build_image index bd49af5054..61b28215b8 100755 --- a/build_image +++ b/build_image @@ -163,7 +163,7 @@ if [ "${FLAGS_fast}" -eq "${FLAGS_TRUE}" ]; then fi # Determine build version. -. "${SCRIPTS_DIR}/chromeos_version.sh" +. ${SRC_ROOT}/third_party/chromiumos-overlay/chromeos/config/chromeos_version.sh # Configure extra USE or packages for this type of build. EXTRA_PACKAGES="" @@ -415,7 +415,8 @@ update_dev_packages() { fi # Install the bare necessary files so that the "emerge" command works - sudo cp ${root_dev_dir}/etc/make.globals ${ROOT_FS_DIR}/etc/ + [ ${FLAGS_statefuldev} -eq ${FLAGS_TRUE} ] && \ + sudo cp ${root_dev_dir}/etc/make.globals ${ROOT_FS_DIR}/etc/ sudo sed -i s,/usr/bin/wget,wget, ${ROOT_FS_DIR}/etc/make.globals sudo mkdir -p ${ROOT_FS_DIR}/etc/make.profile @@ -503,6 +504,14 @@ update_base_packages() { -s "${STATEFUL_FS_DIR}" -e "${ESP_FS_DIR}" } +zero_free_space() { + local fs_mount_point=$1 + info "Zeroing freespace in ${fs_mount_point}" + sudo dd if=/dev/zero of="${fs_mount_point}/filler" oflag=sync bs=4096 || true + sudo rm -f "${fs_mount_point}/filler" + sudo sync +} + create_base_image() { local image_name=$1 @@ -683,6 +692,10 @@ create_base_image() { # Create an empty esp image to be updated in by update_bootloaders.sh. ${SCRIPTS_DIR}/create_esp.sh --to="${ESP_FS_IMG}" + # Zero rootfs free space to make it more compressible so auto-update + # payloads become smaller + zero_free_space "${ROOT_FS_DIR}" + cleanup trap delete_prompt EXIT @@ -702,6 +715,14 @@ create_base_image() { trap - EXIT } +generate_au_zip () { + local lgenerateauzip="${SCRIPTS_DIR}/generate_au_zip.py" + local largs="-o ${OUTPUT_DIR}" + test ! -d "${OUTPUT_DIR}" && mkdir -p "${OUTPUT_DIR}" + info "Running ${lgenerateauzip} ${largs} for generating AU updater zip file" + $lgenerateauzip $largs +} + # Create the output directory. mkdir -p "${OUTPUT_DIR}" mkdir -p "${ROOT_FS_DIR}" @@ -782,6 +803,8 @@ rm -f "${ROOT_FS_IMG}" "${STATEFUL_FS_IMG}" "${OUTPUT_DIR}/vmlinuz.image" \ "${ESP_FS_IMG}" "${OEM_FS_IMG}" "${OUTPUT_DIR}/vmlinuz_hd.vblock" rmdir "${ROOT_FS_DIR}" "${STATEFUL_FS_DIR}" "${OEM_FS_DIR}" "${ESP_FS_DIR}" +# Generating AU generator zip file to run outside chroot +generate_au_zip || echo "Failed generating AU zip file - ignoring Error..." # Create a 'latest' link rm -f ${FLAGS_output_root}/${FLAGS_board}/latest ln -s $(basename ${OUTPUT_DIR}) ${FLAGS_output_root}/${FLAGS_board}/latest diff --git a/build_kernel_image.sh b/build_kernel_image.sh index 98230e41e6..08cca4c726 100755 --- a/build_kernel_image.sh +++ b/build_kernel_image.sh @@ -220,7 +220,7 @@ elif [[ "${FLAGS_arch}" = "arm" ]]; then echo -n 'setenv bootargs ${bootargs} ' > "${kernel_script}" tr '\n' ' ' <"${FLAGS_working_dir}/boot.config" >> "${kernel_script}" echo >> "${kernel_script}" - printf 'read ${devtype} 0:${kernelpart} ${loadaddr} %x %x\n' \ + printf 'read ${devtype} ${devnum}:${kernelpart} ${loadaddr} %x %x\n' \ ${script_size} ${kernel_size} >> "${kernel_script}" echo 'bootm ${loadaddr}' >> ${kernel_script} mkimage -A arm -O linux -T script -C none -a 0 -e 0 \ diff --git a/chromeos_version.sh b/chromeos_version.sh deleted file mode 100755 index 0540c3ab7f..0000000000 --- a/chromeos_version.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/sh - -# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# ChromeOS version information -# -# This file is usually sourced by other build scripts, but can be run -# directly to see what it would do. -# -# Version numbering scheme is much like Chrome's, with the addition of -# double-incrementing branch number so trunk is always odd. - -HOSTNAME=$(hostname) -############################################################################# -# SET VERSION NUMBERS -############################################################################# -# Major/minor versions. -# Primarily for product marketing. -export CHROMEOS_VERSION_MAJOR=0 -export CHROMEOS_VERSION_MINOR=9 - -# Branch number. -# Increment by 1 in a new release branch. -# Increment by 2 in trunk after making a release branch. -# Does not reset on a major/minor change (always increases). -# (Trunk is always odd; branches are always even). -export CHROMEOS_VERSION_BRANCH=131 - -# Patch number. -# Increment by 1 each release on a branch. -# Reset to 0 when increasing branch number. -export CHROMEOS_VERSION_PATCH=0 - -# Codename of this version. -export CHROMEOS_VERSION_CODENAME="" - -############################################################################# -# SET VERSION STRINGS -############################################################################# -# Official builds must set -# CHROMEOS_OFFICIAL=1 -# Note that ${FOO:-0} means default-to-0-if-unset; ${FOO:?} means die-if-unset. -if [ ${CHROMEOS_OFFICIAL:-0} -eq 1 ] -then - # Official builds (i.e., buildbot) - export CHROMEOS_VERSION_NAME="Chrome OS" - export CHROMEOS_VERSION_TRACK="dev-channel" - export CHROMEOS_VERSION_AUSERVER="https://tools.google.com/service/update2" - export CHROMEOS_VERSION_DEVSERVER="" -elif [ "$USER" = "chrome-bot" ] -then - # Continuous builder - # Sets the codename to the user who built the image. This - # will help us figure out who did the build if a different - # person is debugging the system. - export CHROMEOS_VERSION_CODENAME="$USER" - - export CHROMEOS_VERSION_NAME="Chromium OS" - export CHROMEOS_VERSION_TRACK="buildbot-build" - export CHROMEOS_VERSION_AUSERVER="http://$HOSTNAME:8080/update" - export CHROMEOS_VERSION_DEVSERVER="http://$HOSTNAME:8080" -else - # Developer hand-builds - # Sets the codename to the user who built the image. This - # will help us figure out who did the build if a different - # person is debugging the system. - export CHROMEOS_VERSION_CODENAME="$USER" - - export CHROMEOS_VERSION_NAME="Chromium OS" - export CHROMEOS_VERSION_TRACK="developer-build" - export CHROMEOS_VERSION_AUSERVER="http://$HOSTNAME:8080/update" - export CHROMEOS_VERSION_DEVSERVER="http://$HOSTNAME:8080" - # Overwrite CHROMEOS_VERSION_PATCH with a date string for use by auto-updater - export CHROMEOS_VERSION_PATCH=$(date +%Y_%m_%d_%H%M) -fi - -# Version string. Not indentied to appease bash. -export CHROMEOS_VERSION_STRING=\ -"${CHROMEOS_VERSION_MAJOR}.${CHROMEOS_VERSION_MINOR}"\ -".${CHROMEOS_VERSION_BRANCH}.${CHROMEOS_VERSION_PATCH}" - -# Set CHROME values (Used for releases) to pass to chromeos-chrome-bin ebuild -# URL to chrome archive -export CHROME_BASE= -# export CHROME_VERSION from incoming value or NULL and let ebuild default -export CHROME_VERSION="$CHROME_VERSION" - -# Print (and remember) version info. -echo "ChromeOS version information:" -logvers="/tmp/version_${CHROMEOS_VERSION_STRING}" -env | egrep '^CHROMEOS_VERSION|CHROME_' | tee $logvers | sed 's/^/ /' diff --git a/cros_mark_as_stable.py b/cros_mark_as_stable.py index 7e170d2952..9a87062635 100755 --- a/cros_mark_as_stable.py +++ b/cros_mark_as_stable.py @@ -396,7 +396,8 @@ class EBuildStableMarker(object): @classmethod def MarkAsStable(cls, unstable_ebuild_path, new_stable_ebuild_path, - commit_keyword, commit_value, redirect_file=None): + commit_keyword, commit_value, redirect_file=None, + make_stable=True): """Static function that creates a revved stable ebuild. This function assumes you have already figured out the name of the new @@ -412,6 +413,7 @@ class EBuildStableMarker(object): stable. commit_value: Value to set the above keyword to. redirect_file: Optionally redirect output of new ebuild somewhere else. + make_stable: Actually make the ebuild stable. """ shutil.copyfile(unstable_ebuild_path, new_stable_ebuild_path) for line in fileinput.input(new_stable_ebuild_path, inplace=1): @@ -420,7 +422,10 @@ class EBuildStableMarker(object): redirect_file = sys.stdout if line.startswith('KEYWORDS'): # Actually mark this file as stable by removing ~'s. - redirect_file.write(line.replace('~', '')) + if make_stable: + redirect_file.write(line.replace('~', '')) + else: + redirect_file.write(line) elif line.startswith('EAPI'): # Always add new commit_id after EAPI definition. redirect_file.write(line) diff --git a/enter_chroot.sh b/enter_chroot.sh index 606eaf51a6..431afa3bab 100755 --- a/enter_chroot.sh +++ b/enter_chroot.sh @@ -280,6 +280,8 @@ REVISION=$(cd ${FLAGS_trunk}/src/scripts ; git rev-parse --short=8 HEAD) CHROOT_PASSTHRU="CHROMEOS_REVISION=$REVISION BUILDBOT_BUILD=$FLAGS_build_number CHROMEOS_OFFICIAL=$CHROMEOS_OFFICIAL" CHROOT_PASSTHRU="${CHROOT_PASSTHRU} \ CHROMEOS_RELEASE_APPID=${CHROMEOS_RELEASE_APPID:-"{DEV-BUILD}"}" +CHROOT_PASSTHRU="${CHROOT_PASSTHRU} \ +CHROMEOS_VERSION_TRACK=$CHROMEOS_VERSION_TRACK CHROMEOS_VERSION_AUSERVER=$CHROMEOS_VERSION_AUSERVER CHROMEOS_VERSION_DEVSERVER=$CHROMEOS_VERSION_DEVSERVER" if [ -d "$HOME/.subversion" ]; then # Bind mounting .subversion into chroot diff --git a/generate_au_zip.py b/generate_au_zip.py index f38e1e3aad..fa62ba3bd0 100755 --- a/generate_au_zip.py +++ b/generate_au_zip.py @@ -26,7 +26,7 @@ logging.basicConfig(level=logging.INFO, format=logging_format, def CreateTempDir(): """Creates a tempdir and returns the name of the tempdir.""" temp_dir = tempfile.mkdtemp(suffix='au', prefix='tmp') - logging.info('Using tempdir = %s', temp_dir) + logging.debug('Using tempdir = %s', temp_dir) return temp_dir @@ -66,7 +66,7 @@ def DepsToCopy(ldd_files, black_list): library_list: List of files that are dependencies """ for file_name in ldd_files: - logging.info('Running ldd on %s', file_name) + logging.debug('Running ldd on %s', file_name) cmd = ['/usr/bin/ldd', file_name] stdout_data = '' stderr_data = '' @@ -137,18 +137,18 @@ def CopyRequiredFiles(dest_files_root): logging.debug('Given files that need to be copied = %s' % '' .join(all_files)) all_files += DepsToCopy(ldd_files=ldd_files,black_list=black_list) for file_name in all_files: - logging.info('Copying file %s to %s', file_name, dest_files_root) + logging.debug('Copying file %s to %s', file_name, dest_files_root) shutil.copy2(file_name, dest_files_root) for source_dir, target_dir in recurse_dirs.iteritems(): - logging.info('Processing directory %s', source_dir) + logging.debug('Processing directory %s', source_dir) full_path = os.path.expanduser(source_dir) if not os.path.isdir(full_path): logging.error("Directory given for %s expanded to %s doens't exist.", source_dir, full_path) sys.exit(1) dest = os.path.join(dest_files_root, target_dir) - logging.info('Copying directory %s to %s.', full_path, target_dir) + logging.debug('Copying directory %s to %s.', full_path, target_dir) shutil.copytree(full_path, dest) def CleanUp(temp_dir): @@ -158,7 +158,7 @@ def CleanUp(temp_dir): """ if os.path.exists(temp_dir): shutil.rmtree(temp_dir, ignore_errors=True) - logging.info('Removed tempdir = %s', temp_dir) + logging.debug('Removed tempdir = %s', temp_dir) def GenerateZipFile(base_name, root_dir): """Returns true if able to generate zip file @@ -168,7 +168,7 @@ def GenerateZipFile(base_name, root_dir): Returns: True if successfully generates the zip file otherwise False """ - logging.info('Generating zip file %s with contents from %s', base_name, + logging.debug('Generating zip file %s with contents from %s', base_name, root_dir) current_dir = os.getcwd() os.chdir(root_dir) @@ -223,7 +223,7 @@ def CopyZipToFinalDestination(output_dir, zip_file_name): if not os.path.isdir(output_dir): logging.debug('Creating %s', output_dir) os.makedirs(output_dir) - logging.info('Copying %s to %s', zip_file_name, output_dir) + logging.debug('Copying %s to %s', zip_file_name, output_dir) shutil.copy2(zip_file_name, output_dir) return True @@ -255,6 +255,7 @@ def main(): zip_file_name = os.path.join(temp_dir, options.zip_name) GenerateZipFile(zip_file_name, dest_files_root) CopyZipToFinalDestination(options.output_dir, zip_file_name) + logging.info('Generated %s/%s' % (options.output_dir, options.zip_name)) if not options.keep_temp: CleanUp(temp_dir) diff --git a/image_to_live.sh b/image_to_live.sh index f18badf667..6c237820b5 100755 --- a/image_to_live.sh +++ b/image_to_live.sh @@ -38,6 +38,8 @@ DEFINE_string image "" \ "Update with this image path that is in this source checkout." i DEFINE_string payload "" \ "Update with this update payload, ignoring specified images." +DEFINE_string proxy_port "" \ + "Have the client request from this proxy instead of devserver." DEFINE_string src_image "" \ "Create a delta update by passing in the image on the remote machine." DEFINE_boolean update_stateful ${FLAGS_TRUE} \ @@ -139,6 +141,11 @@ function start_dev_server { --payload $(reinterpret_path_for_chroot ${FLAGS_payload})" fi + if [ -n "${FLAGS_proxy_port}" ]; then + devserver_flags="${devserver_flags} \ + --proxy_port ${FLAGS_proxy_port}" + fi + [ ${FLAGS_for_vm} -eq ${FLAGS_TRUE} ] && \ devserver_flags="${devserver_flags} --for_vm" @@ -146,7 +153,7 @@ function start_dev_server { --src_image=\"$(reinterpret_path_for_chroot ${FLAGS_src_image})\"" info "Starting devserver with flags ${devserver_flags}" - ./enter_chroot.sh "sudo ./start_devserver ${devserver_flags} \ + ./enter_chroot.sh -- sudo sh -c "./start_devserver ${devserver_flags} \ --client_prefix=ChromeOSUpdateEngine \ --board=${FLAGS_board} \ --port=${FLAGS_devserver_port} > ${FLAGS_server_log} 2>&1" & @@ -209,9 +216,15 @@ function get_update_args { function get_devserver_url { local devserver_url="" + local port=${FLAGS_devserver_port} + + if [[ -n ${FLAGS_proxy_port} ]]; then + port=${FLAGS_proxy_port} + fi + if [ ${FLAGS_ignore_hostname} -eq ${FLAGS_TRUE} ]; then if [ -z ${FLAGS_update_url} ]; then - devserver_url="http://$(get_hostname):${FLAGS_devserver_port}/update" + devserver_url="http://$(get_hostname):${port}/update" else devserver_url="${FLAGS_update_url}" fi @@ -356,7 +369,7 @@ function main() { if [ ${FLAGS_update_stateful} -eq ${FLAGS_TRUE} ] && \ ! run_stateful_update; then - warn "Stateful update was not successful." + die "Stateful update was not successful." fi remote_reboot diff --git a/lib/cros_build_lib.py b/lib/cros_build_lib.py index b45733eeda..2e6876060c 100644 --- a/lib/cros_build_lib.py +++ b/lib/cros_build_lib.py @@ -177,6 +177,7 @@ def Die(message): """ print >> sys.stderr, ( Color(_STDOUT_IS_TTY).Color(Color.RED, '\nERROR: ' + message)) + sys.stderr.flush() sys.exit(1) @@ -188,6 +189,7 @@ def Warning(message): """ print >> sys.stderr, ( Color(_STDOUT_IS_TTY).Color(Color.YELLOW, '\nWARNING: ' + message)) + sys.stderr.flush() def Info(message): @@ -198,6 +200,7 @@ def Info(message): """ print >> sys.stderr, ( Color(_STDOUT_IS_TTY).Color(Color.BLUE, '\nINFO: ' + message)) + sys.stderr.flush() def FindRepoDir(path=None): diff --git a/lib/cros_test_proxy.py b/lib/cros_test_proxy.py new file mode 100755 index 0000000000..21709829c3 --- /dev/null +++ b/lib/cros_test_proxy.py @@ -0,0 +1,113 @@ +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import select +import socket +import SocketServer +import threading + +class Filter(object): + """Base class for data filters. + + Pass subclass of this to CrosTestProxy which will perform whatever + connection manipulation you prefer. + """ + + def setup(self): + """This setup method is called once per connection.""" + pass + + def InBound(self, data): + """This method is called once per packet of incoming data. + + The value returned is what is sent through the proxy. If + None is returned, the connection will be closed. + """ + return data + + def OutBound(self, data): + """This method is called once per packet of outgoing data. + + The value returned is what is sent through the proxy. If + None is returned, the connection will be closed. + """ + return data + + +class CrosTestProxy(SocketServer.ThreadingMixIn, SocketServer.TCPServer): + """A transparent proxy for simulating network errors""" + + class _Handler(SocketServer.BaseRequestHandler): + """Proxy connection handler that passes data though a filter""" + + def setup(self): + """Setup is called once for each connection proxied.""" + self.server.filter.setup() + + def handle(self): + """Handles each incoming connection. + + Opens a new connection to the port we are proxing to, then + passes each packet along in both directions after passing + them through the filter object passed in. + """ + # Open outgoing socket + s_in = self.request + s_out = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s_out.connect((self.server.address_out, self.server.port_out)) + + while True: + rlist, wlist, xlist = select.select([s_in, s_out], [], []) + + if s_in in rlist: + data = s_in.recv(1024) + data = self.server.filter.InBound(data) + if not data: break + try: + # If there is any error sending data, close both connections. + s_out.sendall(data) + except socket.error: + break + + if s_out in rlist: + data = s_out.recv(1024) + data = self.server.filter.OutBound(data) + if not data: break + try: + # If there is any error sending data, close both connections. + s_in.sendall(data) + except socket.error: + break + + s_in.close() + s_out.close() + + def __init__(self, + filter, + port_in=8081, + address_out='127.0.0.1', port_out=8080): + """Configures the proxy object. + + Args: + filter: An instance of a subclass of Filter. + port_in: Port on which to listen for incoming connections. + address_out: Address to which outgoing connections will go. + address_port: Port to which outgoing connections will go. + """ + self.port_in = port_in + self.address_out = address_out + self.port_out = port_out + self.filter = filter + + SocketServer.TCPServer.__init__(self, + ('', port_in), + self._Handler) + + def serve_forever_in_thread(self): + """Helper method to start the server in a new background thread.""" + server_thread = threading.Thread(target=self.serve_forever) + server_thread.setDaemon(True) + server_thread.start() + + return server_thread diff --git a/lib/cros_vm_lib.sh b/lib/cros_vm_lib.sh index d28539c14d..3048c3dd30 100644 --- a/lib/cros_vm_lib.sh +++ b/lib/cros_vm_lib.sh @@ -19,6 +19,25 @@ function get_pid() { sudo cat "${KVM_PID_FILE}" } +# General purpose blocking kill on a pid. +# This function sends a specified kill signal [0-9] to a pid and waits for it +# die up to a given timeout. It exponentially backs off it's timeout starting +# at 1 second. +# $1 the process id. +# $2 signal to send (-#). +# $3 max timeout in seconds. +# Returns 0 on success. +function blocking_kill() { + local timeout=1 + sudo kill -$2 $1 + while ps -p $1 > /dev/null && [ ${timeout} -le $3 ]; do + warn "Process still running, sleeping for ${timeout}" + sleep ${timeout} + timeout=$((timeout*2)) + done + ! ps -p ${1} > /dev/null +} + # TODO(rtc): These flags assume that we'll be using KVM on Lucid and won't work # on Hardy. # $1: Path to the virtual image to start. @@ -98,7 +117,7 @@ function stop_kvm() { local pid=$(get_pid) if [ -n "${pid}" ]; then echo "Killing ${pid}" >&2 - sudo kill ${pid} + blocking_kill ${pid} 1 16 || blocking_kill 9 1 sudo rm "${KVM_PID_FILE}" else echo "No kvm pid found to stop." >&2 diff --git a/run_remote_tests.sh b/run_remote_tests.sh index 2a54456f73..9cb1318172 100755 --- a/run_remote_tests.sh +++ b/run_remote_tests.sh @@ -268,7 +268,7 @@ function main() { local autoserv_args="-m ${FLAGS_remote} --ssh-port ${FLAGS_ssh_port} \ ${option} ${control_file} -r ${results_dir} ${verbose}" if [ -n "${FLAGS_args}" ]; then - autoserv_args="${autoserv_args} -a \"${FLAGS_args}\"" + autoserv_args="${autoserv_args} --args=${FLAGS_args}" fi sudo chmod a+w ./server/{tests,site_tests} @@ -291,5 +291,5 @@ function main() { print_time_elapsed } -restart_in_chroot_if_needed $* +restart_in_chroot_if_needed "$@" main "$@" diff --git a/update_kernel.sh b/update_kernel.sh index 126786b2f6..4166709aea 100755 --- a/update_kernel.sh +++ b/update_kernel.sh @@ -66,7 +66,7 @@ function main() { --bootloader /lib64/bootstub/bootstub.efi \ --vmlinuz /build/${FLAGS_board}/boot/vmlinuz" - ./enter_chroot.sh -- "${cmd}" + ./enter_chroot.sh -- ${cmd} learn_partition