From 41a2b700a23df7d6952c75c8d2ca37ba6a9653c7 Mon Sep 17 00:00:00 2001 From: Huyen Nguyen Date: Thu, 4 Nov 2010 12:40:28 -0700 Subject: [PATCH 01/38] Fix bug where subfolder is not add to path of firmware. Change-Id: I5521baa3a2b176167f827034166ffcd443a6a793 BUG=8711 TEST=run locally Review URL: http://codereview.chromium.org/4413004 --- make_factory_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/make_factory_package.sh b/make_factory_package.sh index 8f238fb5d2..f59aada962 100755 --- a/make_factory_package.sh +++ b/make_factory_package.sh @@ -190,7 +190,7 @@ echo -n "{ if [ ! -z "${FLAGS_firmware_updater}" ] ; then echo -n " - 'firmware_image': 'firmware.gz', + 'firmware_image': '"${subfolder}"firmware.gz', 'firmware_checksum': '${firmware_hash}'," >> ${OMAHA_DIR}/miniomaha.conf fi From ca3029e0d2ab0858f3066471d3aa8b23ab00d9dd Mon Sep 17 00:00:00 2001 From: Huyen Nguyen Date: Thu, 4 Nov 2010 12:56:02 -0700 Subject: [PATCH 02/38] Run devserver with --factory_config. Add --start_devserver flag. Change-Id: I9ed0a10033ce608c7ef35465debfc874b55d748c BUG=7474 TEST=run locally Review URL: http://codereview.chromium.org/4452001 --- serve_factory_packages.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/serve_factory_packages.py b/serve_factory_packages.py index 41c35176e9..723386780e 100644 --- a/serve_factory_packages.py +++ b/serve_factory_packages.py @@ -37,7 +37,7 @@ FLAGS = gflags.FLAGS gflags.DEFINE_string('board', None, 'Platform to build.') gflags.DEFINE_string('base_image', None, 'Path to base image.') gflags.DEFINE_string('firmware_updater', None, 'Path to firmware updater.') - +gflags.DEFINE_boolean('start_devserver', False, 'Start devserver.') class KillableProcess(): """A killable process. @@ -77,7 +77,7 @@ class KillableProcess(): def start_devserver(): """Starts devserver.""" - cmd = 'python devserver.py' + cmd = 'python devserver.py --factory_config miniomaha.conf' print 'Running command: %s' % cmd devserver_process = KillableProcess(cmd, cwd=DEVSERVER_DIR) devserver_process.start(wait=False) @@ -174,7 +174,8 @@ def main(argv): FLAGS.firmware_updater, folder=FLAGS.board, board=FLAGS.board) - start_devserver() + if FLAGS.start_devserver: + start_devserver() if __name__ == '__main__': From de285e0238de4f0eb8bbbeb2ad596c864f303296 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Thu, 4 Nov 2010 13:03:33 -0700 Subject: [PATCH 03/38] Add 0.0.1 logic to new cros_mark. Change-Id: I413d21dd60ce822af65d5c7c22fd2dba67858d43 BUG=6113 TEST=Ran unit tests and also created my own 9999 ebuild and ran with --all and saw 0.0.1-r1.ebuild get created in stabilizing_branch Review URL: http://codereview.chromium.org/4449001 --- cros_mark_as_stable.py | 23 +++++++++++++++-------- cros_mark_as_stable_unittest.py | 9 +++++---- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/cros_mark_as_stable.py b/cros_mark_as_stable.py index 9a3ab5221c..56a589c682 100755 --- a/cros_mark_as_stable.py +++ b/cros_mark_as_stable.py @@ -93,8 +93,12 @@ def _BestEBuild(ebuilds): return winner -def _FindStableEBuilds(files): - """Return a list of stable ebuilds from specified list of files. +def _FindUprevCandidates(files): + """Return a list of uprev candidates from specified list of files. + + Usually an uprev candidate is a the stable ebuild in a cros_workon directory. + However, if no such stable ebuild exists (someone just checked in the 9999 + ebuild), this is the unstable ebuild. Args: files: List of files. @@ -131,7 +135,8 @@ def _FindStableEBuilds(files): if not unstable_ebuilds: Die('Missing 9999 ebuild in %s' % os.path.dirname(path)) if not stable_ebuilds: - Die('Missing stable ebuild in %s' % os.path.dirname(path)) + Warning('Missing stable ebuild in %s' % os.path.dirname(path)) + return unstable_ebuilds[0] if stable_ebuilds: return stable_ebuilds[0] @@ -153,7 +158,7 @@ def _BuildEBuildDictionary(overlays, all, packages): for package_dir, dirs, files in os.walk(overlay): # Add stable ebuilds to overlays[overlay]. paths = [os.path.join(package_dir, path) for path in files] - ebuild = _FindStableEBuilds(paths) + ebuild = _FindUprevCandidates(paths) # If the --all option isn't used, we only want to update packages that # are in packages. @@ -377,8 +382,8 @@ class _EBuild(object): else: # Has no revision so we stripped the version number instead. ebuild_no_version = ebuild_no_rev - ebuild_no_rev = ebuild_path.rpartition('.ebuild')[0] - rev_string = "0" + ebuild_no_rev = ebuild_path.rpartition('9999.ebuild')[0] + '0.0.1' + rev_string = '0' revision = int(rev_string) return (ebuild_no_rev, ebuild_no_version, revision) @@ -448,8 +453,10 @@ class EBuildStableMarker(object): _Print('Adding new stable ebuild to git') _SimpleRunCommand('git add %s' % new_ebuild_path) - _Print('Removing old ebuild from git') - _SimpleRunCommand('git rm %s' % old_ebuild_path) + if self._ebuild.is_stable: + _Print('Removing old ebuild from git') + _SimpleRunCommand('git rm %s' % old_ebuild_path) + return True def CommitChange(self, message): diff --git a/cros_mark_as_stable_unittest.py b/cros_mark_as_stable_unittest.py index 73985ae4d2..e7fd3317b2 100755 --- a/cros_mark_as_stable_unittest.py +++ b/cros_mark_as_stable_unittest.py @@ -125,7 +125,7 @@ class EBuildTest(mox.MoxTestBase): def testParseEBuildPathNoRevisionNumber(self): # Test with ebuild without revision number. no_rev, no_version, revision = cros_mark_as_stable._EBuild._ParseEBuildPath( - '/path/test_package-0.0.1.ebuild') + '/path/test_package-9999.ebuild') self.assertEquals(no_rev, '/path/test_package-0.0.1') self.assertEquals(no_version, '/path/test_package') self.assertEquals(revision, 0) @@ -139,6 +139,7 @@ class EBuildStableMarkerTest(mox.MoxTestBase): self.mox.StubOutWithMock(cros_mark_as_stable, 'RunCommand') self.mox.StubOutWithMock(os, 'unlink') self.m_ebuild = self.mox.CreateMock(cros_mark_as_stable._EBuild) + self.m_ebuild.is_stable = True self.m_ebuild.package = 'test_package' self.m_ebuild.current_revision = 1 self.m_ebuild.ebuild_path_no_revision = '/path/test_package-0.0.1' @@ -281,13 +282,13 @@ class BuildEBuildDictionaryTest(mox.MoxTestBase): self.package_path = self.root + '/test_package-0.0.1.ebuild' paths = [[self.root, [], []]] cros_mark_as_stable.os.walk("/overlay").AndReturn(paths) - self.mox.StubOutWithMock(cros_mark_as_stable, '_FindStableEBuilds') + self.mox.StubOutWithMock(cros_mark_as_stable, '_FindUprevCandidates') def testWantedPackage(self): overlays = {"/overlay": []} package = _Package(self.package) - cros_mark_as_stable._FindStableEBuilds([]).AndReturn(package) + cros_mark_as_stable._FindUprevCandidates([]).AndReturn(package) self.mox.ReplayAll() cros_mark_as_stable._BuildEBuildDictionary(overlays, False, [self.package]) self.mox.VerifyAll() @@ -297,7 +298,7 @@ class BuildEBuildDictionaryTest(mox.MoxTestBase): def testUnwantedPackage(self): overlays = {"/overlay": []} package = _Package(self.package) - cros_mark_as_stable._FindStableEBuilds([]).AndReturn(package) + cros_mark_as_stable._FindUprevCandidates([]).AndReturn(package) self.mox.ReplayAll() cros_mark_as_stable._BuildEBuildDictionary(overlays, False, []) self.assertEquals(len(overlays), 1) From 4b029a27f81d7d3c64299f2703d7beed544f3469 Mon Sep 17 00:00:00 2001 From: Darin Petkov Date: Thu, 4 Nov 2010 14:06:57 -0700 Subject: [PATCH 04/38] VM: Use -serial none in no-graphics kvm mode... ...so that the console doesn't get messed up due to the serial port being redirected from stdin. BUG=8729 TEST=./bin/cros_run_parallel_vm_tests suite_Smoke suite_Smoke; checked console Change-Id: I5a0cedba2b0ac7f453b4574666c645e84b30b379 Review URL: http://codereview.chromium.org/4501001 --- bin/cros_run_parallel_vm_tests.py | 7 +------ lib/cros_vm_lib.sh | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/bin/cros_run_parallel_vm_tests.py b/bin/cros_run_parallel_vm_tests.py index b6403bfe65..0c0b08b88a 100755 --- a/bin/cros_run_parallel_vm_tests.py +++ b/bin/cros_run_parallel_vm_tests.py @@ -64,10 +64,6 @@ class ParallelTestRunner(object): """ ssh_port = self._base_ssh_port spawned_tests = [] - # Test runs shouldn't need anything from stdin. However, it seems that - # running with stdin leaves the terminal in a bad state so redirect from - # /dev/null. - dev_null = open('/dev/null') for test in self._tests: args = [ os.path.join(os.path.dirname(__file__), 'cros_run_vm_test'), '--snapshot', # The image is shared so don't modify it. @@ -84,8 +80,7 @@ class ParallelTestRunner(object): if self._order_output: output = tempfile.NamedTemporaryFile(prefix='parallel_vm_test_') Info('Piping output to %s.' % output.name) - proc = subprocess.Popen(args, stdin=dev_null, stdout=output, - stderr=output) + proc = subprocess.Popen(args, stdout=output, stderr=output) test_info = { 'test': test, 'proc': proc, 'output': output } diff --git a/lib/cros_vm_lib.sh b/lib/cros_vm_lib.sh index 1809acc9d1..afa481e8fb 100644 --- a/lib/cros_vm_lib.sh +++ b/lib/cros_vm_lib.sh @@ -40,7 +40,7 @@ function start_kvm() { local nographics="" local usesnapshot="" if [ ${FLAGS_no_graphics} -eq ${FLAGS_TRUE} ]; then - nographics="-nographic" + nographics="-nographic -serial none" fi if [ ${FLAGS_snapshot} -eq ${FLAGS_TRUE} ]; then From 5494da3107a51c6cbc038fd6b98fa5008c193473 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Thu, 4 Nov 2010 17:33:46 -0700 Subject: [PATCH 05/38] Pass board to devserver for latest image. Change-Id: I339e1ce3d8f73e15aa554e1111aced163819ec23 BUG=8732 TEST=Ran with other CL and using latest image. Review URL: http://codereview.chromium.org/4527001 --- image_to_live.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/image_to_live.sh b/image_to_live.sh index b27a87db54..ad57811741 100755 --- a/image_to_live.sh +++ b/image_to_live.sh @@ -139,6 +139,7 @@ function start_dev_server { info "Starting devserver with flags ${devserver_flags}" ./enter_chroot.sh "sudo ./start_devserver ${devserver_flags} \ --client_prefix=ChromeOSUpdateEngine \ + --board=${FLAGS_board} \ --port=${FLAGS_devserver_port} > ${FLAGS_server_log} 2>&1" & info "Waiting on devserver to start" From 59752f1667c56f7d26243797433507bfe795ff89 Mon Sep 17 00:00:00 2001 From: Will Drewry Date: Thu, 4 Nov 2010 21:58:02 -0500 Subject: [PATCH 06/38] create_legacy_bootloader_templates.sh: ensure grub escapes quotes Evidently no one has tested the grub.cfg since verified boot landed. Grub eats double quotes which breaks verified boot. This changes adds the escapes. No other code changes should be needed in build and install code since they replace DMTABLEA and DMTABLEB and not the quotes. TEST=nsanders escaped the quotes in his build :); I am doing a new build to ensure the output is escaped now. BUG=chrome-os-partner:1603 Change-Id: I55e823347c91493caf3b5b2e4577a4e4616df7f0 Review URL: http://codereview.chromium.org/4529002 --- create_legacy_bootloader_templates.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/create_legacy_bootloader_templates.sh b/create_legacy_bootloader_templates.sh index 955da4826d..8bfa761617 100755 --- a/create_legacy_bootloader_templates.sh +++ b/create_legacy_bootloader_templates.sh @@ -191,11 +191,11 @@ menuentry "local image B" { } menuentry "verified image A" { - linux \$grubpartA/boot/vmlinuz ${common_args} ${verity_common} i915.modeset=1 cros_efi root=/dev/dm-0 dm="DMTABLEA" + linux \$grubpartA/boot/vmlinuz ${common_args} ${verity_common} i915.modeset=1 cros_efi root=/dev/dm-0 dm=\\"DMTABLEA\\" } menuentry "verified image B" { - linux \$grubpartB/boot/vmlinuz ${common_args} ${verity_common} i915.modeset=1 cros_efi root=/dev/dm-0 dm="DMTABLEB" + linux \$grubpartB/boot/vmlinuz ${common_args} ${verity_common} i915.modeset=1 cros_efi root=/dev/dm-0 dm=\\"DMTABLEB\\" } # FIXME: usb doesn't support verified boot for now From 137eab61299e82ce3c9e495c5ad5e7aa90026171 Mon Sep 17 00:00:00 2001 From: Will Drewry Date: Thu, 4 Nov 2010 22:06:41 -0500 Subject: [PATCH 07/38] build_image: enable new cros specific error handling in dm-verity A new error mode for dm-verity exists which can be called as "cros" or "3". This change enables that mode in builds by default. TEST=build_image had the correct argument; functionality was there [in progress] BUG=chromium-os:8442 Change-Id: I3eb3f4a662c5d36011542af4e50718db50ab63bd Review URL: http://codereview.chromium.org/4530003 --- build_image | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build_image b/build_image index 19fdd4eed4..7b89a3e15b 100755 --- a/build_image +++ b/build_image @@ -71,9 +71,9 @@ DEFINE_string usb_disk /dev/sdb3 \ DEFINE_boolean enable_rootfs_verification ${FLAGS_TRUE} \ "Default all bootloaders to use kernel-based root fs integrity checking." -DEFINE_integer verity_error_behavior 1 \ - "Kernel verified boot error behavior (0: I/O errors, 1: panic, 2: nothing) \ -Default: 1" +DEFINE_integer verity_error_behavior 3 \ + "Kernel verified boot error behavior (0: I/O errors, 1: panic, 2: nothing, \ +3: cros) Default: 3" DEFINE_integer verity_depth 1 \ "Kernel verified boot hash tree depth. Default: 1" DEFINE_integer verity_max_ios -1 \ From 170bde0fd472de545dc10e92ad5135fb3ff2a361 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Thu, 4 Nov 2010 20:49:30 -0700 Subject: [PATCH 08/38] Fix for bug 8764 that breaks prebuilts for files with more than just keyword pairs. I've also modified the unit test so it actually tests for this case. Change-Id: I6688d3be7bf1d8962031fcf1d684ddcaab43ba9f BUG=8764 TEST=Unittests Review URL: http://codereview.chromium.org/4566001 --- prebuilt.py | 4 ++-- prebuilt_unittest.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/prebuilt.py b/prebuilt.py index df7f6e36dc..a029159ed4 100755 --- a/prebuilt.py +++ b/prebuilt.py @@ -92,8 +92,8 @@ def UpdateLocalFile(filename, value, key='PORTAGE_BINHOST'): # Strip newlines from end of line. We already add newlines below. line = line.rstrip("\n") - if '=' not in line: - # Skip any line without an equal in it and just write it out. + if len(line.split('=')) != 2: + # Skip any line that doesn't fit key=val. file_lines.append(line) continue diff --git a/prebuilt_unittest.py b/prebuilt_unittest.py index 37d24afde1..fe2387fbf7 100755 --- a/prebuilt_unittest.py +++ b/prebuilt_unittest.py @@ -17,7 +17,9 @@ class TestUpdateFile(unittest.TestCase): self.contents_str = ['# comment that should be skipped', 'PKGDIR="/var/lib/portage/pkgs"', 'PORTAGE_BINHOST="http://no.thanks.com"', - 'portage portage-20100310.tar.bz2'] + 'portage portage-20100310.tar.bz2', + 'COMPILE_FLAGS="some_value=some_other"', + ] temp_fd, self.version_file = tempfile.mkstemp() os.write(temp_fd, '\n'.join(self.contents_str)) os.close(temp_fd) From 6418c50450ad23ca009f690e111c5ad1b0d846a3 Mon Sep 17 00:00:00 2001 From: David James Date: Thu, 4 Nov 2010 23:48:52 -0700 Subject: [PATCH 09/38] Update parallel_emerge to support --force-remote-binary. This feature is useful if you want to stick to an old binary package of a package you're not working on. BUG=chromium-os:8769 TEST=./parallel_emerge --board=x86-mario -uDNvg chromeos --force-remote-binary=chromeos-chrome --force-remote-binary=libcros Change-Id: I7d3011fa64134158ed848f136bc75e09b0af438e Review URL: http://codereview.chromium.org/4555002 --- parallel_emerge | 217 +++++++++++++++++++++++++++++------------------- 1 file changed, 132 insertions(+), 85 deletions(-) diff --git a/parallel_emerge b/parallel_emerge index 66b2334da6..c1667134c0 100755 --- a/parallel_emerge +++ b/parallel_emerge @@ -7,7 +7,7 @@ Usage: ./parallel_emerge [--board=BOARD] [--workon=PKGS] [--no-workon-deps] - [emerge args] package" + [--force-remote-binary=PKGS] [emerge args] package Basic operation: Runs 'emerge -p --debug' to display dependencies, and stores a @@ -84,6 +84,7 @@ from _emerge.Scheduler import Scheduler from _emerge.stdout_spinner import stdout_spinner import portage import portage.debug +import portage.versions def Usage(): @@ -218,7 +219,8 @@ class DepGraphGenerator(object): """ __slots__ = ["board", "emerge", "mandatory_source", "no_workon_deps", - "nomerge", "package_db", "rebuild", "show_output"] + "nomerge", "package_db", "rebuild", "show_output", + "force_remote_binary", "forced_remote_binary_packages"] def __init__(self): self.board = None @@ -229,6 +231,8 @@ class DepGraphGenerator(object): self.package_db = {} self.rebuild = False self.show_output = False + self.force_remote_binary = set() + self.forced_remote_binary_packages = set() def ParseParallelEmergeArgs(self, argv): """Read the parallel emerge arguments from the command-line. @@ -251,6 +255,11 @@ class DepGraphGenerator(object): workon_str = arg.replace("--workon=", "") package_list = shlex.split(" ".join(shlex.split(workon_str))) self.mandatory_source.update(package_list) + elif arg.startswith("--force-remote-binary="): + force_remote_binary = arg.replace("--force-remote-binary=", "") + force_remote_binary = \ + shlex.split(" ".join(shlex.split(force_remote_binary))) + self.force_remote_binary.update(force_remote_binary) elif arg.startswith("--nomerge="): nomerge_str = arg.replace("--nomerge=", "") package_list = shlex.split(" ".join(shlex.split(nomerge_str))) @@ -460,7 +469,7 @@ class DepGraphGenerator(object): cur_iuse, now_use, now_iuse) return not flags - def GenDependencyTree(self): + def GenDependencyTree(self, remote_pkgs): """Get dependency tree info from emerge. TODO(): Update cros_extract_deps to also use this code. @@ -479,10 +488,7 @@ class DepGraphGenerator(object): # --workon and the dependencies have changed. emerge = self.emerge emerge_opts = emerge.opts.copy() - emerge_opts.pop("--getbinpkg", None) - if "--usepkgonly" not in emerge_opts: - emerge_opts.pop("--usepkg", None) - if self.mandatory_source or self.rebuild: + if self.mandatory_source or self.rebuild or self.force_remote_binary: # Enable --emptytree so that we get the full tree, which we need for # dependency analysis. By default, with this option, emerge optimizes # the graph by removing uninstall instructions from the graph. By @@ -491,10 +497,30 @@ class DepGraphGenerator(object): emerge_opts["--tree"] = True emerge_opts["--emptytree"] = True + # Tell emerge not to worry about use flags yet. We handle those inside + # parallel_emerge itself. Further, when we use the --force-remote-binary + # flag, we don't emerge to reject a package just because it has different + # use flags. + emerge_opts.pop("--newuse", None) + emerge_opts.pop("--reinstall", None) + # Create a list of packages to merge packages = set(emerge.cmdline_packages[:]) if self.mandatory_source: packages.update(self.mandatory_source) + if self.force_remote_binary: + forced_pkgs = {} + for pkg in remote_pkgs: + category, pkgname, _, _ = portage.catpkgsplit(pkg) + full_pkgname = "%s/%s" % (category, pkgname) + if (pkgname in self.force_remote_binary or + full_pkgname in self.force_remote_binary): + forced_pkgs.setdefault(full_pkgname, []).append(pkg) + + for pkgs in forced_pkgs.values(): + forced_package = portage.versions.best(pkgs) + packages.add("=%s" % forced_package) + self.forced_remote_binary_packages.add(forced_package) # Tell emerge to be quiet. We print plenty of info ourselves so we don't # need any extra output from portage. @@ -580,9 +606,6 @@ class DepGraphGenerator(object): optional = True break - # Add the package to our database. - self.package_db[str(pkg.cpv)] = pkg - # Save off info about the package deps_info[str(pkg.cpv)] = {"idx": len(deps_info), "optional": optional} @@ -611,7 +634,65 @@ class DepGraphGenerator(object): print "%s %s (%s)" % (depth, entry, action) self.PrintTree(deps[entry]["deps"], depth=depth + " ") - def GenDependencyGraph(self, deps_tree, deps_info): + def RemotePackageDatabase(self, binhost_url): + """Grab the latest binary package database from the prebuilt server. + + We need to know the modification times of the prebuilt packages so that we + know when it is OK to use these packages and when we should rebuild them + instead. + + Args: + binhost_url: Base URL of remote packages (PORTAGE_BINHOST). + + Returns: + A dict mapping package identifiers to modification times. + """ + + if not binhost_url: + return {} + + def retry_urlopen(url, tries=3): + """Open the specified url, retrying if we run into network errors. + + We do not retry for HTTP errors. + + Args: + url: The specified url. + tries: The number of times to try. + + Returns: + The result of urllib2.urlopen(url). + """ + for i in range(tries): + try: + return urllib2.urlopen(url) + except urllib2.HTTPError as e: + raise + except urllib2.URLError as e: + if i + 1 == tries: + raise + else: + print "Cannot GET %s: %s" % (url, e) + + url = os.path.join(binhost_url, "Packages") + try: + f = retry_urlopen(url) + except urllib2.HTTPError as e: + if e.code == 404: + return {} + else: + raise + prebuilt_pkgs = {} + for line in f: + if line.startswith("CPV: "): + pkg = line.replace("CPV: ", "").rstrip() + elif line.startswith("MTIME: "): + prebuilt_pkgs[pkg] = int(line[:-1].replace("MTIME: ", "")) + f.close() + + return prebuilt_pkgs + + def GenDependencyGraph(self, deps_tree, deps_info, remote_pkgs): """Generate a doubly linked dependency graph. Args: @@ -660,6 +741,10 @@ class DepGraphGenerator(object): # If true, indicates that this package must be installed. We don't care # whether it's binary or source, unless the mandatory_source flag is # also set. + # - force_remote_binary: + # If true, indicates that we want to update to the latest remote prebuilt + # of this package. Packages that depend on this package should be built + # from source. # deps_map = {} @@ -678,7 +763,8 @@ class DepGraphGenerator(object): # Create an entry for the package action = packages[pkg]["action"] default_pkg = {"needs": {}, "provides": set(), "action": action, - "mandatory_source": False, "mandatory": False} + "mandatory_source": False, "mandatory": False, + "force_remote_binary": False} this_pkg = deps_map.setdefault(pkg, default_pkg) # Create entries for dependencies of this package first. @@ -909,64 +995,6 @@ class DepGraphGenerator(object): if this_pkg["action"] == "nomerge": this_pkg["action"] = "merge" - def RemotePackageDatabase(binhost_url): - """Grab the latest binary package database from the prebuilt server. - - We need to know the modification times of the prebuilt packages so that we - know when it is OK to use these packages and when we should rebuild them - instead. - - Args: - binhost_url: Base URL of remote packages (PORTAGE_BINHOST). - - Returns: - A dict mapping package identifiers to modification times. - """ - - if not binhost_url: - return {} - - def retry_urlopen(url, tries=3): - """Open the specified url, retrying if we run into network errors. - - We do not retry for HTTP errors. - - Args: - url: The specified url. - tries: The number of times to try. - - Returns: - The result of urllib2.urlopen(url). - """ - for i in range(tries): - try: - return urllib2.urlopen(url) - except urllib2.HTTPError as e: - raise - except urllib2.URLError as e: - if i + 1 == tries: - raise - else: - print "Cannot GET %s: %s" % (url, e) - - url = binhost_url + "/Packages" - try: - f = retry_urlopen(url) - except urllib2.HTTPError as e: - if e.code == 404: - return {} - else: - raise - prebuilt_pkgs = {} - for line in f: - if line.startswith("CPV: "): - pkg = line.replace("CPV: ", "").rstrip() - elif line.startswith("MTIME: "): - prebuilt_pkgs[pkg] = int(line[:-1].replace("MTIME: ", "")) - f.close() - - return prebuilt_pkgs - def LocalPackageDatabase(): """Get the modification times of the packages in the local database. @@ -1019,7 +1047,7 @@ class DepGraphGenerator(object): """ if pkg in cache: return cache[pkg] - if pkg not in pkg_db: + if pkg not in pkg_db and pkg not in self.forced_remote_binary_packages: cache[pkg] = False else: cache[pkg] = True @@ -1081,7 +1109,7 @@ class DepGraphGenerator(object): else: MergeChildren(pkg, "mandatory_source") - def UsePrebuiltPackages(): + def UsePrebuiltPackages(remote_pkgs): """Update packages that can use prebuilts to do so.""" start = time.time() @@ -1099,13 +1127,18 @@ class DepGraphGenerator(object): # Build list of prebuilt packages for pkg, info in deps_map.iteritems(): - if info and not info["mandatory_source"] and info["action"] == "merge": + if info and info["action"] == "merge": + if (not info["force_remote_binary"] and info["mandatory_source"] or + "--usepkgonly" not in emerge.opts and pkg not in remote_pkgs): + continue + db_keys = list(bindb._aux_cache_keys) try: db_vals = bindb.aux_get(pkg, db_keys + ["MTIME"]) except KeyError: # No binary package continue + mtime = int(db_vals.pop() or 0) metadata = zip(db_keys, db_vals) db_pkg = Package(built=True, cpv=pkg, installed=False, @@ -1116,15 +1149,16 @@ class DepGraphGenerator(object): # Calculate what packages need to be rebuilt due to changes in use flags. for pkg, db_pkg in prebuilt_pkgs.iteritems(): - db_pkg_src = self.package_db[pkg] - if not self.CheckUseFlags(pkgsettings, db_pkg, db_pkg_src): + db_pkg_src = self.package_db.get(pkg) + if db_pkg_src and not self.CheckUseFlags(pkgsettings, db_pkg, + db_pkg_src): MergeChildren(pkg, "mandatory_source") # Convert eligible packages to binaries. for pkg, info in deps_map.iteritems(): - if (info and not info["mandatory_source"] and - info["action"] == "merge" and pkg in prebuilt_pkgs): - self.package_db[pkg] = prebuilt_pkgs[pkg] + if info and info["action"] == "merge" and pkg in prebuilt_pkgs: + if not info["mandatory_source"] or info["force_remote_binary"]: + self.package_db[pkg] = prebuilt_pkgs[pkg] seconds = time.time() - start if "--quiet" not in emerge.opts: @@ -1154,6 +1188,18 @@ class DepGraphGenerator(object): BuildFinalPackageSet() AddSecretDeps() + # Mark that we want to use remote binaries only for a particular package. + vardb = emerge.depgraph._frozen_config.trees[root]["vartree"].dbapi + for pkg in self.force_remote_binary: + for db_pkg in final_db.match_pkgs(pkg): + match = deps_map.get(str(db_pkg.cpv)) + if match: + match["force_remote_binary"] = True + + rebuild_blacklist.add(str(db_pkg.cpv)) + if not vardb.match_pkgs(db_pkg.cpv): + MergeChildren(str(db_pkg.cpv), "mandatory") + if self.no_workon_deps: for pkg in self.mandatory_source.copy(): for db_pkg in final_db.match_pkgs(pkg): @@ -1166,10 +1212,6 @@ class DepGraphGenerator(object): cycles = FindCycles() if self.rebuild: local_pkgs = LocalPackageDatabase() - remote_pkgs = {} - if "--getbinpkg" in emerge.opts: - binhost = emerge.settings["PORTAGE_BINHOST"] - remote_pkgs = RemotePackageDatabase(binhost) AutoRebuildDeps(local_pkgs, remote_pkgs, cycles) # We need to remove installed packages so that we can use the dependency @@ -1180,7 +1222,7 @@ class DepGraphGenerator(object): SanitizeTree() if deps_map: if "--usepkg" in emerge.opts: - UsePrebuiltPackages() + UsePrebuiltPackages(remote_pkgs) AddRemainingPackages() return deps_map @@ -1734,13 +1776,18 @@ def main(): print " Skipping package %s on %s" % (nomerge_packages, deps.board or "root") - deps_tree, deps_info = deps.GenDependencyTree() + remote_pkgs = {} + if "--getbinpkg" in emerge.opts: + binhost = emerge.settings["PORTAGE_BINHOST"] + remote_pkgs = deps.RemotePackageDatabase(binhost) + + deps_tree, deps_info = deps.GenDependencyTree(remote_pkgs) # You want me to be verbose? I'll give you two trees! Twice as much value. if "--tree" in emerge.opts and "--verbose" in emerge.opts: deps.PrintTree(deps_tree) - deps_graph = deps.GenDependencyGraph(deps_tree, deps_info) + deps_graph = deps.GenDependencyGraph(deps_tree, deps_info, remote_pkgs) # OK, time to print out our progress so far. deps.PrintInstallPlan(deps_graph) From 16106bade8fd32de77e91de6c1c8a03088371175 Mon Sep 17 00:00:00 2001 From: Will Drewry Date: Fri, 5 Nov 2010 10:10:22 -0500 Subject: [PATCH 10/38] mod_image_for_recovery: keep the stateful partition around. Developer images may have data on stateful they want to keep. In that case, just swap the kernels. BUG=chromium-os:7451 TEST=tested with a --withdev build and stateful was intact and the recovery install worked. Change-Id: I59d7f2e99892448d1eaf964e6292611accb2035c Review URL: http://codereview.chromium.org/4418002 --- mod_image_for_recovery.sh | 82 ++++++++++++++++++++++++++++++++------- 1 file changed, 68 insertions(+), 14 deletions(-) diff --git a/mod_image_for_recovery.sh b/mod_image_for_recovery.sh index 6a0b261330..3e4a6f9780 100755 --- a/mod_image_for_recovery.sh +++ b/mod_image_for_recovery.sh @@ -27,7 +27,7 @@ get_default_board DEFINE_string board "$DEFAULT_BOARD" "Board for which the image was built" b DEFINE_integer statefulfs_sectors 4096 \ - "Number of sectors to use for the stateful filesystem" + "Number of sectors to use for the stateful filesystem when minimizing" # Skips the build steps and just does the kernel swap. DEFINE_string kernel_image "" \ "Path to a pre-built recovery kernel" @@ -42,6 +42,12 @@ DEFINE_boolean kernel_image_only $FLAGS_FALSE \ "Emit the recovery kernel image only" DEFINE_boolean sync_keys $FLAGS_TRUE \ "Update the kernel to be installed with the vblock from stateful" +DEFINE_boolean minimize_image $FLAGS_TRUE \ + "Decides if the original image is used or a minimal recovery image is \ +created." +DEFINE_boolean modify_in_place $FLAGS_FALSE \ + "Modifies the source image in place. This cannot be used with \ +--minimize_image." DEFINE_integer jobs -1 \ "How many packages to build in parallel at maximum." j DEFINE_string build_root "/build" \ @@ -50,6 +56,9 @@ DEFINE_string build_root "/build" \ DEFINE_string rootfs_hash "/tmp/rootfs.hash" \ "Path where the rootfs hash should be stored." +DEFINE_boolean verbose $FLAGS_FALSE \ + "Log all commands to stdout." v + # Keep in sync with build_image. DEFINE_string keys_dir "/usr/share/vboot/devkeys" \ "Directory containing the signing keys." @@ -58,6 +67,17 @@ DEFINE_string keys_dir "/usr/share/vboot/devkeys" \ FLAGS "$@" || exit 1 eval set -- "${FLAGS_ARGV}" +if [ $FLAGS_verbose -eq $FLAGS_FALSE ]; then + exec 2>/dev/null + # Redirecting to stdout instead of stderr since + # we silence stderr above. + die() { + echo -e "${V_BOLD_RED}ERROR : $1${V_VIDOFF}" + exit 1 + } +fi +set -x # Make debugging with -v easy. + EMERGE_CMD="emerge" EMERGE_BOARD_CMD="emerge-${FLAGS_board}" @@ -178,10 +198,11 @@ create_recovery_kernel_image() { local kern_tmp=$(mktemp) local kern_hash= - dd if="$FLAGS_image" bs=512 count=$kern_size skip=$kern_offset of="$kern_tmp" + dd if="$FLAGS_image" bs=512 count=$kern_size \ + skip=$kern_offset of="$kern_tmp" 1>&2 # We're going to use the real signing block. if [ $FLAGS_sync_keys -eq $FLAGS_TRUE ]; then - dd if="$INSTALL_VBLOCK" of="$kern_tmp" conv=notrunc + dd if="$INSTALL_VBLOCK" of="$kern_tmp" conv=notrunc 1>&2 fi local kern_hash=$(sha1sum "$kern_tmp" | cut -f1 -d' ') rm "$kern_tmp" @@ -200,7 +221,7 @@ create_recovery_kernel_image() { --root=${cros_root} \ --keys_dir="${FLAGS_keys_dir}" \ --nouse_dev_keys \ - ${verity_args} + ${verity_args} 1>&2 sudo rm "$FLAGS_rootfs_hash" sudo losetup -d "$root_dev" trap - RETURN @@ -236,6 +257,13 @@ install_recovery_kernel() { local kern_a_size=$(partsize "$RECOVERY_IMAGE" 2) local kern_b_offset=$(partoffset "$RECOVERY_IMAGE" 4) local kern_b_size=$(partsize "$RECOVERY_IMAGE" 4) + + if [ $kern_b_size -eq 1 ]; then + echo "Image was created with no KERN-B partition reserved!" 1>&2 + echo "Cannot proceed." 1>&2 + return 1 + fi + # Backup original kernel to KERN-B dd if="$RECOVERY_IMAGE" of="$RECOVERY_IMAGE" bs=512 \ count=$kern_a_size \ @@ -277,14 +305,22 @@ install_recovery_kernel() { } maybe_resize_stateful() { + # If we're not minimizing, then just copy and go. + if [ $FLAGS_minimize_image -eq $FLAGS_FALSE ]; then + if [ "$FLAGS_image" != "$RECOVERY_IMAGE" ]; then + cp "$FLAGS_image" "$RECOVERY_IMAGE" + fi + return 0 + fi + # Rebuild the image with a 1 sector stateful partition local err=0 local small_stateful=$(mktemp) dd if=/dev/zero of="$small_stateful" bs=512 \ - count=${FLAGS_statefulfs_sectors} + count=${FLAGS_statefulfs_sectors} 1>&2 trap "rm $small_stateful" RETURN # Don't bother with ext3 for such a small image. - /sbin/mkfs.ext2 -F -b 4096 "$small_stateful" + /sbin/mkfs.ext2 -F -b 4096 "$small_stateful" 1>&2 # If it exists, we need to copy the vblock over to stateful # This is the real vblock and not the recovery vblock. @@ -301,14 +337,20 @@ maybe_resize_stateful() { # Create a recovery image of the right size # TODO(wad) Make the developer script case create a custom GPT with # just the kernel image and stateful. - update_partition_table "$FLAGS_image" "$small_stateful" 4096 "$RECOVERY_IMAGE" + update_partition_table "$FLAGS_image" "$small_stateful" 4096 \ + "$RECOVERY_IMAGE" 1>&2 return $err } -# main process begins here. +cleanup() { + set +e + if [ "$FLAGS_image" != "$RECOVERY_IMAGE" ]; then + rm "$RECOVERY_IMAGE" + fi + rm "$INSTALL_VBLOCK" +} -# Make sure this is really what the user wants, before nuking the device -echo "Creating recovery image ${FLAGS_to} from ${FLAGS_image} . . . " +# main process begins here. set -e set -u @@ -335,6 +377,15 @@ if [ $FLAGS_kernel_image_only -eq $FLAGS_TRUE -a \ die "Cannot use --kernel_image_only with --kernel_image" fi +if [ $FLAGS_modify_in_place -eq $FLAGS_TRUE ]; then + if [ $FLAGS_minimize_image -eq $FLAGS_TRUE ]; then + die "Cannot use --modify_in_place and --minimize_image together." + fi + RECOVERY_IMAGE="${FLAGS_image}" +fi + +echo "Creating recovery image from ${FLAGS_image}" + INSTALL_VBLOCK=$(get_install_vblock) if [ -z "$INSTALL_VBLOCK" ]; then die "Could not copy the vblock from stateful." @@ -343,10 +394,10 @@ fi if [ -z "$FLAGS_kernel_image" ]; then emerge_recovery_kernel create_recovery_kernel_image + echo "Recovery kernel created at $RECOVERY_KERNEL_IMAGE" else RECOVERY_KERNEL_IMAGE="$FLAGS_kernel_image" fi -echo "Kernel emitted: $RECOVERY_KERNEL_IMAGE." if [ $FLAGS_kernel_image_only -eq $FLAGS_TRUE ]; then echo "Kernel emitted. Stopping there." @@ -354,13 +405,16 @@ if [ $FLAGS_kernel_image_only -eq $FLAGS_TRUE ]; then exit 0 fi -rm "$RECOVERY_IMAGE" || true # Start fresh :) +if [ $FLAGS_modify_in_place -eq $FLAGS_FALSE ]; then + rm "$RECOVERY_IMAGE" || true # Start fresh :) +fi -trap "rm \"$RECOVERY_IMAGE\" && rm \"$INSTALL_VBLOCK\"" EXIT +trap cleanup EXIT -maybe_resize_stateful # Also copies the image +maybe_resize_stateful # Also copies the image if needed. install_recovery_kernel +echo "Recovery image created at $RECOVERY_IMAGE" print_time_elapsed trap - EXIT From f010d9365f05bfd4a299993617ce7093f1ebed02 Mon Sep 17 00:00:00 2001 From: Ken Mixter Date: Fri, 5 Nov 2010 10:21:08 -0700 Subject: [PATCH 11/38] crosutils: Make running tests easier for developers Add "fast" mode to run_remote_tests which causes the user to no longer need to emerge autotests before running them (even ones that require compilation.) Change-Id: I60c1e8bfe562a787075b4f65b714e221e51934f7 BUG=8784 TEST=Ran bvts in and out of fast mode, ran UserCrash, SanAngeles, TPM, and backlight tests (which require compilation) in fast mode. Review URL: http://codereview.chromium.org/4020004 --- autotest_workon | 229 -------------------------------------------- run_remote_tests.sh | 229 ++++++++++++++++++++++++++------------------ 2 files changed, 134 insertions(+), 324 deletions(-) delete mode 100755 autotest_workon diff --git a/autotest_workon b/autotest_workon deleted file mode 100755 index ee12f4ad47..0000000000 --- a/autotest_workon +++ /dev/null @@ -1,229 +0,0 @@ -#!/usr/bin/python - -# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -# -# A python wrapper to call autotest ebuild. - -import commands, logging, optparse, os, subprocess, sys - - -def run(cmd): - return subprocess.call(cmd, stdout=sys.stdout, stderr=sys.stderr) - - -class MyOptionParser(optparse.OptionParser): - """Override python's builtin OptionParser to accept any undefined args.""" - - help = False - - def _process_args(self, largs, rargs, values): - # see /usr/lib64/python2.6/optparse.py line 1414-1463 - while rargs: - arg = rargs[0] - # We handle bare "--" explicitly, and bare "-" is handled by the - # standard arg handler since the short arg case ensures that the - # len of the opt string is greater than 1. - if arg == "--": - del rargs[0] - return - elif arg[0:2] == "--": - # process a single long option (possibly with value(s)) - try: - self._process_long_opt(rargs, values) - except optparse.BadOptionError: - largs.append(arg) - elif arg[:1] == "-" and len(arg) > 1: - # process a cluster of short options (possibly with - # value(s) for the last one only) - try: - self._process_short_opts(rargs, values) - except optparse.BadOptionError: - largs.append(arg) - elif self.allow_interspersed_args: - largs.append(arg) - del rargs[0] - else: - return # stop now, leave this arg in rargs - - def print_help(self, file=None): - optparse.OptionParser.print_help(self, file) - MyOptionParser.help = True - - -parser = MyOptionParser() -parser.allow_interspersed_args = True - -DEFAULT_BOARD = os.environ.get('DEFAULT_BOARD', '') - -parser.add_option('--args', dest='args', action='store', - default='', - help='The arguments to pass to the test control file.') -parser.add_option('--autox', dest='autox', action='store_true', - default=True, - help='Build autox along with autotest [default].') -parser.add_option('--noautox', dest='autox', action='store_false', - help='Don\'t build autox along with autotest.') -parser.add_option('--board', dest='board', action='store', - default=DEFAULT_BOARD, - help='The board for which you are building autotest.') -parser.add_option('--build', dest='build', action='store', - help='Only prebuild client tests, do not run tests.') -parser.add_option('--buildcheck', dest='buildcheck', action='store_true', - default=True, - help='Fail if tests fail to build [default].') -parser.add_option('--nobuildcheck', dest='buildcheck', action='store_false', - help='Ignore test build failures.') -parser.add_option('--jobs', dest='jobs', action='store', type=int, - default=-1, - help='How many packages to build in parallel at maximum.') -parser.add_option('--noprompt', dest='noprompt', action='store_true', - help='Prompt user when building all tests.') - - -AUTOSERV='../third_party/autotest/files/server/autoserv' -AUTOTEST_CLIENT='../third_party/autotest/files/client/bin/autotest_client' - -def parse_args_and_help(): - - def nop(_): - pass - - sys_exit = sys.exit - sys.exit = nop - options, args = parser.parse_args() - sys.exit = sys_exit - - if not args and not options.build: - parser.print_help() - - if MyOptionParser.help: - if options.build: - print - print 'Options inherited from autotest_client, which is used in build', - print 'only mode.' - run([AUTOTEST_CLIENT, '--help']) - else: - print - print 'Options inherited from autoserv:' - run([AUTOSERV, '--help']) - sys.exit(0) - return options, args - - -def assert_inside_chroot(common_sh): - status, output = commands.getstatusoutput('/bin/bash -c ". %s && ' - 'assert_inside_chroot"' % common_sh) - if status is not 0: - print >> sys.stderr, output - sys.exit(status) - - -def set_common_env(common_sh, env_var): - env_value = commands.getoutput('/bin/bash -c \'. %s && echo $%s\'' % - (common_sh, env_var)) - os.environ[env_var] = env_value - - -def die(common_sh, msg): - output = commands.getoutput('/bin/bash -c \'. %s && die "%s"\'' % - (common_sh, msg)) - print >> sys.stderr, output - sys.exit(1) - - -def build_autotest(options): - environ = os.environ - if options.jobs != -1: - emerge_jobs = '--jobs=%d' % options.jobs - else: - emerge_jobs = '' - - # Decide on USE flags based on options - use_flag = environ.get('USE', '') - if not options.autox: - use_flag = use_flag + ' -autox' - if options.buildcheck: - use_flag = use_flag + ' buildcheck' - - board_blacklist_file = ('%s/src/overlays/overlay-%s/autotest-blacklist' % - (os.environ['GCLIENT_ROOT'], options.board)) - if os.path.exists(board_blacklist_file): - blacklist = [line.strip() - for line in open(board_blacklist_file).readlines()] - else: - blacklist = [] - - all_tests = ('compilebench,dbench,disktest,fsx,hackbench,iperf,ltp,netperf2,' - 'netpipe,unixbench') - site_tests = '../third_party/autotest/files/client/site_tests' - for site_test in os.listdir(site_tests): - test_path = os.path.join(site_tests, site_test) - test_py = os.path.join(test_path, '%s.py' % site_test) - if (os.path.exists(test_path) and os.path.isdir(test_path) and - os.path.exists(test_py) and os.path.isfile(test_py) and - site_test not in blacklist): - all_tests += ',' + site_test - - if 'all' == options.build.lower(): - if options.noprompt is not True: - print 'You want to pre-build all client tests and it may take a long', - print 'time to finish.' - print 'Are you sure you want to continue?(N/y)', - answer = sys.stdin.readline() - if 'y' != answer[0].lower(): - print 'Use --build to specify tests you like to pre-compile. ' - print 'E.g.: ./autotest --build=disktest,hardware_SAT' - sys.exit(0) - test_list = all_tests - else: - test_list = options.build - - environ['FEATURES'] = ('%s -buildpkg -collision-protect' % - environ.get('FEATURES', '')) - environ['TEST_LIST'] = test_list - environ['USE'] = use_flag - emerge_cmd = ['emerge-%s' % options.board, - 'chromeos-base/autotest'] - if emerge_jobs: - emerge_cmd.append(emerge_jobs) - return run(emerge_cmd) - - -def run_autoserv(options, args): - environ = os.environ - - environ['AUTOSERV_TEST_ARGS'] = options.args - environ['AUTOSERV_ARGS'] = ' '.join(args) - environ['FEATURES'] = ('%s -buildpkg -digest noauto' % - environ.get('FEATURES', '')) - ebuild_cmd = [ './autotest_run.sh', '--board=%s' % options.board] - run(ebuild_cmd) - - -def main(): - me = sys.argv[0] - common_sh = os.path.join(os.path.dirname(me), 'common.sh') - - assert_inside_chroot(common_sh) - set_common_env(common_sh, 'GCLIENT_ROOT') - - options, args = parse_args_and_help() - - if not options.board: - die(common_sh, 'Missing --board argument.') - - if options.build: - status = build_autotest(options) - if status: - die(common_sh, 'build_autotest failed.') - else: - ssh_key_file = os.path.join(os.path.dirname(me), - 'mod_for_test_scripts/ssh_keys/testing_rsa') - os.chmod(ssh_key_file, 0400) - run_autoserv(options, args) - - -if __name__ == '__main__': - main() diff --git a/run_remote_tests.sh b/run_remote_tests.sh index 864d8bb756..babbbdceb0 100755 --- a/run_remote_tests.sh +++ b/run_remote_tests.sh @@ -18,42 +18,20 @@ DEFINE_string args "" \ "Command line arguments for test. Quoted and space separated if multiple." a DEFINE_string board "$DEFAULT_BOARD" \ "The board for which you are building autotest" +DEFINE_boolean build ${FLAGS_FALSE} "Build tests while running" b DEFINE_string chroot "${DEFAULT_CHROOT_DIR}" "alternate chroot location" c DEFINE_boolean cleanup ${FLAGS_FALSE} "Clean up temp directory" DEFINE_integer iterations 1 "Iterations to run every top level test" i DEFINE_string results_dir_root "" "alternate root results directory" DEFINE_boolean verbose ${FLAGS_FALSE} "Show verbose autoserv output" v +DEFINE_boolean use_emerged ${FLAGS_FALSE} \ + "Force use of emerged autotest packages" RAN_ANY_TESTS=${FLAGS_FALSE} -# Check if our stdout is a tty -function is_a_tty() { - local stdout=$(readlink /proc/$$/fd/1) - [[ "${stdout#/dev/tty}" != "${stdout}" ]] && return 0 - [[ "${stdout#/dev/pts}" != "${stdout}" ]] && return 0 - return 1 -} - -# Writes out text in specified color if stdout is a tty -# Arguments: -# $1 - color -# $2 - text to color -# $3 - text following colored text (default colored) -# Returns: -# None -function echo_color() { - local color=0 - [[ "$1" == "red" ]] && color=31 - [[ "$1" == "green" ]] && color=32 - [[ "$1" == "yellow" ]] && color=33 - if is_a_tty; then - echo -e "\033[1;${color}m$2\033[0m$3" - else - echo "$2$3" - fi -} - function cleanup() { + # Always remove the build path in case it was used. + [[ -n "${BUILD_DIR}" ]] && sudo rm -rf "${BUILD_DIR}" if [[ $FLAGS_cleanup -eq ${FLAGS_TRUE} ]] || \ [[ ${RAN_ANY_TESTS} -eq ${FLAGS_FALSE} ]]; then rm -rf "${TMP}" @@ -63,27 +41,6 @@ function cleanup() { cleanup_remote_access } -# Adds attributes to all tests run -# Arguments: -# $1 - results directory -# $2 - attribute name (key) -# $3 - attribute value (value) -function add_test_attribute() { - local results_dir="$1" - local attribute_name="$2" - local attribute_value="$3" - if [[ -z "$attribute_value" ]]; then - return; - fi - - for status_file in $(echo "${results_dir}"/*/status); do - local keyval_file=$(dirname $status_file)/keyval - echo "Updating ${keyval_file}" - echo "${attribute_name}=${attribute_value}" >> "${keyval_file}" - done -} - - # Determine if a control is for a client or server test. Echos # either "server" or "client". # Arguments: @@ -94,17 +51,83 @@ function read_test_type() { local type=$(egrep -m1 \ '^[[:space:]]*TEST_TYPE[[:space:]]*=' "${control_file}") if [[ -z "${type}" ]]; then - echo_color "red" ">>> Unable to find TEST_TYPE line in ${control_file}" - exit 1 + die "Unable to find TEST_TYPE line in ${control_file}" fi type=$(python -c "${type}; print TEST_TYPE.lower()") if [[ "${type}" != "client" ]] && [[ "${type}" != "server" ]]; then - echo_color "red" ">>> Unknown type of test (${type}) in ${control_file}" - exit 1 + die "Unknown type of test (${type}) in ${control_file}" fi echo ${type} } +function create_tmp() { + # Set global TMP for remote_access.sh's sake + # and if --results_dir_root is specified, + # set TMP and create dir appropriately + if [[ ${INSIDE_CHROOT} -eq 0 ]]; then + if [[ -n "${FLAGS_results_dir_root}" ]]; then + TMP=${FLAGS_chroot}${FLAGS_results_dir_root} + mkdir -p -m 777 ${TMP} + else + TMP=$(mktemp -d ${FLAGS_chroot}/tmp/run_remote_tests.XXXX) + fi + TMP_INSIDE_CHROOT=$(echo ${TMP#${FLAGS_chroot}}) + else + if [[ -n "${FLAGS_results_dir_root}" ]]; then + TMP=${FLAGS_results_dir_root} + mkdir -p -m 777 ${TMP} + else + TMP=$(mktemp -d /tmp/run_remote_tests.XXXX) + fi + TMP_INSIDE_CHROOT=${TMP} + fi +} + +function prepare_build_dir() { + local autotest_dir="$1" + INSIDE_BUILD_DIR="${TMP_INSIDE_CHROOT}/build" + BUILD_DIR="${TMP}/build" + info "Copying autotest tree into ${BUILD_DIR}." + sudo mkdir -p "${BUILD_DIR}" + sudo rsync -rl --chmod=ugo=rwx "${autotest_dir}"/ "${BUILD_DIR}" + info "Pilfering toolchain shell environment from Portage." + local outside_ebuild_dir="${TMP}/chromeos-base/autotest-build" + local inside_ebuild_dir="${TMP_INSIDE_CHROOT}/chromeos-base/autotest-build" + mkdir -p "${outside_ebuild_dir}" + local E_only="autotest-build-9999.ebuild" + cat > "${outside_ebuild_dir}/${E_only}" <&1 > /dev/null + local P_tmp="${FLAGS_chroot}/build/${FLAGS_board}/tmp/portage/" + local E_dir="${E%%/*}/${E_only%.*}" + sudo cp "${P_tmp}/${E_dir}/temp/environment" "${BUILD_DIR}" +} + +function autodetect_build() { + if [ ${FLAGS_use_emerged} -eq ${FLAGS_TRUE} ]; then + info \ +"As requested, using emerged autotests already installed in your sysroot." + FLAGS_build=${FLAGS_FALSE} + return + fi + if ${ENTER_CHROOT} ./cros_workon --board=${FLAGS_board} list | \ + grep -q autotest; then + info \ +"Detected cros_workon autotests, building your sources instead of emerged \ +autotest. To use installed autotest, pass --use_emerged." + FLAGS_build=${FLAGS_TRUE} + else + info \ +"Using emerged autotests already installed in your sysroot. To build \ +autotests directly from your source directory instead, pass --build." + FLAGS_build=${FLAGS_FALSE} + fi +} + function main() { cd $(dirname "$0") @@ -130,26 +153,7 @@ function main() { set -e - # Set global TMP for remote_access.sh's sake - # and if --results_dir_root is specified, - # set TMP and create dir appropriately - if [[ ${INSIDE_CHROOT} -eq 0 ]]; then - if [[ -n "${FLAGS_results_dir_root}" ]]; then - TMP=${FLAGS_chroot}${FLAGS_results_dir_root} - mkdir -p -m 777 ${TMP} - else - TMP=$(mktemp -d ${FLAGS_chroot}/tmp/run_remote_tests.XXXX) - fi - TMP_INSIDE_CHROOT=$(echo ${TMP#${FLAGS_chroot}}) - else - if [[ -n "${FLAGS_results_dir_root}" ]]; then - TMP=${FLAGS_results_dir_root} - mkdir -p -m 777 ${TMP} - else - TMP=$(mktemp -d /tmp/run_remote_tests.XXXX) - fi - TMP_INSIDE_CHROOT=${TMP} - fi + create_tmp trap cleanup EXIT @@ -158,6 +162,23 @@ function main() { learn_board autotest_dir="${FLAGS_chroot}/build/${FLAGS_board}/usr/local/autotest" + ENTER_CHROOT="" + if [[ ${INSIDE_CHROOT} -eq 0 ]]; then + ENTER_CHROOT="./enter_chroot.sh --chroot ${FLAGS_chroot} --" + fi + + if [ ${FLAGS_build} -eq ${FLAGS_FALSE} ]; then + autodetect_build + fi + + if [ ${FLAGS_build} -eq ${FLAGS_TRUE} ]; then + autotest_dir="${SRC_ROOT}/third_party/autotest/files" + else + if [ ! -d "${autotest_dir}" ]; then + die "You need to emerge autotest-tests (or use --build)" + fi + fi + local control_files_to_run="" local chrome_autotests="${CHROME_ROOT}/src/chrome/test/chromeos/autotest/files" # Now search for tests which unambiguously include the given identifier @@ -172,8 +193,7 @@ function main() { ! finds=$(find ${search_path} -maxdepth 2 -type f \( -name control.\* -or \ -name control \) | egrep -v "~$" | egrep "${test_request}") if [[ -z "${finds}" ]]; then - echo_color "red" ">>> Cannot find match for \"${test_request}\"" - exit 1 + die "Cannot find match for \"${test_request}\"" fi local matches=$(echo "${finds}" | wc -l) if [[ ${matches} -gt 1 ]]; then @@ -193,13 +213,14 @@ function main() { echo "" if [[ -z "${control_files_to_run}" ]]; then - echo_color "red" ">>> Found no control files" - exit 1 + die "Found no control files" fi - echo_color "yellow" ">>> Running the following control files:" + [ ${FLAGS_build} -eq ${FLAGS_TRUE} ] && prepare_build_dir "${autotest_dir}" + + info "Running the following control files:" for CONTROL_FILE in ${control_files_to_run}; do - echo_color "yellow" " * " "${CONTROL_FILE}" + info " * ${CONTROL_FILE}" done for control_file in ${control_files_to_run}; do @@ -217,7 +238,7 @@ function main() { option="-s" fi echo "" - echo_color "yellow" ">>> Running ${type} test " ${control_file} + info "Running ${type} test ${control_file}" local control_file_name=$(basename "${control_file}") local short_name=$(basename $(dirname "${control_file}")) @@ -243,31 +264,49 @@ function main() { RAN_ANY_TESTS=${FLAGS_TRUE} - local enter_chroot="" - local autotest="${GCLIENT_ROOT}/src/scripts/autotest_workon" - if [[ ${INSIDE_CHROOT} -eq 0 ]]; then - enter_chroot="./enter_chroot.sh --chroot ${FLAGS_chroot} --" - autotest="./autotest_workon" - fi - # Remove chrome autotest location prefix from control_file if needed if [[ ${control_file:0:${#chrome_autotests}} == \ "${chrome_autotests}" ]]; then control_file="${control_file:${#chrome_autotests}+1}" - echo_color "yellow" ">>> Running chrome autotest " ${control_file} - fi - if [[ -n "${FLAGS_args}" ]]; then - passthrough_args="--args=${FLAGS_args}" + info "Running chrome autotest ${control_file}" fi - ${enter_chroot} ${autotest} --board "${FLAGS_board}" -m "${FLAGS_remote}" \ - --ssh-port ${FLAGS_ssh_port} \ - "${option}" "${control_file}" -r "${results_dir}" ${verbose} \ - "${passthrough_args}" >&2 + export AUTOSERV_TEST_ARGS="${FLAGS_args}" + export AUTOSERV_ARGS="-m ${FLAGS_remote} \ + --ssh-port ${FLAGS_ssh_port} \ + ${option} ${control_file} -r ${results_dir} ${verbose}" + if [ ${FLAGS_build} -eq ${FLAGS_FALSE} ]; then + cat > "${TMP}/run_test.sh" <&2 + else + cp "${BUILD_DIR}/environment" "${TMP}/run_test.sh" + GRAPHICS_BACKEND=${GRAPHICS_BACKEND:-OPENGL} + if [ -n "${AUTOSERV_TEST_ARGS}" ]; then + AUTOSERV_TEST_ARGS="-a \"${AUTOSERV_TEST_ARGS}\"" + fi + cat >> "${TMP}/run_test.sh" <&2 + fi done echo "" - echo_color "yellow" ">>> Test results:" + info "Test results:" ./generate_test_report "${TMP}" --strip="${TMP}/" print_time_elapsed From 6c4a20ba375de219ed52432eba8582180e53f9d0 Mon Sep 17 00:00:00 2001 From: Anush Elangovan Date: Fri, 5 Nov 2010 11:37:14 -0700 Subject: [PATCH 12/38] export CHROME_VERSION by default Change-Id: I788ec487f7c63e71a8a7012c1c7c19c02df103e4 BUG=8786 TEST=eval the script to see it is set right Review URL: http://codereview.chromium.org/4521003 --- chromeos_version.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/chromeos_version.sh b/chromeos_version.sh index 1ff4478b12..77d4e0158f 100755 --- a/chromeos_version.sh +++ b/chromeos_version.sh @@ -86,6 +86,7 @@ export CHROMEOS_VERSION_STRING=\ export CHROME_BASE= # directory containing chrome-chromeos.zip - an svn rev or a full version export CHROME_BUILD= +export CHROME_VERSION="$CHROME_BUILD" # Print (and remember) version info. echo "ChromeOS version information:" From 139ad9dca075dee06c51e2c9a5bcf67b278fd220 Mon Sep 17 00:00:00 2001 From: David James Date: Fri, 5 Nov 2010 14:39:02 -0700 Subject: [PATCH 13/38] Update parallel_emerge to correct use flags, fixing a regression. BUG=chromium-os:8814 TEST=Verify packages with old use flags are updated, and that binary packages with wrong use flags aren't used. Change-Id: I7064a10796f1541ff9ae36f2ba6ef1d65dbe1aa3 Review URL: http://codereview.chromium.org/4552007 --- parallel_emerge | 68 ++++++++++++++++++++++--------------------------- 1 file changed, 31 insertions(+), 37 deletions(-) diff --git a/parallel_emerge b/parallel_emerge index c1667134c0..ab342334c2 100755 --- a/parallel_emerge +++ b/parallel_emerge @@ -464,7 +464,6 @@ class DepGraphGenerator(object): forced_flags = set(pkgsettings.useforce).union(pkgsettings.usemask) depgraph = self.emerge.depgraph - flags = depgraph._reinstall_for_flags(forced_flags, cur_use, cur_iuse, now_use, now_iuse) return not flags @@ -488,21 +487,21 @@ class DepGraphGenerator(object): # --workon and the dependencies have changed. emerge = self.emerge emerge_opts = emerge.opts.copy() - if self.mandatory_source or self.rebuild or self.force_remote_binary: - # Enable --emptytree so that we get the full tree, which we need for - # dependency analysis. By default, with this option, emerge optimizes - # the graph by removing uninstall instructions from the graph. By - # specifying --tree as well, we tell emerge that it's not safe to remove - # uninstall instructions because we're planning on analyzing the output. - emerge_opts["--tree"] = True - emerge_opts["--emptytree"] = True - # Tell emerge not to worry about use flags yet. We handle those inside - # parallel_emerge itself. Further, when we use the --force-remote-binary - # flag, we don't emerge to reject a package just because it has different - # use flags. - emerge_opts.pop("--newuse", None) - emerge_opts.pop("--reinstall", None) + # Enable --emptytree so that we get the full tree, which we need for + # dependency analysis. By default, with this option, emerge optimizes + # the graph by removing uninstall instructions from the graph. By + # specifying --tree as well, we tell emerge that it's not safe to remove + # uninstall instructions because we're planning on analyzing the output. + emerge_opts["--tree"] = True + emerge_opts["--emptytree"] = True + + # Tell emerge not to worry about use flags yet. We handle those inside + # parallel_emerge itself. Further, when we use the --force-remote-binary + # flag, we don't emerge to reject a package just because it has different + # use flags. + emerge_opts.pop("--newuse", None) + emerge_opts.pop("--reinstall", None) # Create a list of packages to merge packages = set(emerge.cmdline_packages[:]) @@ -593,9 +592,25 @@ class DepGraphGenerator(object): frozen_config = depgraph._frozen_config vardb = frozen_config.trees[root]["vartree"].dbapi pkgsettings = frozen_config.pkgsettings[root] + + # It's time to start worrying about use flags, if necessary. + for flag in ("--newuse", "--reinstall"): + if flag in emerge.opts: + emerge_opts[flag] = emerge.opts[flag] + deps_info = {} for pkg in depgraph.altlist(): if isinstance(pkg, Package): + # If we're not using --force-remote-binary, check what flags are being + # used by the real package. + if "--usepkgonly" not in emerge.opts: + try: + pkg = emerge.depgraph._pkg(pkg.cpv, "ebuild", emerge.root_config) + except portage.exception.PackageNotFound: + # This is a --force-remote-binary package. + pass + self.package_db[pkg.cpv] = pkg + # If we're not in emptytree mode, and we're going to replace a package # that is already installed, then this operation is possibly optional. # ("--selective" mode is handled later, in RemoveInstalledPackages()) @@ -1149,9 +1164,7 @@ class DepGraphGenerator(object): # Calculate what packages need to be rebuilt due to changes in use flags. for pkg, db_pkg in prebuilt_pkgs.iteritems(): - db_pkg_src = self.package_db.get(pkg) - if db_pkg_src and not self.CheckUseFlags(pkgsettings, db_pkg, - db_pkg_src): + if not self.CheckUseFlags(pkgsettings, db_pkg, self.package_db[pkg]): MergeChildren(pkg, "mandatory_source") # Convert eligible packages to binaries. @@ -1166,24 +1179,6 @@ class DepGraphGenerator(object): return prebuilt_pkgs - def AddRemainingPackages(): - """Fill in packages that don't have entries in the package db. - - Every package we are installing needs an entry in the package db. - This function should only be called after we have removed the - packages that are not being merged from our deps_map. - """ - for pkg in deps_map: - if pkg not in self.package_db: - if deps_map[pkg]["action"] != "merge": - # We should only fill in packages that are being merged. If - # there's any other packages here, something funny is going on. - print "Missing entry for %s in package db" % pkg - sys.exit(1) - - db_pkg = emerge.depgraph._pkg(pkg, "ebuild", emerge.root_config) - self.package_db[pkg] = db_pkg - ReverseTree(deps_tree) BuildFinalPackageSet() AddSecretDeps() @@ -1223,7 +1218,6 @@ class DepGraphGenerator(object): if deps_map: if "--usepkg" in emerge.opts: UsePrebuiltPackages(remote_pkgs) - AddRemainingPackages() return deps_map def PrintInstallPlan(self, deps_map): From 1330edb593ca90b806f59a3d341ab534317e768c Mon Sep 17 00:00:00 2001 From: Huyen Nguyen Date: Fri, 5 Nov 2010 16:15:46 -0700 Subject: [PATCH 14/38] Fix bug where square bracket is not removed before appending new rule. Change-Id: I5618b211abd69b4e3774fa6e4009183ef996a8d8 BUG=8815 TEST=run locally Review URL: http://codereview.chromium.org/4498004 --- make_factory_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/make_factory_package.sh b/make_factory_package.sh index f59aada962..9a44f71dd2 100755 --- a/make_factory_package.sh +++ b/make_factory_package.sh @@ -166,7 +166,7 @@ fi if [ -n "${FLAGS_subfolder}" ] && \ [ -f "${OMAHA_DIR}"/miniomaha.conf"" ] ; then # Remove the ']' from the last line of the file so we can add another config. - sed -i '$d' < ${OMAHA_DIR}/miniomaha.conf + sed -i '$d' ${OMAHA_DIR}/miniomaha.conf else echo -e "config = [" > ${OMAHA_DIR}/miniomaha.conf fi From 2c002821ac6cc8143fdc0115fe95cdd3be13271d Mon Sep 17 00:00:00 2001 From: "J. Richard Barnette" Date: Fri, 5 Nov 2010 17:43:04 -0700 Subject: [PATCH 15/38] Create bootperf-bin tools for running boot time tests and reporting results BUG=chromium-os:8655 TEST=run bootperf as described by 'bootperf --help' TEST=run showbootstats as described by 'showbootstats --help' Review URL: http://codereview.chromium.org/4148011 Change-Id: Idcfe206f7708fe5290960149952239219d1a96f6 --- bootperf-bin/__init__.py | 0 bootperf-bin/bootperf | 213 ++++++++++++++++++++++++++++ bootperf-bin/perfprinter.py | 132 +++++++++++++++++ bootperf-bin/resultset.py | 274 ++++++++++++++++++++++++++++++++++++ bootperf-bin/showbootdata | 116 +++++++++++++++ 5 files changed, 735 insertions(+) create mode 100644 bootperf-bin/__init__.py create mode 100755 bootperf-bin/bootperf create mode 100644 bootperf-bin/perfprinter.py create mode 100644 bootperf-bin/resultset.py create mode 100755 bootperf-bin/showbootdata diff --git a/bootperf-bin/__init__.py b/bootperf-bin/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bootperf-bin/bootperf b/bootperf-bin/bootperf new file mode 100755 index 0000000000..4aea1a8971 --- /dev/null +++ b/bootperf-bin/bootperf @@ -0,0 +1,213 @@ +#!/bin/bash + +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Wrapper to run the platform_BootPerfServer autotest, and store the +# results for later analysis by the 'showbootdata' script. +# +# NOTE: This script must be run from inside the chromeos build +# chroot environment. +# + +# SCRIPT_DIR="$(cd "$(dirname $0)/.." ; pwd)" +SCRIPT_DIR=$HOME/trunk/src/scripts +. "$SCRIPT_DIR/common.sh" + +DEFINE_string output_dir "" "output directory for results" o +DEFINE_boolean keep_logs "$FLAGS_FALSE" "keep autotest results" k + +RUN_TEST="$SCRIPT_DIR/run_remote_tests.sh" +TEST=server/site_tests/platform_BootPerfServer/control +TMP_RESULTS="/tmp/bootperf.$(date '+%Y%j%H%M').$$" +RESULTS_KEYVAL=platform_BootPerfServer/platform_BootPerfServer/results/keyval +RESULTS_SUMMARY_FILES=( + $RESULTS_KEYVAL + platform_BootPerfServer/keyval + platform_BootPerfServer/platform_BootPerfServer/keyval + platform_BootPerfServer/platform_BootPerfServer/platform_BootPerf/keyval + platform_BootPerfServer/platform_BootPerfServer/status + platform_BootPerfServer/platform_BootPerfServer/status.log + platform_BootPerfServer/status + platform_BootPerfServer/status.log + platform_BootPerfServer/sysinfo/cmdline + platform_BootPerfServer/sysinfo/cpuinfo + platform_BootPerfServer/sysinfo/modules + platform_BootPerfServer/sysinfo/uname + platform_BootPerfServer/sysinfo/version +) + +# Structure of a results directory: +# $RUNDIR.$ITER/ - directory +# $RUNDIR_LOG - file +# $RUNDIR_SUMMARY/ - directory +# $RUNDIR_ALL_RESULTS/ - optional directory +# $KEYVAL_SUMMARY/ - file +# If you add any other content under the results directory, you'll +# probably need to change extra_files(), below. +RUNDIR=run +RUNDIR_LOG=log.txt +RUNDIR_SUMMARY=summary +RUNDIR_ALL_RESULTS=logs +KEYVAL_SUMMARY=results_keyval + + +# Usage/help function. This function is known to the shflags library, +# and mustn't be renamed. +flags_help() { + cat <&2 +usage: $(basename $0) [ ] [ ] +Options: + --output_dir + --o Specify output directory for results + + --[no]keep_logs + -k Keep [don't keep] autotest log files +Summary: + Run the platform_BootPerfServer autotest, and store results in the + given destination directory. The test target is specified by + . + + By default, the test is run once; if is given, the test is + run that many times. Note that the platform_BootPerfServer test + reboots the target 10 times, so the total number of reboots will + be 10*. + + If the destination directory doesn't exist, it is created. If the + destination directory already holds test results, additional + results are added in without overwriting earlier results. + + If no destination is specified, the current directory is used, + provided that the directory is empty, or has been previously used + as a destination directory for this command. + + By default, only a summary subset of the log files created by + autotest is preserved; with --keep_logs the (potentially large) + autotest logs are preserved with the test results. +END_USAGE + return $FLAGS_TRUE +} + +usage() { + if [ $# -gt 0 ]; then + error "$(basename $0): $*" + echo >&2 + fi + flags_help + exit 1 +} + +# List any files in the current directory not created as output +# from running this script. +extra_files() { + ls | grep -v "^$RUNDIR[.]...\$" | + grep -v $KEYVAL_SUMMARY +} + +# Main function to run the boot performance test. Run the boot +# performance test for the given count, putting output into the +# current directory. +# +# Arguments are and arguments, as for the main +# command. +# +# We terminate test runs if "run_remote_tests" ever fails to produce +# the results file; generally this is the result of a serious error +# (e.g. disk full) that won't go away if we just plow on. +run_boot_test() { + local remote="$1" + local count="${2:-1}" + + local iter=$(expr "$(echo $RUNDIR.???)" : '.*\(...\)') + if [ "$iter" != "???" ]; then + iter=$(echo $iter | awk '{printf "%03d\n", $1 + 1}') + else + iter=000 + fi + + i=0 + while [ $i -lt $count ]; do + local iter_rundir=$RUNDIR.$iter + local logfile=$iter_rundir/$RUNDIR_LOG + local summary_dir=$iter_rundir/$RUNDIR_SUMMARY + local all_results_dir=$iter_rundir/$RUNDIR_ALL_RESULTS + + mkdir $iter_rundir + echo "run $iter start at $(date)" + $RUN_TEST --results_dir_root="$TMP_RESULTS" \ + --remote="$remote" $TEST >$logfile 2>&1 + if [ ! -e "$TMP_RESULTS/$RESULTS_KEYVAL" ]; then + error "No results file; terminating test runs." + error "Check $logfile for output from the test run," + error "and see $TMP_RESULTS for full test logs and output." + break + fi + mkdir $summary_dir + tar cf - -C $TMP_RESULTS "${RESULTS_SUMMARY_FILES[@]}" | + tar xf - -C $summary_dir + if [ $FLAGS_keep_logs -eq $FLAGS_TRUE ]; then + mv $TMP_RESULTS $all_results_dir + chmod 755 $all_results_dir + else + rm -rf $TMP_RESULTS + fi + i=$(expr $i + 1) + iter=$(echo $iter | awk '{printf "%03d\n", $1 + 1}') + done + # "run 000 start at $(date)" + echo " ... end at $(date)" + cat $RUNDIR.???/$RUNDIR_SUMMARY/$RESULTS_KEYVAL >$KEYVAL_SUMMARY +} + +# Main routine - check validity of the (already parsed) command line +# options. 'cd' to the results directory, if it was specified. If +# all the arguments checks pass, hand control to run_boot_test +main() { + if [ $# -lt 1 ]; then + usage "Missing target host address" + elif [ $# -gt 2 ]; then + usage "Too many arguments" + fi + + if [ -n "${FLAGS_output_dir}" ]; then + if [ ! -d "${FLAGS_output_dir}" ]; then + if ! mkdir "${FLAGS_output_dir}"; then + usage "Unable to create ${FLAGS_output_dir}" + fi + fi + cd "${FLAGS_output_dir}" || + usage "No permissions to chdir to ${FLAGS_output_dir}" + elif [ -n "$(extra_files)" ]; then + error "No results directory specified, and current directory" + error "contains contents other than run results." + error "You can override this error by using the --output_dir option" + usage + fi + + # Check the count argument. + # N.B. the test [ "$2" -eq "$2" ] tests whether "$2" is valid as a + # number; when it fails it will also report a syntax error (which + # we suppress). + if [ -n "$2" ]; then + if ! [ "$2" -eq "$2" ] 2>/dev/null || [ "$2" -le 0 ]; then + usage " argument must be a positive number" + fi + fi + + run_boot_test "$@" +} + +# shflags defines --help implicitly; if it's used on the command +# line FLAGS will invoke flags_help, set FLAGS_help to TRUE, and +# then return false. To avoid printing help twice, we have to check +# for that case here. +if ! FLAGS "$@"; then + if [ ${FLAGS_help} -eq ${FLAGS_TRUE} ]; then + exit 0 + else + usage + fi +fi + +eval main "${FLAGS_ARGV}" diff --git a/bootperf-bin/perfprinter.py b/bootperf-bin/perfprinter.py new file mode 100644 index 0000000000..d69637016e --- /dev/null +++ b/bootperf-bin/perfprinter.py @@ -0,0 +1,132 @@ +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Routines for printing boot time performance test results.""" + +import fnmatch +import os +import os.path +import re + +import resultset + + +_PERF_KEYVAL_PATTERN = re.compile("(.*){perf}=(.*)\n") + + +def ReadKeyvalFile(results, file_): + """Read an autotest keyval file, and process the results. + + The `file_` parameter is a file object with contents in autotest + perf keyval format: + {perf}= + + Each iteration of the test is terminated with a single blank line, + including the last iteration. Each iteration's results are added + to the `results` parameter, which should be an instance of + TestResultSet. + + """ + kvd = {} + for line in iter(file_): + if line == "\n": + results.AddIterationResults(kvd) + kvd = {} + continue + m = _PERF_KEYVAL_PATTERN.match(line) + if m is None: + continue + kvd[m.group(1)] = m.group(2) + + +_RESULTS_PATH = ( + "summary/platform_BootPerfServer/platform_BootPerfServer/results/keyval") + + +def ReadResultsDirectory(dir_): + """Process results from a 'bootperf' output directory. + + The accumulated results are returned in a newly created + TestResultSet object. + + """ + res_set = resultset.TestResultSet(dir_) + dirlist = fnmatch.filter(os.listdir(dir_), "run.???") + dirlist.sort() + for run in dirlist: + keyval_path = os.path.join(dir_, run, _RESULTS_PATH) + try: + kvf = open(keyval_path) + except IOError: + continue + ReadKeyvalFile(res_set, kvf) + res_set.FinalizeResults() + return res_set + + +def PrintRawData(dirlist, use_timestats, keylist): + """Print 'bootperf' results in "raw data" format.""" + for dir_ in dirlist: + if use_timestats: + keyset = ReadResultsDirectory(dir_).TimeKeySet() + else: + keyset = ReadResultsDirectory(dir_).DiskKeySet() + for i in range(0, keyset.num_iterations): + if len(dirlist) > 1: + line = "%s %3d" % (dir_, i) + else: + line = "%3d" % i + if keylist is not None: + markers = keylist + else: + markers = keyset.markers + for stat in markers: + (_, v) = keyset.PrintableStatistic(keyset.RawData(stat)[i]) + line += " %5s" % str(v) + print line + + +def PrintStatisticsSummary(dirlist, use_timestats, keylist): + """Print 'bootperf' results in "summary of averages" format.""" + if use_timestats: + header = "%5s %3s %5s %3s %s" % ( + "time", "s%", "dt", "s%", "event") + format = "%5s %2d%% %5s %2d%% %s" + else: + header = "%6s %3s %6s %3s %s" % ( + "diskrd", "s%", "delta", "s%", "event") + format = "%6s %2d%% %6s %2d%% %s" + havedata = False + for dir_ in dirlist: + if use_timestats: + keyset = ReadResultsDirectory(dir_).TimeKeySet() + else: + keyset = ReadResultsDirectory(dir_).DiskKeySet() + if keylist is not None: + markers = keylist + else: + markers = keyset.markers + if havedata: + print + if len(dirlist) > 1: + print "%s" % dir_, + print "(on %d cycles):" % keyset.num_iterations + print header + prevvalue = 0 + prevstat = None + for stat in markers: + (valueavg, valuedev) = keyset.Statistics(stat) + valuepct = int(100 * valuedev / valueavg + 0.5) + if prevstat: + (deltaavg, deltadev) = keyset.DeltaStatistics(prevstat, stat) + deltapct = int(100 * deltadev / deltaavg + 0.5) + else: + deltapct = valuepct + (valstring, val_printed) = keyset.PrintableStatistic(valueavg) + delta = val_printed - prevvalue + (deltastring, _) = keyset.PrintableStatistic(delta) + print format % (valstring, valuepct, "+" + deltastring, deltapct, stat) + prevvalue = val_printed + prevstat = stat + havedata = True diff --git a/bootperf-bin/resultset.py b/bootperf-bin/resultset.py new file mode 100644 index 0000000000..a1cee30b79 --- /dev/null +++ b/bootperf-bin/resultset.py @@ -0,0 +1,274 @@ +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Classes and functions for managing platform_BootPerf results. + +Results from the platform_BootPerf test in the ChromiumOS autotest +package are stored in as performance 'keyvals', that is, a mapping +of names to numeric values. For each iteration of the test, one +set of keyvals is recorded. + +This module currently tracks two kinds of keyval results, the boot +time results, and the disk read results. These results are stored +with keyval names such as 'seconds_kernel_to_login' and +'rdbytes_kernel_to_login'. Additionally, some older versions of the +test produced keyval names such as 'sectors_read_kernel_to_login'. +These keyvals record an accumulated total measured from a fixed +time in the past (kernel startup), e.g. 'seconds_kernel_to_login' +records the total seconds from kernel startup to login screen +ready. + +The boot time keyval names all start with the prefix +'seconds_kernel_to_', and record time in seconds since kernel +startup. + +The disk read keyval names all start with the prefix +'rdbytes_kernel_to_', and record bytes read from the boot device +since kernel startup. The obsolete disk keyvals start with the +prefix 'sectors_read_kernel_to_' and record the same statistic +measured in 512-byte sectors. + +Boot time and disk kevyal values have a consistent ordering +across iterations. For instance, if in one iteration the value of +'seconds_kernel_to_login' is greater than the value of +'seconds_kernel_to_x_started', then it will be greater in *all* +iterations. This property is a consequence of the underlying +measurement procedure; it is not enforced by this module. + +""" + +import math + + +def _ListStats(list_): + # Utility function - calculate the average and (sample) standard + # deviation of a list of numbers. Result is float, even if the + # input list is full of int's + sum_ = 0.0 + sumsq = 0.0 + for v in list_: + sum_ += v + sumsq += v * v + n = len(list_) + avg = sum_ / n + var = (sumsq - sum_ * avg) / (n - 1) + if var < 0.0: + var = 0.0 + dev = math.sqrt(var) + return (avg, dev) + + +def _DoCheck(dict_): + # Utility function - check the that all keyvals occur the same + # number of times. On success, return the number of occurrences; + # on failure return None + check = map(len, dict_.values()) + if not check: + return None + for i in range(1, len(check)): + if check[i] != check[i-1]: + return None + return check[0] + + +def _KeyDelta(dict_, key0, key1): + # Utility function - return a list of the vector difference between + # two keyvals. + return map(lambda a, b: b - a, dict_[key0], dict_[key1]) + + +class TestResultSet(object): + """A set of boot time and disk usage result statistics. + + Objects of this class consist of two sets of result statistics: + the boot time statistics and the disk statistics. + + Class TestResultsSet does not interpret or store keyval mappings + directly; iteration results are processed by attached _KeySet + objects, one for boot time (`_timekeys`), one for disk read + (`_diskkeys`). These attached _KeySet objects can be obtained + with appropriate methods; various methods on these objects will + calculate statistics on the results, and provide the raw data. + + """ + + def __init__(self, name): + self.name = name + self._timekeys = _TimeKeySet() + self._diskkeys = _DiskKeySet() + self._olddiskkeys = _OldDiskKeySet() + + def AddIterationResults(self, runkeys): + """Add keyval results from a single iteration. + + A TestResultSet is constructed by repeatedly calling + AddRunResults(), iteration by iteration. Iteration results are + passed in as a dictionary mapping keyval attributes to values. + When all iteration results have been added, FinalizeResults() + makes the results available for analysis. + + """ + + self._timekeys.AddRunResults(runkeys) + self._diskkeys.AddRunResults(runkeys) + self._olddiskkeys.AddRunResults(runkeys) + + def FinalizeResults(self): + """Make results available for analysis. + + A TestResultSet is constructed by repeatedly feeding it results, + iteration by iteration. Iteration results are passed in as a + dictionary mapping keyval attributes to values. When all iteration + results have been added, FinalizeResults() makes the results + available for analysis. + + """ + + self._timekeys.FinalizeResults() + if not self._diskkeys.FinalizeResults(): + self._olddiskkeys.FinalizeResults() + self._diskkeys = self._olddiskkeys + self._olddiskkeys = None + + def TimeKeySet(self): + """Return the boot time statistics result set.""" + return self._timekeys + + def DiskKeySet(self): + """Return the disk read statistics result set.""" + return self._diskkeys + + +class _KeySet(object): + """Container for a set of related statistics. + + _KeySet is an abstract superclass for containing collections of + either boot time or disk read statistics. Statistics are stored + as a dictionary (`_keyvals`) mapping keyval names to lists of + values. + + The mapped keyval names are shortened by stripping the prefix + that identifies the type of prefix (keyvals that don't start with + the proper prefix are ignored). So, for example, with boot time + keyvals, 'seconds_kernel_to_login' becomes 'login' (and + 'rdbytes_kernel_to_login' is ignored). + + A list of all valid keyval names is stored in the `markers` + instance variable. The list is sorted by the natural ordering of + the underlying values (see the module comments for more details). + + The list of values associated with a given keyval name are indexed + in the order in which they were added. So, all values for a given + iteration are stored at the same index. + + """ + + def __init__(self): + self._keyvals = {} + + def AddRunResults(self, runkeys): + """Add results for one iteration.""" + + for key, value in runkeys.iteritems(): + if not key.startswith(self.PREFIX): + continue + shortkey = key[len(self.PREFIX):] + keylist = self._keyvals.setdefault(shortkey, []) + keylist.append(self._ConvertVal(value)) + + def FinalizeResults(self): + """Finalize this object's results. + + This method makes available the `markers` and `num_iterations` + instance variables. It also ensures that every keyval occurred + in every iteration by requiring that all keyvals have the same + number of data points. + + """ + + count = _DoCheck(self._keyvals) + if count is None: + self.num_iterations = 0 + self.markers = [] + return False + self.num_iterations = count + keylist = map(lambda k: (self._keyvals[k][0], k), + self._keyvals.keys()) + keylist.sort(key=lambda tp: tp[0]) + self.markers = map(lambda tp: tp[1], keylist) + return True + + def RawData(self, key): + """Return the list of values for the given marker key.""" + return self._keyvals[key] + + def DeltaData(self, key0, key1): + """Return vector difference of the values of the given keys.""" + return _KeyDelta(self._keyvals, key0, key1) + + def Statistics(self, key): + """Return the average and standard deviation of the key's values.""" + return _ListStats(self._keyvals[key]) + + def DeltaStatistics(self, key0, key1): + """Return the average and standard deviation of the differences + between two keys. + + """ + + return _ListStats(self.DeltaData(key0, key1)) + + +class _TimeKeySet(_KeySet): + """Concrete subclass of _KeySet for boot time statistics.""" + + # TIME_KEY_PREFIX = 'seconds_kernel_to_' + PREFIX = 'seconds_kernel_to_' + + # Time-based keyvals are reported in seconds and get converted to + # milliseconds + TIME_SCALE = 1000 + + def _ConvertVal(self, value): + # We use a "round to nearest int" formula here to make sure we + # don't lose anything in the conversion from decimal. + return int(self.TIME_SCALE * float(value) + 0.5) + + def PrintableStatistic(self, value): + v = int(value + 0.5) + return ("%d" % v, v) + + +class _DiskKeySet(_KeySet): + """Concrete subclass of _KeySet for disk read statistics.""" + + PREFIX = 'rdbytes_kernel_to_' + + # Disk read keyvals are reported in bytes and get converted to + # MBytes (1 MByte = 1 million bytes, not 2**20) + DISK_SCALE = 1.0e-6 + + def _ConvertVal(self, value): + return self.DISK_SCALE * float(value) + + def PrintableStatistic(self, value): + v = round(value, 1) + return ("%.1fM" % v, v) + + +class _OldDiskKeySet(_DiskKeySet): + """Concrete subclass of _KeySet for the old-style disk read statistics.""" + + # Older versions of platform_BootPerf reported total sectors read + # using names of the form sectors_read_kernel_to_* (instead of the + # more recent rdbytes_kernel_to_*), but some of those names + # exceeded the 30-character limit in the MySQL database schema. + PREFIX = 'sectors_read_kernel_to_' + + # Old sytle disk read keyvals are reported in 512-byte sectors and + # get converted to MBytes (1 MByte = 1 million bytes, not 2**20) + SECTOR_SCALE = 512 * _DiskKeySet.DISK_SCALE + + def _ConvertVal(self, value): + return self.SECTOR_SCALE * float(value) diff --git a/bootperf-bin/showbootdata b/bootperf-bin/showbootdata new file mode 100755 index 0000000000..b5261d12bb --- /dev/null +++ b/bootperf-bin/showbootdata @@ -0,0 +1,116 @@ +#!/usr/bin/python + +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""A command to display summary statistics from runs of 'bootperf'. + +Command line options allow selecting from one of two sets of +performance statistics: The boot time statistics (selected by +--timestats) measure time spent from kernel startup in milliseconds. +The disk statistics (selected by --diskstats) measure total bytes +read from the boot device since kernel startup. + +Boot time statistics are recorded as cumulative time (or disk read) +since kernel startup, measured when specific events occur during +boot. Events include such things as 'startup', the moment when the +upstart 'startup' job begins running, and 'login', when the +Chrome OS login screen is displayed. By default, all recorded events +are included in the output; command line options allow restricting +the view to a selected subset of events. + +Separate command line options allow selecting from one of two +different display modes. When --averages is selected, the display +shows the average value and sample standard deviation (as a percent +of the average) for all selected events. The --averages display also +calculates the time (or bytes) between adjacent events, and shows +the avearage and sample standard deviation of the differences. + +The --rawdata display shows the raw data value associated with each +event for each boot: Each line of output represents the event values +for one boot cycle. + +""" + +import sys +import optparse + +import perfprinter + + +_usage = "%prog [options] [results-directory ...]" +_description = """\ +Summarize boot time performance results. The result directory +arguments are directories previously specified as output for the +'bootperf' script. +""" +optparser = optparse.OptionParser(usage=_usage, description=_description) + +optgroup = optparse.OptionGroup( + optparser, "Selecting boot time or disk statistics (choose one)") +optgroup.add_option( + "-d", "--diskstats", action="store_true", + dest="use_diskstats", + help="use statistics for bytes read since kernel startup") +optgroup.add_option( + "-t", "--timestats", action="store_true", + dest="use_timestats", + help="use statistics for time since kernel startup (default)") +optparser.add_option_group(optgroup) +optparser.set_defaults(use_diskstats=False) +optparser.set_defaults(use_timestats=False) + +optgroup = optparse.OptionGroup(optparser, "Event selection") +optgroup.add_option( + "-e", "--event", action="append", + dest="eventnames", + help="restrict statistics to the comma-separated list of events") +optparser.add_option_group(optgroup) + +optgroup = optparse.OptionGroup( + optparser, "Display mode selection (choose one)") +optgroup.add_option( + "-a", "--averages", action="store_true", + dest="print_averages", + help="display a summary of the averages of chosen statistics (default)") +optgroup.add_option( + "-r", "--rawdata", action="store_true", + dest="print_raw", + help="display raw data from all boot iterations") +optparser.add_option_group(optgroup) +optparser.set_defaults(print_averages=False) +optparser.set_defaults(print_raw=False) + + +def main(argv): + (options, args) = optparser.parse_args(argv) + if options.print_averages and options.print_raw: + print >>sys.stderr, "Can't use -a and -r together.\n" + optparser.print_help() + sys.exit(1) + elif options.print_raw: + printfunc = perfprinter.PrintRawData + else: + printfunc = perfprinter.PrintStatisticsSummary + if options.use_timestats and options.use_diskstats: + print >>sys.stderr, "Can't use -t and -d together.\n" + optparser.print_help() + sys.exit(1) + elif options.use_diskstats: + use_timestats = False + else: + use_timestats = True + if options.eventnames: + keylist = [] + for kl in options.eventnames: + keylist.extend(kl.split(',')) + else: + keylist = None + printfunc(args, use_timestats, keylist) + +if __name__ == "__main__": + if len(sys.argv) > 1: + main(sys.argv[1:]) + else: + main(["."]) From 226522a011c313d043d39e97a7eb42a71a1ec092 Mon Sep 17 00:00:00 2001 From: Will Drewry Date: Fri, 5 Nov 2010 20:20:47 -0500 Subject: [PATCH 16/38] build_image: rollback default error behavior until kernel-next merge kernel-next lacks error mode 3. Rolling back until merged. TEST=none: rolling back to old value BUG=chromium-os:8442 Change-Id: I0c1a89dc3e00fe5f315b3c15c5070209b0fd79d0 Review URL: http://codereview.chromium.org/4512004 --- build_image | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_image b/build_image index 7b89a3e15b..f727b7497f 100755 --- a/build_image +++ b/build_image @@ -71,9 +71,9 @@ DEFINE_string usb_disk /dev/sdb3 \ DEFINE_boolean enable_rootfs_verification ${FLAGS_TRUE} \ "Default all bootloaders to use kernel-based root fs integrity checking." -DEFINE_integer verity_error_behavior 3 \ +DEFINE_integer verity_error_behavior 1 \ "Kernel verified boot error behavior (0: I/O errors, 1: panic, 2: nothing, \ -3: cros) Default: 3" +3: cros) Default: 1" DEFINE_integer verity_depth 1 \ "Kernel verified boot hash tree depth. Default: 1" DEFINE_integer verity_max_ios -1 \ From 47fd7b5c51242fea096c653cdf5de03d9a7d0d21 Mon Sep 17 00:00:00 2001 From: Ken Mixter Date: Mon, 8 Nov 2010 13:35:05 -0800 Subject: [PATCH 17/38] crosutils: Always pass an image to devserver so it can pregenerate Change-Id: Ib366064df269bd0d4aee30e5b1628938bd8fedf9 BUG=8313 TEST=Running it once without passing an image Review URL: http://codereview.chromium.org/4208003 --- image_to_live.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/image_to_live.sh b/image_to_live.sh index ad57811741..88d75b9659 100755 --- a/image_to_live.sh +++ b/image_to_live.sh @@ -30,6 +30,7 @@ DEFINE_boolean verify ${FLAGS_TRUE} "Verify image on device after update." # Flags for devserver. DEFINE_string archive_dir "" \ "Update using the test image in the image.zip in this directory." a +DEFINE_string board "" "Override the board reported by the target" DEFINE_integer devserver_port 8080 \ "Port to use for devserver." DEFINE_boolean for_vm ${FLAGS_FALSE} "Image is for a vm." @@ -124,10 +125,11 @@ function start_dev_server { else # IMAGE_PATH should be the newest image and learn the board from # the target. - FLAGS_board="" learn_board IMAGE_PATH="$($(dirname "$0")/get_latest_image.sh --board="${FLAGS_board}")" IMAGE_PATH="${IMAGE_PATH}/chromiumos_image.bin" + devserver_flags="${devserver_flags} \ + --image $(reinterpret_path_for_chroot ${IMAGE_PATH})" fi [ ${FLAGS_for_vm} -eq ${FLAGS_TRUE} ] && \ @@ -148,6 +150,10 @@ function start_dev_server { do sleep 5 echo -n "." + if ! pgrep -f start_devserver > /dev/null; then + echo "Devserver failed, see dev_server.log." + exit 1 + fi done echo "" } From ec5d7f0d2d95b9adfb83578f698aa41923fbb40e Mon Sep 17 00:00:00 2001 From: Doug Anderson Date: Mon, 8 Nov 2010 14:46:03 -0800 Subject: [PATCH 18/38] Code cleanup: comments, docstrings, and gpylint fixes. Most of this is from this code review: http://codereview.chromium.org/4175007/diff/1/2 I also ran the code through gpylint and fixed some of the stuff, like making quoting consistent. Change-Id: Icb3896dbd4e975c7ea4687d58efd6372adfcc3c9 BUG=chromium-os:8205 TEST=Ran ./cros_changelog 0.7.48.0 and saw error; also ran ./cros_changelog 0.9.98.3 Review URL: http://codereview.chromium.org/4263001 --- chromite/bin/cros_changelog | 84 ++++++++++++++++++++++++++++--------- 1 file changed, 65 insertions(+), 19 deletions(-) diff --git a/chromite/bin/cros_changelog b/chromite/bin/cros_changelog index 73b5435451..f75ceab6a8 100755 --- a/chromite/bin/cros_changelog +++ b/chromite/bin/cros_changelog @@ -52,6 +52,7 @@ def _GrabOutput(cmd): def _GrabTags(): """Returns list of tags from current git repository.""" + # TODO(dianders): replace this with the python equivalent. cmd = ("git for-each-ref refs/tags | awk '{print $3}' | " "sed 's,refs/tags/,,g' | sort -t. -k3,3rn -k4,4rn") return _GrabOutput(cmd).split() @@ -129,7 +130,20 @@ class Commit(object): def __init__(self, commit, projectname, commit_email, commit_date, subject, body, tracker_acc): - """Create commit logs.""" + """Create commit logs. + + Args: + commit: The commit hash (sha) from git. + projectname: The project name, from: + git config --get remote.cros.projectname + commit_email: The email address associated with the commit (%ce in git + log) + commit_date: The date of the commit, like "Mon Nov 1 17:34:14 2010 -0500" + (%cd in git log)) + subject: The subject of the commit (%s in git log) + body: The body of the commit (%b in git log) + tracker_acc: A tracker_access.TrackerAccess object. + """ self.commit = commit self.projectname = projectname self.commit_email = commit_email @@ -149,7 +163,12 @@ class Commit(object): Returns: A list of Issue objects, each of which holds info about a bug. """ + # NOTE: most of this code is copied from bugdroid: + # + # Get a list of bugs. Handle lots of possibilities: + # - Multiple "BUG=" lines, with varying amounts of whitespace. + # - For each BUG= line, bugs can be split by commas _or_ by whitespace (!) entries = [] for line in self.body.split('\n'): match = re.match(r'^ *BUG *=(.*)', line) @@ -157,6 +176,13 @@ class Commit(object): for i in match.group(1).split(','): entries.extend(filter(None, [x.strip() for x in i.split()])) + # Try to parse the bugs. Handle lots of different formats: + # - The whole URL, from which we parse the project and bug. + # - A simple string that looks like "project:bug" + # - A string that looks like "bug", which will always refer to the previous + # tracker referenced (defaulting to the default tracker). + # + # We will create an "Issue" object for each bug. issues = [] last_tracker = DEFAULT_TRACKER regex = (r'http://code.google.com/p/(\S+)/issues/detail\?id=([0-9]+)' @@ -174,6 +200,7 @@ class Commit(object): elif bug_tuple[4]: issues.append(Issue(last_tracker, bug_tuple[4], self._tracker_acc)) + # Sort the issues and return... issues.sort() return issues @@ -205,12 +232,12 @@ class Commit(object): bug_str = 'none' cols = [ - cgi.escape(self.projectname), - str(self.commit_date), - commit_desc, - cgi.escape(self.commit_email), - bug_str, - cgi.escape(self.subject[:100]), + cgi.escape(self.projectname), + str(self.commit_date), + commit_desc, + cgi.escape(self.commit_email), + bug_str, + cgi.escape(self.subject[:100]), ] return '%s' % (''.join(cols)) @@ -221,7 +248,17 @@ class Commit(object): def _GrabChanges(path, tag1, tag2, tracker_acc): - """Return list of commits to path between tag1 and tag2.""" + """Return list of commits to path between tag1 and tag2. + + Args: + path: One of the directories managed by repo. + tag1: The first of the two tags to pass to git log. + tag2: The second of the two tags to pass to git log. + tracker_acc: A tracker_access.TrackerAccess object. + + Returns: + A list of "Commit" objects. + """ cmd = 'cd %s && git config --get remote.cros.projectname' % path projectname = _GrabOutput(cmd).strip() @@ -239,19 +276,24 @@ def _GrabChanges(path, tag1, tag2, tracker_acc): def _ParseArgs(): + """Parse command-line arguments. + + Returns: + An optparse.OptionParser object. + """ parser = optparse.OptionParser() parser.add_option( - "--sort-by-date", dest="sort_by_date", default=False, - action='store_true', help="Sort commits by date.") + '--sort-by-date', dest='sort_by_date', default=False, + action='store_true', help='Sort commits by date.') parser.add_option( - "--tracker-user", dest="tracker_user", default=None, - help="Specify a username to login to code.google.com.") + '--tracker-user', dest='tracker_user', default=None, + help='Specify a username to login to code.google.com.') parser.add_option( - "--tracker-pass", dest="tracker_pass", default=None, - help="Specify a password to go w/ user.") + '--tracker-pass', dest='tracker_pass', default=None, + help='Specify a password to go w/ user.') parser.add_option( - "--tracker-passfile", dest="tracker_passfile", default=None, - help="Specify a file containing a password to go w/ user.") + '--tracker-passfile', dest='tracker_passfile', default=None, + help='Specify a file containing a password to go w/ user.') return parser.parse_args() @@ -264,7 +306,11 @@ def main(): elif len(args) == 1: tag2, = args if tag2 in tags: - tag1 = tags[tags.index(tag2) + 1] + tag2_index = tags.index(tag2) + if tag2_index == len(tags) - 1: + print >>sys.stderr, 'No previous tag for %s' % tag2 + sys.exit(1) + tag1 = tags[tag2_index + 1] else: print >>sys.stderr, 'Unrecognized tag: %s' % tag2 sys.exit(1) @@ -287,7 +333,7 @@ def main(): print >>sys.stderr, INSTRS_FOR_GDATA sys.exit(1) if options.tracker_passfile is not None: - options.tracker_pass = open(options.tracker_passfile, "r").read().strip() + options.tracker_pass = open(options.tracker_passfile, 'r').read().strip() tracker_acc = tracker_access.TrackerAccess(options.tracker_user, options.tracker_pass) else: @@ -307,7 +353,7 @@ def main(): print '' print '' % ('
%s'.join(cols)) if options.sort_by_date: - changes.sort(key=operator.attrgetter("commit_date")) + changes.sort(key=operator.attrgetter('commit_date')) else: changes.sort() for change in changes: From 08145697a5e6a5d07c2ba47e28de0d3804e6980e Mon Sep 17 00:00:00 2001 From: Ken Mixter Date: Mon, 8 Nov 2010 18:10:08 -0800 Subject: [PATCH 19/38] crosutils: Add vnc view of tests running multiple tests for VM tests Change-Id: I87c41e60797848192cee1f0c165215fcec8f5bd2 BUG=8850 TEST=Ran a few tests including suite_Smoke Review URL: http://codereview.chromium.org/4647001 --- bin/cros_run_vm_test | 18 +++++++++++++++--- bin/cros_start_vm | 7 ++++++- lib/cros_vm_lib.sh | 4 ++++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/bin/cros_run_vm_test b/bin/cros_run_vm_test index cef3d115b9..6f5ded2f72 100755 --- a/bin/cros_run_vm_test +++ b/bin/cros_run_vm_test @@ -24,7 +24,6 @@ set -e # Parse command line. FLAGS "$@" || exit 1 -eval set -- "${FLAGS_ARGV}" # Use latest if not specified. if [ -z "${FLAGS_image_path}" ]; then @@ -36,7 +35,20 @@ fi [ -e "${FLAGS_image_path}" ] || die "Image ${FLAGS_image_path} does not exist." -[ -n "${FLAGS_test_case}" ] || die "You must specify a test case." +if [ -n "${FLAGS_test_case}" ]; then + warn "Use of --test_case= is being deprecated. Just pass test names \ +as separate command line arguments." +fi + +if [ -z "${FLAGS_test_case}" ] && [ -z "${FLAGS_ARGV}" ]; then + die "You must specify a test case." +fi + +tests=( ) +[ -n "${FLAGS_test_case}" ] && tests=( "${FLAGS_test_case}" ) +for test in ${FLAGS_ARGV}; do + tests=( "${tests[@]}" "$(remove_quotes "${test}")" ) +done trap stop_kvm EXIT start_kvm "${FLAGS_image_path}" @@ -47,4 +59,4 @@ retry_until_ssh ${MAX_RETRIES} --ssh_port=${FLAGS_ssh_port} \ --remote=127.0.0.1 \ --results_dir_root="${FLAGS_results_dir_root}" \ - "${FLAGS_test_case}" + "${tests[@]}" diff --git a/bin/cros_start_vm b/bin/cros_start_vm index 3ecced9f69..6307920349 100755 --- a/bin/cros_start_vm +++ b/bin/cros_start_vm @@ -10,6 +10,10 @@ . "$(dirname $0)/../lib/cros_vm_lib.sh" . "$(dirname "$0")/../lib/cros_vm_constants.sh" +get_default_board + +DEFINE_string board "${DEFAULT_BOARD}" \ + "Board for VM image (unnecessary if path given)" DEFINE_string image_path "" "Full path of the VM image" set -e @@ -20,7 +24,8 @@ eval set -- "${FLAGS_ARGV}" # Use latest if not specified. if [ -z "${FLAGS_image_path}" ]; then - LATEST_IMAGE="$(${SCRIPTS_DIR}/get_latest_image.sh)/${DEFAULT_QEMU_IMAGE}" + LATEST_IMAGE="$(${SCRIPTS_DIR}/get_latest_image.sh \ + --board=${FLAGS_board})/${DEFAULT_QEMU_IMAGE}" info "Using latest vm image ${LATEST_IMAGE}" FLAGS_image_path=${LATEST_IMAGE} fi diff --git a/lib/cros_vm_lib.sh b/lib/cros_vm_lib.sh index afa481e8fb..7a99cbebe2 100644 --- a/lib/cros_vm_lib.sh +++ b/lib/cros_vm_lib.sh @@ -10,6 +10,7 @@ DEFINE_boolean no_graphics ${FLAGS_FALSE} "Runs the KVM instance silently." DEFINE_boolean persist "${FLAGS_FALSE}" "Persist vm." DEFINE_boolean snapshot ${FLAGS_FALSE} "Don't commit changes to image." DEFINE_integer ssh_port 9222 "Port to tunnel ssh traffic over." +DEFINE_string vnc "" "VNC Server to display to instead of SDL." KVM_PID_FILE=/tmp/kvm.$$.pid LIVE_VM_IMAGE= @@ -42,6 +43,9 @@ function start_kvm() { if [ ${FLAGS_no_graphics} -eq ${FLAGS_TRUE} ]; then nographics="-nographic -serial none" fi + if [ -n "${FLAGS_vnc}" ]; then + nographics="-vnc ${FLAGS_vnc}" + fi if [ ${FLAGS_snapshot} -eq ${FLAGS_TRUE} ]; then snapshot="-snapshot" From 4800e04fe80dbd00a32e404aa12d4a528c2754d3 Mon Sep 17 00:00:00 2001 From: "J. Richard Barnette" Date: Mon, 8 Nov 2010 19:29:26 -0800 Subject: [PATCH 20/38] Try and get the internal buildbot smoke tests to pass BUG=chromium-os:8910 TEST=None Review URL: http://codereview.chromium.org/4683007 Change-Id: I3ff1ccdd038255d8c3594f853ced614ccafc2af3 --- autotest_run.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/autotest_run.sh b/autotest_run.sh index 73907d9f76..17774074e2 100755 --- a/autotest_run.sh +++ b/autotest_run.sh @@ -37,6 +37,11 @@ CONFIG_SITE=/usr/share/config.site function setup_ssh() { eval $(ssh-agent) > /dev/null + # TODO(jrbarnette): This is a temporary hack, slated for removal + # before it was ever created. It's a bug, and you should fix it + # right away! + chmod 400 \ + ${CHROMEOS_ROOT}/src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa ssh-add \ ${CHROMEOS_ROOT}/src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa } From 7d04c7aacb3105f4cc0a852e3d05e1c38754e5dc Mon Sep 17 00:00:00 2001 From: Chris Masone Date: Tue, 9 Nov 2010 08:24:13 -0800 Subject: [PATCH 21/38] [crosutils] bump default size of VM disk to allow for bigger stateful partition BUG=8544 TEST=create an VM image with image_to_vm.sh; the stateful partition should be 2GB in size. Change-Id: I6c0ee9e2cb513e42c25e4f0d1616835a41f378d7 Review URL: http://codereview.chromium.org/4737001 --- image_to_vm.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/image_to_vm.sh b/image_to_vm.sh index fdf23a413c..50e00e5612 100755 --- a/image_to_vm.sh +++ b/image_to_vm.sh @@ -37,7 +37,7 @@ DEFINE_integer rootfs_partition_size 1024 \ "rootfs parition size in MBs." DEFINE_string state_image "" \ "Stateful partition image (defaults to creating new statful partition)" -DEFINE_integer statefulfs_size -1 \ +DEFINE_integer statefulfs_size 2048 \ "Stateful partition size in MBs." DEFINE_boolean test_image "${FLAGS_FALSE}" \ "Copies normal image to chromiumos_test_image.bin, modifies it for test." @@ -285,6 +285,6 @@ fi if [ "${FLAGS_format}" == "qemu" ]; then echo "If you have qemu-kvm installed, you can start the image by:" echo "sudo kvm -m ${FLAGS_mem} -vga std -pidfile /tmp/kvm.pid -net nic,model=e1000 " \ - "-net user,hostfwd=tcp::922-:22 \\" + "-net user,hostfwd=tcp::9222-:22 \\" echo " -hda ${FLAGS_to}/${DEFAULT_QEMU_IMAGE}" fi From d11ce17167d744646084138dba7b2693a8879b17 Mon Sep 17 00:00:00 2001 From: Chris Masone Date: Tue, 9 Nov 2010 14:22:08 -0800 Subject: [PATCH 22/38] [crosutils] Mask out /usr/local/autotest-pkgs on install BUG=8544 TEST=build image with --factory_install, ensure that image does not have data in /usr/local/autotest-pkgs Change-Id: I5ed17d794392ec654bc122fa15f7ab8a7abbba2d Review URL: http://codereview.chromium.org/4756001 --- common.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common.sh b/common.sh index 25abd9b4c3..502a8c6c10 100644 --- a/common.sh +++ b/common.sh @@ -129,6 +129,7 @@ fi CHROOT_TRUNK_DIR="/home/$USER/trunk" # Install make for portage ebuilds. Used by build_image and gmergefs. +# TODO: Is /usr/local/autotest-chrome still used by anyone? DEFAULT_INSTALL_MASK="/usr/include /usr/man /usr/share/man /usr/share/doc \ /usr/share/gtk-doc /usr/share/gtk-2.0 /usr/lib/gtk-2.0/include \ /usr/share/info /usr/share/aclocal /usr/lib/gcc /usr/lib/pkgconfig \ @@ -143,7 +144,7 @@ FACTORY_INSTALL_MASK="/opt/google/chrome /opt/google/o3d /opt/netscape \ /usr/share/ibus-pinyin /usr/share/libhangul /usr/share/locale \ /usr/share/m17n /usr/share/mime /usr/share/sounds /usr/share/tts \ /usr/share/X11 /usr/share/zoneinfo /usr/lib/debug - /usr/local/autotest /usr/local/autotest-chrome" + /usr/local/autotest /usr/local/autotest-chrome /usr/local/autotest-pkgs" # Check to ensure not running old scripts V_REVERSE='' From f8513d84fec1c26ea1d46ca1de6f6bd29114f987 Mon Sep 17 00:00:00 2001 From: Chris Masone Date: Tue, 9 Nov 2010 14:23:29 -0800 Subject: [PATCH 23/38] [crosutils] Propagate use_emerged flag up from run_remote_tests BUG=8544 TEST=build chrome with build_tests, cros_workon start autotest, and then run desktopui_BrowserTest with cros_run_parallel_vm_tests Change-Id: I19e181238fd75450d41f6debb3a8f7d0c17d15dc Review URL: http://codereview.chromium.org/4669007 --- bin/cros_run_parallel_vm_tests.py | 9 +++++++-- bin/cros_run_vm_test | 10 +++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/bin/cros_run_parallel_vm_tests.py b/bin/cros_run_parallel_vm_tests.py index 0c0b08b88a..2ca51a3afa 100755 --- a/bin/cros_run_parallel_vm_tests.py +++ b/bin/cros_run_parallel_vm_tests.py @@ -27,7 +27,8 @@ class ParallelTestRunner(object): """ def __init__(self, tests, base_ssh_port=_DEFAULT_BASE_SSH_PORT, board=None, - image_path=None, order_output=False, results_dir_root=None): + image_path=None, order_output=False, results_dir_root=None, + use_emerged=False): """Constructs and initializes the test runner class. Args: @@ -50,6 +51,7 @@ class ParallelTestRunner(object): self._image_path = image_path self._order_output = order_output self._results_dir_root = results_dir_root + self._use_emerged = use_emerged def _SpawnTests(self): """Spawns VMs and starts the test runs on them. @@ -75,6 +77,7 @@ class ParallelTestRunner(object): if self._results_dir_root: args.append('--results_dir_root=%s/%s.%d' % (self._results_dir_root, test, ssh_port)) + if self._use_emerged: args.append('--use_emerged') Info('Running %r...' % args) output = None if self._order_output: @@ -142,6 +145,8 @@ def main(): parser.add_option('--results_dir_root', help='Root results directory. If none specified, each test ' 'will store its results in a separate /tmp directory.') + parser.add_option('--use_emerged', action='store_true', default=False, + help='Force use of emerged autotest packages') (options, args) = parser.parse_args() if not args: @@ -150,7 +155,7 @@ def main(): runner = ParallelTestRunner(args, options.base_ssh_port, options.board, options.image_path, options.order_output, - options.results_dir_root) + options.results_dir_root, options.use_emerged) runner.Run() diff --git a/bin/cros_run_vm_test b/bin/cros_run_vm_test index 6f5ded2f72..fbbee23c7c 100755 --- a/bin/cros_run_vm_test +++ b/bin/cros_run_vm_test @@ -15,10 +15,12 @@ MAX_RETRIES=3 get_default_board DEFINE_string board "$DEFAULT_BOARD" \ - "The board for which you built autotest." + "The board for which you built autotest." b DEFINE_string image_path "" "Full path of the VM image" DEFINE_string results_dir_root "" "alternate root results directory" DEFINE_string test_case "" "Name of the test case to run" +DEFINE_boolean use_emerged ${FLAGS_FALSE} \ + "Force use of emerged autotest packages" set -e @@ -44,6 +46,11 @@ if [ -z "${FLAGS_test_case}" ] && [ -z "${FLAGS_ARGV}" ]; then die "You must specify a test case." fi +USE_EMERGED= +if [[ ${FLAGS_use_emerged} -eq ${FLAGS_TRUE} ]]; then + USE_EMERGED="--use_emerged" +fi + tests=( ) [ -n "${FLAGS_test_case}" ] && tests=( "${FLAGS_test_case}" ) for test in ${FLAGS_ARGV}; do @@ -59,4 +66,5 @@ retry_until_ssh ${MAX_RETRIES} --ssh_port=${FLAGS_ssh_port} \ --remote=127.0.0.1 \ --results_dir_root="${FLAGS_results_dir_root}" \ + ${USE_EMERGED} \ "${tests[@]}" From 5bedbc9ad7a892c6d9a89f063c50a31419a79154 Mon Sep 17 00:00:00 2001 From: Ken Mixter Date: Wed, 10 Nov 2010 16:42:43 -0800 Subject: [PATCH 24/38] crosutils: Require cros_show_stacks to run inside chroot Change-Id: I28e01fb8ddb542eddf3750def7d49cd40822d6d9 BUG=9026 TEST=Try outside chroot Review URL: http://codereview.chromium.org/4782003 --- cros_show_stacks | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cros_show_stacks b/cros_show_stacks index c5bc02cc64..fa6b769321 100755 --- a/cros_show_stacks +++ b/cros_show_stacks @@ -12,7 +12,7 @@ . "$(dirname $0)/common.sh" . "$(dirname $0)/remote_access.sh" -restart_in_chroot_if_needed $* +assert_inside_chroot MINIDUMP_DUMP=/usr/bin/minidump_dump MINIDUMP_STACKWALK=/usr/bin/minidump_stackwalk From 9a40915f0093c3ab4e1f7f56b54de05b9dc88e87 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Wed, 10 Nov 2010 17:31:51 -0800 Subject: [PATCH 25/38] Add new stateful upload payload generator to crosutils. Change-Id: I53a24d6188711a248a1dc7c869aa9258b10970d2 BUG=chromium-os:8885 TEST=Ran by itself with latest image and current directory as my output dir as well as with new dev server changes. Review URL: http://codereview.chromium.org/4688006 --- cros_generate_stateful_update_payload | 1 + cros_generate_stateful_update_payload.py | 94 ++++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 120000 cros_generate_stateful_update_payload create mode 100755 cros_generate_stateful_update_payload.py diff --git a/cros_generate_stateful_update_payload b/cros_generate_stateful_update_payload new file mode 120000 index 0000000000..05cb0081bc --- /dev/null +++ b/cros_generate_stateful_update_payload @@ -0,0 +1 @@ +cros_generate_stateful_update_payload.py \ No newline at end of file diff --git a/cros_generate_stateful_update_payload.py b/cros_generate_stateful_update_payload.py new file mode 100755 index 0000000000..b8bc1eb65e --- /dev/null +++ b/cros_generate_stateful_update_payload.py @@ -0,0 +1,94 @@ +#!/usr/bin/python2.6 + +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""This module is responsible for generate a stateful update payload.""" + +import logging +import optparse +import os +import subprocess +import tempfile + +STATEFUL_FILE = 'stateful.tgz' + + +def GenerateStatefulPayload(image_path, output_directory, logger): + """Generates a stateful update payload given a full path to an image. + + Args: + image_path: Full path to the image. + output_directory: Path to the directory to leave the resulting output. + logger: logging instance. + """ + logger.info('Generating stateful update file.') + from_dir = os.path.dirname(image_path) + image = os.path.basename(image_path) + output_gz = os.path.join(output_directory, STATEFUL_FILE) + crosutils_dir = os.path.dirname(__file__) + + # Temporary directories for this function. + rootfs_dir = tempfile.mkdtemp(suffix='rootfs', prefix='tmp') + stateful_dir = tempfile.mkdtemp(suffix='stateful', prefix='tmp') + + # Mount the image to pull out the important directories. + try: + # Only need stateful partition, but this saves us having to manage our + # own loopback device. + subprocess.check_call(['%s/mount_gpt_image.sh' % crosutils_dir, + '--from=%s' % from_dir, + '--image=%s' % image, + '--read_only', + '--rootfs_mountpt=%s' % rootfs_dir, + '--stateful_mountpt=%s' % stateful_dir, + ]) + logger.info('Tarring up /usr/local and /var!') + subprocess.check_call(['sudo', + 'tar', + '-czf', + output_gz, + '--directory=%s' % stateful_dir, + '--transform=s,^dev_image,dev_image_new,', + '--transform=s,^var,var_new,', + 'dev_image', + 'var', + ]) + except: + logger.error('Failed to create stateful update file') + raise + finally: + # Unmount best effort regardless. + subprocess.call(['%s/mount_gpt_image.sh' % crosutils_dir, + '--unmount', + '--rootfs_mountpt=%s' % rootfs_dir, + '--stateful_mountpt=%s' % stateful_dir, + ]) + # Clean up our directories. + os.rmdir(rootfs_dir) + os.rmdir(stateful_dir) + + logger.info('Successfully generated %s' % output_gz) + + +def main(): + logging.basicConfig(level=logging.INFO) + logger = logging.getLogger(os.path.basename(__file__)) + parser = optparse.OptionParser() + parser.add_option('-i', '--image_path', + help='The image to generate the stateful update for.') + parser.add_option('-o', '--output_dir', + help='The path to the directory to output the update file.') + options, unused_args = parser.parse_args() + if not options.image_path: + parser.error('Missing image for stateful payload generator') + if not options.output_dir: + parser.error('Missing output directory for the payload generator') + + GenerateStatefulPayload(os.path.abspath(options.image_path), + options.output_dir, logger) + + +if __name__ == '__main__': + main() From b4258c8bd48f89d87f9d1aa2fc23b0556ffc611b Mon Sep 17 00:00:00 2001 From: David James Date: Thu, 11 Nov 2010 10:37:04 -0800 Subject: [PATCH 26/38] Add more error checking to preflight queue. What's new? - cros_mark_as_stable now exits with errors if directories are specified that don't exist. - cbuildbot.py always explicitly specifies overlay directories so cros_mark_as_stable can rely on them existing. - Package names and paths are now separated with colons instead of spaces, so as to allow for us using the same syntax with enter_chroot.sh as we use without the same script. (enter_chroot.sh mucks with command-lines that contain spaces or quotes.) - All buildbots now explicitly specify what overlays they want to rev. Public buildbots only rev public ebuilds and private buildbots now only rev private ebuilds. BUG=chromium-os:8647 TEST=Ran unit tests. Manually marked packages as stable. Change-Id: I1df6d428973d91329c4f5159e2886889a3ebb7c7 Review URL: http://codereview.chromium.org/4442001 --- bin/cbuildbot.py | 83 ++++++++++++++++++++++++--------------- bin/cbuildbot_config.py | 39 +++++------------- bin/cbuildbot_unittest.py | 13 ++++-- cros_mark_as_stable.py | 18 ++++++--- 4 files changed, 83 insertions(+), 70 deletions(-) diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index 372ddcaf5f..b0a26af94c 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -181,37 +181,35 @@ def _ParseRevisionString(revision_string, repo_dictionary): return revisions.items() -def _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board): +def _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board, + overlays): """Uprevs based on revision list.""" if not revision_list: Info('No packages found to uprev') return - package_str = '' + packages = [] for package, revision in revision_list: - package_str += package + ' ' - - package_str = package_str.strip() + assert ':' not in package, 'Invalid package name: %s' % package + packages.append(package) cwd = os.path.join(buildroot, 'src', 'scripts') - # TODO(davidjames): --foo="bar baz" only works here because we're using - # enter_chroot. RunCommand(['./cros_mark_as_stable', '--board=%s' % board, - '--tracking_branch="%s"' % tracking_branch, - '--packages="%s"' % package_str, + '--tracking_branch=%s' % tracking_branch, + '--overlays=%s' % ':'.join(overlays), + '--packages=%s' % ':'.join(packages), 'commit'], cwd=cwd, enter_chroot=True) -def _UprevAllPackages(buildroot, tracking_branch, board): +def _UprevAllPackages(buildroot, tracking_branch, board, overlays): """Uprevs all packages that have been updated since last uprev.""" cwd = os.path.join(buildroot, 'src', 'scripts') - # TODO(davidjames): --foo="bar baz" only works here because we're using - # enter_chroot. RunCommand(['./cros_mark_as_stable', '--all', '--board=%s' % board, - '--tracking_branch="%s"' % tracking_branch, 'commit'], + '--overlays=%s' % ':'.join(overlays), + '--tracking_branch=%s' % tracking_branch, 'commit'], cwd=cwd, enter_chroot=True) @@ -228,12 +226,13 @@ def _GetVMConstants(buildroot): return (vdisk_size.strip(), statefulfs_size.strip()) -def _GitCleanup(buildroot, board, tracking_branch): +def _GitCleanup(buildroot, board, tracking_branch, overlays): """Clean up git branch after previous uprev attempt.""" cwd = os.path.join(buildroot, 'src', 'scripts') if os.path.exists(cwd): RunCommand(['./cros_mark_as_stable', '--srcroot=..', '--board=%s' % board, + '--overlays=%s' % ':'.join(overlays), '--tracking_branch=%s' % tracking_branch, 'clean'], cwd=cwd, error_ok=True) @@ -257,9 +256,9 @@ def _WipeOldOutput(buildroot): # =========================== Main Commands =================================== -def _PreFlightRinse(buildroot, board, tracking_branch): +def _PreFlightRinse(buildroot, board, tracking_branch, overlays): """Cleans up any leftover state from previous runs.""" - _GitCleanup(buildroot, board, tracking_branch) + _GitCleanup(buildroot, board, tracking_branch, overlays) _CleanUpMountPoints(buildroot) RunCommand(['sudo', 'killall', 'kvm'], error_ok=True) @@ -347,7 +346,7 @@ def _RunSmokeSuite(buildroot, results_dir): ], cwd=cwd, error_ok=False) -def _UprevPackages(buildroot, tracking_branch, revisionfile, board): +def _UprevPackages(buildroot, tracking_branch, revisionfile, board, overlays): """Uprevs a package based on given revisionfile. If revisionfile is set to None or does not resolve to an actual file, this @@ -376,26 +375,19 @@ def _UprevPackages(buildroot, tracking_branch, revisionfile, board): # print >> sys.stderr, 'CBUILDBOT Revision list found %s' % revisions # revision_list = _ParseRevisionString(revisions, # _CreateRepoDictionary(buildroot, board)) - # _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board) + # _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board, + # overlays) #else: Info('CBUILDBOT Revving all') - _UprevAllPackages(buildroot, tracking_branch, board) + _UprevAllPackages(buildroot, tracking_branch, board, overlays) def _UprevPush(buildroot, tracking_branch, board, overlays): """Pushes uprev changes to the main line.""" cwd = os.path.join(buildroot, 'src', 'scripts') - public_overlay = '%s/src/third_party/chromiumos-overlay' % buildroot - private_overlay = '%s/src/private-overlays/chromeos-overlay' % buildroot - if overlays == 'private': - overlays = [private_overlay] - elif overlays == 'public': - overlays = [public_overlay] - else: - overlays = [public_overlay, private_overlay] RunCommand(['./cros_mark_as_stable', '--srcroot=..', '--board=%s' % board, - '--overlays=%s' % " ".join(overlays), + '--overlays=%s' % ':'.join(overlays), '--tracking_branch=%s' % tracking_branch, '--push_options=--bypass-hooks -f', 'push'], cwd=cwd) @@ -467,6 +459,32 @@ def _GetConfig(config_name): return buildconfig +def ResolveOverlays(overlays): + """Return the list of overlays to use for a given buildbot. + + Args: + overlays: A string describing which overlays you want. + 'private': Just the private overlay. + 'public': Just the public overlay. + 'both': Both the public and private overlays. + """ + public_overlay = '%s/src/third_party/chromiumos-overlay' % buildroot + private_overlay = '%s/src/private-overlays/chromeos-overlay' % buildroot + if overlays == 'private': + dirs = [private_overlay] + elif overlays == 'public': + dirs = [public_overlay] + elif overlays == 'both': + dirs = [public_overlay, private_overlay] + else: + Die('Incorrect overlay configuration: %s' % overlays) + for dir in dirs: + assert ':' not in dir, 'Overlay must not contain colons: %s' % dir + if not os.path.exists(dir): + Die('Missing overlay: %s' % dir) + return dirs + + def main(): # Parse options usage = "usage: %prog [options] cbuildbot_config" @@ -502,8 +520,11 @@ def main(): parser.print_usage() sys.exit(1) + # Calculate list of overlay directories. + overlays = ResolveOverlays(buildconfig['overlays']) + try: - _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch) + _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays) if options.clobber or not os.path.isdir(buildroot): _FullCheckout(buildroot, tracking_branch, url=options.url) else: @@ -519,7 +540,7 @@ def main(): if buildconfig['uprev']: _UprevPackages(buildroot, tracking_branch, revisionfile, - board=buildconfig['board']) + buildconfig['board'], overlays) _EnableLocalAccount(buildroot) _Build(buildroot) @@ -545,7 +566,7 @@ def main(): # Master bot needs to check if the other slaves completed. if cbuildbot_comm.HaveSlavesCompleted(config): _UprevPush(buildroot, tracking_branch, buildconfig['board'], - buildconfig['overlays']) + overlays) else: Die('CBUILDBOT - One of the slaves has failed!!!') diff --git a/bin/cbuildbot_config.py b/bin/cbuildbot_config.py index 831ef06bd1..718cd36a70 100644 --- a/bin/cbuildbot_config.py +++ b/bin/cbuildbot_config.py @@ -19,9 +19,9 @@ hostname -- Needed for 'important' slaves. The hostname of the bot. Should match hostname in slaves.cfg in buildbot checkout. unittests -- Runs unittests for packages. smoke_bvt -- Runs the test smoke suite in a qemu-based VM using KVM. -overlays -- If this bot is a master bot, select what overlays to push changes - to. This can be 'public', 'private', or 'both'. There should only - be one bot pushing changes to each overlay. +overlays -- Select what overlays to look at. This can be 'public', 'private' + or 'both'. There should only be one master bot pushing changes to + each overlay per branch. """ @@ -33,6 +33,7 @@ config['default'] = { 'important' : False, 'unittests' : False, 'smoke_bvt' : False, + 'overlays': 'public', } config['x86-generic-pre-flight-queue'] = { 'board' : 'x86-generic', @@ -69,6 +70,7 @@ config['x86_agz_bin'] = { 'important' : False, 'unittests' : True, 'smoke_bvt' : True, + 'overlays': 'private', } config['x86_dogfood_bin'] = { 'board' : 'x86-dogfood', @@ -77,52 +79,29 @@ config['x86_dogfood_bin'] = { 'important' : False, 'unittests' : True, 'smoke_bvt' : True, + 'overlays': 'private', } config['x86_pineview_bin'] = { 'board' : 'x86-pineview', 'uprev' : True, 'master' : False, 'important' : False, - 'hostname' : 'codf200.jail', 'unittests': True, + 'overlays': 'public', } config['arm_tegra2_bin'] = { 'board' : 'tegra2', 'uprev' : True, 'master' : False, 'important' : False, - 'hostname' : 'codg172.jail', - 'unittests' : False, -} -config['arm_voguev210_bin'] = { - 'board' : 'voguev210', - 'uprev' : True, - 'master' : False, - 'important' : False, - 'hostname' : 'codf196.jail', - 'unittests' : False, -} -config['arm_beagleboard_bin'] = { - 'board' : 'beagleboard', - 'master' : False, - 'uprev' : True, - 'important' : False, - 'hostname' : 'codf202.jail', - 'unittests' : False, -} -config['arm_st1q_bin'] = { - 'board' : 'st1q', - 'uprev' : True, - 'master' : False, - 'important' : False, - 'hostname' : 'codg158.jail', 'unittests' : False, + 'overlays': 'public', } config['arm_generic_bin'] = { 'board' : 'arm-generic', 'uprev' : True, 'master' : False, 'important' : False, - 'hostname' : 'codg175.jail', 'unittests' : False, + 'overlays': 'public', } diff --git a/bin/cbuildbot_unittest.py b/bin/cbuildbot_unittest.py index bbbc56c6f7..7c642f1789 100755 --- a/bin/cbuildbot_unittest.py +++ b/bin/cbuildbot_unittest.py @@ -166,15 +166,17 @@ class CBuildBotTest(mox.MoxTestBase): m_file.read().AndReturn(self._test_string) m_file.close() + overlays = [ '%s/src/third_party/chromiumos-overlay' % self._buildroot ] cbuildbot.RunCommand(['./cros_mark_as_stable', '--all', '--board=%s' % self._test_board, - '--tracking_branch="cros/master"', 'commit'], + '--overlays=%s' % ':'.join(overlays), + '--tracking_branch=cros/master', 'commit'], cwd='%s/src/scripts' % self._buildroot, enter_chroot=True) self.mox.ReplayAll() cbuildbot._UprevPackages(self._buildroot, self.tracking_branch, - self._revision_file, self._test_board) + self._revision_file, self._test_board, overlays) self.mox.VerifyAll() def testUprevAllPackages(self): @@ -187,15 +189,18 @@ class CBuildBotTest(mox.MoxTestBase): m_file.read().AndReturn('None') m_file.close() + overlays = [ '%s/src/third_party/chromiumos-overlay' % self._buildroot ] cbuildbot.RunCommand(['./cros_mark_as_stable', '--all', '--board=%s' % self._test_board, - '--tracking_branch="cros/master"', 'commit'], + '--overlays=%s' % ':'.join(overlays), + '--tracking_branch=cros/master', 'commit'], cwd='%s/src/scripts' % self._buildroot, enter_chroot=True) self.mox.ReplayAll() + overlays = [ '%s/src/third_party/chromiumos-overlay' % self._buildroot ] cbuildbot._UprevPackages(self._buildroot, self.tracking_branch, - self._revision_file, self._test_board) + self._revision_file, self._test_board, overlays) self.mox.VerifyAll() diff --git a/cros_mark_as_stable.py b/cros_mark_as_stable.py index 56a589c682..10578e3916 100755 --- a/cros_mark_as_stable.py +++ b/cros_mark_as_stable.py @@ -22,10 +22,10 @@ from cros_build_lib import Info, RunCommand, Warning, Die gflags.DEFINE_string('board', '', 'Board for which the package belongs.', short_name='b') gflags.DEFINE_string('overlays', '', - 'Space separated list of overlays to modify.', + 'Colon-separated list of overlays to modify.', short_name='o') gflags.DEFINE_string('packages', '', - 'Space separated list of packages to mark as stable.', + 'Colon-separated list of packages to mark as stable.', short_name='p') gflags.DEFINE_string('push_options', '', 'Options to use with git-cl push using push command.') @@ -245,7 +245,10 @@ def _SimpleRunCommand(command): """Runs a shell command and returns stdout back to caller.""" _Print(' + %s' % command) proc_handle = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) - return proc_handle.communicate()[0] + stdout = proc_handle.communicate()[0] + retcode = proc_handle.wait() + assert retcode == 0, "Return code %s for command: %s" % (retcode, command) + return stdout # ======================= End Global Helper Functions ======================== @@ -487,11 +490,16 @@ def main(argv): except gflags.FlagsError, e : _PrintUsageAndDie(str(e)) - package_list = gflags.FLAGS.packages.split() + package_list = gflags.FLAGS.packages.split(':') _CheckSaneArguments(package_list, command) if gflags.FLAGS.overlays: - overlays = dict((path, []) for path in gflags.FLAGS.overlays.split()) + overlays = {} + for path in gflags.FLAGS.overlays.split(':'): + if not os.path.exists(path): + Die('Cannot find overlay: %s' % path) + overlays[path] = [] else: + Warning('Missing --overlays argument') overlays = { '%s/private-overlays/chromeos-overlay' % gflags.FLAGS.srcroot: [], '%s/third_party/chromiumos-overlay' % gflags.FLAGS.srcroot: [] From f9b84d61a13673957c404cda0703386a659431bc Mon Sep 17 00:00:00 2001 From: David James Date: Thu, 11 Nov 2010 10:45:12 -0800 Subject: [PATCH 27/38] Define buildroot properly. Followup to "Add more error checking to preflight queue." BUG=Br0ken build TBR=kliegs@chromium.org TEST=Ran unit tests Review URL: http://codereview.chromium.org/4742004 Change-Id: I63e52265ae404f0193cb1f9ab35f3f1ec266d2c1 --- bin/cbuildbot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index b0a26af94c..c255f2f7e0 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -459,7 +459,7 @@ def _GetConfig(config_name): return buildconfig -def ResolveOverlays(overlays): +def ResolveOverlays(buildroot, overlays): """Return the list of overlays to use for a given buildbot. Args: @@ -521,7 +521,7 @@ def main(): sys.exit(1) # Calculate list of overlay directories. - overlays = ResolveOverlays(buildconfig['overlays']) + overlays = ResolveOverlays(buildroot, buildconfig['overlays']) try: _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays) From 4beb5cb58fdbfdee335e06fc98d5fa7b9f5da2d5 Mon Sep 17 00:00:00 2001 From: David James Date: Thu, 11 Nov 2010 10:51:07 -0800 Subject: [PATCH 28/38] Revert "Add more error checking to preflight queue." This reverts commit b4258c8bd48f89d87f9d1aa2fc23b0556ffc611b. BUG=br0ken build TBR=rspangler TEST= Review URL: http://codereview.chromium.org/4697007 Change-Id: I839ffd08d33fc5943696cfe31f2978fe32ce5fec --- bin/cbuildbot.py | 83 +++++++++++++++------------------------ bin/cbuildbot_config.py | 39 +++++++++++++----- bin/cbuildbot_unittest.py | 13 ++---- cros_mark_as_stable.py | 18 +++------ 4 files changed, 70 insertions(+), 83 deletions(-) diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index c255f2f7e0..372ddcaf5f 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -181,35 +181,37 @@ def _ParseRevisionString(revision_string, repo_dictionary): return revisions.items() -def _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board, - overlays): +def _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board): """Uprevs based on revision list.""" if not revision_list: Info('No packages found to uprev') return - packages = [] + package_str = '' for package, revision in revision_list: - assert ':' not in package, 'Invalid package name: %s' % package - packages.append(package) + package_str += package + ' ' + + package_str = package_str.strip() cwd = os.path.join(buildroot, 'src', 'scripts') + # TODO(davidjames): --foo="bar baz" only works here because we're using + # enter_chroot. RunCommand(['./cros_mark_as_stable', '--board=%s' % board, - '--tracking_branch=%s' % tracking_branch, - '--overlays=%s' % ':'.join(overlays), - '--packages=%s' % ':'.join(packages), + '--tracking_branch="%s"' % tracking_branch, + '--packages="%s"' % package_str, 'commit'], cwd=cwd, enter_chroot=True) -def _UprevAllPackages(buildroot, tracking_branch, board, overlays): +def _UprevAllPackages(buildroot, tracking_branch, board): """Uprevs all packages that have been updated since last uprev.""" cwd = os.path.join(buildroot, 'src', 'scripts') + # TODO(davidjames): --foo="bar baz" only works here because we're using + # enter_chroot. RunCommand(['./cros_mark_as_stable', '--all', '--board=%s' % board, - '--overlays=%s' % ':'.join(overlays), - '--tracking_branch=%s' % tracking_branch, 'commit'], + '--tracking_branch="%s"' % tracking_branch, 'commit'], cwd=cwd, enter_chroot=True) @@ -226,13 +228,12 @@ def _GetVMConstants(buildroot): return (vdisk_size.strip(), statefulfs_size.strip()) -def _GitCleanup(buildroot, board, tracking_branch, overlays): +def _GitCleanup(buildroot, board, tracking_branch): """Clean up git branch after previous uprev attempt.""" cwd = os.path.join(buildroot, 'src', 'scripts') if os.path.exists(cwd): RunCommand(['./cros_mark_as_stable', '--srcroot=..', '--board=%s' % board, - '--overlays=%s' % ':'.join(overlays), '--tracking_branch=%s' % tracking_branch, 'clean'], cwd=cwd, error_ok=True) @@ -256,9 +257,9 @@ def _WipeOldOutput(buildroot): # =========================== Main Commands =================================== -def _PreFlightRinse(buildroot, board, tracking_branch, overlays): +def _PreFlightRinse(buildroot, board, tracking_branch): """Cleans up any leftover state from previous runs.""" - _GitCleanup(buildroot, board, tracking_branch, overlays) + _GitCleanup(buildroot, board, tracking_branch) _CleanUpMountPoints(buildroot) RunCommand(['sudo', 'killall', 'kvm'], error_ok=True) @@ -346,7 +347,7 @@ def _RunSmokeSuite(buildroot, results_dir): ], cwd=cwd, error_ok=False) -def _UprevPackages(buildroot, tracking_branch, revisionfile, board, overlays): +def _UprevPackages(buildroot, tracking_branch, revisionfile, board): """Uprevs a package based on given revisionfile. If revisionfile is set to None or does not resolve to an actual file, this @@ -375,19 +376,26 @@ def _UprevPackages(buildroot, tracking_branch, revisionfile, board, overlays): # print >> sys.stderr, 'CBUILDBOT Revision list found %s' % revisions # revision_list = _ParseRevisionString(revisions, # _CreateRepoDictionary(buildroot, board)) - # _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board, - # overlays) + # _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board) #else: Info('CBUILDBOT Revving all') - _UprevAllPackages(buildroot, tracking_branch, board, overlays) + _UprevAllPackages(buildroot, tracking_branch, board) def _UprevPush(buildroot, tracking_branch, board, overlays): """Pushes uprev changes to the main line.""" cwd = os.path.join(buildroot, 'src', 'scripts') + public_overlay = '%s/src/third_party/chromiumos-overlay' % buildroot + private_overlay = '%s/src/private-overlays/chromeos-overlay' % buildroot + if overlays == 'private': + overlays = [private_overlay] + elif overlays == 'public': + overlays = [public_overlay] + else: + overlays = [public_overlay, private_overlay] RunCommand(['./cros_mark_as_stable', '--srcroot=..', '--board=%s' % board, - '--overlays=%s' % ':'.join(overlays), + '--overlays=%s' % " ".join(overlays), '--tracking_branch=%s' % tracking_branch, '--push_options=--bypass-hooks -f', 'push'], cwd=cwd) @@ -459,32 +467,6 @@ def _GetConfig(config_name): return buildconfig -def ResolveOverlays(buildroot, overlays): - """Return the list of overlays to use for a given buildbot. - - Args: - overlays: A string describing which overlays you want. - 'private': Just the private overlay. - 'public': Just the public overlay. - 'both': Both the public and private overlays. - """ - public_overlay = '%s/src/third_party/chromiumos-overlay' % buildroot - private_overlay = '%s/src/private-overlays/chromeos-overlay' % buildroot - if overlays == 'private': - dirs = [private_overlay] - elif overlays == 'public': - dirs = [public_overlay] - elif overlays == 'both': - dirs = [public_overlay, private_overlay] - else: - Die('Incorrect overlay configuration: %s' % overlays) - for dir in dirs: - assert ':' not in dir, 'Overlay must not contain colons: %s' % dir - if not os.path.exists(dir): - Die('Missing overlay: %s' % dir) - return dirs - - def main(): # Parse options usage = "usage: %prog [options] cbuildbot_config" @@ -520,11 +502,8 @@ def main(): parser.print_usage() sys.exit(1) - # Calculate list of overlay directories. - overlays = ResolveOverlays(buildroot, buildconfig['overlays']) - try: - _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays) + _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch) if options.clobber or not os.path.isdir(buildroot): _FullCheckout(buildroot, tracking_branch, url=options.url) else: @@ -540,7 +519,7 @@ def main(): if buildconfig['uprev']: _UprevPackages(buildroot, tracking_branch, revisionfile, - buildconfig['board'], overlays) + board=buildconfig['board']) _EnableLocalAccount(buildroot) _Build(buildroot) @@ -566,7 +545,7 @@ def main(): # Master bot needs to check if the other slaves completed. if cbuildbot_comm.HaveSlavesCompleted(config): _UprevPush(buildroot, tracking_branch, buildconfig['board'], - overlays) + buildconfig['overlays']) else: Die('CBUILDBOT - One of the slaves has failed!!!') diff --git a/bin/cbuildbot_config.py b/bin/cbuildbot_config.py index 718cd36a70..831ef06bd1 100644 --- a/bin/cbuildbot_config.py +++ b/bin/cbuildbot_config.py @@ -19,9 +19,9 @@ hostname -- Needed for 'important' slaves. The hostname of the bot. Should match hostname in slaves.cfg in buildbot checkout. unittests -- Runs unittests for packages. smoke_bvt -- Runs the test smoke suite in a qemu-based VM using KVM. -overlays -- Select what overlays to look at. This can be 'public', 'private' - or 'both'. There should only be one master bot pushing changes to - each overlay per branch. +overlays -- If this bot is a master bot, select what overlays to push changes + to. This can be 'public', 'private', or 'both'. There should only + be one bot pushing changes to each overlay. """ @@ -33,7 +33,6 @@ config['default'] = { 'important' : False, 'unittests' : False, 'smoke_bvt' : False, - 'overlays': 'public', } config['x86-generic-pre-flight-queue'] = { 'board' : 'x86-generic', @@ -70,7 +69,6 @@ config['x86_agz_bin'] = { 'important' : False, 'unittests' : True, 'smoke_bvt' : True, - 'overlays': 'private', } config['x86_dogfood_bin'] = { 'board' : 'x86-dogfood', @@ -79,29 +77,52 @@ config['x86_dogfood_bin'] = { 'important' : False, 'unittests' : True, 'smoke_bvt' : True, - 'overlays': 'private', } config['x86_pineview_bin'] = { 'board' : 'x86-pineview', 'uprev' : True, 'master' : False, 'important' : False, + 'hostname' : 'codf200.jail', 'unittests': True, - 'overlays': 'public', } config['arm_tegra2_bin'] = { 'board' : 'tegra2', 'uprev' : True, 'master' : False, 'important' : False, + 'hostname' : 'codg172.jail', + 'unittests' : False, +} +config['arm_voguev210_bin'] = { + 'board' : 'voguev210', + 'uprev' : True, + 'master' : False, + 'important' : False, + 'hostname' : 'codf196.jail', + 'unittests' : False, +} +config['arm_beagleboard_bin'] = { + 'board' : 'beagleboard', + 'master' : False, + 'uprev' : True, + 'important' : False, + 'hostname' : 'codf202.jail', + 'unittests' : False, +} +config['arm_st1q_bin'] = { + 'board' : 'st1q', + 'uprev' : True, + 'master' : False, + 'important' : False, + 'hostname' : 'codg158.jail', 'unittests' : False, - 'overlays': 'public', } config['arm_generic_bin'] = { 'board' : 'arm-generic', 'uprev' : True, 'master' : False, 'important' : False, + 'hostname' : 'codg175.jail', 'unittests' : False, - 'overlays': 'public', } diff --git a/bin/cbuildbot_unittest.py b/bin/cbuildbot_unittest.py index 7c642f1789..bbbc56c6f7 100755 --- a/bin/cbuildbot_unittest.py +++ b/bin/cbuildbot_unittest.py @@ -166,17 +166,15 @@ class CBuildBotTest(mox.MoxTestBase): m_file.read().AndReturn(self._test_string) m_file.close() - overlays = [ '%s/src/third_party/chromiumos-overlay' % self._buildroot ] cbuildbot.RunCommand(['./cros_mark_as_stable', '--all', '--board=%s' % self._test_board, - '--overlays=%s' % ':'.join(overlays), - '--tracking_branch=cros/master', 'commit'], + '--tracking_branch="cros/master"', 'commit'], cwd='%s/src/scripts' % self._buildroot, enter_chroot=True) self.mox.ReplayAll() cbuildbot._UprevPackages(self._buildroot, self.tracking_branch, - self._revision_file, self._test_board, overlays) + self._revision_file, self._test_board) self.mox.VerifyAll() def testUprevAllPackages(self): @@ -189,18 +187,15 @@ class CBuildBotTest(mox.MoxTestBase): m_file.read().AndReturn('None') m_file.close() - overlays = [ '%s/src/third_party/chromiumos-overlay' % self._buildroot ] cbuildbot.RunCommand(['./cros_mark_as_stable', '--all', '--board=%s' % self._test_board, - '--overlays=%s' % ':'.join(overlays), - '--tracking_branch=cros/master', 'commit'], + '--tracking_branch="cros/master"', 'commit'], cwd='%s/src/scripts' % self._buildroot, enter_chroot=True) self.mox.ReplayAll() - overlays = [ '%s/src/third_party/chromiumos-overlay' % self._buildroot ] cbuildbot._UprevPackages(self._buildroot, self.tracking_branch, - self._revision_file, self._test_board, overlays) + self._revision_file, self._test_board) self.mox.VerifyAll() diff --git a/cros_mark_as_stable.py b/cros_mark_as_stable.py index 10578e3916..56a589c682 100755 --- a/cros_mark_as_stable.py +++ b/cros_mark_as_stable.py @@ -22,10 +22,10 @@ from cros_build_lib import Info, RunCommand, Warning, Die gflags.DEFINE_string('board', '', 'Board for which the package belongs.', short_name='b') gflags.DEFINE_string('overlays', '', - 'Colon-separated list of overlays to modify.', + 'Space separated list of overlays to modify.', short_name='o') gflags.DEFINE_string('packages', '', - 'Colon-separated list of packages to mark as stable.', + 'Space separated list of packages to mark as stable.', short_name='p') gflags.DEFINE_string('push_options', '', 'Options to use with git-cl push using push command.') @@ -245,10 +245,7 @@ def _SimpleRunCommand(command): """Runs a shell command and returns stdout back to caller.""" _Print(' + %s' % command) proc_handle = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) - stdout = proc_handle.communicate()[0] - retcode = proc_handle.wait() - assert retcode == 0, "Return code %s for command: %s" % (retcode, command) - return stdout + return proc_handle.communicate()[0] # ======================= End Global Helper Functions ======================== @@ -490,16 +487,11 @@ def main(argv): except gflags.FlagsError, e : _PrintUsageAndDie(str(e)) - package_list = gflags.FLAGS.packages.split(':') + package_list = gflags.FLAGS.packages.split() _CheckSaneArguments(package_list, command) if gflags.FLAGS.overlays: - overlays = {} - for path in gflags.FLAGS.overlays.split(':'): - if not os.path.exists(path): - Die('Cannot find overlay: %s' % path) - overlays[path] = [] + overlays = dict((path, []) for path in gflags.FLAGS.overlays.split()) else: - Warning('Missing --overlays argument') overlays = { '%s/private-overlays/chromeos-overlay' % gflags.FLAGS.srcroot: [], '%s/third_party/chromiumos-overlay' % gflags.FLAGS.srcroot: [] From d44bcf9a45f55f2d5c840b7afb702b61ee277ead Mon Sep 17 00:00:00 2001 From: Raja Aluri Date: Thu, 11 Nov 2010 17:46:53 -0800 Subject: [PATCH 29/38] Adding a script to generate au-geneate.zip file Change-Id: Ifc47ed28dc3efc0e7ebd018f6703b36913ffd39c BUG=8716 TEST=Ran the script inside the chroot to makesure it is generating the package. Review URL: http://codereview.chromium.org/4425004 --- common.sh | 4 +- generate_au_zip.py | 263 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 266 insertions(+), 1 deletion(-) create mode 100755 generate_au_zip.py diff --git a/common.sh b/common.sh index 502a8c6c10..b7d428dab5 100644 --- a/common.sh +++ b/common.sh @@ -104,7 +104,9 @@ DEFAULT_CHROOT_DIR=${CHROMEOS_CHROOT_DIR:-"$GCLIENT_ROOT/chroot"} DEFAULT_BUILD_ROOT=${CHROMEOS_BUILD_ROOT:-"$SRC_ROOT/build"} # Set up a global ALL_BOARDS value -ALL_BOARDS=$(cd $SRC_ROOT/overlays;ls -1d overlay-* 2>&-|sed 's,overlay-,,g') +if [ -d $SRC_ROOT/overlays ]; then + ALL_BOARDS=$(cd $SRC_ROOT/overlays;ls -1d overlay-* 2>&-|sed 's,overlay-,,g') +fi # Strip CR ALL_BOARDS=$(echo $ALL_BOARDS) # Set a default BOARD diff --git a/generate_au_zip.py b/generate_au_zip.py new file mode 100755 index 0000000000..f38e1e3aad --- /dev/null +++ b/generate_au_zip.py @@ -0,0 +1,263 @@ +#!/usr/bin/python + +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" + Script to generate a zip file of delta-generator and it's dependencies. +""" +import logging.handlers +import optparse +import os +import re +import shutil +import subprocess +import sys +import tempfile + +# GLOBALS + +logging_format = '%(asctime)s - %(filename)s - %(levelname)-8s: %(message)s' +date_format = '%Y/%m/%d %H:%M:%S' +logging.basicConfig(level=logging.INFO, format=logging_format, + datefmt=date_format) + +def CreateTempDir(): + """Creates a tempdir and returns the name of the tempdir.""" + temp_dir = tempfile.mkdtemp(suffix='au', prefix='tmp') + logging.info('Using tempdir = %s', temp_dir) + return temp_dir + + +def _SplitAndStrip(data): + """Prunes the ldd output, and return a list of needed library names + Example of data: + linux-vdso.so.1 => (0x00007ffffc96a000) + libbz2.so.1 => /lib/libbz2.so.1 (0x00007f3ff8782000) + libc.so.6 => /lib/libc.so.6 (0x00007f3ff83ff000) + /lib64/ld-linux-x86-64.so.2 (0x00007f3ff89b3000) + Args: + data: list of libraries from ldd output + Returns: + list of libararies that we should copy + + """ + return_list = [] + for line in data.split('\n'): + line = re.sub('.*not a dynamic executable.*', '', line) + line = re.sub('.* =>\s+', '', line) + line = re.sub('\(0x.*\)\s?', '', line) + line = line.strip() + if not len(line): + continue + logging.debug('MATCHED line = %s', line) + return_list.append(line) + + return return_list + + +def DepsToCopy(ldd_files, black_list): + """Returns a list of deps for a given dynamic executables list. + Args: + ldd_files: List of dynamic files that needs to have the deps evaluated + black_list: List of files that we should ignore + Returns: + library_list: List of files that are dependencies + """ + for file_name in ldd_files: + logging.info('Running ldd on %s', file_name) + cmd = ['/usr/bin/ldd', file_name] + stdout_data = '' + stderr_data = '' + + try: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + (stdout_data, stderr_data) = proc.communicate(input=None) + except subprocess.CalledProcessError, e: + logging.error('Command %s failed', cmd) + logging.error('error code %s', e.returncode) + logging.error('ouput %s', e.output) + raise + + library_list = [] + if not stdout_data: + return library_list + + logging.debug('ldd for %s = stdout = %s stderr =%s', file_name, + stdout_data, stderr_data) + + library_list = _SplitAndStrip(stdout_data) + + return _ExcludeBlacklist(library_list, black_list) + + +def CopyRequiredFiles(dest_files_root): + """Generates a list of files that are required for au-generator zip file + Args: + dest_files_root: location of the directory where we should copy the files + """ + if not dest_files_root: + logging.error('Invalid option passed for dest_files_root') + sys.exit(1) + # Files that need to go through ldd + ldd_files = ['/usr/bin/delta_generator', '/usr/bin/bsdiff', + '/usr/bin/bspatch', '/usr/bin/cgpt'] + # statically linked files and scripts etc., + static_files = ['~/trunk/src/scripts/common.sh', + '~/trunk/src/scripts/cros_generate_update_payload', + '~/trunk/src/scripts/chromeos-common.sh'] + # We need directories to be copied recursively to a destination within tempdir + recurse_dirs = {'~/trunk/src/scripts/lib/shflags': 'lib/shflags'} + + black_list = [ + 'linux-vdso.so', + 'libgcc_s.so', + 'libgthread-2.0.so', + 'libpthread.so', + 'librt.so', + 'libstdc', + 'libgcc_s.so', + 'libc.so', + 'ld-linux-x86-64', + 'libm.so', + 'libdl.so', + 'libresolv.so', + ] + + all_files = ldd_files + static_files + all_files = map(os.path.expanduser, all_files) + + for file_name in all_files: + if not os.path.isfile(file_name): + logging.error('file = %s does not exist', file_name) + sys.exit(1) + + logging.debug('Given files that need to be copied = %s' % '' .join(all_files)) + all_files += DepsToCopy(ldd_files=ldd_files,black_list=black_list) + for file_name in all_files: + logging.info('Copying file %s to %s', file_name, dest_files_root) + shutil.copy2(file_name, dest_files_root) + + for source_dir, target_dir in recurse_dirs.iteritems(): + logging.info('Processing directory %s', source_dir) + full_path = os.path.expanduser(source_dir) + if not os.path.isdir(full_path): + logging.error("Directory given for %s expanded to %s doens't exist.", + source_dir, full_path) + sys.exit(1) + dest = os.path.join(dest_files_root, target_dir) + logging.info('Copying directory %s to %s.', full_path, target_dir) + shutil.copytree(full_path, dest) + +def CleanUp(temp_dir): + """Cleans up the tempdir + Args: + temp_dir = name of the directory to cleanup + """ + if os.path.exists(temp_dir): + shutil.rmtree(temp_dir, ignore_errors=True) + logging.info('Removed tempdir = %s', temp_dir) + +def GenerateZipFile(base_name, root_dir): + """Returns true if able to generate zip file + Args: + base_name: name of the zip file + root_dir: location of the directory that we should zip + Returns: + True if successfully generates the zip file otherwise False + """ + logging.info('Generating zip file %s with contents from %s', base_name, + root_dir) + current_dir = os.getcwd() + os.chdir(root_dir) + try: + subprocess.Popen(['zip', '-r', '-9', base_name, '.'], + stdout=subprocess.PIPE).communicate()[0] + except OSError, e: + logging.error('Execution failed:%s', e.strerror) + return False + finally: + os.chdir(current_dir) + + return True + + +def _ExcludeBlacklist(library_list, black_list=[]): + """Deletes the set of files from black_list from the library_list + Args: + library_list: List of the library names to filter through black_list + black_list: List of the black listed names to filter + Returns: + Filtered library_list + """ + return_list = [] + pattern = re.compile(r'|'.join(black_list)) + + logging.debug('PATTERN: %s=', pattern) + + for library in library_list: + if pattern.search(library): + logging.debug('BLACK-LISTED = %s=', library) + continue + return_list.append(library) + + logging.debug('Returning return_list=%s=', return_list) + + return return_list + + +def CopyZipToFinalDestination(output_dir, zip_file_name): + """Copies the generated zip file to a final destination + Args: + output_dir: Directory where the file should be copied to + zip_file_name: name of the zip file that should be copied + Returns: + True on Success False on Failure + """ + if not os.path.isfile(zip_file_name): + logging.error("Zip file %s doesn't exist. Returning False", zip_file_name) + return False + + if not os.path.isdir(output_dir): + logging.debug('Creating %s', output_dir) + os.makedirs(output_dir) + logging.info('Copying %s to %s', zip_file_name, output_dir) + shutil.copy2(zip_file_name, output_dir) + return True + + +def main(): + """Main function to start the script""" + parser = optparse.OptionParser() + + parser.add_option( '-d', '--debug', dest='debug', action='store_true', + default=False, help='Verbose Default: False',) + parser.add_option('-o', '--output-dir', dest='output_dir', + default='/tmp/au-generator', + help='Specify the output location for copying the zipfile') + parser.add_option('-z', '--zip-name', dest='zip_name', + default='au-generator.zip', help='Name of the zip file') + parser.add_option('-k', '--keep-temp', dest='keep_temp', default=False, + action='store_true', help='Keep the temp files...',) + + (options, args) = parser.parse_args() + if options.debug: + logging.setLevel(logging.DEBUG) + + logging.debug('Options are %s ', options) + + temp_dir = CreateTempDir() + dest_files_root = os.path.join(temp_dir, 'au-generator') + os.makedirs(dest_files_root) + CopyRequiredFiles(dest_files_root=dest_files_root) + zip_file_name = os.path.join(temp_dir, options.zip_name) + GenerateZipFile(zip_file_name, dest_files_root) + CopyZipToFinalDestination(options.output_dir, zip_file_name) + + if not options.keep_temp: + CleanUp(temp_dir) + +if __name__ == '__main__': + main() From 4ca4b147e5d8eb2453749f37569041b8a24cb1b0 Mon Sep 17 00:00:00 2001 From: Raja Aluri Date: Thu, 11 Nov 2010 18:26:41 -0800 Subject: [PATCH 30/38] Created branch 0.9.112.B. Update CHROMEOS_VERSION_BRANCH=113 Change-Id: Ia23fb80872a902750d113a6bbe65595ffbd6d720 --- chromeos_version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chromeos_version.sh b/chromeos_version.sh index 77d4e0158f..fcebafb516 100755 --- a/chromeos_version.sh +++ b/chromeos_version.sh @@ -26,7 +26,7 @@ export CHROMEOS_VERSION_MINOR=9 # Increment by 2 in trunk after making a release branch. # Does not reset on a major/minor change (always increases). # (Trunk is always odd; branches are always even). -export CHROMEOS_VERSION_BRANCH=111 +export CHROMEOS_VERSION_BRANCH=113 # Patch number. # Increment by 1 each release on a branch. From 51484a96e9867957507f7bfecd31fabc8ce2e190 Mon Sep 17 00:00:00 2001 From: Hung-Te Lin Date: Sat, 13 Nov 2010 02:43:14 +0800 Subject: [PATCH 31/38] crosutils: refine memento image / factory package creation This CL improves creation of memento / factory image payload by: - verbose progress report - allowing to compress by pigz, the parallel version of gzip - prevents unpacking entire image if partition tools (cgpt/parted) is available. BUG=chromium-os:6536,chromium-os:5208 TEST=Verified executing "time ./make_factory_package.sh ..." for ToT factory bundle: - before this CL (memento gzip param is not -9): 3m53.126s - after this CL, without pigz, with cgpt: 2m34.897s - after this CL, with pigz+cgpt, memento_gz=-9: 0m45.603s - after this CL, with pigz, without cgpt/parted, memento_gz=-9: 1m49.748s Also verified such bundle can be installed on a real netbook device. Change-Id: Ie182844ea5482d6d321b9549fa584377edf7dfe3 Review URL: http://codereview.chromium.org/4824003 --- image_common.sh | 106 ++++++++++++++++++++++++++++++++++++++++ make_factory_package.sh | 79 ++++++++++++++++++++++++------ mk_memento_images.sh | 98 +++++++++++++++++++++++++------------ 3 files changed, 237 insertions(+), 46 deletions(-) create mode 100644 image_common.sh diff --git a/image_common.sh b/image_common.sh new file mode 100644 index 0000000000..0ad9d0bbcd --- /dev/null +++ b/image_common.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +# Copyright (c) 2009 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# This script contains common utility function to deal with disk images, +# especially for being redistributed into platforms without complete Chromium OS +# developing environment. + +# Check if given command is available in current system +has_command() { + type "$1" >/dev/null 2>&1 +} + +err_die() { + echo "ERROR: $@" >&2 + exit 1 +} + +# Finds the best gzip compressor and invoke it. +gzip_compress() { + if has_command pigz; then + # echo " ** Using parallel gzip **" >&2 + # Tested with -b 32, 64, 128(default), 256, 1024, 16384, and -b 32 (max + # window size of Deflate) seems to be the best in output size. + pigz -b 32 "$@" + else + gzip "$@" + fi +} + +# Finds if current system has tools for part_* commands +has_part_tools() { + has_command cgpt || has_command parted +} + +# Finds the best partition tool and print partition offset +part_offset() { + local file="$1" + local partno="$2" + + if has_command cgpt; then + cgpt show -b -i "$partno" "$file" + elif has_command parted; then + parted -m "$file" unit s print | + grep "^$partno:" | cut -d ':' -f 2 | sed 's/s$//' + else + exit 1 + fi +} + +# Finds the best partition tool and print partition size +part_size() { + local file="$1" + local partno="$2" + + if has_command cgpt; then + cgpt show -s -i "$partno" "$file" + elif has_command parted; then + parted -m "$file" unit s print | + grep "^$partno:" | cut -d ':' -f 4 | sed 's/s$//' + else + exit 1 + fi +} + +# Dumps a file by given offset and size (in sectors) +dump_partial_file() { + local file="$1" + local offset="$2" + local sectors="$3" + local bs=512 + + # Try to use larger buffer if offset/size can be re-aligned. + # 2M / 512 = 4096 + local buffer_ratio=4096 + if [ $((offset % buffer_ratio)) -eq 0 -a \ + $((sectors % buffer_ratio)) -eq 0 ]; then + offset=$((offset / buffer_ratio)) + sectors=$((sectors / buffer_ratio)) + bs=$((bs * buffer_ratio)) + fi + + if has_command pv; then + dd if="$file" bs=$bs skip="$offset" count="$sectors" \ + oflag=sync status=noxfer 2>/dev/null | + pv -ptreb -B 4m -s $((sectors * $bs)) + else + dd if="$file" bs=$bs skip="$offset" count="$sectors" \ + oflag=sync status=noxfer 2>/dev/null + fi +} + +# Dumps a specific partition from given image file +dump_partition() { + local file="$1" + local part_num="$2" + local offset="$(part_offset "$file" "$part_num")" || + err_die "failed to dump partition #$part_num from: $file" + local size="$(part_size "$file" "$part_num")" || + err_die "failed to dump partition #$part_num from: $file" + + dump_partial_file "$file" "$offset" "$size" +} + diff --git a/make_factory_package.sh b/make_factory_package.sh index 9a44f71dd2..a251980a12 100755 --- a/make_factory_package.sh +++ b/make_factory_package.sh @@ -18,6 +18,9 @@ # Load functions and constants for chromeos-install . "$(dirname "$0")/chromeos-common.sh" +# Load functions designed for image processing +. "$(dirname "$0")/image_common.sh" + get_default_board # Flags @@ -95,6 +98,50 @@ prepare_dir() { rm -rf state.gz } +compress_and_hash_memento_image() { + local input_file="$1" + + if has_part_tools; then + sudo "${SCRIPTS_DIR}/mk_memento_images.sh" "$input_file" 2 3 | + grep hash | + awk '{print $4}' + else + sudo "${SCRIPTS_DIR}/mk_memento_images.sh" part_2 part_3 | + grep hash | + awk '{print $4}' + fi +} + +compress_and_hash_file() { + local input_file="$1" + local output_file="$2" + + if [ -z "$input_file" ]; then + # Runs as a pipe processor + gzip_compress -c -9 | + tee "$output_file" | + openssl sha1 -binary | + openssl base64 + else + gzip_compress -c -9 "$input_file" | + tee "$output_file" | + openssl sha1 -binary | + openssl base64 + fi +} + +compress_and_hash_partition() { + local input_file="$1" + local part_num="$2" + local output_file="$3" + + if has_part_tools; then + dump_partition "$input_file" "$part_num" | + compress_and_hash_file "" "$output_file" + else + compress_and_hash_file "part_$part_num" "$output_file" + fi +} # Clean up stale config and data files. prepare_omaha @@ -108,21 +155,25 @@ echo "Output omaha config to ${OMAHA_DIR}/miniomaha.conf" prepare_dir -sudo ./unpack_partitions.sh ${RELEASE_IMAGE} &> /dev/null -release_hash=`sudo ${SCRIPTS_DIR}/mk_memento_images.sh part_2 part_3 \ - | grep hash | awk '{print $4}'` +if ! has_part_tools; then + #TODO(hungte) we can still avoid running unpack_partitions.sh + # by $(cat unpack_partitions.sh | grep Label | sed "s/#//" | grep ${name}" | + # awk '{ print $1}') to fetch offset/size. + echo "Unpacking image ${RELEASE_IMAGE} ..." >&2 + sudo ./unpack_partitions.sh "${RELEASE_IMAGE}" 2>/dev/null +fi + +release_hash="$(compress_and_hash_memento_image "${RELEASE_IMAGE}")" sudo chmod a+rw update.gz mv update.gz rootfs-release.gz mv rootfs-release.gz ${OMAHA_DATA_DIR} echo "release: ${release_hash}" -cat part_8 | gzip -9 > oem.gz -oem_hash=`cat oem.gz | openssl sha1 -binary | openssl base64` +oem_hash="$(compress_and_hash_partition "${RELEASE_IMAGE}" 8 "oem.gz")" mv oem.gz ${OMAHA_DATA_DIR} echo "oem: ${oem_hash}" -cat part_12 | gzip -9 > efi.gz -efi_hash=`cat efi.gz | openssl sha1 -binary | openssl base64` +efi_hash="$(compress_and_hash_partition "${RELEASE_IMAGE}" 12 "efi.gz")" mv efi.gz ${OMAHA_DATA_DIR} echo "efi: ${efi_hash}" @@ -132,17 +183,18 @@ popd > /dev/null pushd ${FACTORY_DIR} > /dev/null prepare_dir +if ! has_part_tools; then + echo "Unpacking image ${FACTORY_IMAGE} ..." >&2 + sudo ./unpack_partitions.sh "${FACTORY_IMAGE}" 2>/dev/null +fi -sudo ./unpack_partitions.sh ${FACTORY_IMAGE} &> /dev/null -test_hash=`sudo ${SCRIPTS_DIR}//mk_memento_images.sh part_2 part_3 \ - | grep hash | awk '{print $4}'` +test_hash="$(compress_and_hash_memento_image "${FACTORY_IMAGE}")" sudo chmod a+rw update.gz mv update.gz rootfs-test.gz mv rootfs-test.gz ${OMAHA_DATA_DIR} echo "test: ${test_hash}" -cat part_1 | gzip -9 > state.gz -state_hash=`cat state.gz | openssl sha1 -binary | openssl base64` +state_hash="$(compress_and_hash_partition "${FACTORY_IMAGE}" 1 "state.gz")" mv state.gz ${OMAHA_DATA_DIR} echo "state: ${state_hash}" @@ -155,8 +207,7 @@ if [ ! -z ${FLAGS_firmware_updater} ] ; then exit 1 fi - cat $SHELLBALL | gzip -9 > firmware.gz - firmware_hash=`cat firmware.gz | openssl sha1 -binary | openssl base64` + firmware_hash="$(compress_and_hash_file "$SHELLBALL" "firmware.gz")" mv firmware.gz ${OMAHA_DATA_DIR} echo "firmware: ${firmware_hash}" fi diff --git a/mk_memento_images.sh b/mk_memento_images.sh index de921bb579..b5d786135c 100755 --- a/mk_memento_images.sh +++ b/mk_memento_images.sh @@ -10,25 +10,64 @@ set -e +LIB_IMAGE_COMMON="$(dirname "$0")/image_common.sh" +if ! . "$LIB_IMAGE_COMMON"; then + echo "Missing required library: $LIB_IMAGE_COMMON. Cannot continue." + exit 1 +fi + if [ -z "$2" -o -z "$1" ]; then echo "usage: $0 path/to/kernel_partition_img path/to/rootfs_partition_img" + echo " or $0 path/to/chromiumos_img kern_part_no rootfs_part_no" exit 1 fi if [ "$CROS_GENERATE_UPDATE_PAYLOAD_CALLED" != "1" ]; then echo "WARNING:" - echo "This script should only be called from cros_generate_update_payload" - echo "Please run that script with --help to see how to use it." + echo " This script should only be called from cros_generate_update_payload" + echo " Please run that script with --help to see how to use it." +fi + +if ! has_command pigz; then + (echo "WARNING:" + echo " Your system does not have pigz (parallel gzip) installed." + echo " COMPRESSING WILL BE VERY SLOW. It is recommended to install pigz" + if has_command apt-get; then + echo " by 'sudo apt-get install pigz'." + elif has_command emerge; then + echo " by 'sudo emerge pigz'." + fi) >&2 fi if [ $(whoami) = "root" ]; then echo "running $0 as root which is unneccessary" fi -KPART="$1" -ROOT_PART="$2" - -KPART_SIZE=$(stat -c%s "$KPART") +# Determine the offset size, and file name of parameters +if [ -z "$3" ]; then + # kernnel_img rootfs_img + KPART="$1" + ROOT_PART="$2" + KPART_SIZE=$(stat -c%s "$KPART") + ROOT_PART_SIZE=$(stat -c%s "$ROOT_PART") + KPART_OFFSET=0 + KPART_SECTORS=$((KPART_SIZE / 512)) + ROOT_OFFSET=0 + ROOT_SECTORS=$((ROOT_PART_SIZE / 512)) +else + # chromiumos_img kern_part_no rootfs_part_no + KPART="$1" + ROOT_PART="$1" + KPART_OFFSET="$(part_offset "$KPART" "$2")" || + err_die "cannot retieve kernel partition offset" + KPART_SECTORS="$(part_size "$KPART" "$2")" || + err_die "cannot retieve kernel partition size" + ROOT_OFFSET="$(part_offset "$ROOT_PART" "$3")" || + err_die "cannot retieve root partition offset" + ROOT_SECTORS="$(part_size "$ROOT_PART" "$3")" || + err_die "cannot retieve root partition size" + KPART_SIZE=$((KPART_SECTORS * 512)) +fi # Sanity check size. if [ "$KPART_SIZE" -gt $((16 * 1024 * 1024)) ]; then @@ -38,34 +77,31 @@ if [ "$KPART_SIZE" -gt $((16 * 1024 * 1024)) ]; then fi FINAL_OUT_FILE=$(dirname "$1")/update.gz -UNCOMPRESSED_OUT_FILE="$FINAL_OUT_FILE.uncompressed" -# First, write size of kernel partition in big endian as uint64 to out file -# printf converts it to a number like 00000000003d0900. sed converts it to: -# \\x00\\x00\\x00\\x00\\x00\\x3d\\x09\\x00, then xargs converts it to binary -# with echo. -printf %016x "$KPART_SIZE" | \ - sed 's/\([0-9a-f][0-9a-f]\)/\\\\x\1/g' | \ - xargs echo -ne > "$UNCOMPRESSED_OUT_FILE" +# Update payload format: +# [kernel_size: big-endian uint64][kernel_blob][rootfs_blob] -# Next, write kernel partition to the out file -cat "$KPART" >> "$UNCOMPRESSED_OUT_FILE" +# Prepare kernel_size by using printf as a number like 00000000003d0900, then +# sed to convert as: \x00\x00\x00\x00\x00\x3d\x09\x00, finally echo -e to +# convert into binary. +KPART_SIZE_SIGNATURE="$(printf "%016x" "$KPART_SIZE" | + sed 's/\([0-9a-f][0-9a-f]\)/\\x\1/g')" -# Sanity check size of output file now -if [ $(stat -c%s "$UNCOMPRESSED_OUT_FILE") -ne $((8 + $KPART_SIZE)) ]; then - echo "Kernel partition changed size during image generation. Aborting." - exit 1 -fi +# Build the blob! +CS_AND_RET_CODES="$( + (echo -en "$KPART_SIZE_SIGNATURE" + echo "Compressing kernel..." >&2 + dump_partial_file "$KPART" "$KPART_OFFSET" "$KPART_SECTORS" + echo "Compressing rootfs..." >&2 + dump_partial_file "$ROOT_PART" "$ROOT_OFFSET" "$ROOT_SECTORS") | + gzip_compress -9 -c | + tee "$FINAL_OUT_FILE" | + openssl sha1 -binary | + openssl base64 | + tr '\n' ' ' + echo ${PIPESTATUS[*]})" -# Put rootfs into the out file -cat "$ROOT_PART" >> "$UNCOMPRESSED_OUT_FILE" - -# compress and hash -CS_AND_RET_CODES=$(gzip -c "$UNCOMPRESSED_OUT_FILE" | \ - tee "$FINAL_OUT_FILE" | openssl sha1 -binary | \ - openssl base64 | tr '\n' ' '; \ - echo ${PIPESTATUS[*]}) -EXPECTED_RET_CODES="0 0 0 0 0" +EXPECTED_RET_CODES="0 0 0 0 0 0" set -- $CS_AND_RET_CODES CALC_CS="$1" shift @@ -75,6 +111,4 @@ if [ "$RET_CODES" != "$EXPECTED_RET_CODES" ]; then exit 1 fi -rm "$UNCOMPRESSED_OUT_FILE" - echo Success. hash is "$CALC_CS" From c60e5677997769ef957d2f9f387f00c599647fc7 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Fri, 12 Nov 2010 10:54:25 -0800 Subject: [PATCH 32/38] Convert rootfs and stateful mountpts to absolute paths. Change-Id: Ib9b6b9ba7787fba337e71e4968a5c152b5a21e14 BUG=chromiumos:8885 TEST=Ran with default options and setting --rootfs_mountpt and stateful_mountpt to dirs in my current workig directory ... saw cleanup correctly on unmount Review URL: http://codereview.chromium.org/4853001 --- mount_gpt_image.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mount_gpt_image.sh b/mount_gpt_image.sh index 63dba3d8b6..3156c1586e 100755 --- a/mount_gpt_image.sh +++ b/mount_gpt_image.sh @@ -139,8 +139,10 @@ if [ ${FLAGS_most_recent} -eq ${FLAGS_TRUE} ] ; then FLAGS_from="$(./get_latest_image.sh --board="${FLAGS_board}")" fi -# Turn path into an absolute path. +# Turn paths into absolute paths. FLAGS_from=`eval readlink -f ${FLAGS_from}` +FLAGS_rootfs_mountpt=`eval readlink -f ${FLAGS_rootfs_mountpt}` +FLAGS_stateful_mountpt=`eval readlink -f ${FLAGS_stateful_mountpt}` # Perform desired operation. if [ ${FLAGS_unmount} -eq ${FLAGS_TRUE} ] ; then From bcc60b625d3115f37fff3d3c2e3c526a285db4e2 Mon Sep 17 00:00:00 2001 From: Hung-Te Lin Date: Sat, 13 Nov 2010 05:36:20 +0800 Subject: [PATCH 33/38] crosutils: move image_common.sh to lib/cros_image_common.sh The team is moving utility/library scripts into 'lib' folder. image_common.sh should follow this policy. Also refined the parameter check of mk_memento_images.sh BUG=chromium-os:5208 TEST=./make_factory_package.sh --factory PATH_TO_FACTORY --release PATH_TO_RELEASE; # factory bundle created successfully ./mk_memento_images.sh PATH_TO_PART2 PATH_TO_PART3 # update.gz created successfully ./mk_memento_images.sh PATH_TO_IMGE 2 3 # update.gz created successfully Change-Id: I3afecf05da2832986723f28b595045d0540ea9e9 Review URL: http://codereview.chromium.org/4825004 --- image_common.sh => lib/cros_image_common.sh | 0 make_factory_package.sh | 3 ++- mk_memento_images.sh | 8 ++++---- 3 files changed, 6 insertions(+), 5 deletions(-) rename image_common.sh => lib/cros_image_common.sh (100%) diff --git a/image_common.sh b/lib/cros_image_common.sh similarity index 100% rename from image_common.sh rename to lib/cros_image_common.sh diff --git a/make_factory_package.sh b/make_factory_package.sh index a251980a12..a62cfe2e3c 100755 --- a/make_factory_package.sh +++ b/make_factory_package.sh @@ -19,7 +19,8 @@ . "$(dirname "$0")/chromeos-common.sh" # Load functions designed for image processing -. "$(dirname "$0")/image_common.sh" +. "$(dirname "$0")/lib/cros_image_common.sh" || + die "Cannot load required library: lib/cros_image_common.sh; Abort." get_default_board diff --git a/mk_memento_images.sh b/mk_memento_images.sh index b5d786135c..415768fb4b 100755 --- a/mk_memento_images.sh +++ b/mk_memento_images.sh @@ -10,13 +10,13 @@ set -e -LIB_IMAGE_COMMON="$(dirname "$0")/image_common.sh" -if ! . "$LIB_IMAGE_COMMON"; then - echo "Missing required library: $LIB_IMAGE_COMMON. Cannot continue." +# Load functions designed for image processing +if ! . "$(dirname "$0")/lib/cros_image_common.sh"; then + echo "ERROR: Cannot load required library: lib/cros_image_common.sh; Abort." exit 1 fi -if [ -z "$2" -o -z "$1" ]; then +if [ -z "$2" -o -z "$1" ] || [ "${#@}" -ne 2 -a "${#@}" -ne 3 ]; then echo "usage: $0 path/to/kernel_partition_img path/to/rootfs_partition_img" echo " or $0 path/to/chromiumos_img kern_part_no rootfs_part_no" exit 1 From 795bd30f069bc64da6f68d8c3e23ed15fa62e072 Mon Sep 17 00:00:00 2001 From: David James Date: Fri, 12 Nov 2010 14:33:14 -0800 Subject: [PATCH 34/38] Add more error checking to preflight queue. What's new? - cros_mark_as_stable now exits with errors if directories are specified that don't exist. - cbuildbot.py always explicitly specifies overlay directories so cros_mark_as_stable can rely on them existing. - Package names and paths are now separated with colons instead of spaces, so as to allow for us using the same syntax with enter_chroot.sh as we use without the same script. (enter_chroot.sh mucks with command-lines that contain spaces or quotes.) - cbuildbot.py now ensures its build path is a absolute path. This ensures we don't kill the wrong processes, if, for instance, the buildpath is '../..' - All buildbots now explicitly specify what overlays they want to rev. Public buildbots only rev public ebuilds and private buildbots now only rev private ebuilds. BUG=chromium-os:8647 TEST=Ran unit tests. Manually marked packages as stable. Ran cbuildbot.py test run. Change-Id: I1df6d428973d91329c4f5159e2886889a3ebb7c7 Review URL: http://codereview.chromium.org/4442001 --- bin/cbuildbot.py | 93 +++++++++++++++++++++++++-------------- bin/cbuildbot_config.py | 39 ++++------------ bin/cbuildbot_unittest.py | 17 +++++-- cros_mark_as_stable.py | 31 +++++++++---- 4 files changed, 104 insertions(+), 76 deletions(-) diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index 372ddcaf5f..58d5c3e2c3 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -18,7 +18,8 @@ import cbuildbot_comm from cbuildbot_config import config sys.path.append(os.path.join(os.path.dirname(__file__), '../lib')) -from cros_build_lib import Die, Info, RunCommand, Warning +from cros_build_lib import (Die, Info, ReinterpretPathForChroot, RunCommand, + Warning) _DEFAULT_RETRIES = 3 ARCHIVE_BASE = '/var/www/archive' @@ -181,37 +182,38 @@ def _ParseRevisionString(revision_string, repo_dictionary): return revisions.items() -def _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board): +def _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board, + overlays): """Uprevs based on revision list.""" if not revision_list: Info('No packages found to uprev') return - package_str = '' + packages = [] for package, revision in revision_list: - package_str += package + ' ' + assert ':' not in package, 'Invalid package name: %s' % package + packages.append(package) - package_str = package_str.strip() + chroot_overlays = [ReinterpretPathForChroot(path) for path in overlays] cwd = os.path.join(buildroot, 'src', 'scripts') - # TODO(davidjames): --foo="bar baz" only works here because we're using - # enter_chroot. RunCommand(['./cros_mark_as_stable', '--board=%s' % board, - '--tracking_branch="%s"' % tracking_branch, - '--packages="%s"' % package_str, + '--tracking_branch=%s' % tracking_branch, + '--overlays=%s' % ':'.join(chroot_overlays), + '--packages=%s' % ':'.join(packages), 'commit'], - cwd=cwd, enter_chroot=True) + cwd=cwd, enter_chroot=True) -def _UprevAllPackages(buildroot, tracking_branch, board): +def _UprevAllPackages(buildroot, tracking_branch, board, overlays): """Uprevs all packages that have been updated since last uprev.""" cwd = os.path.join(buildroot, 'src', 'scripts') - # TODO(davidjames): --foo="bar baz" only works here because we're using - # enter_chroot. + chroot_overlays = [ReinterpretPathForChroot(path) for path in overlays] RunCommand(['./cros_mark_as_stable', '--all', '--board=%s' % board, - '--tracking_branch="%s"' % tracking_branch, 'commit'], + '--overlays=%s' % ':'.join(chroot_overlays), + '--tracking_branch=%s' % tracking_branch, 'commit'], cwd=cwd, enter_chroot=True) @@ -228,12 +230,13 @@ def _GetVMConstants(buildroot): return (vdisk_size.strip(), statefulfs_size.strip()) -def _GitCleanup(buildroot, board, tracking_branch): +def _GitCleanup(buildroot, board, tracking_branch, overlays): """Clean up git branch after previous uprev attempt.""" cwd = os.path.join(buildroot, 'src', 'scripts') if os.path.exists(cwd): RunCommand(['./cros_mark_as_stable', '--srcroot=..', '--board=%s' % board, + '--overlays=%s' % ':'.join(overlays), '--tracking_branch=%s' % tracking_branch, 'clean'], cwd=cwd, error_ok=True) @@ -257,9 +260,9 @@ def _WipeOldOutput(buildroot): # =========================== Main Commands =================================== -def _PreFlightRinse(buildroot, board, tracking_branch): +def _PreFlightRinse(buildroot, board, tracking_branch, overlays): """Cleans up any leftover state from previous runs.""" - _GitCleanup(buildroot, board, tracking_branch) + _GitCleanup(buildroot, board, tracking_branch, overlays) _CleanUpMountPoints(buildroot) RunCommand(['sudo', 'killall', 'kvm'], error_ok=True) @@ -347,7 +350,7 @@ def _RunSmokeSuite(buildroot, results_dir): ], cwd=cwd, error_ok=False) -def _UprevPackages(buildroot, tracking_branch, revisionfile, board): +def _UprevPackages(buildroot, tracking_branch, revisionfile, board, overlays): """Uprevs a package based on given revisionfile. If revisionfile is set to None or does not resolve to an actual file, this @@ -376,26 +379,19 @@ def _UprevPackages(buildroot, tracking_branch, revisionfile, board): # print >> sys.stderr, 'CBUILDBOT Revision list found %s' % revisions # revision_list = _ParseRevisionString(revisions, # _CreateRepoDictionary(buildroot, board)) - # _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board) + # _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board, + # overlays) #else: Info('CBUILDBOT Revving all') - _UprevAllPackages(buildroot, tracking_branch, board) + _UprevAllPackages(buildroot, tracking_branch, board, overlays) def _UprevPush(buildroot, tracking_branch, board, overlays): """Pushes uprev changes to the main line.""" cwd = os.path.join(buildroot, 'src', 'scripts') - public_overlay = '%s/src/third_party/chromiumos-overlay' % buildroot - private_overlay = '%s/src/private-overlays/chromeos-overlay' % buildroot - if overlays == 'private': - overlays = [private_overlay] - elif overlays == 'public': - overlays = [public_overlay] - else: - overlays = [public_overlay, private_overlay] RunCommand(['./cros_mark_as_stable', '--srcroot=..', '--board=%s' % board, - '--overlays=%s' % " ".join(overlays), + '--overlays=%s' % ':'.join(overlays), '--tracking_branch=%s' % tracking_branch, '--push_options=--bypass-hooks -f', 'push'], cwd=cwd) @@ -467,6 +463,34 @@ def _GetConfig(config_name): return buildconfig +def _ResolveOverlays(buildroot, overlays): + """Return the list of overlays to use for a given buildbot. + + Args: + buildroot: The root directory where the build occurs. Must be an absolute + path. + overlays: A string describing which overlays you want. + 'private': Just the private overlay. + 'public': Just the public overlay. + 'both': Both the public and private overlays. + """ + public_overlay = '%s/src/third_party/chromiumos-overlay' % buildroot + private_overlay = '%s/src/private-overlays/chromeos-overlay' % buildroot + if overlays == 'private': + paths = [private_overlay] + elif overlays == 'public': + paths = [public_overlay] + elif overlays == 'both': + paths = [public_overlay, private_overlay] + else: + Die('Incorrect overlay configuration: %s' % overlays) + for path in paths: + assert ':' not in path, 'Overlay must not contain colons: %s' % path + if not os.path.isdir(path): + Die('Missing overlay: %s' % path) + return paths + + def main(): # Parse options usage = "usage: %prog [options] cbuildbot_config" @@ -491,7 +515,7 @@ def main(): (options, args) = parser.parse_args() - buildroot = options.buildroot + buildroot = os.path.abspath(options.buildroot) revisionfile = options.revisionfile tracking_branch = options.tracking_branch @@ -502,8 +526,11 @@ def main(): parser.print_usage() sys.exit(1) + # Calculate list of overlay directories. + overlays = _ResolveOverlays(buildroot, buildconfig['overlays']) + try: - _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch) + _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays) if options.clobber or not os.path.isdir(buildroot): _FullCheckout(buildroot, tracking_branch, url=options.url) else: @@ -519,7 +546,7 @@ def main(): if buildconfig['uprev']: _UprevPackages(buildroot, tracking_branch, revisionfile, - board=buildconfig['board']) + buildconfig['board'], overlays) _EnableLocalAccount(buildroot) _Build(buildroot) @@ -545,7 +572,7 @@ def main(): # Master bot needs to check if the other slaves completed. if cbuildbot_comm.HaveSlavesCompleted(config): _UprevPush(buildroot, tracking_branch, buildconfig['board'], - buildconfig['overlays']) + overlays) else: Die('CBUILDBOT - One of the slaves has failed!!!') diff --git a/bin/cbuildbot_config.py b/bin/cbuildbot_config.py index 831ef06bd1..718cd36a70 100644 --- a/bin/cbuildbot_config.py +++ b/bin/cbuildbot_config.py @@ -19,9 +19,9 @@ hostname -- Needed for 'important' slaves. The hostname of the bot. Should match hostname in slaves.cfg in buildbot checkout. unittests -- Runs unittests for packages. smoke_bvt -- Runs the test smoke suite in a qemu-based VM using KVM. -overlays -- If this bot is a master bot, select what overlays to push changes - to. This can be 'public', 'private', or 'both'. There should only - be one bot pushing changes to each overlay. +overlays -- Select what overlays to look at. This can be 'public', 'private' + or 'both'. There should only be one master bot pushing changes to + each overlay per branch. """ @@ -33,6 +33,7 @@ config['default'] = { 'important' : False, 'unittests' : False, 'smoke_bvt' : False, + 'overlays': 'public', } config['x86-generic-pre-flight-queue'] = { 'board' : 'x86-generic', @@ -69,6 +70,7 @@ config['x86_agz_bin'] = { 'important' : False, 'unittests' : True, 'smoke_bvt' : True, + 'overlays': 'private', } config['x86_dogfood_bin'] = { 'board' : 'x86-dogfood', @@ -77,52 +79,29 @@ config['x86_dogfood_bin'] = { 'important' : False, 'unittests' : True, 'smoke_bvt' : True, + 'overlays': 'private', } config['x86_pineview_bin'] = { 'board' : 'x86-pineview', 'uprev' : True, 'master' : False, 'important' : False, - 'hostname' : 'codf200.jail', 'unittests': True, + 'overlays': 'public', } config['arm_tegra2_bin'] = { 'board' : 'tegra2', 'uprev' : True, 'master' : False, 'important' : False, - 'hostname' : 'codg172.jail', - 'unittests' : False, -} -config['arm_voguev210_bin'] = { - 'board' : 'voguev210', - 'uprev' : True, - 'master' : False, - 'important' : False, - 'hostname' : 'codf196.jail', - 'unittests' : False, -} -config['arm_beagleboard_bin'] = { - 'board' : 'beagleboard', - 'master' : False, - 'uprev' : True, - 'important' : False, - 'hostname' : 'codf202.jail', - 'unittests' : False, -} -config['arm_st1q_bin'] = { - 'board' : 'st1q', - 'uprev' : True, - 'master' : False, - 'important' : False, - 'hostname' : 'codg158.jail', 'unittests' : False, + 'overlays': 'public', } config['arm_generic_bin'] = { 'board' : 'arm-generic', 'uprev' : True, 'master' : False, 'important' : False, - 'hostname' : 'codg175.jail', 'unittests' : False, + 'overlays': 'public', } diff --git a/bin/cbuildbot_unittest.py b/bin/cbuildbot_unittest.py index bbbc56c6f7..979cda55d4 100755 --- a/bin/cbuildbot_unittest.py +++ b/bin/cbuildbot_unittest.py @@ -16,6 +16,7 @@ import unittest # Fixes circular dependency error. import cbuildbot_comm import cbuildbot +from cros_build_lib import ReinterpretPathForChroot class CBuildBotTest(mox.MoxTestBase): @@ -42,6 +43,10 @@ class CBuildBotTest(mox.MoxTestBase): ['dev-util/perf', '12345test'], ['chromos-base/libcros', '12345test'] ] + self._overlays = ['%s/src/third_party/chromiumos-overlay' % self._buildroot] + self._chroot_overlays = [ + ReinterpretPathForChroot(p) for p in self._overlays + ] def testParseRevisionString(self): """Test whether _ParseRevisionString parses string correctly.""" @@ -168,13 +173,15 @@ class CBuildBotTest(mox.MoxTestBase): cbuildbot.RunCommand(['./cros_mark_as_stable', '--all', '--board=%s' % self._test_board, - '--tracking_branch="cros/master"', 'commit'], + '--overlays=%s' % ':'.join(self._chroot_overlays), + '--tracking_branch=cros/master', 'commit'], cwd='%s/src/scripts' % self._buildroot, enter_chroot=True) self.mox.ReplayAll() cbuildbot._UprevPackages(self._buildroot, self.tracking_branch, - self._revision_file, self._test_board) + self._revision_file, self._test_board, + self._overlays) self.mox.VerifyAll() def testUprevAllPackages(self): @@ -189,13 +196,15 @@ class CBuildBotTest(mox.MoxTestBase): cbuildbot.RunCommand(['./cros_mark_as_stable', '--all', '--board=%s' % self._test_board, - '--tracking_branch="cros/master"', 'commit'], + '--overlays=%s' % ':'.join(self._chroot_overlays), + '--tracking_branch=cros/master', 'commit'], cwd='%s/src/scripts' % self._buildroot, enter_chroot=True) self.mox.ReplayAll() cbuildbot._UprevPackages(self._buildroot, self.tracking_branch, - self._revision_file, self._test_board) + self._revision_file, self._test_board, + self._overlays) self.mox.VerifyAll() diff --git a/cros_mark_as_stable.py b/cros_mark_as_stable.py index 56a589c682..a6888439ed 100755 --- a/cros_mark_as_stable.py +++ b/cros_mark_as_stable.py @@ -22,10 +22,10 @@ from cros_build_lib import Info, RunCommand, Warning, Die gflags.DEFINE_string('board', '', 'Board for which the package belongs.', short_name='b') gflags.DEFINE_string('overlays', '', - 'Space separated list of overlays to modify.', + 'Colon-separated list of overlays to modify.', short_name='o') gflags.DEFINE_string('packages', '', - 'Space separated list of packages to mark as stable.', + 'Colon-separated list of packages to mark as stable.', short_name='p') gflags.DEFINE_string('push_options', '', 'Options to use with git-cl push using push command.') @@ -245,7 +245,11 @@ def _SimpleRunCommand(command): """Runs a shell command and returns stdout back to caller.""" _Print(' + %s' % command) proc_handle = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) - return proc_handle.communicate()[0] + stdout = proc_handle.communicate()[0] + retcode = proc_handle.wait() + if retcode != 0: + raise subprocess.CalledProcessError(retcode, command, output=stdout) + return stdout # ======================= End Global Helper Functions ======================== @@ -323,7 +327,7 @@ class _EBuild(object): # Grab and evaluate CROS_WORKON variables from this ebuild. unstable_ebuild = '%s-9999.ebuild' % self.ebuild_path_no_version - cmd = ('CROS_WORKON_LOCALNAME="%s" CROS_WORKON_PROJECT="%s" ' + cmd = ('export CROS_WORKON_LOCALNAME="%s" CROS_WORKON_PROJECT="%s"; ' 'eval $(grep -E "^CROS_WORKON" %s) && ' 'echo $CROS_WORKON_PROJECT ' '$CROS_WORKON_LOCALNAME/$CROS_WORKON_SUBDIR' @@ -341,10 +345,10 @@ class _EBuild(object): # TODO(anush): This hack is only necessary because the kernel ebuild has # 'if' statements, so we can't grab the CROS_WORKON_LOCALNAME properly. # We should clean up the kernel ebuild and remove this hack. - if not os.path.exists(srcdir) and subdir == 'kernel/': + if not os.path.isdir(srcdir) and subdir == 'kernel/': srcdir = os.path.join(srcroot, 'third_party/kernel/files') - if not os.path.exists(srcdir): + if not os.path.isdir(srcdir): Die('Cannot find commit id for %s' % self.ebuild_path) # Verify that we're grabbing the commit id from the right project name. @@ -487,11 +491,16 @@ def main(argv): except gflags.FlagsError, e : _PrintUsageAndDie(str(e)) - package_list = gflags.FLAGS.packages.split() + package_list = gflags.FLAGS.packages.split(':') _CheckSaneArguments(package_list, command) if gflags.FLAGS.overlays: - overlays = dict((path, []) for path in gflags.FLAGS.overlays.split()) + overlays = {} + for path in gflags.FLAGS.overlays.split(':'): + if not os.path.isdir(path): + Die('Cannot find overlay: %s' % path) + overlays[path] = [] else: + Warning('Missing --overlays argument') overlays = { '%s/private-overlays/chromeos-overlay' % gflags.FLAGS.srcroot: [], '%s/third_party/chromiumos-overlay' % gflags.FLAGS.srcroot: [] @@ -501,9 +510,13 @@ def main(argv): _BuildEBuildDictionary(overlays, gflags.FLAGS.all, package_list) for overlay, ebuilds in overlays.items(): - if not os.path.exists(overlay): + if not os.path.isdir(overlay): Warning("Skipping %s" % overlay) continue + + # TODO(davidjames): Currently, all code that interacts with git depends on + # the cwd being set to the overlay directory. We should instead pass in + # this parameter so that we don't need to modify the cwd globally. os.chdir(overlay) if command == 'clean': From 02b3e4f3d52cd8182219c07df56d4030d41d3958 Mon Sep 17 00:00:00 2001 From: David James Date: Fri, 12 Nov 2010 14:47:57 -0800 Subject: [PATCH 35/38] Revert "Add more error checking to preflight queue." This reverts commit 795bd30f069bc64da6f68d8c3e23ed15fa62e072. BUG= TEST= Review URL: http://codereview.chromium.org/4904003 Change-Id: I9148026ecd0df75c253cfcf0c67af3d26771dd21 --- bin/cbuildbot.py | 93 ++++++++++++++------------------------- bin/cbuildbot_config.py | 39 ++++++++++++---- bin/cbuildbot_unittest.py | 17 ++----- cros_mark_as_stable.py | 31 ++++--------- 4 files changed, 76 insertions(+), 104 deletions(-) diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index 58d5c3e2c3..372ddcaf5f 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -18,8 +18,7 @@ import cbuildbot_comm from cbuildbot_config import config sys.path.append(os.path.join(os.path.dirname(__file__), '../lib')) -from cros_build_lib import (Die, Info, ReinterpretPathForChroot, RunCommand, - Warning) +from cros_build_lib import Die, Info, RunCommand, Warning _DEFAULT_RETRIES = 3 ARCHIVE_BASE = '/var/www/archive' @@ -182,38 +181,37 @@ def _ParseRevisionString(revision_string, repo_dictionary): return revisions.items() -def _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board, - overlays): +def _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board): """Uprevs based on revision list.""" if not revision_list: Info('No packages found to uprev') return - packages = [] + package_str = '' for package, revision in revision_list: - assert ':' not in package, 'Invalid package name: %s' % package - packages.append(package) + package_str += package + ' ' - chroot_overlays = [ReinterpretPathForChroot(path) for path in overlays] + package_str = package_str.strip() cwd = os.path.join(buildroot, 'src', 'scripts') + # TODO(davidjames): --foo="bar baz" only works here because we're using + # enter_chroot. RunCommand(['./cros_mark_as_stable', '--board=%s' % board, - '--tracking_branch=%s' % tracking_branch, - '--overlays=%s' % ':'.join(chroot_overlays), - '--packages=%s' % ':'.join(packages), + '--tracking_branch="%s"' % tracking_branch, + '--packages="%s"' % package_str, 'commit'], - cwd=cwd, enter_chroot=True) + cwd=cwd, enter_chroot=True) -def _UprevAllPackages(buildroot, tracking_branch, board, overlays): +def _UprevAllPackages(buildroot, tracking_branch, board): """Uprevs all packages that have been updated since last uprev.""" cwd = os.path.join(buildroot, 'src', 'scripts') - chroot_overlays = [ReinterpretPathForChroot(path) for path in overlays] + # TODO(davidjames): --foo="bar baz" only works here because we're using + # enter_chroot. RunCommand(['./cros_mark_as_stable', '--all', '--board=%s' % board, - '--overlays=%s' % ':'.join(chroot_overlays), - '--tracking_branch=%s' % tracking_branch, 'commit'], + '--tracking_branch="%s"' % tracking_branch, 'commit'], cwd=cwd, enter_chroot=True) @@ -230,13 +228,12 @@ def _GetVMConstants(buildroot): return (vdisk_size.strip(), statefulfs_size.strip()) -def _GitCleanup(buildroot, board, tracking_branch, overlays): +def _GitCleanup(buildroot, board, tracking_branch): """Clean up git branch after previous uprev attempt.""" cwd = os.path.join(buildroot, 'src', 'scripts') if os.path.exists(cwd): RunCommand(['./cros_mark_as_stable', '--srcroot=..', '--board=%s' % board, - '--overlays=%s' % ':'.join(overlays), '--tracking_branch=%s' % tracking_branch, 'clean'], cwd=cwd, error_ok=True) @@ -260,9 +257,9 @@ def _WipeOldOutput(buildroot): # =========================== Main Commands =================================== -def _PreFlightRinse(buildroot, board, tracking_branch, overlays): +def _PreFlightRinse(buildroot, board, tracking_branch): """Cleans up any leftover state from previous runs.""" - _GitCleanup(buildroot, board, tracking_branch, overlays) + _GitCleanup(buildroot, board, tracking_branch) _CleanUpMountPoints(buildroot) RunCommand(['sudo', 'killall', 'kvm'], error_ok=True) @@ -350,7 +347,7 @@ def _RunSmokeSuite(buildroot, results_dir): ], cwd=cwd, error_ok=False) -def _UprevPackages(buildroot, tracking_branch, revisionfile, board, overlays): +def _UprevPackages(buildroot, tracking_branch, revisionfile, board): """Uprevs a package based on given revisionfile. If revisionfile is set to None or does not resolve to an actual file, this @@ -379,19 +376,26 @@ def _UprevPackages(buildroot, tracking_branch, revisionfile, board, overlays): # print >> sys.stderr, 'CBUILDBOT Revision list found %s' % revisions # revision_list = _ParseRevisionString(revisions, # _CreateRepoDictionary(buildroot, board)) - # _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board, - # overlays) + # _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board) #else: Info('CBUILDBOT Revving all') - _UprevAllPackages(buildroot, tracking_branch, board, overlays) + _UprevAllPackages(buildroot, tracking_branch, board) def _UprevPush(buildroot, tracking_branch, board, overlays): """Pushes uprev changes to the main line.""" cwd = os.path.join(buildroot, 'src', 'scripts') + public_overlay = '%s/src/third_party/chromiumos-overlay' % buildroot + private_overlay = '%s/src/private-overlays/chromeos-overlay' % buildroot + if overlays == 'private': + overlays = [private_overlay] + elif overlays == 'public': + overlays = [public_overlay] + else: + overlays = [public_overlay, private_overlay] RunCommand(['./cros_mark_as_stable', '--srcroot=..', '--board=%s' % board, - '--overlays=%s' % ':'.join(overlays), + '--overlays=%s' % " ".join(overlays), '--tracking_branch=%s' % tracking_branch, '--push_options=--bypass-hooks -f', 'push'], cwd=cwd) @@ -463,34 +467,6 @@ def _GetConfig(config_name): return buildconfig -def _ResolveOverlays(buildroot, overlays): - """Return the list of overlays to use for a given buildbot. - - Args: - buildroot: The root directory where the build occurs. Must be an absolute - path. - overlays: A string describing which overlays you want. - 'private': Just the private overlay. - 'public': Just the public overlay. - 'both': Both the public and private overlays. - """ - public_overlay = '%s/src/third_party/chromiumos-overlay' % buildroot - private_overlay = '%s/src/private-overlays/chromeos-overlay' % buildroot - if overlays == 'private': - paths = [private_overlay] - elif overlays == 'public': - paths = [public_overlay] - elif overlays == 'both': - paths = [public_overlay, private_overlay] - else: - Die('Incorrect overlay configuration: %s' % overlays) - for path in paths: - assert ':' not in path, 'Overlay must not contain colons: %s' % path - if not os.path.isdir(path): - Die('Missing overlay: %s' % path) - return paths - - def main(): # Parse options usage = "usage: %prog [options] cbuildbot_config" @@ -515,7 +491,7 @@ def main(): (options, args) = parser.parse_args() - buildroot = os.path.abspath(options.buildroot) + buildroot = options.buildroot revisionfile = options.revisionfile tracking_branch = options.tracking_branch @@ -526,11 +502,8 @@ def main(): parser.print_usage() sys.exit(1) - # Calculate list of overlay directories. - overlays = _ResolveOverlays(buildroot, buildconfig['overlays']) - try: - _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays) + _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch) if options.clobber or not os.path.isdir(buildroot): _FullCheckout(buildroot, tracking_branch, url=options.url) else: @@ -546,7 +519,7 @@ def main(): if buildconfig['uprev']: _UprevPackages(buildroot, tracking_branch, revisionfile, - buildconfig['board'], overlays) + board=buildconfig['board']) _EnableLocalAccount(buildroot) _Build(buildroot) @@ -572,7 +545,7 @@ def main(): # Master bot needs to check if the other slaves completed. if cbuildbot_comm.HaveSlavesCompleted(config): _UprevPush(buildroot, tracking_branch, buildconfig['board'], - overlays) + buildconfig['overlays']) else: Die('CBUILDBOT - One of the slaves has failed!!!') diff --git a/bin/cbuildbot_config.py b/bin/cbuildbot_config.py index 718cd36a70..831ef06bd1 100644 --- a/bin/cbuildbot_config.py +++ b/bin/cbuildbot_config.py @@ -19,9 +19,9 @@ hostname -- Needed for 'important' slaves. The hostname of the bot. Should match hostname in slaves.cfg in buildbot checkout. unittests -- Runs unittests for packages. smoke_bvt -- Runs the test smoke suite in a qemu-based VM using KVM. -overlays -- Select what overlays to look at. This can be 'public', 'private' - or 'both'. There should only be one master bot pushing changes to - each overlay per branch. +overlays -- If this bot is a master bot, select what overlays to push changes + to. This can be 'public', 'private', or 'both'. There should only + be one bot pushing changes to each overlay. """ @@ -33,7 +33,6 @@ config['default'] = { 'important' : False, 'unittests' : False, 'smoke_bvt' : False, - 'overlays': 'public', } config['x86-generic-pre-flight-queue'] = { 'board' : 'x86-generic', @@ -70,7 +69,6 @@ config['x86_agz_bin'] = { 'important' : False, 'unittests' : True, 'smoke_bvt' : True, - 'overlays': 'private', } config['x86_dogfood_bin'] = { 'board' : 'x86-dogfood', @@ -79,29 +77,52 @@ config['x86_dogfood_bin'] = { 'important' : False, 'unittests' : True, 'smoke_bvt' : True, - 'overlays': 'private', } config['x86_pineview_bin'] = { 'board' : 'x86-pineview', 'uprev' : True, 'master' : False, 'important' : False, + 'hostname' : 'codf200.jail', 'unittests': True, - 'overlays': 'public', } config['arm_tegra2_bin'] = { 'board' : 'tegra2', 'uprev' : True, 'master' : False, 'important' : False, + 'hostname' : 'codg172.jail', + 'unittests' : False, +} +config['arm_voguev210_bin'] = { + 'board' : 'voguev210', + 'uprev' : True, + 'master' : False, + 'important' : False, + 'hostname' : 'codf196.jail', + 'unittests' : False, +} +config['arm_beagleboard_bin'] = { + 'board' : 'beagleboard', + 'master' : False, + 'uprev' : True, + 'important' : False, + 'hostname' : 'codf202.jail', + 'unittests' : False, +} +config['arm_st1q_bin'] = { + 'board' : 'st1q', + 'uprev' : True, + 'master' : False, + 'important' : False, + 'hostname' : 'codg158.jail', 'unittests' : False, - 'overlays': 'public', } config['arm_generic_bin'] = { 'board' : 'arm-generic', 'uprev' : True, 'master' : False, 'important' : False, + 'hostname' : 'codg175.jail', 'unittests' : False, - 'overlays': 'public', } diff --git a/bin/cbuildbot_unittest.py b/bin/cbuildbot_unittest.py index 979cda55d4..bbbc56c6f7 100755 --- a/bin/cbuildbot_unittest.py +++ b/bin/cbuildbot_unittest.py @@ -16,7 +16,6 @@ import unittest # Fixes circular dependency error. import cbuildbot_comm import cbuildbot -from cros_build_lib import ReinterpretPathForChroot class CBuildBotTest(mox.MoxTestBase): @@ -43,10 +42,6 @@ class CBuildBotTest(mox.MoxTestBase): ['dev-util/perf', '12345test'], ['chromos-base/libcros', '12345test'] ] - self._overlays = ['%s/src/third_party/chromiumos-overlay' % self._buildroot] - self._chroot_overlays = [ - ReinterpretPathForChroot(p) for p in self._overlays - ] def testParseRevisionString(self): """Test whether _ParseRevisionString parses string correctly.""" @@ -173,15 +168,13 @@ class CBuildBotTest(mox.MoxTestBase): cbuildbot.RunCommand(['./cros_mark_as_stable', '--all', '--board=%s' % self._test_board, - '--overlays=%s' % ':'.join(self._chroot_overlays), - '--tracking_branch=cros/master', 'commit'], + '--tracking_branch="cros/master"', 'commit'], cwd='%s/src/scripts' % self._buildroot, enter_chroot=True) self.mox.ReplayAll() cbuildbot._UprevPackages(self._buildroot, self.tracking_branch, - self._revision_file, self._test_board, - self._overlays) + self._revision_file, self._test_board) self.mox.VerifyAll() def testUprevAllPackages(self): @@ -196,15 +189,13 @@ class CBuildBotTest(mox.MoxTestBase): cbuildbot.RunCommand(['./cros_mark_as_stable', '--all', '--board=%s' % self._test_board, - '--overlays=%s' % ':'.join(self._chroot_overlays), - '--tracking_branch=cros/master', 'commit'], + '--tracking_branch="cros/master"', 'commit'], cwd='%s/src/scripts' % self._buildroot, enter_chroot=True) self.mox.ReplayAll() cbuildbot._UprevPackages(self._buildroot, self.tracking_branch, - self._revision_file, self._test_board, - self._overlays) + self._revision_file, self._test_board) self.mox.VerifyAll() diff --git a/cros_mark_as_stable.py b/cros_mark_as_stable.py index a6888439ed..56a589c682 100755 --- a/cros_mark_as_stable.py +++ b/cros_mark_as_stable.py @@ -22,10 +22,10 @@ from cros_build_lib import Info, RunCommand, Warning, Die gflags.DEFINE_string('board', '', 'Board for which the package belongs.', short_name='b') gflags.DEFINE_string('overlays', '', - 'Colon-separated list of overlays to modify.', + 'Space separated list of overlays to modify.', short_name='o') gflags.DEFINE_string('packages', '', - 'Colon-separated list of packages to mark as stable.', + 'Space separated list of packages to mark as stable.', short_name='p') gflags.DEFINE_string('push_options', '', 'Options to use with git-cl push using push command.') @@ -245,11 +245,7 @@ def _SimpleRunCommand(command): """Runs a shell command and returns stdout back to caller.""" _Print(' + %s' % command) proc_handle = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) - stdout = proc_handle.communicate()[0] - retcode = proc_handle.wait() - if retcode != 0: - raise subprocess.CalledProcessError(retcode, command, output=stdout) - return stdout + return proc_handle.communicate()[0] # ======================= End Global Helper Functions ======================== @@ -327,7 +323,7 @@ class _EBuild(object): # Grab and evaluate CROS_WORKON variables from this ebuild. unstable_ebuild = '%s-9999.ebuild' % self.ebuild_path_no_version - cmd = ('export CROS_WORKON_LOCALNAME="%s" CROS_WORKON_PROJECT="%s"; ' + cmd = ('CROS_WORKON_LOCALNAME="%s" CROS_WORKON_PROJECT="%s" ' 'eval $(grep -E "^CROS_WORKON" %s) && ' 'echo $CROS_WORKON_PROJECT ' '$CROS_WORKON_LOCALNAME/$CROS_WORKON_SUBDIR' @@ -345,10 +341,10 @@ class _EBuild(object): # TODO(anush): This hack is only necessary because the kernel ebuild has # 'if' statements, so we can't grab the CROS_WORKON_LOCALNAME properly. # We should clean up the kernel ebuild and remove this hack. - if not os.path.isdir(srcdir) and subdir == 'kernel/': + if not os.path.exists(srcdir) and subdir == 'kernel/': srcdir = os.path.join(srcroot, 'third_party/kernel/files') - if not os.path.isdir(srcdir): + if not os.path.exists(srcdir): Die('Cannot find commit id for %s' % self.ebuild_path) # Verify that we're grabbing the commit id from the right project name. @@ -491,16 +487,11 @@ def main(argv): except gflags.FlagsError, e : _PrintUsageAndDie(str(e)) - package_list = gflags.FLAGS.packages.split(':') + package_list = gflags.FLAGS.packages.split() _CheckSaneArguments(package_list, command) if gflags.FLAGS.overlays: - overlays = {} - for path in gflags.FLAGS.overlays.split(':'): - if not os.path.isdir(path): - Die('Cannot find overlay: %s' % path) - overlays[path] = [] + overlays = dict((path, []) for path in gflags.FLAGS.overlays.split()) else: - Warning('Missing --overlays argument') overlays = { '%s/private-overlays/chromeos-overlay' % gflags.FLAGS.srcroot: [], '%s/third_party/chromiumos-overlay' % gflags.FLAGS.srcroot: [] @@ -510,13 +501,9 @@ def main(argv): _BuildEBuildDictionary(overlays, gflags.FLAGS.all, package_list) for overlay, ebuilds in overlays.items(): - if not os.path.isdir(overlay): + if not os.path.exists(overlay): Warning("Skipping %s" % overlay) continue - - # TODO(davidjames): Currently, all code that interacts with git depends on - # the cwd being set to the overlay directory. We should instead pass in - # this parameter so that we don't need to modify the cwd globally. os.chdir(overlay) if command == 'clean': From 5a886ff53d1eb05a299dc0a6a2c99e585bc8c145 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Fri, 12 Nov 2010 15:20:27 -0800 Subject: [PATCH 36/38] Don't allow hardlinks in tarball. These errors on extraction if you don't have the original targets of the links on the image (i.e. if the stateful_partition gets wiped). Change-Id: Id5d3f770830de6165613d506733c04edefccd3fb BUG=chromium-os:9103 TEST=Ran update, wiped partition, ran stateful_update and ran bvt's. Review URL: http://codereview.chromium.org/4882006 --- cros_generate_stateful_update_payload.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cros_generate_stateful_update_payload.py b/cros_generate_stateful_update_payload.py index b8bc1eb65e..a2c170a84e 100755 --- a/cros_generate_stateful_update_payload.py +++ b/cros_generate_stateful_update_payload.py @@ -50,6 +50,7 @@ def GenerateStatefulPayload(image_path, output_directory, logger): '-czf', output_gz, '--directory=%s' % stateful_dir, + '--hard-dereference', '--transform=s,^dev_image,dev_image_new,', '--transform=s,^var,var_new,', 'dev_image', From 132ff4320dba21b167aee9ec5888eca265e32897 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Fri, 12 Nov 2010 15:29:30 -0800 Subject: [PATCH 37/38] cros_generate_upload_payload --patch_kernel should mount from the source image read only to avoid unintended changes. Change-Id: Id3a83786f2c1240a4401978c8cacffc0dcb9d66f BUG=9073 TEST=Ran script with --patch_kernel, verified source image unmodified. Ran ./image_to_live.sh to verify valid update was generated. Review URL: http://codereview.chromium.org/4892002 --- cros_generate_update_payload | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cros_generate_update_payload b/cros_generate_update_payload index dd0a67c06e..3b5b9f9d6d 100755 --- a/cros_generate_update_payload +++ b/cros_generate_update_payload @@ -87,7 +87,7 @@ patch_kernel() { offset=$(($offset * 512)) sudo losetup -o "$offset" "$STATE_LOOP_DEV" "$IMAGE" STATE_MNT=$(mktemp -d /tmp/state.XXXXXX) - sudo mount "$STATE_LOOP_DEV" "$STATE_MNT" + sudo mount --read-only "$STATE_LOOP_DEV" "$STATE_MNT" dd if="$STATE_MNT"/vmlinuz_hd.vblock of="$KERN_FILE" conv=notrunc sudo umount "$STATE_MNT" STATE_MNT="" From 37f42226326855989306e85e57db29a691c51276 Mon Sep 17 00:00:00 2001 From: David McMahon Date: Fri, 12 Nov 2010 16:17:59 -0800 Subject: [PATCH 38/38] Stop using CHROME_BUILD altogether - companion change coming in crostools. BUG= TEST= Review URL: http://codereview.chromium.org/4917001 --- chromeos_version.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/chromeos_version.sh b/chromeos_version.sh index fcebafb516..649898719e 100755 --- a/chromeos_version.sh +++ b/chromeos_version.sh @@ -84,9 +84,8 @@ export CHROMEOS_VERSION_STRING=\ # Set CHROME values (Used for releases) to pass to chromeos-chrome-bin ebuild # URL to chrome archive export CHROME_BASE= -# directory containing chrome-chromeos.zip - an svn rev or a full version -export CHROME_BUILD= -export CHROME_VERSION="$CHROME_BUILD" +# export CHROME_VERSION from incoming value or NULL and let ebuild default +export CHROME_VERSION="$CHROME_VERSION" # Print (and remember) version info. echo "ChromeOS version information:"