diff --git a/chromite/bin/cros_build_packages b/chromite/bin/cros_build_packages new file mode 100755 index 0000000000..0cf2f4f301 --- /dev/null +++ b/chromite/bin/cros_build_packages @@ -0,0 +1,275 @@ +#!/usr/bin/python2.6 +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import optparse +import os +import multiprocessing +import sys +import tempfile +sys.path.insert(0, os.path.abspath(__file__ + "/../../lib")) +from cros_build_lib import Die +from cros_build_lib import Info +from cros_build_lib import RunCommand +from cros_build_lib import Warning + + +def BuildPackages(): + """Build packages according to options specified on command-line.""" + + if os.getuid() != 0: + Die("superuser access required") + + scripts_dir = os.path.abspath(__file__ + "/../../..") + builder = PackageBuilder(scripts_dir) + options, _ = builder.ParseArgs() + + # Calculate packages to install. + # TODO(davidjames): Grab these from a spec file. + packages = ["chromeos-base/chromeos"] + if options.withdev: + packages.append("chromeos-base/chromeos-dev") + if options.withfactory: + packages.append("chromeos-base/chromeos-factoryinstall") + if options.withtest: + packages.append("chromeos-base/chromeos-test") + + if options.usetarball: + builder.ExtractTarball(options, packages) + else: + builder.BuildTarball(options, packages) + + +def _Apply(args): + """Call the function specified in args[0], with arguments in args[1:].""" + return apply(args[0], args[1:]) + + +def _GetLatestPrebuiltPrefix(board): + """Get the latest prebuilt prefix for the specified board. + + Args: + board: The board you want prebuilts for. + Returns: + Latest prebuilt prefix. + """ + # TODO(davidjames): Also append profile names here. + prefix = "http://commondatastorage.googleapis.com/chromeos-prebuilt/board" + tmpfile = tempfile.NamedTemporaryFile() + _Run("curl '%s/%s-latest' -o %s" % (prefix, board, tmpfile.name), retries=3) + tmpfile.seek(0) + latest = tmpfile.read().strip() + tmpfile.close() + return "%s/%s" % (prefix, latest) + + +def _GetPrebuiltDownloadCommands(prefix): + """Return a list of commands for grabbing packages. + + There must be a file called "packages/Packages" that contains the list of + packages. The specified list of commands will fill the packages directory + with the bzipped packages from the specified prefix. + + Args: + prefix: Url prefix to download packages from. + Returns: + List of commands for grabbing packages. + """ + + cmds = [] + for line in file("packages/Packages"): + if line.startswith("CPV: "): + pkgpath, pkgname = line.replace("CPV: ", "").strip().split("/") + path = "%s/%s.tbz2" % (pkgpath, pkgname) + url = "%s/%s" % (prefix, path) + dirname = "packages/%s" % pkgpath + fullpath = "packages/%s" % path + if not os.path.exists(dirname): + os.makedirs(dirname) + if not os.path.exists(fullpath): + cmds.append("curl -s %s -o %s" % (url, fullpath)) + return cmds + + +def _Run(cmd, retries=0): + """Run the specified command. + + If the command fails, and the retries have been exhausted, the program exits + with an appropriate error message. + + Args: + cmd: The command to run. + retries: If exit code is non-zero, retry this many times. + """ + # TODO(davidjames): Move this to common library. + for _ in range(retries+1): + result = RunCommand(cmd, shell=True, exit_code=True, error_ok=True) + if result.returncode == 0: + Info("Command succeeded: %s" % cmd) + break + Warning("Command failed: %s" % cmd) + else: + Die("Command failed, exiting: %s" % cmd) + + +def _RunManyParallel(cmds, retries=0): + """Run list of provided commands in parallel. + + To work around a bug in the multiprocessing module, we use map_async instead + of the usual map function. See http://bugs.python.org/issue9205 + + Args: + cmds: List of commands to run. + retries: Number of retries per command. + """ + # TODO(davidjames): Move this to common library. + pool = multiprocessing.Pool() + args = [] + for cmd in cmds: + args.append((_Run, cmd, retries)) + result = pool.map_async(_Apply, args, chunksize=1) + while True: + try: + result.get(60*60) + break + except multiprocessing.TimeoutError: + pass + + +class PackageBuilder(object): + """A class for building and extracting tarballs of Chromium OS packages.""" + + def __init__(self, scripts_dir): + self.scripts_dir = scripts_dir + + def BuildTarball(self, options, packages): + """Build a tarball with the specified packages. + + Args: + options: Options object, as output by ParseArgs. + packages: List of packages to build. + """ + + board = options.board + + # Run setup_board. TODO(davidjames): Integrate the logic used in + # setup_board into chromite. + _Run("%s/setup_board --force --board=%s" % (self.scripts_dir, board)) + + # Create complete build directory + _Run(self._EmergeBoardCmd(options, packages)) + + # Archive build directory as tarballs + os.chdir("/build/%s" % board) + cmds = [ + "tar -c --wildcards --exclude='usr/lib/debug/*' " + "--exclude='packages/*' * | pigz -c > packages/%s-build.tgz" % board, + "tar -c usr/lib/debug/* | pigz -c > packages/%s-debug.tgz" % board + ] + + # Run list of commands. + _RunManyParallel(cmds) + + def ExtractTarball(self, options, packages): + """Extract the latest build tarball, then update the specified packages. + + Args: + options: Options object, as output by ParseArgs. + packages: List of packages to update. + """ + + board = options.board + prefix = _GetLatestPrebuiltPrefix(board) + + # If the user doesn't have emerge-${BOARD} setup yet, we need to run + # setup_board. TODO(davidjames): Integrate the logic used in setup_board + # into chromite. + if not os.path.exists("/usr/local/bin/emerge-%s" % board): + _Run("%s/setup_board --force --board=%s" % (self.scripts_dir, board)) + + # Delete old build directory. This process might take a while, so do it in + # the background. + cmds = [] + if os.path.exists("/build/%s" % board): + tempdir = tempfile.mkdtemp() + _Run("mv /build/%s %s" % (board, tempdir)) + cmds.append("rm -rf %s" % tempdir) + + # Create empty build directory, and chdir into it. + os.makedirs("/build/%s/packages" % board) + os.chdir("/build/%s" % board) + + # Download and expand build tarball. + build_url = "%s/%s-build.tgz" % (prefix, board) + cmds.append("curl -s %s | tar -xz" % build_url) + + # Download and expand debug tarball (if requested). + if options.debug: + debug_url = "%s/%s-debug.tgz" % (prefix, board) + cmds.append("curl -s %s | tar -xz" % debug_url) + + # Download prebuilt packages. + _Run("curl '%s/Packages' -o packages/Packages" % prefix, retries=3) + cmds.extend(_GetPrebuiltDownloadCommands(prefix)) + + # Run list of commands, with three retries per command, in case the network + # is flaky. + _RunManyParallel(cmds, retries=3) + + # Emerge remaining packages. + _Run(self._EmergeBoardCmd(options, packages)) + + def ParseArgs(self): + """Parse arguments from the command line using optparse.""" + + # TODO(davidjames): We should use spec files for this. + default_board = self._GetDefaultBoard() + parser = optparse.OptionParser() + parser.add_option("--board", dest="board", default=default_board, + help="The board to build packages for.") + parser.add_option("--debug", action="store_true", dest="debug", + default=False, help="Include debug symbols.") + parser.add_option("--nowithdev", action="store_false", dest="withdev", + default=True, + help="Don't build useful developer friendly utilities.") + parser.add_option("--nowithtest", action="store_false", dest="withtest", + default=True, help="Build packages required for testing.") + parser.add_option("--nowithfactory", action="store_false", + dest="withfactory", default=True, + help="Build factory installer") + parser.add_option("--nousepkg", action="store_false", + dest="usepkg", default=True, + help="Don't use binary packages.") + parser.add_option("--nousetarball", action="store_false", + dest="usetarball", default=True, + help="Don't use tarball.") + parser.add_option("--nofast", action="store_false", dest="fast", + default=True, + help="Don't merge packages in parallel.") + + return parser.parse_args() + + def _EmergeBoardCmd(self, options, packages): + """Calculate board emerge command.""" + board = options.board + scripts_dir = self.scripts_dir + emerge_board = "emerge-%s" % board + if options.fast: + emerge_board = "%s/parallel_emerge --board=%s" % (scripts_dir, board) + usepkg = "" + if options.usepkg: + usepkg = "g" + return "%s -uDNv%s %s" % (emerge_board, usepkg, " ".join(packages)) + + def _GetDefaultBoard(self): + """Get the default board configured by the user.""" + + default_board_file = "%s/.default_board" % self.scripts_dir + default_board = None + if os.path.exists(default_board_file): + default_board = file(default_board_file).read().strip() + return default_board + +if __name__ == "__main__": + BuildPackages()