mirror of
https://github.com/flatcar/scripts.git
synced 2025-09-24 15:11:19 +02:00
a48cb7e19d
Usage: cros_build_packages [options] Options: -h, --help show this help message and exit --board=BOARD The board to build packages for. --debug Include debug symbols. --nowithdev Don't build useful developer friendly utilities. --nowithtest Build packages required for testing. --nowithfactory Build factory installer --nousepkg Don't use binary packages. --nousetarball Don't use tarball. --nofast Don't merge packages in parallel. BUG=chromium-os:6604 TEST=Ran cros_build_packages and ./cros_build_packages --nousetarball Review URL: http://codereview.chromium.org/3315020 Change-Id: Ida9ed6375c2ce4acde304bd3f2796b4419f1d376
276 lines
9.0 KiB
Python
Executable File
276 lines
9.0 KiB
Python
Executable File
#!/usr/bin/python2.6
|
|
# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
import optparse
|
|
import os
|
|
import multiprocessing
|
|
import sys
|
|
import tempfile
|
|
sys.path.insert(0, os.path.abspath(__file__ + "/../../lib"))
|
|
from cros_build_lib import Die
|
|
from cros_build_lib import Info
|
|
from cros_build_lib import RunCommand
|
|
from cros_build_lib import Warning
|
|
|
|
|
|
def BuildPackages():
|
|
"""Build packages according to options specified on command-line."""
|
|
|
|
if os.getuid() != 0:
|
|
Die("superuser access required")
|
|
|
|
scripts_dir = os.path.abspath(__file__ + "/../../..")
|
|
builder = PackageBuilder(scripts_dir)
|
|
options, _ = builder.ParseArgs()
|
|
|
|
# Calculate packages to install.
|
|
# TODO(davidjames): Grab these from a spec file.
|
|
packages = ["chromeos-base/chromeos"]
|
|
if options.withdev:
|
|
packages.append("chromeos-base/chromeos-dev")
|
|
if options.withfactory:
|
|
packages.append("chromeos-base/chromeos-factoryinstall")
|
|
if options.withtest:
|
|
packages.append("chromeos-base/chromeos-test")
|
|
|
|
if options.usetarball:
|
|
builder.ExtractTarball(options, packages)
|
|
else:
|
|
builder.BuildTarball(options, packages)
|
|
|
|
|
|
def _Apply(args):
|
|
"""Call the function specified in args[0], with arguments in args[1:]."""
|
|
return apply(args[0], args[1:])
|
|
|
|
|
|
def _GetLatestPrebuiltPrefix(board):
|
|
"""Get the latest prebuilt prefix for the specified board.
|
|
|
|
Args:
|
|
board: The board you want prebuilts for.
|
|
Returns:
|
|
Latest prebuilt prefix.
|
|
"""
|
|
# TODO(davidjames): Also append profile names here.
|
|
prefix = "http://commondatastorage.googleapis.com/chromeos-prebuilt/board"
|
|
tmpfile = tempfile.NamedTemporaryFile()
|
|
_Run("curl '%s/%s-latest' -o %s" % (prefix, board, tmpfile.name), retries=3)
|
|
tmpfile.seek(0)
|
|
latest = tmpfile.read().strip()
|
|
tmpfile.close()
|
|
return "%s/%s" % (prefix, latest)
|
|
|
|
|
|
def _GetPrebuiltDownloadCommands(prefix):
|
|
"""Return a list of commands for grabbing packages.
|
|
|
|
There must be a file called "packages/Packages" that contains the list of
|
|
packages. The specified list of commands will fill the packages directory
|
|
with the bzipped packages from the specified prefix.
|
|
|
|
Args:
|
|
prefix: Url prefix to download packages from.
|
|
Returns:
|
|
List of commands for grabbing packages.
|
|
"""
|
|
|
|
cmds = []
|
|
for line in file("packages/Packages"):
|
|
if line.startswith("CPV: "):
|
|
pkgpath, pkgname = line.replace("CPV: ", "").strip().split("/")
|
|
path = "%s/%s.tbz2" % (pkgpath, pkgname)
|
|
url = "%s/%s" % (prefix, path)
|
|
dirname = "packages/%s" % pkgpath
|
|
fullpath = "packages/%s" % path
|
|
if not os.path.exists(dirname):
|
|
os.makedirs(dirname)
|
|
if not os.path.exists(fullpath):
|
|
cmds.append("curl -s %s -o %s" % (url, fullpath))
|
|
return cmds
|
|
|
|
|
|
def _Run(cmd, retries=0):
|
|
"""Run the specified command.
|
|
|
|
If the command fails, and the retries have been exhausted, the program exits
|
|
with an appropriate error message.
|
|
|
|
Args:
|
|
cmd: The command to run.
|
|
retries: If exit code is non-zero, retry this many times.
|
|
"""
|
|
# TODO(davidjames): Move this to common library.
|
|
for _ in range(retries+1):
|
|
result = RunCommand(cmd, shell=True, exit_code=True, error_ok=True)
|
|
if result.returncode == 0:
|
|
Info("Command succeeded: %s" % cmd)
|
|
break
|
|
Warning("Command failed: %s" % cmd)
|
|
else:
|
|
Die("Command failed, exiting: %s" % cmd)
|
|
|
|
|
|
def _RunManyParallel(cmds, retries=0):
|
|
"""Run list of provided commands in parallel.
|
|
|
|
To work around a bug in the multiprocessing module, we use map_async instead
|
|
of the usual map function. See http://bugs.python.org/issue9205
|
|
|
|
Args:
|
|
cmds: List of commands to run.
|
|
retries: Number of retries per command.
|
|
"""
|
|
# TODO(davidjames): Move this to common library.
|
|
pool = multiprocessing.Pool()
|
|
args = []
|
|
for cmd in cmds:
|
|
args.append((_Run, cmd, retries))
|
|
result = pool.map_async(_Apply, args, chunksize=1)
|
|
while True:
|
|
try:
|
|
result.get(60*60)
|
|
break
|
|
except multiprocessing.TimeoutError:
|
|
pass
|
|
|
|
|
|
class PackageBuilder(object):
|
|
"""A class for building and extracting tarballs of Chromium OS packages."""
|
|
|
|
def __init__(self, scripts_dir):
|
|
self.scripts_dir = scripts_dir
|
|
|
|
def BuildTarball(self, options, packages):
|
|
"""Build a tarball with the specified packages.
|
|
|
|
Args:
|
|
options: Options object, as output by ParseArgs.
|
|
packages: List of packages to build.
|
|
"""
|
|
|
|
board = options.board
|
|
|
|
# Run setup_board. TODO(davidjames): Integrate the logic used in
|
|
# setup_board into chromite.
|
|
_Run("%s/setup_board --force --board=%s" % (self.scripts_dir, board))
|
|
|
|
# Create complete build directory
|
|
_Run(self._EmergeBoardCmd(options, packages))
|
|
|
|
# Archive build directory as tarballs
|
|
os.chdir("/build/%s" % board)
|
|
cmds = [
|
|
"tar -c --wildcards --exclude='usr/lib/debug/*' "
|
|
"--exclude='packages/*' * | pigz -c > packages/%s-build.tgz" % board,
|
|
"tar -c usr/lib/debug/* | pigz -c > packages/%s-debug.tgz" % board
|
|
]
|
|
|
|
# Run list of commands.
|
|
_RunManyParallel(cmds)
|
|
|
|
def ExtractTarball(self, options, packages):
|
|
"""Extract the latest build tarball, then update the specified packages.
|
|
|
|
Args:
|
|
options: Options object, as output by ParseArgs.
|
|
packages: List of packages to update.
|
|
"""
|
|
|
|
board = options.board
|
|
prefix = _GetLatestPrebuiltPrefix(board)
|
|
|
|
# If the user doesn't have emerge-${BOARD} setup yet, we need to run
|
|
# setup_board. TODO(davidjames): Integrate the logic used in setup_board
|
|
# into chromite.
|
|
if not os.path.exists("/usr/local/bin/emerge-%s" % board):
|
|
_Run("%s/setup_board --force --board=%s" % (self.scripts_dir, board))
|
|
|
|
# Delete old build directory. This process might take a while, so do it in
|
|
# the background.
|
|
cmds = []
|
|
if os.path.exists("/build/%s" % board):
|
|
tempdir = tempfile.mkdtemp()
|
|
_Run("mv /build/%s %s" % (board, tempdir))
|
|
cmds.append("rm -rf %s" % tempdir)
|
|
|
|
# Create empty build directory, and chdir into it.
|
|
os.makedirs("/build/%s/packages" % board)
|
|
os.chdir("/build/%s" % board)
|
|
|
|
# Download and expand build tarball.
|
|
build_url = "%s/%s-build.tgz" % (prefix, board)
|
|
cmds.append("curl -s %s | tar -xz" % build_url)
|
|
|
|
# Download and expand debug tarball (if requested).
|
|
if options.debug:
|
|
debug_url = "%s/%s-debug.tgz" % (prefix, board)
|
|
cmds.append("curl -s %s | tar -xz" % debug_url)
|
|
|
|
# Download prebuilt packages.
|
|
_Run("curl '%s/Packages' -o packages/Packages" % prefix, retries=3)
|
|
cmds.extend(_GetPrebuiltDownloadCommands(prefix))
|
|
|
|
# Run list of commands, with three retries per command, in case the network
|
|
# is flaky.
|
|
_RunManyParallel(cmds, retries=3)
|
|
|
|
# Emerge remaining packages.
|
|
_Run(self._EmergeBoardCmd(options, packages))
|
|
|
|
def ParseArgs(self):
|
|
"""Parse arguments from the command line using optparse."""
|
|
|
|
# TODO(davidjames): We should use spec files for this.
|
|
default_board = self._GetDefaultBoard()
|
|
parser = optparse.OptionParser()
|
|
parser.add_option("--board", dest="board", default=default_board,
|
|
help="The board to build packages for.")
|
|
parser.add_option("--debug", action="store_true", dest="debug",
|
|
default=False, help="Include debug symbols.")
|
|
parser.add_option("--nowithdev", action="store_false", dest="withdev",
|
|
default=True,
|
|
help="Don't build useful developer friendly utilities.")
|
|
parser.add_option("--nowithtest", action="store_false", dest="withtest",
|
|
default=True, help="Build packages required for testing.")
|
|
parser.add_option("--nowithfactory", action="store_false",
|
|
dest="withfactory", default=True,
|
|
help="Build factory installer")
|
|
parser.add_option("--nousepkg", action="store_false",
|
|
dest="usepkg", default=True,
|
|
help="Don't use binary packages.")
|
|
parser.add_option("--nousetarball", action="store_false",
|
|
dest="usetarball", default=True,
|
|
help="Don't use tarball.")
|
|
parser.add_option("--nofast", action="store_false", dest="fast",
|
|
default=True,
|
|
help="Don't merge packages in parallel.")
|
|
|
|
return parser.parse_args()
|
|
|
|
def _EmergeBoardCmd(self, options, packages):
|
|
"""Calculate board emerge command."""
|
|
board = options.board
|
|
scripts_dir = self.scripts_dir
|
|
emerge_board = "emerge-%s" % board
|
|
if options.fast:
|
|
emerge_board = "%s/parallel_emerge --board=%s" % (scripts_dir, board)
|
|
usepkg = ""
|
|
if options.usepkg:
|
|
usepkg = "g"
|
|
return "%s -uDNv%s %s" % (emerge_board, usepkg, " ".join(packages))
|
|
|
|
def _GetDefaultBoard(self):
|
|
"""Get the default board configured by the user."""
|
|
|
|
default_board_file = "%s/.default_board" % self.scripts_dir
|
|
default_board = None
|
|
if os.path.exists(default_board_file):
|
|
default_board = file(default_board_file).read().strip()
|
|
return default_board
|
|
|
|
if __name__ == "__main__":
|
|
BuildPackages()
|