diff --git a/autotest_run.sh b/autotest_run.sh index 73907d9f76..17774074e2 100755 --- a/autotest_run.sh +++ b/autotest_run.sh @@ -37,6 +37,11 @@ CONFIG_SITE=/usr/share/config.site function setup_ssh() { eval $(ssh-agent) > /dev/null + # TODO(jrbarnette): This is a temporary hack, slated for removal + # before it was ever created. It's a bug, and you should fix it + # right away! + chmod 400 \ + ${CHROMEOS_ROOT}/src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa ssh-add \ ${CHROMEOS_ROOT}/src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa } diff --git a/autotest_workon b/autotest_workon deleted file mode 100755 index ee12f4ad47..0000000000 --- a/autotest_workon +++ /dev/null @@ -1,229 +0,0 @@ -#!/usr/bin/python - -# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -# -# A python wrapper to call autotest ebuild. - -import commands, logging, optparse, os, subprocess, sys - - -def run(cmd): - return subprocess.call(cmd, stdout=sys.stdout, stderr=sys.stderr) - - -class MyOptionParser(optparse.OptionParser): - """Override python's builtin OptionParser to accept any undefined args.""" - - help = False - - def _process_args(self, largs, rargs, values): - # see /usr/lib64/python2.6/optparse.py line 1414-1463 - while rargs: - arg = rargs[0] - # We handle bare "--" explicitly, and bare "-" is handled by the - # standard arg handler since the short arg case ensures that the - # len of the opt string is greater than 1. - if arg == "--": - del rargs[0] - return - elif arg[0:2] == "--": - # process a single long option (possibly with value(s)) - try: - self._process_long_opt(rargs, values) - except optparse.BadOptionError: - largs.append(arg) - elif arg[:1] == "-" and len(arg) > 1: - # process a cluster of short options (possibly with - # value(s) for the last one only) - try: - self._process_short_opts(rargs, values) - except optparse.BadOptionError: - largs.append(arg) - elif self.allow_interspersed_args: - largs.append(arg) - del rargs[0] - else: - return # stop now, leave this arg in rargs - - def print_help(self, file=None): - optparse.OptionParser.print_help(self, file) - MyOptionParser.help = True - - -parser = MyOptionParser() -parser.allow_interspersed_args = True - -DEFAULT_BOARD = os.environ.get('DEFAULT_BOARD', '') - -parser.add_option('--args', dest='args', action='store', - default='', - help='The arguments to pass to the test control file.') -parser.add_option('--autox', dest='autox', action='store_true', - default=True, - help='Build autox along with autotest [default].') -parser.add_option('--noautox', dest='autox', action='store_false', - help='Don\'t build autox along with autotest.') -parser.add_option('--board', dest='board', action='store', - default=DEFAULT_BOARD, - help='The board for which you are building autotest.') -parser.add_option('--build', dest='build', action='store', - help='Only prebuild client tests, do not run tests.') -parser.add_option('--buildcheck', dest='buildcheck', action='store_true', - default=True, - help='Fail if tests fail to build [default].') -parser.add_option('--nobuildcheck', dest='buildcheck', action='store_false', - help='Ignore test build failures.') -parser.add_option('--jobs', dest='jobs', action='store', type=int, - default=-1, - help='How many packages to build in parallel at maximum.') -parser.add_option('--noprompt', dest='noprompt', action='store_true', - help='Prompt user when building all tests.') - - -AUTOSERV='../third_party/autotest/files/server/autoserv' -AUTOTEST_CLIENT='../third_party/autotest/files/client/bin/autotest_client' - -def parse_args_and_help(): - - def nop(_): - pass - - sys_exit = sys.exit - sys.exit = nop - options, args = parser.parse_args() - sys.exit = sys_exit - - if not args and not options.build: - parser.print_help() - - if MyOptionParser.help: - if options.build: - print - print 'Options inherited from autotest_client, which is used in build', - print 'only mode.' - run([AUTOTEST_CLIENT, '--help']) - else: - print - print 'Options inherited from autoserv:' - run([AUTOSERV, '--help']) - sys.exit(0) - return options, args - - -def assert_inside_chroot(common_sh): - status, output = commands.getstatusoutput('/bin/bash -c ". %s && ' - 'assert_inside_chroot"' % common_sh) - if status is not 0: - print >> sys.stderr, output - sys.exit(status) - - -def set_common_env(common_sh, env_var): - env_value = commands.getoutput('/bin/bash -c \'. %s && echo $%s\'' % - (common_sh, env_var)) - os.environ[env_var] = env_value - - -def die(common_sh, msg): - output = commands.getoutput('/bin/bash -c \'. %s && die "%s"\'' % - (common_sh, msg)) - print >> sys.stderr, output - sys.exit(1) - - -def build_autotest(options): - environ = os.environ - if options.jobs != -1: - emerge_jobs = '--jobs=%d' % options.jobs - else: - emerge_jobs = '' - - # Decide on USE flags based on options - use_flag = environ.get('USE', '') - if not options.autox: - use_flag = use_flag + ' -autox' - if options.buildcheck: - use_flag = use_flag + ' buildcheck' - - board_blacklist_file = ('%s/src/overlays/overlay-%s/autotest-blacklist' % - (os.environ['GCLIENT_ROOT'], options.board)) - if os.path.exists(board_blacklist_file): - blacklist = [line.strip() - for line in open(board_blacklist_file).readlines()] - else: - blacklist = [] - - all_tests = ('compilebench,dbench,disktest,fsx,hackbench,iperf,ltp,netperf2,' - 'netpipe,unixbench') - site_tests = '../third_party/autotest/files/client/site_tests' - for site_test in os.listdir(site_tests): - test_path = os.path.join(site_tests, site_test) - test_py = os.path.join(test_path, '%s.py' % site_test) - if (os.path.exists(test_path) and os.path.isdir(test_path) and - os.path.exists(test_py) and os.path.isfile(test_py) and - site_test not in blacklist): - all_tests += ',' + site_test - - if 'all' == options.build.lower(): - if options.noprompt is not True: - print 'You want to pre-build all client tests and it may take a long', - print 'time to finish.' - print 'Are you sure you want to continue?(N/y)', - answer = sys.stdin.readline() - if 'y' != answer[0].lower(): - print 'Use --build to specify tests you like to pre-compile. ' - print 'E.g.: ./autotest --build=disktest,hardware_SAT' - sys.exit(0) - test_list = all_tests - else: - test_list = options.build - - environ['FEATURES'] = ('%s -buildpkg -collision-protect' % - environ.get('FEATURES', '')) - environ['TEST_LIST'] = test_list - environ['USE'] = use_flag - emerge_cmd = ['emerge-%s' % options.board, - 'chromeos-base/autotest'] - if emerge_jobs: - emerge_cmd.append(emerge_jobs) - return run(emerge_cmd) - - -def run_autoserv(options, args): - environ = os.environ - - environ['AUTOSERV_TEST_ARGS'] = options.args - environ['AUTOSERV_ARGS'] = ' '.join(args) - environ['FEATURES'] = ('%s -buildpkg -digest noauto' % - environ.get('FEATURES', '')) - ebuild_cmd = [ './autotest_run.sh', '--board=%s' % options.board] - run(ebuild_cmd) - - -def main(): - me = sys.argv[0] - common_sh = os.path.join(os.path.dirname(me), 'common.sh') - - assert_inside_chroot(common_sh) - set_common_env(common_sh, 'GCLIENT_ROOT') - - options, args = parse_args_and_help() - - if not options.board: - die(common_sh, 'Missing --board argument.') - - if options.build: - status = build_autotest(options) - if status: - die(common_sh, 'build_autotest failed.') - else: - ssh_key_file = os.path.join(os.path.dirname(me), - 'mod_for_test_scripts/ssh_keys/testing_rsa') - os.chmod(ssh_key_file, 0400) - run_autoserv(options, args) - - -if __name__ == '__main__': - main() diff --git a/bin/cros_run_parallel_vm_tests.py b/bin/cros_run_parallel_vm_tests.py index b6403bfe65..2ca51a3afa 100755 --- a/bin/cros_run_parallel_vm_tests.py +++ b/bin/cros_run_parallel_vm_tests.py @@ -27,7 +27,8 @@ class ParallelTestRunner(object): """ def __init__(self, tests, base_ssh_port=_DEFAULT_BASE_SSH_PORT, board=None, - image_path=None, order_output=False, results_dir_root=None): + image_path=None, order_output=False, results_dir_root=None, + use_emerged=False): """Constructs and initializes the test runner class. Args: @@ -50,6 +51,7 @@ class ParallelTestRunner(object): self._image_path = image_path self._order_output = order_output self._results_dir_root = results_dir_root + self._use_emerged = use_emerged def _SpawnTests(self): """Spawns VMs and starts the test runs on them. @@ -64,10 +66,6 @@ class ParallelTestRunner(object): """ ssh_port = self._base_ssh_port spawned_tests = [] - # Test runs shouldn't need anything from stdin. However, it seems that - # running with stdin leaves the terminal in a bad state so redirect from - # /dev/null. - dev_null = open('/dev/null') for test in self._tests: args = [ os.path.join(os.path.dirname(__file__), 'cros_run_vm_test'), '--snapshot', # The image is shared so don't modify it. @@ -79,13 +77,13 @@ class ParallelTestRunner(object): if self._results_dir_root: args.append('--results_dir_root=%s/%s.%d' % (self._results_dir_root, test, ssh_port)) + if self._use_emerged: args.append('--use_emerged') Info('Running %r...' % args) output = None if self._order_output: output = tempfile.NamedTemporaryFile(prefix='parallel_vm_test_') Info('Piping output to %s.' % output.name) - proc = subprocess.Popen(args, stdin=dev_null, stdout=output, - stderr=output) + proc = subprocess.Popen(args, stdout=output, stderr=output) test_info = { 'test': test, 'proc': proc, 'output': output } @@ -147,6 +145,8 @@ def main(): parser.add_option('--results_dir_root', help='Root results directory. If none specified, each test ' 'will store its results in a separate /tmp directory.') + parser.add_option('--use_emerged', action='store_true', default=False, + help='Force use of emerged autotest packages') (options, args) = parser.parse_args() if not args: @@ -155,7 +155,7 @@ def main(): runner = ParallelTestRunner(args, options.base_ssh_port, options.board, options.image_path, options.order_output, - options.results_dir_root) + options.results_dir_root, options.use_emerged) runner.Run() diff --git a/bin/cros_run_vm_test b/bin/cros_run_vm_test index cef3d115b9..fbbee23c7c 100755 --- a/bin/cros_run_vm_test +++ b/bin/cros_run_vm_test @@ -15,16 +15,17 @@ MAX_RETRIES=3 get_default_board DEFINE_string board "$DEFAULT_BOARD" \ - "The board for which you built autotest." + "The board for which you built autotest." b DEFINE_string image_path "" "Full path of the VM image" DEFINE_string results_dir_root "" "alternate root results directory" DEFINE_string test_case "" "Name of the test case to run" +DEFINE_boolean use_emerged ${FLAGS_FALSE} \ + "Force use of emerged autotest packages" set -e # Parse command line. FLAGS "$@" || exit 1 -eval set -- "${FLAGS_ARGV}" # Use latest if not specified. if [ -z "${FLAGS_image_path}" ]; then @@ -36,7 +37,25 @@ fi [ -e "${FLAGS_image_path}" ] || die "Image ${FLAGS_image_path} does not exist." -[ -n "${FLAGS_test_case}" ] || die "You must specify a test case." +if [ -n "${FLAGS_test_case}" ]; then + warn "Use of --test_case= is being deprecated. Just pass test names \ +as separate command line arguments." +fi + +if [ -z "${FLAGS_test_case}" ] && [ -z "${FLAGS_ARGV}" ]; then + die "You must specify a test case." +fi + +USE_EMERGED= +if [[ ${FLAGS_use_emerged} -eq ${FLAGS_TRUE} ]]; then + USE_EMERGED="--use_emerged" +fi + +tests=( ) +[ -n "${FLAGS_test_case}" ] && tests=( "${FLAGS_test_case}" ) +for test in ${FLAGS_ARGV}; do + tests=( "${tests[@]}" "$(remove_quotes "${test}")" ) +done trap stop_kvm EXIT start_kvm "${FLAGS_image_path}" @@ -47,4 +66,5 @@ retry_until_ssh ${MAX_RETRIES} --ssh_port=${FLAGS_ssh_port} \ --remote=127.0.0.1 \ --results_dir_root="${FLAGS_results_dir_root}" \ - "${FLAGS_test_case}" + ${USE_EMERGED} \ + "${tests[@]}" diff --git a/bin/cros_start_vm b/bin/cros_start_vm index 3ecced9f69..6307920349 100755 --- a/bin/cros_start_vm +++ b/bin/cros_start_vm @@ -10,6 +10,10 @@ . "$(dirname $0)/../lib/cros_vm_lib.sh" . "$(dirname "$0")/../lib/cros_vm_constants.sh" +get_default_board + +DEFINE_string board "${DEFAULT_BOARD}" \ + "Board for VM image (unnecessary if path given)" DEFINE_string image_path "" "Full path of the VM image" set -e @@ -20,7 +24,8 @@ eval set -- "${FLAGS_ARGV}" # Use latest if not specified. if [ -z "${FLAGS_image_path}" ]; then - LATEST_IMAGE="$(${SCRIPTS_DIR}/get_latest_image.sh)/${DEFAULT_QEMU_IMAGE}" + LATEST_IMAGE="$(${SCRIPTS_DIR}/get_latest_image.sh \ + --board=${FLAGS_board})/${DEFAULT_QEMU_IMAGE}" info "Using latest vm image ${LATEST_IMAGE}" FLAGS_image_path=${LATEST_IMAGE} fi diff --git a/bootperf-bin/__init__.py b/bootperf-bin/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bootperf-bin/bootperf b/bootperf-bin/bootperf new file mode 100755 index 0000000000..4aea1a8971 --- /dev/null +++ b/bootperf-bin/bootperf @@ -0,0 +1,213 @@ +#!/bin/bash + +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Wrapper to run the platform_BootPerfServer autotest, and store the +# results for later analysis by the 'showbootdata' script. +# +# NOTE: This script must be run from inside the chromeos build +# chroot environment. +# + +# SCRIPT_DIR="$(cd "$(dirname $0)/.." ; pwd)" +SCRIPT_DIR=$HOME/trunk/src/scripts +. "$SCRIPT_DIR/common.sh" + +DEFINE_string output_dir "" "output directory for results" o +DEFINE_boolean keep_logs "$FLAGS_FALSE" "keep autotest results" k + +RUN_TEST="$SCRIPT_DIR/run_remote_tests.sh" +TEST=server/site_tests/platform_BootPerfServer/control +TMP_RESULTS="/tmp/bootperf.$(date '+%Y%j%H%M').$$" +RESULTS_KEYVAL=platform_BootPerfServer/platform_BootPerfServer/results/keyval +RESULTS_SUMMARY_FILES=( + $RESULTS_KEYVAL + platform_BootPerfServer/keyval + platform_BootPerfServer/platform_BootPerfServer/keyval + platform_BootPerfServer/platform_BootPerfServer/platform_BootPerf/keyval + platform_BootPerfServer/platform_BootPerfServer/status + platform_BootPerfServer/platform_BootPerfServer/status.log + platform_BootPerfServer/status + platform_BootPerfServer/status.log + platform_BootPerfServer/sysinfo/cmdline + platform_BootPerfServer/sysinfo/cpuinfo + platform_BootPerfServer/sysinfo/modules + platform_BootPerfServer/sysinfo/uname + platform_BootPerfServer/sysinfo/version +) + +# Structure of a results directory: +# $RUNDIR.$ITER/ - directory +# $RUNDIR_LOG - file +# $RUNDIR_SUMMARY/ - directory +# $RUNDIR_ALL_RESULTS/ - optional directory +# $KEYVAL_SUMMARY/ - file +# If you add any other content under the results directory, you'll +# probably need to change extra_files(), below. +RUNDIR=run +RUNDIR_LOG=log.txt +RUNDIR_SUMMARY=summary +RUNDIR_ALL_RESULTS=logs +KEYVAL_SUMMARY=results_keyval + + +# Usage/help function. This function is known to the shflags library, +# and mustn't be renamed. +flags_help() { + cat <&2 +usage: $(basename $0) [ ] [ ] +Options: + --output_dir + --o Specify output directory for results + + --[no]keep_logs + -k Keep [don't keep] autotest log files +Summary: + Run the platform_BootPerfServer autotest, and store results in the + given destination directory. The test target is specified by + . + + By default, the test is run once; if is given, the test is + run that many times. Note that the platform_BootPerfServer test + reboots the target 10 times, so the total number of reboots will + be 10*. + + If the destination directory doesn't exist, it is created. If the + destination directory already holds test results, additional + results are added in without overwriting earlier results. + + If no destination is specified, the current directory is used, + provided that the directory is empty, or has been previously used + as a destination directory for this command. + + By default, only a summary subset of the log files created by + autotest is preserved; with --keep_logs the (potentially large) + autotest logs are preserved with the test results. +END_USAGE + return $FLAGS_TRUE +} + +usage() { + if [ $# -gt 0 ]; then + error "$(basename $0): $*" + echo >&2 + fi + flags_help + exit 1 +} + +# List any files in the current directory not created as output +# from running this script. +extra_files() { + ls | grep -v "^$RUNDIR[.]...\$" | + grep -v $KEYVAL_SUMMARY +} + +# Main function to run the boot performance test. Run the boot +# performance test for the given count, putting output into the +# current directory. +# +# Arguments are and arguments, as for the main +# command. +# +# We terminate test runs if "run_remote_tests" ever fails to produce +# the results file; generally this is the result of a serious error +# (e.g. disk full) that won't go away if we just plow on. +run_boot_test() { + local remote="$1" + local count="${2:-1}" + + local iter=$(expr "$(echo $RUNDIR.???)" : '.*\(...\)') + if [ "$iter" != "???" ]; then + iter=$(echo $iter | awk '{printf "%03d\n", $1 + 1}') + else + iter=000 + fi + + i=0 + while [ $i -lt $count ]; do + local iter_rundir=$RUNDIR.$iter + local logfile=$iter_rundir/$RUNDIR_LOG + local summary_dir=$iter_rundir/$RUNDIR_SUMMARY + local all_results_dir=$iter_rundir/$RUNDIR_ALL_RESULTS + + mkdir $iter_rundir + echo "run $iter start at $(date)" + $RUN_TEST --results_dir_root="$TMP_RESULTS" \ + --remote="$remote" $TEST >$logfile 2>&1 + if [ ! -e "$TMP_RESULTS/$RESULTS_KEYVAL" ]; then + error "No results file; terminating test runs." + error "Check $logfile for output from the test run," + error "and see $TMP_RESULTS for full test logs and output." + break + fi + mkdir $summary_dir + tar cf - -C $TMP_RESULTS "${RESULTS_SUMMARY_FILES[@]}" | + tar xf - -C $summary_dir + if [ $FLAGS_keep_logs -eq $FLAGS_TRUE ]; then + mv $TMP_RESULTS $all_results_dir + chmod 755 $all_results_dir + else + rm -rf $TMP_RESULTS + fi + i=$(expr $i + 1) + iter=$(echo $iter | awk '{printf "%03d\n", $1 + 1}') + done + # "run 000 start at $(date)" + echo " ... end at $(date)" + cat $RUNDIR.???/$RUNDIR_SUMMARY/$RESULTS_KEYVAL >$KEYVAL_SUMMARY +} + +# Main routine - check validity of the (already parsed) command line +# options. 'cd' to the results directory, if it was specified. If +# all the arguments checks pass, hand control to run_boot_test +main() { + if [ $# -lt 1 ]; then + usage "Missing target host address" + elif [ $# -gt 2 ]; then + usage "Too many arguments" + fi + + if [ -n "${FLAGS_output_dir}" ]; then + if [ ! -d "${FLAGS_output_dir}" ]; then + if ! mkdir "${FLAGS_output_dir}"; then + usage "Unable to create ${FLAGS_output_dir}" + fi + fi + cd "${FLAGS_output_dir}" || + usage "No permissions to chdir to ${FLAGS_output_dir}" + elif [ -n "$(extra_files)" ]; then + error "No results directory specified, and current directory" + error "contains contents other than run results." + error "You can override this error by using the --output_dir option" + usage + fi + + # Check the count argument. + # N.B. the test [ "$2" -eq "$2" ] tests whether "$2" is valid as a + # number; when it fails it will also report a syntax error (which + # we suppress). + if [ -n "$2" ]; then + if ! [ "$2" -eq "$2" ] 2>/dev/null || [ "$2" -le 0 ]; then + usage " argument must be a positive number" + fi + fi + + run_boot_test "$@" +} + +# shflags defines --help implicitly; if it's used on the command +# line FLAGS will invoke flags_help, set FLAGS_help to TRUE, and +# then return false. To avoid printing help twice, we have to check +# for that case here. +if ! FLAGS "$@"; then + if [ ${FLAGS_help} -eq ${FLAGS_TRUE} ]; then + exit 0 + else + usage + fi +fi + +eval main "${FLAGS_ARGV}" diff --git a/bootperf-bin/perfprinter.py b/bootperf-bin/perfprinter.py new file mode 100644 index 0000000000..d69637016e --- /dev/null +++ b/bootperf-bin/perfprinter.py @@ -0,0 +1,132 @@ +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Routines for printing boot time performance test results.""" + +import fnmatch +import os +import os.path +import re + +import resultset + + +_PERF_KEYVAL_PATTERN = re.compile("(.*){perf}=(.*)\n") + + +def ReadKeyvalFile(results, file_): + """Read an autotest keyval file, and process the results. + + The `file_` parameter is a file object with contents in autotest + perf keyval format: + {perf}= + + Each iteration of the test is terminated with a single blank line, + including the last iteration. Each iteration's results are added + to the `results` parameter, which should be an instance of + TestResultSet. + + """ + kvd = {} + for line in iter(file_): + if line == "\n": + results.AddIterationResults(kvd) + kvd = {} + continue + m = _PERF_KEYVAL_PATTERN.match(line) + if m is None: + continue + kvd[m.group(1)] = m.group(2) + + +_RESULTS_PATH = ( + "summary/platform_BootPerfServer/platform_BootPerfServer/results/keyval") + + +def ReadResultsDirectory(dir_): + """Process results from a 'bootperf' output directory. + + The accumulated results are returned in a newly created + TestResultSet object. + + """ + res_set = resultset.TestResultSet(dir_) + dirlist = fnmatch.filter(os.listdir(dir_), "run.???") + dirlist.sort() + for run in dirlist: + keyval_path = os.path.join(dir_, run, _RESULTS_PATH) + try: + kvf = open(keyval_path) + except IOError: + continue + ReadKeyvalFile(res_set, kvf) + res_set.FinalizeResults() + return res_set + + +def PrintRawData(dirlist, use_timestats, keylist): + """Print 'bootperf' results in "raw data" format.""" + for dir_ in dirlist: + if use_timestats: + keyset = ReadResultsDirectory(dir_).TimeKeySet() + else: + keyset = ReadResultsDirectory(dir_).DiskKeySet() + for i in range(0, keyset.num_iterations): + if len(dirlist) > 1: + line = "%s %3d" % (dir_, i) + else: + line = "%3d" % i + if keylist is not None: + markers = keylist + else: + markers = keyset.markers + for stat in markers: + (_, v) = keyset.PrintableStatistic(keyset.RawData(stat)[i]) + line += " %5s" % str(v) + print line + + +def PrintStatisticsSummary(dirlist, use_timestats, keylist): + """Print 'bootperf' results in "summary of averages" format.""" + if use_timestats: + header = "%5s %3s %5s %3s %s" % ( + "time", "s%", "dt", "s%", "event") + format = "%5s %2d%% %5s %2d%% %s" + else: + header = "%6s %3s %6s %3s %s" % ( + "diskrd", "s%", "delta", "s%", "event") + format = "%6s %2d%% %6s %2d%% %s" + havedata = False + for dir_ in dirlist: + if use_timestats: + keyset = ReadResultsDirectory(dir_).TimeKeySet() + else: + keyset = ReadResultsDirectory(dir_).DiskKeySet() + if keylist is not None: + markers = keylist + else: + markers = keyset.markers + if havedata: + print + if len(dirlist) > 1: + print "%s" % dir_, + print "(on %d cycles):" % keyset.num_iterations + print header + prevvalue = 0 + prevstat = None + for stat in markers: + (valueavg, valuedev) = keyset.Statistics(stat) + valuepct = int(100 * valuedev / valueavg + 0.5) + if prevstat: + (deltaavg, deltadev) = keyset.DeltaStatistics(prevstat, stat) + deltapct = int(100 * deltadev / deltaavg + 0.5) + else: + deltapct = valuepct + (valstring, val_printed) = keyset.PrintableStatistic(valueavg) + delta = val_printed - prevvalue + (deltastring, _) = keyset.PrintableStatistic(delta) + print format % (valstring, valuepct, "+" + deltastring, deltapct, stat) + prevvalue = val_printed + prevstat = stat + havedata = True diff --git a/bootperf-bin/resultset.py b/bootperf-bin/resultset.py new file mode 100644 index 0000000000..a1cee30b79 --- /dev/null +++ b/bootperf-bin/resultset.py @@ -0,0 +1,274 @@ +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Classes and functions for managing platform_BootPerf results. + +Results from the platform_BootPerf test in the ChromiumOS autotest +package are stored in as performance 'keyvals', that is, a mapping +of names to numeric values. For each iteration of the test, one +set of keyvals is recorded. + +This module currently tracks two kinds of keyval results, the boot +time results, and the disk read results. These results are stored +with keyval names such as 'seconds_kernel_to_login' and +'rdbytes_kernel_to_login'. Additionally, some older versions of the +test produced keyval names such as 'sectors_read_kernel_to_login'. +These keyvals record an accumulated total measured from a fixed +time in the past (kernel startup), e.g. 'seconds_kernel_to_login' +records the total seconds from kernel startup to login screen +ready. + +The boot time keyval names all start with the prefix +'seconds_kernel_to_', and record time in seconds since kernel +startup. + +The disk read keyval names all start with the prefix +'rdbytes_kernel_to_', and record bytes read from the boot device +since kernel startup. The obsolete disk keyvals start with the +prefix 'sectors_read_kernel_to_' and record the same statistic +measured in 512-byte sectors. + +Boot time and disk kevyal values have a consistent ordering +across iterations. For instance, if in one iteration the value of +'seconds_kernel_to_login' is greater than the value of +'seconds_kernel_to_x_started', then it will be greater in *all* +iterations. This property is a consequence of the underlying +measurement procedure; it is not enforced by this module. + +""" + +import math + + +def _ListStats(list_): + # Utility function - calculate the average and (sample) standard + # deviation of a list of numbers. Result is float, even if the + # input list is full of int's + sum_ = 0.0 + sumsq = 0.0 + for v in list_: + sum_ += v + sumsq += v * v + n = len(list_) + avg = sum_ / n + var = (sumsq - sum_ * avg) / (n - 1) + if var < 0.0: + var = 0.0 + dev = math.sqrt(var) + return (avg, dev) + + +def _DoCheck(dict_): + # Utility function - check the that all keyvals occur the same + # number of times. On success, return the number of occurrences; + # on failure return None + check = map(len, dict_.values()) + if not check: + return None + for i in range(1, len(check)): + if check[i] != check[i-1]: + return None + return check[0] + + +def _KeyDelta(dict_, key0, key1): + # Utility function - return a list of the vector difference between + # two keyvals. + return map(lambda a, b: b - a, dict_[key0], dict_[key1]) + + +class TestResultSet(object): + """A set of boot time and disk usage result statistics. + + Objects of this class consist of two sets of result statistics: + the boot time statistics and the disk statistics. + + Class TestResultsSet does not interpret or store keyval mappings + directly; iteration results are processed by attached _KeySet + objects, one for boot time (`_timekeys`), one for disk read + (`_diskkeys`). These attached _KeySet objects can be obtained + with appropriate methods; various methods on these objects will + calculate statistics on the results, and provide the raw data. + + """ + + def __init__(self, name): + self.name = name + self._timekeys = _TimeKeySet() + self._diskkeys = _DiskKeySet() + self._olddiskkeys = _OldDiskKeySet() + + def AddIterationResults(self, runkeys): + """Add keyval results from a single iteration. + + A TestResultSet is constructed by repeatedly calling + AddRunResults(), iteration by iteration. Iteration results are + passed in as a dictionary mapping keyval attributes to values. + When all iteration results have been added, FinalizeResults() + makes the results available for analysis. + + """ + + self._timekeys.AddRunResults(runkeys) + self._diskkeys.AddRunResults(runkeys) + self._olddiskkeys.AddRunResults(runkeys) + + def FinalizeResults(self): + """Make results available for analysis. + + A TestResultSet is constructed by repeatedly feeding it results, + iteration by iteration. Iteration results are passed in as a + dictionary mapping keyval attributes to values. When all iteration + results have been added, FinalizeResults() makes the results + available for analysis. + + """ + + self._timekeys.FinalizeResults() + if not self._diskkeys.FinalizeResults(): + self._olddiskkeys.FinalizeResults() + self._diskkeys = self._olddiskkeys + self._olddiskkeys = None + + def TimeKeySet(self): + """Return the boot time statistics result set.""" + return self._timekeys + + def DiskKeySet(self): + """Return the disk read statistics result set.""" + return self._diskkeys + + +class _KeySet(object): + """Container for a set of related statistics. + + _KeySet is an abstract superclass for containing collections of + either boot time or disk read statistics. Statistics are stored + as a dictionary (`_keyvals`) mapping keyval names to lists of + values. + + The mapped keyval names are shortened by stripping the prefix + that identifies the type of prefix (keyvals that don't start with + the proper prefix are ignored). So, for example, with boot time + keyvals, 'seconds_kernel_to_login' becomes 'login' (and + 'rdbytes_kernel_to_login' is ignored). + + A list of all valid keyval names is stored in the `markers` + instance variable. The list is sorted by the natural ordering of + the underlying values (see the module comments for more details). + + The list of values associated with a given keyval name are indexed + in the order in which they were added. So, all values for a given + iteration are stored at the same index. + + """ + + def __init__(self): + self._keyvals = {} + + def AddRunResults(self, runkeys): + """Add results for one iteration.""" + + for key, value in runkeys.iteritems(): + if not key.startswith(self.PREFIX): + continue + shortkey = key[len(self.PREFIX):] + keylist = self._keyvals.setdefault(shortkey, []) + keylist.append(self._ConvertVal(value)) + + def FinalizeResults(self): + """Finalize this object's results. + + This method makes available the `markers` and `num_iterations` + instance variables. It also ensures that every keyval occurred + in every iteration by requiring that all keyvals have the same + number of data points. + + """ + + count = _DoCheck(self._keyvals) + if count is None: + self.num_iterations = 0 + self.markers = [] + return False + self.num_iterations = count + keylist = map(lambda k: (self._keyvals[k][0], k), + self._keyvals.keys()) + keylist.sort(key=lambda tp: tp[0]) + self.markers = map(lambda tp: tp[1], keylist) + return True + + def RawData(self, key): + """Return the list of values for the given marker key.""" + return self._keyvals[key] + + def DeltaData(self, key0, key1): + """Return vector difference of the values of the given keys.""" + return _KeyDelta(self._keyvals, key0, key1) + + def Statistics(self, key): + """Return the average and standard deviation of the key's values.""" + return _ListStats(self._keyvals[key]) + + def DeltaStatistics(self, key0, key1): + """Return the average and standard deviation of the differences + between two keys. + + """ + + return _ListStats(self.DeltaData(key0, key1)) + + +class _TimeKeySet(_KeySet): + """Concrete subclass of _KeySet for boot time statistics.""" + + # TIME_KEY_PREFIX = 'seconds_kernel_to_' + PREFIX = 'seconds_kernel_to_' + + # Time-based keyvals are reported in seconds and get converted to + # milliseconds + TIME_SCALE = 1000 + + def _ConvertVal(self, value): + # We use a "round to nearest int" formula here to make sure we + # don't lose anything in the conversion from decimal. + return int(self.TIME_SCALE * float(value) + 0.5) + + def PrintableStatistic(self, value): + v = int(value + 0.5) + return ("%d" % v, v) + + +class _DiskKeySet(_KeySet): + """Concrete subclass of _KeySet for disk read statistics.""" + + PREFIX = 'rdbytes_kernel_to_' + + # Disk read keyvals are reported in bytes and get converted to + # MBytes (1 MByte = 1 million bytes, not 2**20) + DISK_SCALE = 1.0e-6 + + def _ConvertVal(self, value): + return self.DISK_SCALE * float(value) + + def PrintableStatistic(self, value): + v = round(value, 1) + return ("%.1fM" % v, v) + + +class _OldDiskKeySet(_DiskKeySet): + """Concrete subclass of _KeySet for the old-style disk read statistics.""" + + # Older versions of platform_BootPerf reported total sectors read + # using names of the form sectors_read_kernel_to_* (instead of the + # more recent rdbytes_kernel_to_*), but some of those names + # exceeded the 30-character limit in the MySQL database schema. + PREFIX = 'sectors_read_kernel_to_' + + # Old sytle disk read keyvals are reported in 512-byte sectors and + # get converted to MBytes (1 MByte = 1 million bytes, not 2**20) + SECTOR_SCALE = 512 * _DiskKeySet.DISK_SCALE + + def _ConvertVal(self, value): + return self.SECTOR_SCALE * float(value) diff --git a/bootperf-bin/showbootdata b/bootperf-bin/showbootdata new file mode 100755 index 0000000000..b5261d12bb --- /dev/null +++ b/bootperf-bin/showbootdata @@ -0,0 +1,116 @@ +#!/usr/bin/python + +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""A command to display summary statistics from runs of 'bootperf'. + +Command line options allow selecting from one of two sets of +performance statistics: The boot time statistics (selected by +--timestats) measure time spent from kernel startup in milliseconds. +The disk statistics (selected by --diskstats) measure total bytes +read from the boot device since kernel startup. + +Boot time statistics are recorded as cumulative time (or disk read) +since kernel startup, measured when specific events occur during +boot. Events include such things as 'startup', the moment when the +upstart 'startup' job begins running, and 'login', when the +Chrome OS login screen is displayed. By default, all recorded events +are included in the output; command line options allow restricting +the view to a selected subset of events. + +Separate command line options allow selecting from one of two +different display modes. When --averages is selected, the display +shows the average value and sample standard deviation (as a percent +of the average) for all selected events. The --averages display also +calculates the time (or bytes) between adjacent events, and shows +the avearage and sample standard deviation of the differences. + +The --rawdata display shows the raw data value associated with each +event for each boot: Each line of output represents the event values +for one boot cycle. + +""" + +import sys +import optparse + +import perfprinter + + +_usage = "%prog [options] [results-directory ...]" +_description = """\ +Summarize boot time performance results. The result directory +arguments are directories previously specified as output for the +'bootperf' script. +""" +optparser = optparse.OptionParser(usage=_usage, description=_description) + +optgroup = optparse.OptionGroup( + optparser, "Selecting boot time or disk statistics (choose one)") +optgroup.add_option( + "-d", "--diskstats", action="store_true", + dest="use_diskstats", + help="use statistics for bytes read since kernel startup") +optgroup.add_option( + "-t", "--timestats", action="store_true", + dest="use_timestats", + help="use statistics for time since kernel startup (default)") +optparser.add_option_group(optgroup) +optparser.set_defaults(use_diskstats=False) +optparser.set_defaults(use_timestats=False) + +optgroup = optparse.OptionGroup(optparser, "Event selection") +optgroup.add_option( + "-e", "--event", action="append", + dest="eventnames", + help="restrict statistics to the comma-separated list of events") +optparser.add_option_group(optgroup) + +optgroup = optparse.OptionGroup( + optparser, "Display mode selection (choose one)") +optgroup.add_option( + "-a", "--averages", action="store_true", + dest="print_averages", + help="display a summary of the averages of chosen statistics (default)") +optgroup.add_option( + "-r", "--rawdata", action="store_true", + dest="print_raw", + help="display raw data from all boot iterations") +optparser.add_option_group(optgroup) +optparser.set_defaults(print_averages=False) +optparser.set_defaults(print_raw=False) + + +def main(argv): + (options, args) = optparser.parse_args(argv) + if options.print_averages and options.print_raw: + print >>sys.stderr, "Can't use -a and -r together.\n" + optparser.print_help() + sys.exit(1) + elif options.print_raw: + printfunc = perfprinter.PrintRawData + else: + printfunc = perfprinter.PrintStatisticsSummary + if options.use_timestats and options.use_diskstats: + print >>sys.stderr, "Can't use -t and -d together.\n" + optparser.print_help() + sys.exit(1) + elif options.use_diskstats: + use_timestats = False + else: + use_timestats = True + if options.eventnames: + keylist = [] + for kl in options.eventnames: + keylist.extend(kl.split(',')) + else: + keylist = None + printfunc(args, use_timestats, keylist) + +if __name__ == "__main__": + if len(sys.argv) > 1: + main(sys.argv[1:]) + else: + main(["."]) diff --git a/build_image b/build_image index 19fdd4eed4..f727b7497f 100755 --- a/build_image +++ b/build_image @@ -72,8 +72,8 @@ DEFINE_string usb_disk /dev/sdb3 \ DEFINE_boolean enable_rootfs_verification ${FLAGS_TRUE} \ "Default all bootloaders to use kernel-based root fs integrity checking." DEFINE_integer verity_error_behavior 1 \ - "Kernel verified boot error behavior (0: I/O errors, 1: panic, 2: nothing) \ -Default: 1" + "Kernel verified boot error behavior (0: I/O errors, 1: panic, 2: nothing, \ +3: cros) Default: 1" DEFINE_integer verity_depth 1 \ "Kernel verified boot hash tree depth. Default: 1" DEFINE_integer verity_max_ios -1 \ diff --git a/chromeos_version.sh b/chromeos_version.sh index 1ff4478b12..649898719e 100755 --- a/chromeos_version.sh +++ b/chromeos_version.sh @@ -26,7 +26,7 @@ export CHROMEOS_VERSION_MINOR=9 # Increment by 2 in trunk after making a release branch. # Does not reset on a major/minor change (always increases). # (Trunk is always odd; branches are always even). -export CHROMEOS_VERSION_BRANCH=111 +export CHROMEOS_VERSION_BRANCH=113 # Patch number. # Increment by 1 each release on a branch. @@ -84,8 +84,8 @@ export CHROMEOS_VERSION_STRING=\ # Set CHROME values (Used for releases) to pass to chromeos-chrome-bin ebuild # URL to chrome archive export CHROME_BASE= -# directory containing chrome-chromeos.zip - an svn rev or a full version -export CHROME_BUILD= +# export CHROME_VERSION from incoming value or NULL and let ebuild default +export CHROME_VERSION="$CHROME_VERSION" # Print (and remember) version info. echo "ChromeOS version information:" diff --git a/chromite/bin/cros_changelog b/chromite/bin/cros_changelog index 73b5435451..f75ceab6a8 100755 --- a/chromite/bin/cros_changelog +++ b/chromite/bin/cros_changelog @@ -52,6 +52,7 @@ def _GrabOutput(cmd): def _GrabTags(): """Returns list of tags from current git repository.""" + # TODO(dianders): replace this with the python equivalent. cmd = ("git for-each-ref refs/tags | awk '{print $3}' | " "sed 's,refs/tags/,,g' | sort -t. -k3,3rn -k4,4rn") return _GrabOutput(cmd).split() @@ -129,7 +130,20 @@ class Commit(object): def __init__(self, commit, projectname, commit_email, commit_date, subject, body, tracker_acc): - """Create commit logs.""" + """Create commit logs. + + Args: + commit: The commit hash (sha) from git. + projectname: The project name, from: + git config --get remote.cros.projectname + commit_email: The email address associated with the commit (%ce in git + log) + commit_date: The date of the commit, like "Mon Nov 1 17:34:14 2010 -0500" + (%cd in git log)) + subject: The subject of the commit (%s in git log) + body: The body of the commit (%b in git log) + tracker_acc: A tracker_access.TrackerAccess object. + """ self.commit = commit self.projectname = projectname self.commit_email = commit_email @@ -149,7 +163,12 @@ class Commit(object): Returns: A list of Issue objects, each of which holds info about a bug. """ + # NOTE: most of this code is copied from bugdroid: + # + # Get a list of bugs. Handle lots of possibilities: + # - Multiple "BUG=" lines, with varying amounts of whitespace. + # - For each BUG= line, bugs can be split by commas _or_ by whitespace (!) entries = [] for line in self.body.split('\n'): match = re.match(r'^ *BUG *=(.*)', line) @@ -157,6 +176,13 @@ class Commit(object): for i in match.group(1).split(','): entries.extend(filter(None, [x.strip() for x in i.split()])) + # Try to parse the bugs. Handle lots of different formats: + # - The whole URL, from which we parse the project and bug. + # - A simple string that looks like "project:bug" + # - A string that looks like "bug", which will always refer to the previous + # tracker referenced (defaulting to the default tracker). + # + # We will create an "Issue" object for each bug. issues = [] last_tracker = DEFAULT_TRACKER regex = (r'http://code.google.com/p/(\S+)/issues/detail\?id=([0-9]+)' @@ -174,6 +200,7 @@ class Commit(object): elif bug_tuple[4]: issues.append(Issue(last_tracker, bug_tuple[4], self._tracker_acc)) + # Sort the issues and return... issues.sort() return issues @@ -205,12 +232,12 @@ class Commit(object): bug_str = 'none' cols = [ - cgi.escape(self.projectname), - str(self.commit_date), - commit_desc, - cgi.escape(self.commit_email), - bug_str, - cgi.escape(self.subject[:100]), + cgi.escape(self.projectname), + str(self.commit_date), + commit_desc, + cgi.escape(self.commit_email), + bug_str, + cgi.escape(self.subject[:100]), ] return '%s' % (''.join(cols)) @@ -221,7 +248,17 @@ class Commit(object): def _GrabChanges(path, tag1, tag2, tracker_acc): - """Return list of commits to path between tag1 and tag2.""" + """Return list of commits to path between tag1 and tag2. + + Args: + path: One of the directories managed by repo. + tag1: The first of the two tags to pass to git log. + tag2: The second of the two tags to pass to git log. + tracker_acc: A tracker_access.TrackerAccess object. + + Returns: + A list of "Commit" objects. + """ cmd = 'cd %s && git config --get remote.cros.projectname' % path projectname = _GrabOutput(cmd).strip() @@ -239,19 +276,24 @@ def _GrabChanges(path, tag1, tag2, tracker_acc): def _ParseArgs(): + """Parse command-line arguments. + + Returns: + An optparse.OptionParser object. + """ parser = optparse.OptionParser() parser.add_option( - "--sort-by-date", dest="sort_by_date", default=False, - action='store_true', help="Sort commits by date.") + '--sort-by-date', dest='sort_by_date', default=False, + action='store_true', help='Sort commits by date.') parser.add_option( - "--tracker-user", dest="tracker_user", default=None, - help="Specify a username to login to code.google.com.") + '--tracker-user', dest='tracker_user', default=None, + help='Specify a username to login to code.google.com.') parser.add_option( - "--tracker-pass", dest="tracker_pass", default=None, - help="Specify a password to go w/ user.") + '--tracker-pass', dest='tracker_pass', default=None, + help='Specify a password to go w/ user.') parser.add_option( - "--tracker-passfile", dest="tracker_passfile", default=None, - help="Specify a file containing a password to go w/ user.") + '--tracker-passfile', dest='tracker_passfile', default=None, + help='Specify a file containing a password to go w/ user.') return parser.parse_args() @@ -264,7 +306,11 @@ def main(): elif len(args) == 1: tag2, = args if tag2 in tags: - tag1 = tags[tags.index(tag2) + 1] + tag2_index = tags.index(tag2) + if tag2_index == len(tags) - 1: + print >>sys.stderr, 'No previous tag for %s' % tag2 + sys.exit(1) + tag1 = tags[tag2_index + 1] else: print >>sys.stderr, 'Unrecognized tag: %s' % tag2 sys.exit(1) @@ -287,7 +333,7 @@ def main(): print >>sys.stderr, INSTRS_FOR_GDATA sys.exit(1) if options.tracker_passfile is not None: - options.tracker_pass = open(options.tracker_passfile, "r").read().strip() + options.tracker_pass = open(options.tracker_passfile, 'r').read().strip() tracker_acc = tracker_access.TrackerAccess(options.tracker_user, options.tracker_pass) else: @@ -307,7 +353,7 @@ def main(): print '' print '' % ('
%s'.join(cols)) if options.sort_by_date: - changes.sort(key=operator.attrgetter("commit_date")) + changes.sort(key=operator.attrgetter('commit_date')) else: changes.sort() for change in changes: diff --git a/common.sh b/common.sh index 25abd9b4c3..b7d428dab5 100644 --- a/common.sh +++ b/common.sh @@ -104,7 +104,9 @@ DEFAULT_CHROOT_DIR=${CHROMEOS_CHROOT_DIR:-"$GCLIENT_ROOT/chroot"} DEFAULT_BUILD_ROOT=${CHROMEOS_BUILD_ROOT:-"$SRC_ROOT/build"} # Set up a global ALL_BOARDS value -ALL_BOARDS=$(cd $SRC_ROOT/overlays;ls -1d overlay-* 2>&-|sed 's,overlay-,,g') +if [ -d $SRC_ROOT/overlays ]; then + ALL_BOARDS=$(cd $SRC_ROOT/overlays;ls -1d overlay-* 2>&-|sed 's,overlay-,,g') +fi # Strip CR ALL_BOARDS=$(echo $ALL_BOARDS) # Set a default BOARD @@ -129,6 +131,7 @@ fi CHROOT_TRUNK_DIR="/home/$USER/trunk" # Install make for portage ebuilds. Used by build_image and gmergefs. +# TODO: Is /usr/local/autotest-chrome still used by anyone? DEFAULT_INSTALL_MASK="/usr/include /usr/man /usr/share/man /usr/share/doc \ /usr/share/gtk-doc /usr/share/gtk-2.0 /usr/lib/gtk-2.0/include \ /usr/share/info /usr/share/aclocal /usr/lib/gcc /usr/lib/pkgconfig \ @@ -143,7 +146,7 @@ FACTORY_INSTALL_MASK="/opt/google/chrome /opt/google/o3d /opt/netscape \ /usr/share/ibus-pinyin /usr/share/libhangul /usr/share/locale \ /usr/share/m17n /usr/share/mime /usr/share/sounds /usr/share/tts \ /usr/share/X11 /usr/share/zoneinfo /usr/lib/debug - /usr/local/autotest /usr/local/autotest-chrome" + /usr/local/autotest /usr/local/autotest-chrome /usr/local/autotest-pkgs" # Check to ensure not running old scripts V_REVERSE='' diff --git a/create_legacy_bootloader_templates.sh b/create_legacy_bootloader_templates.sh index 955da4826d..8bfa761617 100755 --- a/create_legacy_bootloader_templates.sh +++ b/create_legacy_bootloader_templates.sh @@ -191,11 +191,11 @@ menuentry "local image B" { } menuentry "verified image A" { - linux \$grubpartA/boot/vmlinuz ${common_args} ${verity_common} i915.modeset=1 cros_efi root=/dev/dm-0 dm="DMTABLEA" + linux \$grubpartA/boot/vmlinuz ${common_args} ${verity_common} i915.modeset=1 cros_efi root=/dev/dm-0 dm=\\"DMTABLEA\\" } menuentry "verified image B" { - linux \$grubpartB/boot/vmlinuz ${common_args} ${verity_common} i915.modeset=1 cros_efi root=/dev/dm-0 dm="DMTABLEB" + linux \$grubpartB/boot/vmlinuz ${common_args} ${verity_common} i915.modeset=1 cros_efi root=/dev/dm-0 dm=\\"DMTABLEB\\" } # FIXME: usb doesn't support verified boot for now diff --git a/cros_generate_stateful_update_payload b/cros_generate_stateful_update_payload new file mode 120000 index 0000000000..05cb0081bc --- /dev/null +++ b/cros_generate_stateful_update_payload @@ -0,0 +1 @@ +cros_generate_stateful_update_payload.py \ No newline at end of file diff --git a/cros_generate_stateful_update_payload.py b/cros_generate_stateful_update_payload.py new file mode 100755 index 0000000000..a2c170a84e --- /dev/null +++ b/cros_generate_stateful_update_payload.py @@ -0,0 +1,95 @@ +#!/usr/bin/python2.6 + +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""This module is responsible for generate a stateful update payload.""" + +import logging +import optparse +import os +import subprocess +import tempfile + +STATEFUL_FILE = 'stateful.tgz' + + +def GenerateStatefulPayload(image_path, output_directory, logger): + """Generates a stateful update payload given a full path to an image. + + Args: + image_path: Full path to the image. + output_directory: Path to the directory to leave the resulting output. + logger: logging instance. + """ + logger.info('Generating stateful update file.') + from_dir = os.path.dirname(image_path) + image = os.path.basename(image_path) + output_gz = os.path.join(output_directory, STATEFUL_FILE) + crosutils_dir = os.path.dirname(__file__) + + # Temporary directories for this function. + rootfs_dir = tempfile.mkdtemp(suffix='rootfs', prefix='tmp') + stateful_dir = tempfile.mkdtemp(suffix='stateful', prefix='tmp') + + # Mount the image to pull out the important directories. + try: + # Only need stateful partition, but this saves us having to manage our + # own loopback device. + subprocess.check_call(['%s/mount_gpt_image.sh' % crosutils_dir, + '--from=%s' % from_dir, + '--image=%s' % image, + '--read_only', + '--rootfs_mountpt=%s' % rootfs_dir, + '--stateful_mountpt=%s' % stateful_dir, + ]) + logger.info('Tarring up /usr/local and /var!') + subprocess.check_call(['sudo', + 'tar', + '-czf', + output_gz, + '--directory=%s' % stateful_dir, + '--hard-dereference', + '--transform=s,^dev_image,dev_image_new,', + '--transform=s,^var,var_new,', + 'dev_image', + 'var', + ]) + except: + logger.error('Failed to create stateful update file') + raise + finally: + # Unmount best effort regardless. + subprocess.call(['%s/mount_gpt_image.sh' % crosutils_dir, + '--unmount', + '--rootfs_mountpt=%s' % rootfs_dir, + '--stateful_mountpt=%s' % stateful_dir, + ]) + # Clean up our directories. + os.rmdir(rootfs_dir) + os.rmdir(stateful_dir) + + logger.info('Successfully generated %s' % output_gz) + + +def main(): + logging.basicConfig(level=logging.INFO) + logger = logging.getLogger(os.path.basename(__file__)) + parser = optparse.OptionParser() + parser.add_option('-i', '--image_path', + help='The image to generate the stateful update for.') + parser.add_option('-o', '--output_dir', + help='The path to the directory to output the update file.') + options, unused_args = parser.parse_args() + if not options.image_path: + parser.error('Missing image for stateful payload generator') + if not options.output_dir: + parser.error('Missing output directory for the payload generator') + + GenerateStatefulPayload(os.path.abspath(options.image_path), + options.output_dir, logger) + + +if __name__ == '__main__': + main() diff --git a/cros_generate_update_payload b/cros_generate_update_payload index dd0a67c06e..3b5b9f9d6d 100755 --- a/cros_generate_update_payload +++ b/cros_generate_update_payload @@ -87,7 +87,7 @@ patch_kernel() { offset=$(($offset * 512)) sudo losetup -o "$offset" "$STATE_LOOP_DEV" "$IMAGE" STATE_MNT=$(mktemp -d /tmp/state.XXXXXX) - sudo mount "$STATE_LOOP_DEV" "$STATE_MNT" + sudo mount --read-only "$STATE_LOOP_DEV" "$STATE_MNT" dd if="$STATE_MNT"/vmlinuz_hd.vblock of="$KERN_FILE" conv=notrunc sudo umount "$STATE_MNT" STATE_MNT="" diff --git a/cros_mark_as_stable.py b/cros_mark_as_stable.py index 9a3ab5221c..56a589c682 100755 --- a/cros_mark_as_stable.py +++ b/cros_mark_as_stable.py @@ -93,8 +93,12 @@ def _BestEBuild(ebuilds): return winner -def _FindStableEBuilds(files): - """Return a list of stable ebuilds from specified list of files. +def _FindUprevCandidates(files): + """Return a list of uprev candidates from specified list of files. + + Usually an uprev candidate is a the stable ebuild in a cros_workon directory. + However, if no such stable ebuild exists (someone just checked in the 9999 + ebuild), this is the unstable ebuild. Args: files: List of files. @@ -131,7 +135,8 @@ def _FindStableEBuilds(files): if not unstable_ebuilds: Die('Missing 9999 ebuild in %s' % os.path.dirname(path)) if not stable_ebuilds: - Die('Missing stable ebuild in %s' % os.path.dirname(path)) + Warning('Missing stable ebuild in %s' % os.path.dirname(path)) + return unstable_ebuilds[0] if stable_ebuilds: return stable_ebuilds[0] @@ -153,7 +158,7 @@ def _BuildEBuildDictionary(overlays, all, packages): for package_dir, dirs, files in os.walk(overlay): # Add stable ebuilds to overlays[overlay]. paths = [os.path.join(package_dir, path) for path in files] - ebuild = _FindStableEBuilds(paths) + ebuild = _FindUprevCandidates(paths) # If the --all option isn't used, we only want to update packages that # are in packages. @@ -377,8 +382,8 @@ class _EBuild(object): else: # Has no revision so we stripped the version number instead. ebuild_no_version = ebuild_no_rev - ebuild_no_rev = ebuild_path.rpartition('.ebuild')[0] - rev_string = "0" + ebuild_no_rev = ebuild_path.rpartition('9999.ebuild')[0] + '0.0.1' + rev_string = '0' revision = int(rev_string) return (ebuild_no_rev, ebuild_no_version, revision) @@ -448,8 +453,10 @@ class EBuildStableMarker(object): _Print('Adding new stable ebuild to git') _SimpleRunCommand('git add %s' % new_ebuild_path) - _Print('Removing old ebuild from git') - _SimpleRunCommand('git rm %s' % old_ebuild_path) + if self._ebuild.is_stable: + _Print('Removing old ebuild from git') + _SimpleRunCommand('git rm %s' % old_ebuild_path) + return True def CommitChange(self, message): diff --git a/cros_mark_as_stable_unittest.py b/cros_mark_as_stable_unittest.py index 73985ae4d2..e7fd3317b2 100755 --- a/cros_mark_as_stable_unittest.py +++ b/cros_mark_as_stable_unittest.py @@ -125,7 +125,7 @@ class EBuildTest(mox.MoxTestBase): def testParseEBuildPathNoRevisionNumber(self): # Test with ebuild without revision number. no_rev, no_version, revision = cros_mark_as_stable._EBuild._ParseEBuildPath( - '/path/test_package-0.0.1.ebuild') + '/path/test_package-9999.ebuild') self.assertEquals(no_rev, '/path/test_package-0.0.1') self.assertEquals(no_version, '/path/test_package') self.assertEquals(revision, 0) @@ -139,6 +139,7 @@ class EBuildStableMarkerTest(mox.MoxTestBase): self.mox.StubOutWithMock(cros_mark_as_stable, 'RunCommand') self.mox.StubOutWithMock(os, 'unlink') self.m_ebuild = self.mox.CreateMock(cros_mark_as_stable._EBuild) + self.m_ebuild.is_stable = True self.m_ebuild.package = 'test_package' self.m_ebuild.current_revision = 1 self.m_ebuild.ebuild_path_no_revision = '/path/test_package-0.0.1' @@ -281,13 +282,13 @@ class BuildEBuildDictionaryTest(mox.MoxTestBase): self.package_path = self.root + '/test_package-0.0.1.ebuild' paths = [[self.root, [], []]] cros_mark_as_stable.os.walk("/overlay").AndReturn(paths) - self.mox.StubOutWithMock(cros_mark_as_stable, '_FindStableEBuilds') + self.mox.StubOutWithMock(cros_mark_as_stable, '_FindUprevCandidates') def testWantedPackage(self): overlays = {"/overlay": []} package = _Package(self.package) - cros_mark_as_stable._FindStableEBuilds([]).AndReturn(package) + cros_mark_as_stable._FindUprevCandidates([]).AndReturn(package) self.mox.ReplayAll() cros_mark_as_stable._BuildEBuildDictionary(overlays, False, [self.package]) self.mox.VerifyAll() @@ -297,7 +298,7 @@ class BuildEBuildDictionaryTest(mox.MoxTestBase): def testUnwantedPackage(self): overlays = {"/overlay": []} package = _Package(self.package) - cros_mark_as_stable._FindStableEBuilds([]).AndReturn(package) + cros_mark_as_stable._FindUprevCandidates([]).AndReturn(package) self.mox.ReplayAll() cros_mark_as_stable._BuildEBuildDictionary(overlays, False, []) self.assertEquals(len(overlays), 1) diff --git a/cros_show_stacks b/cros_show_stacks index c5bc02cc64..fa6b769321 100755 --- a/cros_show_stacks +++ b/cros_show_stacks @@ -12,7 +12,7 @@ . "$(dirname $0)/common.sh" . "$(dirname $0)/remote_access.sh" -restart_in_chroot_if_needed $* +assert_inside_chroot MINIDUMP_DUMP=/usr/bin/minidump_dump MINIDUMP_STACKWALK=/usr/bin/minidump_stackwalk diff --git a/generate_au_zip.py b/generate_au_zip.py new file mode 100755 index 0000000000..f38e1e3aad --- /dev/null +++ b/generate_au_zip.py @@ -0,0 +1,263 @@ +#!/usr/bin/python + +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" + Script to generate a zip file of delta-generator and it's dependencies. +""" +import logging.handlers +import optparse +import os +import re +import shutil +import subprocess +import sys +import tempfile + +# GLOBALS + +logging_format = '%(asctime)s - %(filename)s - %(levelname)-8s: %(message)s' +date_format = '%Y/%m/%d %H:%M:%S' +logging.basicConfig(level=logging.INFO, format=logging_format, + datefmt=date_format) + +def CreateTempDir(): + """Creates a tempdir and returns the name of the tempdir.""" + temp_dir = tempfile.mkdtemp(suffix='au', prefix='tmp') + logging.info('Using tempdir = %s', temp_dir) + return temp_dir + + +def _SplitAndStrip(data): + """Prunes the ldd output, and return a list of needed library names + Example of data: + linux-vdso.so.1 => (0x00007ffffc96a000) + libbz2.so.1 => /lib/libbz2.so.1 (0x00007f3ff8782000) + libc.so.6 => /lib/libc.so.6 (0x00007f3ff83ff000) + /lib64/ld-linux-x86-64.so.2 (0x00007f3ff89b3000) + Args: + data: list of libraries from ldd output + Returns: + list of libararies that we should copy + + """ + return_list = [] + for line in data.split('\n'): + line = re.sub('.*not a dynamic executable.*', '', line) + line = re.sub('.* =>\s+', '', line) + line = re.sub('\(0x.*\)\s?', '', line) + line = line.strip() + if not len(line): + continue + logging.debug('MATCHED line = %s', line) + return_list.append(line) + + return return_list + + +def DepsToCopy(ldd_files, black_list): + """Returns a list of deps for a given dynamic executables list. + Args: + ldd_files: List of dynamic files that needs to have the deps evaluated + black_list: List of files that we should ignore + Returns: + library_list: List of files that are dependencies + """ + for file_name in ldd_files: + logging.info('Running ldd on %s', file_name) + cmd = ['/usr/bin/ldd', file_name] + stdout_data = '' + stderr_data = '' + + try: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + (stdout_data, stderr_data) = proc.communicate(input=None) + except subprocess.CalledProcessError, e: + logging.error('Command %s failed', cmd) + logging.error('error code %s', e.returncode) + logging.error('ouput %s', e.output) + raise + + library_list = [] + if not stdout_data: + return library_list + + logging.debug('ldd for %s = stdout = %s stderr =%s', file_name, + stdout_data, stderr_data) + + library_list = _SplitAndStrip(stdout_data) + + return _ExcludeBlacklist(library_list, black_list) + + +def CopyRequiredFiles(dest_files_root): + """Generates a list of files that are required for au-generator zip file + Args: + dest_files_root: location of the directory where we should copy the files + """ + if not dest_files_root: + logging.error('Invalid option passed for dest_files_root') + sys.exit(1) + # Files that need to go through ldd + ldd_files = ['/usr/bin/delta_generator', '/usr/bin/bsdiff', + '/usr/bin/bspatch', '/usr/bin/cgpt'] + # statically linked files and scripts etc., + static_files = ['~/trunk/src/scripts/common.sh', + '~/trunk/src/scripts/cros_generate_update_payload', + '~/trunk/src/scripts/chromeos-common.sh'] + # We need directories to be copied recursively to a destination within tempdir + recurse_dirs = {'~/trunk/src/scripts/lib/shflags': 'lib/shflags'} + + black_list = [ + 'linux-vdso.so', + 'libgcc_s.so', + 'libgthread-2.0.so', + 'libpthread.so', + 'librt.so', + 'libstdc', + 'libgcc_s.so', + 'libc.so', + 'ld-linux-x86-64', + 'libm.so', + 'libdl.so', + 'libresolv.so', + ] + + all_files = ldd_files + static_files + all_files = map(os.path.expanduser, all_files) + + for file_name in all_files: + if not os.path.isfile(file_name): + logging.error('file = %s does not exist', file_name) + sys.exit(1) + + logging.debug('Given files that need to be copied = %s' % '' .join(all_files)) + all_files += DepsToCopy(ldd_files=ldd_files,black_list=black_list) + for file_name in all_files: + logging.info('Copying file %s to %s', file_name, dest_files_root) + shutil.copy2(file_name, dest_files_root) + + for source_dir, target_dir in recurse_dirs.iteritems(): + logging.info('Processing directory %s', source_dir) + full_path = os.path.expanduser(source_dir) + if not os.path.isdir(full_path): + logging.error("Directory given for %s expanded to %s doens't exist.", + source_dir, full_path) + sys.exit(1) + dest = os.path.join(dest_files_root, target_dir) + logging.info('Copying directory %s to %s.', full_path, target_dir) + shutil.copytree(full_path, dest) + +def CleanUp(temp_dir): + """Cleans up the tempdir + Args: + temp_dir = name of the directory to cleanup + """ + if os.path.exists(temp_dir): + shutil.rmtree(temp_dir, ignore_errors=True) + logging.info('Removed tempdir = %s', temp_dir) + +def GenerateZipFile(base_name, root_dir): + """Returns true if able to generate zip file + Args: + base_name: name of the zip file + root_dir: location of the directory that we should zip + Returns: + True if successfully generates the zip file otherwise False + """ + logging.info('Generating zip file %s with contents from %s', base_name, + root_dir) + current_dir = os.getcwd() + os.chdir(root_dir) + try: + subprocess.Popen(['zip', '-r', '-9', base_name, '.'], + stdout=subprocess.PIPE).communicate()[0] + except OSError, e: + logging.error('Execution failed:%s', e.strerror) + return False + finally: + os.chdir(current_dir) + + return True + + +def _ExcludeBlacklist(library_list, black_list=[]): + """Deletes the set of files from black_list from the library_list + Args: + library_list: List of the library names to filter through black_list + black_list: List of the black listed names to filter + Returns: + Filtered library_list + """ + return_list = [] + pattern = re.compile(r'|'.join(black_list)) + + logging.debug('PATTERN: %s=', pattern) + + for library in library_list: + if pattern.search(library): + logging.debug('BLACK-LISTED = %s=', library) + continue + return_list.append(library) + + logging.debug('Returning return_list=%s=', return_list) + + return return_list + + +def CopyZipToFinalDestination(output_dir, zip_file_name): + """Copies the generated zip file to a final destination + Args: + output_dir: Directory where the file should be copied to + zip_file_name: name of the zip file that should be copied + Returns: + True on Success False on Failure + """ + if not os.path.isfile(zip_file_name): + logging.error("Zip file %s doesn't exist. Returning False", zip_file_name) + return False + + if not os.path.isdir(output_dir): + logging.debug('Creating %s', output_dir) + os.makedirs(output_dir) + logging.info('Copying %s to %s', zip_file_name, output_dir) + shutil.copy2(zip_file_name, output_dir) + return True + + +def main(): + """Main function to start the script""" + parser = optparse.OptionParser() + + parser.add_option( '-d', '--debug', dest='debug', action='store_true', + default=False, help='Verbose Default: False',) + parser.add_option('-o', '--output-dir', dest='output_dir', + default='/tmp/au-generator', + help='Specify the output location for copying the zipfile') + parser.add_option('-z', '--zip-name', dest='zip_name', + default='au-generator.zip', help='Name of the zip file') + parser.add_option('-k', '--keep-temp', dest='keep_temp', default=False, + action='store_true', help='Keep the temp files...',) + + (options, args) = parser.parse_args() + if options.debug: + logging.setLevel(logging.DEBUG) + + logging.debug('Options are %s ', options) + + temp_dir = CreateTempDir() + dest_files_root = os.path.join(temp_dir, 'au-generator') + os.makedirs(dest_files_root) + CopyRequiredFiles(dest_files_root=dest_files_root) + zip_file_name = os.path.join(temp_dir, options.zip_name) + GenerateZipFile(zip_file_name, dest_files_root) + CopyZipToFinalDestination(options.output_dir, zip_file_name) + + if not options.keep_temp: + CleanUp(temp_dir) + +if __name__ == '__main__': + main() diff --git a/image_to_live.sh b/image_to_live.sh index b27a87db54..88d75b9659 100755 --- a/image_to_live.sh +++ b/image_to_live.sh @@ -30,6 +30,7 @@ DEFINE_boolean verify ${FLAGS_TRUE} "Verify image on device after update." # Flags for devserver. DEFINE_string archive_dir "" \ "Update using the test image in the image.zip in this directory." a +DEFINE_string board "" "Override the board reported by the target" DEFINE_integer devserver_port 8080 \ "Port to use for devserver." DEFINE_boolean for_vm ${FLAGS_FALSE} "Image is for a vm." @@ -124,10 +125,11 @@ function start_dev_server { else # IMAGE_PATH should be the newest image and learn the board from # the target. - FLAGS_board="" learn_board IMAGE_PATH="$($(dirname "$0")/get_latest_image.sh --board="${FLAGS_board}")" IMAGE_PATH="${IMAGE_PATH}/chromiumos_image.bin" + devserver_flags="${devserver_flags} \ + --image $(reinterpret_path_for_chroot ${IMAGE_PATH})" fi [ ${FLAGS_for_vm} -eq ${FLAGS_TRUE} ] && \ @@ -139,6 +141,7 @@ function start_dev_server { info "Starting devserver with flags ${devserver_flags}" ./enter_chroot.sh "sudo ./start_devserver ${devserver_flags} \ --client_prefix=ChromeOSUpdateEngine \ + --board=${FLAGS_board} \ --port=${FLAGS_devserver_port} > ${FLAGS_server_log} 2>&1" & info "Waiting on devserver to start" @@ -147,6 +150,10 @@ function start_dev_server { do sleep 5 echo -n "." + if ! pgrep -f start_devserver > /dev/null; then + echo "Devserver failed, see dev_server.log." + exit 1 + fi done echo "" } diff --git a/image_to_vm.sh b/image_to_vm.sh index fdf23a413c..50e00e5612 100755 --- a/image_to_vm.sh +++ b/image_to_vm.sh @@ -37,7 +37,7 @@ DEFINE_integer rootfs_partition_size 1024 \ "rootfs parition size in MBs." DEFINE_string state_image "" \ "Stateful partition image (defaults to creating new statful partition)" -DEFINE_integer statefulfs_size -1 \ +DEFINE_integer statefulfs_size 2048 \ "Stateful partition size in MBs." DEFINE_boolean test_image "${FLAGS_FALSE}" \ "Copies normal image to chromiumos_test_image.bin, modifies it for test." @@ -285,6 +285,6 @@ fi if [ "${FLAGS_format}" == "qemu" ]; then echo "If you have qemu-kvm installed, you can start the image by:" echo "sudo kvm -m ${FLAGS_mem} -vga std -pidfile /tmp/kvm.pid -net nic,model=e1000 " \ - "-net user,hostfwd=tcp::922-:22 \\" + "-net user,hostfwd=tcp::9222-:22 \\" echo " -hda ${FLAGS_to}/${DEFAULT_QEMU_IMAGE}" fi diff --git a/lib/cros_image_common.sh b/lib/cros_image_common.sh new file mode 100644 index 0000000000..0ad9d0bbcd --- /dev/null +++ b/lib/cros_image_common.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +# Copyright (c) 2009 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# This script contains common utility function to deal with disk images, +# especially for being redistributed into platforms without complete Chromium OS +# developing environment. + +# Check if given command is available in current system +has_command() { + type "$1" >/dev/null 2>&1 +} + +err_die() { + echo "ERROR: $@" >&2 + exit 1 +} + +# Finds the best gzip compressor and invoke it. +gzip_compress() { + if has_command pigz; then + # echo " ** Using parallel gzip **" >&2 + # Tested with -b 32, 64, 128(default), 256, 1024, 16384, and -b 32 (max + # window size of Deflate) seems to be the best in output size. + pigz -b 32 "$@" + else + gzip "$@" + fi +} + +# Finds if current system has tools for part_* commands +has_part_tools() { + has_command cgpt || has_command parted +} + +# Finds the best partition tool and print partition offset +part_offset() { + local file="$1" + local partno="$2" + + if has_command cgpt; then + cgpt show -b -i "$partno" "$file" + elif has_command parted; then + parted -m "$file" unit s print | + grep "^$partno:" | cut -d ':' -f 2 | sed 's/s$//' + else + exit 1 + fi +} + +# Finds the best partition tool and print partition size +part_size() { + local file="$1" + local partno="$2" + + if has_command cgpt; then + cgpt show -s -i "$partno" "$file" + elif has_command parted; then + parted -m "$file" unit s print | + grep "^$partno:" | cut -d ':' -f 4 | sed 's/s$//' + else + exit 1 + fi +} + +# Dumps a file by given offset and size (in sectors) +dump_partial_file() { + local file="$1" + local offset="$2" + local sectors="$3" + local bs=512 + + # Try to use larger buffer if offset/size can be re-aligned. + # 2M / 512 = 4096 + local buffer_ratio=4096 + if [ $((offset % buffer_ratio)) -eq 0 -a \ + $((sectors % buffer_ratio)) -eq 0 ]; then + offset=$((offset / buffer_ratio)) + sectors=$((sectors / buffer_ratio)) + bs=$((bs * buffer_ratio)) + fi + + if has_command pv; then + dd if="$file" bs=$bs skip="$offset" count="$sectors" \ + oflag=sync status=noxfer 2>/dev/null | + pv -ptreb -B 4m -s $((sectors * $bs)) + else + dd if="$file" bs=$bs skip="$offset" count="$sectors" \ + oflag=sync status=noxfer 2>/dev/null + fi +} + +# Dumps a specific partition from given image file +dump_partition() { + local file="$1" + local part_num="$2" + local offset="$(part_offset "$file" "$part_num")" || + err_die "failed to dump partition #$part_num from: $file" + local size="$(part_size "$file" "$part_num")" || + err_die "failed to dump partition #$part_num from: $file" + + dump_partial_file "$file" "$offset" "$size" +} + diff --git a/lib/cros_vm_lib.sh b/lib/cros_vm_lib.sh index 1809acc9d1..7a99cbebe2 100644 --- a/lib/cros_vm_lib.sh +++ b/lib/cros_vm_lib.sh @@ -10,6 +10,7 @@ DEFINE_boolean no_graphics ${FLAGS_FALSE} "Runs the KVM instance silently." DEFINE_boolean persist "${FLAGS_FALSE}" "Persist vm." DEFINE_boolean snapshot ${FLAGS_FALSE} "Don't commit changes to image." DEFINE_integer ssh_port 9222 "Port to tunnel ssh traffic over." +DEFINE_string vnc "" "VNC Server to display to instead of SDL." KVM_PID_FILE=/tmp/kvm.$$.pid LIVE_VM_IMAGE= @@ -40,7 +41,10 @@ function start_kvm() { local nographics="" local usesnapshot="" if [ ${FLAGS_no_graphics} -eq ${FLAGS_TRUE} ]; then - nographics="-nographic" + nographics="-nographic -serial none" + fi + if [ -n "${FLAGS_vnc}" ]; then + nographics="-vnc ${FLAGS_vnc}" fi if [ ${FLAGS_snapshot} -eq ${FLAGS_TRUE} ]; then diff --git a/make_factory_package.sh b/make_factory_package.sh index 8f238fb5d2..a62cfe2e3c 100755 --- a/make_factory_package.sh +++ b/make_factory_package.sh @@ -18,6 +18,10 @@ # Load functions and constants for chromeos-install . "$(dirname "$0")/chromeos-common.sh" +# Load functions designed for image processing +. "$(dirname "$0")/lib/cros_image_common.sh" || + die "Cannot load required library: lib/cros_image_common.sh; Abort." + get_default_board # Flags @@ -95,6 +99,50 @@ prepare_dir() { rm -rf state.gz } +compress_and_hash_memento_image() { + local input_file="$1" + + if has_part_tools; then + sudo "${SCRIPTS_DIR}/mk_memento_images.sh" "$input_file" 2 3 | + grep hash | + awk '{print $4}' + else + sudo "${SCRIPTS_DIR}/mk_memento_images.sh" part_2 part_3 | + grep hash | + awk '{print $4}' + fi +} + +compress_and_hash_file() { + local input_file="$1" + local output_file="$2" + + if [ -z "$input_file" ]; then + # Runs as a pipe processor + gzip_compress -c -9 | + tee "$output_file" | + openssl sha1 -binary | + openssl base64 + else + gzip_compress -c -9 "$input_file" | + tee "$output_file" | + openssl sha1 -binary | + openssl base64 + fi +} + +compress_and_hash_partition() { + local input_file="$1" + local part_num="$2" + local output_file="$3" + + if has_part_tools; then + dump_partition "$input_file" "$part_num" | + compress_and_hash_file "" "$output_file" + else + compress_and_hash_file "part_$part_num" "$output_file" + fi +} # Clean up stale config and data files. prepare_omaha @@ -108,21 +156,25 @@ echo "Output omaha config to ${OMAHA_DIR}/miniomaha.conf" prepare_dir -sudo ./unpack_partitions.sh ${RELEASE_IMAGE} &> /dev/null -release_hash=`sudo ${SCRIPTS_DIR}/mk_memento_images.sh part_2 part_3 \ - | grep hash | awk '{print $4}'` +if ! has_part_tools; then + #TODO(hungte) we can still avoid running unpack_partitions.sh + # by $(cat unpack_partitions.sh | grep Label | sed "s/#//" | grep ${name}" | + # awk '{ print $1}') to fetch offset/size. + echo "Unpacking image ${RELEASE_IMAGE} ..." >&2 + sudo ./unpack_partitions.sh "${RELEASE_IMAGE}" 2>/dev/null +fi + +release_hash="$(compress_and_hash_memento_image "${RELEASE_IMAGE}")" sudo chmod a+rw update.gz mv update.gz rootfs-release.gz mv rootfs-release.gz ${OMAHA_DATA_DIR} echo "release: ${release_hash}" -cat part_8 | gzip -9 > oem.gz -oem_hash=`cat oem.gz | openssl sha1 -binary | openssl base64` +oem_hash="$(compress_and_hash_partition "${RELEASE_IMAGE}" 8 "oem.gz")" mv oem.gz ${OMAHA_DATA_DIR} echo "oem: ${oem_hash}" -cat part_12 | gzip -9 > efi.gz -efi_hash=`cat efi.gz | openssl sha1 -binary | openssl base64` +efi_hash="$(compress_and_hash_partition "${RELEASE_IMAGE}" 12 "efi.gz")" mv efi.gz ${OMAHA_DATA_DIR} echo "efi: ${efi_hash}" @@ -132,17 +184,18 @@ popd > /dev/null pushd ${FACTORY_DIR} > /dev/null prepare_dir +if ! has_part_tools; then + echo "Unpacking image ${FACTORY_IMAGE} ..." >&2 + sudo ./unpack_partitions.sh "${FACTORY_IMAGE}" 2>/dev/null +fi -sudo ./unpack_partitions.sh ${FACTORY_IMAGE} &> /dev/null -test_hash=`sudo ${SCRIPTS_DIR}//mk_memento_images.sh part_2 part_3 \ - | grep hash | awk '{print $4}'` +test_hash="$(compress_and_hash_memento_image "${FACTORY_IMAGE}")" sudo chmod a+rw update.gz mv update.gz rootfs-test.gz mv rootfs-test.gz ${OMAHA_DATA_DIR} echo "test: ${test_hash}" -cat part_1 | gzip -9 > state.gz -state_hash=`cat state.gz | openssl sha1 -binary | openssl base64` +state_hash="$(compress_and_hash_partition "${FACTORY_IMAGE}" 1 "state.gz")" mv state.gz ${OMAHA_DATA_DIR} echo "state: ${state_hash}" @@ -155,8 +208,7 @@ if [ ! -z ${FLAGS_firmware_updater} ] ; then exit 1 fi - cat $SHELLBALL | gzip -9 > firmware.gz - firmware_hash=`cat firmware.gz | openssl sha1 -binary | openssl base64` + firmware_hash="$(compress_and_hash_file "$SHELLBALL" "firmware.gz")" mv firmware.gz ${OMAHA_DATA_DIR} echo "firmware: ${firmware_hash}" fi @@ -166,7 +218,7 @@ fi if [ -n "${FLAGS_subfolder}" ] && \ [ -f "${OMAHA_DIR}"/miniomaha.conf"" ] ; then # Remove the ']' from the last line of the file so we can add another config. - sed -i '$d' < ${OMAHA_DIR}/miniomaha.conf + sed -i '$d' ${OMAHA_DIR}/miniomaha.conf else echo -e "config = [" > ${OMAHA_DIR}/miniomaha.conf fi @@ -190,7 +242,7 @@ echo -n "{ if [ ! -z "${FLAGS_firmware_updater}" ] ; then echo -n " - 'firmware_image': 'firmware.gz', + 'firmware_image': '"${subfolder}"firmware.gz', 'firmware_checksum': '${firmware_hash}'," >> ${OMAHA_DIR}/miniomaha.conf fi diff --git a/mk_memento_images.sh b/mk_memento_images.sh index de921bb579..415768fb4b 100755 --- a/mk_memento_images.sh +++ b/mk_memento_images.sh @@ -10,25 +10,64 @@ set -e -if [ -z "$2" -o -z "$1" ]; then +# Load functions designed for image processing +if ! . "$(dirname "$0")/lib/cros_image_common.sh"; then + echo "ERROR: Cannot load required library: lib/cros_image_common.sh; Abort." + exit 1 +fi + +if [ -z "$2" -o -z "$1" ] || [ "${#@}" -ne 2 -a "${#@}" -ne 3 ]; then echo "usage: $0 path/to/kernel_partition_img path/to/rootfs_partition_img" + echo " or $0 path/to/chromiumos_img kern_part_no rootfs_part_no" exit 1 fi if [ "$CROS_GENERATE_UPDATE_PAYLOAD_CALLED" != "1" ]; then echo "WARNING:" - echo "This script should only be called from cros_generate_update_payload" - echo "Please run that script with --help to see how to use it." + echo " This script should only be called from cros_generate_update_payload" + echo " Please run that script with --help to see how to use it." +fi + +if ! has_command pigz; then + (echo "WARNING:" + echo " Your system does not have pigz (parallel gzip) installed." + echo " COMPRESSING WILL BE VERY SLOW. It is recommended to install pigz" + if has_command apt-get; then + echo " by 'sudo apt-get install pigz'." + elif has_command emerge; then + echo " by 'sudo emerge pigz'." + fi) >&2 fi if [ $(whoami) = "root" ]; then echo "running $0 as root which is unneccessary" fi -KPART="$1" -ROOT_PART="$2" - -KPART_SIZE=$(stat -c%s "$KPART") +# Determine the offset size, and file name of parameters +if [ -z "$3" ]; then + # kernnel_img rootfs_img + KPART="$1" + ROOT_PART="$2" + KPART_SIZE=$(stat -c%s "$KPART") + ROOT_PART_SIZE=$(stat -c%s "$ROOT_PART") + KPART_OFFSET=0 + KPART_SECTORS=$((KPART_SIZE / 512)) + ROOT_OFFSET=0 + ROOT_SECTORS=$((ROOT_PART_SIZE / 512)) +else + # chromiumos_img kern_part_no rootfs_part_no + KPART="$1" + ROOT_PART="$1" + KPART_OFFSET="$(part_offset "$KPART" "$2")" || + err_die "cannot retieve kernel partition offset" + KPART_SECTORS="$(part_size "$KPART" "$2")" || + err_die "cannot retieve kernel partition size" + ROOT_OFFSET="$(part_offset "$ROOT_PART" "$3")" || + err_die "cannot retieve root partition offset" + ROOT_SECTORS="$(part_size "$ROOT_PART" "$3")" || + err_die "cannot retieve root partition size" + KPART_SIZE=$((KPART_SECTORS * 512)) +fi # Sanity check size. if [ "$KPART_SIZE" -gt $((16 * 1024 * 1024)) ]; then @@ -38,34 +77,31 @@ if [ "$KPART_SIZE" -gt $((16 * 1024 * 1024)) ]; then fi FINAL_OUT_FILE=$(dirname "$1")/update.gz -UNCOMPRESSED_OUT_FILE="$FINAL_OUT_FILE.uncompressed" -# First, write size of kernel partition in big endian as uint64 to out file -# printf converts it to a number like 00000000003d0900. sed converts it to: -# \\x00\\x00\\x00\\x00\\x00\\x3d\\x09\\x00, then xargs converts it to binary -# with echo. -printf %016x "$KPART_SIZE" | \ - sed 's/\([0-9a-f][0-9a-f]\)/\\\\x\1/g' | \ - xargs echo -ne > "$UNCOMPRESSED_OUT_FILE" +# Update payload format: +# [kernel_size: big-endian uint64][kernel_blob][rootfs_blob] -# Next, write kernel partition to the out file -cat "$KPART" >> "$UNCOMPRESSED_OUT_FILE" +# Prepare kernel_size by using printf as a number like 00000000003d0900, then +# sed to convert as: \x00\x00\x00\x00\x00\x3d\x09\x00, finally echo -e to +# convert into binary. +KPART_SIZE_SIGNATURE="$(printf "%016x" "$KPART_SIZE" | + sed 's/\([0-9a-f][0-9a-f]\)/\\x\1/g')" -# Sanity check size of output file now -if [ $(stat -c%s "$UNCOMPRESSED_OUT_FILE") -ne $((8 + $KPART_SIZE)) ]; then - echo "Kernel partition changed size during image generation. Aborting." - exit 1 -fi +# Build the blob! +CS_AND_RET_CODES="$( + (echo -en "$KPART_SIZE_SIGNATURE" + echo "Compressing kernel..." >&2 + dump_partial_file "$KPART" "$KPART_OFFSET" "$KPART_SECTORS" + echo "Compressing rootfs..." >&2 + dump_partial_file "$ROOT_PART" "$ROOT_OFFSET" "$ROOT_SECTORS") | + gzip_compress -9 -c | + tee "$FINAL_OUT_FILE" | + openssl sha1 -binary | + openssl base64 | + tr '\n' ' ' + echo ${PIPESTATUS[*]})" -# Put rootfs into the out file -cat "$ROOT_PART" >> "$UNCOMPRESSED_OUT_FILE" - -# compress and hash -CS_AND_RET_CODES=$(gzip -c "$UNCOMPRESSED_OUT_FILE" | \ - tee "$FINAL_OUT_FILE" | openssl sha1 -binary | \ - openssl base64 | tr '\n' ' '; \ - echo ${PIPESTATUS[*]}) -EXPECTED_RET_CODES="0 0 0 0 0" +EXPECTED_RET_CODES="0 0 0 0 0 0" set -- $CS_AND_RET_CODES CALC_CS="$1" shift @@ -75,6 +111,4 @@ if [ "$RET_CODES" != "$EXPECTED_RET_CODES" ]; then exit 1 fi -rm "$UNCOMPRESSED_OUT_FILE" - echo Success. hash is "$CALC_CS" diff --git a/mod_image_for_recovery.sh b/mod_image_for_recovery.sh index 6a0b261330..3e4a6f9780 100755 --- a/mod_image_for_recovery.sh +++ b/mod_image_for_recovery.sh @@ -27,7 +27,7 @@ get_default_board DEFINE_string board "$DEFAULT_BOARD" "Board for which the image was built" b DEFINE_integer statefulfs_sectors 4096 \ - "Number of sectors to use for the stateful filesystem" + "Number of sectors to use for the stateful filesystem when minimizing" # Skips the build steps and just does the kernel swap. DEFINE_string kernel_image "" \ "Path to a pre-built recovery kernel" @@ -42,6 +42,12 @@ DEFINE_boolean kernel_image_only $FLAGS_FALSE \ "Emit the recovery kernel image only" DEFINE_boolean sync_keys $FLAGS_TRUE \ "Update the kernel to be installed with the vblock from stateful" +DEFINE_boolean minimize_image $FLAGS_TRUE \ + "Decides if the original image is used or a minimal recovery image is \ +created." +DEFINE_boolean modify_in_place $FLAGS_FALSE \ + "Modifies the source image in place. This cannot be used with \ +--minimize_image." DEFINE_integer jobs -1 \ "How many packages to build in parallel at maximum." j DEFINE_string build_root "/build" \ @@ -50,6 +56,9 @@ DEFINE_string build_root "/build" \ DEFINE_string rootfs_hash "/tmp/rootfs.hash" \ "Path where the rootfs hash should be stored." +DEFINE_boolean verbose $FLAGS_FALSE \ + "Log all commands to stdout." v + # Keep in sync with build_image. DEFINE_string keys_dir "/usr/share/vboot/devkeys" \ "Directory containing the signing keys." @@ -58,6 +67,17 @@ DEFINE_string keys_dir "/usr/share/vboot/devkeys" \ FLAGS "$@" || exit 1 eval set -- "${FLAGS_ARGV}" +if [ $FLAGS_verbose -eq $FLAGS_FALSE ]; then + exec 2>/dev/null + # Redirecting to stdout instead of stderr since + # we silence stderr above. + die() { + echo -e "${V_BOLD_RED}ERROR : $1${V_VIDOFF}" + exit 1 + } +fi +set -x # Make debugging with -v easy. + EMERGE_CMD="emerge" EMERGE_BOARD_CMD="emerge-${FLAGS_board}" @@ -178,10 +198,11 @@ create_recovery_kernel_image() { local kern_tmp=$(mktemp) local kern_hash= - dd if="$FLAGS_image" bs=512 count=$kern_size skip=$kern_offset of="$kern_tmp" + dd if="$FLAGS_image" bs=512 count=$kern_size \ + skip=$kern_offset of="$kern_tmp" 1>&2 # We're going to use the real signing block. if [ $FLAGS_sync_keys -eq $FLAGS_TRUE ]; then - dd if="$INSTALL_VBLOCK" of="$kern_tmp" conv=notrunc + dd if="$INSTALL_VBLOCK" of="$kern_tmp" conv=notrunc 1>&2 fi local kern_hash=$(sha1sum "$kern_tmp" | cut -f1 -d' ') rm "$kern_tmp" @@ -200,7 +221,7 @@ create_recovery_kernel_image() { --root=${cros_root} \ --keys_dir="${FLAGS_keys_dir}" \ --nouse_dev_keys \ - ${verity_args} + ${verity_args} 1>&2 sudo rm "$FLAGS_rootfs_hash" sudo losetup -d "$root_dev" trap - RETURN @@ -236,6 +257,13 @@ install_recovery_kernel() { local kern_a_size=$(partsize "$RECOVERY_IMAGE" 2) local kern_b_offset=$(partoffset "$RECOVERY_IMAGE" 4) local kern_b_size=$(partsize "$RECOVERY_IMAGE" 4) + + if [ $kern_b_size -eq 1 ]; then + echo "Image was created with no KERN-B partition reserved!" 1>&2 + echo "Cannot proceed." 1>&2 + return 1 + fi + # Backup original kernel to KERN-B dd if="$RECOVERY_IMAGE" of="$RECOVERY_IMAGE" bs=512 \ count=$kern_a_size \ @@ -277,14 +305,22 @@ install_recovery_kernel() { } maybe_resize_stateful() { + # If we're not minimizing, then just copy and go. + if [ $FLAGS_minimize_image -eq $FLAGS_FALSE ]; then + if [ "$FLAGS_image" != "$RECOVERY_IMAGE" ]; then + cp "$FLAGS_image" "$RECOVERY_IMAGE" + fi + return 0 + fi + # Rebuild the image with a 1 sector stateful partition local err=0 local small_stateful=$(mktemp) dd if=/dev/zero of="$small_stateful" bs=512 \ - count=${FLAGS_statefulfs_sectors} + count=${FLAGS_statefulfs_sectors} 1>&2 trap "rm $small_stateful" RETURN # Don't bother with ext3 for such a small image. - /sbin/mkfs.ext2 -F -b 4096 "$small_stateful" + /sbin/mkfs.ext2 -F -b 4096 "$small_stateful" 1>&2 # If it exists, we need to copy the vblock over to stateful # This is the real vblock and not the recovery vblock. @@ -301,14 +337,20 @@ maybe_resize_stateful() { # Create a recovery image of the right size # TODO(wad) Make the developer script case create a custom GPT with # just the kernel image and stateful. - update_partition_table "$FLAGS_image" "$small_stateful" 4096 "$RECOVERY_IMAGE" + update_partition_table "$FLAGS_image" "$small_stateful" 4096 \ + "$RECOVERY_IMAGE" 1>&2 return $err } -# main process begins here. +cleanup() { + set +e + if [ "$FLAGS_image" != "$RECOVERY_IMAGE" ]; then + rm "$RECOVERY_IMAGE" + fi + rm "$INSTALL_VBLOCK" +} -# Make sure this is really what the user wants, before nuking the device -echo "Creating recovery image ${FLAGS_to} from ${FLAGS_image} . . . " +# main process begins here. set -e set -u @@ -335,6 +377,15 @@ if [ $FLAGS_kernel_image_only -eq $FLAGS_TRUE -a \ die "Cannot use --kernel_image_only with --kernel_image" fi +if [ $FLAGS_modify_in_place -eq $FLAGS_TRUE ]; then + if [ $FLAGS_minimize_image -eq $FLAGS_TRUE ]; then + die "Cannot use --modify_in_place and --minimize_image together." + fi + RECOVERY_IMAGE="${FLAGS_image}" +fi + +echo "Creating recovery image from ${FLAGS_image}" + INSTALL_VBLOCK=$(get_install_vblock) if [ -z "$INSTALL_VBLOCK" ]; then die "Could not copy the vblock from stateful." @@ -343,10 +394,10 @@ fi if [ -z "$FLAGS_kernel_image" ]; then emerge_recovery_kernel create_recovery_kernel_image + echo "Recovery kernel created at $RECOVERY_KERNEL_IMAGE" else RECOVERY_KERNEL_IMAGE="$FLAGS_kernel_image" fi -echo "Kernel emitted: $RECOVERY_KERNEL_IMAGE." if [ $FLAGS_kernel_image_only -eq $FLAGS_TRUE ]; then echo "Kernel emitted. Stopping there." @@ -354,13 +405,16 @@ if [ $FLAGS_kernel_image_only -eq $FLAGS_TRUE ]; then exit 0 fi -rm "$RECOVERY_IMAGE" || true # Start fresh :) +if [ $FLAGS_modify_in_place -eq $FLAGS_FALSE ]; then + rm "$RECOVERY_IMAGE" || true # Start fresh :) +fi -trap "rm \"$RECOVERY_IMAGE\" && rm \"$INSTALL_VBLOCK\"" EXIT +trap cleanup EXIT -maybe_resize_stateful # Also copies the image +maybe_resize_stateful # Also copies the image if needed. install_recovery_kernel +echo "Recovery image created at $RECOVERY_IMAGE" print_time_elapsed trap - EXIT diff --git a/mount_gpt_image.sh b/mount_gpt_image.sh index 63dba3d8b6..3156c1586e 100755 --- a/mount_gpt_image.sh +++ b/mount_gpt_image.sh @@ -139,8 +139,10 @@ if [ ${FLAGS_most_recent} -eq ${FLAGS_TRUE} ] ; then FLAGS_from="$(./get_latest_image.sh --board="${FLAGS_board}")" fi -# Turn path into an absolute path. +# Turn paths into absolute paths. FLAGS_from=`eval readlink -f ${FLAGS_from}` +FLAGS_rootfs_mountpt=`eval readlink -f ${FLAGS_rootfs_mountpt}` +FLAGS_stateful_mountpt=`eval readlink -f ${FLAGS_stateful_mountpt}` # Perform desired operation. if [ ${FLAGS_unmount} -eq ${FLAGS_TRUE} ] ; then diff --git a/parallel_emerge b/parallel_emerge index 66b2334da6..ab342334c2 100755 --- a/parallel_emerge +++ b/parallel_emerge @@ -7,7 +7,7 @@ Usage: ./parallel_emerge [--board=BOARD] [--workon=PKGS] [--no-workon-deps] - [emerge args] package" + [--force-remote-binary=PKGS] [emerge args] package Basic operation: Runs 'emerge -p --debug' to display dependencies, and stores a @@ -84,6 +84,7 @@ from _emerge.Scheduler import Scheduler from _emerge.stdout_spinner import stdout_spinner import portage import portage.debug +import portage.versions def Usage(): @@ -218,7 +219,8 @@ class DepGraphGenerator(object): """ __slots__ = ["board", "emerge", "mandatory_source", "no_workon_deps", - "nomerge", "package_db", "rebuild", "show_output"] + "nomerge", "package_db", "rebuild", "show_output", + "force_remote_binary", "forced_remote_binary_packages"] def __init__(self): self.board = None @@ -229,6 +231,8 @@ class DepGraphGenerator(object): self.package_db = {} self.rebuild = False self.show_output = False + self.force_remote_binary = set() + self.forced_remote_binary_packages = set() def ParseParallelEmergeArgs(self, argv): """Read the parallel emerge arguments from the command-line. @@ -251,6 +255,11 @@ class DepGraphGenerator(object): workon_str = arg.replace("--workon=", "") package_list = shlex.split(" ".join(shlex.split(workon_str))) self.mandatory_source.update(package_list) + elif arg.startswith("--force-remote-binary="): + force_remote_binary = arg.replace("--force-remote-binary=", "") + force_remote_binary = \ + shlex.split(" ".join(shlex.split(force_remote_binary))) + self.force_remote_binary.update(force_remote_binary) elif arg.startswith("--nomerge="): nomerge_str = arg.replace("--nomerge=", "") package_list = shlex.split(" ".join(shlex.split(nomerge_str))) @@ -455,12 +464,11 @@ class DepGraphGenerator(object): forced_flags = set(pkgsettings.useforce).union(pkgsettings.usemask) depgraph = self.emerge.depgraph - flags = depgraph._reinstall_for_flags(forced_flags, cur_use, cur_iuse, now_use, now_iuse) return not flags - def GenDependencyTree(self): + def GenDependencyTree(self, remote_pkgs): """Get dependency tree info from emerge. TODO(): Update cros_extract_deps to also use this code. @@ -479,22 +487,39 @@ class DepGraphGenerator(object): # --workon and the dependencies have changed. emerge = self.emerge emerge_opts = emerge.opts.copy() - emerge_opts.pop("--getbinpkg", None) - if "--usepkgonly" not in emerge_opts: - emerge_opts.pop("--usepkg", None) - if self.mandatory_source or self.rebuild: - # Enable --emptytree so that we get the full tree, which we need for - # dependency analysis. By default, with this option, emerge optimizes - # the graph by removing uninstall instructions from the graph. By - # specifying --tree as well, we tell emerge that it's not safe to remove - # uninstall instructions because we're planning on analyzing the output. - emerge_opts["--tree"] = True - emerge_opts["--emptytree"] = True + + # Enable --emptytree so that we get the full tree, which we need for + # dependency analysis. By default, with this option, emerge optimizes + # the graph by removing uninstall instructions from the graph. By + # specifying --tree as well, we tell emerge that it's not safe to remove + # uninstall instructions because we're planning on analyzing the output. + emerge_opts["--tree"] = True + emerge_opts["--emptytree"] = True + + # Tell emerge not to worry about use flags yet. We handle those inside + # parallel_emerge itself. Further, when we use the --force-remote-binary + # flag, we don't emerge to reject a package just because it has different + # use flags. + emerge_opts.pop("--newuse", None) + emerge_opts.pop("--reinstall", None) # Create a list of packages to merge packages = set(emerge.cmdline_packages[:]) if self.mandatory_source: packages.update(self.mandatory_source) + if self.force_remote_binary: + forced_pkgs = {} + for pkg in remote_pkgs: + category, pkgname, _, _ = portage.catpkgsplit(pkg) + full_pkgname = "%s/%s" % (category, pkgname) + if (pkgname in self.force_remote_binary or + full_pkgname in self.force_remote_binary): + forced_pkgs.setdefault(full_pkgname, []).append(pkg) + + for pkgs in forced_pkgs.values(): + forced_package = portage.versions.best(pkgs) + packages.add("=%s" % forced_package) + self.forced_remote_binary_packages.add(forced_package) # Tell emerge to be quiet. We print plenty of info ourselves so we don't # need any extra output from portage. @@ -567,9 +592,25 @@ class DepGraphGenerator(object): frozen_config = depgraph._frozen_config vardb = frozen_config.trees[root]["vartree"].dbapi pkgsettings = frozen_config.pkgsettings[root] + + # It's time to start worrying about use flags, if necessary. + for flag in ("--newuse", "--reinstall"): + if flag in emerge.opts: + emerge_opts[flag] = emerge.opts[flag] + deps_info = {} for pkg in depgraph.altlist(): if isinstance(pkg, Package): + # If we're not using --force-remote-binary, check what flags are being + # used by the real package. + if "--usepkgonly" not in emerge.opts: + try: + pkg = emerge.depgraph._pkg(pkg.cpv, "ebuild", emerge.root_config) + except portage.exception.PackageNotFound: + # This is a --force-remote-binary package. + pass + self.package_db[pkg.cpv] = pkg + # If we're not in emptytree mode, and we're going to replace a package # that is already installed, then this operation is possibly optional. # ("--selective" mode is handled later, in RemoveInstalledPackages()) @@ -580,9 +621,6 @@ class DepGraphGenerator(object): optional = True break - # Add the package to our database. - self.package_db[str(pkg.cpv)] = pkg - # Save off info about the package deps_info[str(pkg.cpv)] = {"idx": len(deps_info), "optional": optional} @@ -611,7 +649,65 @@ class DepGraphGenerator(object): print "%s %s (%s)" % (depth, entry, action) self.PrintTree(deps[entry]["deps"], depth=depth + " ") - def GenDependencyGraph(self, deps_tree, deps_info): + def RemotePackageDatabase(self, binhost_url): + """Grab the latest binary package database from the prebuilt server. + + We need to know the modification times of the prebuilt packages so that we + know when it is OK to use these packages and when we should rebuild them + instead. + + Args: + binhost_url: Base URL of remote packages (PORTAGE_BINHOST). + + Returns: + A dict mapping package identifiers to modification times. + """ + + if not binhost_url: + return {} + + def retry_urlopen(url, tries=3): + """Open the specified url, retrying if we run into network errors. + + We do not retry for HTTP errors. + + Args: + url: The specified url. + tries: The number of times to try. + + Returns: + The result of urllib2.urlopen(url). + """ + for i in range(tries): + try: + return urllib2.urlopen(url) + except urllib2.HTTPError as e: + raise + except urllib2.URLError as e: + if i + 1 == tries: + raise + else: + print "Cannot GET %s: %s" % (url, e) + + url = os.path.join(binhost_url, "Packages") + try: + f = retry_urlopen(url) + except urllib2.HTTPError as e: + if e.code == 404: + return {} + else: + raise + prebuilt_pkgs = {} + for line in f: + if line.startswith("CPV: "): + pkg = line.replace("CPV: ", "").rstrip() + elif line.startswith("MTIME: "): + prebuilt_pkgs[pkg] = int(line[:-1].replace("MTIME: ", "")) + f.close() + + return prebuilt_pkgs + + def GenDependencyGraph(self, deps_tree, deps_info, remote_pkgs): """Generate a doubly linked dependency graph. Args: @@ -660,6 +756,10 @@ class DepGraphGenerator(object): # If true, indicates that this package must be installed. We don't care # whether it's binary or source, unless the mandatory_source flag is # also set. + # - force_remote_binary: + # If true, indicates that we want to update to the latest remote prebuilt + # of this package. Packages that depend on this package should be built + # from source. # deps_map = {} @@ -678,7 +778,8 @@ class DepGraphGenerator(object): # Create an entry for the package action = packages[pkg]["action"] default_pkg = {"needs": {}, "provides": set(), "action": action, - "mandatory_source": False, "mandatory": False} + "mandatory_source": False, "mandatory": False, + "force_remote_binary": False} this_pkg = deps_map.setdefault(pkg, default_pkg) # Create entries for dependencies of this package first. @@ -909,64 +1010,6 @@ class DepGraphGenerator(object): if this_pkg["action"] == "nomerge": this_pkg["action"] = "merge" - def RemotePackageDatabase(binhost_url): - """Grab the latest binary package database from the prebuilt server. - - We need to know the modification times of the prebuilt packages so that we - know when it is OK to use these packages and when we should rebuild them - instead. - - Args: - binhost_url: Base URL of remote packages (PORTAGE_BINHOST). - - Returns: - A dict mapping package identifiers to modification times. - """ - - if not binhost_url: - return {} - - def retry_urlopen(url, tries=3): - """Open the specified url, retrying if we run into network errors. - - We do not retry for HTTP errors. - - Args: - url: The specified url. - tries: The number of times to try. - - Returns: - The result of urllib2.urlopen(url). - """ - for i in range(tries): - try: - return urllib2.urlopen(url) - except urllib2.HTTPError as e: - raise - except urllib2.URLError as e: - if i + 1 == tries: - raise - else: - print "Cannot GET %s: %s" % (url, e) - - url = binhost_url + "/Packages" - try: - f = retry_urlopen(url) - except urllib2.HTTPError as e: - if e.code == 404: - return {} - else: - raise - prebuilt_pkgs = {} - for line in f: - if line.startswith("CPV: "): - pkg = line.replace("CPV: ", "").rstrip() - elif line.startswith("MTIME: "): - prebuilt_pkgs[pkg] = int(line[:-1].replace("MTIME: ", "")) - f.close() - - return prebuilt_pkgs - def LocalPackageDatabase(): """Get the modification times of the packages in the local database. @@ -1019,7 +1062,7 @@ class DepGraphGenerator(object): """ if pkg in cache: return cache[pkg] - if pkg not in pkg_db: + if pkg not in pkg_db and pkg not in self.forced_remote_binary_packages: cache[pkg] = False else: cache[pkg] = True @@ -1081,7 +1124,7 @@ class DepGraphGenerator(object): else: MergeChildren(pkg, "mandatory_source") - def UsePrebuiltPackages(): + def UsePrebuiltPackages(remote_pkgs): """Update packages that can use prebuilts to do so.""" start = time.time() @@ -1099,13 +1142,18 @@ class DepGraphGenerator(object): # Build list of prebuilt packages for pkg, info in deps_map.iteritems(): - if info and not info["mandatory_source"] and info["action"] == "merge": + if info and info["action"] == "merge": + if (not info["force_remote_binary"] and info["mandatory_source"] or + "--usepkgonly" not in emerge.opts and pkg not in remote_pkgs): + continue + db_keys = list(bindb._aux_cache_keys) try: db_vals = bindb.aux_get(pkg, db_keys + ["MTIME"]) except KeyError: # No binary package continue + mtime = int(db_vals.pop() or 0) metadata = zip(db_keys, db_vals) db_pkg = Package(built=True, cpv=pkg, installed=False, @@ -1116,15 +1164,14 @@ class DepGraphGenerator(object): # Calculate what packages need to be rebuilt due to changes in use flags. for pkg, db_pkg in prebuilt_pkgs.iteritems(): - db_pkg_src = self.package_db[pkg] - if not self.CheckUseFlags(pkgsettings, db_pkg, db_pkg_src): + if not self.CheckUseFlags(pkgsettings, db_pkg, self.package_db[pkg]): MergeChildren(pkg, "mandatory_source") # Convert eligible packages to binaries. for pkg, info in deps_map.iteritems(): - if (info and not info["mandatory_source"] and - info["action"] == "merge" and pkg in prebuilt_pkgs): - self.package_db[pkg] = prebuilt_pkgs[pkg] + if info and info["action"] == "merge" and pkg in prebuilt_pkgs: + if not info["mandatory_source"] or info["force_remote_binary"]: + self.package_db[pkg] = prebuilt_pkgs[pkg] seconds = time.time() - start if "--quiet" not in emerge.opts: @@ -1132,28 +1179,22 @@ class DepGraphGenerator(object): return prebuilt_pkgs - def AddRemainingPackages(): - """Fill in packages that don't have entries in the package db. - - Every package we are installing needs an entry in the package db. - This function should only be called after we have removed the - packages that are not being merged from our deps_map. - """ - for pkg in deps_map: - if pkg not in self.package_db: - if deps_map[pkg]["action"] != "merge": - # We should only fill in packages that are being merged. If - # there's any other packages here, something funny is going on. - print "Missing entry for %s in package db" % pkg - sys.exit(1) - - db_pkg = emerge.depgraph._pkg(pkg, "ebuild", emerge.root_config) - self.package_db[pkg] = db_pkg - ReverseTree(deps_tree) BuildFinalPackageSet() AddSecretDeps() + # Mark that we want to use remote binaries only for a particular package. + vardb = emerge.depgraph._frozen_config.trees[root]["vartree"].dbapi + for pkg in self.force_remote_binary: + for db_pkg in final_db.match_pkgs(pkg): + match = deps_map.get(str(db_pkg.cpv)) + if match: + match["force_remote_binary"] = True + + rebuild_blacklist.add(str(db_pkg.cpv)) + if not vardb.match_pkgs(db_pkg.cpv): + MergeChildren(str(db_pkg.cpv), "mandatory") + if self.no_workon_deps: for pkg in self.mandatory_source.copy(): for db_pkg in final_db.match_pkgs(pkg): @@ -1166,10 +1207,6 @@ class DepGraphGenerator(object): cycles = FindCycles() if self.rebuild: local_pkgs = LocalPackageDatabase() - remote_pkgs = {} - if "--getbinpkg" in emerge.opts: - binhost = emerge.settings["PORTAGE_BINHOST"] - remote_pkgs = RemotePackageDatabase(binhost) AutoRebuildDeps(local_pkgs, remote_pkgs, cycles) # We need to remove installed packages so that we can use the dependency @@ -1180,8 +1217,7 @@ class DepGraphGenerator(object): SanitizeTree() if deps_map: if "--usepkg" in emerge.opts: - UsePrebuiltPackages() - AddRemainingPackages() + UsePrebuiltPackages(remote_pkgs) return deps_map def PrintInstallPlan(self, deps_map): @@ -1734,13 +1770,18 @@ def main(): print " Skipping package %s on %s" % (nomerge_packages, deps.board or "root") - deps_tree, deps_info = deps.GenDependencyTree() + remote_pkgs = {} + if "--getbinpkg" in emerge.opts: + binhost = emerge.settings["PORTAGE_BINHOST"] + remote_pkgs = deps.RemotePackageDatabase(binhost) + + deps_tree, deps_info = deps.GenDependencyTree(remote_pkgs) # You want me to be verbose? I'll give you two trees! Twice as much value. if "--tree" in emerge.opts and "--verbose" in emerge.opts: deps.PrintTree(deps_tree) - deps_graph = deps.GenDependencyGraph(deps_tree, deps_info) + deps_graph = deps.GenDependencyGraph(deps_tree, deps_info, remote_pkgs) # OK, time to print out our progress so far. deps.PrintInstallPlan(deps_graph) diff --git a/prebuilt.py b/prebuilt.py index df7f6e36dc..a029159ed4 100755 --- a/prebuilt.py +++ b/prebuilt.py @@ -92,8 +92,8 @@ def UpdateLocalFile(filename, value, key='PORTAGE_BINHOST'): # Strip newlines from end of line. We already add newlines below. line = line.rstrip("\n") - if '=' not in line: - # Skip any line without an equal in it and just write it out. + if len(line.split('=')) != 2: + # Skip any line that doesn't fit key=val. file_lines.append(line) continue diff --git a/prebuilt_unittest.py b/prebuilt_unittest.py index 37d24afde1..fe2387fbf7 100755 --- a/prebuilt_unittest.py +++ b/prebuilt_unittest.py @@ -17,7 +17,9 @@ class TestUpdateFile(unittest.TestCase): self.contents_str = ['# comment that should be skipped', 'PKGDIR="/var/lib/portage/pkgs"', 'PORTAGE_BINHOST="http://no.thanks.com"', - 'portage portage-20100310.tar.bz2'] + 'portage portage-20100310.tar.bz2', + 'COMPILE_FLAGS="some_value=some_other"', + ] temp_fd, self.version_file = tempfile.mkstemp() os.write(temp_fd, '\n'.join(self.contents_str)) os.close(temp_fd) diff --git a/run_remote_tests.sh b/run_remote_tests.sh index 864d8bb756..babbbdceb0 100755 --- a/run_remote_tests.sh +++ b/run_remote_tests.sh @@ -18,42 +18,20 @@ DEFINE_string args "" \ "Command line arguments for test. Quoted and space separated if multiple." a DEFINE_string board "$DEFAULT_BOARD" \ "The board for which you are building autotest" +DEFINE_boolean build ${FLAGS_FALSE} "Build tests while running" b DEFINE_string chroot "${DEFAULT_CHROOT_DIR}" "alternate chroot location" c DEFINE_boolean cleanup ${FLAGS_FALSE} "Clean up temp directory" DEFINE_integer iterations 1 "Iterations to run every top level test" i DEFINE_string results_dir_root "" "alternate root results directory" DEFINE_boolean verbose ${FLAGS_FALSE} "Show verbose autoserv output" v +DEFINE_boolean use_emerged ${FLAGS_FALSE} \ + "Force use of emerged autotest packages" RAN_ANY_TESTS=${FLAGS_FALSE} -# Check if our stdout is a tty -function is_a_tty() { - local stdout=$(readlink /proc/$$/fd/1) - [[ "${stdout#/dev/tty}" != "${stdout}" ]] && return 0 - [[ "${stdout#/dev/pts}" != "${stdout}" ]] && return 0 - return 1 -} - -# Writes out text in specified color if stdout is a tty -# Arguments: -# $1 - color -# $2 - text to color -# $3 - text following colored text (default colored) -# Returns: -# None -function echo_color() { - local color=0 - [[ "$1" == "red" ]] && color=31 - [[ "$1" == "green" ]] && color=32 - [[ "$1" == "yellow" ]] && color=33 - if is_a_tty; then - echo -e "\033[1;${color}m$2\033[0m$3" - else - echo "$2$3" - fi -} - function cleanup() { + # Always remove the build path in case it was used. + [[ -n "${BUILD_DIR}" ]] && sudo rm -rf "${BUILD_DIR}" if [[ $FLAGS_cleanup -eq ${FLAGS_TRUE} ]] || \ [[ ${RAN_ANY_TESTS} -eq ${FLAGS_FALSE} ]]; then rm -rf "${TMP}" @@ -63,27 +41,6 @@ function cleanup() { cleanup_remote_access } -# Adds attributes to all tests run -# Arguments: -# $1 - results directory -# $2 - attribute name (key) -# $3 - attribute value (value) -function add_test_attribute() { - local results_dir="$1" - local attribute_name="$2" - local attribute_value="$3" - if [[ -z "$attribute_value" ]]; then - return; - fi - - for status_file in $(echo "${results_dir}"/*/status); do - local keyval_file=$(dirname $status_file)/keyval - echo "Updating ${keyval_file}" - echo "${attribute_name}=${attribute_value}" >> "${keyval_file}" - done -} - - # Determine if a control is for a client or server test. Echos # either "server" or "client". # Arguments: @@ -94,17 +51,83 @@ function read_test_type() { local type=$(egrep -m1 \ '^[[:space:]]*TEST_TYPE[[:space:]]*=' "${control_file}") if [[ -z "${type}" ]]; then - echo_color "red" ">>> Unable to find TEST_TYPE line in ${control_file}" - exit 1 + die "Unable to find TEST_TYPE line in ${control_file}" fi type=$(python -c "${type}; print TEST_TYPE.lower()") if [[ "${type}" != "client" ]] && [[ "${type}" != "server" ]]; then - echo_color "red" ">>> Unknown type of test (${type}) in ${control_file}" - exit 1 + die "Unknown type of test (${type}) in ${control_file}" fi echo ${type} } +function create_tmp() { + # Set global TMP for remote_access.sh's sake + # and if --results_dir_root is specified, + # set TMP and create dir appropriately + if [[ ${INSIDE_CHROOT} -eq 0 ]]; then + if [[ -n "${FLAGS_results_dir_root}" ]]; then + TMP=${FLAGS_chroot}${FLAGS_results_dir_root} + mkdir -p -m 777 ${TMP} + else + TMP=$(mktemp -d ${FLAGS_chroot}/tmp/run_remote_tests.XXXX) + fi + TMP_INSIDE_CHROOT=$(echo ${TMP#${FLAGS_chroot}}) + else + if [[ -n "${FLAGS_results_dir_root}" ]]; then + TMP=${FLAGS_results_dir_root} + mkdir -p -m 777 ${TMP} + else + TMP=$(mktemp -d /tmp/run_remote_tests.XXXX) + fi + TMP_INSIDE_CHROOT=${TMP} + fi +} + +function prepare_build_dir() { + local autotest_dir="$1" + INSIDE_BUILD_DIR="${TMP_INSIDE_CHROOT}/build" + BUILD_DIR="${TMP}/build" + info "Copying autotest tree into ${BUILD_DIR}." + sudo mkdir -p "${BUILD_DIR}" + sudo rsync -rl --chmod=ugo=rwx "${autotest_dir}"/ "${BUILD_DIR}" + info "Pilfering toolchain shell environment from Portage." + local outside_ebuild_dir="${TMP}/chromeos-base/autotest-build" + local inside_ebuild_dir="${TMP_INSIDE_CHROOT}/chromeos-base/autotest-build" + mkdir -p "${outside_ebuild_dir}" + local E_only="autotest-build-9999.ebuild" + cat > "${outside_ebuild_dir}/${E_only}" <&1 > /dev/null + local P_tmp="${FLAGS_chroot}/build/${FLAGS_board}/tmp/portage/" + local E_dir="${E%%/*}/${E_only%.*}" + sudo cp "${P_tmp}/${E_dir}/temp/environment" "${BUILD_DIR}" +} + +function autodetect_build() { + if [ ${FLAGS_use_emerged} -eq ${FLAGS_TRUE} ]; then + info \ +"As requested, using emerged autotests already installed in your sysroot." + FLAGS_build=${FLAGS_FALSE} + return + fi + if ${ENTER_CHROOT} ./cros_workon --board=${FLAGS_board} list | \ + grep -q autotest; then + info \ +"Detected cros_workon autotests, building your sources instead of emerged \ +autotest. To use installed autotest, pass --use_emerged." + FLAGS_build=${FLAGS_TRUE} + else + info \ +"Using emerged autotests already installed in your sysroot. To build \ +autotests directly from your source directory instead, pass --build." + FLAGS_build=${FLAGS_FALSE} + fi +} + function main() { cd $(dirname "$0") @@ -130,26 +153,7 @@ function main() { set -e - # Set global TMP for remote_access.sh's sake - # and if --results_dir_root is specified, - # set TMP and create dir appropriately - if [[ ${INSIDE_CHROOT} -eq 0 ]]; then - if [[ -n "${FLAGS_results_dir_root}" ]]; then - TMP=${FLAGS_chroot}${FLAGS_results_dir_root} - mkdir -p -m 777 ${TMP} - else - TMP=$(mktemp -d ${FLAGS_chroot}/tmp/run_remote_tests.XXXX) - fi - TMP_INSIDE_CHROOT=$(echo ${TMP#${FLAGS_chroot}}) - else - if [[ -n "${FLAGS_results_dir_root}" ]]; then - TMP=${FLAGS_results_dir_root} - mkdir -p -m 777 ${TMP} - else - TMP=$(mktemp -d /tmp/run_remote_tests.XXXX) - fi - TMP_INSIDE_CHROOT=${TMP} - fi + create_tmp trap cleanup EXIT @@ -158,6 +162,23 @@ function main() { learn_board autotest_dir="${FLAGS_chroot}/build/${FLAGS_board}/usr/local/autotest" + ENTER_CHROOT="" + if [[ ${INSIDE_CHROOT} -eq 0 ]]; then + ENTER_CHROOT="./enter_chroot.sh --chroot ${FLAGS_chroot} --" + fi + + if [ ${FLAGS_build} -eq ${FLAGS_FALSE} ]; then + autodetect_build + fi + + if [ ${FLAGS_build} -eq ${FLAGS_TRUE} ]; then + autotest_dir="${SRC_ROOT}/third_party/autotest/files" + else + if [ ! -d "${autotest_dir}" ]; then + die "You need to emerge autotest-tests (or use --build)" + fi + fi + local control_files_to_run="" local chrome_autotests="${CHROME_ROOT}/src/chrome/test/chromeos/autotest/files" # Now search for tests which unambiguously include the given identifier @@ -172,8 +193,7 @@ function main() { ! finds=$(find ${search_path} -maxdepth 2 -type f \( -name control.\* -or \ -name control \) | egrep -v "~$" | egrep "${test_request}") if [[ -z "${finds}" ]]; then - echo_color "red" ">>> Cannot find match for \"${test_request}\"" - exit 1 + die "Cannot find match for \"${test_request}\"" fi local matches=$(echo "${finds}" | wc -l) if [[ ${matches} -gt 1 ]]; then @@ -193,13 +213,14 @@ function main() { echo "" if [[ -z "${control_files_to_run}" ]]; then - echo_color "red" ">>> Found no control files" - exit 1 + die "Found no control files" fi - echo_color "yellow" ">>> Running the following control files:" + [ ${FLAGS_build} -eq ${FLAGS_TRUE} ] && prepare_build_dir "${autotest_dir}" + + info "Running the following control files:" for CONTROL_FILE in ${control_files_to_run}; do - echo_color "yellow" " * " "${CONTROL_FILE}" + info " * ${CONTROL_FILE}" done for control_file in ${control_files_to_run}; do @@ -217,7 +238,7 @@ function main() { option="-s" fi echo "" - echo_color "yellow" ">>> Running ${type} test " ${control_file} + info "Running ${type} test ${control_file}" local control_file_name=$(basename "${control_file}") local short_name=$(basename $(dirname "${control_file}")) @@ -243,31 +264,49 @@ function main() { RAN_ANY_TESTS=${FLAGS_TRUE} - local enter_chroot="" - local autotest="${GCLIENT_ROOT}/src/scripts/autotest_workon" - if [[ ${INSIDE_CHROOT} -eq 0 ]]; then - enter_chroot="./enter_chroot.sh --chroot ${FLAGS_chroot} --" - autotest="./autotest_workon" - fi - # Remove chrome autotest location prefix from control_file if needed if [[ ${control_file:0:${#chrome_autotests}} == \ "${chrome_autotests}" ]]; then control_file="${control_file:${#chrome_autotests}+1}" - echo_color "yellow" ">>> Running chrome autotest " ${control_file} - fi - if [[ -n "${FLAGS_args}" ]]; then - passthrough_args="--args=${FLAGS_args}" + info "Running chrome autotest ${control_file}" fi - ${enter_chroot} ${autotest} --board "${FLAGS_board}" -m "${FLAGS_remote}" \ - --ssh-port ${FLAGS_ssh_port} \ - "${option}" "${control_file}" -r "${results_dir}" ${verbose} \ - "${passthrough_args}" >&2 + export AUTOSERV_TEST_ARGS="${FLAGS_args}" + export AUTOSERV_ARGS="-m ${FLAGS_remote} \ + --ssh-port ${FLAGS_ssh_port} \ + ${option} ${control_file} -r ${results_dir} ${verbose}" + if [ ${FLAGS_build} -eq ${FLAGS_FALSE} ]; then + cat > "${TMP}/run_test.sh" <&2 + else + cp "${BUILD_DIR}/environment" "${TMP}/run_test.sh" + GRAPHICS_BACKEND=${GRAPHICS_BACKEND:-OPENGL} + if [ -n "${AUTOSERV_TEST_ARGS}" ]; then + AUTOSERV_TEST_ARGS="-a \"${AUTOSERV_TEST_ARGS}\"" + fi + cat >> "${TMP}/run_test.sh" <&2 + fi done echo "" - echo_color "yellow" ">>> Test results:" + info "Test results:" ./generate_test_report "${TMP}" --strip="${TMP}/" print_time_elapsed diff --git a/serve_factory_packages.py b/serve_factory_packages.py index 41c35176e9..723386780e 100644 --- a/serve_factory_packages.py +++ b/serve_factory_packages.py @@ -37,7 +37,7 @@ FLAGS = gflags.FLAGS gflags.DEFINE_string('board', None, 'Platform to build.') gflags.DEFINE_string('base_image', None, 'Path to base image.') gflags.DEFINE_string('firmware_updater', None, 'Path to firmware updater.') - +gflags.DEFINE_boolean('start_devserver', False, 'Start devserver.') class KillableProcess(): """A killable process. @@ -77,7 +77,7 @@ class KillableProcess(): def start_devserver(): """Starts devserver.""" - cmd = 'python devserver.py' + cmd = 'python devserver.py --factory_config miniomaha.conf' print 'Running command: %s' % cmd devserver_process = KillableProcess(cmd, cwd=DEVSERVER_DIR) devserver_process.start(wait=False) @@ -174,7 +174,8 @@ def main(argv): FLAGS.firmware_updater, folder=FLAGS.board, board=FLAGS.board) - start_devserver() + if FLAGS.start_devserver: + start_devserver() if __name__ == '__main__':