From cd33866c896d7d52cd520a75aef8adaa82a818ff Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Mon, 29 Nov 2010 13:30:42 -0800 Subject: [PATCH 01/31] Add RunCommandCaptureOutput as a helper method. It's like RunCommand, but you can run the command, capture it's error code and output together. This is to support tests that need to capture output of image_to_live, and to know if it succeeded. Change-Id: If674b6d79697c0f0b5c96be9fc83adbed9b9893e BUG=chromium-os:9502 TEST=Ran by hand Review URL: http://codereview.chromium.org/5339006 --- lib/cros_build_lib.py | 46 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/lib/cros_build_lib.py b/lib/cros_build_lib.py index 0b728efd38..b45733eeda 100644 --- a/lib/cros_build_lib.py +++ b/lib/cros_build_lib.py @@ -93,6 +93,51 @@ def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None, return output +def RunCommandCaptureOutput(cmd, print_cmd=True, cwd=None, input=None, + enter_chroot=False, + combine_stdout_stderr=True): + """Runs a shell command. Differs from RunCommand, because it allows + you to run a command and capture the exit code, output, and stderr + all at the same time. + + Arguments: + cmd: cmd to run. Should be input to subprocess.POpen. If a string, + converted to an array using split(). + print_cmd: prints the command before running it. + cwd: the working directory to run this cmd. + input: input to pipe into this command through stdin. + enter_chroot: this command should be run from within the chroot. If set, + cwd must point to the scripts directory. + combine_stdout_stderr -- combine outputs together. + + Returns: + Returns a tuple: (exit_code, stdout, stderr) (integer, string, string) + stderr is None if combine_stdout_stderr is True + """ + # Set default for variables. + stdout = subprocess.PIPE + stderr = subprocess.PIPE + stdin = None + + # Modify defaults based on parameters. + if input: stdin = subprocess.PIPE + if combine_stdout_stderr: stderr = subprocess.STDOUT + + if enter_chroot: cmd = ['./enter_chroot.sh', '--'] + cmd + + # Print out the command before running. + if print_cmd: + Info('PROGRAM(%s) -> RunCommand: %r in dir %s' % + (GetCallerName(), cmd, cwd)) + + proc = subprocess.Popen(cmd, cwd=cwd, stdin=stdin, + stdout=stdout, stderr=stderr) + (output, error) = proc.communicate(input) + + # Error is None if stdout, stderr are combined. + return proc.returncode, output, error + + class Color(object): """Conditionally wraps text in ANSI color escape sequences.""" BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8) @@ -191,4 +236,3 @@ def ReinterpretPathForChroot(path): new_path = os.path.join('/home', os.getenv('USER'), 'trunk', relative_path) return new_path - From 6fbd3d123cfb8d68ebd666b02255fe713a438a9d Mon Sep 17 00:00:00 2001 From: Eric Li Date: Mon, 29 Nov 2010 14:13:15 -0800 Subject: [PATCH 02/31] Refactor run_remote_tests script. What I did: 1. enter_chroot once and only once if necessary. The previous version would enter/exit chroot 3 times in worst case. And the entire logic happens inside chroot only, this has greatly simplified the overall workflow, and reduced the number of variables used. Shell script variables could be tricky. 2. Change variable type to test_type since type is a bash builtin. 3. get rid of {$TMP}/run_test.sh script, since it is not necessary any more. 4. get rid of rsync step from third_party/autotest/files to ${BUILD_DIR}, since it is not neccessary either. overall, reduced ~40 lines of code. All the change should be transparent to end users and there should be no regression changes at all. w/wo emerge autotest w/wo cros_workon in/outside of chroot. and all its combinations. Change-Id: I9c1532e9cb6cc0e724d4b6d870723df3e2a147ec BUG=9291 TEST=Run storageFio test since it need prebuild test and deps. Review URL: http://codereview.chromium.org/5176009 --- run_remote_tests.sh | 186 ++++++++++++++++++++------------------------ 1 file changed, 83 insertions(+), 103 deletions(-) diff --git a/run_remote_tests.sh b/run_remote_tests.sh index 0c644d46f1..e920bb61bc 100755 --- a/run_remote_tests.sh +++ b/run_remote_tests.sh @@ -48,83 +48,94 @@ function cleanup() { function read_test_type() { local control_file=$1 # Assume a line starts with TEST_TYPE = - local type=$(egrep -m1 \ - '^[[:space:]]*TEST_TYPE[[:space:]]*=' "${control_file}") - if [[ -z "${type}" ]]; then + local test_type=$(egrep -m1 \ + '^[[:space:]]*TEST_TYPE[[:space:]]*=' "${control_file}") + if [[ -z "${test_type}" ]]; then die "Unable to find TEST_TYPE line in ${control_file}" fi - type=$(python -c "${type}; print TEST_TYPE.lower()") - if [[ "${type}" != "client" ]] && [[ "${type}" != "server" ]]; then - die "Unknown type of test (${type}) in ${control_file}" + test_type=$(python -c "${test_type}; print TEST_TYPE.lower()") + if [[ "${test_type}" != "client" ]] && [[ "${test_type}" != "server" ]]; then + die "Unknown type of test (${test_type}) in ${control_file}" fi - echo ${type} + echo ${test_type} } function create_tmp() { # Set global TMP for remote_access.sh's sake # and if --results_dir_root is specified, # set TMP and create dir appropriately - if [[ ${INSIDE_CHROOT} -eq 0 ]]; then - if [[ -n "${FLAGS_results_dir_root}" ]]; then - TMP=${FLAGS_chroot}${FLAGS_results_dir_root} - mkdir -p -m 777 ${TMP} - else - TMP=$(mktemp -d ${FLAGS_chroot}/tmp/run_remote_tests.XXXX) - fi - TMP_INSIDE_CHROOT=$(echo ${TMP#${FLAGS_chroot}}) + if [[ -n "${FLAGS_results_dir_root}" ]]; then + TMP=${FLAGS_results_dir_root} + mkdir -p -m 777 ${TMP} else - if [[ -n "${FLAGS_results_dir_root}" ]]; then - TMP=${FLAGS_results_dir_root} - mkdir -p -m 777 ${TMP} - else - TMP=$(mktemp -d /tmp/run_remote_tests.XXXX) - fi - TMP_INSIDE_CHROOT=${TMP} + TMP=$(mktemp -d /tmp/run_remote_tests.XXXX) fi } -function prepare_build_dir() { - local autotest_dir="$1" - INSIDE_BUILD_DIR="${TMP_INSIDE_CHROOT}/build" - BUILD_DIR="${TMP}/build" - info "Copying autotest tree into ${BUILD_DIR}." - sudo mkdir -p "${BUILD_DIR}" - sudo rsync -rl --chmod=ugo=rwx "${autotest_dir}"/ "${BUILD_DIR}" +function prepare_build_env() { info "Pilfering toolchain shell environment from Portage." - local outside_ebuild_dir="${TMP}/chromeos-base/autotest-build" - local inside_ebuild_dir="${TMP_INSIDE_CHROOT}/chromeos-base/autotest-build" - mkdir -p "${outside_ebuild_dir}" + local ebuild_dir="${TMP}/chromeos-base/autotest-build" + mkdir -p "${ebuild_dir}" local E_only="autotest-build-9999.ebuild" - cat > "${outside_ebuild_dir}/${E_only}" < "${ebuild_dir}/${E_only}" <&1 > /dev/null - local P_tmp="${FLAGS_chroot}/build/${FLAGS_board}/tmp/portage/" + local P_tmp="/build/${FLAGS_board}/tmp/portage/" local E_dir="${E%%/*}/${E_only%.*}" - sudo cp "${P_tmp}/${E_dir}/temp/environment" "${BUILD_DIR}" + export BUILD_ENV="${P_tmp}/${E_dir}/temp/environment" } function autodetect_build() { if [ ${FLAGS_use_emerged} -eq ${FLAGS_TRUE} ]; then - info \ -"As requested, using emerged autotests already installed in your sysroot." + AUTOTEST_DIR="/build/${FLAGS_board}/usr/local/autotest" FLAGS_build=${FLAGS_FALSE} + if [ ! -d "${AUTOTEST_DIR}" ]; then + die \ +"Could not find pre-installed autotest, you need to emerge-${FLAGS_board} \ +autotest autotest-tests (or use --build)." + fi + info \ +"As requested, using emerged autotests already installed at ${AUTOTEST_DIR}." return fi - if ${ENTER_CHROOT} ./cros_workon --board=${FLAGS_board} list | \ - grep -q autotest; then - info \ -"Detected cros_workon autotests, building your sources instead of emerged \ -autotest. To use installed autotest, pass --use_emerged." + + if [ ${FLAGS_build} -eq ${FLAGS_FALSE} ] && + $(dirname $0)/cros_workon --board=${FLAGS_board} list | + grep -q autotest; then + AUTOTEST_DIR="${SRC_ROOT}/third_party/autotest/files" FLAGS_build=${FLAGS_TRUE} - else + if [ ! -d "${AUTOTEST_DIR}" ]; then + die \ +"Detected cros_workon autotest but ${AUTOTEST_DIR} does not exist. Run \ +repo sync autotest." + fi info \ -"Using emerged autotests already installed in your sysroot. To build \ -autotests directly from your source directory instead, pass --build." - FLAGS_build=${FLAGS_FALSE} +"Detected cros_workon autotests. Building and running your autotests from \ +${AUTOTEST_DIR}. To use emerged autotest, pass --use_emerged." + return + fi + + # flag use_emerged should be false once the code reaches here. + if [ ${FLAGS_build} -eq ${FLAGS_TRUE} ]; then + AUTOTEST_DIR="${SRC_ROOT}/third_party/autotest/files" + if [ ! -d "${AUTOTEST_DIR}" ]; then + die \ +"Build flag was turned on but ${AUTOTEST_DIR} is not found. Run cros_workon \ +start autotest and repo sync to continue." + fi + info "Build and run autotests from ${AUTOTEST_DIR}." + else + AUTOTEST_DIR="/build/${FLAGS_board}/usr/local/autotest" + if [ ! -d "${AUTOTEST_DIR}" ]; then + die \ +"Autotest was not emerged. Run emerge-${FLAGS_board} autotest \ +autotest-tests to continue." + fi + info "Using emerged autotests already installed at ${AUTOTEST_DIR}." fi } @@ -160,24 +171,7 @@ function main() { remote_access_init learn_board - autotest_dir="${FLAGS_chroot}/build/${FLAGS_board}/usr/local/autotest" - - ENTER_CHROOT="" - if [[ ${INSIDE_CHROOT} -eq 0 ]]; then - ENTER_CHROOT="./enter_chroot.sh --chroot ${FLAGS_chroot} --" - fi - - if [ ${FLAGS_build} -eq ${FLAGS_FALSE} ]; then - autodetect_build - fi - - if [ ${FLAGS_build} -eq ${FLAGS_TRUE} ]; then - autotest_dir="${SRC_ROOT}/third_party/autotest/files" - else - if [ ! -d "${autotest_dir}" ]; then - die "You need to emerge autotest-tests (or use --build)" - fi - fi + autodetect_build local control_files_to_run="" local chrome_autotests="${CHROME_ROOT}/src/chrome/test/chromeos/autotest/files" @@ -187,7 +181,8 @@ function main() { if [ -n "${CHROME_ROOT}" ]; then search_path="${search_path} ${chrome_autotests}/client/site_tests" fi - pushd ${autotest_dir} > /dev/null + + pushd ${AUTOTEST_DIR} > /dev/null for test_request in $FLAGS_ARGV; do test_request=$(remove_quotes "${test_request}") ! finds=$(find ${search_path} -maxdepth 2 -type f \( -name control.\* -or \ @@ -208,7 +203,6 @@ function main() { control_files_to_run="${control_files_to_run} '${finds}'" done done - popd > /dev/null echo "" @@ -216,29 +210,29 @@ function main() { die "Found no control files" fi - [ ${FLAGS_build} -eq ${FLAGS_TRUE} ] && prepare_build_dir "${autotest_dir}" + [ ${FLAGS_build} -eq ${FLAGS_TRUE} ] && prepare_build_env info "Running the following control files:" - for CONTROL_FILE in ${control_files_to_run}; do - info " * ${CONTROL_FILE}" + for control_file in ${control_files_to_run}; do + info " * ${control_file}" done for control_file in ${control_files_to_run}; do # Assume a line starts with TEST_TYPE = control_file=$(remove_quotes "${control_file}") - local type=$(read_test_type "${autotest_dir}/${control_file}") + local test_type=$(read_test_type "${AUTOTEST_DIR}/${control_file}") # Check if the control file is an absolute path (i.e. chrome autotests case) if [[ ${control_file:0:1} == "/" ]]; then - type=$(read_test_type "${control_file}") + test_type=$(read_test_type "${control_file}") fi local option - if [[ "${type}" == "client" ]]; then + if [[ "${test_type}" == "client" ]]; then option="-c" else option="-s" fi echo "" - info "Running ${type} test ${control_file}" + info "Running ${test_type} test ${control_file}" local control_file_name=$(basename "${control_file}") local short_name=$(basename $(dirname "${control_file}")) @@ -255,7 +249,7 @@ function main() { fi local results_dir_name="${short_name}" - local results_dir="${TMP_INSIDE_CHROOT}/${results_dir_name}" + local results_dir="${TMP}/${results_dir_name}" rm -rf "${results_dir}" local verbose="" if [[ ${FLAGS_verbose} -eq $FLAGS_TRUE ]]; then @@ -271,39 +265,24 @@ function main() { info "Running chrome autotest ${control_file}" fi - local autoserv_test_args="${FLAGS_args}" - if [ -n "${autoserv_test_args}" ]; then - autoserv_test_args="-a \"${autoserv_test_args}\"" - fi local autoserv_args="-m ${FLAGS_remote} --ssh-port ${FLAGS_ssh_port} \ ${option} ${control_file} -r ${results_dir} ${verbose}" - if [ ${FLAGS_build} -eq ${FLAGS_FALSE} ]; then - cat > "${TMP}/run_test.sh" <&2 + if [ -n "${FLAGS_args}" ]; then + autoserv_args="${autoserv_args} -a \""${FLAGS_args}"\"" + fi + + sudo chmod a+w ./server/{tests,site_tests} + echo ./server/autoserv ${autoserv_args} + + if [ ${FLAGS_build} -eq ${FLAGS_TRUE} ]; then + # run autoserv in subshell + (. ${BUILD_ENV} && tc-export CC CXX PKG_CONFIG && + ./server/autoserv ${autoserv_args}) else - cp "${BUILD_DIR}/environment" "${TMP}/run_test.sh" - GRAPHICS_BACKEND=${GRAPHICS_BACKEND:-OPENGL} - cat >> "${TMP}/run_test.sh" <&2 + ./server/autoserv ${autoserv_args} fi done + popd > /dev/null echo "" info "Test results:" @@ -312,4 +291,5 @@ EOF print_time_elapsed } +restart_in_chroot_if_needed $* main "$@" From 330375f6926420014caf267cb25103431d75f74c Mon Sep 17 00:00:00 2001 From: Chris Masone Date: Mon, 29 Nov 2010 15:06:53 -0800 Subject: [PATCH 03/31] [crosutils] Add confirmation warning to cros_workon_make --scrub BUG=9717 TEST=run cros_workon_make --scrub and ensure that it only scrubs if the user types y or yes (case insensitive) Change-Id: Ic1f8020fcca8bd57924391fcbf075d3e2adc5e4a Review URL: http://codereview.chromium.org/5265009 --- bin/cros_workon_make | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/bin/cros_workon_make b/bin/cros_workon_make index 95cc709a5d..da2a3ebca5 100755 --- a/bin/cros_workon_make +++ b/bin/cros_workon_make @@ -58,12 +58,18 @@ if ! pkgfile=$("${EQUERYCMD}" which "${workon_name}" 2> /dev/null); then fi if [ "${FLAGS_scrub}" = "${FLAGS_TRUE}" ]; then - eval $(${EBUILDCMD} $(${EQUERYCMD} which ${workon_name}) info) - srcdir=$(readlink -m ${CROS_WORKON_SRCDIR}) - trunkdir=$(readlink -m ${CHROOT_TRUNK_DIR}) - project_path=${srcdir#${trunkdir}/} - if ! (cd "${GCLIENT_ROOT}/${project_path}" && git clean -xf); then - die "Could not scrub source directory" + warn "--scrub will destroy ALL FILES unknown to git!" + read -p "Are you sure you want to do this? [y|N]" resp + if egrep -qi "^y(es)?$" <(echo -n "${resp}"); then + eval $(${EBUILDCMD} $(${EQUERYCMD} which ${workon_name}) info) + srcdir=$(readlink -m ${CROS_WORKON_SRCDIR}) + trunkdir=$(readlink -m ${CHROOT_TRUNK_DIR}) + project_path=${srcdir#${trunkdir}/} + if ! (cd "${GCLIENT_ROOT}/${project_path}" && git clean -xf); then + die "Could not scrub source directory" + fi + else + info "Not scrubbing; exiting gracefully" fi exit 0 fi From 53698c3865f89d8c33735611ccdb65ca101c4670 Mon Sep 17 00:00:00 2001 From: Eric Li Date: Mon, 29 Nov 2010 16:35:47 -0800 Subject: [PATCH 04/31] Remove unused autotest scripts. Change-Id: Ia627995d235732602c34494f11aa6ae7058209c3 BUG=9291 TEST=None Review URL: http://codereview.chromium.org/5278013 --- autotest | 232 ---------------------------------------------- autotest_lib.sh | 56 ----------- build_autotest.sh | 32 ------- 3 files changed, 320 deletions(-) delete mode 100755 autotest delete mode 100644 autotest_lib.sh delete mode 100755 build_autotest.sh diff --git a/autotest b/autotest deleted file mode 100755 index e5065950d7..0000000000 --- a/autotest +++ /dev/null @@ -1,232 +0,0 @@ -#!/usr/bin/python - -# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -# -# A python wrapper to call autotest ebuild. - -import commands, logging, optparse, os, subprocess, sys - - -def run(cmd): - return subprocess.call(cmd, stdout=sys.stdout, stderr=sys.stderr) - - -class MyOptionParser(optparse.OptionParser): - """Override python's builtin OptionParser to accept any undefined args.""" - - help = False - - def _process_args(self, largs, rargs, values): - # see /usr/lib64/python2.6/optparse.py line 1414-1463 - while rargs: - arg = rargs[0] - # We handle bare "--" explicitly, and bare "-" is handled by the - # standard arg handler since the short arg case ensures that the - # len of the opt string is greater than 1. - if arg == "--": - del rargs[0] - return - elif arg[0:2] == "--": - # process a single long option (possibly with value(s)) - try: - self._process_long_opt(rargs, values) - except optparse.BadOptionError: - largs.append(arg) - elif arg[:1] == "-" and len(arg) > 1: - # process a cluster of short options (possibly with - # value(s) for the last one only) - try: - self._process_short_opts(rargs, values) - except optparse.BadOptionError: - largs.append(arg) - elif self.allow_interspersed_args: - largs.append(arg) - del rargs[0] - else: - return # stop now, leave this arg in rargs - - def print_help(self, file=None): - optparse.OptionParser.print_help(self, file) - MyOptionParser.help = True - - -parser = MyOptionParser() -parser.allow_interspersed_args = True - -DEFAULT_BOARD = os.environ.get('DEFAULT_BOARD', '') - -parser.add_option('--args', dest='args', action='store', - default='', - help='The arguments to pass to the test control file.') -parser.add_option('--autox', dest='autox', action='store_true', - default=True, - help='Build autox along with autotest [default].') -parser.add_option('--noautox', dest='autox', action='store_false', - help='Don\'t build autox along with autotest.') -parser.add_option('--board', dest='board', action='store', - default=DEFAULT_BOARD, - help='The board for which you are building autotest.') -parser.add_option('--build', dest='build', action='store', - help='Only prebuild client tests, do not run tests.') -parser.add_option('--buildcheck', dest='buildcheck', action='store_true', - default=True, - help='Fail if tests fail to build [default].') -parser.add_option('--nobuildcheck', dest='buildcheck', action='store_false', - help='Ignore test build failures.') -parser.add_option('--jobs', dest='jobs', action='store', type=int, - default=-1, - help='How many packages to build in parallel at maximum.') -parser.add_option('--noprompt', dest='noprompt', action='store_true', - help='Prompt user when building all tests.') - - -AUTOSERV='../third_party/autotest/files/server/autoserv' -AUTOTEST_CLIENT='../third_party/autotest/files/client/bin/autotest_client' - -def parse_args_and_help(): - - def nop(_): - pass - - sys_exit = sys.exit - sys.exit = nop - options, args = parser.parse_args() - sys.exit = sys_exit - - if not args and not options.build: - parser.print_help() - - if MyOptionParser.help: - if options.build: - print - print 'Options inherited from autotest_client, which is used in build', - print 'only mode.' - run([AUTOTEST_CLIENT, '--help']) - else: - print - print 'Options inherited from autoserv:' - run([AUTOSERV, '--help']) - sys.exit(0) - return options, args - - -def assert_inside_chroot(common_sh): - status, output = commands.getstatusoutput('/bin/bash -c ". %s && ' - 'assert_inside_chroot"' % common_sh) - if status is not 0: - print >> sys.stderr, output - sys.exit(status) - - -def set_common_env(common_sh, env_var): - env_value = commands.getoutput('/bin/bash -c \'. %s && echo $%s\'' % - (common_sh, env_var)) - os.environ[env_var] = env_value - - -def die(common_sh, msg): - output = commands.getoutput('/bin/bash -c \'. %s && die "%s"\'' % - (common_sh, msg)) - print >> sys.stderr, output - sys.exit(1) - - -def build_autotest(options): - environ = os.environ - if options.jobs != -1: - emerge_jobs = '--jobs=%d' % options.jobs - else: - emerge_jobs = '' - - # Decide on USE flags based on options - use_flag = environ.get('USE', '') - if not options.autox: - use_flag = use_flag + ' -autox' - if options.buildcheck: - use_flag = use_flag + ' buildcheck' - - board_blacklist_file = ('%s/src/overlays/overlay-%s/autotest-blacklist' % - (os.environ['GCLIENT_ROOT'], options.board)) - if os.path.exists(board_blacklist_file): - blacklist = [line.strip() - for line in open(board_blacklist_file).readlines()] - else: - blacklist = [] - - all_tests = ('compilebench,dbench,disktest,fsx,hackbench,iperf,netperf2,' - 'netpipe,unixbench') - site_tests = '../third_party/autotest/files/client/site_tests' - for site_test in os.listdir(site_tests): - test_path = os.path.join(site_tests, site_test) - test_py = os.path.join(test_path, '%s.py' % site_test) - if (os.path.exists(test_path) and os.path.isdir(test_path) and - os.path.exists(test_py) and os.path.isfile(test_py) and - site_test not in blacklist): - all_tests += ',' + site_test - - if 'all' == options.build.lower(): - if options.noprompt is not True: - print 'You want to pre-build all client tests and it may take a long', - print 'time to finish.' - print 'Are you sure you want to continue?(N/y)', - answer = sys.stdin.readline() - if 'y' != answer[0].lower(): - print 'Use --build to specify tests you like to pre-compile. ' - print 'E.g.: ./autotest --build=disktest,hardware_SAT' - sys.exit(0) - test_list = all_tests - else: - test_list = options.build - - environ['FEATURES'] = ('%s -buildpkg -collision-protect' % - environ.get('FEATURES', '')) - environ['TEST_LIST'] = test_list - environ['USE'] = use_flag - emerge_cmd = ['emerge-%s' % options.board, - 'chromeos-base/autotest'] - if emerge_jobs: - emerge_cmd.append(emerge_jobs) - return run(emerge_cmd) - - -def run_autoserv(options, args): - environ = os.environ - - environ['AUTOSERV_TEST_ARGS'] = options.args - environ['AUTOSERV_ARGS'] = ' '.join(args) - environ['FEATURES'] = ('%s -buildpkg -digest noauto' % - environ.get('FEATURES', '')) - ebuild_cmd = ['ebuild-%s' % options.board, - '../third_party/chromiumos-overlay/chromeos-base/' - 'autotest/autotest-0.0.1.ebuild', - 'clean', 'unpack', 'test'] - run(ebuild_cmd) - - -def main(): - me = sys.argv[0] - common_sh = os.path.join(os.path.dirname(me), 'common.sh') - - assert_inside_chroot(common_sh) - set_common_env(common_sh, 'GCLIENT_ROOT') - - options, args = parse_args_and_help() - - if not options.board: - die(common_sh, 'Missing --board argument.') - - if options.build: - status = build_autotest(options) - if status: - die(common_sh, 'build_autotest failed.') - else: - ssh_key_file = os.path.join(os.path.dirname(me), - 'mod_for_test_scripts/ssh_keys/testing_rsa') - os.chmod(ssh_key_file, 0400) - run_autoserv(options, args) - - -if __name__ == '__main__': - main() diff --git a/autotest_lib.sh b/autotest_lib.sh deleted file mode 100644 index 311bad46fe..0000000000 --- a/autotest_lib.sh +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -# -# Provides common commands for dealing running/building autotest - -. "$(dirname "$0")/common.sh" - -get_default_board - -DEFINE_string board "$DEFAULT_BOARD" \ - "The board for which you are building autotest" - -function check_board() { - local board_names="" - local index=1 - local found=0 - local board_basename=$(echo "${FLAGS_board}" |cut -d '_' -f 1) - for overlay_path in "${SRC_ROOT}"/overlays/overlay-* - do - local overlay=$(basename "${overlay_path}") - local board="${overlay#overlay-}" - board_names[index]="${board}" - index+=1 - if [ "${board_basename}" == "${board}" ] - then - found=1 - fi - done - - if [ ${found} -eq 0 ] - then - echo "You are required to specify a supported board from the command line." - echo "Supported boards are:" - for board in ${board_names[@]} - do - echo ${board} - done - exit 0 - fi -} - - -# Populates the chroot's /usr/local/autotest/$FLAGS_board directory based on -# the given source directory. -# args: -# $1 - original source directory -# $2 - target directory -function update_chroot_autotest() { - local original=$1 - local target=$2 - echo "Updating chroot Autotest from ${original} to ${target}..." - sudo mkdir -p "${target}" - sudo chmod 777 "${target}" - cp -fpru ${original}/{client,conmux,server,tko,utils,global_config.ini,shadow_config.ini} ${target} -} diff --git a/build_autotest.sh b/build_autotest.sh deleted file mode 100755 index d974e4d31e..0000000000 --- a/build_autotest.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2009 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# This script makes autotest client tests inside a chroot environment. The idea -# is to compile any platform-dependent autotest client tests in the build -# environment, since client systems under test lack the proper toolchain. -# -# The user can later run autotest against an ssh enabled test client system, or -# install the compiled client tests directly onto the rootfs image. - -. "$(dirname "$0")/common.sh" - -get_default_board - -DEFINE_string board "$DEFAULT_BOARD" \ - "The board for which you are building autotest" - -FLAGS "$@" || exit 1 - -if [[ -n "${CROS_WORKON_SRCROOT}" ]]; then - if [[ -z "${FLAGS_board}" ]]; then - setup_board_warning - exit 1 - fi - emerge-${FLAGS_board} autotest-all -else - ./autotest --noprompt --build=all --board="${FLAGS_board}" $@ -fi - From fb5ce169eec96811a359a7fdc6a608584a16a2ed Mon Sep 17 00:00:00 2001 From: Tom Wai-Hong Tam Date: Tue, 30 Nov 2010 11:22:50 +0800 Subject: [PATCH 05/31] For the boards already have DB directories, remove the default DB. The default DB is an all-pass DB. If no DB found, the hardware_Components test falls back to the all-pass one and skip the real check. In order to prevent skipping the check, remove the default DB in the boards already have DB directories. TEST=run "image_to_usb.sh --factory" and check the default DB removed BUG=chrome-os-partner:1737 Change-Id: Ia25f2551a5eba3a95fe736e86bfe52c340f329a8 Review URL: http://codereview.chromium.org/5339011 --- mod_for_factory_scripts/500populateQualDbs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mod_for_factory_scripts/500populateQualDbs b/mod_for_factory_scripts/500populateQualDbs index 77f73ec1e2..2d869413b3 100755 --- a/mod_for_factory_scripts/500populateQualDbs +++ b/mod_for_factory_scripts/500populateQualDbs @@ -13,10 +13,14 @@ if [ -d "${TEST_DIR}" ]; then KEEPDB="data_${BOARD}" ls -d data_* 2>/dev/null | grep -v "${KEEPDB}" | xargs rm -fr - # Ensure there is DB directory in x86-agz and x86-mario. - if [ ! -d "${KEEPDB}" ] && - [ "${BOARD}" = "x86-agz" -o "${BOARD}" = "x86-mario" ]; then - echo "No component DB directory found at: ${KEEPDB}" + if [ "${BOARD}" = "x86-agz" -o "${BOARD}" = "x86-mario" ]; then + # Ensure there is a DB directory in x86-agz or x86-mario. + if [ ! -d "${KEEPDB}" ]; then + echo "No component DB directory found at: ${KEEPDB}" + fi + # Remove the default DB since it is unnecessary. + DEFAULTDB="approved_components.default" + rm -f "${DEFAULTDB}" fi popd >/dev/null From c509a906da80a5b6b918a2816ed4d285dfef37b0 Mon Sep 17 00:00:00 2001 From: David James Date: Tue, 30 Nov 2010 11:36:57 -0800 Subject: [PATCH 06/31] Update cbuildbot.py to upload prebuilts from preflight buildbot. This change updates cbuildbot.py to upload prebuilts and reference them via PREFLIGHT_BINHOST. Also ensure make.conf ends in a newline, as it looks nicer. BUG=chromium-os:5311 TEST=Test that prebuilt.py updates PREFLIGHT_BINHOST in right make.conf file. Test that cbuildbot.py runs prebuilt.py with right arguments. Run unit tests for cbuildbot and prebuilt.py. Test runs of cbuildbot.py with --dryrun. Test upload of Packages file to local web server and check that emerge works with them. Change-Id: Iad03c6c469e05b9ee1cceff69cbe2bdd51225e25 Review URL: http://codereview.chromium.org/4969003 --- bin/cbuildbot.py | 71 ++++++++++++++++++++++++++++++++++++--- bin/cbuildbot_unittest.py | 39 +++++++++++++++++++-- prebuilt.py | 70 +++++++++++++++++++++++++++++++------- 3 files changed, 161 insertions(+), 19 deletions(-) diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index f6258d12fc..2a6e22b68e 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -26,6 +26,13 @@ _PACKAGE_FILE = '%(buildroot)s/src/scripts/cbuildbot_package.list' ARCHIVE_BASE = '/var/www/archive' ARCHIVE_COUNT = 10 +# Currently, both the full buildbot and the preflight buildbot store their +# data in a variable named PORTAGE_BINHOST, but they're in different files. +# We're planning on joining the two files soon and renaming the full binhost +# to FULL_BINHOST. +_FULL_BINHOST = 'PORTAGE_BINHOST' +_PREFLIGHT_BINHOST = 'PORTAGE_BINHOST' + # ======================== Utility functions ================================ def MakeDir(path, parents=False): @@ -305,6 +312,25 @@ def _MakeChroot(buildroot): RunCommand(['./make_chroot', '--fast'], cwd=cwd) +def _GetPortageEnvVar(buildroot, board, envvar): + """Get a portage environment variable for the specified board, if any. + + buildroot: The root directory where the build occurs. Must be an absolute + path. + board: Board type that was built on this machine. E.g. x86-generic. + envvar: The environment variable to get. E.g. "PORTAGE_BINHOST". + + Returns: + The value of the environment variable, as a string. If no such variable + can be found, return the empty string. + """ + cwd = os.path.join(buildroot, 'src', 'scripts') + binhost = RunCommand(['portageq-%s' % board, 'envvar', envvar], + cwd=cwd, redirect_stdout=True, enter_chroot=True, + error_ok=True) + return binhost.rstrip('\n') + + def _SetupBoard(buildroot, board='x86-generic'): """Wrapper around setup_board.""" cwd = os.path.join(buildroot, 'src', 'scripts') @@ -507,6 +533,35 @@ def _ResolveOverlays(buildroot, overlays): return paths +def _UploadPrebuilts(buildroot, board, overlay_config): + """Upload prebuilts. + + Args: + buildroot: The root directory where the build occurs. + board: Board type that was built on this machine + overlay_config: A string describing which overlays you want. + 'private': Just the private overlay. + 'public': Just the public overlay. + 'both': Both the public and private overlays. + """ + + cwd = os.path.join(buildroot, 'src', 'scripts') + cmd = [os.path.join(cwd, 'prebuilt.py'), + '--sync-binhost-conf', + '--build-path', buildroot, + '--board', board, + '--prepend-version', 'preflight', + '--key', _PREFLIGHT_BINHOST] + if overlay_config == 'public': + cmd.extend(['--upload', 'gs://chromeos-prebuilt']) + else: + assert overlay_config in ('private', 'both') + cmd.extend(['--upload', 'chromeos-images:/var/www/prebuilt/', + '--binhost-base-url', 'http://chromeos-prebuilt']) + + RunCommand(cmd, cwd=cwd) + + def main(): # Parse options usage = "usage: %prog [options] cbuildbot_config" @@ -558,16 +613,23 @@ def main(): parser.print_usage() sys.exit(1) - # Calculate list of overlay directories. - overlays = _ResolveOverlays(buildroot, buildconfig['overlays']) - try: + # Calculate list of overlay directories. + overlays = _ResolveOverlays(buildroot, buildconfig['overlays']) + board = buildconfig['board'] + _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays) + chroot_path = os.path.join(buildroot, 'chroot') + boardpath = os.path.join(chroot_path, 'build', board) if options.sync: if options.clobber or not os.path.isdir(buildroot): _FullCheckout(buildroot, tracking_branch, url=options.url) else: + old_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST) _IncrementalCheckout(buildroot) + new_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST) + if old_binhost != new_binhost: + RunCommand(['sudo', 'rm', '-rf', boardpath]) # Check that all overlays can be found. for path in overlays: @@ -575,11 +637,9 @@ def main(): if not os.path.isdir(path): Die('Missing overlay: %s' % path) - chroot_path = os.path.join(buildroot, 'chroot') if not os.path.isdir(chroot_path): _MakeChroot(buildroot) - boardpath = os.path.join(chroot_path, 'build', buildconfig['board']) if not os.path.isdir(boardpath): _SetupBoard(buildroot, board=buildconfig['board']) @@ -624,6 +684,7 @@ def main(): if buildconfig['master']: # Master bot needs to check if the other slaves completed. if cbuildbot_comm.HaveSlavesCompleted(config): + _UploadPrebuilts(buildroot, board, buildconfig['overlays']) _UprevPush(buildroot, tracking_branch, buildconfig['board'], overlays, options.debug) else: diff --git a/bin/cbuildbot_unittest.py b/bin/cbuildbot_unittest.py index ba9b545b0d..87bfc363c2 100755 --- a/bin/cbuildbot_unittest.py +++ b/bin/cbuildbot_unittest.py @@ -151,10 +151,13 @@ class CBuildBotTest(mox.MoxTestBase): m_file.read().AndReturn(self._test_string) m_file.close() + drop_file = cbuildbot._PACKAGE_FILE % {'buildroot': self._buildroot} cbuildbot.RunCommand(['./cros_mark_as_stable', '--all', '--board=%s' % self._test_board, '--overlays=%s' % ':'.join(self._chroot_overlays), - '--tracking_branch=cros/master', 'commit'], + '--tracking_branch=cros/master', + '--drop_file=%s' % ReinterpretPathForChroot(drop_file), + 'commit'], cwd='%s/src/scripts' % self._buildroot, enter_chroot=True) @@ -174,10 +177,13 @@ class CBuildBotTest(mox.MoxTestBase): m_file.read().AndReturn('None') m_file.close() + drop_file = cbuildbot._PACKAGE_FILE % {'buildroot': self._buildroot} cbuildbot.RunCommand(['./cros_mark_as_stable', '--all', '--board=%s' % self._test_board, '--overlays=%s' % ':'.join(self._chroot_overlays), - '--tracking_branch=cros/master', 'commit'], + '--tracking_branch=cros/master', + '--drop_file=%s' % ReinterpretPathForChroot(drop_file), + 'commit'], cwd='%s/src/scripts' % self._buildroot, enter_chroot=True) @@ -187,6 +193,35 @@ class CBuildBotTest(mox.MoxTestBase): self._overlays) self.mox.VerifyAll() + def testGetPortageEnvVar(self): + """Basic test case for _GetPortageEnvVar function.""" + envvar = 'EXAMPLE' + cbuildbot.RunCommand(mox.And(mox.IsA(list), mox.In(envvar)), + cwd='%s/src/scripts' % self._buildroot, + redirect_stdout=True, enter_chroot=True, + error_ok=True).AndReturn('RESULT\n') + self.mox.ReplayAll() + result = cbuildbot._GetPortageEnvVar(self._buildroot, self._test_board, + envvar) + self.mox.VerifyAll() + self.assertEqual(result, 'RESULT') + + def testUploadPublicPrebuilts(self): + """Test _UploadPrebuilts with a public location.""" + check = mox.And(mox.IsA(list), mox.In('gs://chromeos-prebuilt')) + cbuildbot.RunCommand(check, cwd='%s/src/scripts' % self._buildroot) + self.mox.ReplayAll() + cbuildbot._UploadPrebuilts(self._buildroot, self._test_board, 'public') + self.mox.VerifyAll() + + def testUploadPrivatePrebuilts(self): + """Test _UploadPrebuilts with a private location.""" + check = mox.And(mox.IsA(list), mox.In('chromeos-images:/var/www/prebuilt/')) + cbuildbot.RunCommand(check, cwd='%s/src/scripts' % self._buildroot) + self.mox.ReplayAll() + cbuildbot._UploadPrebuilts(self._buildroot, self._test_board, 'private') + self.mox.VerifyAll() + if __name__ == '__main__': unittest.main() diff --git a/prebuilt.py b/prebuilt.py index ee9ecf1887..cb4e04f84a 100755 --- a/prebuilt.py +++ b/prebuilt.py @@ -59,6 +59,7 @@ _PREBUILT_BASE_DIR = 'src/third_party/chromiumos-overlay/chromeos/config/' # Created in the event of new host targets becoming available _PREBUILT_MAKE_CONF = {'amd64': os.path.join(_PREBUILT_BASE_DIR, 'make.conf.amd64-host')} +_BINHOST_CONF_DIR = 'src/third_party/chromiumos-overlay/chromeos/binhost' class FiltersEmpty(Exception): @@ -92,6 +93,7 @@ def UpdateLocalFile(filename, value, key='PORTAGE_BINHOST'): file_fh = open(filename) file_lines = [] found = False + keyval_str = '%(key)s=%(value)s' for line in file_fh: # Strip newlines from end of line. We already add newlines below. line = line.rstrip("\n") @@ -102,7 +104,6 @@ def UpdateLocalFile(filename, value, key='PORTAGE_BINHOST'): continue file_var, file_val = line.split('=') - keyval_str = '%(key)s=%(value)s' if file_var == key: found = True print 'Updating %s=%s to %s="%s"' % (file_var, file_val, key, value) @@ -117,7 +118,7 @@ def UpdateLocalFile(filename, value, key='PORTAGE_BINHOST'): file_fh.close() # write out new file new_file_fh = open(filename, 'w') - new_file_fh.write('\n'.join(file_lines)) + new_file_fh.write('\n'.join(file_lines) + '\n') new_file_fh.close() @@ -143,7 +144,7 @@ def RevGitPushWithRetry(retries=5): raise GitPushFailed('Failed to push change after %s retries' % retries) -def RevGitFile(filename, value, retries=5): +def RevGitFile(filename, value, retries=5, key='PORTAGE_BINHOST'): """Update and push the git file. Args: @@ -151,6 +152,8 @@ def RevGitFile(filename, value, retries=5): value: string representing the version of the prebuilt that has been uploaded. retries: The number of times to retry before giving up, default: 5 + key: The variable key to update in the git file. + (Default: PORTAGE_BINHOST) """ prebuilt_branch = 'prebuilt_branch' old_cwd = os.getcwd() @@ -162,10 +165,10 @@ def RevGitFile(filename, value, retries=5): 'git config url.ssh://git@gitrw.chromium.org:9222.pushinsteadof ' 'http://git.chromium.org/git') cros_build_lib.RunCommand(git_ssh_config_cmd, shell=True) - description = 'Update PORTAGE_BINHOST="%s" in %s' % (value, filename) + description = 'Update %s="%s" in %s' % (key, value, filename) print description try: - UpdateLocalFile(filename, value) + UpdateLocalFile(filename, value, key) cros_build_lib.RunCommand('git config push.default tracking', shell=True) cros_build_lib.RunCommand('git commit -am "%s"' % description, shell=True) RevGitPushWithRetry(retries) @@ -405,8 +408,32 @@ def DetermineMakeConfFile(target): return os.path.join(make_path) +def UpdateBinhostConfFile(path, key, value): + """Update binhost config file file with key=value. + + Args: + path: Filename to update. + key: Key to update. + value: New value for key. + """ + cwd = os.path.dirname(os.path.abspath(path)) + filename = os.path.basename(path) + if not os.path.isdir(cwd): + os.makedirs(cwd) + if not os.path.isfile(path): + config_file = file(path, 'w') + config_file.write('FULL_BINHOST="$PORTAGE_BINHOST"\n') + config_file.close() + UpdateLocalFile(path, value, key) + cros_build_lib.RunCommand('git add %s' % filename, cwd=cwd, shell=True) + description = 'Update %s=%s in %s' % (key, value, filename) + cros_build_lib.RunCommand('git commit -m "%s"' % description, cwd=cwd, + shell=True) + + def UploadPrebuilt(build_path, upload_location, version, binhost_base_url, - board=None, git_sync=False, git_sync_retries=5): + board=None, git_sync=False, git_sync_retries=5, + key='PORTAGE_BINHOST', sync_binhost_conf=False): """Upload Host prebuilt files to Google Storage space. Args: @@ -415,10 +442,13 @@ def UploadPrebuilt(build_path, upload_location, version, binhost_base_url, board: The board to upload to Google Storage, if this is None upload host packages. git_sync: If set, update make.conf of target to reference the latest - prebuilt packages genereated here. + prebuilt packages generated here. git_sync_retries: How many times to retry pushing when updating git files. This helps avoid failures when multiple bots are modifying the same Repo. default: 5 + key: The variable key to update in the git file. (Default: PORTAGE_BINHOST) + sync_binhost_conf: If set, update binhost config file in chromiumos-overlay + for the current board or host. """ if not board: @@ -428,12 +458,16 @@ def UploadPrebuilt(build_path, upload_location, version, binhost_base_url, url_suffix = _REL_HOST_PATH % {'version': version, 'target': _HOST_TARGET} package_string = _HOST_TARGET git_file = os.path.join(build_path, _PREBUILT_MAKE_CONF[_HOST_TARGET]) + binhost_conf = os.path.join(build_path, _BINHOST_CONF_DIR, 'host', + '%s.conf' % _HOST_TARGET) else: board_path = os.path.join(build_path, _BOARD_PATH % {'board': board}) package_path = os.path.join(board_path, 'packages') package_string = board url_suffix = _REL_BOARD_PATH % {'board': board, 'version': version} git_file = os.path.join(build_path, DetermineMakeConfFile(board)) + binhost_conf = os.path.join(build_path, _BINHOST_CONF_DIR, 'target', + '%s.conf' % board) remote_location = os.path.join(upload_location, url_suffix) if upload_location.startswith('gs://'): @@ -452,10 +486,13 @@ def UploadPrebuilt(build_path, upload_location, version, binhost_base_url, if not _RetryRun(cmd, shell=True): raise UploadFailed('Could not run %s' % cmd) - if git_sync: - url_value = '%s/%s/' % (binhost_base_url, url_suffix) - RevGitFile(git_file, url_value, retries=git_sync_retries) + url_value = '%s/%s/' % (binhost_base_url, url_suffix) + if git_sync: + RevGitFile(git_file, url_value, retries=git_sync_retries, key=key) + + if sync_binhost_conf: + UpdateBinhostConfFile(binhost_conf, key, url_value) def usage(parser, msg): """Display usage message and parser help then exit with 1.""" @@ -488,6 +525,12 @@ def main(): parser.add_option('-f', '--filters', dest='filters', action='store_true', default=False, help='Turn on filtering of private ebuild packages') + parser.add_option('-k', '--key', dest='key', + default='PORTAGE_BINHOST', + help='Key to update in make.conf / binhost.conf') + parser.add_option('', '--sync-binhost-conf', dest='sync_binhost_conf', + default=False, action='store_true', + help='Update binhost.conf') options, args = parser.parse_args() # Setup boto environment for gsutil to use @@ -511,12 +554,15 @@ def main(): if options.sync_host: UploadPrebuilt(options.build_path, options.upload, version, - options.binhost_base_url, git_sync=options.git_sync) + options.binhost_base_url, git_sync=options.git_sync, + key=options.key, + sync_binhost_conf=options.sync_binhost_conf) if options.board: UploadPrebuilt(options.build_path, options.upload, version, options.binhost_base_url, board=options.board, - git_sync=options.git_sync) + git_sync=options.git_sync, key=options.key, + sync_binhost_conf=options.sync_binhost_conf) if __name__ == '__main__': From 63c51c4d3cb4fd1fe19eee9e9ce764775ef65da4 Mon Sep 17 00:00:00 2001 From: David James Date: Tue, 30 Nov 2010 11:45:11 -0800 Subject: [PATCH 07/31] Update cbuildbot.py and prebuilt.py to deduplicate preflight prebuilts. Preflight prebuilts reference the last full build, so they should always be complete as long as the last full build is still there. Also add host prebuilts to the preflight prebuilts. BUG=chromium-os:5311 TEST=Run unit tests for cbuildbot and prebuilt.py. Test runs of cbuildbot.py with --dryrun. Review URL: http://codereview.chromium.org/5344002 Change-Id: Id95f94c02cc2f6cbd70a029d4f8b94617b7cc071 --- bin/cbuildbot.py | 31 ++-- bin/cbuildbot_unittest.py | 16 +- chromite/lib/binpkg.py | 307 ++++++++++++++++++++++++++++++++++++++ prebuilt.py | 147 ++++++++---------- prebuilt_unittest.py | 143 ++++++++++++------ 5 files changed, 497 insertions(+), 147 deletions(-) create mode 100644 chromite/lib/binpkg.py diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index 2a6e22b68e..d49fa06dc6 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -317,17 +317,20 @@ def _GetPortageEnvVar(buildroot, board, envvar): buildroot: The root directory where the build occurs. Must be an absolute path. - board: Board type that was built on this machine. E.g. x86-generic. - envvar: The environment variable to get. E.g. "PORTAGE_BINHOST". + board: Board type that was built on this machine. E.g. x86-generic. If this + is None, get the env var from the host. + envvar: The environment variable to get. E.g. 'PORTAGE_BINHOST'. Returns: The value of the environment variable, as a string. If no such variable can be found, return the empty string. """ cwd = os.path.join(buildroot, 'src', 'scripts') - binhost = RunCommand(['portageq-%s' % board, 'envvar', envvar], - cwd=cwd, redirect_stdout=True, enter_chroot=True, - error_ok=True) + portageq = 'portageq' + if board: + portageq += '-%s' % board + binhost = RunCommand([portageq, 'envvar', envvar], cwd=cwd, + redirect_stdout=True, enter_chroot=True, error_ok=True) return binhost.rstrip('\n') @@ -533,7 +536,7 @@ def _ResolveOverlays(buildroot, overlays): return paths -def _UploadPrebuilts(buildroot, board, overlay_config): +def _UploadPrebuilts(buildroot, board, overlay_config, binhosts): """Upload prebuilts. Args: @@ -543,6 +546,8 @@ def _UploadPrebuilts(buildroot, board, overlay_config): 'private': Just the private overlay. 'public': Just the public overlay. 'both': Both the public and private overlays. + binhosts: The URLs of the current binhosts. Binaries that are already + present will not be uploaded twice. Empty URLs will be ignored. """ cwd = os.path.join(buildroot, 'src', 'scripts') @@ -552,6 +557,9 @@ def _UploadPrebuilts(buildroot, board, overlay_config): '--board', board, '--prepend-version', 'preflight', '--key', _PREFLIGHT_BINHOST] + for binhost in binhosts: + if binhost: + cmd.extend(['--previous-binhost-url', binhost]) if overlay_config == 'public': cmd.extend(['--upload', 'gs://chromeos-prebuilt']) else: @@ -617,6 +625,7 @@ def main(): # Calculate list of overlay directories. overlays = _ResolveOverlays(buildroot, buildconfig['overlays']) board = buildconfig['board'] + old_binhost = None _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays) chroot_path = os.path.join(buildroot, 'chroot') @@ -627,9 +636,10 @@ def main(): else: old_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST) _IncrementalCheckout(buildroot) - new_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST) - if old_binhost != new_binhost: - RunCommand(['sudo', 'rm', '-rf', boardpath]) + + new_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST) + if old_binhost and old_binhost != new_binhost: + RunCommand(['sudo', 'rm', '-rf', boardpath]) # Check that all overlays can be found. for path in overlays: @@ -684,7 +694,8 @@ def main(): if buildconfig['master']: # Master bot needs to check if the other slaves completed. if cbuildbot_comm.HaveSlavesCompleted(config): - _UploadPrebuilts(buildroot, board, buildconfig['overlays']) + _UploadPrebuilts(buildroot, board, buildconfig['overlays'], + [new_binhost]) _UprevPush(buildroot, tracking_branch, buildconfig['board'], overlays, options.debug) else: diff --git a/bin/cbuildbot_unittest.py b/bin/cbuildbot_unittest.py index 87bfc363c2..b3a4009a90 100755 --- a/bin/cbuildbot_unittest.py +++ b/bin/cbuildbot_unittest.py @@ -208,18 +208,26 @@ class CBuildBotTest(mox.MoxTestBase): def testUploadPublicPrebuilts(self): """Test _UploadPrebuilts with a public location.""" - check = mox.And(mox.IsA(list), mox.In('gs://chromeos-prebuilt')) + binhost = 'http://www.example.com' + binhosts = [binhost, None] + check = mox.And(mox.IsA(list), mox.In(binhost), mox.Not(mox.In(None)), + mox.In('gs://chromeos-prebuilt')) cbuildbot.RunCommand(check, cwd='%s/src/scripts' % self._buildroot) self.mox.ReplayAll() - cbuildbot._UploadPrebuilts(self._buildroot, self._test_board, 'public') + cbuildbot._UploadPrebuilts(self._buildroot, self._test_board, 'public', + binhosts) self.mox.VerifyAll() def testUploadPrivatePrebuilts(self): """Test _UploadPrebuilts with a private location.""" - check = mox.And(mox.IsA(list), mox.In('chromeos-images:/var/www/prebuilt/')) + binhost = 'http://www.example.com' + binhosts = [binhost, None] + check = mox.And(mox.IsA(list), mox.In(binhost), mox.Not(mox.In(None)), + mox.In('chromeos-images:/var/www/prebuilt/')) cbuildbot.RunCommand(check, cwd='%s/src/scripts' % self._buildroot) self.mox.ReplayAll() - cbuildbot._UploadPrebuilts(self._buildroot, self._test_board, 'private') + cbuildbot._UploadPrebuilts(self._buildroot, self._test_board, 'private', + binhosts) self.mox.VerifyAll() diff --git a/chromite/lib/binpkg.py b/chromite/lib/binpkg.py new file mode 100644 index 0000000000..40a29dbcd2 --- /dev/null +++ b/chromite/lib/binpkg.py @@ -0,0 +1,307 @@ +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +# +# Adapted from portage/getbinpkg.py -- Portage binary-package helper functions +# Copyright 2003-2004 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import operator +import os +import tempfile +import time +import urllib2 +import urlparse + +class PackageIndex(object): + """A parser for the Portage Packages index file. + + The Portage Packages index file serves to keep track of what packages are + included in a tree. It contains the following sections: + 1) The header. The header tracks general key/value pairs that don't apply + to any specific package. E.g., it tracks the base URL of the packages + file, and the number of packages included in the file. The header is + terminated by a blank line. + 2) The body. The body is a list of packages. Each package contains a list + of key/value pairs. Packages are either terminated by a blank line or + by the end of the file. Every package has a CPV entry, which serves as + a unique identifier for the package. + """ + + def __init__(self): + """Constructor.""" + + # The header tracks general key/value pairs that don't apply to any + # specific package. E.g., it tracks the base URL of the packages. + self.header = {} + + # A list of packages (stored as a list of dictionaries). + self.packages = [] + + # Whether or not the PackageIndex has been modified since the last time it + # was written. + self.modified = False + + def _PopulateDuplicateDB(self, db): + """Populate db with SHA1 -> URL mapping for packages. + + Args: + db: Dictionary to populate with SHA1 -> URL mapping for packages. + """ + + uri = self.header['URI'] + for pkg in self.packages: + cpv, sha1 = pkg['CPV'], pkg['SHA1'] + path = pkg.get('PATH', cpv + '.tbz2') + db[sha1] = urlparse.urljoin(uri, path) + + def _ReadPkgIndex(self, pkgfile): + """Read a list of key/value pairs from the Packages file into a dictionary. + + Both header entries and package entries are lists of key/value pairs, so + they can both be read by this function. Entries can be terminated by empty + lines or by the end of the file. + + This function will read lines from the specified file until it encounters + the a blank line or the end of the file. + + Keys and values in the Packages file are separated by a colon and a space. + Keys may contain capital letters, numbers, and underscores, but may not + contain colons. Values may contain any character except a newline. In + particular, it is normal for values to contain colons. + + Lines that have content, and do not contain a valid key/value pair, are + ignored. This is for compatibility with the Portage package parser, and + to allow for future extensions to the Packages file format. + + All entries must contain at least one key/value pair. If the end of the + fils is reached, an empty dictionary is returned. + + Args: + pkgfile: A python file object. + + Returns the dictionary of key-value pairs that was read from the file. + """ + d = {} + for line in pkgfile: + line = line.rstrip('\n') + if not line: + assert d, 'Packages entry must contain at least one key/value pair' + break + line = line.split(': ', 1) + if len(line) == 2: + k, v = line + d[k] = v + return d + + def _WritePkgIndex(self, pkgfile, entry): + """Write header entry or package entry to packages file. + + The keys and values will be separated by a colon and a space. The entry + will be terminated by a blank line. + + Args: + pkgfile: A python file object. + entry: A dictionary of the key/value pairs to write. + """ + lines = ['%s: %s' % (k, v) for k, v in sorted(entry.items()) if v] + pkgfile.write('%s\n\n' % '\n'.join(lines)) + + def _ReadHeader(self, pkgfile): + """Read header of packages file. + + Args: + pkgfile: A python file object. + """ + assert not self.header, 'Should only read header once.' + self.header = self._ReadPkgIndex(pkgfile) + + def _ReadBody(self, pkgfile): + """Read body of packages file. + + Before calling this function, you must first read the header (using + _ReadHeader). + + Args: + pkgfile: A python file object. + """ + assert self.header, 'Should read header first.' + assert not self.packages, 'Should only read body once.' + + # Read all of the sections in the body by looping until we reach the end + # of the file. + while True: + d = self._ReadPkgIndex(pkgfile) + if not d: + break + if 'CPV' in d: + self.packages.append(d) + + def Read(self, pkgfile): + """Read the entire packages file. + + Args: + pkgfile: A python file object. + """ + self._ReadHeader(pkgfile) + self._ReadBody(pkgfile) + + def RemoveFilteredPackages(self, filter_fn): + """Remove packages which match filter_fn. + + Args: + filter_fn: A function which operates on packages. If it returns True, + the package should be removed. + """ + + filtered = [p for p in self.packages if not filter_fn(p)] + if filtered != self.packages: + self.modified = True + self.packages = filtered + + def ResolveDuplicateUploads(self, pkgindexes): + """Point packages at files that have already been uploaded. + + For each package in our index, check if there is an existing package that + has already been uploaded to the same base URI. If so, point that package + at the existing file, so that we don't have to upload the file. + + Args: + pkgindexes: A list of PackageIndex objects containing info about packages + that have already been uploaded. + + Returns: + A list of the packages that still need to be uploaded. + """ + db = {} + for pkgindex in pkgindexes: + pkgindex._PopulateDuplicateDB(db) + + uploads = [] + base_uri = self.header['URI'] + for pkg in self.packages: + sha1 = pkg['SHA1'] + uri = db.get(sha1) + if uri and uri.startswith(base_uri): + pkg['PATH'] = uri[len(base_uri):].lstrip('/') + else: + uploads.append(pkg) + return uploads + + def SetUploadLocation(self, base_uri, path_prefix): + """Set upload location to base_uri + path_prefix. + + Args: + base_uri: Base URI for all packages in the file. We set + self.header['URI'] to this value, so all packages must live under + this directory. + path_prefix: Path prefix to use for all current packages in the file. + This will be added to the beginning of the path for every package. + """ + self.header['URI'] = base_uri + for pkg in self.packages: + pkg['PATH'] = urlparse.urljoin(path_prefix, pkg['CPV'] + '.tbz2') + + def Write(self, pkgfile): + """Write a packages file to disk. + + If 'modified' flag is set, the TIMESTAMP and PACKAGES fields in the header + will be updated before writing to disk. + + Args: + pkgfile: A python file object. + """ + if self.modified: + self.header['TIMESTAMP'] = str(long(time.time())) + self.header['PACKAGES'] = str(len(self.packages)) + self.modified = False + self._WritePkgIndex(pkgfile, self.header) + for metadata in sorted(self.packages, key=operator.itemgetter('CPV')): + self._WritePkgIndex(pkgfile, metadata) + + def WriteToNamedTemporaryFile(self): + """Write pkgindex to a temporary file. + + Args: + pkgindex: The PackageIndex object. + + Returns: + A temporary file containing the packages from pkgindex. + """ + f = tempfile.NamedTemporaryFile() + self.Write(f) + f.flush() + f.seek(0) + return f + + +def _RetryUrlOpen(url, tries=3): + """Open the specified url, retrying if we run into temporary errors. + + We retry for both network errors and 5xx Server Errors. We do not retry + for HTTP errors with a non-5xx code. + + Args: + url: The specified url. + tries: The number of times to try. + + Returns: + The result of urllib2.urlopen(url). + """ + for i in range(tries): + try: + return urllib2.urlopen(url) + except urllib2.HTTPError as e: + if i + 1 >= tries or e.code < 500: + raise + else: + print 'Cannot GET %s: %s' % (url, str(e)) + except urllib2.URLError as e: + if i + 1 >= tries: + raise + else: + print 'Cannot GET %s: %s' % (url, str(e)) + print 'Sleeping for 10 seconds before retrying...' + time.sleep(10) + + +def GrabRemotePackageIndex(binhost_url): + """Grab the latest binary package database from the specified URL. + + Args: + binhost_url: Base URL of remote packages (PORTAGE_BINHOST). + + Returns: + A PackageIndex object, if the Packages file can be retrieved. If the + server returns status code 404, None is returned. + """ + + url = urlparse.urljoin(binhost_url, 'Packages') + try: + f = _RetryUrlOpen(url) + except urllib2.HTTPError as e: + if e.code == 404: + return None + raise + + pkgindex = PackageIndex() + pkgindex.Read(f) + pkgindex.header.setdefault('URI', binhost_url) + f.close() + return pkgindex + + +def GrabLocalPackageIndex(package_path): + """Read a local packages file from disk into a PackageIndex() object. + + Args: + package_path: Directory containing Packages file. + + Returns: + A PackageIndex object. + """ + packages_file = file(os.path.join(package_path, 'Packages')) + pkgindex = PackageIndex() + pkgindex.Read(packages_file) + packages_file.close() + return pkgindex diff --git a/prebuilt.py b/prebuilt.py index cb4e04f84a..4baf95dab6 100755 --- a/prebuilt.py +++ b/prebuilt.py @@ -11,8 +11,11 @@ import re import sys import tempfile import time +import urlparse from chromite.lib import cros_build_lib +from chromite.lib.binpkg import (GrabLocalPackageIndex, GrabRemotePackageIndex, + PackageIndex) """ This script is used to upload host prebuilts as well as board BINHOSTS. @@ -228,70 +231,14 @@ def ShouldFilterPackage(file_path): return False -def _ShouldFilterPackageFileSection(section): - """Return whether an section in the package file should be filtered out. - - Args: - section: The section, as a list of strings. - - Returns: - True if the section should be excluded. - """ - - for line in section: - if line.startswith("CPV: "): - package = line.replace("CPV: ", "").rstrip() - if ShouldFilterPackage(package): - return True - else: - return False - - -def FilterPackagesFile(packages_filename): - """Read a portage Packages file and filter out private packages. - - The new, filtered packages file is written to a temporary file. - - Args: - packages_filename: The filename of the Packages file. - - Returns: - filtered_packages: A filtered Packages file, as a NamedTemporaryFile. - """ - - packages_file = open(packages_filename) - filtered_packages = tempfile.NamedTemporaryFile() - section = [] - for line in packages_file: - if line == "\n": - if not _ShouldFilterPackageFileSection(section): - # Looks like this section doesn't contain a private package. Write it - # out. - filtered_packages.write("".join(section)) - - # Start next section. - section = [] - - section.append(line) - else: - if not _ShouldFilterPackageFileSection(section): - filtered_packages.write("".join(section)) - packages_file.close() - - # Flush contents to disk. - filtered_packages.flush() - filtered_packages.seek(0) - - return filtered_packages - - -def _RetryRun(cmd, print_cmd=True, shell=False): +def _RetryRun(cmd, print_cmd=True, shell=False, cwd=None): """Run the specified command, retrying if necessary. Args: cmd: The command to run. print_cmd: Whether to print out the cmd. shell: Whether to treat the command as a shell. + cwd: Working directory to run command in. Returns: True if the command succeeded. Otherwise, returns False. @@ -301,7 +248,8 @@ def _RetryRun(cmd, print_cmd=True, shell=False): # cros_build_lib. for attempt in range(_RETRIES): try: - output = cros_build_lib.RunCommand(cmd, print_cmd=print_cmd, shell=shell) + output = cros_build_lib.RunCommand(cmd, print_cmd=print_cmd, shell=shell, + cwd=cwd) return True except cros_build_lib.RunCommandError: print 'Failed to run %s' % cmd @@ -320,12 +268,6 @@ def _GsUpload(args): Return the arg tuple of two if the upload failed """ (local_file, remote_file) = args - if ShouldFilterPackage(local_file): - return - - if local_file.endswith("/Packages"): - filtered_packages_file = FilterPackagesFile(local_file) - local_file = filtered_packages_file.name cmd = '%s cp -a public-read %s %s' % (_GSUTIL_BIN, local_file, remote_file) if not _RetryRun(cmd, print_cmd=False, shell=True): @@ -359,22 +301,24 @@ def RemoteUpload(files, pool=10): pass -def GenerateUploadDict(local_path, gs_path): - """Build a dictionary of local remote file key pairs for gsutil to upload. +def GenerateUploadDict(base_local_path, base_remote_path, pkgs): + """Build a dictionary of local remote file key pairs to upload. Args: - local_path: A path to the file on the local hard drive. - gs_path: Path to upload in Google Storage. + base_local_path: The base path to the files on the local hard drive. + remote_path: The base path to the remote paths. + pkgs: The packages to upload. Returns: - Returns a dictionary of file path/gs_dest_path pairs + Returns a dictionary of local_path/remote_path pairs """ - files_to_sync = cros_build_lib.ListFiles(local_path) upload_files = {} - for file_path in files_to_sync: - filename = file_path.replace(local_path, '').lstrip('/') - gs_file_path = os.path.join(gs_path, filename) - upload_files[file_path] = gs_file_path + for pkg in pkgs: + suffix = pkg['CPV'] + '.tbz2' + local_path = os.path.join(base_local_path, suffix) + assert os.path.exists(local_path) + remote_path = urlparse.urljoin(base_remote_path, suffix) + upload_files[local_path] = remote_path return upload_files @@ -433,13 +377,14 @@ def UpdateBinhostConfFile(path, key, value): def UploadPrebuilt(build_path, upload_location, version, binhost_base_url, board=None, git_sync=False, git_sync_retries=5, - key='PORTAGE_BINHOST', sync_binhost_conf=False): + key='PORTAGE_BINHOST', pkg_indexes=[], + sync_binhost_conf=False): """Upload Host prebuilt files to Google Storage space. Args: build_path: The path to the root of the chroot. upload_location: The upload location. - board: The board to upload to Google Storage, if this is None upload + board: The board to upload to Google Storage. If this is None, upload host packages. git_sync: If set, update make.conf of target to reference the latest prebuilt packages generated here. @@ -447,6 +392,8 @@ def UploadPrebuilt(build_path, upload_location, version, binhost_base_url, This helps avoid failures when multiple bots are modifying the same Repo. default: 5 key: The variable key to update in the git file. (Default: PORTAGE_BINHOST) + pkg_indexes: Old uploaded prebuilts to compare against. Instead of + uploading duplicate files, we just link to the old files. sync_binhost_conf: If set, update binhost config file in chromiumos-overlay for the current board or host. """ @@ -468,10 +415,22 @@ def UploadPrebuilt(build_path, upload_location, version, binhost_base_url, git_file = os.path.join(build_path, DetermineMakeConfFile(board)) binhost_conf = os.path.join(build_path, _BINHOST_CONF_DIR, 'target', '%s.conf' % board) - remote_location = os.path.join(upload_location, url_suffix) + remote_location = urlparse.urljoin(upload_location, url_suffix) + + # Process Packages file, removing duplicates and filtered packages. + pkg_index = GrabLocalPackageIndex(package_path) + pkg_index.SetUploadLocation(binhost_base_url, url_suffix) + pkg_index.RemoveFilteredPackages(lambda pkg: ShouldFilterPackage(pkg)) + uploads = pkg_index.ResolveDuplicateUploads(pkg_indexes) + + # Write Packages file. + tmp_packages_file = pkg_index.WriteToNamedTemporaryFile() if upload_location.startswith('gs://'): - upload_files = GenerateUploadDict(package_path, remote_location) + # Build list of files to upload. + upload_files = GenerateUploadDict(package_path, remote_location, uploads) + remote_file = urlparse.urljoin(remote_location, 'Packages') + upload_files[tmp_packages_file.name] = remote_file print 'Uploading %s' % package_string failed_uploads = RemoteUpload(upload_files) @@ -479,11 +438,19 @@ def UploadPrebuilt(build_path, upload_location, version, binhost_base_url, error_msg = ['%s -> %s\n' % args for args in failed_uploads] raise UploadFailed('Error uploading:\n%s' % error_msg) else: + pkgs = ' '.join(p['CPV'] + '.tbz2' for p in uploads) ssh_server, remote_path = remote_location.split(':', 1) - cmds = ['ssh %s mkdir -p %s' % (ssh_server, remote_path), - 'rsync -av %s/ %s/' % (package_path, remote_location)] + d = { 'pkg_index': tmp_packages_file.name, + 'pkgs': pkgs, + 'remote_path': remote_path, + 'remote_location': remote_location, + 'ssh_server': ssh_server } + cmds = ['ssh %(ssh_server)s mkdir -p %(remote_path)s' % d, + 'rsync -av %(pkg_index)s %(remote_location)s/Packages' % d] + if pkgs: + cmds.append('rsync -Rav %(pkgs)s %(remote_location)s/' % d) for cmd in cmds: - if not _RetryRun(cmd, shell=True): + if not _RetryRun(cmd, shell=True, cwd=package_path): raise UploadFailed('Could not run %s' % cmd) url_value = '%s/%s/' % (binhost_base_url, url_suffix) @@ -506,6 +473,9 @@ def main(): parser.add_option('-H', '--binhost-base-url', dest='binhost_base_url', default=_BINHOST_BASE_URL, help='Base URL to use for binhost in make.conf updates') + parser.add_option('', '--previous-binhost-url', action='append', + default=[], dest='previous_binhost_url', + help='Previous binhost URL') parser.add_option('-b', '--board', dest='board', default=None, help='Board type that was built on this machine') parser.add_option('-p', '--build-path', dest='build_path', @@ -542,26 +512,29 @@ def main(): usage(parser, 'Error: you need to provide an upload location using -u') if options.filters: - # TODO(davidjames): It might be nice to be able to filter private ebuilds - # from rsync uploads as well, some day. But for now it's not needed. - if not options.upload.startswith("gs://"): - usage(parser, 'Error: filtering only works with gs:// paths') LoadPrivateFilters(options.build_path) version = GetVersion() if options.prepend_version: version = '%s-%s' % (options.prepend_version, version) + pkg_indexes = [] + for url in options.previous_binhost_url: + pkg_index = GrabRemotePackageIndex(url) + if pkg_index: + pkg_indexes.append(pkg_index) + if options.sync_host: UploadPrebuilt(options.build_path, options.upload, version, options.binhost_base_url, git_sync=options.git_sync, - key=options.key, + key=options.key, pkg_indexes=pkg_indexes, sync_binhost_conf=options.sync_binhost_conf) if options.board: UploadPrebuilt(options.build_path, options.upload, version, options.binhost_base_url, board=options.board, git_sync=options.git_sync, key=options.key, + pkg_indexes=pkg_indexes, sync_binhost_conf=options.sync_binhost_conf) diff --git a/prebuilt_unittest.py b/prebuilt_unittest.py index fe2387fbf7..9d26692ccf 100755 --- a/prebuilt_unittest.py +++ b/prebuilt_unittest.py @@ -3,6 +3,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import copy import mox import os import prebuilt @@ -10,6 +11,21 @@ import shutil import tempfile import unittest from chromite.lib import cros_build_lib +from chromite.lib.binpkg import PackageIndex + +PUBLIC_PACKAGES = [{'CPV': 'public1', 'SHA1': '1'}, + {'CPV': 'public2', 'SHA1': '2', 'PATH': 'foo.tgz'}] +PRIVATE_PACKAGES = [{'CPV': 'private', 'SHA1': '3'}] + + +def SimplePackageIndex(header=True, packages=True): + pkgindex = PackageIndex() + if header: + pkgindex.header['URI'] = 'http://www.example.com' + if packages: + pkgindex.packages = copy.deepcopy(PUBLIC_PACKAGES + PRIVATE_PACKAGES) + return pkgindex + class TestUpdateFile(unittest.TestCase): @@ -137,14 +153,6 @@ class TestPrebuiltFilters(unittest.TestCase): class TestPrebuilt(unittest.TestCase): - fake_path = '/b/cbuild/build/chroot/build/x86-dogfood/' - bin_package_mock = ['packages/x11-misc/shared-mime-info-0.70.tbz2', - 'packages/x11-misc/util-macros-1.5.0.tbz2', - 'packages/x11-misc/xbitmaps-1.1.0.tbz2', - 'packages/x11-misc/read-edid-1.4.2.tbz2', - 'packages/x11-misc/xdg-utils-1.0.2-r3.tbz2'] - - files_to_sync = [os.path.join(fake_path, file) for file in bin_package_mock] def setUp(self): self.mox = mox.Mox() @@ -153,23 +161,17 @@ class TestPrebuilt(unittest.TestCase): self.mox.UnsetStubs() self.mox.VerifyAll() - def _generate_dict_results(self, gs_bucket_path): - """ - Generate a dictionary result similar to GenerateUploadDict - """ - results = {} - for entry in self.files_to_sync: - results[entry] = os.path.join( - gs_bucket_path, entry.replace(self.fake_path, '').lstrip('/')) - return results - def testGenerateUploadDict(self): + base_local_path = '/b/cbuild/build/chroot/build/x86-dogfood/' gs_bucket_path = 'gs://chromeos-prebuilt/host/version' - self.mox.StubOutWithMock(cros_build_lib, 'ListFiles') - cros_build_lib.ListFiles(self.fake_path).AndReturn(self.files_to_sync) + local_path = os.path.join(base_local_path, 'public1.tbz2') + self.mox.StubOutWithMock(prebuilt.os.path, 'exists') + prebuilt.os.path.exists(local_path).AndReturn(True) self.mox.ReplayAll() - result = prebuilt.GenerateUploadDict(self.fake_path, gs_bucket_path) - self.assertEqual(result, self._generate_dict_results(gs_bucket_path)) + pkgs = [{ 'CPV': 'public1' }] + result = prebuilt.GenerateUploadDict(base_local_path, gs_bucket_path, pkgs) + expected = { local_path: gs_bucket_path + '/public1.tbz2' } + self.assertEqual(result, expected) def testFailonUploadFail(self): """Make sure we fail if one of the upload processes fail.""" @@ -195,6 +197,73 @@ class TestPrebuilt(unittest.TestCase): class TestPackagesFileFiltering(unittest.TestCase): + def testFilterPkgIndex(self): + pkgindex = SimplePackageIndex() + pkgindex.RemoveFilteredPackages(lambda pkg: pkg in PRIVATE_PACKAGES) + self.assertEqual(pkgindex.packages, PUBLIC_PACKAGES) + self.assertEqual(pkgindex.modified, True) + + +class TestPopulateDuplicateDB(unittest.TestCase): + + def testEmptyIndex(self): + pkgindex = SimplePackageIndex(packages=False) + db = {} + pkgindex._PopulateDuplicateDB(db) + self.assertEqual(db, {}) + + def testNormalIndex(self): + pkgindex = SimplePackageIndex() + db = {} + pkgindex._PopulateDuplicateDB(db) + self.assertEqual(len(db), 3) + self.assertEqual(db['1'], 'http://www.example.com/public1.tbz2') + self.assertEqual(db['2'], 'http://www.example.com/foo.tgz') + self.assertEqual(db['3'], 'http://www.example.com/private.tbz2') + + def testFailedPopulate(self): + db = {} + pkgindex = SimplePackageIndex(header=False) + self.assertRaises(KeyError, pkgindex._PopulateDuplicateDB, db) + pkgindex = SimplePackageIndex() + del pkgindex.packages[0]['CPV'] + self.assertRaises(KeyError, pkgindex._PopulateDuplicateDB, db) + pkgindex = SimplePackageIndex() + del pkgindex.packages[0]['SHA1'] + self.assertRaises(KeyError, pkgindex._PopulateDuplicateDB, db) + + +class TestResolveDuplicateUploads(unittest.TestCase): + + def testEmptyList(self): + pkgindex = SimplePackageIndex() + pristine = SimplePackageIndex() + uploads = pkgindex.ResolveDuplicateUploads([]) + self.assertEqual(uploads, pristine.packages) + self.assertEqual(pkgindex.packages, pristine.packages) + self.assertEqual(pkgindex.modified, False) + + def testEmptyIndex(self): + pkgindex = SimplePackageIndex() + pristine = SimplePackageIndex() + empty = SimplePackageIndex(packages=False) + uploads = pkgindex.ResolveDuplicateUploads([empty]) + self.assertEqual(uploads, pristine.packages) + self.assertEqual(pkgindex.packages, pristine.packages) + self.assertEqual(pkgindex.modified, False) + + def testDuplicates(self): + pkgindex = SimplePackageIndex() + dup_pkgindex = SimplePackageIndex() + expected_pkgindex = SimplePackageIndex() + for pkg in expected_pkgindex.packages: + pkg.setdefault('PATH', pkg['CPV'] + '.tbz2') + uploads = pkgindex.ResolveDuplicateUploads([dup_pkgindex]) + self.assertEqual(pkgindex.packages, expected_pkgindex.packages) + + +class TestWritePackageIndex(unittest.TestCase): + def setUp(self): self.mox = mox.Mox() @@ -202,31 +271,13 @@ class TestPackagesFileFiltering(unittest.TestCase): self.mox.UnsetStubs() self.mox.VerifyAll() - def testFilterAllPackages(self): - self.mox.StubOutWithMock(prebuilt, 'ShouldFilterPackage') - prebuilt.ShouldFilterPackage("public1").AndReturn(False) - prebuilt.ShouldFilterPackage("private").AndReturn(True) - prebuilt.ShouldFilterPackage("public2").AndReturn(False) - full_packages_file = [ - "foo: bar\n", "\n", - "CPV: public1\n", "foo: bar1\n", "\n", - "CPV: private\n", "foo: bar2\n", "\n", - "CPV: public2\n", "foo: bar3\n", "\n", - ] - private_packages_file = [ - "foo: bar\n", "\n", - "CPV: public1\n", "foo: bar1\n", "\n", - "CPV: public2\n", "foo: bar3\n", "\n", - ] + def testSimple(self): + pkgindex = SimplePackageIndex() + self.mox.StubOutWithMock(pkgindex, 'Write') + pkgindex.Write(mox.IgnoreArg()) self.mox.ReplayAll() - temp_packages_file = tempfile.NamedTemporaryFile() - temp_packages_file.write("".join(full_packages_file)) - temp_packages_file.flush() - new_packages_file = prebuilt.FilterPackagesFile(temp_packages_file.name) - new_contents = open(new_packages_file.name).read() - self.assertEqual("".join(private_packages_file), new_contents) - self.assertEqual("".join(private_packages_file), new_packages_file.read()) - new_packages_file.close() + f = pkgindex.WriteToNamedTemporaryFile() + self.assertEqual(f.read(), '') if __name__ == '__main__': From ff07201a8cbaebc01d78198aa9be38d927208296 Mon Sep 17 00:00:00 2001 From: David James Date: Tue, 30 Nov 2010 13:22:05 -0800 Subject: [PATCH 08/31] Turn on --fast on official builds. The --fast support for build_packages is stable now, and should significantly improve the speed of our official builders. We should turn it on so that builders can finish their builds in less than 8 hours. BUG=chromium-os:6706 TEST=Run ./enter_chroot.sh --official_build and verify that --fast is on by default now Change-Id: I6ad126b9b6ce16ffc9887a7af22c2e3f85afbf42 Review URL: http://codereview.chromium.org/3418001 --- common.sh | 5 +---- parallel_emerge | 23 +++++++++++------------ 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/common.sh b/common.sh index b7d428dab5..20a473cff8 100644 --- a/common.sh +++ b/common.sh @@ -113,11 +113,8 @@ ALL_BOARDS=$(echo $ALL_BOARDS) #DEFAULT_BOARD=x86-generic # or... DEFAULT_BOARD=$(echo $ALL_BOARDS | awk '{print $NF}') -# Enable --fast by default on non-official builds +# Enable --fast by default. DEFAULT_FAST="${FLAGS_TRUE}" -if [ "${CHROMEOS_OFFICIAL:-0}" = "1" ]; then - DEFAULT_FAST="${FLAGS_FALSE}" -fi # Detect whether we're inside a chroot or not if [ -e /etc/debian_chroot ] diff --git a/parallel_emerge b/parallel_emerge index c912a115ac..ce40be4b07 100755 --- a/parallel_emerge +++ b/parallel_emerge @@ -310,13 +310,6 @@ class DepGraphGenerator(object): # will be going away soon as we migrate to CROS_WORKON_SRCROOT. os.environ.setdefault("CHROMEOS_ROOT", os.environ["HOME"] + "/trunk") - # Modify the environment to disable locking by default. - # TODO(davidjames): This option can cause problems if packages muck - # with each other during the post-install step. There are a few host - # packages that do this, so we only do this environment modification for - # board builds. - os.environ.setdefault("PORTAGE_LOCKS", "false") - # Turn off interactive delays os.environ["EBEEP_IGNORE"] = "1" os.environ["EPAUSE_IGNORE"] = "1" @@ -357,11 +350,17 @@ class DepGraphGenerator(object): # TODO(davidjames): Look for a better solution. features = os.environ.get("FEATURES", "") + " -collision-protect" - # If we're cross-compiling, updating the environment every time we install - # a package isn't necessary, and leads to race conditions when - # PORTAGE_LOCKS is false. In this case, do environment updates at the end, - # instead. - if self.board and os.environ.get("PORTAGE_LOCKS") == "false": + # If we're installing packages to the board, and we're not using the + # official flag, we can enable the following optimizations: + # 1) Don't lock during install step. This allows multiple packages to be + # installed at once. This is safe because our board packages do not + # muck with each other during the post-install step. + # 2) Don't update the environment until the end of the build. This is + # safe because board packages don't need to run during the build -- + # they're cross-compiled, so our CPU architecture doesn't support them + # anyway. + if self.board and os.environ.get("CHROMEOS_OFFICIAL") != "1": + os.environ.setdefault("PORTAGE_LOCKS", "false") features = features + " no-env-update" os.environ["FEATURES"] = features From 74f0f17ad86ee04975501d1949c5a3d94f54b463 Mon Sep 17 00:00:00 2001 From: David James Date: Tue, 30 Nov 2010 13:56:52 -0800 Subject: [PATCH 09/31] Use regular string ops, instead of urlparse.urljoin. urlparse.urljoin('scheme:foo/bar/', 'baz') == 'baz', so it doesn't work the way I expected. Instead, let's use regular string joins. BUG=chromium-os:5311 TEST=Ran buildbot in test mode. Change-Id: Iea4205573db602e0cc20af24a77d255d17e79221 Review URL: http://codereview.chromium.org/5357012 --- prebuilt.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/prebuilt.py b/prebuilt.py index 4baf95dab6..607397ee8d 100755 --- a/prebuilt.py +++ b/prebuilt.py @@ -11,7 +11,6 @@ import re import sys import tempfile import time -import urlparse from chromite.lib import cros_build_lib from chromite.lib.binpkg import (GrabLocalPackageIndex, GrabRemotePackageIndex, @@ -317,7 +316,7 @@ def GenerateUploadDict(base_local_path, base_remote_path, pkgs): suffix = pkg['CPV'] + '.tbz2' local_path = os.path.join(base_local_path, suffix) assert os.path.exists(local_path) - remote_path = urlparse.urljoin(base_remote_path, suffix) + remote_path = '%s/%s' % (base_remote_path.rstrip('/'), suffix) upload_files[local_path] = remote_path return upload_files @@ -415,7 +414,7 @@ def UploadPrebuilt(build_path, upload_location, version, binhost_base_url, git_file = os.path.join(build_path, DetermineMakeConfFile(board)) binhost_conf = os.path.join(build_path, _BINHOST_CONF_DIR, 'target', '%s.conf' % board) - remote_location = urlparse.urljoin(upload_location, url_suffix) + remote_location = '%s/%s' % (upload_location.rstrip('/'), url_suffix) # Process Packages file, removing duplicates and filtered packages. pkg_index = GrabLocalPackageIndex(package_path) @@ -429,7 +428,7 @@ def UploadPrebuilt(build_path, upload_location, version, binhost_base_url, if upload_location.startswith('gs://'): # Build list of files to upload. upload_files = GenerateUploadDict(package_path, remote_location, uploads) - remote_file = urlparse.urljoin(remote_location, 'Packages') + remote_file = '%s/Packages' % remote_location.rstrip('/') upload_files[tmp_packages_file.name] = remote_file print 'Uploading %s' % package_string From adcb314d9f3437e285b9255797b0f9107160ea7d Mon Sep 17 00:00:00 2001 From: David James Date: Tue, 30 Nov 2010 15:34:20 -0800 Subject: [PATCH 10/31] Treat packages with missing SHA1 as not duplicated. TBR=Needed to fix preflight build. BUG=chromium-os:5311 TEST=Remove SHA1 from one of the packages in the Packages file, make sure that prebuilt.py treats that as a duplicated package. Also run new unit tests. Also ran buildbot.py Change-Id: Ie35a7c818fd659bcc2296605ace850aee2a88448 Review URL: http://codereview.chromium.org/5370005 --- chromite/lib/binpkg.py | 11 ++++++----- prebuilt_unittest.py | 24 +++++++++++++++++++++--- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/chromite/lib/binpkg.py b/chromite/lib/binpkg.py index 40a29dbcd2..48adccb209 100644 --- a/chromite/lib/binpkg.py +++ b/chromite/lib/binpkg.py @@ -51,9 +51,10 @@ class PackageIndex(object): uri = self.header['URI'] for pkg in self.packages: - cpv, sha1 = pkg['CPV'], pkg['SHA1'] - path = pkg.get('PATH', cpv + '.tbz2') - db[sha1] = urlparse.urljoin(uri, path) + cpv, sha1 = pkg['CPV'], pkg.get('SHA1') + if sha1: + path = pkg.get('PATH', cpv + '.tbz2') + db[sha1] = urlparse.urljoin(uri, path) def _ReadPkgIndex(self, pkgfile): """Read a list of key/value pairs from the Packages file into a dictionary. @@ -180,9 +181,9 @@ class PackageIndex(object): uploads = [] base_uri = self.header['URI'] for pkg in self.packages: - sha1 = pkg['SHA1'] + sha1 = pkg.get('SHA1') uri = db.get(sha1) - if uri and uri.startswith(base_uri): + if sha1 and uri and uri.startswith(base_uri): pkg['PATH'] = uri[len(base_uri):].lstrip('/') else: uploads.append(pkg) diff --git a/prebuilt_unittest.py b/prebuilt_unittest.py index 9d26692ccf..4c7255e3c5 100755 --- a/prebuilt_unittest.py +++ b/prebuilt_unittest.py @@ -221,6 +221,15 @@ class TestPopulateDuplicateDB(unittest.TestCase): self.assertEqual(db['2'], 'http://www.example.com/foo.tgz') self.assertEqual(db['3'], 'http://www.example.com/private.tbz2') + def testMissingSHA1(self): + db = {} + pkgindex = SimplePackageIndex() + del pkgindex.packages[0]['SHA1'] + pkgindex._PopulateDuplicateDB(db) + self.assertEqual(len(db), 2) + self.assertEqual(db['2'], 'http://www.example.com/foo.tgz') + self.assertEqual(db['3'], 'http://www.example.com/private.tbz2') + def testFailedPopulate(self): db = {} pkgindex = SimplePackageIndex(header=False) @@ -228,9 +237,6 @@ class TestPopulateDuplicateDB(unittest.TestCase): pkgindex = SimplePackageIndex() del pkgindex.packages[0]['CPV'] self.assertRaises(KeyError, pkgindex._PopulateDuplicateDB, db) - pkgindex = SimplePackageIndex() - del pkgindex.packages[0]['SHA1'] - self.assertRaises(KeyError, pkgindex._PopulateDuplicateDB, db) class TestResolveDuplicateUploads(unittest.TestCase): @@ -261,6 +267,18 @@ class TestResolveDuplicateUploads(unittest.TestCase): uploads = pkgindex.ResolveDuplicateUploads([dup_pkgindex]) self.assertEqual(pkgindex.packages, expected_pkgindex.packages) + def testMissingSHA1(self): + db = {} + pkgindex = SimplePackageIndex() + dup_pkgindex = SimplePackageIndex() + expected_pkgindex = SimplePackageIndex() + del pkgindex.packages[0]['SHA1'] + del expected_pkgindex.packages[0]['SHA1'] + for pkg in expected_pkgindex.packages[1:]: + pkg.setdefault('PATH', pkg['CPV'] + '.tbz2') + uploads = pkgindex.ResolveDuplicateUploads([dup_pkgindex]) + self.assertEqual(pkgindex.packages, expected_pkgindex.packages) + class TestWritePackageIndex(unittest.TestCase): From 6edcc4d952ee77e12d0d2650dc2cd781c6634ec8 Mon Sep 17 00:00:00 2001 From: David Hendricks Date: Tue, 30 Nov 2010 17:21:39 -0800 Subject: [PATCH 11/31] Copy recovery kernel to syslinux path only on x86. This is intended to fix a build error seen on ARM platforms where the path "$esp_mnt/syslinux" does not exist. Change-Id: I0b9e2d1fae376a7137d7e196901fbea4f2e796bc BUG=http://code.google.com/p/chromium-os/issues/detail?id=9713 TEST=Not tested. Review URL: http://codereview.chromium.org/5404001 --- mod_image_for_recovery.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/mod_image_for_recovery.sh b/mod_image_for_recovery.sh index e6cc8859a8..2d6e03151b 100755 --- a/mod_image_for_recovery.sh +++ b/mod_image_for_recovery.sh @@ -292,15 +292,20 @@ install_recovery_kernel() { # Replace vmlinuz.A with the recovery version local sysroot="${FLAGS_build_root}/${FLAGS_board}" local vmlinuz="$sysroot/boot/vmlinuz" - local esp_offset=$(partoffset "$RECOVERY_IMAGE" 12) - local esp_mnt=$(mktemp -d) - set +e local failed=0 - sudo mount -o loop,offset=$((esp_offset * 512)) "$RECOVERY_IMAGE" "$esp_mnt" - sudo cp "$vmlinuz" "$esp_mnt/syslinux/vmlinuz.A" || failed=1 - sudo umount -d "$esp_mnt" - rmdir "$esp_mnt" - set -e + + if [ "$ARCH" = "x86" ]; then + # There is no syslinux on ARM, so this copy only makes sense for x86. + set +e + local esp_offset=$(partoffset "$RECOVERY_IMAGE" 12) + local esp_mnt=$(mktemp -d) + sudo mount -o loop,offset=$((esp_offset * 512)) "$RECOVERY_IMAGE" "$esp_mnt" + sudo cp "$vmlinuz" "$esp_mnt/syslinux/vmlinuz.A" || failed=1 + sudo umount -d "$esp_mnt" + rmdir "$esp_mnt" + set -e + fi + if [ $failed -eq 1 ]; then echo "Failed to copy recovery kernel to ESP" return 1 From 2df2dd5d234039e2ddb1998fbe280a9be86140d6 Mon Sep 17 00:00:00 2001 From: David James Date: Tue, 30 Nov 2010 19:06:30 -0800 Subject: [PATCH 12/31] Remove more broken urlparse.urljoin examples. urlparse.urljoin('scheme:foo/bar/', 'baz') == 'baz', so it doesn't work the way I expected. Instead, let's use regular string joins. BUG=chromium-os:5311 TEST=Examine example Packages file. Review URL: http://codereview.chromium.org/5414001 Change-Id: I1d81dd7a3a79ca6c1f0750993535d3b66696ec23 --- chromite/lib/binpkg.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/chromite/lib/binpkg.py b/chromite/lib/binpkg.py index 48adccb209..75c86bcbac 100644 --- a/chromite/lib/binpkg.py +++ b/chromite/lib/binpkg.py @@ -11,7 +11,6 @@ import os import tempfile import time import urllib2 -import urlparse class PackageIndex(object): """A parser for the Portage Packages index file. @@ -54,7 +53,7 @@ class PackageIndex(object): cpv, sha1 = pkg['CPV'], pkg.get('SHA1') if sha1: path = pkg.get('PATH', cpv + '.tbz2') - db[sha1] = urlparse.urljoin(uri, path) + db[sha1] = '%s/%s' % (uri.rstrip('/'), path) def _ReadPkgIndex(self, pkgfile): """Read a list of key/value pairs from the Packages file into a dictionary. @@ -201,7 +200,7 @@ class PackageIndex(object): """ self.header['URI'] = base_uri for pkg in self.packages: - pkg['PATH'] = urlparse.urljoin(path_prefix, pkg['CPV'] + '.tbz2') + pkg['PATH'] = '%s/%s' % (path_prefix.rstrip('/'), pkg['CPV'] + '.tbz2') def Write(self, pkgfile): """Write a packages file to disk. @@ -277,7 +276,7 @@ def GrabRemotePackageIndex(binhost_url): server returns status code 404, None is returned. """ - url = urlparse.urljoin(binhost_url, 'Packages') + url = '%s/Packages' % binhost_url.rstrip('/') try: f = _RetryUrlOpen(url) except urllib2.HTTPError as e: From 7efdebf35f18b74240843a5511f994ba906cb96a Mon Sep 17 00:00:00 2001 From: Dave Parker Date: Wed, 1 Dec 2010 11:34:32 -0800 Subject: [PATCH 13/31] Added support for boards with only private overlays. (crosutils) If a board has a public overlay it will be used as the primary overlay. However, with this change, if a board only has a private overlay it will be used as the primary. BUG=7339 TEST=Ran setup_board, build_packages, and build_image on a board with only a private overlay (locally created). Ran setup_board, build_packages, and build_image on a board with both public and private overlays. Change-Id: Ic56e4951272ddb80511a213e3a2e54419267a49a Related Issues: http://codereview.chromium.org/3544009 http://codereview.chromium.org/3571015 Review URL: http://codereview.chromium.org/3622003 --- bin/cros_overlay_list | 103 ++++++++++++++++++++++++++++-------------- parallel_emerge | 16 ++++++- 2 files changed, 84 insertions(+), 35 deletions(-) diff --git a/bin/cros_overlay_list b/bin/cros_overlay_list index 1867da25a6..2cf911a858 100755 --- a/bin/cros_overlay_list +++ b/bin/cros_overlay_list @@ -15,6 +15,8 @@ get_default_board # Flags DEFINE_string board "$DEFAULT_BOARD" "The name of the board to set up." DEFINE_string board_overlay "" "Location of the board overlay." +DEFINE_boolean primary_only ${FLAGS_FALSE} \ + "Only return the path to the board's primary overlay. (Default: false)" DEFINE_string variant "" "Board variant." # Parse command line flags @@ -40,6 +42,53 @@ if [[ $FLAGS_variant =~ [_\ ] ]] ; then exit 1 fi +# +# Check that the provided variant overlay name is valid. +# +if [ -n "$FLAGS_variant" ] ; then + VARIANT_NAME="overlay-variant-${FLAGS_board}-${FLAGS_variant}" + VARIANT_OVERLAY="${SRC_ROOT}/overlays/${VARIANT_NAME}" + PRIVATE_VARIANT_NAME="overlay-variant-${FLAGS_board}-${FLAGS_variant}-private" + PRIVATE_VARIANT_OVERLAY="${SRC_ROOT}/private-overlays/${PRIVATE_VARIANT_NAME}" + if [ ! -d "${VARIANT_OVERLAY}" ] && \ + [ ! -d "${PRIVATE_VARIANT_OVERLAY}" ] ; then + error "There is no variant overlay called '${FLAGS_variant}'" + exit 1 + fi +fi + +function is_primary_overlay() { + local directory=$1 + [ -f "${directory}/make.conf" ] || return 1 + [ -f "${directory}/toolchain.conf" ] || return 1 + return 0 +} + +BOARD_OVERLAY="${SRC_ROOT}/overlays/overlay-${FLAGS_board}" +PRIVATE_OVERLAY_NAME="overlay-${FLAGS_board}-private" +PRIVATE_BOARD_OVERLAY="${SRC_ROOT}/private-overlays/${PRIVATE_OVERLAY_NAME}" + +# +# Identify the primary board overlay or die. +# +if is_primary_overlay ${BOARD_OVERLAY}; then + PRIMARY_OVERLAY="${BOARD_OVERLAY}" +elif is_primary_overlay "${PRIVATE_BOARD_OVERLAY}"; then + PRIMARY_OVERLAY="${PRIVATE_BOARD_OVERLAY}" +fi +if [ ! -n "${PRIMARY_OVERLAY}" ]; then + error "There is no primary board overlay for ${FLAGS_board}" + exit 1 +fi + +# +# If only the primary overlay is needed, provide it and exit. +# +if [ "${FLAGS_primary_only}" -eq "${FLAGS_TRUE}" ]; then + echo "${PRIMARY_OVERLAY}" + exit 0 +fi + # # Check for chromeos-overlay. # @@ -50,50 +99,38 @@ if [ -d "${CHROMEOS_OVERLAY}" ]; then fi # -# Check if there are any board overlays. There should be at least a top -# level board specific overlay. +# Check if there are any public board overlays. # -PRIMARY_BOARD_OVERLAY="${SRC_ROOT}/overlays/overlay-${FLAGS_board}" - -if [ -d "${PRIMARY_BOARD_OVERLAY}" ]; then - echo "${PRIMARY_BOARD_OVERLAY}" +if [ -d "${BOARD_OVERLAY}" ]; then + echo "${BOARD_OVERLAY}" # - # Add the public variant overlay + # Add the public variant overlay if it exists. # if [ -n "$FLAGS_variant" ] ; then - VARIANT_NAME="overlay-variant-${FLAGS_board}-${FLAGS_variant}" - VARIANT_OVERLAY="${SRC_ROOT}/overlays/${VARIANT_NAME}" - - if [ ! -d "$VARIANT_OVERLAY" ] ; then - error "Can't find variant overlay directory $VARIANT_OVERLAY" - exit 1 + if [ -d "$VARIANT_OVERLAY" ] ; then + echo "${VARIANT_OVERLAY}" fi + fi +fi - echo "${VARIANT_OVERLAY}" +# +# Add any private overlays and private variant overlays for this board. +# +if [ -d "${SRC_ROOT}/private-overlays" ] ; then + OVERLAY_NAME="overlay-${FLAGS_board}-private" + PRIVATE_OVERLAY="${SRC_ROOT}/private-overlays/${OVERLAY_NAME}" + + if [ -d "${PRIVATE_OVERLAY}" ] ; then + echo "${PRIVATE_OVERLAY}" fi # - # Add any private overlays and variant overlays for this board. + # Add the private variant overlay if it exists. # - if [ -d "${SRC_ROOT}/private-overlays" ] ; then - OVERLAY_NAME="overlay-${FLAGS_board}-private" - PRIVATE_OVERLAY="${SRC_ROOT}/private-overlays/${OVERLAY_NAME}" - - if [ -d "${PRIVATE_OVERLAY}" ] ; then - echo "${PRIVATE_OVERLAY}" - fi - - # - # Add the public and private variant overlays - # - if [ -n "$FLAGS_variant" ] ; then - VARIANT_NAME="overlay-variant-${FLAGS_board}-${FLAGS_variant}-private" - PRIVATE_VARIANT_OVERLAY="${SRC_ROOT}/private-overlays/${VARIANT_NAME}" - - if [ -d "${PRIVATE_VARIANT_OVERLAY}" ] ; then - echo "${PRIVATE_VARIANT_OVERLAY}" - fi + if [ -n "$FLAGS_variant" ] ; then + if [ -d "${PRIVATE_VARIANT_OVERLAY}" ] ; then + echo "${PRIVATE_VARIANT_OVERLAY}" fi fi fi diff --git a/parallel_emerge b/parallel_emerge index ce40be4b07..6516c29f5e 100755 --- a/parallel_emerge +++ b/parallel_emerge @@ -297,11 +297,23 @@ class DepGraphGenerator(object): os.environ["PORTAGE_SYSROOT"] = "/build/" + self.board os.environ["SYSROOT"] = "/build/" + self.board scripts_dir = os.path.dirname(os.path.realpath(__file__)) - toolchain_path = "%s/../overlays/overlay-%s/toolchain.conf" # Strip the variant out of the board name to look for the toolchain. This # is similar to what setup_board does. board_no_variant = self.board.split('_')[0] - f = open(toolchain_path % (scripts_dir, board_no_variant)) + public_toolchain_path = ("%s/../overlays/overlay-%s/toolchain.conf" % + (scripts_dir, board_no_variant)) + private_toolchain_path = ( + "%s/../private-overlays/overlay-%s-private/toolchain.conf" % + (scripts_dir, board_no_variant)) + if os.path.isfile(public_toolchain_path): + toolchain_path = public_toolchain_path + elif os.path.isfile(private_toolchain_path): + toolchain_path = private_toolchain_path + else: + print "Not able to locate toolchain.conf in board overlays" + sys.exit(1) + + f = open(toolchain_path) os.environ["CHOST"] = f.readline().strip() f.close() From a8ae0bc63b359b13a6d53afb50ac2efccba5eac6 Mon Sep 17 00:00:00 2001 From: David James Date: Wed, 1 Dec 2010 13:43:48 -0800 Subject: [PATCH 14/31] Update prebuilt.py to set permissions correctly on created Packages file. Since the local Packages file is a temporary file, other users won't have permissions to read it. rsync copies those permissions over to the remote server. To ensure remote server has the right permissions, we need to use chmod a+r on either the local file or the remote file. In this case, I chose to run rsync on the remote file. TEST=rsync -av --chmod=a+r foo bar and verify permissions are added to bar. BUG=chromium-os:5311 Change-Id: Iee4c8df00b36b9f9434546603195ea1906b289f1 Review URL: http://codereview.chromium.org/5501001 --- prebuilt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prebuilt.py b/prebuilt.py index 607397ee8d..2bada2ea35 100755 --- a/prebuilt.py +++ b/prebuilt.py @@ -441,11 +441,12 @@ def UploadPrebuilt(build_path, upload_location, version, binhost_base_url, ssh_server, remote_path = remote_location.split(':', 1) d = { 'pkg_index': tmp_packages_file.name, 'pkgs': pkgs, + 'remote_packages': '%s/Packages' % remote_location.rstrip('/'), 'remote_path': remote_path, 'remote_location': remote_location, 'ssh_server': ssh_server } cmds = ['ssh %(ssh_server)s mkdir -p %(remote_path)s' % d, - 'rsync -av %(pkg_index)s %(remote_location)s/Packages' % d] + 'rsync -av --chmod=a+r %(pkg_index)s %(remote_packages)s' % d] if pkgs: cmds.append('rsync -Rav %(pkgs)s %(remote_location)s/' % d) for cmd in cmds: From 3b9107014b9f4c0013169fab923737bc8ae561ac Mon Sep 17 00:00:00 2001 From: David Rochberg Date: Thu, 2 Dec 2010 10:45:21 -0500 Subject: [PATCH 15/31] restart_in_chroot_if_needed can run scripts from bin/. cros_workon_now does this BUG=chromium-os:9877 TEST=ran bin/cros_workon_now, cros_workon, set_shared_user_password Note---this introduces another bashism to common.sh. However, despite comments to the contrary, common.sh does not actually parse under dash without this change Change-Id: I1e6b9751afa8341c100f3b8e0b96284835a53d17 Review URL: http://codereview.chromium.org/5444002 --- bin/cros_workon_make | 2 +- common.sh | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bin/cros_workon_make b/bin/cros_workon_make index da2a3ebca5..1556149b82 100755 --- a/bin/cros_workon_make +++ b/bin/cros_workon_make @@ -10,7 +10,7 @@ . "$(dirname $0)/../common.sh" # Script must be run inside the chroot. -assert_inside_chroot +restart_in_chroot_if_needed "$@" get_default_board diff --git a/common.sh b/common.sh index 20a473cff8..a11f257289 100644 --- a/common.sh +++ b/common.sh @@ -106,7 +106,7 @@ DEFAULT_BUILD_ROOT=${CHROMEOS_BUILD_ROOT:-"$SRC_ROOT/build"} # Set up a global ALL_BOARDS value if [ -d $SRC_ROOT/overlays ]; then ALL_BOARDS=$(cd $SRC_ROOT/overlays;ls -1d overlay-* 2>&-|sed 's,overlay-,,g') -fi +fi # Strip CR ALL_BOARDS=$(echo $ALL_BOARDS) # Set a default BOARD @@ -231,11 +231,15 @@ function make_pkg_common { # Enter a chroot and restart the current script if needed function restart_in_chroot_if_needed { + # NB: Pass in ARGV: restart_in_chroot_if_needed "$@" if [ $INSIDE_CHROOT -ne 1 ] then - # Equivalent to enter_chroot.sh -- + local abspath=$(readlink -f "$0") + # strip everything up to (and including) /src/scripts/ from abspath + local path_from_scripts="${abspath##*/src/scripts/}" exec $SCRIPTS_DIR/enter_chroot.sh -- \ - $CHROOT_TRUNK_DIR/src/scripts/$(basename $0) $* + "$CHROOT_TRUNK_DIR/src/scripts/$path_from_scripts" "$@" + exit fi } From c15b609efacb7b8240d8cc00133c6194bb70cdfa Mon Sep 17 00:00:00 2001 From: David James Date: Thu, 2 Dec 2010 10:21:49 -0800 Subject: [PATCH 16/31] Fix handling of array arguments in chromite cros_build_lib. Change-Id: Ibc8d603708bb239a125d0c642deecdc3a171ca63 BUG=chromium-os:9759 TEST=cros_build_lib.RunCommand(['ls', '-l']) Review URL: http://codereview.chromium.org/5347008 --- chromite/lib/cros_build_lib.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/chromite/lib/cros_build_lib.py b/chromite/lib/cros_build_lib.py index 1d2860ba93..87fa133b75 100644 --- a/chromite/lib/cros_build_lib.py +++ b/chromite/lib/cros_build_lib.py @@ -78,9 +78,8 @@ def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None, cmd_result.cmd = cmd_str try: - proc = subprocess.Popen(cmd_str, cwd=cwd, stdin=stdin, - stdout=stdout, stderr=stderr, - shell=shell) + proc = subprocess.Popen(cmd, cwd=cwd, stdin=stdin, stdout=stdout, + stderr=stderr, shell=shell) (cmd_result.output, cmd_result.error) = proc.communicate(input) if exit_code: cmd_result.returncode = proc.returncode From a63cd2d64848568316bb8d69c4b47d37732abad8 Mon Sep 17 00:00:00 2001 From: Mandeep Singh Baines Date: Thu, 2 Dec 2010 11:58:26 -0800 Subject: [PATCH 17/31] Create a new command update_kernel.sh This command can be used to update the kernel of a ChromiumOS target. Much thanks to snanda for the inspiration and instructions. BUG=9864 TEST=Verified that it works. Change-Id: I66f7d940e0dc59b0e1a50409a7155fe0ba7157fe Review URL: http://codereview.chromium.org/5550001 --- image_to_live.sh | 25 -------------- remote_access.sh | 25 ++++++++++++++ update_kernel.sh | 86 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 25 deletions(-) create mode 100755 update_kernel.sh diff --git a/image_to_live.sh b/image_to_live.sh index 58b83e9138..f18badf667 100755 --- a/image_to_live.sh +++ b/image_to_live.sh @@ -291,31 +291,6 @@ function run_auto_update { fi } -function remote_reboot { - info "Rebooting." - remote_sh "touch /tmp/awaiting_reboot; reboot" - local output_file - output_file="${TMP}/output" - - while true; do - REMOTE_OUT="" - # This may fail while the machine is down so generate output and a - # boolean result to distinguish between down/timeout and real failure - ! remote_sh_allow_changed_host_key \ - "echo 0; [ -e /tmp/awaiting_reboot ] && echo '1'; true" - echo "${REMOTE_OUT}" > "${output_file}" - if grep -q "0" "${output_file}"; then - if grep -q "1" "${output_file}"; then - info "Not yet rebooted" - else - info "Rebooted and responding" - break - fi - fi - sleep .5 - done -} - function verify_image { info "Verifying image." "${SCRIPTS_DIR}/mount_gpt_image.sh" --from "$(dirname ${IMAGE_PATH})" \ diff --git a/remote_access.sh b/remote_access.sh index f3d490ab97..794deff8e7 100644 --- a/remote_access.sh +++ b/remote_access.sh @@ -68,6 +68,31 @@ function learn_board() { info "Target reports board is ${FLAGS_board}" } +function remote_reboot { + info "Rebooting." + remote_sh "touch /tmp/awaiting_reboot; reboot" + local output_file + output_file="${TMP}/output" + + while true; do + REMOTE_OUT="" + # This may fail while the machine is down so generate output and a + # boolean result to distinguish between down/timeout and real failure + ! remote_sh_allow_changed_host_key \ + "echo 0; [ -e /tmp/awaiting_reboot ] && echo '1'; true" + echo "${REMOTE_OUT}" > "${output_file}" + if grep -q "0" "${output_file}"; then + if grep -q "1" "${output_file}"; then + info "Not yet rebooted" + else + info "Rebooted and responding" + break + fi + fi + sleep .5 + done +} + function cleanup_remote_access() { # Call this function from the exit trap of the main script. # Iff we started ssh-agent, be nice and clean it up. diff --git a/update_kernel.sh b/update_kernel.sh new file mode 100755 index 0000000000..126786b2f6 --- /dev/null +++ b/update_kernel.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +# Copyright (c) 2009-2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Script to update the kernel on a live running ChromiumOS instance. + +# Load common constants. This should be the first executable line. +# The path to common.sh should be relative to your script's location. + +. "$(dirname $0)/common.sh" +. "$(dirname $0)/remote_access.sh" + +DEFINE_string board "" "Override board reported by target" +DEFINE_string partition "" "Override kernel partition reported by target" + +function cleanup { + cleanup_remote_access + rm -rf "${TMP}" +} + +# Ask the target what the kernel partition is +function learn_partition() { + [ -n "${FLAGS_partition}" ] && return + remote_sh cat /proc/cmdline + if echo "${REMOTE_OUT}" | grep -q "/dev/sda3"; then + FLAGS_partition="/dev/sda2" + else + FLAGS_partition="/dev/sda4" + fi + if [ -z "${FLAGS_partition}" ]; then + error "Partition required" + exit 1 + fi + info "Target reports kernel partition is ${FLAGS_partition}" +} + +function main() { + assert_outside_chroot + + cd $(dirname "$0") + + FLAGS "$@" || exit 1 + eval set -- "${FLAGS_ARGV}" + + set -e + + trap cleanup EXIT + + TMP=$(mktemp -d /tmp/image_to_live.XXXX) + + remote_access_init + + learn_board + + remote_sh uname -r -v + + old_kernel="${REMOTE_OUT}" + + cmd="vbutil_kernel --pack new_kern.bin \ + --keyblock /usr/share/vboot/devkeys/kernel.keyblock \ + --signprivate /usr/share/vboot/devkeys/kernel_data_key.vbprivk \ + --version 1 \ + --config ../build/images/${FLAGS_board}/latest/config.txt \ + --bootloader /lib64/bootstub/bootstub.efi \ + --vmlinuz /build/${FLAGS_board}/boot/vmlinuz" + + ./enter_chroot.sh -- "${cmd}" + + learn_partition + + remote_cp_to new_kern.bin /tmp + + remote_sh dd if=/tmp/new_kern.bin of="${FLAGS_partition}" + + remote_reboot + + remote_sh uname -r -v + + info "old kernel: ${old_kernel}" + + info "new kernel: ${REMOTE_OUT}" +} + +main $@ From 59a6600edd0d22775ce48db385d78b8d1e37a98e Mon Sep 17 00:00:00 2001 From: David James Date: Thu, 2 Dec 2010 13:54:48 -0800 Subject: [PATCH 18/31] Update scripts to avoid using deprecated --test_case syntax. TEST=Used cbuildbot.py to run tests BUG=chromium-os:9893 Change-Id: I3c3856fee6fa09c4e2c829f3c8f23d098b4b6bb5 Review URL: http://codereview.chromium.org/5555003 --- bin/cbuildbot.py | 2 +- bin/cros_run_parallel_vm_tests.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index d49fa06dc6..ad0c073698 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -397,8 +397,8 @@ def _RunSmokeSuite(buildroot, results_dir): cwd = os.path.join(buildroot, 'src', 'scripts') RunCommand(['bin/cros_run_vm_test', '--no_graphics', - '--test_case=suite_Smoke', '--results_dir_root=%s' % results_dir, + 'suite_Smoke', ], cwd=cwd, error_ok=False) diff --git a/bin/cros_run_parallel_vm_tests.py b/bin/cros_run_parallel_vm_tests.py index 2ca51a3afa..3d0d1d6d90 100755 --- a/bin/cros_run_parallel_vm_tests.py +++ b/bin/cros_run_parallel_vm_tests.py @@ -70,14 +70,14 @@ class ParallelTestRunner(object): args = [ os.path.join(os.path.dirname(__file__), 'cros_run_vm_test'), '--snapshot', # The image is shared so don't modify it. '--no_graphics', - '--ssh_port=%d' % ssh_port, - '--test_case=%s' % test ] + '--ssh_port=%d' % ssh_port ] if self._board: args.append('--board=%s' % self._board) if self._image_path: args.append('--image_path=%s' % self._image_path) if self._results_dir_root: args.append('--results_dir_root=%s/%s.%d' % (self._results_dir_root, test, ssh_port)) if self._use_emerged: args.append('--use_emerged') + args.append(test) Info('Running %r...' % args) output = None if self._order_output: From 55430beaefb1ef0df07e6f597fe4c0e9b8fd7400 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Thu, 2 Dec 2010 15:49:00 -0800 Subject: [PATCH 19/31] Add ability to push to subset of overlays we rev. We also fix some of the configs for existing configurations. Change-Id: I34baa9664ff9e91268c8bac976ab2ab0aee51bfb BUG=chromium-os:9714 TEST=Running now with cbuildbot x86-generic-pre-flight-queue. Review URL: http://codereview.chromium.org/5531002 --- bin/cbuildbot.py | 25 ++++++++++++++++--------- bin/cbuildbot_config.py | 36 ++++++++++++++++++++++++------------ 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index ad0c073698..c75e467cc7 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -532,7 +532,8 @@ def _ResolveOverlays(buildroot, overlays): elif overlays == 'both': paths = [public_overlay, private_overlay] else: - Die('Incorrect overlay configuration: %s' % overlays) + Info('No overlays found.') + paths = [] return paths @@ -623,11 +624,18 @@ def main(): try: # Calculate list of overlay directories. - overlays = _ResolveOverlays(buildroot, buildconfig['overlays']) + rev_overlays = _ResolveOverlays(buildroot, buildconfig['rev_overlays']) + push_overlays = _ResolveOverlays(buildroot, buildconfig['push_overlays']) + # We cannot push to overlays that we don't rev. + assert set(push_overlays).issubset(set(rev_overlays)) + # Either has to be a master or not have any push overlays. + assert buildconfig['master'] or not push_overlays + board = buildconfig['board'] old_binhost = None - _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays) + _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, + rev_overlays) chroot_path = os.path.join(buildroot, 'chroot') boardpath = os.path.join(chroot_path, 'build', board) if options.sync: @@ -642,8 +650,7 @@ def main(): RunCommand(['sudo', 'rm', '-rf', boardpath]) # Check that all overlays can be found. - for path in overlays: - assert ':' not in path, 'Overlay must not contain colons: %s' % path + for path in rev_overlays: if not os.path.isdir(path): Die('Missing overlay: %s' % path) @@ -659,7 +666,7 @@ def main(): options.chrome_rev) elif buildconfig['uprev']: _UprevPackages(buildroot, tracking_branch, revisionfile, - buildconfig['board'], overlays) + buildconfig['board'], rev_overlays) _EnableLocalAccount(buildroot) # Doesn't rebuild without acquiring more source. @@ -681,7 +688,7 @@ def main(): _RunSmokeSuite(buildroot, test_results_dir) finally: if not options.debug: - archive_full_path=os.path.join(options.gsutil_archive, + archive_full_path = os.path.join(options.gsutil_archive, str(options.buildnumber)) _ArchiveTestResults(buildroot, buildconfig['board'], test_results_dir=test_results_dir, @@ -694,10 +701,10 @@ def main(): if buildconfig['master']: # Master bot needs to check if the other slaves completed. if cbuildbot_comm.HaveSlavesCompleted(config): - _UploadPrebuilts(buildroot, board, buildconfig['overlays'], + _UploadPrebuilts(buildroot, board, buildconfig['rev_overlays'], [new_binhost]) _UprevPush(buildroot, tracking_branch, buildconfig['board'], - overlays, options.debug) + push_overlays, options.debug) else: Die('CBUILDBOT - One of the slaves has failed!!!') diff --git a/bin/cbuildbot_config.py b/bin/cbuildbot_config.py index 718cd36a70..ba3706e0f5 100644 --- a/bin/cbuildbot_config.py +++ b/bin/cbuildbot_config.py @@ -19,9 +19,12 @@ hostname -- Needed for 'important' slaves. The hostname of the bot. Should match hostname in slaves.cfg in buildbot checkout. unittests -- Runs unittests for packages. smoke_bvt -- Runs the test smoke suite in a qemu-based VM using KVM. -overlays -- Select what overlays to look at. This can be 'public', 'private' - or 'both'. There should only be one master bot pushing changes to - each overlay per branch. +rev_overlays -- Select what overlays to look at for revving. This can be + 'public', 'private' or 'both'. +push_overlays -- Select what overlays to push at. This should be a subset of + rev_overlays for the particular builder. Must be None if + not a master. There should only be one master bot pushing + changes to each overlay per branch. """ @@ -33,7 +36,8 @@ config['default'] = { 'important' : False, 'unittests' : False, 'smoke_bvt' : False, - 'overlays': 'public', + 'rev_overlays': 'public', + 'push_overlays': None, } config['x86-generic-pre-flight-queue'] = { 'board' : 'x86-generic', @@ -43,7 +47,8 @@ config['x86-generic-pre-flight-queue'] = { 'hostname' : 'chromeosbuild2', 'unittests' : True, 'smoke_bvt' : True, - 'overlays': 'public', + 'rev_overlays': 'public', + 'push_overlays': 'public', } config['x86-mario-pre-flight-queue'] = { 'board' : 'x86-mario', @@ -52,7 +57,8 @@ config['x86-mario-pre-flight-queue'] = { 'important' : False, 'unittests' : True, 'smoke_bvt' : True, - 'overlays': 'private', + 'rev_overlays': 'both', + 'push_overlays': 'private', } config['x86-mario-pre-flight-branch'] = { 'board' : 'x86-mario', @@ -61,7 +67,8 @@ config['x86-mario-pre-flight-branch'] = { 'important' : False, 'unittests' : True, 'smoke_bvt' : True, - 'overlays': 'both', + 'rev_overlays': 'both', + 'push_overlays': 'both', } config['x86_agz_bin'] = { 'board' : 'x86-agz', @@ -70,7 +77,8 @@ config['x86_agz_bin'] = { 'important' : False, 'unittests' : True, 'smoke_bvt' : True, - 'overlays': 'private', + 'rev_overlays': 'both', + 'push_overlays': None, } config['x86_dogfood_bin'] = { 'board' : 'x86-dogfood', @@ -79,7 +87,8 @@ config['x86_dogfood_bin'] = { 'important' : False, 'unittests' : True, 'smoke_bvt' : True, - 'overlays': 'private', + 'rev_overlays': 'both', + 'push_overlays': None, } config['x86_pineview_bin'] = { 'board' : 'x86-pineview', @@ -87,7 +96,8 @@ config['x86_pineview_bin'] = { 'master' : False, 'important' : False, 'unittests': True, - 'overlays': 'public', + 'rev_overlays': 'public', + 'push_overlays': None, } config['arm_tegra2_bin'] = { 'board' : 'tegra2', @@ -95,7 +105,8 @@ config['arm_tegra2_bin'] = { 'master' : False, 'important' : False, 'unittests' : False, - 'overlays': 'public', + 'rev_overlays': 'public', + 'push_overlays': None, } config['arm_generic_bin'] = { 'board' : 'arm-generic', @@ -103,5 +114,6 @@ config['arm_generic_bin'] = { 'master' : False, 'important' : False, 'unittests' : False, - 'overlays': 'public', + 'rev_overlays': 'public', + 'push_overlays': None, } From 969a67304525cae2d03af218137f445591964072 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Thu, 2 Dec 2010 17:54:40 -0800 Subject: [PATCH 20/31] Add testPartialUpdate, testCorruptedUpdate. Add two new tests that test update-engine handling of a corrupted image, or a truncated image. BUG=chromium-os:9502 TEST=Ran against Dell L13, Mario Review URL: http://codereview.chromium.org/5373008 Change-Id: Ie1043004f01131ea837798c06327c486e1901c15 --- bin/cros_au_test_harness.py | 170 +++++++++++++++++++++++++++++++----- bin/cros_run_vm_update | 13 ++- 2 files changed, 158 insertions(+), 25 deletions(-) diff --git a/bin/cros_au_test_harness.py b/bin/cros_au_test_harness.py index bedd16e094..eacf837d18 100755 --- a/bin/cros_au_test_harness.py +++ b/bin/cros_au_test_harness.py @@ -6,14 +6,17 @@ import optparse import os +import re import sys import unittest +import urllib sys.path.append(os.path.join(os.path.dirname(__file__), '../lib')) from cros_build_lib import Die from cros_build_lib import Info from cros_build_lib import ReinterpretPathForChroot from cros_build_lib import RunCommand +from cros_build_lib import RunCommandCaptureOutput from cros_build_lib import Warning # VM Constants. @@ -29,6 +32,11 @@ global remote global target_image_path global vm_graphics_flag +class UpdateException(Exception): + """Exception thrown when UpdateImage or UpdateUsingPayload fail""" + def __init__(self, code, stdout): + self.code = code + self.stdout = stdout class AUTest(object): """Abstract interface that defines an Auto Update test.""" @@ -40,6 +48,7 @@ class AUTest(object): # Set these up as they are used often. self.crosutils = os.path.join(os.path.dirname(__file__), '..') self.crosutilsbin = os.path.join(os.path.dirname(__file__)) + self.download_folder = os.path.join(self.crosutilsbin, 'latest_download') def GetStatefulChangeFlag(self, stateful_change): """Returns the flag to pass to image_to_vm for the stateful change.""" @@ -74,11 +83,36 @@ class AUTest(object): Warning('Delta update failed, disabling delta updates and retrying.') self.use_delta_updates = False self.source_image = '' - self.UpdateImage(image) + self._UpdateImageReportError(image) else: - self.UpdateImage(image) + self._UpdateImageReportError(image) - def PrepareBase(self): + def _UpdateImageReportError(self, image_path, stateful_change='old'): + """Calls UpdateImage and reports any error to the console. + + Still throws the exception. + """ + try: + self.UpdateImage(image_path, stateful_change) + except UpdateException as err: + # If the update fails, print it out + Warning(err.stdout) + raise + + def _AttemptUpdateWithPayloadExpectedFailure(self, payload, expected_msg): + # This update is expected to fail... + try: + self.UpdateUsingPayload(payload) + except UpdateException as err: + # Will raise ValueError if expected is not found. + if re.search(re.escape(expected_msg), err.stdout, re.MULTILINE): + return + + Warning("Didn't find '%s' in:" % expected_msg) + Warning(err.stdout) + self.fail('We managed to update when failure was expected') + + def PrepareBase(self, image_path): """Prepares target with base_image_path.""" pass @@ -95,6 +129,15 @@ class AUTest(object): """ pass + def UpdateUsingPayload(self, update_path, stateful_change='old'): + """Updates target with the pre-generated update stored in update_path + + Args: + update_path: Path to the image to update with. This directory should + contain both update.gz, and stateful.image.gz + """ + pass + def VerifyImage(self, percent_required_to_pass): """Verifies the image with tests. @@ -140,7 +183,7 @@ class AUTest(object): """ # Just make sure some tests pass on original image. Some old images # don't pass many tests. - self.PrepareBase() + self.PrepareBase(image_path=base_image_path) # TODO(sosa): move to 100% once we start testing using the autotest paired # with the dev channel. percent_passed = self.VerifyImage(10) @@ -163,7 +206,7 @@ class AUTest(object): """ # Just make sure some tests pass on original image. Some old images # don't pass many tests. - self.PrepareBase() + self.PrepareBase(image_path=base_image_path) # TODO(sosa): move to 100% once we start testing using the autotest paired # with the dev channel. percent_passed = self.VerifyImage(10) @@ -178,6 +221,40 @@ class AUTest(object): self.TryDeltaAndFallbackToFull(target_image_path, base_image_path, 'clean') self.VerifyImage(percent_passed) + def testPartialUpdate(self): + """Tests what happens if we attempt to update with a truncated payload.""" + # Preload with the version we are trying to test. + self.PrepareBase(image_path=target_image_path) + + # Image can be updated at: + # ~chrome-eng/chromeos/localmirror/autest-images + url = 'http://gsdview.appspot.com/chromeos-localmirror/' \ + 'autest-images/truncated_image.gz' + payload = os.path.join(self.download_folder, 'truncated_image.gz') + + # Read from the URL and write to the local file + urllib.urlretrieve(url, payload) + + expected_msg='download_hash_data == update_check_response_hash failed' + self._AttemptUpdateWithPayloadExpectedFailure(payload, expected_msg) + + def testCorruptedUpdate(self): + """Tests what happens if we attempt to update with a corrupted payload.""" + # Preload with the version we are trying to test. + self.PrepareBase(image_path=target_image_path) + + # Image can be updated at: + # ~chrome-eng/chromeos/localmirror/autest-images + url = 'http://gsdview.appspot.com/chromeos-localmirror/' \ + 'autest-images/corrupted_image.gz' + payload = os.path.join(self.download_folder, 'corrupted.gz') + + # Read from the URL and write to the local file + urllib.urlretrieve(url, payload) + + # This update is expected to fail... + expected_msg='zlib inflate() error:-3' + self._AttemptUpdateWithPayloadExpectedFailure(payload, expected_msg) class RealAUTest(unittest.TestCase, AUTest): """Test harness for updating real images.""" @@ -185,23 +262,40 @@ class RealAUTest(unittest.TestCase, AUTest): def setUp(self): AUTest.setUp(self) - def PrepareBase(self): + def PrepareBase(self, image_path): """Auto-update to base image to prepare for test.""" - self.UpdateImage(base_image_path) + self._UpdateImageReportError(image_path) def UpdateImage(self, image_path, stateful_change='old'): """Updates a remote image using image_to_live.sh.""" stateful_change_flag = self.GetStatefulChangeFlag(stateful_change) - RunCommand([ + (code, stdout, stderr) = RunCommandCaptureOutput([ '%s/image_to_live.sh' % self.crosutils, '--image=%s' % image_path, '--remote=%s' % remote, stateful_change_flag, '--verify', - '--src_image=%s' % self.source_image, - ], enter_chroot=False) + '--src_image=%s' % self.source_image + ]) + if code != 0: + raise UpdateException(code, stdout) + + def UpdateUsingPayload(self, update_path, stateful_change='old'): + """Updates a remote image using image_to_live.sh.""" + stateful_change_flag = self.GetStatefulChangeFlag(stateful_change) + + (code, stdout, stderr) = RunCommandCaptureOutput([ + '%s/image_to_live.sh' % self.crosutils, + '--payload=%s' % update_path, + '--remote=%s' % remote, + stateful_change_flag, + '--verify', + ]) + + if code != 0: + raise UpdateException(code, stdout) def VerifyImage(self, percent_required_to_pass): """Verifies an image using run_remote_tests.sh with verification suite.""" @@ -233,17 +327,19 @@ class VirtualAUTest(unittest.TestCase, AUTest): AUTest.setUp(self) self._KillExistingVM(_KVM_PID_FILE) - def PrepareBase(self): + def PrepareBase(self, image_path): """Creates an update-able VM based on base image.""" self.vm_image_path = '%s/chromiumos_qemu_image.bin' % os.path.dirname( - base_image_path) + image_path) + + Info('Creating: %s' % self.vm_image_path) if not os.path.exists(self.vm_image_path): Info('Qemu image %s not found, creating one.' % self.vm_image_path) RunCommand(['%s/image_to_vm.sh' % self.crosutils, '--full', '--from=%s' % ReinterpretPathForChroot( - os.path.dirname(base_image_path)), + os.path.dirname(image_path)), '--vdisk_size=%s' % _FULL_VDISK_SIZE, '--statefulfs_size=%s' % _FULL_STATEFULFS_SIZE, '--board=%s' % board, @@ -251,6 +347,9 @@ class VirtualAUTest(unittest.TestCase, AUTest): else: Info('Using existing VM image %s' % self.vm_image_path) + + Info('Testing for %s' % self.vm_image_path) + self.assertTrue(os.path.exists(self.vm_image_path)) def UpdateImage(self, image_path, stateful_change='old'): @@ -259,16 +358,41 @@ class VirtualAUTest(unittest.TestCase, AUTest): if self.source_image == base_image_path: self.source_image = self.vm_image_path - RunCommand(['%s/cros_run_vm_update' % self.crosutilsbin, - '--update_image_path=%s' % image_path, - '--vm_image_path=%s' % self.vm_image_path, - '--snapshot', - vm_graphics_flag, - '--persist', - '--kvm_pid=%s' % _KVM_PID_FILE, - stateful_change_flag, - '--src_image=%s' % self.source_image, - ], enter_chroot=False) + (code, stdout, stderr) = RunCommandCaptureOutput([ + '%s/cros_run_vm_update' % self.crosutilsbin, + '--update_image_path=%s' % image_path, + '--vm_image_path=%s' % self.vm_image_path, + '--snapshot', + vm_graphics_flag, + '--persist', + '--kvm_pid=%s' % _KVM_PID_FILE, + stateful_change_flag, + '--src_image=%s' % self.source_image, + ]) + + if code != 0: + raise UpdateException(code, stdout) + + def UpdateUsingPayload(self, update_path, stateful_change='old'): + """Updates a remote image using image_to_live.sh.""" + stateful_change_flag = self.GetStatefulChangeFlag(stateful_change) + if self.source_image == base_image_path: + self.source_image = self.vm_image_path + + (code, stdout, stderr) = RunCommandCaptureOutput([ + '%s/cros_run_vm_update' % self.crosutilsbin, + '--payload=%s' % update_path, + '--vm_image_path=%s' % self.vm_image_path, + '--snapshot', + vm_graphics_flag, + '--persist', + '--kvm_pid=%s' % _KVM_PID_FILE, + stateful_change_flag, + '--src_image=%s' % self.source_image, + ]) + + if code != 0: + raise UpdateException(code, stdout) def VerifyImage(self, percent_required_to_pass): """Runs vm smoke suite to verify image.""" diff --git a/bin/cros_run_vm_update b/bin/cros_run_vm_update index 0af8fb3a3e..abd0071462 100755 --- a/bin/cros_run_vm_update +++ b/bin/cros_run_vm_update @@ -9,6 +9,7 @@ . "$(dirname $0)/../common.sh" . "$(dirname $0)/../lib/cros_vm_lib.sh" +DEFINE_string payload "" "Full name of the payload to update with." DEFINE_string src_image "" \ "Create a delta update by passing in the image on the remote machine." DEFINE_string stateful_update_flag "" "Flags to pass to stateful update." s @@ -21,7 +22,7 @@ set -e FLAGS "$@" || exit 1 eval set -- "${FLAGS_ARGV}" -[ -n "${FLAGS_update_image_path}" ] || \ +[ -n "${FLAGS_update_image_path}" ] || [ -n "${FLAGS_payload}" ] || \ die "You must specify a path to an image to use as an update." [ -n "${FLAGS_vm_image_path}" ] || \ die "You must specify a path to a vm image." @@ -29,6 +30,14 @@ eval set -- "${FLAGS_ARGV}" trap stop_kvm EXIT start_kvm "${FLAGS_vm_image_path}" +if [ -n "${FLAGS_update_image_path}" ]; then + IMAGE_ARGS="--image=$(readlink -f ${FLAGS_update_image_path})" +fi + +if [ -n "${FLAGS_payload}" ]; then + IMAGE_ARGS="--payload="${FLAGS_payload}"" +fi + $(dirname $0)/../image_to_live.sh \ --remote=127.0.0.1 \ --ssh_port=${FLAGS_ssh_port} \ @@ -36,5 +45,5 @@ $(dirname $0)/../image_to_live.sh \ --src_image="${FLAGS_src_image}" \ --verify \ --for_vm \ - --image=$(readlink -f ${FLAGS_update_image_path}) + ${IMAGE_ARGS} From 4f09b96e70a2a8cc3b70e1ae9fa4642a7f16b405 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Thu, 2 Dec 2010 20:07:48 -0800 Subject: [PATCH 21/31] Use release candidates as a backup in case we fail to find the latest release. Change-Id: Ia94f1c983aec533fc2813f9c300266d394b8ee7c BUG=chromium-os:9946 TEST=Ran ctest and saw through the download and start of the test harness Review URL: http://codereview.chromium.org/5628002 --- bin/ctest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/ctest.py b/bin/ctest.py index c1c806e558..3310056df7 100755 --- a/bin/ctest.py +++ b/bin/ctest.py @@ -167,7 +167,11 @@ def GetLatestZipUrl(board, channel, latest_url_base, zip_server_base): Warning(('Could not use latest link provided, defaulting to parsing' ' latest from zip url base.')) - return GetNewestLinkFromZipBase(board, channel, zip_server_base) + try: + return GetNewestLinkFromZipBase(board, channel, zip_server_base) + except: + Warning('Failed to get url from standard zip base. Trying rc.') + return GetNewestLinkFromZipBase(board + '-rc', channel, zip_server_base) def GrabZipAndExtractImage(zip_url, download_folder, image_name) : From 46b5e1d2860cdbf2376ab05fcce59de154125998 Mon Sep 17 00:00:00 2001 From: Mandeep Singh Baines Date: Fri, 3 Dec 2010 16:18:30 -0800 Subject: [PATCH 22/31] cros_workon_make: actually inplace when doing an install Also, fixed to honour external FEATURES for the install case. BUG=n0ne TEST=Built a kernel with this CL. Change-Id: I17d50cc382b297766761f12d47a555630fedc016 Review URL: http://codereview.chromium.org/5530004 --- bin/cros_workon_make | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/cros_workon_make b/bin/cros_workon_make index 1556149b82..df5f0a7686 100755 --- a/bin/cros_workon_make +++ b/bin/cros_workon_make @@ -85,7 +85,8 @@ if [ "${FLAGS_test}" = "${FLAGS_TRUE}" ]; then emerge_features="test" fi if [ "${FLAGS_install}" = "${FLAGS_TRUE}" ]; then - FEATURES="${emerge_features}" "${EMERGECMD}" "${1}" + SANDBOX_WRITE=~/trunk CROS_WORKON_INPLACE=1 \ + FEATURES="${emerge_features} ${FEATURES}" "${EMERGECMD}" "${1}" exit $? fi From 1a4a9665d4c6e976cba2200294dbd61c8936bfcc Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Fri, 3 Dec 2010 16:28:33 -0800 Subject: [PATCH 23/31] Have upload_prebuilts respect the debug flag. Chrome PFQ right now is uploading prebuilts which wasn't meant to happen :( Change-Id: I502a0621a6c9f655e5a3f890dd79e00652925902 BUG=chromium-os:8693 TEST=Ran cbuildbot with --debug Review URL: http://codereview.chromium.org/5595004 --- bin/cbuildbot.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index c75e467cc7..2e86ae4173 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -701,8 +701,9 @@ def main(): if buildconfig['master']: # Master bot needs to check if the other slaves completed. if cbuildbot_comm.HaveSlavesCompleted(config): - _UploadPrebuilts(buildroot, board, buildconfig['rev_overlays'], - [new_binhost]) + if not options.debug: + _UploadPrebuilts(buildroot, board, buildconfig['rev_overlays'], + [new_binhost]) _UprevPush(buildroot, tracking_branch, buildconfig['board'], push_overlays, options.debug) else: From dcd5b59d4a22a7fd63e42679ca5b077f744bfa71 Mon Sep 17 00:00:00 2001 From: Anush Elangovan Date: Sat, 4 Dec 2010 17:21:38 -0800 Subject: [PATCH 24/31] Disable chrome version check in vm_test since it is in a ebuild Change-Id: I1a3f46cf37341255dbb01866cf68e5308bb36d83 BUG=10024 TEST=adhoc Review URL: http://codereview.chromium.org/5603007 --- bin/cros_run_vm_test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cros_run_vm_test b/bin/cros_run_vm_test index d62a2b9852..c583a462cf 100755 --- a/bin/cros_run_vm_test +++ b/bin/cros_run_vm_test @@ -77,7 +77,7 @@ if [ -n "${FLAGS_verify_chrome_version}" ]; then --remote=127.0.0.1 \ --ssh_port=${FLAGS_ssh_port}) [[ ${chrome_version_on_vm} == ${FLAGS_verify_chrome_version} ]] || \ - die "Chrome version mismatch. VM reported ${chrome_version_on_vm}" + warn "CHROME_VERSION is no longer set.This check will be removed" else warn "${FLAGS_verify_chrome_version} is not a valid Chrome version" fi From c9f365d707dceffac53299640fdf87ac7dbbb8ce Mon Sep 17 00:00:00 2001 From: David McMahon Date: Mon, 6 Dec 2010 15:19:53 -0800 Subject: [PATCH 25/31] Created branch 0.9.130.B. Update CHROMEOS_VERSION_BRANCH=131 --- chromeos_version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chromeos_version.sh b/chromeos_version.sh index b836d7e976..0540c3ab7f 100755 --- a/chromeos_version.sh +++ b/chromeos_version.sh @@ -26,7 +26,7 @@ export CHROMEOS_VERSION_MINOR=9 # Increment by 2 in trunk after making a release branch. # Does not reset on a major/minor change (always increases). # (Trunk is always odd; branches are always even). -export CHROMEOS_VERSION_BRANCH=129 +export CHROMEOS_VERSION_BRANCH=131 # Patch number. # Increment by 1 each release on a branch. From e55c89ab447531e081497a8c4fbc47b4bc1b20dd Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Mon, 6 Dec 2010 15:59:14 -0800 Subject: [PATCH 26/31] Create the latest_download dir if it doesn't exist. I started to use /tmp, but it doesn't exist both inside and outside of the chroot. Using a consistent name like this has the advantage that the files will be cached and not redownloaded every time. Well, except on the build servers that wipe the related directories between builds. Change-Id: I58b42d9f1bd18cc4eb44043ed2436d4ffa482767 BUG=chromium-os:10012 TEST= Review URL: http://codereview.chromium.org/5597004 --- bin/cros_au_test_harness.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bin/cros_au_test_harness.py b/bin/cros_au_test_harness.py index eacf837d18..3ffdac53cc 100755 --- a/bin/cros_au_test_harness.py +++ b/bin/cros_au_test_harness.py @@ -48,7 +48,9 @@ class AUTest(object): # Set these up as they are used often. self.crosutils = os.path.join(os.path.dirname(__file__), '..') self.crosutilsbin = os.path.join(os.path.dirname(__file__)) - self.download_folder = os.path.join(self.crosutilsbin, 'latest_download') + self.download_folder = os.path.join(self.crosutils, 'latest_download') + if not os.path.exists(self.download_folder): + os.makedirs(self.download_folder) def GetStatefulChangeFlag(self, stateful_change): """Returns the flag to pass to image_to_vm for the stateful change.""" @@ -89,7 +91,7 @@ class AUTest(object): def _UpdateImageReportError(self, image_path, stateful_change='old'): """Calls UpdateImage and reports any error to the console. - + Still throws the exception. """ try: @@ -331,7 +333,7 @@ class VirtualAUTest(unittest.TestCase, AUTest): """Creates an update-able VM based on base image.""" self.vm_image_path = '%s/chromiumos_qemu_image.bin' % os.path.dirname( image_path) - + Info('Creating: %s' % self.vm_image_path) if not os.path.exists(self.vm_image_path): From 0614c4a11c5a5270e3c54d868a1114ab6746be13 Mon Sep 17 00:00:00 2001 From: David James Date: Tue, 7 Dec 2010 13:39:16 -0800 Subject: [PATCH 27/31] Switch from gsdview.appspot.com to commondatastorage.googleapis.com. This server requires that unsafe characters are quoted in URLs, so we do that here. Change-Id: I6b7c430671b02a8da11777029b0fdb59a87dcae1 BUG=chromium-os:9902 TEST=Ran unit tests. Ran example buildbot run and diff'd new version and old version of Packages file. Review URL: http://codereview.chromium.org/5542002 --- chromite/lib/binpkg.py | 6 ++++-- prebuilt.py | 3 +-- prebuilt_unittest.py | 14 ++++++++------ 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/chromite/lib/binpkg.py b/chromite/lib/binpkg.py index 75c86bcbac..a594571cfa 100644 --- a/chromite/lib/binpkg.py +++ b/chromite/lib/binpkg.py @@ -10,6 +10,7 @@ import operator import os import tempfile import time +import urllib import urllib2 class PackageIndex(object): @@ -52,7 +53,7 @@ class PackageIndex(object): for pkg in self.packages: cpv, sha1 = pkg['CPV'], pkg.get('SHA1') if sha1: - path = pkg.get('PATH', cpv + '.tbz2') + path = pkg.get('PATH', urllib.quote(cpv + '.tbz2')) db[sha1] = '%s/%s' % (uri.rstrip('/'), path) def _ReadPkgIndex(self, pkgfile): @@ -200,7 +201,8 @@ class PackageIndex(object): """ self.header['URI'] = base_uri for pkg in self.packages: - pkg['PATH'] = '%s/%s' % (path_prefix.rstrip('/'), pkg['CPV'] + '.tbz2') + path = urllib.quote(pkg['CPV'] + '.tbz2') + pkg['PATH'] = '%s/%s' % (path_prefix.rstrip('/'), path) def Write(self, pkgfile): """Write a packages file to disk. diff --git a/prebuilt.py b/prebuilt.py index 2bada2ea35..5330191b3a 100755 --- a/prebuilt.py +++ b/prebuilt.py @@ -55,8 +55,7 @@ _REL_HOST_PATH = 'host/%(target)s/%(version)s/packages' # relative to build path _PRIVATE_OVERLAY_DIR = 'src/private-overlays' _BINHOST_BASE_DIR = 'src/overlays' -#_BINHOST_BASE_URL = 'http://commondatastorage.googleapis.com/chromeos-prebuilt' -_BINHOST_BASE_URL = 'http://gsdview.appspot.com/chromeos-prebuilt' +_BINHOST_BASE_URL = 'http://commondatastorage.googleapis.com/chromeos-prebuilt' _PREBUILT_BASE_DIR = 'src/third_party/chromiumos-overlay/chromeos/config/' # Created in the event of new host targets becoming available _PREBUILT_MAKE_CONF = {'amd64': os.path.join(_PREBUILT_BASE_DIR, diff --git a/prebuilt_unittest.py b/prebuilt_unittest.py index 4c7255e3c5..a39aabcef6 100755 --- a/prebuilt_unittest.py +++ b/prebuilt_unittest.py @@ -10,11 +10,13 @@ import prebuilt import shutil import tempfile import unittest +import urllib from chromite.lib import cros_build_lib from chromite.lib.binpkg import PackageIndex -PUBLIC_PACKAGES = [{'CPV': 'public1', 'SHA1': '1'}, - {'CPV': 'public2', 'SHA1': '2', 'PATH': 'foo.tgz'}] +PUBLIC_PACKAGES = [{'CPV': 'gtk+/public1', 'SHA1': '1'}, + {'CPV': 'gtk+/public2', 'SHA1': '2', + 'PATH': 'gtk%2B/foo.tgz'}] PRIVATE_PACKAGES = [{'CPV': 'private', 'SHA1': '3'}] @@ -217,8 +219,8 @@ class TestPopulateDuplicateDB(unittest.TestCase): db = {} pkgindex._PopulateDuplicateDB(db) self.assertEqual(len(db), 3) - self.assertEqual(db['1'], 'http://www.example.com/public1.tbz2') - self.assertEqual(db['2'], 'http://www.example.com/foo.tgz') + self.assertEqual(db['1'], 'http://www.example.com/gtk%2B/public1.tbz2') + self.assertEqual(db['2'], 'http://www.example.com/gtk%2B/foo.tgz') self.assertEqual(db['3'], 'http://www.example.com/private.tbz2') def testMissingSHA1(self): @@ -227,7 +229,7 @@ class TestPopulateDuplicateDB(unittest.TestCase): del pkgindex.packages[0]['SHA1'] pkgindex._PopulateDuplicateDB(db) self.assertEqual(len(db), 2) - self.assertEqual(db['2'], 'http://www.example.com/foo.tgz') + self.assertEqual(db['2'], 'http://www.example.com/gtk%2B/foo.tgz') self.assertEqual(db['3'], 'http://www.example.com/private.tbz2') def testFailedPopulate(self): @@ -263,7 +265,7 @@ class TestResolveDuplicateUploads(unittest.TestCase): dup_pkgindex = SimplePackageIndex() expected_pkgindex = SimplePackageIndex() for pkg in expected_pkgindex.packages: - pkg.setdefault('PATH', pkg['CPV'] + '.tbz2') + pkg.setdefault('PATH', urllib.quote(pkg['CPV'] + '.tbz2')) uploads = pkgindex.ResolveDuplicateUploads([dup_pkgindex]) self.assertEqual(pkgindex.packages, expected_pkgindex.packages) From 2fb9892ced1301e1b8bde0dd53cf57fd52d7a4ec Mon Sep 17 00:00:00 2001 From: Ken Mixter Date: Tue, 7 Dec 2010 15:10:25 -0800 Subject: [PATCH 28/31] crosutils: Use upstream dump_syms parameters for supporting splitdebug Change-Id: If4eba8779496c4fb0283865da48ba38941de9c08 BUG=9281 TEST=ran without obvious errors Review URL: http://codereview.chromium.org/5308005 --- cros_generate_breakpad_symbols | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cros_generate_breakpad_symbols b/cros_generate_breakpad_symbols index cee8731260..8bfe8792c8 100755 --- a/cros_generate_breakpad_symbols +++ b/cros_generate_breakpad_symbols @@ -77,11 +77,12 @@ function verify_not_64b_elf() { function dump_file() { local debug_file="$1" local text_file="$2" + local debug_directory="$(dirname "${debug_file}")" # 64b ELF files may be installed on the target in PERL directories verify_not_64b_elf "${debug_file}" || return 1 verify_not_64b_elf "${text_file}" || return 1 # Dump symbols as root in order to read all files. - if ! sudo "${DUMP_SYMS}" "${debug_file}" "${text_file}" > "${SYM_FILE}" \ + if ! sudo "${DUMP_SYMS}" "${text_file}" "${debug_directory}" > "${SYM_FILE}" \ 2> "${ERR_FILE}"; then # A lot of files (like kernel files) contain no debug information, do # not consider such occurrences as errors. From 0de6de66db6b598a31c15773e2227eaaa6ee6da1 Mon Sep 17 00:00:00 2001 From: Anush Elangovan Date: Tue, 7 Dec 2010 17:31:48 -0800 Subject: [PATCH 29/31] Enable virtio for tests Change-Id: I1d0e546f6ea837928ba426e4690007ef2649f68e BUG=10102 TEST=test KVM with -net virtio and ensure it boots and can copy files in etc Review URL: http://codereview.chromium.org/5543006 --- image_to_vm.sh | 2 +- lib/cros_vm_lib.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/image_to_vm.sh b/image_to_vm.sh index 50e00e5612..2ff4984607 100755 --- a/image_to_vm.sh +++ b/image_to_vm.sh @@ -284,7 +284,7 @@ fi if [ "${FLAGS_format}" == "qemu" ]; then echo "If you have qemu-kvm installed, you can start the image by:" - echo "sudo kvm -m ${FLAGS_mem} -vga std -pidfile /tmp/kvm.pid -net nic,model=e1000 " \ + echo "sudo kvm -m ${FLAGS_mem} -vga std -pidfile /tmp/kvm.pid -net nic,model=virtio " \ "-net user,hostfwd=tcp::9222-:22 \\" echo " -hda ${FLAGS_to}/${DEFAULT_QEMU_IMAGE}" fi diff --git a/lib/cros_vm_lib.sh b/lib/cros_vm_lib.sh index 7a99cbebe2..d28539c14d 100644 --- a/lib/cros_vm_lib.sh +++ b/lib/cros_vm_lib.sh @@ -55,7 +55,7 @@ function start_kvm() { -vga std \ -pidfile "${KVM_PID_FILE}" \ -daemonize \ - -net nic,model=e1000 \ + -net nic,model=virtio \ ${nographics} \ ${snapshot} \ -net user,hostfwd=tcp::${FLAGS_ssh_port}-:22 \ From b2b446631b4127ea03d643e62b20350a40dbbef6 Mon Sep 17 00:00:00 2001 From: Eric Li Date: Tue, 7 Dec 2010 17:35:47 -0800 Subject: [PATCH 30/31] Remove extra double quotes while passing args into autoserv. Change-Id: Iaee4c276b8f03546cc32b30849e424c84c729aef BUG=9841 TEST=run with sam's test script. cd chromeos/src/scripts echo chromelab > .default_wifi_test_lab sudo echo 172.22.18.5 chromelab >>/etc/hosts bin/cros_run_wifi_tests.sh --cell=4 MatFunc Review URL: http://codereview.chromium.org/5523005 --- run_remote_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_remote_tests.sh b/run_remote_tests.sh index e920bb61bc..2a54456f73 100755 --- a/run_remote_tests.sh +++ b/run_remote_tests.sh @@ -268,7 +268,7 @@ function main() { local autoserv_args="-m ${FLAGS_remote} --ssh-port ${FLAGS_ssh_port} \ ${option} ${control_file} -r ${results_dir} ${verbose}" if [ -n "${FLAGS_args}" ]; then - autoserv_args="${autoserv_args} -a \""${FLAGS_args}"\"" + autoserv_args="${autoserv_args} -a \"${FLAGS_args}\"" fi sudo chmod a+w ./server/{tests,site_tests} From b5ed3128ee43548d89c0ec55d1585f0f7c69e0ff Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Tue, 7 Dec 2010 17:40:34 -0800 Subject: [PATCH 31/31] Change config to use stable vs. sticky. Change-Id: I0dda2cfb6e11cb69a6a1a6a4bae478cab78b8947 BUG=chromium-os:8693 TEST=Ran it with cros_mark_chrome with stable_release TBR=Config change to get chrome builder up. --- bin/cros_mark_chrome_as_stable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cros_mark_chrome_as_stable.py b/bin/cros_mark_chrome_as_stable.py index 7545c641a7..356b97d6de 100755 --- a/bin/cros_mark_chrome_as_stable.py +++ b/bin/cros_mark_chrome_as_stable.py @@ -31,7 +31,7 @@ from cros_build_lib import RunCommand, Info, Warning BASE_CHROME_SVN_URL = 'http://src.chromium.org/svn' # Command for which chrome ebuild to uprev. -TIP_OF_TRUNK, LATEST_RELEASE, STICKY = 'tot', 'latest_release', 'sticky_release' +TIP_OF_TRUNK, LATEST_RELEASE, STICKY = 'tot', 'latest_release', 'stable_release' CHROME_REV = [TIP_OF_TRUNK, LATEST_RELEASE, STICKY] # Helper regex's for finding ebuilds.