From 4fc5227fac085dc36478af913db82c17c2c9ff69 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Wed, 8 Dec 2010 14:22:11 -0800 Subject: [PATCH 01/29] Factored out the code to copy an image and modify it for test into mod_image_for_test.sh rather than slightly different versions of the same in image_to_usb.sh and image_to_vm.sh Added a function to get a test image into common.sh Added --inplace option to mod_image_for_test, which is the default, and preserves the original behaviour. But using --noinplace it will now do the copy for you. Found that chromiumos_image.bin appears throughout the scripts, so added it and the test variant to common.sh BUG=chromiumos-10126 TEST=run mod_image_for_test.sh with and without --noinplace run image_to_usb.sh and image_to_vm.sh with both options test on Seaboard that correct image is provided Really we should have automated testing for these scripts Change-Id: I5cfa91792c7fded35e7f4ca8f8f27c6b270817fb Review URL: http://codereview.chromium.org/5271010 --- common.sh | 57 +++++++++++++++++++++++++++++++++++++ image_to_usb.sh | 66 ++++++++++++------------------------------- image_to_vm.sh | 42 ++++++--------------------- mod_image_for_test.sh | 27 +++++++++++++++++- 4 files changed, 110 insertions(+), 82 deletions(-) diff --git a/common.sh b/common.sh index a11f257289..6b55b707dc 100644 --- a/common.sh +++ b/common.sh @@ -14,6 +14,14 @@ # and dpkg-buildpackage NUM_JOBS=`grep -c "^processor" /proc/cpuinfo` +# True if we have the 'pv' utility - also set up COMMON_PV_CAT for convenience +COMMON_PV_OK=1 +COMMON_PV_CAT=pv +pv -V >/dev/null 2>&1 || COMMON_PV_OK=0 +if [ $COMMON_PV_OK -eq 0 ]; then + COMMON_PV_CAT=cat +fi + # Store location of the calling script. TOP_SCRIPT_DIR="${TOP_SCRIPT_DIR:-$(dirname $0)}" @@ -124,6 +132,12 @@ else INSIDE_CHROOT=0 fi + +# Standard filenames +CHROMEOS_IMAGE_NAME="chromiumos_image.bin" +CHROMEOS_TEST_IMAGE_NAME="chromiumos_test_image.bin" + + # Directory locations inside the dev chroot CHROOT_TRUNK_DIR="/home/$USER/trunk" @@ -527,3 +541,46 @@ chroot_hacks_from_outside() { sudo bash -c "echo root ALL=\(ALL\) ALL >> \"${chroot_dir}/etc/sudoers\"" fi } + +# This function converts a chromiumos image into a test image, either +# in place or by copying to a new test image filename first. It honors +# the following flags (see mod_image_for_test.sh) +# +# --factory +# --factory_install +# --force_copy +# +# On entry, pass the directory containing the image, and the image filename +# On exit, it puts the pathname of the resulting test image into +# CHROMEOS_RETURN_VAL +# (yes this is ugly, but perhaps less ugly than the alternatives) +# +# Usage: +# SRC_IMAGE=$(prepare_test_image "directory" "imagefile") +prepare_test_image() { + # If we're asked to modify the image for test, then let's make a copy and + # modify that instead. + # Check for manufacturing image. + local args + + if [ ${FLAGS_factory} -eq ${FLAGS_TRUE} ]; then + args="--factory" + fi + + # Check for install shim. + if [ ${FLAGS_factory_install} -eq ${FLAGS_TRUE} ]; then + args="--factory_install" + fi + + # Check for forcing copy of image + if [ ${FLAGS_force_copy} -eq ${FLAGS_TRUE} ]; then + args="${args} --force_copy" + fi + + # Modify the image for test, creating a new test image + "${SCRIPTS_DIR}/mod_image_for_test.sh" --board=${FLAGS_board} \ + --image="$1/$2" --noinplace ${args} + + # From now on we use the just-created test image + CHROMEOS_RETURN_VAL="$1/${CHROMEOS_TEST_IMAGE_NAME}" +} diff --git a/image_to_usb.sh b/image_to_usb.sh index 19eefb1f63..4b32ff8f88 100755 --- a/image_to_usb.sh +++ b/image_to_usb.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2009 The Chromium OS Authors. All rights reserved. +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -18,7 +18,7 @@ get_default_board # Flags DEFINE_string board "${DEFAULT_BOARD}" "Board for which the image was built" DEFINE_string from "" \ - "Directory containing chromiumos_image.bin" + "Directory containing ${CHROMEOS_IMAGE_NAME}" DEFINE_string to "/dev/sdX" "${DEFAULT_TO_HELP}" DEFINE_boolean yes ${FLAGS_FALSE} "Answer yes to all prompts" "y" DEFINE_boolean force_copy ${FLAGS_FALSE} "Always rebuild test image" @@ -31,8 +31,8 @@ DEFINE_boolean factory ${FLAGS_FALSE} \ DEFINE_boolean copy_kernel ${FLAGS_FALSE} \ "Copy the kernel to the fourth partition." DEFINE_boolean test_image "${FLAGS_FALSE}" \ - "Copies normal image to chromiumos_test_image.bin, modifies it for test." -DEFINE_string image_name "chromiumos_image.bin" \ + "Copies normal image to ${CHROMEOS_TEST_IMAGE_NAME}, modifies it for test." +DEFINE_string image_name "${CHROMEOS_IMAGE_NAME}" \ "Base name of the image" i DEFINE_string build_root "/build" \ "The root location for board sysroots." @@ -81,6 +81,8 @@ fi # We have a board name but no image set. Use image at default location if [ -z "${FLAGS_from}" ]; then IMAGES_DIR="${DEFAULT_BUILD_ROOT}/images/${FLAGS_board}" + + # Get latest image directory FLAGS_from="${IMAGES_DIR}/$(ls -t ${IMAGES_DIR} 2>&-| head -1)" fi @@ -131,9 +133,6 @@ if [ -b "${FLAGS_to}" ]; then fi fi -# Use this image as the source image to copy -SRC_IMAGE="${FLAGS_from}/${FLAGS_image_name}" - STATEFUL_DIR="${FLAGS_from}/stateful_partition" mkdir -p "${STATEFUL_DIR}" @@ -147,41 +146,18 @@ function do_cleanup { fi } - -# If we're asked to modify the image for test, then let's make a copy and -# modify that instead. if [ ${FLAGS_test_image} -eq ${FLAGS_TRUE} ] ; then - if [ ! -f "${FLAGS_from}/chromiumos_test_image.bin" ] || \ - [ ${FLAGS_force_copy} -eq ${FLAGS_TRUE} ] ; then - # Copy it. - echo "Creating test image from original..." - cp -f "${SRC_IMAGE}" "${FLAGS_from}/chromiumos_test_image.bin" + # Make a test image - this returns the test filename in CHROMEOS_RETURN_VAL + prepare_test_image "${FLAGS_from}" "${FLAGS_image_name}" + SRC_IMAGE="${CHROMEOS_RETURN_VAL}" - # Check for manufacturing image. - if [ ${FLAGS_factory} -eq ${FLAGS_TRUE} ] ; then - EXTRA_ARGS="--factory" - fi - - # Check for instqall shim. - if [ ${FLAGS_factory_install} -eq ${FLAGS_TRUE} ] ; then - EXTRA_ARGS="--factory_install" - fi - - # Modify it. Pass --yes so that mod_image_for_test.sh won't ask us if we - # really want to modify the image; the user gave their assent already with - # --test-image and the original image is going to be preserved. - "${SCRIPTS_DIR}/mod_image_for_test.sh" --image \ - "${FLAGS_from}/chromiumos_test_image.bin" --board=${FLAGS_board} \ - ${EXTRA_ARGS} --yes - echo "Done with mod_image_for_test." - else - echo "Using cached test image." - fi - SRC_IMAGE="${FLAGS_from}/chromiumos_test_image.bin" - echo "Source test image is: ${SRC_IMAGE}" + # No need to check with the user, as we are preserving the base image + FLAGS_yes="${FLAGS_TRUE}" +else + # Use the standard image + SRC_IMAGE="${FLAGS_from}/${FLAGS_image_name}" fi - # Let's do it. if [ -b "${FLAGS_to}" ] then @@ -235,15 +211,9 @@ then sleep 3 if [ ${FLAGS_install} -ne ${FLAGS_TRUE} ]; then - echo "Copying ${SRC_IMAGE} to ${FLAGS_to}..." - if type pv >/dev/null 2>&1; then - # pv displays file size in k=1024 while dd uses k=1000. - # To prevent confusion, we suppress the summary report from dd. - sudo pv -ptreb -B 4m "${SRC_IMAGE}" | - sudo dd of="${FLAGS_to}" bs=4M oflag=sync status=noxfer - else - sudo dd if="${SRC_IMAGE}" of="${FLAGS_to}" bs=4M oflag=sync - fi + echo "Copying with dd ${SRC_IMAGE} to ${FLAGS_to}..." + sudo ${COMMON_PV_CAT} "${SRC_IMAGE}" | + sudo dd of="${FLAGS_to}" bs=4M oflag=sync status=noxfer sync else if [ ${INSIDE_CHROOT} -ne 1 ]; then @@ -264,7 +234,7 @@ then else # Output to a file, so just make a copy. echo "Copying ${SRC_IMAGE} to ${FLAGS_to}..." - cp -f "${SRC_IMAGE}" "${FLAGS_to}" + ${COMMON_PV_CAT} "${SRC_IMAGE}" >"${FLAGS_to}" echo "Done. To copy to a USB drive, do something like:" echo " sudo dd if=${FLAGS_to} of=/dev/sdX bs=4M oflag=sync" diff --git a/image_to_vm.sh b/image_to_vm.sh index 2ff4984607..12f421a95e 100755 --- a/image_to_vm.sh +++ b/image_to_vm.sh @@ -10,6 +10,8 @@ # Load common constants. This should be the first executable line. # The path to common.sh should be relative to your script's location. . "$(dirname "$0")/common.sh" + +# Load functions and constants for chromeos-install . "$(dirname "$0")/chromeos-common.sh" . "$(dirname "$0")/lib/cros_vm_constants.sh" @@ -40,7 +42,7 @@ DEFINE_string state_image "" \ DEFINE_integer statefulfs_size 2048 \ "Stateful partition size in MBs." DEFINE_boolean test_image "${FLAGS_FALSE}" \ - "Copies normal image to chromiumos_test_image.bin, modifies it for test." + "Copies normal image to ${CHROMEOS_TEST_IMAGE_NAME}, modifies it for test." DEFINE_string to "" \ "Destination folder for VM output file(s)" DEFINE_string vbox_disk "${DEFAULT_VBOX_DISK}" \ @@ -84,39 +86,13 @@ if [ -z "${FLAGS_to}" ] ; then FLAGS_to="${FLAGS_from}" fi -# Use this image as the source image to copy -SRC_IMAGE="${FLAGS_from}/chromiumos_image.bin" - -# If we're asked to modify the image for test, then let's make a copy and -# modify that instead. if [ ${FLAGS_test_image} -eq ${FLAGS_TRUE} ] ; then - if [ ! -f "${FLAGS_from}/chromiumos_test_image.bin" ] || \ - [ ${FLAGS_force_copy} -eq ${FLAGS_TRUE} ] ; then - # Copy it. - echo "Creating test image from original..." - cp -f "${SRC_IMAGE}" "${FLAGS_from}/chromiumos_test_image.bin" - - # Check for manufacturing image. - if [ ${FLAGS_factory} -eq ${FLAGS_TRUE} ] ; then - EXTRA_ARGS="--factory" - fi - - # Check for install shim. - if [ ${FLAGS_factory_install} -eq ${FLAGS_TRUE} ] ; then - EXTRA_ARGS="--factory_install" - fi - - # Modify it. Pass --yes so that mod_image_for_test.sh won't ask us if we - # really want to modify the image; the user gave their assent already with - # --test-image and the original image is going to be preserved. - "${SCRIPTS_DIR}/mod_image_for_test.sh" --board=${FLAGS_board} --image \ - "${FLAGS_from}/chromiumos_test_image.bin" ${EXTRA_ARGS} --yes - echo "Done with mod_image_for_test." - else - echo "Using cached test image." - fi - SRC_IMAGE="${FLAGS_from}/chromiumos_test_image.bin" - echo "Source test image is: ${SRC_IMAGE}" + # Make a test image - this returns the test filename in CHROMEOS_RETURN_VAL + prepare_test_image "${FLAGS_from}" "${CHROMEOS_IMAGE_NAME}" + SRC_IMAGE="${CHROMEOS_RETURN_VAL}" +else + # Use the standard image + SRC_IMAGE="${FLAGS_from}/${CHROMEOS_IMAGE_NAME}" fi # Memory units are in MBs diff --git a/mod_image_for_test.sh b/mod_image_for_test.sh index 97aa5e0cc7..52bd99d3a7 100755 --- a/mod_image_for_test.sh +++ b/mod_image_for_test.sh @@ -33,6 +33,12 @@ DEFINE_boolean yes $FLAGS_FALSE "Answer yes to all prompts" y DEFINE_string build_root "/build" \ "The root location for board sysroots." DEFINE_boolean fast ${DEFAULT_FAST} "Call many emerges in parallel" +DEFINE_boolean inplace $FLAGS_TRUE \ + "Modify/overwrite the image ${CHROMEOS_IMAGE_NAME} in place. \ +Otherwise the image will be copied to ${CHROMEOS_TEST_IMAGE_NAME} \ +if needed, and modified there" +DEFINE_boolean force_copy ${FLAGS_FALSE} \ + "Always rebuild test image if --noinplace" # Parse command line @@ -56,7 +62,7 @@ fi # We have a board name but no image set. Use image at default location if [ -z $FLAGS_image ] ; then IMAGES_DIR="${DEFAULT_BUILD_ROOT}/images/${FLAGS_board}" - FILENAME="chromiumos_image.bin" + FILENAME="${CHROMEOS_IMAGE_NAME}" FLAGS_image="${IMAGES_DIR}/$(ls -t $IMAGES_DIR 2>&-| head -1)/${FILENAME}" fi @@ -155,6 +161,25 @@ install_autotest() { # main process begins here. +IMAGE_DIR="$(dirname "${FLAGS_image}")" + +# Copy the image to a test location if required +if [ ${FLAGS_inplace} -eq ${FLAGS_FALSE} ]; then + TEST_PATHNAME="${IMAGE_DIR}/${CHROMEOS_TEST_IMAGE_NAME}" + if [ ! -f "${TEST_PATHNAME}" ] || \ + [ ${FLAGS_force_copy} -eq ${FLAGS_TRUE} ] ; then + echo "Creating test image from original..." + ${COMMON_PV_CAT} "${FLAGS_image}" >"${TEST_PATHNAME}" \ + || die "Cannot copy ${FLAGS_image} to test image" + FLAGS_image="${TEST_PATHNAME}" + else + echo "Using cached test image" + fi + + # No need to confirm now, since we are not overwriting the main image + FLAGS_yes="$FLAGS_TRUE" +fi + # Make sure this is really what the user wants, before nuking the device if [ $FLAGS_yes -ne $FLAGS_TRUE ]; then read -p "Modifying image ${FLAGS_image} for test; are you sure (y/N)? " SURE From 84e72fab23c1359785f93685df6a3a45e7612d6e Mon Sep 17 00:00:00 2001 From: Anush Elangovan Date: Wed, 8 Dec 2010 23:22:15 -0800 Subject: [PATCH 02/29] Revert "Factored out the code to copy an image and modify it for test" Reverting for now since preflight fails. This reverts commit 4fc5227fac085dc36478af913db82c17c2c9ff69. TBR=sjg Change-Id: I0ada89c15d0f5ecc07cb15b22d4084a37fa4ddb8 --- common.sh | 57 ------------------------------------ image_to_usb.sh | 68 +++++++++++++++++++++++++++++++------------ image_to_vm.sh | 42 ++++++++++++++++++++------ mod_image_for_test.sh | 27 +---------------- 4 files changed, 83 insertions(+), 111 deletions(-) diff --git a/common.sh b/common.sh index 6b55b707dc..a11f257289 100644 --- a/common.sh +++ b/common.sh @@ -14,14 +14,6 @@ # and dpkg-buildpackage NUM_JOBS=`grep -c "^processor" /proc/cpuinfo` -# True if we have the 'pv' utility - also set up COMMON_PV_CAT for convenience -COMMON_PV_OK=1 -COMMON_PV_CAT=pv -pv -V >/dev/null 2>&1 || COMMON_PV_OK=0 -if [ $COMMON_PV_OK -eq 0 ]; then - COMMON_PV_CAT=cat -fi - # Store location of the calling script. TOP_SCRIPT_DIR="${TOP_SCRIPT_DIR:-$(dirname $0)}" @@ -132,12 +124,6 @@ else INSIDE_CHROOT=0 fi - -# Standard filenames -CHROMEOS_IMAGE_NAME="chromiumos_image.bin" -CHROMEOS_TEST_IMAGE_NAME="chromiumos_test_image.bin" - - # Directory locations inside the dev chroot CHROOT_TRUNK_DIR="/home/$USER/trunk" @@ -541,46 +527,3 @@ chroot_hacks_from_outside() { sudo bash -c "echo root ALL=\(ALL\) ALL >> \"${chroot_dir}/etc/sudoers\"" fi } - -# This function converts a chromiumos image into a test image, either -# in place or by copying to a new test image filename first. It honors -# the following flags (see mod_image_for_test.sh) -# -# --factory -# --factory_install -# --force_copy -# -# On entry, pass the directory containing the image, and the image filename -# On exit, it puts the pathname of the resulting test image into -# CHROMEOS_RETURN_VAL -# (yes this is ugly, but perhaps less ugly than the alternatives) -# -# Usage: -# SRC_IMAGE=$(prepare_test_image "directory" "imagefile") -prepare_test_image() { - # If we're asked to modify the image for test, then let's make a copy and - # modify that instead. - # Check for manufacturing image. - local args - - if [ ${FLAGS_factory} -eq ${FLAGS_TRUE} ]; then - args="--factory" - fi - - # Check for install shim. - if [ ${FLAGS_factory_install} -eq ${FLAGS_TRUE} ]; then - args="--factory_install" - fi - - # Check for forcing copy of image - if [ ${FLAGS_force_copy} -eq ${FLAGS_TRUE} ]; then - args="${args} --force_copy" - fi - - # Modify the image for test, creating a new test image - "${SCRIPTS_DIR}/mod_image_for_test.sh" --board=${FLAGS_board} \ - --image="$1/$2" --noinplace ${args} - - # From now on we use the just-created test image - CHROMEOS_RETURN_VAL="$1/${CHROMEOS_TEST_IMAGE_NAME}" -} diff --git a/image_to_usb.sh b/image_to_usb.sh index 4b32ff8f88..19eefb1f63 100755 --- a/image_to_usb.sh +++ b/image_to_usb.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Copyright (c) 2009 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -18,7 +18,7 @@ get_default_board # Flags DEFINE_string board "${DEFAULT_BOARD}" "Board for which the image was built" DEFINE_string from "" \ - "Directory containing ${CHROMEOS_IMAGE_NAME}" + "Directory containing chromiumos_image.bin" DEFINE_string to "/dev/sdX" "${DEFAULT_TO_HELP}" DEFINE_boolean yes ${FLAGS_FALSE} "Answer yes to all prompts" "y" DEFINE_boolean force_copy ${FLAGS_FALSE} "Always rebuild test image" @@ -31,8 +31,8 @@ DEFINE_boolean factory ${FLAGS_FALSE} \ DEFINE_boolean copy_kernel ${FLAGS_FALSE} \ "Copy the kernel to the fourth partition." DEFINE_boolean test_image "${FLAGS_FALSE}" \ - "Copies normal image to ${CHROMEOS_TEST_IMAGE_NAME}, modifies it for test." -DEFINE_string image_name "${CHROMEOS_IMAGE_NAME}" \ + "Copies normal image to chromiumos_test_image.bin, modifies it for test." +DEFINE_string image_name "chromiumos_image.bin" \ "Base name of the image" i DEFINE_string build_root "/build" \ "The root location for board sysroots." @@ -81,8 +81,6 @@ fi # We have a board name but no image set. Use image at default location if [ -z "${FLAGS_from}" ]; then IMAGES_DIR="${DEFAULT_BUILD_ROOT}/images/${FLAGS_board}" - - # Get latest image directory FLAGS_from="${IMAGES_DIR}/$(ls -t ${IMAGES_DIR} 2>&-| head -1)" fi @@ -133,6 +131,9 @@ if [ -b "${FLAGS_to}" ]; then fi fi +# Use this image as the source image to copy +SRC_IMAGE="${FLAGS_from}/${FLAGS_image_name}" + STATEFUL_DIR="${FLAGS_from}/stateful_partition" mkdir -p "${STATEFUL_DIR}" @@ -146,18 +147,41 @@ function do_cleanup { fi } -if [ ${FLAGS_test_image} -eq ${FLAGS_TRUE} ] ; then - # Make a test image - this returns the test filename in CHROMEOS_RETURN_VAL - prepare_test_image "${FLAGS_from}" "${FLAGS_image_name}" - SRC_IMAGE="${CHROMEOS_RETURN_VAL}" - # No need to check with the user, as we are preserving the base image - FLAGS_yes="${FLAGS_TRUE}" -else - # Use the standard image - SRC_IMAGE="${FLAGS_from}/${FLAGS_image_name}" +# If we're asked to modify the image for test, then let's make a copy and +# modify that instead. +if [ ${FLAGS_test_image} -eq ${FLAGS_TRUE} ] ; then + if [ ! -f "${FLAGS_from}/chromiumos_test_image.bin" ] || \ + [ ${FLAGS_force_copy} -eq ${FLAGS_TRUE} ] ; then + # Copy it. + echo "Creating test image from original..." + cp -f "${SRC_IMAGE}" "${FLAGS_from}/chromiumos_test_image.bin" + + # Check for manufacturing image. + if [ ${FLAGS_factory} -eq ${FLAGS_TRUE} ] ; then + EXTRA_ARGS="--factory" + fi + + # Check for instqall shim. + if [ ${FLAGS_factory_install} -eq ${FLAGS_TRUE} ] ; then + EXTRA_ARGS="--factory_install" + fi + + # Modify it. Pass --yes so that mod_image_for_test.sh won't ask us if we + # really want to modify the image; the user gave their assent already with + # --test-image and the original image is going to be preserved. + "${SCRIPTS_DIR}/mod_image_for_test.sh" --image \ + "${FLAGS_from}/chromiumos_test_image.bin" --board=${FLAGS_board} \ + ${EXTRA_ARGS} --yes + echo "Done with mod_image_for_test." + else + echo "Using cached test image." + fi + SRC_IMAGE="${FLAGS_from}/chromiumos_test_image.bin" + echo "Source test image is: ${SRC_IMAGE}" fi + # Let's do it. if [ -b "${FLAGS_to}" ] then @@ -211,9 +235,15 @@ then sleep 3 if [ ${FLAGS_install} -ne ${FLAGS_TRUE} ]; then - echo "Copying with dd ${SRC_IMAGE} to ${FLAGS_to}..." - sudo ${COMMON_PV_CAT} "${SRC_IMAGE}" | - sudo dd of="${FLAGS_to}" bs=4M oflag=sync status=noxfer + echo "Copying ${SRC_IMAGE} to ${FLAGS_to}..." + if type pv >/dev/null 2>&1; then + # pv displays file size in k=1024 while dd uses k=1000. + # To prevent confusion, we suppress the summary report from dd. + sudo pv -ptreb -B 4m "${SRC_IMAGE}" | + sudo dd of="${FLAGS_to}" bs=4M oflag=sync status=noxfer + else + sudo dd if="${SRC_IMAGE}" of="${FLAGS_to}" bs=4M oflag=sync + fi sync else if [ ${INSIDE_CHROOT} -ne 1 ]; then @@ -234,7 +264,7 @@ then else # Output to a file, so just make a copy. echo "Copying ${SRC_IMAGE} to ${FLAGS_to}..." - ${COMMON_PV_CAT} "${SRC_IMAGE}" >"${FLAGS_to}" + cp -f "${SRC_IMAGE}" "${FLAGS_to}" echo "Done. To copy to a USB drive, do something like:" echo " sudo dd if=${FLAGS_to} of=/dev/sdX bs=4M oflag=sync" diff --git a/image_to_vm.sh b/image_to_vm.sh index 12f421a95e..2ff4984607 100755 --- a/image_to_vm.sh +++ b/image_to_vm.sh @@ -10,8 +10,6 @@ # Load common constants. This should be the first executable line. # The path to common.sh should be relative to your script's location. . "$(dirname "$0")/common.sh" - -# Load functions and constants for chromeos-install . "$(dirname "$0")/chromeos-common.sh" . "$(dirname "$0")/lib/cros_vm_constants.sh" @@ -42,7 +40,7 @@ DEFINE_string state_image "" \ DEFINE_integer statefulfs_size 2048 \ "Stateful partition size in MBs." DEFINE_boolean test_image "${FLAGS_FALSE}" \ - "Copies normal image to ${CHROMEOS_TEST_IMAGE_NAME}, modifies it for test." + "Copies normal image to chromiumos_test_image.bin, modifies it for test." DEFINE_string to "" \ "Destination folder for VM output file(s)" DEFINE_string vbox_disk "${DEFAULT_VBOX_DISK}" \ @@ -86,13 +84,39 @@ if [ -z "${FLAGS_to}" ] ; then FLAGS_to="${FLAGS_from}" fi +# Use this image as the source image to copy +SRC_IMAGE="${FLAGS_from}/chromiumos_image.bin" + +# If we're asked to modify the image for test, then let's make a copy and +# modify that instead. if [ ${FLAGS_test_image} -eq ${FLAGS_TRUE} ] ; then - # Make a test image - this returns the test filename in CHROMEOS_RETURN_VAL - prepare_test_image "${FLAGS_from}" "${CHROMEOS_IMAGE_NAME}" - SRC_IMAGE="${CHROMEOS_RETURN_VAL}" -else - # Use the standard image - SRC_IMAGE="${FLAGS_from}/${CHROMEOS_IMAGE_NAME}" + if [ ! -f "${FLAGS_from}/chromiumos_test_image.bin" ] || \ + [ ${FLAGS_force_copy} -eq ${FLAGS_TRUE} ] ; then + # Copy it. + echo "Creating test image from original..." + cp -f "${SRC_IMAGE}" "${FLAGS_from}/chromiumos_test_image.bin" + + # Check for manufacturing image. + if [ ${FLAGS_factory} -eq ${FLAGS_TRUE} ] ; then + EXTRA_ARGS="--factory" + fi + + # Check for install shim. + if [ ${FLAGS_factory_install} -eq ${FLAGS_TRUE} ] ; then + EXTRA_ARGS="--factory_install" + fi + + # Modify it. Pass --yes so that mod_image_for_test.sh won't ask us if we + # really want to modify the image; the user gave their assent already with + # --test-image and the original image is going to be preserved. + "${SCRIPTS_DIR}/mod_image_for_test.sh" --board=${FLAGS_board} --image \ + "${FLAGS_from}/chromiumos_test_image.bin" ${EXTRA_ARGS} --yes + echo "Done with mod_image_for_test." + else + echo "Using cached test image." + fi + SRC_IMAGE="${FLAGS_from}/chromiumos_test_image.bin" + echo "Source test image is: ${SRC_IMAGE}" fi # Memory units are in MBs diff --git a/mod_image_for_test.sh b/mod_image_for_test.sh index 52bd99d3a7..97aa5e0cc7 100755 --- a/mod_image_for_test.sh +++ b/mod_image_for_test.sh @@ -33,12 +33,6 @@ DEFINE_boolean yes $FLAGS_FALSE "Answer yes to all prompts" y DEFINE_string build_root "/build" \ "The root location for board sysroots." DEFINE_boolean fast ${DEFAULT_FAST} "Call many emerges in parallel" -DEFINE_boolean inplace $FLAGS_TRUE \ - "Modify/overwrite the image ${CHROMEOS_IMAGE_NAME} in place. \ -Otherwise the image will be copied to ${CHROMEOS_TEST_IMAGE_NAME} \ -if needed, and modified there" -DEFINE_boolean force_copy ${FLAGS_FALSE} \ - "Always rebuild test image if --noinplace" # Parse command line @@ -62,7 +56,7 @@ fi # We have a board name but no image set. Use image at default location if [ -z $FLAGS_image ] ; then IMAGES_DIR="${DEFAULT_BUILD_ROOT}/images/${FLAGS_board}" - FILENAME="${CHROMEOS_IMAGE_NAME}" + FILENAME="chromiumos_image.bin" FLAGS_image="${IMAGES_DIR}/$(ls -t $IMAGES_DIR 2>&-| head -1)/${FILENAME}" fi @@ -161,25 +155,6 @@ install_autotest() { # main process begins here. -IMAGE_DIR="$(dirname "${FLAGS_image}")" - -# Copy the image to a test location if required -if [ ${FLAGS_inplace} -eq ${FLAGS_FALSE} ]; then - TEST_PATHNAME="${IMAGE_DIR}/${CHROMEOS_TEST_IMAGE_NAME}" - if [ ! -f "${TEST_PATHNAME}" ] || \ - [ ${FLAGS_force_copy} -eq ${FLAGS_TRUE} ] ; then - echo "Creating test image from original..." - ${COMMON_PV_CAT} "${FLAGS_image}" >"${TEST_PATHNAME}" \ - || die "Cannot copy ${FLAGS_image} to test image" - FLAGS_image="${TEST_PATHNAME}" - else - echo "Using cached test image" - fi - - # No need to confirm now, since we are not overwriting the main image - FLAGS_yes="$FLAGS_TRUE" -fi - # Make sure this is really what the user wants, before nuking the device if [ $FLAGS_yes -ne $FLAGS_TRUE ]; then read -p "Modifying image ${FLAGS_image} for test; are you sure (y/N)? " SURE From 3146eaa74e9ec84fa649d15cd383d2bfb8c93078 Mon Sep 17 00:00:00 2001 From: David James Date: Thu, 9 Dec 2010 09:37:14 -0800 Subject: [PATCH 03/29] Fix typo where host prebuilts aren't getting updated. archive_build.sh has code to archive host prebuilts, but the code isn't working because of a typo in archive_build.sh. The typo: $FLAGS_board is misspelled as $FLAGS_BOARD. BUG=chromium-os:10148 TEST=Ran prebuilt.py with --sync-host option Change-Id: I50129b1fdfb931b1c076a064b970d7d943f556ac Review URL: http://codereview.chromium.org/5724001 --- archive_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive_build.sh b/archive_build.sh index 74d577ea84..fdce71f8ce 100755 --- a/archive_build.sh +++ b/archive_build.sh @@ -275,7 +275,7 @@ then prebuilt_cmd="$prebuilt_cmd -u gs://chromeos-prebuilt --git-sync -V master" prebuilt_cmd="$prebuilt_cmd -p ${GCLIENT_ROOT} -b ${FLAGS_board}" - if [ "${FLAGS_BOARD}" == "x86-generic" ] + if [ "${FLAGS_board}" == "x86-generic" ] then prebuilt_cmd="$prebuilt_cmd --sync-host" fi From 53ac954a36d1a62a6e819695f7a3b2788201c667 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Thu, 9 Dec 2010 11:29:31 -0800 Subject: [PATCH 04/29] Add some desired options to the au test harness. Adds verbose flag to print output while updates are running rather than on error. Adds test_prefix to specify a prefix of tests you want to run Flushes stderr more reliably for builders and debugging purposes Dies when stateful_updates fail Change-Id: I9e4289deec3688c52b92763eee9334a8868f948e BUG=chromium-os:6910 TEST=Ran with test_prefix and verbose on and off Review URL: http://codereview.chromium.org/5573007 --- bin/cros_au_test_harness.py | 176 +++++++++++++++++++++--------------- image_to_live.sh | 2 +- lib/cros_build_lib.py | 3 + 3 files changed, 105 insertions(+), 76 deletions(-) diff --git a/bin/cros_au_test_harness.py b/bin/cros_au_test_harness.py index 3ffdac53cc..4cb6653d17 100755 --- a/bin/cros_au_test_harness.py +++ b/bin/cros_au_test_harness.py @@ -42,6 +42,7 @@ class AUTest(object): """Abstract interface that defines an Auto Update test.""" source_image = '' use_delta_updates = False + verbose = False def setUp(self): unittest.TestCase.setUp(self) @@ -80,7 +81,7 @@ class AUTest(object): if self.use_delta_updates: try: self.source_image = src_image - self.UpdateImage(image) + self._UpdateImageReportError(image) except: Warning('Delta update failed, disabling delta updates and retrying.') self.use_delta_updates = False @@ -110,9 +111,9 @@ class AUTest(object): if re.search(re.escape(expected_msg), err.stdout, re.MULTILINE): return - Warning("Didn't find '%s' in:" % expected_msg) - Warning(err.stdout) - self.fail('We managed to update when failure was expected') + Warning("Didn't find '%s' in:" % expected_msg) + Warning(err.stdout) + self.fail('We managed to update when failure was expected') def PrepareBase(self, image_path): """Prepares target with base_image_path.""" @@ -169,7 +170,8 @@ class AUTest(object): percent that passed. """ Info('Output from VerifyImage():') - print output + print >> sys.stderr, output + sys.stderr.flush() percent_passed = self.ParseGenerateTestReportOutput(output) Info('Percent passed: %d vs. Percent required: %d' % ( percent_passed, percent_required_to_pass)) @@ -237,7 +239,7 @@ class AUTest(object): # Read from the URL and write to the local file urllib.urlretrieve(url, payload) - expected_msg='download_hash_data == update_check_response_hash failed' + expected_msg = 'download_hash_data == update_check_response_hash failed' self._AttemptUpdateWithPayloadExpectedFailure(payload, expected_msg) def testCorruptedUpdate(self): @@ -255,7 +257,7 @@ class AUTest(object): urllib.urlretrieve(url, payload) # This update is expected to fail... - expected_msg='zlib inflate() error:-3' + expected_msg = 'zlib inflate() error:-3' self._AttemptUpdateWithPayloadExpectedFailure(payload, expected_msg) class RealAUTest(unittest.TestCase, AUTest): @@ -271,33 +273,43 @@ class RealAUTest(unittest.TestCase, AUTest): def UpdateImage(self, image_path, stateful_change='old'): """Updates a remote image using image_to_live.sh.""" stateful_change_flag = self.GetStatefulChangeFlag(stateful_change) + cmd = ['%s/image_to_live.sh' % self.crosutils, + '--image=%s' % image_path, + '--remote=%s' % remote, + stateful_change_flag, + '--verify', + '--src_image=%s' % self.source_image + ] - (code, stdout, stderr) = RunCommandCaptureOutput([ - '%s/image_to_live.sh' % self.crosutils, - '--image=%s' % image_path, - '--remote=%s' % remote, - stateful_change_flag, - '--verify', - '--src_image=%s' % self.source_image - ]) - - if code != 0: - raise UpdateException(code, stdout) + if self.verbose: + try: + RunCommand(cmd) + except Exception, e: + raise UpdateException(1, e.message) + else: + (code, stdout, stderr) = RunCommandCaptureOutput(cmd) + if code != 0: + raise UpdateException(code, stdout) def UpdateUsingPayload(self, update_path, stateful_change='old'): """Updates a remote image using image_to_live.sh.""" stateful_change_flag = self.GetStatefulChangeFlag(stateful_change) + cmd = ['%s/image_to_live.sh' % self.crosutils, + '--payload=%s' % update_path, + '--remote=%s' % remote, + stateful_change_flag, + '--verify', + ] - (code, stdout, stderr) = RunCommandCaptureOutput([ - '%s/image_to_live.sh' % self.crosutils, - '--payload=%s' % update_path, - '--remote=%s' % remote, - stateful_change_flag, - '--verify', - ]) - - if code != 0: - raise UpdateException(code, stdout) + if self.verbose: + try: + RunCommand(cmd) + except Exception, e: + raise UpdateException(1, e.message) + else: + (code, stdout, stderr) = RunCommandCaptureOutput(cmd) + if code != 0: + raise UpdateException(code, stdout) def VerifyImage(self, percent_required_to_pass): """Verifies an image using run_remote_tests.sh with verification suite.""" @@ -360,20 +372,25 @@ class VirtualAUTest(unittest.TestCase, AUTest): if self.source_image == base_image_path: self.source_image = self.vm_image_path - (code, stdout, stderr) = RunCommandCaptureOutput([ - '%s/cros_run_vm_update' % self.crosutilsbin, - '--update_image_path=%s' % image_path, - '--vm_image_path=%s' % self.vm_image_path, - '--snapshot', - vm_graphics_flag, - '--persist', - '--kvm_pid=%s' % _KVM_PID_FILE, - stateful_change_flag, - '--src_image=%s' % self.source_image, - ]) - - if code != 0: - raise UpdateException(code, stdout) + cmd = ['%s/cros_run_vm_update' % self.crosutilsbin, + '--update_image_path=%s' % image_path, + '--vm_image_path=%s' % self.vm_image_path, + '--snapshot', + vm_graphics_flag, + '--persist', + '--kvm_pid=%s' % _KVM_PID_FILE, + stateful_change_flag, + '--src_image=%s' % self.source_image, + ] + if self.verbose: + try: + RunCommand(cmd) + except Exception, e: + raise UpdateException(1, e.message) + else: + (code, stdout, stderr) = RunCommandCaptureOutput(cmd) + if code != 0: + raise UpdateException(code, stdout) def UpdateUsingPayload(self, update_path, stateful_change='old'): """Updates a remote image using image_to_live.sh.""" @@ -381,20 +398,26 @@ class VirtualAUTest(unittest.TestCase, AUTest): if self.source_image == base_image_path: self.source_image = self.vm_image_path - (code, stdout, stderr) = RunCommandCaptureOutput([ - '%s/cros_run_vm_update' % self.crosutilsbin, - '--payload=%s' % update_path, - '--vm_image_path=%s' % self.vm_image_path, - '--snapshot', - vm_graphics_flag, - '--persist', - '--kvm_pid=%s' % _KVM_PID_FILE, - stateful_change_flag, - '--src_image=%s' % self.source_image, - ]) + cmd = ['%s/cros_run_vm_update' % self.crosutilsbin, + '--payload=%s' % update_path, + '--vm_image_path=%s' % self.vm_image_path, + '--snapshot', + vm_graphics_flag, + '--persist', + '--kvm_pid=%s' % _KVM_PID_FILE, + stateful_change_flag, + '--src_image=%s' % self.source_image, + ] - if code != 0: - raise UpdateException(code, stdout) + if self.verbose: + try: + RunCommand(cmd) + except Exception, e: + raise UpdateException(1, e.message) + else: + (code, stdout, stderr) = RunCommandCaptureOutput(cmd) + if code != 0: + raise UpdateException(code, stdout) def VerifyImage(self, percent_required_to_pass): """Runs vm smoke suite to verify image.""" @@ -421,26 +444,33 @@ if __name__ == '__main__': parser = optparse.OptionParser() parser.add_option('-b', '--base_image', help='path to the base image.') - parser.add_option('-t', '--target_image', - help='path to the target image.') parser.add_option('-r', '--board', help='board for the images.') - parser.add_option('-p', '--type', default='vm', - help='type of test to run: [vm, real]. Default: vm.') - parser.add_option('-m', '--remote', - help='Remote address for real test.') - parser.add_option('--no_graphics', action='store_true', - help='Disable graphics for the vm test.') parser.add_option('--no_delta', action='store_false', default=True, dest='delta', help='Disable using delta updates.') + parser.add_option('--no_graphics', action='store_true', + help='Disable graphics for the vm test.') + parser.add_option('-m', '--remote', + help='Remote address for real test.') parser.add_option('-q', '--quick_test', default=False, action='store_true', help='Use a basic test to verify image.') + parser.add_option('-t', '--target_image', + help='path to the target image.') + parser.add_option('--test_prefix', default='test', + help='Only runs tests with specific prefix i.e. ' + 'testFullUpdateWipeStateful.') + parser.add_option('-p', '--type', default='vm', + help='type of test to run: [vm, real]. Default: vm.') + parser.add_option('--verbose', default=False, action='store_true', + help='Print out rather than capture output as much as ' + 'possible.') # Set the usage to include flags. parser.set_usage(parser.format_help()) # Parse existing sys.argv so we can pass rest to unittest.main. (options, sys.argv) = parser.parse_args(sys.argv) + AUTest.verbose = options.verbose base_image_path = options.base_image target_image_path = options.target_image board = options.board @@ -465,19 +495,15 @@ if __name__ == '__main__': AUTest.use_delta_updates = options.delta # Only run the test harness we care about. - if options.type == 'vm': - suite = unittest.TestLoader().loadTestsFromTestCase(VirtualAUTest) - test_result = unittest.TextTestRunner(verbosity=2).run(suite) - elif options.type == 'real': - if not options.remote: - parser.error('Real tests require a remote test machine.') - else: - remote = options.remote + test_loader = unittest.TestLoader() + test_loader.testMethodPrefix = options.test_prefix - suite = unittest.TestLoader().loadTestsFromTestCase(RealAUTest) - test_result = unittest.TextTestRunner(verbosity=2).run(suite) - else: - parser.error('Could not parse harness type %s.' % options.type) + if options.type == 'vm': test_class = VirtualAUTest + elif options.type == 'real': test_class = RealAUTest + else: parser.error('Could not parse harness type %s.' % options.type) + + test_suite = test_loader.loadTestsFromTestCase(test_class) + test_result = unittest.TextTestRunner(verbosity=2).run(test_suite) if not test_result.wasSuccessful(): Die('Test harness was not successful') diff --git a/image_to_live.sh b/image_to_live.sh index f18badf667..3988470254 100755 --- a/image_to_live.sh +++ b/image_to_live.sh @@ -356,7 +356,7 @@ function main() { if [ ${FLAGS_update_stateful} -eq ${FLAGS_TRUE} ] && \ ! run_stateful_update; then - warn "Stateful update was not successful." + die "Stateful update was not successful." fi remote_reboot diff --git a/lib/cros_build_lib.py b/lib/cros_build_lib.py index b45733eeda..2e6876060c 100644 --- a/lib/cros_build_lib.py +++ b/lib/cros_build_lib.py @@ -177,6 +177,7 @@ def Die(message): """ print >> sys.stderr, ( Color(_STDOUT_IS_TTY).Color(Color.RED, '\nERROR: ' + message)) + sys.stderr.flush() sys.exit(1) @@ -188,6 +189,7 @@ def Warning(message): """ print >> sys.stderr, ( Color(_STDOUT_IS_TTY).Color(Color.YELLOW, '\nWARNING: ' + message)) + sys.stderr.flush() def Info(message): @@ -198,6 +200,7 @@ def Info(message): """ print >> sys.stderr, ( Color(_STDOUT_IS_TTY).Color(Color.BLUE, '\nINFO: ' + message)) + sys.stderr.flush() def FindRepoDir(path=None): From ab5c6917c6892b847a7559559dc2a6ae244bee52 Mon Sep 17 00:00:00 2001 From: David James Date: Thu, 9 Dec 2010 14:52:46 -0800 Subject: [PATCH 05/29] Update cbuildbot.py to use emptytree option instead of deleting board. This allows us to avoid deleting the board and still ensure our packages are in sync with the full buildbot. This allows us to detect conflicts between old and new packages. BUG=chromium-os:10127 TEST=Run cbuildbot.py with emptytree feature forced to on and verify that build_packages rebuilds all packages. Also run unit tests. Change-Id: I83e186937e28eef44285724697895a4583910b58 Review URL: http://codereview.chromium.org/5652007 --- bin/cbuildbot.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index 2e86ae4173..8b44fdf253 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -341,10 +341,14 @@ def _SetupBoard(buildroot, board='x86-generic'): cwd=cwd, enter_chroot=True) -def _Build(buildroot): +def _Build(buildroot, emptytree): """Wrapper around build_packages.""" cwd = os.path.join(buildroot, 'src', 'scripts') - RunCommand(['./build_packages'], cwd=cwd, enter_chroot=True) + cmd = ['./build_packages'] + if emptytree: + cmd.insert(0, 'EXTRA_BOARD_FLAGS=--emptytree') + + RunCommand(cmd, cwd=cwd, enter_chroot=True) def _BuildChrome(buildroot, board, chrome_atom_to_build): @@ -646,8 +650,7 @@ def main(): _IncrementalCheckout(buildroot) new_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST) - if old_binhost and old_binhost != new_binhost: - RunCommand(['sudo', 'rm', '-rf', boardpath]) + emptytree = (old_binhost and old_binhost != new_binhost) # Check that all overlays can be found. for path in rev_overlays: @@ -671,7 +674,7 @@ def main(): _EnableLocalAccount(buildroot) # Doesn't rebuild without acquiring more source. if options.sync: - _Build(buildroot) + _Build(buildroot, emptytree) if chrome_atom_to_build: _BuildChrome(buildroot, buildconfig['board'], chrome_atom_to_build) From 4a5d7d9858d407582917edcf7473f63a0bad5566 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Thu, 9 Dec 2010 15:46:09 -0800 Subject: [PATCH 06/29] Wait for child to exit after killing it. Use general purpose wait in case your connecting to a vm that you didn't start. Change-Id: Ie67e2d3235882d372d04e4a44d0781ddfc8d4645 BUG=chromium-os:10121 TEST=Ran it, caused some crashes. Still ran test after retries. Review URL: http://codereview.chromium.org/5757001 --- lib/cros_vm_lib.sh | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/cros_vm_lib.sh b/lib/cros_vm_lib.sh index d28539c14d..3048c3dd30 100644 --- a/lib/cros_vm_lib.sh +++ b/lib/cros_vm_lib.sh @@ -19,6 +19,25 @@ function get_pid() { sudo cat "${KVM_PID_FILE}" } +# General purpose blocking kill on a pid. +# This function sends a specified kill signal [0-9] to a pid and waits for it +# die up to a given timeout. It exponentially backs off it's timeout starting +# at 1 second. +# $1 the process id. +# $2 signal to send (-#). +# $3 max timeout in seconds. +# Returns 0 on success. +function blocking_kill() { + local timeout=1 + sudo kill -$2 $1 + while ps -p $1 > /dev/null && [ ${timeout} -le $3 ]; do + warn "Process still running, sleeping for ${timeout}" + sleep ${timeout} + timeout=$((timeout*2)) + done + ! ps -p ${1} > /dev/null +} + # TODO(rtc): These flags assume that we'll be using KVM on Lucid and won't work # on Hardy. # $1: Path to the virtual image to start. @@ -98,7 +117,7 @@ function stop_kvm() { local pid=$(get_pid) if [ -n "${pid}" ]; then echo "Killing ${pid}" >&2 - sudo kill ${pid} + blocking_kill ${pid} 1 16 || blocking_kill 9 1 sudo rm "${KVM_PID_FILE}" else echo "No kvm pid found to stop." >&2 From 86965c9baf5dcda6b6419f38fc2049fd2f48318f Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Thu, 9 Dec 2010 15:47:38 -0800 Subject: [PATCH 07/29] Save off --remote flag Change-Id: I5621ff8ab81408ac3f7daed970a065641495c4eb BUG=chromium-os:10179 TEST=none Review URL: http://codereview.chromium.org/5676002 --- bin/cros_au_test_harness.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/cros_au_test_harness.py b/bin/cros_au_test_harness.py index 4cb6653d17..2449587df8 100755 --- a/bin/cros_au_test_harness.py +++ b/bin/cros_au_test_harness.py @@ -502,6 +502,8 @@ if __name__ == '__main__': elif options.type == 'real': test_class = RealAUTest else: parser.error('Could not parse harness type %s.' % options.type) + remote = options.remote + test_suite = test_loader.loadTestsFromTestCase(test_class) test_result = unittest.TextTestRunner(verbosity=2).run(test_suite) From 8be02f96b6a1bf49eed4a9b5ad58e85fcff441ed Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Thu, 9 Dec 2010 15:54:11 -0800 Subject: [PATCH 08/29] Create a transparent proxy for test programs to use, with the ability to filter or manipulate content as it's passed through. Create two example tests using this utility that cause updates to close early to test resumes, and to add delays to make sure the update still completes correctly. What other tests should be created for this? BUG=chromium-os:8207 TEST=Run by hand Review URL: http://codereview.chromium.org/5632002 Change-Id: Iefb8c8e223fb2ba6bad2c551f7c4403a0bec6ecf --- bin/cros_au_test_harness.py | 145 ++++++++++++++++++++++++++++++++---- image_to_live.sh | 15 +++- lib/cros_test_proxy.py | 113 ++++++++++++++++++++++++++++ 3 files changed, 258 insertions(+), 15 deletions(-) create mode 100755 lib/cros_test_proxy.py diff --git a/bin/cros_au_test_harness.py b/bin/cros_au_test_harness.py index 2449587df8..0e484947ab 100755 --- a/bin/cros_au_test_harness.py +++ b/bin/cros_au_test_harness.py @@ -8,6 +8,8 @@ import optparse import os import re import sys +import thread +import time import unittest import urllib @@ -19,6 +21,8 @@ from cros_build_lib import RunCommand from cros_build_lib import RunCommandCaptureOutput from cros_build_lib import Warning +import cros_test_proxy + # VM Constants. _FULL_VDISK_SIZE = 6072 _FULL_STATEFULFS_SIZE = 3074 @@ -90,20 +94,21 @@ class AUTest(object): else: self._UpdateImageReportError(image) - def _UpdateImageReportError(self, image_path, stateful_change='old'): + def _UpdateImageReportError(self, image_path, stateful_change='old', + proxy_port=None): """Calls UpdateImage and reports any error to the console. Still throws the exception. """ try: - self.UpdateImage(image_path, stateful_change) + self.UpdateImage(image_path, stateful_change, proxy_port) except UpdateException as err: # If the update fails, print it out Warning(err.stdout) raise def _AttemptUpdateWithPayloadExpectedFailure(self, payload, expected_msg): - # This update is expected to fail... + """Attempt a payload update, expect it to fail with expected log""" try: self.UpdateUsingPayload(payload) except UpdateException as err: @@ -115,11 +120,32 @@ class AUTest(object): Warning(err.stdout) self.fail('We managed to update when failure was expected') + def _AttemptUpdateWithFilter(self, filter): + """Update through a proxy, with a specified filter, and expect success.""" + + self.PrepareBase(target_image_path) + + # The devserver runs at port 8080 by default. We assume that here, and + # start our proxy at 8081. We then tell our update tools to have the + # client connect to 8081 instead of 8080. + proxy_port = 8081 + proxy = cros_test_proxy.CrosTestProxy(port_in=proxy_port, + address_out='127.0.0.1', + port_out=8080, + filter=filter) + proxy.serve_forever_in_thread() + + # This update is expected to fail... + try: + self._UpdateImageReportError(target_image_path, proxy_port=proxy_port) + finally: + proxy.shutdown() + def PrepareBase(self, image_path): """Prepares target with base_image_path.""" pass - def UpdateImage(self, image_path, stateful_change='old'): + def UpdateImage(self, image_path, stateful_change='old', proxy_port=None): """Updates target with the image given by the image_path. Args: @@ -129,15 +155,22 @@ class AUTest(object): 'old': Don't modify stateful partition. Just update normally. 'clean': Uses clobber-state to wipe the stateful partition with the exception of code needed for ssh. + proxy_port: Port to have the client connect to. For use with + CrosTestProxy. """ pass - def UpdateUsingPayload(self, update_path, stateful_change='old'): + def UpdateUsingPayload(self, + update_path, + stateful_change='old', + proxy_port=None): """Updates target with the pre-generated update stored in update_path Args: update_path: Path to the image to update with. This directory should - contain both update.gz, and stateful.image.gz + contain both update.gz, and stateful.image.gz + proxy_port: Port to have the client connect to. For use with + CrosTestProxy. """ pass @@ -187,7 +220,7 @@ class AUTest(object): """ # Just make sure some tests pass on original image. Some old images # don't pass many tests. - self.PrepareBase(image_path=base_image_path) + self.PrepareBase(base_image_path) # TODO(sosa): move to 100% once we start testing using the autotest paired # with the dev channel. percent_passed = self.VerifyImage(10) @@ -210,7 +243,7 @@ class AUTest(object): """ # Just make sure some tests pass on original image. Some old images # don't pass many tests. - self.PrepareBase(image_path=base_image_path) + self.PrepareBase(base_image_path) # TODO(sosa): move to 100% once we start testing using the autotest paired # with the dev channel. percent_passed = self.VerifyImage(10) @@ -228,7 +261,7 @@ class AUTest(object): def testPartialUpdate(self): """Tests what happens if we attempt to update with a truncated payload.""" # Preload with the version we are trying to test. - self.PrepareBase(image_path=target_image_path) + self.PrepareBase(target_image_path) # Image can be updated at: # ~chrome-eng/chromeos/localmirror/autest-images @@ -245,7 +278,7 @@ class AUTest(object): def testCorruptedUpdate(self): """Tests what happens if we attempt to update with a corrupted payload.""" # Preload with the version we are trying to test. - self.PrepareBase(image_path=target_image_path) + self.PrepareBase(target_image_path) # Image can be updated at: # ~chrome-eng/chromeos/localmirror/autest-images @@ -260,6 +293,71 @@ class AUTest(object): expected_msg = 'zlib inflate() error:-3' self._AttemptUpdateWithPayloadExpectedFailure(payload, expected_msg) + def testInterruptedUpdate(self): + """Tests what happens if we interrupt payload delivery 3 times.""" + + class InterruptionFilter(cros_test_proxy.Filter): + """This filter causes the proxy to interrupt the download 3 times + + It does this by closing the first three connections to transfer + 2M total in the outbound connection after they transfer the + 2M. + """ + def __init__(self): + """Defines variable shared across all connections""" + self.close_count = 0 + + def setup(self): + """Called once at the start of each connection.""" + self.data_size = 0 + + def OutBound(self, data): + """Called once per packet for outgoing data. + + The first three connections transferring more than 2M + outbound will be closed. + """ + if self.close_count < 3: + if self.data_size > (2 * 1024 * 1024): + self.close_count += 1 + return None + + self.data_size += len(data) + return data + + self._AttemptUpdateWithFilter(InterruptionFilter()) + + def testDelayedUpdate(self): + """Tests what happens if some data is delayed during update delivery""" + + class DelayedFilter(cros_test_proxy.Filter): + """Causes intermittent delays in data transmission. + + It does this by inserting 3 20 second delays when transmitting + data after 2M has been sent. + """ + def setup(self): + """Called once at the start of each connection.""" + self.data_size = 0 + self.delay_count = 0 + + def OutBound(self, data): + """Called once per packet for outgoing data. + + The first three packets after we reach 2M transferred + are delayed by 20 seconds. + """ + if self.delay_count < 3: + if self.data_size > (2 * 1024 * 1024): + self.delay_count += 1 + time.sleep(20) + + self.data_size += len(data) + return data + + + self._AttemptUpdateWithFilter(DelayedFilter()) + class RealAUTest(unittest.TestCase, AUTest): """Test harness for updating real images.""" @@ -270,7 +368,7 @@ class RealAUTest(unittest.TestCase, AUTest): """Auto-update to base image to prepare for test.""" self._UpdateImageReportError(image_path) - def UpdateImage(self, image_path, stateful_change='old'): + def UpdateImage(self, image_path, stateful_change='old', proxy_port=None): """Updates a remote image using image_to_live.sh.""" stateful_change_flag = self.GetStatefulChangeFlag(stateful_change) cmd = ['%s/image_to_live.sh' % self.crosutils, @@ -281,6 +379,9 @@ class RealAUTest(unittest.TestCase, AUTest): '--src_image=%s' % self.source_image ] + if proxy_port: + cmd.append('--proxy_port=%s' % proxy_port) + if self.verbose: try: RunCommand(cmd) @@ -291,7 +392,10 @@ class RealAUTest(unittest.TestCase, AUTest): if code != 0: raise UpdateException(code, stdout) - def UpdateUsingPayload(self, update_path, stateful_change='old'): + def UpdateUsingPayload(self, + update_path, + stateful_change='old', + proxy_port=None): """Updates a remote image using image_to_live.sh.""" stateful_change_flag = self.GetStatefulChangeFlag(stateful_change) cmd = ['%s/image_to_live.sh' % self.crosutils, @@ -301,6 +405,9 @@ class RealAUTest(unittest.TestCase, AUTest): '--verify', ] + if proxy_port: + cmd.append('--proxy_port=%s' % proxy_port) + if self.verbose: try: RunCommand(cmd) @@ -366,7 +473,7 @@ class VirtualAUTest(unittest.TestCase, AUTest): self.assertTrue(os.path.exists(self.vm_image_path)) - def UpdateImage(self, image_path, stateful_change='old'): + def UpdateImage(self, image_path, stateful_change='old', proxy_port=None): """Updates VM image with image_path.""" stateful_change_flag = self.GetStatefulChangeFlag(stateful_change) if self.source_image == base_image_path: @@ -382,6 +489,10 @@ class VirtualAUTest(unittest.TestCase, AUTest): stateful_change_flag, '--src_image=%s' % self.source_image, ] + + if proxy_port: + cmd.append('--proxy_port=%s' % proxy_port) + if self.verbose: try: RunCommand(cmd) @@ -392,7 +503,10 @@ class VirtualAUTest(unittest.TestCase, AUTest): if code != 0: raise UpdateException(code, stdout) - def UpdateUsingPayload(self, update_path, stateful_change='old'): + def UpdateUsingPayload(self, + update_path, + stateful_change='old', + proxy_port=None): """Updates a remote image using image_to_live.sh.""" stateful_change_flag = self.GetStatefulChangeFlag(stateful_change) if self.source_image == base_image_path: @@ -409,6 +523,9 @@ class VirtualAUTest(unittest.TestCase, AUTest): '--src_image=%s' % self.source_image, ] + if proxy_port: + cmd.append('--proxy_port=%s' % proxy_port) + if self.verbose: try: RunCommand(cmd) diff --git a/image_to_live.sh b/image_to_live.sh index 3988470254..545e428b92 100755 --- a/image_to_live.sh +++ b/image_to_live.sh @@ -38,6 +38,8 @@ DEFINE_string image "" \ "Update with this image path that is in this source checkout." i DEFINE_string payload "" \ "Update with this update payload, ignoring specified images." +DEFINE_string proxy_port "" \ + "Have the client request from this proxy instead of devserver." DEFINE_string src_image "" \ "Create a delta update by passing in the image on the remote machine." DEFINE_boolean update_stateful ${FLAGS_TRUE} \ @@ -139,6 +141,11 @@ function start_dev_server { --payload $(reinterpret_path_for_chroot ${FLAGS_payload})" fi + if [ -n "${FLAGS_proxy_port}" ]; then + devserver_flags="${devserver_flags} \ + --proxy_port ${FLAGS_proxy_port}" + fi + [ ${FLAGS_for_vm} -eq ${FLAGS_TRUE} ] && \ devserver_flags="${devserver_flags} --for_vm" @@ -209,9 +216,15 @@ function get_update_args { function get_devserver_url { local devserver_url="" + local port=${FLAGS_devserver_port} + + if [[ -n ${FLAGS_proxy_port} ]]; then + port=${FLAGS_proxy_port} + fi + if [ ${FLAGS_ignore_hostname} -eq ${FLAGS_TRUE} ]; then if [ -z ${FLAGS_update_url} ]; then - devserver_url="http://$(get_hostname):${FLAGS_devserver_port}/update" + devserver_url="http://$(get_hostname):${port}/update" else devserver_url="${FLAGS_update_url}" fi diff --git a/lib/cros_test_proxy.py b/lib/cros_test_proxy.py new file mode 100755 index 0000000000..21709829c3 --- /dev/null +++ b/lib/cros_test_proxy.py @@ -0,0 +1,113 @@ +# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import select +import socket +import SocketServer +import threading + +class Filter(object): + """Base class for data filters. + + Pass subclass of this to CrosTestProxy which will perform whatever + connection manipulation you prefer. + """ + + def setup(self): + """This setup method is called once per connection.""" + pass + + def InBound(self, data): + """This method is called once per packet of incoming data. + + The value returned is what is sent through the proxy. If + None is returned, the connection will be closed. + """ + return data + + def OutBound(self, data): + """This method is called once per packet of outgoing data. + + The value returned is what is sent through the proxy. If + None is returned, the connection will be closed. + """ + return data + + +class CrosTestProxy(SocketServer.ThreadingMixIn, SocketServer.TCPServer): + """A transparent proxy for simulating network errors""" + + class _Handler(SocketServer.BaseRequestHandler): + """Proxy connection handler that passes data though a filter""" + + def setup(self): + """Setup is called once for each connection proxied.""" + self.server.filter.setup() + + def handle(self): + """Handles each incoming connection. + + Opens a new connection to the port we are proxing to, then + passes each packet along in both directions after passing + them through the filter object passed in. + """ + # Open outgoing socket + s_in = self.request + s_out = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s_out.connect((self.server.address_out, self.server.port_out)) + + while True: + rlist, wlist, xlist = select.select([s_in, s_out], [], []) + + if s_in in rlist: + data = s_in.recv(1024) + data = self.server.filter.InBound(data) + if not data: break + try: + # If there is any error sending data, close both connections. + s_out.sendall(data) + except socket.error: + break + + if s_out in rlist: + data = s_out.recv(1024) + data = self.server.filter.OutBound(data) + if not data: break + try: + # If there is any error sending data, close both connections. + s_in.sendall(data) + except socket.error: + break + + s_in.close() + s_out.close() + + def __init__(self, + filter, + port_in=8081, + address_out='127.0.0.1', port_out=8080): + """Configures the proxy object. + + Args: + filter: An instance of a subclass of Filter. + port_in: Port on which to listen for incoming connections. + address_out: Address to which outgoing connections will go. + address_port: Port to which outgoing connections will go. + """ + self.port_in = port_in + self.address_out = address_out + self.port_out = port_out + self.filter = filter + + SocketServer.TCPServer.__init__(self, + ('', port_in), + self._Handler) + + def serve_forever_in_thread(self): + """Helper method to start the server in a new background thread.""" + server_thread = threading.Thread(target=self.serve_forever) + server_thread.setDaemon(True) + server_thread.start() + + return server_thread From 5b2f49deeb8489d967ba4e9959e5418c496b40e7 Mon Sep 17 00:00:00 2001 From: Allen Martin Date: Thu, 9 Dec 2010 16:35:39 -0800 Subject: [PATCH 09/29] Support booting from non zero MMC device numbers On ARM platform read kernel from device passed in from u-boot instead of hardcoding it to 0 BUG=none TEST=emerged on seaboard, booted from SD card and emmc Change-Id: Ia4506ed9f85d94eb37a9ac57430e1490d106c403 Review URL: http://codereview.chromium.org/5612008 Patch from Allen Martin . --- build_kernel_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_kernel_image.sh b/build_kernel_image.sh index 98230e41e6..08cca4c726 100755 --- a/build_kernel_image.sh +++ b/build_kernel_image.sh @@ -220,7 +220,7 @@ elif [[ "${FLAGS_arch}" = "arm" ]]; then echo -n 'setenv bootargs ${bootargs} ' > "${kernel_script}" tr '\n' ' ' <"${FLAGS_working_dir}/boot.config" >> "${kernel_script}" echo >> "${kernel_script}" - printf 'read ${devtype} 0:${kernelpart} ${loadaddr} %x %x\n' \ + printf 'read ${devtype} ${devnum}:${kernelpart} ${loadaddr} %x %x\n' \ ${script_size} ${kernel_size} >> "${kernel_script}" echo 'bootm ${loadaddr}' >> ${kernel_script} mkimage -A arm -O linux -T script -C none -a 0 -e 0 \ From aeb1048742e7e07bf9cce64972d61c60d0403f7e Mon Sep 17 00:00:00 2001 From: Raja Aluri Date: Fri, 10 Dec 2010 13:03:09 -0800 Subject: [PATCH 10/29] Moving generating au-generator.zip file to build_image. Change-Id: Id4ed6af96e0b621f4ad36966fad45c887b991373 BUG=None TEST=Ran a fresh build_packages and build_image for x86-generic. Review URL: http://codereview.chromium.org/5689004 --- archive_build.sh | 1 + build_image | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/archive_build.sh b/archive_build.sh index fdce71f8ce..ceefc65cd8 100755 --- a/archive_build.sh +++ b/archive_build.sh @@ -234,6 +234,7 @@ echo "$LAST_CHANGE" > "${FLAGS_to}/LATEST" # Make sure files are readable chmod 644 "$ZIPFILE" "${FLAGS_to}/LATEST" chmod 755 "$OUTDIR" +cp -f "${FLAGS_from}/au-generator.zip" "${OUTDIR}/" function gsutil_archive() { diff --git a/build_image b/build_image index bd49af5054..722d704f96 100755 --- a/build_image +++ b/build_image @@ -702,6 +702,14 @@ create_base_image() { trap - EXIT } +generate_au_zip () { + local lgenerateauzip="${SCRIPTS_DIR}/generate_au_zip.py" + local largs="-o ${OUTPUT_DIR}" + ! -d "${OUTPUT_DIR}" && mkdir -p "${OUTPUT_DIR}" + info "Running ${lgenerateauzip} ${largs} for generating AU updater zip file" + $lgenerateauzip $largs +} + # Create the output directory. mkdir -p "${OUTPUT_DIR}" mkdir -p "${ROOT_FS_DIR}" @@ -782,6 +790,8 @@ rm -f "${ROOT_FS_IMG}" "${STATEFUL_FS_IMG}" "${OUTPUT_DIR}/vmlinuz.image" \ "${ESP_FS_IMG}" "${OEM_FS_IMG}" "${OUTPUT_DIR}/vmlinuz_hd.vblock" rmdir "${ROOT_FS_DIR}" "${STATEFUL_FS_DIR}" "${OEM_FS_DIR}" "${ESP_FS_DIR}" +# Generating AU generator zip file to run outside chroot +generate_au_zip || echo "Failed generating AU zip file - ignoring Error..." # Create a 'latest' link rm -f ${FLAGS_output_root}/${FLAGS_board}/latest ln -s $(basename ${OUTPUT_DIR}) ${FLAGS_output_root}/${FLAGS_board}/latest From 230756fc83931697a28aa94b51d9c58f53e1e35d Mon Sep 17 00:00:00 2001 From: Raja Aluri Date: Fri, 10 Dec 2010 15:16:17 -0800 Subject: [PATCH 11/29] Switching most of the logging to debug Change-Id: I812305b173050e4539ce0f77fa5193e214d7dc2d BUG=None TEST=Ran the au_generator.py script inside chroot Review URL: http://codereview.chromium.org/5686004 --- generate_au_zip.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/generate_au_zip.py b/generate_au_zip.py index f38e1e3aad..fa62ba3bd0 100755 --- a/generate_au_zip.py +++ b/generate_au_zip.py @@ -26,7 +26,7 @@ logging.basicConfig(level=logging.INFO, format=logging_format, def CreateTempDir(): """Creates a tempdir and returns the name of the tempdir.""" temp_dir = tempfile.mkdtemp(suffix='au', prefix='tmp') - logging.info('Using tempdir = %s', temp_dir) + logging.debug('Using tempdir = %s', temp_dir) return temp_dir @@ -66,7 +66,7 @@ def DepsToCopy(ldd_files, black_list): library_list: List of files that are dependencies """ for file_name in ldd_files: - logging.info('Running ldd on %s', file_name) + logging.debug('Running ldd on %s', file_name) cmd = ['/usr/bin/ldd', file_name] stdout_data = '' stderr_data = '' @@ -137,18 +137,18 @@ def CopyRequiredFiles(dest_files_root): logging.debug('Given files that need to be copied = %s' % '' .join(all_files)) all_files += DepsToCopy(ldd_files=ldd_files,black_list=black_list) for file_name in all_files: - logging.info('Copying file %s to %s', file_name, dest_files_root) + logging.debug('Copying file %s to %s', file_name, dest_files_root) shutil.copy2(file_name, dest_files_root) for source_dir, target_dir in recurse_dirs.iteritems(): - logging.info('Processing directory %s', source_dir) + logging.debug('Processing directory %s', source_dir) full_path = os.path.expanduser(source_dir) if not os.path.isdir(full_path): logging.error("Directory given for %s expanded to %s doens't exist.", source_dir, full_path) sys.exit(1) dest = os.path.join(dest_files_root, target_dir) - logging.info('Copying directory %s to %s.', full_path, target_dir) + logging.debug('Copying directory %s to %s.', full_path, target_dir) shutil.copytree(full_path, dest) def CleanUp(temp_dir): @@ -158,7 +158,7 @@ def CleanUp(temp_dir): """ if os.path.exists(temp_dir): shutil.rmtree(temp_dir, ignore_errors=True) - logging.info('Removed tempdir = %s', temp_dir) + logging.debug('Removed tempdir = %s', temp_dir) def GenerateZipFile(base_name, root_dir): """Returns true if able to generate zip file @@ -168,7 +168,7 @@ def GenerateZipFile(base_name, root_dir): Returns: True if successfully generates the zip file otherwise False """ - logging.info('Generating zip file %s with contents from %s', base_name, + logging.debug('Generating zip file %s with contents from %s', base_name, root_dir) current_dir = os.getcwd() os.chdir(root_dir) @@ -223,7 +223,7 @@ def CopyZipToFinalDestination(output_dir, zip_file_name): if not os.path.isdir(output_dir): logging.debug('Creating %s', output_dir) os.makedirs(output_dir) - logging.info('Copying %s to %s', zip_file_name, output_dir) + logging.debug('Copying %s to %s', zip_file_name, output_dir) shutil.copy2(zip_file_name, output_dir) return True @@ -255,6 +255,7 @@ def main(): zip_file_name = os.path.join(temp_dir, options.zip_name) GenerateZipFile(zip_file_name, dest_files_root) CopyZipToFinalDestination(options.output_dir, zip_file_name) + logging.info('Generated %s/%s' % (options.output_dir, options.zip_name)) if not options.keep_temp: CleanUp(temp_dir) From 8e6abb04c6e8b070860928afa67c9ad2376a225e Mon Sep 17 00:00:00 2001 From: Trevor Bourget Date: Mon, 13 Dec 2010 09:44:32 -0700 Subject: [PATCH 12/29] scripts: repair build_image nostatefuldev collision Change-Id: Ib9fcc078d8e0223cd1b1376db91d8e71ae8359a6 BUG=cp file to itself fails in case of nostatefuldev TEST=ran build_image with and without statefuldev Review URL: http://codereview.chromium.org/4959001 Patch from Trevor Bourget . --- build_image | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build_image b/build_image index 722d704f96..9e4df1229f 100755 --- a/build_image +++ b/build_image @@ -415,7 +415,8 @@ update_dev_packages() { fi # Install the bare necessary files so that the "emerge" command works - sudo cp ${root_dev_dir}/etc/make.globals ${ROOT_FS_DIR}/etc/ + [ ${FLAGS_statefuldev} -eq ${FLAGS_TRUE} ] && \ + sudo cp ${root_dev_dir}/etc/make.globals ${ROOT_FS_DIR}/etc/ sudo sed -i s,/usr/bin/wget,wget, ${ROOT_FS_DIR}/etc/make.globals sudo mkdir -p ${ROOT_FS_DIR}/etc/make.profile From c663f209f730646e1e05cb5ec980deeab7ff9fd2 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Mon, 13 Dec 2010 11:11:35 -0800 Subject: [PATCH 13/29] Add ability to store the manifest in the manifest directory. Change-Id: I1ac3ed12df00424ba5ba2d1c61fd6fe9716e13b9 BUG=chromium-os:9714 TEST=Ran new unittests as well as ran cbuildbot on local b/cbuild Review URL: http://codereview.chromium.org/5689003 --- bin/cbuildbot.py | 74 ++++++++++++++++++++++++++++++++++++--- bin/cbuildbot_unittest.py | 66 +++++++++++++++++++++++++++++++++- 2 files changed, 134 insertions(+), 6 deletions(-) diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index 8b44fdf253..c332a50aae 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -25,6 +25,8 @@ _DEFAULT_RETRIES = 3 _PACKAGE_FILE = '%(buildroot)s/src/scripts/cbuildbot_package.list' ARCHIVE_BASE = '/var/www/archive' ARCHIVE_COUNT = 10 +PUBLIC_OVERLAY = '%(buildroot)s/src/third_party/chromiumos-overlay' +PRIVATE_OVERLAY = '%(buildroot)s/src/private-overlays/chromeos-overlay' # Currently, both the full buildbot and the preflight buildbot store their # data in a variable named PORTAGE_BINHOST, but they're in different files. @@ -35,6 +37,14 @@ _PREFLIGHT_BINHOST = 'PORTAGE_BINHOST' # ======================== Utility functions ================================ +def _PrintFile(path): + """Prints out the contents of a file to stderr.""" + file_handle = open(path) + print >> sys.stderr, file_handle.read() + file_handle.close() + sys.stderr.flush() + + def MakeDir(path, parents=False): """Basic wrapper around os.mkdirs. @@ -76,9 +86,6 @@ def RepoSync(buildroot, retries=_DEFAULT_RETRIES): Warning('CBUILDBOT -- Retries exhausted') raise - # Output manifest - RunCommand(['repo', 'manifest', '-r', '-o', '-'], cwd=buildroot) - # =========================== Command Helpers ================================= def _GetAllGitRepos(buildroot, debug=False): @@ -278,6 +285,31 @@ def _WipeOldOutput(buildroot): RunCommand(['rm', '-rf', 'src/build/images'], cwd=buildroot) +def _GetChromeOSVersion(buildroot): + """Returns the tuple version of the Chrome OS version of the buildroot.""" + cwd = os.path.join(buildroot, 'src', 'scripts') + version_cmd = './chromeos_version.sh' + output = RunCommand(version_cmd, cwd=cwd, redirect_stdout=True, + redirect_stderr=True) + version_re = re.compile('\s+CHROMEOS_VERSION_STRING=' + '(\d+)\.(\d+)\.(\d+)\.(\w+)') + for line in output.splitlines(): + match = version_re.match(line) + if match: + return match.group(1), match.group(2), match.group(3), match.group(4) + + raise Exception('Chrome OS version not found.') + + +def _GetManifestPath(buildroot): + """Returns the relative path that a manifest should be saved into.""" + version_tuple = _GetChromeOSVersion(buildroot) + (major, minor, branch, patch) = version_tuple + relative_path = os.path.join('.'.join([major, minor]), + '%s.xml' % '.'.join(version_tuple)) + return relative_path + + # =========================== Main Commands =================================== @@ -306,6 +338,36 @@ def _IncrementalCheckout(buildroot, retries=_DEFAULT_RETRIES): RepoSync(buildroot, retries) +def _DumpManifest(buildroot, url): + """Stores the manifest in the public | private overlay depending on url.""" + public_overlay = PUBLIC_OVERLAY % {'buildroot': buildroot} + private_overlay = PRIVATE_OVERLAY % {'buildroot': buildroot} + if url.endswith('manifest-internal'): + overlay = PRIVATE_OVERLAY % {'buildroot': buildroot} + else: + overlay = PUBLIC_OVERLAY % {'buildroot': buildroot} + + # Generate paths for manifests. + relative_path = _GetManifestPath(buildroot) + manifest_path = os.path.join(overlay, 'manifests', relative_path) + symlink_path = os.path.join(overlay, 'manifests', 'LATEST') + if not os.path.isdir(os.path.dirname(manifest_path)): + os.makedirs(os.path.dirname(manifest_path)) + + # Dump the manifest and create a symlink to it. + RunCommand(['repo', 'manifest', '-r', '-o', manifest_path], cwd=buildroot) + if os.path.exists(symlink_path): + os.unlink(symlink_path) + + os.symlink(relative_path, symlink_path) + + # Add it to git and print it to stderr. + RunCommand(['git', 'add', os.path.join('manifests', relative_path)], + cwd=overlay) + RunCommand(['git', 'add', os.path.join('manifests', 'LATEST')], cwd=overlay) + _PrintFile(manifest_path) + + def _MakeChroot(buildroot): """Wrapper around make_chroot.""" cwd = os.path.join(buildroot, 'src', 'scripts') @@ -527,8 +589,8 @@ def _ResolveOverlays(buildroot, overlays): 'public': Just the public overlay. 'both': Both the public and private overlays. """ - public_overlay = '%s/src/third_party/chromiumos-overlay' % buildroot - private_overlay = '%s/src/private-overlays/chromeos-overlay' % buildroot + public_overlay = PUBLIC_OVERLAY % {'buildroot': buildroot} + private_overlay = PRIVATE_OVERLAY % {'buildroot': buildroot} if overlays == 'private': paths = [private_overlay] elif overlays == 'public': @@ -657,6 +719,8 @@ def main(): if not os.path.isdir(path): Die('Missing overlay: %s' % path) + _DumpManifest(buildroot, options.url) + if not os.path.isdir(chroot_path): _MakeChroot(buildroot) diff --git a/bin/cbuildbot_unittest.py b/bin/cbuildbot_unittest.py index b3a4009a90..edacbe55d9 100755 --- a/bin/cbuildbot_unittest.py +++ b/bin/cbuildbot_unittest.py @@ -11,6 +11,7 @@ import mox import os import posix import shutil +import tempfile import unittest # Fixes circular dependency error. @@ -18,6 +19,7 @@ import cbuildbot_comm import cbuildbot from cros_build_lib import ReinterpretPathForChroot + class CBuildBotTest(mox.MoxTestBase): def setUp(self): @@ -117,7 +119,7 @@ class CBuildBotTest(mox.MoxTestBase): buildroot = '/fake_dir' board = 'fake-board' test_results_dir = 'fake_results_dir' - gsutil_path='/fake/gsutil/path' + gsutil_path = '/fake/gsutil/path' archive_dir = 1234 acl = 'fake_acl' num_retries = 5 @@ -230,6 +232,68 @@ class CBuildBotTest(mox.MoxTestBase): binhosts) self.mox.VerifyAll() + def testGetChromeOSVersion(self): + """Tests that we can parse the correct chromeos version using method.""" + fake_data = '\n'.join(['ChromeOS version information:', + ' SOME_OTHER_DATA=blah', + ' CHROMEOS_VERSION_STRING=9.0.232.1', + ' CHROMEOS_VERSION_CODENAME=test-bot', + ]) + cbuildbot.RunCommand('./chromeos_version.sh', + cwd='src/scripts', + redirect_stderr=True, + redirect_stdout=True).AndReturn(fake_data) + self.mox.ReplayAll() + return_tuple = cbuildbot._GetChromeOSVersion('') + self.assertEquals('.'.join(return_tuple), '9.0.232.1') + self.mox.VerifyAll() + + def testGetManifestPath(self): + """Tests whether our logic to get the manifest path is correct.""" + self.mox.StubOutWithMock(cbuildbot, '_GetChromeOSVersion') + return_tuple = cbuildbot._GetChromeOSVersion('').AndReturn( + ('9', '0', '232', '1')) + self.mox.ReplayAll() + relative_path = cbuildbot._GetManifestPath('') + self.assertEquals(relative_path, '9.0/9.0.232.1.xml') + self.mox.VerifyAll() + + def _CommonManifestTest(self, url, overlay_no_buildroot): + """Common method for dump manifest tests.""" + self.mox.StubOutWithMock(cbuildbot, '_GetManifestPath') + self.mox.StubOutWithMock(shutil, 'copy') + self.mox.StubOutWithMock(os, 'symlink') + temp_root = tempfile.mkdtemp('_unittest') + + overlay = overlay_no_buildroot % {'buildroot': temp_root} + relative_path = 'fake/manifest/path.xml' + full_path = os.path.join(overlay, 'manifests', relative_path) + + cbuildbot._GetManifestPath(temp_root).AndReturn(relative_path) + cbuildbot.RunCommand(['repo', 'manifest', '-r', '-o', full_path], + cwd=temp_root) + os.symlink(relative_path, os.path.join(overlay, 'manifests', 'LATEST')) + cbuildbot.RunCommand(['git', 'add', 'manifests/' + relative_path], + cwd=overlay) + cbuildbot.RunCommand(['git', 'add', 'manifests/LATEST'], cwd=overlay) + shutil.copy(full_path, '/dev/stderr') + + self.mox.ReplayAll() + cbuildbot._DumpManifest(temp_root, url) + self.mox.VerifyAll() + + shutil.rmtree(temp_root) + + def testDumpManifestPublic(self): + """Tests whether we push the manifest to the public overlay correctly.""" + self._CommonManifestTest('http://some_url/manifest', + cbuildbot.PUBLIC_OVERLAY) + + def testDumpManifestPrivate(self): + """Tests whether we push the manifest to the private overlay correctly.""" + self._CommonManifestTest('http://some_url/manifest-internal', + cbuildbot.PRIVATE_OVERLAY) + if __name__ == '__main__': unittest.main() From d8a0a57c30d488cf0071967b6dfe6b35ba25d2af Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Mon, 13 Dec 2010 11:27:26 -0800 Subject: [PATCH 14/29] Add support to pushing unstable changes for the chrome pfq. This specifically gets the pfq one step closer to being completely ready to rev. We add --noprebuilts for the chrome pfq as its not ready to host prebuilts and instead we push unstable ebuilds. Anush noticed an Anush today where we are revving even when we don't change, and I've addressed this also in this CL. Specifically: 1) Sort in cbuildbot long options by long format 2) Add --noprebuilts so that the Chrome PFQ can use this for now. 3) Fixed a unittest that had broken in a previous CL for cros_mark_chrome_as_stable. 4) Add ability for chrome to only rev if something really has changed: -- This means, if the new ebuild either: -- doesn't have the exact same chrome_version -- or diff -Bu (actual diff) between last ebuild and new ebuild isn't the same (i.e. detecting a 9999 change or different CROS_SVN_COMMIT) Change-Id: I9b289c2168d6868299573f5d7c7a676380ee497e BUG=chromium-os:8693 TEST=Ran unittests (cbuildbot, cros_mark_*) and ran cbuildbot with new chrome pfq items. Review URL: http://codereview.chromium.org/5783001 --- bin/cbuildbot.py | 25 ++++++--- bin/cros_mark_chrome_as_stable.py | 64 +++++++++++++--------- bin/cros_mark_chrome_as_stable_unittest.py | 13 +++-- cros_mark_as_stable.py | 9 ++- 4 files changed, 68 insertions(+), 43 deletions(-) diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index c332a50aae..b5c4b8b151 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -416,7 +416,8 @@ def _Build(buildroot, emptytree): def _BuildChrome(buildroot, board, chrome_atom_to_build): """Wrapper for emerge call to build Chrome.""" cwd = os.path.join(buildroot, 'src', 'scripts') - RunCommand(['emerge-%s' % board, '=%s' % chrome_atom_to_build], + RunCommand(['ACCEPT_KEYWORDS="* ~*"', 'emerge-%s' % board, + '=%s' % chrome_atom_to_build], cwd=cwd, enter_chroot=True) @@ -641,6 +642,8 @@ def main(): # Parse options usage = "usage: %prog [options] cbuildbot_config" parser = optparse.OptionParser(usage=usage) + parser.add_option('-a', '--acl', default='private', + help='ACL to set on GSD archives') parser.add_option('-r', '--buildroot', help='root directory where build occurs', default=".") parser.add_option('-n', '--buildnumber', @@ -649,30 +652,30 @@ def main(): dest='chrome_rev', help=('Chrome_rev of type [tot|latest_release|' 'sticky_release]')) - parser.add_option('-f', '--revisionfile', - help='file where new revisions are stored') + parser.add_option('-g', '--gsutil', default='', help='Location of gsutil') + parser.add_option('-c', '--gsutil_archive', default='', + help='Datastore archive location') parser.add_option('--clobber', action='store_true', dest='clobber', default=False, help='Clobbers an old checkout before syncing') parser.add_option('--debug', action='store_true', dest='debug', default=False, help='Override some options to run as a developer.') + parser.add_option('--noprebuilts', action='store_false', dest='prebuilts', + help="Don't upload prebuilts.") parser.add_option('--nosync', action='store_false', dest='sync', default=True, help="Don't sync before building.") parser.add_option('--notests', action='store_false', dest='tests', default=True, help='Override values from buildconfig and run no tests.') + parser.add_option('-f', '--revisionfile', + help='file where new revisions are stored') parser.add_option('-t', '--tracking-branch', dest='tracking_branch', default='cros/master', help='Run the buildbot on a branch') parser.add_option('-u', '--url', dest='url', default='http://git.chromium.org/git/manifest', help='Run the buildbot on internal manifest') - parser.add_option('-g', '--gsutil', default='', help='Location of gsutil') - parser.add_option('-c', '--gsutil_archive', default='', - help='Datastore archive location') - parser.add_option('-a', '--acl', default='private', - help='ACL to set on GSD archives') (options, args) = parser.parse_args() @@ -731,6 +734,10 @@ def main(): if options.chrome_rev: chrome_atom_to_build = _MarkChromeAsStable(buildroot, tracking_branch, options.chrome_rev) + # If we found nothing to rev, we're done here. + if not chrome_atom_to_build: + return + elif buildconfig['uprev']: _UprevPackages(buildroot, tracking_branch, revisionfile, buildconfig['board'], rev_overlays) @@ -768,7 +775,7 @@ def main(): if buildconfig['master']: # Master bot needs to check if the other slaves completed. if cbuildbot_comm.HaveSlavesCompleted(config): - if not options.debug: + if not options.debug and options.prebuilts: _UploadPrebuilts(buildroot, board, buildconfig['rev_overlays'], [new_binhost]) _UprevPush(buildroot, tracking_branch, buildconfig['board'], diff --git a/bin/cros_mark_chrome_as_stable.py b/bin/cros_mark_chrome_as_stable.py index 356b97d6de..23655500f9 100755 --- a/bin/cros_mark_chrome_as_stable.py +++ b/bin/cros_mark_chrome_as_stable.py @@ -109,8 +109,8 @@ def _GetLatestRelease(branch=None): return current_version.rstrip('/') -def _GetStickyVersion(stable_ebuilds): - """Discovers the sticky version from the current stable_ebuilds.""" +def _GetStickyEBuild(stable_ebuilds): + """Returns the sticky ebuild.""" sticky_ebuilds = [] non_sticky_re = re.compile(_NON_STICKY_REGEX) for ebuild in stable_ebuilds: @@ -122,7 +122,7 @@ def _GetStickyVersion(stable_ebuilds): elif len(sticky_ebuilds) > 1: Warning('More than one sticky ebuild found') - return cros_mark_as_stable.BestEBuild(sticky_ebuilds).chrome_version + return cros_mark_as_stable.BestEBuild(sticky_ebuilds) class ChromeEBuild(cros_mark_as_stable.EBuild): @@ -203,7 +203,7 @@ def FindChromeUprevCandidate(stable_ebuilds, chrome_rev, sticky_branch): candidates.append(ebuild) elif chrome_rev == STICKY: - chrome_branch_re = re.compile('%s\.\d+.*_rc.*' % sticky_branch) + chrome_branch_re = re.compile('%s\..*' % sticky_branch) for ebuild in stable_ebuilds: if chrome_branch_re.search(ebuild.version): candidates.append(ebuild) @@ -222,7 +222,8 @@ def FindChromeUprevCandidate(stable_ebuilds, chrome_rev, sticky_branch): def MarkChromeEBuildAsStable(stable_candidate, unstable_ebuild, chrome_rev, - chrome_version, commit, overlay_dir): + chrome_version, commit, overlay_dir, + sticky_ebuild): """Uprevs the chrome ebuild specified by chrome_rev. This is the main function that uprevs the chrome_rev from a stable candidate @@ -242,6 +243,7 @@ def MarkChromeEBuildAsStable(stable_candidate, unstable_ebuild, chrome_rev, chrome_version: The \d.\d.\d.\d version of Chrome. commit: Used with TIP_OF_TRUNK. The svn revision of chrome. overlay_dir: Path to the chromeos-chrome package dir. + sticky_ebuild: EBuild class for the sticky ebuild. Returns: Full portage version atom (including rc's, etc) that was revved. """ @@ -260,9 +262,21 @@ def MarkChromeEBuildAsStable(stable_candidate, unstable_ebuild, chrome_rev, new_ebuild_path = base_path + ('%s-r1.ebuild' % portage_suffix) cros_mark_as_stable.EBuildStableMarker.MarkAsStable( - unstable_ebuild.ebuild_path, new_ebuild_path, 'CROS_SVN_COMMIT', commit) + unstable_ebuild.ebuild_path, new_ebuild_path, 'CROS_SVN_COMMIT', commit, + make_stable=False) + new_ebuild = ChromeEBuild(new_ebuild_path) + if stable_candidate and ( + stable_candidate.chrome_version == new_ebuild.chrome_version): + if 0 == RunCommand(['diff', '-Bu', stable_candidate.ebuild_path, + new_ebuild_path], + redirect_stderr=True, + redirect_stdout=True, + exit_code=True): + os.unlink(new_ebuild_path) + return None + RunCommand(['git', 'add', new_ebuild_path]) - if stable_candidate: + if stable_candidate and stable_candidate != sticky_ebuild: RunCommand(['git', 'rm', stable_candidate.ebuild_path]) cros_mark_as_stable.EBuildStableMarker.CommitChange( @@ -293,10 +307,10 @@ def main(): commit_to_use = None (unstable_ebuild, stable_ebuilds) = FindChromeCandidates(overlay_dir) - sticky_version = _GetStickyVersion(stable_ebuilds) + sticky_ebuild = _GetStickyEBuild(stable_ebuilds) + sticky_version = sticky_ebuild.chrome_version sticky_branch = sticky_version.rpartition('.')[0] - if chrome_rev == TIP_OF_TRUNK: version_to_uprev = _GetTipOfTrunkVersion() commit_to_use = _GetTipOfTrunkSvnRevision() @@ -307,25 +321,23 @@ def main(): stable_candidate = FindChromeUprevCandidate(stable_ebuilds, chrome_rev, sticky_branch) - # There are some cases we don't need to do anything. Check for them. - if stable_candidate and (version_to_uprev == stable_candidate.chrome_version - and not commit_to_use): - Info('Found nothing to do for chrome_rev %s with version %s.' % ( - chrome_rev, version_to_uprev)) - else: - os.chdir(overlay_dir) - work_branch = cros_mark_as_stable.GitBranch( - cros_mark_as_stable.STABLE_BRANCH_NAME, options.tracking_branch) - work_branch.CreateBranch() - try: - chrome_version_atom = MarkChromeEBuildAsStable( - stable_candidate, unstable_ebuild, chrome_rev, version_to_uprev, - commit_to_use, overlay_dir) - # Explicit print to communicate to caller. + + os.chdir(overlay_dir) + work_branch = cros_mark_as_stable.GitBranch( + cros_mark_as_stable.STABLE_BRANCH_NAME, options.tracking_branch) + work_branch.CreateBranch() + try: + chrome_version_atom = MarkChromeEBuildAsStable( + stable_candidate, unstable_ebuild, chrome_rev, version_to_uprev, + commit_to_use, overlay_dir, sticky_ebuild) + # Explicit print to communicate to caller. + if chrome_version_atom: print 'CHROME_VERSION_ATOM=%s' % chrome_version_atom - except: + else: work_branch.Delete() - raise + except: + work_branch.Delete() + raise if __name__ == '__main__': diff --git a/bin/cros_mark_chrome_as_stable_unittest.py b/bin/cros_mark_chrome_as_stable_unittest.py index ab16cd68ea..dfc981de8b 100755 --- a/bin/cros_mark_chrome_as_stable_unittest.py +++ b/bin/cros_mark_chrome_as_stable_unittest.py @@ -200,12 +200,12 @@ class CrosMarkChromeAsStable(mox.MoxTestBase): self.mox.VerifyAll() self.assertEqual('8.0.224.2', release) - def testStickyVersion(self): - """Tests if we can find the sticky version from our mock directories.""" + def testStickyEBuild(self): + """Tests if we can find the sticky ebuild from our mock directories.""" stable_ebuilds = self._GetStableEBuilds() - sticky_version = cros_mark_chrome_as_stable._GetStickyVersion( + sticky_ebuild = cros_mark_chrome_as_stable._GetStickyEBuild( stable_ebuilds) - self.assertEqual(sticky_version, self.sticky_version) + self.assertEqual(sticky_ebuild.chrome_version, self.sticky_version) def testChromeEBuildInit(self): """Tests if the chrome_version is set correctly in a ChromeEBuild.""" @@ -231,6 +231,7 @@ class CrosMarkChromeAsStable(mox.MoxTestBase): 'CommitChange') stable_candidate = cros_mark_chrome_as_stable.ChromeEBuild(old_ebuild_path) unstable_ebuild = cros_mark_chrome_as_stable.ChromeEBuild(self.unstable) + sticky_ebuild = cros_mark_chrome_as_stable.ChromeEBuild(self.sticky) chrome_version = new_version commit = None overlay_dir = self.mock_chrome_dir @@ -243,14 +244,14 @@ class CrosMarkChromeAsStable(mox.MoxTestBase): self.mox.ReplayAll() cros_mark_chrome_as_stable.MarkChromeEBuildAsStable( stable_candidate, unstable_ebuild, chrome_rev, chrome_version, commit, - overlay_dir) + overlay_dir, sticky_ebuild) self.mox.VerifyAll() def testStickyMarkAsStable(self): """Tests to see if we can mark chrome as stable for a new sticky release.""" self._CommonMarkAsStableTest(cros_mark_chrome_as_stable.STICKY, self.sticky_new_rc_version, self.sticky_rc, - self.sticky_new_rc, 'sticky_release') + self.sticky_new_rc, 'stable_release') def testLatestMarkAsStable(self): """Tests to see if we can mark chrome for a latest release.""" diff --git a/cros_mark_as_stable.py b/cros_mark_as_stable.py index 7e170d2952..9a87062635 100755 --- a/cros_mark_as_stable.py +++ b/cros_mark_as_stable.py @@ -396,7 +396,8 @@ class EBuildStableMarker(object): @classmethod def MarkAsStable(cls, unstable_ebuild_path, new_stable_ebuild_path, - commit_keyword, commit_value, redirect_file=None): + commit_keyword, commit_value, redirect_file=None, + make_stable=True): """Static function that creates a revved stable ebuild. This function assumes you have already figured out the name of the new @@ -412,6 +413,7 @@ class EBuildStableMarker(object): stable. commit_value: Value to set the above keyword to. redirect_file: Optionally redirect output of new ebuild somewhere else. + make_stable: Actually make the ebuild stable. """ shutil.copyfile(unstable_ebuild_path, new_stable_ebuild_path) for line in fileinput.input(new_stable_ebuild_path, inplace=1): @@ -420,7 +422,10 @@ class EBuildStableMarker(object): redirect_file = sys.stdout if line.startswith('KEYWORDS'): # Actually mark this file as stable by removing ~'s. - redirect_file.write(line.replace('~', '')) + if make_stable: + redirect_file.write(line.replace('~', '')) + else: + redirect_file.write(line) elif line.startswith('EAPI'): # Always add new commit_id after EAPI definition. redirect_file.write(line) From 171321f8ff88e1edeec50820c42d6bec40992c28 Mon Sep 17 00:00:00 2001 From: Thieu Le Date: Mon, 13 Dec 2010 11:37:42 -0800 Subject: [PATCH 15/29] Zero free space on rootfs so it's more compressible for auto-update BUG=chromium-os:10228 TEST=Build image, install image to Mario, make sure image still boots Change-Id: Idc3c2a929515d39cb06926611eff37960074889c Review URL: http://codereview.chromium.org/5704004 --- build_image | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/build_image b/build_image index 9e4df1229f..6dcb05ebd5 100755 --- a/build_image +++ b/build_image @@ -504,6 +504,14 @@ update_base_packages() { -s "${STATEFUL_FS_DIR}" -e "${ESP_FS_DIR}" } +zero_free_space() { + local fs_mount_point=$1 + info "Zeroing freespace in ${fs_mount_point}" + sudo dd if=/dev/zero of="${fs_mount_point}/filler" oflag=sync bs=4096 || true + sudo rm -f "${fs_mount_point}/filler" + sudo sync +} + create_base_image() { local image_name=$1 @@ -684,6 +692,10 @@ create_base_image() { # Create an empty esp image to be updated in by update_bootloaders.sh. ${SCRIPTS_DIR}/create_esp.sh --to="${ESP_FS_IMG}" + # Zero rootfs free space to make it more compressible so auto-update + # payloads become smaller + zero_free_space "${ROOT_FS_DIR}" + cleanup trap delete_prompt EXIT @@ -706,7 +718,7 @@ create_base_image() { generate_au_zip () { local lgenerateauzip="${SCRIPTS_DIR}/generate_au_zip.py" local largs="-o ${OUTPUT_DIR}" - ! -d "${OUTPUT_DIR}" && mkdir -p "${OUTPUT_DIR}" + ! -d "${OUTPUT_DIR}" && mkdir -p "${OUTPUT_DIR}" info "Running ${lgenerateauzip} ${largs} for generating AU updater zip file" $lgenerateauzip $largs } From 97432e7e9e9a1f5aeaf5ea86bf4084379b2531c6 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Mon, 13 Dec 2010 16:20:51 -0800 Subject: [PATCH 16/29] Don't push manifests for chrome pfq. Change-Id: I252616725936b2c471026d39d06c3fbb2fa1ed65 BUG=chromium-os:8693 TEST=Ran with chrome_rev=tot Review URL: http://codereview.chromium.org/5744003 --- bin/cbuildbot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index b5c4b8b151..076487fb20 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -722,7 +722,8 @@ def main(): if not os.path.isdir(path): Die('Missing overlay: %s' % path) - _DumpManifest(buildroot, options.url) + if not options.chrome_rev: + _DumpManifest(buildroot, options.url) if not os.path.isdir(chroot_path): _MakeChroot(buildroot) From 888e07c9c872c7c9df27b30d633442263f42f1b1 Mon Sep 17 00:00:00 2001 From: Raja Aluri Date: Tue, 14 Dec 2010 10:36:34 -0800 Subject: [PATCH 17/29] Adding missing test Change-Id: I1bbb4b0b54632878bcebdf2fac9fcbe105e2d6cd BUG=None TEST=Ran build_image with empty directory. Review URL: http://codereview.chromium.org/5781009 --- build_image | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_image b/build_image index 6dcb05ebd5..9d1c262b2a 100755 --- a/build_image +++ b/build_image @@ -718,7 +718,7 @@ create_base_image() { generate_au_zip () { local lgenerateauzip="${SCRIPTS_DIR}/generate_au_zip.py" local largs="-o ${OUTPUT_DIR}" - ! -d "${OUTPUT_DIR}" && mkdir -p "${OUTPUT_DIR}" + test ! -d "${OUTPUT_DIR}" && mkdir -p "${OUTPUT_DIR}" info "Running ${lgenerateauzip} ${largs} for generating AU updater zip file" $lgenerateauzip $largs } From fdefe8664b06d86eccd02bfd89f42154bce54f09 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Tue, 14 Dec 2010 11:09:19 -0800 Subject: [PATCH 18/29] Fix cros_run_vm_update --proxy_port support BUG=chromium-os:10318 TEST=Manual Review URL: http://codereview.chromium.org/5801001 Change-Id: I26f0cad29e2b36b045883f0e153e8994e6b436cd --- bin/cros_run_vm_update | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/cros_run_vm_update b/bin/cros_run_vm_update index abd0071462..863b66dc65 100755 --- a/bin/cros_run_vm_update +++ b/bin/cros_run_vm_update @@ -10,6 +10,8 @@ . "$(dirname $0)/../lib/cros_vm_lib.sh" DEFINE_string payload "" "Full name of the payload to update with." +DEFINE_string proxy_port "" \ + "Have the client request from this proxy instead of devserver." DEFINE_string src_image "" \ "Create a delta update by passing in the image on the remote machine." DEFINE_string stateful_update_flag "" "Flags to pass to stateful update." s @@ -35,7 +37,11 @@ if [ -n "${FLAGS_update_image_path}" ]; then fi if [ -n "${FLAGS_payload}" ]; then - IMAGE_ARGS="--payload="${FLAGS_payload}"" + IMAGE_ARGS="--payload=${FLAGS_payload}" +fi + +if [ -n "${FLAGS_proxy_port}" ]; then + IMAGE_ARGS="--proxy_port=${FLAGS_proxy_port}" fi $(dirname $0)/../image_to_live.sh \ From 48db5d20683802a150870322781f4c6d088b1688 Mon Sep 17 00:00:00 2001 From: David McMahon Date: Tue, 14 Dec 2010 16:19:24 -0800 Subject: [PATCH 19/29] Moving chromeos_version.sh to chromiumos_overlay. BUG=chromium-os:9714 TEST= Review URL: http://codereview.chromium.org/5513009 Change-Id: Ib569fe59f4e14551b198568b24014208b5d65647 --- bin/cbuildbot.py | 3 +- build_image | 2 +- chromeos_version.sh | 93 --------------------------------------------- 3 files changed, 3 insertions(+), 95 deletions(-) delete mode 100755 chromeos_version.sh diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index 076487fb20..1cc076e290 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -287,7 +287,8 @@ def _WipeOldOutput(buildroot): def _GetChromeOSVersion(buildroot): """Returns the tuple version of the Chrome OS version of the buildroot.""" - cwd = os.path.join(buildroot, 'src', 'scripts') + cwd = os.path.join(buildroot, 'src', 'third_party', 'chromiumos-overlay', + 'chromeos', 'config') version_cmd = './chromeos_version.sh' output = RunCommand(version_cmd, cwd=cwd, redirect_stdout=True, redirect_stderr=True) diff --git a/build_image b/build_image index 9d1c262b2a..61b28215b8 100755 --- a/build_image +++ b/build_image @@ -163,7 +163,7 @@ if [ "${FLAGS_fast}" -eq "${FLAGS_TRUE}" ]; then fi # Determine build version. -. "${SCRIPTS_DIR}/chromeos_version.sh" +. ${SRC_ROOT}/third_party/chromiumos-overlay/chromeos/config/chromeos_version.sh # Configure extra USE or packages for this type of build. EXTRA_PACKAGES="" diff --git a/chromeos_version.sh b/chromeos_version.sh deleted file mode 100755 index 0540c3ab7f..0000000000 --- a/chromeos_version.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/sh - -# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# ChromeOS version information -# -# This file is usually sourced by other build scripts, but can be run -# directly to see what it would do. -# -# Version numbering scheme is much like Chrome's, with the addition of -# double-incrementing branch number so trunk is always odd. - -HOSTNAME=$(hostname) -############################################################################# -# SET VERSION NUMBERS -############################################################################# -# Major/minor versions. -# Primarily for product marketing. -export CHROMEOS_VERSION_MAJOR=0 -export CHROMEOS_VERSION_MINOR=9 - -# Branch number. -# Increment by 1 in a new release branch. -# Increment by 2 in trunk after making a release branch. -# Does not reset on a major/minor change (always increases). -# (Trunk is always odd; branches are always even). -export CHROMEOS_VERSION_BRANCH=131 - -# Patch number. -# Increment by 1 each release on a branch. -# Reset to 0 when increasing branch number. -export CHROMEOS_VERSION_PATCH=0 - -# Codename of this version. -export CHROMEOS_VERSION_CODENAME="" - -############################################################################# -# SET VERSION STRINGS -############################################################################# -# Official builds must set -# CHROMEOS_OFFICIAL=1 -# Note that ${FOO:-0} means default-to-0-if-unset; ${FOO:?} means die-if-unset. -if [ ${CHROMEOS_OFFICIAL:-0} -eq 1 ] -then - # Official builds (i.e., buildbot) - export CHROMEOS_VERSION_NAME="Chrome OS" - export CHROMEOS_VERSION_TRACK="dev-channel" - export CHROMEOS_VERSION_AUSERVER="https://tools.google.com/service/update2" - export CHROMEOS_VERSION_DEVSERVER="" -elif [ "$USER" = "chrome-bot" ] -then - # Continuous builder - # Sets the codename to the user who built the image. This - # will help us figure out who did the build if a different - # person is debugging the system. - export CHROMEOS_VERSION_CODENAME="$USER" - - export CHROMEOS_VERSION_NAME="Chromium OS" - export CHROMEOS_VERSION_TRACK="buildbot-build" - export CHROMEOS_VERSION_AUSERVER="http://$HOSTNAME:8080/update" - export CHROMEOS_VERSION_DEVSERVER="http://$HOSTNAME:8080" -else - # Developer hand-builds - # Sets the codename to the user who built the image. This - # will help us figure out who did the build if a different - # person is debugging the system. - export CHROMEOS_VERSION_CODENAME="$USER" - - export CHROMEOS_VERSION_NAME="Chromium OS" - export CHROMEOS_VERSION_TRACK="developer-build" - export CHROMEOS_VERSION_AUSERVER="http://$HOSTNAME:8080/update" - export CHROMEOS_VERSION_DEVSERVER="http://$HOSTNAME:8080" - # Overwrite CHROMEOS_VERSION_PATCH with a date string for use by auto-updater - export CHROMEOS_VERSION_PATCH=$(date +%Y_%m_%d_%H%M) -fi - -# Version string. Not indentied to appease bash. -export CHROMEOS_VERSION_STRING=\ -"${CHROMEOS_VERSION_MAJOR}.${CHROMEOS_VERSION_MINOR}"\ -".${CHROMEOS_VERSION_BRANCH}.${CHROMEOS_VERSION_PATCH}" - -# Set CHROME values (Used for releases) to pass to chromeos-chrome-bin ebuild -# URL to chrome archive -export CHROME_BASE= -# export CHROME_VERSION from incoming value or NULL and let ebuild default -export CHROME_VERSION="$CHROME_VERSION" - -# Print (and remember) version info. -echo "ChromeOS version information:" -logvers="/tmp/version_${CHROMEOS_VERSION_STRING}" -env | egrep '^CHROMEOS_VERSION|CHROME_' | tee $logvers | sed 's/^/ /' From cd2196d6c560351cc1f26a16fa3769fea6758f65 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Tue, 14 Dec 2010 17:57:02 -0800 Subject: [PATCH 20/29] Add package.keywords logic to build version of chrome we specify. Change-Id: I32c17e7353b420d7a895acffab4579c9b861095c BUG=chromium-os:8693 TEST=Ran it with chrome_rev=tot, latest_release and stable_release Review URL: http://codereview.chromium.org/5854001 --- bin/cbuildbot.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index 1cc076e290..519ecc83db 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -27,6 +27,7 @@ ARCHIVE_BASE = '/var/www/archive' ARCHIVE_COUNT = 10 PUBLIC_OVERLAY = '%(buildroot)s/src/third_party/chromiumos-overlay' PRIVATE_OVERLAY = '%(buildroot)s/src/private-overlays/chromeos-overlay' +CHROME_KEYWORDS_FILE = ('/build/%(board)s/etc/portage/package.keywords/chrome') # Currently, both the full buildbot and the preflight buildbot store their # data in a variable named PORTAGE_BINHOST, but they're in different files. @@ -217,7 +218,7 @@ def _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board, cwd=cwd, enter_chroot=True) -def _MarkChromeAsStable(buildroot, tracking_branch, chrome_rev): +def _MarkChromeAsStable(buildroot, tracking_branch, chrome_rev, board): """Returns the portage atom for the revved chrome ebuild - see man emerge.""" cwd = os.path.join(buildroot, 'src', 'scripts') portage_atom_string = RunCommand(['bin/cros_mark_chrome_as_stable', @@ -228,7 +229,12 @@ def _MarkChromeAsStable(buildroot, tracking_branch, chrome_rev): Info('Found nothing to rev.') return None else: - return portage_atom_string.split('=')[1] + chrome_atom = portage_atom_string.split('=')[1] + # TODO(sosa): Workaround to build unstable chrome ebuild we uprevved. + RunCommand(['sudo', 'tee', CHROME_KEYWORDS_FILE % {'board': board}], + input='=%s\n' % chrome_atom, enter_chroot=True, + cwd=cwd) + return chrome_atom def _UprevAllPackages(buildroot, tracking_branch, board, overlays): @@ -417,7 +423,7 @@ def _Build(buildroot, emptytree): def _BuildChrome(buildroot, board, chrome_atom_to_build): """Wrapper for emerge call to build Chrome.""" cwd = os.path.join(buildroot, 'src', 'scripts') - RunCommand(['ACCEPT_KEYWORDS="* ~*"', 'emerge-%s' % board, + RunCommand(['emerge-%s' % board, '=%s' % chrome_atom_to_build], cwd=cwd, enter_chroot=True) @@ -735,7 +741,7 @@ def main(): # Perform uprev. If chrome_uprev is set, rev Chrome ebuilds. if options.chrome_rev: chrome_atom_to_build = _MarkChromeAsStable(buildroot, tracking_branch, - options.chrome_rev) + options.chrome_rev, board) # If we found nothing to rev, we're done here. if not chrome_atom_to_build: return From 27c473d6c7c33ef4a8f22d063fdf58bc29c4c25e Mon Sep 17 00:00:00 2001 From: Yasuhiro Matsuda Date: Wed, 15 Dec 2010 17:35:54 +0900 Subject: [PATCH 21/29] Fix the path of chromeos_version.sh in archive_build.sh Change-Id: I16fd83aef4cfc11245a95af66813f7689386949b TBR=djmm BUG=chromium-os:9714, chromium-os:10354 TEST=none Review URL: http://codereview.chromium.org/5884001 --- archive_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive_build.sh b/archive_build.sh index ceefc65cd8..233540a627 100755 --- a/archive_build.sh +++ b/archive_build.sh @@ -93,7 +93,7 @@ then fi # Get version information -. "${SCRIPTS_DIR}/chromeos_version.sh" +. ${SRC_ROOT}/third_party/chromiumos-overlay/chromeos/config/chromeos_version.sh # Get git hash # Use git:8 chars of sha1 From b38b3c81bcd609f6ae51637e0dceb09d352d2077 Mon Sep 17 00:00:00 2001 From: Daniel Kurtz Date: Wed, 15 Dec 2010 10:31:11 -0800 Subject: [PATCH 22/29] run_remote_tests.sh: pass multiple arguments to autoserv Change-Id: I7f54a0e078d0c7eb70e0a614bbb991cd372dec60 BUG=chromium-os:9841 TEST= ## Test with these 2 scripts ./bin/cros_run_wifi_tests.sh --cell 3 MatFunc --args "test_pat=000*" ## --and-- ./run_remote_tests.sh --remote=${DUT} --args="test_pat=000* config_file=wifi_testbed_chromelab server_addr=${SERVER} router_addr=${ROUTER}" MatFunc Review URL: http://codereview.chromium.org/5821001 --- run_remote_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_remote_tests.sh b/run_remote_tests.sh index 2a54456f73..9cb1318172 100755 --- a/run_remote_tests.sh +++ b/run_remote_tests.sh @@ -268,7 +268,7 @@ function main() { local autoserv_args="-m ${FLAGS_remote} --ssh-port ${FLAGS_ssh_port} \ ${option} ${control_file} -r ${results_dir} ${verbose}" if [ -n "${FLAGS_args}" ]; then - autoserv_args="${autoserv_args} -a \"${FLAGS_args}\"" + autoserv_args="${autoserv_args} --args=${FLAGS_args}" fi sudo chmod a+w ./server/{tests,site_tests} @@ -291,5 +291,5 @@ function main() { print_time_elapsed } -restart_in_chroot_if_needed $* +restart_in_chroot_if_needed "$@" main "$@" From 72d0b08d922d4184c67176f76fcf5ed85325c8ef Mon Sep 17 00:00:00 2001 From: hexxeh Date: Wed, 15 Dec 2010 13:17:37 -0800 Subject: [PATCH 23/29] Added option to set ChromeOS version name, track and AU/dev server URLs via environment variables Change-Id: I80914aa49e74edf71914aa93f310104210581ffd BUG=none TEST=Ran a x86-generic build and tested from USB drive Committed: http://chrome-svn/viewvc/chromeos?view=rev&revision=4a0a239 Review URL: http://codereview.chromium.org/5760004 Patch from hexxeh . --- enter_chroot.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/enter_chroot.sh b/enter_chroot.sh index 606eaf51a6..431afa3bab 100755 --- a/enter_chroot.sh +++ b/enter_chroot.sh @@ -280,6 +280,8 @@ REVISION=$(cd ${FLAGS_trunk}/src/scripts ; git rev-parse --short=8 HEAD) CHROOT_PASSTHRU="CHROMEOS_REVISION=$REVISION BUILDBOT_BUILD=$FLAGS_build_number CHROMEOS_OFFICIAL=$CHROMEOS_OFFICIAL" CHROOT_PASSTHRU="${CHROOT_PASSTHRU} \ CHROMEOS_RELEASE_APPID=${CHROMEOS_RELEASE_APPID:-"{DEV-BUILD}"}" +CHROOT_PASSTHRU="${CHROOT_PASSTHRU} \ +CHROMEOS_VERSION_TRACK=$CHROMEOS_VERSION_TRACK CHROMEOS_VERSION_AUSERVER=$CHROMEOS_VERSION_AUSERVER CHROMEOS_VERSION_DEVSERVER=$CHROMEOS_VERSION_DEVSERVER" if [ -d "$HOME/.subversion" ]; then # Bind mounting .subversion into chroot From ebe1baa4ba68159299904a3437d0ea3c267ebf47 Mon Sep 17 00:00:00 2001 From: Doug Anderson Date: Wed, 15 Dec 2010 13:56:09 -0800 Subject: [PATCH 24/29] Fixed update_kernel.sh not to put an extra set of quotes w/ enter_chroot. This change is in preparation for the new sudo, which corrects how parameters are passed to subcommands. Change-Id: I34d09d2716fbef137d02748a2934459a8a10d693 BUG=chromium-os:7072 TEST=Ran modified "update_kernel.sh" w/ old sudo and new sudo. Review URL: http://codereview.chromium.org/5859002 --- update_kernel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update_kernel.sh b/update_kernel.sh index 126786b2f6..4166709aea 100755 --- a/update_kernel.sh +++ b/update_kernel.sh @@ -66,7 +66,7 @@ function main() { --bootloader /lib64/bootstub/bootstub.efi \ --vmlinuz /build/${FLAGS_board}/boot/vmlinuz" - ./enter_chroot.sh -- "${cmd}" + ./enter_chroot.sh -- ${cmd} learn_partition From 7361be58e0938426102588bd44ed18455d5b3376 Mon Sep 17 00:00:00 2001 From: Doug Anderson Date: Wed, 15 Dec 2010 16:05:56 -0800 Subject: [PATCH 25/29] Fixed image_to_live so that it will work with the new sudo This change will not work properly unless the chroot sudo is updated to handle escaping properly: http://codereview.chromium.org/5923001 ...and the chroot sudo will break image_to_live unless this change goes in. Thus, the two changes are intertwined. Change-Id: Ib1602ee47178fc5d32c286e9a914111c1140252b BUG=chromium-os:7072 TEST=Valided that the dev server seemed to be getting proper args. Review URL: http://codereview.chromium.org/5929001 --- image_to_live.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/image_to_live.sh b/image_to_live.sh index 545e428b92..6c237820b5 100755 --- a/image_to_live.sh +++ b/image_to_live.sh @@ -153,7 +153,7 @@ function start_dev_server { --src_image=\"$(reinterpret_path_for_chroot ${FLAGS_src_image})\"" info "Starting devserver with flags ${devserver_flags}" - ./enter_chroot.sh "sudo ./start_devserver ${devserver_flags} \ + ./enter_chroot.sh -- sudo sh -c "./start_devserver ${devserver_flags} \ --client_prefix=ChromeOSUpdateEngine \ --board=${FLAGS_board} \ --port=${FLAGS_devserver_port} > ${FLAGS_server_log} 2>&1" & @@ -217,11 +217,11 @@ function get_update_args { function get_devserver_url { local devserver_url="" local port=${FLAGS_devserver_port} - + if [[ -n ${FLAGS_proxy_port} ]]; then port=${FLAGS_proxy_port} fi - + if [ ${FLAGS_ignore_hostname} -eq ${FLAGS_TRUE} ]; then if [ -z ${FLAGS_update_url} ]; then devserver_url="http://$(get_hostname):${port}/update" From 9b0127c6f57b06f06167b9186bd732214065170d Mon Sep 17 00:00:00 2001 From: Doug Anderson Date: Wed, 15 Dec 2010 16:51:49 -0800 Subject: [PATCH 26/29] Fixed build failure w/ new sudo Change-Id: I2760e33e1473ce04f7adeda39d0db5725eaa81ae BUG=chromium-os:7072 TEST=Replaced ./build_packages with bash and ran command; valided env var set right. Review URL: http://codereview.chromium.org/5930002 --- bin/cbuildbot.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index 519ecc83db..de395947c2 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -413,9 +413,10 @@ def _SetupBoard(buildroot, board='x86-generic'): def _Build(buildroot, emptytree): """Wrapper around build_packages.""" cwd = os.path.join(buildroot, 'src', 'scripts') - cmd = ['./build_packages'] if emptytree: - cmd.insert(0, 'EXTRA_BOARD_FLAGS=--emptytree') + cmd = ['sh', '-c', 'EXTRA_BOARD_FLAGS=--emptytree ./build_packages'] + else: + cmd = ['./build_packages'] RunCommand(cmd, cwd=cwd, enter_chroot=True) From d75d3b43c8179f74d39908d30984be9e06771a45 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Wed, 15 Dec 2010 17:42:07 -0800 Subject: [PATCH 27/29] Undo all the manifest logic from cbuildbot. We also print to /dev/stderr directly to avoid flushing issues which have been occassionally given us broken manifests printed out Change-Id: I97819bbc31142432aa5ef926f99f2c4f90fd50ef BUG=chromium-os:9714 TEST=Ran cbuildbot through sync Review URL: http://codereview.chromium.org/5888003 --- bin/cbuildbot.py | 61 ++------------------------------------ bin/cbuildbot_unittest.py | 62 --------------------------------------- 2 files changed, 2 insertions(+), 121 deletions(-) diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index de395947c2..29ab5c66f0 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -87,6 +87,8 @@ def RepoSync(buildroot, retries=_DEFAULT_RETRIES): Warning('CBUILDBOT -- Retries exhausted') raise + RunCommand(['repo', 'manifest', '-r', '-o', '/dev/stderr'], cwd=buildroot) + # =========================== Command Helpers ================================= def _GetAllGitRepos(buildroot, debug=False): @@ -291,32 +293,6 @@ def _WipeOldOutput(buildroot): RunCommand(['rm', '-rf', 'src/build/images'], cwd=buildroot) -def _GetChromeOSVersion(buildroot): - """Returns the tuple version of the Chrome OS version of the buildroot.""" - cwd = os.path.join(buildroot, 'src', 'third_party', 'chromiumos-overlay', - 'chromeos', 'config') - version_cmd = './chromeos_version.sh' - output = RunCommand(version_cmd, cwd=cwd, redirect_stdout=True, - redirect_stderr=True) - version_re = re.compile('\s+CHROMEOS_VERSION_STRING=' - '(\d+)\.(\d+)\.(\d+)\.(\w+)') - for line in output.splitlines(): - match = version_re.match(line) - if match: - return match.group(1), match.group(2), match.group(3), match.group(4) - - raise Exception('Chrome OS version not found.') - - -def _GetManifestPath(buildroot): - """Returns the relative path that a manifest should be saved into.""" - version_tuple = _GetChromeOSVersion(buildroot) - (major, minor, branch, patch) = version_tuple - relative_path = os.path.join('.'.join([major, minor]), - '%s.xml' % '.'.join(version_tuple)) - return relative_path - - # =========================== Main Commands =================================== @@ -345,36 +321,6 @@ def _IncrementalCheckout(buildroot, retries=_DEFAULT_RETRIES): RepoSync(buildroot, retries) -def _DumpManifest(buildroot, url): - """Stores the manifest in the public | private overlay depending on url.""" - public_overlay = PUBLIC_OVERLAY % {'buildroot': buildroot} - private_overlay = PRIVATE_OVERLAY % {'buildroot': buildroot} - if url.endswith('manifest-internal'): - overlay = PRIVATE_OVERLAY % {'buildroot': buildroot} - else: - overlay = PUBLIC_OVERLAY % {'buildroot': buildroot} - - # Generate paths for manifests. - relative_path = _GetManifestPath(buildroot) - manifest_path = os.path.join(overlay, 'manifests', relative_path) - symlink_path = os.path.join(overlay, 'manifests', 'LATEST') - if not os.path.isdir(os.path.dirname(manifest_path)): - os.makedirs(os.path.dirname(manifest_path)) - - # Dump the manifest and create a symlink to it. - RunCommand(['repo', 'manifest', '-r', '-o', manifest_path], cwd=buildroot) - if os.path.exists(symlink_path): - os.unlink(symlink_path) - - os.symlink(relative_path, symlink_path) - - # Add it to git and print it to stderr. - RunCommand(['git', 'add', os.path.join('manifests', relative_path)], - cwd=overlay) - RunCommand(['git', 'add', os.path.join('manifests', 'LATEST')], cwd=overlay) - _PrintFile(manifest_path) - - def _MakeChroot(buildroot): """Wrapper around make_chroot.""" cwd = os.path.join(buildroot, 'src', 'scripts') @@ -730,9 +676,6 @@ def main(): if not os.path.isdir(path): Die('Missing overlay: %s' % path) - if not options.chrome_rev: - _DumpManifest(buildroot, options.url) - if not os.path.isdir(chroot_path): _MakeChroot(buildroot) diff --git a/bin/cbuildbot_unittest.py b/bin/cbuildbot_unittest.py index edacbe55d9..9e727423f9 100755 --- a/bin/cbuildbot_unittest.py +++ b/bin/cbuildbot_unittest.py @@ -232,68 +232,6 @@ class CBuildBotTest(mox.MoxTestBase): binhosts) self.mox.VerifyAll() - def testGetChromeOSVersion(self): - """Tests that we can parse the correct chromeos version using method.""" - fake_data = '\n'.join(['ChromeOS version information:', - ' SOME_OTHER_DATA=blah', - ' CHROMEOS_VERSION_STRING=9.0.232.1', - ' CHROMEOS_VERSION_CODENAME=test-bot', - ]) - cbuildbot.RunCommand('./chromeos_version.sh', - cwd='src/scripts', - redirect_stderr=True, - redirect_stdout=True).AndReturn(fake_data) - self.mox.ReplayAll() - return_tuple = cbuildbot._GetChromeOSVersion('') - self.assertEquals('.'.join(return_tuple), '9.0.232.1') - self.mox.VerifyAll() - - def testGetManifestPath(self): - """Tests whether our logic to get the manifest path is correct.""" - self.mox.StubOutWithMock(cbuildbot, '_GetChromeOSVersion') - return_tuple = cbuildbot._GetChromeOSVersion('').AndReturn( - ('9', '0', '232', '1')) - self.mox.ReplayAll() - relative_path = cbuildbot._GetManifestPath('') - self.assertEquals(relative_path, '9.0/9.0.232.1.xml') - self.mox.VerifyAll() - - def _CommonManifestTest(self, url, overlay_no_buildroot): - """Common method for dump manifest tests.""" - self.mox.StubOutWithMock(cbuildbot, '_GetManifestPath') - self.mox.StubOutWithMock(shutil, 'copy') - self.mox.StubOutWithMock(os, 'symlink') - temp_root = tempfile.mkdtemp('_unittest') - - overlay = overlay_no_buildroot % {'buildroot': temp_root} - relative_path = 'fake/manifest/path.xml' - full_path = os.path.join(overlay, 'manifests', relative_path) - - cbuildbot._GetManifestPath(temp_root).AndReturn(relative_path) - cbuildbot.RunCommand(['repo', 'manifest', '-r', '-o', full_path], - cwd=temp_root) - os.symlink(relative_path, os.path.join(overlay, 'manifests', 'LATEST')) - cbuildbot.RunCommand(['git', 'add', 'manifests/' + relative_path], - cwd=overlay) - cbuildbot.RunCommand(['git', 'add', 'manifests/LATEST'], cwd=overlay) - shutil.copy(full_path, '/dev/stderr') - - self.mox.ReplayAll() - cbuildbot._DumpManifest(temp_root, url) - self.mox.VerifyAll() - - shutil.rmtree(temp_root) - - def testDumpManifestPublic(self): - """Tests whether we push the manifest to the public overlay correctly.""" - self._CommonManifestTest('http://some_url/manifest', - cbuildbot.PUBLIC_OVERLAY) - - def testDumpManifestPrivate(self): - """Tests whether we push the manifest to the private overlay correctly.""" - self._CommonManifestTest('http://some_url/manifest-internal', - cbuildbot.PRIVATE_OVERLAY) - if __name__ == '__main__': unittest.main() From 7c3fdc13b08998d120c3da9bf10fd3975aa5120d Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Wed, 15 Dec 2010 17:42:44 -0800 Subject: [PATCH 28/29] Push prebuilts by default. Change-Id: I0d03b037253f299e36c3621ff6e5970e9af2bb11 BUG=chromium-os:10375 TEST=NA Review URL: http://codereview.chromium.org/5876002 --- bin/cbuildbot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py index 29ab5c66f0..8424e9e527 100755 --- a/bin/cbuildbot.py +++ b/bin/cbuildbot.py @@ -616,6 +616,7 @@ def main(): default=False, help='Override some options to run as a developer.') parser.add_option('--noprebuilts', action='store_false', dest='prebuilts', + default=True, help="Don't upload prebuilts.") parser.add_option('--nosync', action='store_false', dest='sync', default=True, From f19dea58e9b12aa927c27a216f7f4631554c6b56 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Thu, 16 Dec 2010 12:36:45 -0800 Subject: [PATCH 29/29] Fix discovering of stable candidate with unstable ebuilds. Also add some more information output when we detect that there is nothing to do. Change-Id: Ia3dcf0d3c276e10d3368d6bd508f3a0d72be79cd BUG=chromium-os:8693 TEST=Ran it with all 3 paramaters and current overlay. Review URL: http://codereview.chromium.org/5884004 --- bin/cros_mark_chrome_as_stable.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bin/cros_mark_chrome_as_stable.py b/bin/cros_mark_chrome_as_stable.py index 23655500f9..d7a75605e6 100755 --- a/bin/cros_mark_chrome_as_stable.py +++ b/bin/cros_mark_chrome_as_stable.py @@ -101,7 +101,7 @@ def _GetLatestRelease(branch=None): if branch: chrome_version_re = re.compile('^%s\.\d+.*' % branch) else: - chrome_version_re = re.compile('^[0-9]\..*') + chrome_version_re = re.compile('^[0-9]+\..*') for chrome_version in sorted_ls.splitlines(): if chrome_version_re.match(chrome_version): current_version = chrome_version @@ -146,6 +146,9 @@ class ChromeEBuild(cros_mark_as_stable.EBuild): else: return (-1) + def __str__(self): + return self.ebuild_path + def FindChromeCandidates(overlay_dir): """Return a tuple of chrome's unstable ebuild and stable ebuilds. @@ -166,7 +169,7 @@ def FindChromeCandidates(overlay_dir): if not ebuild.chrome_version: Warning('Poorly formatted ebuild found at %s' % path) else: - if not ebuild.is_stable: + if '9999' in ebuild.version: unstable_ebuilds.append(ebuild) else: stable_ebuilds.append(ebuild) @@ -272,6 +275,8 @@ def MarkChromeEBuildAsStable(stable_candidate, unstable_ebuild, chrome_rev, redirect_stderr=True, redirect_stdout=True, exit_code=True): + Info('Previous ebuild with same version found and no 9999 changes found.' + ' Nothing to do.') os.unlink(new_ebuild_path) return None