diff --git a/archive_build.sh b/archive_build.sh deleted file mode 100755 index 77b7be8199..0000000000 --- a/archive_build.sh +++ /dev/null @@ -1,377 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2010 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# Script to archive build results. Used by the buildbots. - -# --- BEGIN COMMON.SH BOILERPLATE --- -# Load common CrOS utilities. Inside the chroot this file is installed in -# /usr/lib/crosutils. Outside the chroot we find it relative to the script's -# location. -find_common_sh() { - local common_paths=(/usr/lib/crosutils $(dirname "$(readlink -f "$0")")) - local path - - SCRIPT_ROOT= - for path in "${common_paths[@]}"; do - if [ -r "${path}/common.sh" ]; then - SCRIPT_ROOT=${path} - break - fi - done -} - -find_common_sh -. "${SCRIPT_ROOT}/common.sh" || { echo "Unable to load common.sh"; exit 1; } -# --- END COMMON.SH BOILERPLATE --- - -# Script must be run outside the chroot -assert_outside_chroot - -IMAGES_DIR="${DEFAULT_BUILD_ROOT}/images" -# Default to the most recent image -DEFAULT_TO="${GCLIENT_ROOT}/archive" -DEFAULT_FROM="${IMAGES_DIR}/$DEFAULT_BOARD/$(ls -t1 \ - $IMAGES_DIR/$DEFAULT_BOARD 2>&-| head -1)" -# The set of environment assignment we need to pass to enter_chroots for this -# build. e.g. USE flags alter some of the commands' effect. -CHROOT_ENV="" - -# Flags -DEFINE_boolean archive_debug $FLAGS_TRUE \ - "Archive debug information for build" -DEFINE_string board "$DEFAULT_BOARD" \ - "The board to build packages for." -DEFINE_string build_number "" \ - "The build-bot build number (when called by buildbot only)." "b" -DEFINE_string chroot "$DEFAULT_CHROOT_DIR" \ - "The chroot of the build to archive." -DEFINE_boolean debug $FLAGS_FALSE "Run, but don't upload anything." -DEFINE_boolean factory_install_mod $FLAGS_FALSE \ - "Modify image for factory install purposes" -DEFINE_boolean factory_test_mod $FLAGS_FALSE \ - "Modify image for factory testing purposes" -DEFINE_string from "$DEFAULT_FROM" \ - "Directory to archive" -DEFINE_string gsutil "gsutil" \ - "Location of gsutil" -DEFINE_string gsd_gen_index "" \ - "Location of gsd_generate_index.py" -DEFINE_string acl "private" \ - "ACL to set on GSD archives" -DEFINE_string gsutil_archive "" \ - "Optional datastore archive location" -DEFINE_boolean gsutil_append_last_change $FLAGS_TRUE \ - "Optional whether to append build # and chrome os hash to GS uploads" -DEFINE_integer keep_max 0 "Maximum builds to keep in archive (0=all)" -DEFINE_boolean official_build $FLAGS_FALSE \ - "Set CHROMEOS_OFFICIAL=1 for release builds." -DEFINE_string test_tarball "" "Optional path to test tarball to archive" -DEFINE_boolean test_mod $FLAGS_TRUE "Modify image for testing purposes" -DEFINE_boolean prebuilt_upload $FLAGS_FALSE "Upload prebuilt binary packages." -DEFINE_boolean remove_dev $FLAGS_TRUE "Remove the de image during archive." -DEFINE_string set_version "" "Set version/change identifier." -DEFINE_string to "$DEFAULT_TO" "Directory of build archive" -DEFINE_string zipname "image.zip" "Name of zip file to create." -DEFINE_string useflags "" "USE flags to pass into mod_image_* and build_image." - -# Parse command line -FLAGS "$@" || exit 1 -eval set -- "${FLAGS_ARGV}" - -# Set if default from path is used -DEFAULT_USED= - -# Reset "default" FLAGS_from based on passed-in board if not set on cmd-line -if [ "$FLAGS_from" = "$DEFAULT_FROM" ]; then - # Keep the directory name of the current image set (*.bin). - IMG_DIR="$(readlink ${IMAGES_DIR}/${FLAGS_board}/latest)" - FLAGS_from="${IMAGES_DIR}/${FLAGS_board}/${IMG_DIR}" - DEFAULT_USED=1 -fi - -SYSROOT="${GCLIENT_ROOT}/chroot/build/${FLAGS_board}" - -# Die on any errors. -set -e - -if [ -n "${FLAGS_useflags}" ]; then - CHROOT_ENV="${CHROOT_ENV} USE='${FLAGS_useflags}'" -fi - -if [ -z "$DEFAULT_USED" ]; then - if [ $FLAGS_test_mod -eq $FLAGS_TRUE ] || \ - [ $FLAGS_factory_install_mod -eq $FLAGS_TRUE ] || \ - [ $FLAGS_factory_test_mod -eq $FLAGS_TRUE ] - then - echo "test_mod requires that the default from path be used." - echo "If non default behavior is desired, run mod_image_for_test manually" - echo "re-run archive build without test_mod" - exit 1 - fi -fi - -if [ ! -d "$FLAGS_from" ]; then - echo "$FLAGS_from does not exist. Exiting..." - exit 1 -fi - -if [ $FLAGS_official_build -eq $FLAGS_TRUE ]; then - CHROMEOS_OFFICIAL=1 -fi - -# Get version information -. ${SRC_ROOT}/third_party/chromiumos-overlay/chromeos/config/chromeos_version.sh - -# Get git hash -# Use git:8 chars of sha1 -REVISION=$(git rev-parse HEAD) -REVISION=${REVISION:0:8} - -# Use the version number plus revision as the last change. (Need both, since -# trunk builds multiple times with the same version string.) -LAST_CHANGE="${CHROMEOS_VERSION_STRING}-r${REVISION}" -if [ -n "$FLAGS_set_version" ]; then - LAST_CHANGE="$FLAGS_set_version" -elif [ -n "$FLAGS_build_number" ]; then - LAST_CHANGE="$LAST_CHANGE-b${FLAGS_build_number}" -fi - -# The Chromium buildbot scripts only create a clickable link to the archive -# if an output line of the form "last change: XXX" exists -echo "last change: $LAST_CHANGE" -echo "archive from: $FLAGS_from" - -# Create the output directory -OUTDIR="${FLAGS_to}/${LAST_CHANGE}" -ZIPFILE="${OUTDIR}/${FLAGS_zipname}" -FACTORY_ZIPFILE="${OUTDIR}/factory_${FLAGS_zipname}" -echo "archive to dir: $OUTDIR" -echo "archive to file: $ZIPFILE" - -mkdir -p "$OUTDIR" - -# Modify image for test if flag set. -CHROOT_IMAGE_DIR=$(reinterpret_path_for_chroot "$FLAGS_from") -if [ $FLAGS_test_mod -eq $FLAGS_TRUE ]; then - echo "Modifying image for test" - ./enter_chroot.sh $CHROOT_ENV -- ./mod_image_for_test.sh \ - --board $FLAGS_board --noinplace --yes \ - --image $CHROOT_IMAGE_DIR/$CHROMEOS_IMAGE_NAME - - pushd "${FLAGS_chroot}/build/${FLAGS_board}/usr/local" - echo "Archiving autotest build artifacts" - if which pbzip2 >/dev/null 2>/dev/null; then - TAR_BZIP2="tar --use-compress-program=pbzip2" - else - TAR_BZIP2="tar --bzip2" - fi - ${TAR_BZIP2} -cf "${FLAGS_from}/autotest.tar.bz2" --checkpoint=10000 autotest - popd -fi - -# Modify for recovery -if [ $FLAGS_official_build -eq $FLAGS_TRUE ]; then - ./enter_chroot.sh $CHROOT_ENV -- ./mod_image_for_recovery.sh \ - --board $FLAGS_board --image $CHROOT_IMAGE_DIR/chromiumos_base_image.bin -fi - -# Remove the developer build if test image is also built. -if [ $FLAGS_remove_dev -eq $FLAGS_TRUE ]; then - rm -f "${FLAGS_from}/${CHROMEOS_IMAGE_NAME}" -fi - -if [ $FLAGS_factory_test_mod -eq $FLAGS_TRUE ]; then - echo "Generating image for factory test" - # HACK: The build system can't currently handle more than one image size - # at a time. Therefore it's necessary to do another round of build after - # archiving the original build. This should be fixed in Chromite. - - # HACK: cbuild has a special case when running on chrome-bot that - # zeroes out the current revision, which makes calling build_image directly - # fail. You must explictly call replace and specify a unique name numerically - # using build_attempt. - ./enter_chroot.sh $CHROOT_ENV -- ./build_image --board $FLAGS_board \ - --replace --noenable_rootfs_verification --build_attempt 4 \ - --symlink factory_test --factory --test -fi - -# Build differently sized shims. Currently only factory install shim is -# supported, TODO(tgao): Add developer shim. -if [ $FLAGS_factory_install_mod -eq $FLAGS_TRUE ]; then - echo "Building factory install shim." - # HACK: The build system can't currently handle more than one image size - # at a time. Therefore it's necessary to do another round of build after - # archiving the original build. This should be fixed in Chromite. - - # HACK: cbuild has a special case when running on chrome-bot that - # zeroes out the current revision, which makes calling build_image directly - # fail. You must explictly call replace and specify a unique name numerically - # using build_attempt. - ./enter_chroot.sh $CHROOT_ENV -- ./build_image --board $FLAGS_board \ - --factory_install --replace --build_attempt 7 --symlink factory_shim - - # Get the install shim dir: It is the newest factory_shim build. - SHIM_DIR="${IMAGES_DIR}/${FLAGS_board}/factory_shim" - CHROOT_SHIM_DIR=$(reinterpret_path_for_chroot "${SHIM_DIR}") - - # For ARM we should creat a netboot image. - # Only ARM has "uimg" uboot format kernels. - # TODO(anush): We should have build infrastructure that can directly - # query architecture. - if [ -f "${SYSROOT}/boot/vmlinux.uimg" ]; then - ./enter_chroot.sh $CHROOT_ENV -- ./make_netboot.sh --board $FLAGS_board \ - --image "$CHROOT_SHIM_DIR/factory_install_shim.bin" - fi -fi - -# Zip the build -echo "Compressing and archiving build..." -cd "$FLAGS_from" -MANIFEST=`ls | grep -v factory | grep -v netboot | grep -v hwid` -rm -f "${ZIPFILE}" -zip -r "${ZIPFILE}" ${MANIFEST} - -if [ $FLAGS_factory_test_mod -eq $FLAGS_TRUE ] || \ - [ $FLAGS_factory_install_mod -eq $FLAGS_TRUE ]; then - # We need to have directory structure for factory package, as - # signing and packaging utilities need unpack_partitions.sh. - echo "Compressing factory software" - pushd .. - - FACTORY_MANIFEST=$(find factory_shim factory_test -follow -type f | - grep -E "(factory_image|factory_install|partition|netboot|hwid)") - rm -f "${FACTORY_ZIPFILE}" - zip "${FACTORY_ZIPFILE}" ${FACTORY_MANIFEST} - echo "Zipped" - popd - chmod 644 "${FACTORY_ZIPFILE}" -fi -cd - - -# Update LATEST file -echo "$LAST_CHANGE" > "${FLAGS_to}/LATEST" - -# Make sure files are readable -chmod 644 "$ZIPFILE" "${FLAGS_to}/LATEST" -chmod 755 "$OUTDIR" -cp -f "${FLAGS_from}/au-generator.zip" "${OUTDIR}/" - - -function gsutil_archive_noappend() { - local in_path="$1" - local out_path="$2" - echo "Using gsutil to archive to ${out_path}..." - if [ -z "$FLAGS_gsutil_archive" -o ${FLAGS_debug} -eq ${FLAGS_TRUE} ]; then - echo -n "In debug mode. Would have run: " - echo "gsutil cp ${in_path} /${out_path}" - return - fi - - local full_out_path="${FLAGS_gsutil_archive}/${out_path}" - ${FLAGS_gsutil} cp ${in_path} ${full_out_path} - ${FLAGS_gsutil} setacl ${FLAGS_acl} ${full_out_path} - if [ -n "$FLAGS_gsd_gen_index" ]; then - echo "Updating indexes..." - ${FLAGS_gsd_gen_index} \ - --gsutil=${FLAGS_gsutil} \ - -a ${FLAGS_acl} \ - -p ${full_out_path} ${FLAGS_gsutil_archive} - fi -} - - -function gsutil_archive() { - local in_path="$1" - local out_path="$2" - if [ ${FLAGS_gsutil_append_last_change} -eq ${FLAGS_TRUE} ]; then - out_path="${LAST_CHANGE}/${out_path}" - fi - gsutil_archive_noappend "${in_path}" "${out_path}" -} - -if [ $FLAGS_test_mod -eq $FLAGS_TRUE -a $FLAGS_official_build -eq $FLAGS_TRUE ] -then - echo "Creating hwqual archive" - HWQUAL_NAME="chromeos-hwqual-${FLAGS_board}-${CHROMEOS_VERSION_STRING}" - "${SCRIPTS_DIR}/archive_hwqual" --from "${OUTDIR}" \ - --output_tag "${HWQUAL_NAME}" - gsutil_archive "${OUTDIR}/${HWQUAL_NAME}.tar.bz2" \ - "${HWQUAL_NAME}.tar.bz2" -fi - - -if [ $FLAGS_prebuilt_upload -eq $FLAGS_TRUE ]; then - # Construct prebuilt upload command. - # This will upload prebuilt packages to Google Storage. - prebuilt_cmd="${GCLIENT_ROOT}/chromite/buildbot/prebuilt.py" - prebuilt_cmd="$prebuilt_cmd -u gs://chromeos-prebuilt --git-sync -V master" - prebuilt_cmd="$prebuilt_cmd -p ${GCLIENT_ROOT} -b ${FLAGS_board}" - - if [ "${FLAGS_board}" == "x86-generic" ] - then - prebuilt_cmd="$prebuilt_cmd --sync-host" - fi - - if [ ${FLAGS_debug} -eq ${FLAGS_FALSE} ]; then - echo "Running $prebuilt_cmd" - $prebuilt_cmd - else - echo "Would have run $prebuilt_cmd" - fi -fi - -gsutil_archive "${ZIPFILE}" "${FLAGS_zipname}" - -if [ $FLAGS_archive_debug -eq $FLAGS_TRUE ]; then - echo "Generating Breakpad symbols" - ! ${SCRIPTS_DIR}/cros_generate_breakpad_symbols --board=${FLAGS_board} - echo "Creating debug archive" - pushd "${FLAGS_chroot}/build/${FLAGS_board}/usr/lib" - sudo tar czf "${OUTDIR}/debug.tgz" --checkpoint=1000 --exclude\ - debug/usr/local/autotest --exclude debug/tests debug - CMD="chown \${SUDO_UID}:\${SUDO_GID} ${OUTDIR}/debug.tgz" - sudo sh -c "${CMD}" - popd - gsutil_archive "${OUTDIR}/debug.tgz" "debug.tgz" -fi - -if [ $FLAGS_factory_test_mod -eq $FLAGS_TRUE ] || \ - [ $FLAGS_factory_install_mod -eq $FLAGS_TRUE ]; then - gsutil_archive "${FACTORY_ZIPFILE}" \ - "factory_${FLAGS_zipname}" -fi - -gsutil_archive_noappend "${FLAGS_to}/LATEST" "LATEST" - -if [ -n "${FLAGS_gsutil_archive}" ]; then - if [ ${FLAGS_gsutil_append_last_change} -eq ${FLAGS_TRUE} ]; then - FULL_INDEX_PATH="${FLAGS_gsutil_archive}/${LAST_CHANGE}/_index.html" - else - FULL_INDEX_PATH="${FLAGS_gsutil_archive}/_index.html" - fi - RELATIVE_ARCHIVE_URL_PATH="${FULL_INDEX_PATH#gs://}" - echo "CROS_ARCHIVE_URL=\ -https://sandbox.google.com/storage/${RELATIVE_ARCHIVE_URL_PATH}" -fi - -if [ -n "${FLAGS_test_tarball}" ]; then - # Copy to local outdir first. - TEST_TARBALL_OUT="${OUTDIR}/$(basename "${FLAGS_test_tarball}")" - cp "${FLAGS_test_tarball}" "${TEST_TARBALL_OUT}" - chmod 644 "${TEST_TARBALL_OUT}" - gsutil_archive "${TEST_TARBALL_OUT}" "test_results.tgz" -fi - -# Purge old builds if necessary -if [ $FLAGS_keep_max -gt 0 ]; then - echo "Deleting old builds (all but the newest ${FLAGS_keep_max})..." - cd "$FLAGS_to" - # +2 because line numbers start at 1 and need to skip LATEST file - rm -rf `ls -t1 | tail --lines=+$(($FLAGS_keep_max + 2))` - cd - -fi - -echo "Done."