diff --git a/bootstrap_sdk b/bootstrap_sdk index 7aa9668b18..17f1e00f0b 100755 --- a/bootstrap_sdk +++ b/bootstrap_sdk @@ -117,4 +117,4 @@ if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]]; then "${UPLOAD_ROOT}/${FLAGS_version}/pkgs/" fi -info "Done!" +command_completed diff --git a/build_image b/build_image index 09a7eedb84..c3eac64783 100755 --- a/build_image +++ b/build_image @@ -34,6 +34,9 @@ DEFINE_string usb_disk /dev/sdb4 \ DEFINE_string enable_serial "" \ "Enable serial port for printks. Example values: ttyS0" +# include upload options +. "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 + FLAGS_HELP="USAGE: build_image [flags] [list of images to build]. This script is used to build a CoreOS image. CoreOS comes in many different forms. This scripts can be used to build the following: @@ -86,6 +89,8 @@ eval set -- "${FLAGS_ARGV}" # so will die prematurely if 'switch_to_strict_mode' is specified before now. switch_to_strict_mode +check_gsutil_opts + # N.B. Ordering matters for some of the libraries below, because # some of the files contain initialization used by later files. . "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 @@ -146,6 +151,7 @@ mkdir -p "${BUILD_DIR}" # Create the base image. create_base_image ${PRISTINE_IMAGE_NAME} ${FLAGS_enable_rootfs_verification} \ ${FLAGS_enable_bootcache} +upload_image "${BUILD_DIR}/${PRISTINE_IMAGE_NAME}" # Running board-specific setup if any exists. if type board_setup &>/dev/null; then @@ -157,22 +163,26 @@ if should_build_image ${CHROMEOS_DEVELOPER_IMAGE_NAME} \ ${CHROMEOS_TEST_IMAGE_NAME}; then copy_image ${CHROMEOS_BASE_IMAGE_NAME} ${CHROMEOS_DEVELOPER_IMAGE_NAME} install_dev_packages ${CHROMEOS_DEVELOPER_IMAGE_NAME} + upload_image "${BUILD_DIR}/${CHROMEOS_DEVELOPER_IMAGE_NAME}" fi if should_build_image ${COREOS_PRODUCTION_IMAGE_NAME}; then copy_image ${CHROMEOS_BASE_IMAGE_NAME} ${COREOS_PRODUCTION_IMAGE_NAME} ${SCRIPTS_DIR}/bin/cros_make_image_bootable "${BUILD_DIR}" \ ${COREOS_PRODUCTION_IMAGE_NAME} + upload_image "${BUILD_DIR}/${COREOS_PRODUCTION_IMAGE_NAME}" fi # From a developer image create a test|factory_test image. if should_build_image ${CHROMEOS_TEST_IMAGE_NAME}; then copy_image ${CHROMEOS_DEVELOPER_IMAGE_NAME} ${CHROMEOS_TEST_IMAGE_NAME} mod_image_for_test ${CHROMEOS_TEST_IMAGE_NAME} + upload_image "${BUILD_DIR}/${CHROMEOS_TEST_IMAGE_NAME}" fi # Generating AU generator zip file to run outside chroot generate_au_zip || echo "Failed generating AU zip file - ignoring Error..." +upload_image "${BUILD_DIR}/au-generator.zip" # Create a named symlink. LINK_NAME="${FLAGS_output_root}/${BOARD}/${FLAGS_symlink}" diff --git a/build_library/release_util.sh b/build_library/release_util.sh new file mode 100644 index 0000000000..c36ad0101d --- /dev/null +++ b/build_library/release_util.sh @@ -0,0 +1,80 @@ +# Copyright (c) 2013 The CoreOS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +GSUTIL_OPTS= +UPLOAD_ROOT="gs://storage.core-os.net/coreos" +UPLOAD_DEFAULT=${FLAGS_FALSE} +if [[ ${COREOS_OFFICIAL:-0} -eq 1 ]]; then + UPLOAD_DEFAULT=${FLAGS_TRUE} +fi + +IMAGE_ZIPPER="lbzip2 --compress --keep" +IMAGE_ZIPEXT=".bz2" + +DEFINE_boolean parallel ${FLAGS_TRUE} \ + "Enable parallelism in gsutil." +DEFINE_boolean upload ${UPLOAD_DEFAULT} \ + "Upload all packages/images via gsutil." + +check_gsutil_opts() { + [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0 + + if [[ ${FLAGS_parallel} -eq ${FLAGS_TRUE} ]]; then + GSUTIL_OPTS="-m" + fi + + if [[ ! -f "$HOME/.boto" ]]; then + die_notrace "Please run gsutil config to create ~/.boto" + fi +} + +upload_packages() { + [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0 + [[ -n "${BOARD}" ]] || die "board_options.sh must be sourced first" + + local BOARD_PACKAGES="${1:-"${BOARD_ROOT}/packages"}" + local UPLOAD_PATH="${UPLOAD_ROOT}/${BOARD}/${COREOS_VERSION_STRING}/pkgs/" + info "Uploading packages" + gsutil ${GSUTIL_OPTS} cp -R "${BOARD_PACKAGES}"/* "${UPLOAD_PATH}" +} + +make_digests() { + local dirname=$(dirname "$1") + local filename=$(basename "$1") + cd "${dirname}" + info "Computing DIGESTS for ${filename}" + echo -n > "${filename}.DIGESTS" + for hash in md5 sha1 sha512; do + echo "# $hash HASH" | tr "a-z" "A-Z" >> "${filename}.DIGESTS" + ${hash}sum "${filename}" >> "${filename}.DIGESTS" + done + cd - +} + +upload_image() { + [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0 + [[ -n "${BOARD}" ]] || die "board_options.sh must be sourced first" + + local BUILT_IMAGE="$1" + local UPLOAD_PATH="${UPLOAD_ROOT}/${BOARD}/${COREOS_VERSION_STRING}/" + + if [[ ! -f "${BUILT_IMAGE}" ]]; then + die "Image '${BUILT_IMAGE}' does not exist!" + fi + + # Compress raw images + if [[ "${BUILT_IMAGE}" =~ \.(img|bin)$ ]]; then + info "Compressing ${BUILT_IMAGE##*/}" + $IMAGE_ZIPPER "${BUILT_IMAGE}" + BUILT_IMAGE="${BUILT_IMAGE}${IMAGE_ZIPEXT}" + fi + + # For consistency generate a .DIGESTS file similar to the one catalyst + # produces for the SDK tarballs and up upload it too. + make_digests "${BUILT_IMAGE}" + + info "Uploading ${BUILT_IMAGE##*/}" + gsutil ${GSUTIL_OPTS} cp "${BUILT_IMAGE}" \ + "${BUILT_IMAGE}.DIGESTS" "${UPLOAD_PATH}" +} diff --git a/build_packages b/build_packages index 73228d16cd..53334fd7c9 100755 --- a/build_packages +++ b/build_packages @@ -11,12 +11,6 @@ restart_in_chroot_if_needed "$@" assert_not_root_user -UPLOAD_ROOT="gs://storage.core-os.net/coreos" -UPLOAD_DEFAULT=${FLAGS_FALSE} -if [[ ${COREOS_OFFICIAL:-0} -eq 1 ]]; then - UPLOAD_DEFAULT=${FLAGS_TRUE} -fi - # Developer-visible flags. DEFINE_string board "${DEFAULT_BOARD}" \ "The board to build packages for." @@ -32,10 +26,9 @@ DEFINE_boolean withautotest "${FLAGS_TRUE}" \ "Build autotest client code." DEFINE_boolean fetchonly "${FLAGS_FALSE}" \ "Don't build anything, instead only fetch what is needed." -DEFINE_boolean parallel ${FLAGS_TRUE} \ - "Enable parallelism in gsutil." -DEFINE_boolean upload ${UPLOAD_DEFAULT} \ - "Upload all packages via gsutil." + +# include upload options +. "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 FLAGS_HELP="usage: $(basename $0) [flags] [packages] @@ -95,9 +88,7 @@ if [[ -z "${FLAGS_board}" ]]; then exit 1 fi -if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} && ! -f "$HOME/.boto" ]]; then - die_notrace "Please run gsutil config to create ~/.boto" -fi +check_gsutil_opts CHROMITE_BIN="${GCLIENT_ROOT}/chromite/bin" @@ -246,17 +237,8 @@ info "Merging board packages now" tee "${tmpfile}" ) -if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]]; then - GSUTIL_OPTS= - if [[ ${FLAGS_parallel} -eq ${FLAGS_TRUE} ]]; then - GSUTIL_OPTS="-m" - fi - - BOARD_PACKAGES="${BOARD_ROOT}/packages" - UPLOAD_PATH="${UPLOAD_ROOT}/${BOARD}/${COREOS_VERSION_STRING}/pkgs/" - info "Uploading packages" - gsutil ${GSUTIL_OPTS} cp -R "${BOARD_PACKAGES}"/* "${UPLOAD_PATH}" -fi +# upload packages if enabled +upload_packages # Extract total package count from emerge output. package_count=$(awk '$0 ~ /^Total: [0-9]+ packages/ { print $2 }' "${tmpfile}") diff --git a/common.sh b/common.sh index e1a83dc40e..a72e7a8451 100644 --- a/common.sh +++ b/common.sh @@ -82,7 +82,7 @@ _dump_trace() { ! (( p -= ${BASH_ARGC[${n} - 1]} )) fi if [[ ${n} == ${#FUNCNAME[@]} ]]; then - error "script called: ${0##/*} ${args}" + error "script called: ${0##*/} ${args}" error "Backtrace: (most recent call is last)" else error "$(printf ' file %s, line %s, called: %s %s' \