Merge pull request #23 from marineam/images

Upload official filesystem images

bootstrap_sdk

@@ -117,4 +117,4 @@ if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]]; then
         "${UPLOAD_ROOT}/${FLAGS_version}/pkgs/"
 fi
 
-info "Done!"
+command_completed

build_image

@@ -34,6 +34,9 @@ DEFINE_string usb_disk /dev/sdb4 \
 DEFINE_string enable_serial "" \
   "Enable serial port for printks. Example values: ttyS0"
 
+# include upload options
+. "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
+
 FLAGS_HELP="USAGE: build_image [flags] [list of images to build].
 This script is used to build a CoreOS image. CoreOS comes in many
 different forms.  This scripts can be used to build the following:
@@ -86,6 +89,8 @@ eval set -- "${FLAGS_ARGV}"
 # so will die prematurely if 'switch_to_strict_mode' is specified before now.
 switch_to_strict_mode
 
+check_gsutil_opts
+
 # N.B.  Ordering matters for some of the libraries below, because
 # some of the files contain initialization used by later files.
 . "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
@@ -146,6 +151,7 @@ mkdir -p "${BUILD_DIR}"
 # Create the base image.
 create_base_image ${PRISTINE_IMAGE_NAME} ${FLAGS_enable_rootfs_verification} \
   ${FLAGS_enable_bootcache}
+upload_image "${BUILD_DIR}/${PRISTINE_IMAGE_NAME}"
 
 # Running board-specific setup if any exists.
 if type board_setup &>/dev/null; then
@@ -157,22 +163,26 @@ if should_build_image ${CHROMEOS_DEVELOPER_IMAGE_NAME} \
     ${CHROMEOS_TEST_IMAGE_NAME}; then
   copy_image ${CHROMEOS_BASE_IMAGE_NAME} ${CHROMEOS_DEVELOPER_IMAGE_NAME}
   install_dev_packages ${CHROMEOS_DEVELOPER_IMAGE_NAME}
+  upload_image "${BUILD_DIR}/${CHROMEOS_DEVELOPER_IMAGE_NAME}"
 fi
 
 if should_build_image ${COREOS_PRODUCTION_IMAGE_NAME}; then
   copy_image ${CHROMEOS_BASE_IMAGE_NAME} ${COREOS_PRODUCTION_IMAGE_NAME}
   ${SCRIPTS_DIR}/bin/cros_make_image_bootable "${BUILD_DIR}" \
      ${COREOS_PRODUCTION_IMAGE_NAME}
+  upload_image "${BUILD_DIR}/${COREOS_PRODUCTION_IMAGE_NAME}"
 fi
 
 # From a developer image create a test|factory_test image.
 if should_build_image ${CHROMEOS_TEST_IMAGE_NAME}; then
   copy_image  ${CHROMEOS_DEVELOPER_IMAGE_NAME} ${CHROMEOS_TEST_IMAGE_NAME}
   mod_image_for_test  ${CHROMEOS_TEST_IMAGE_NAME}
+  upload_image "${BUILD_DIR}/${CHROMEOS_TEST_IMAGE_NAME}"
 fi
 
 # Generating AU generator zip file to run outside chroot
 generate_au_zip || echo "Failed generating AU zip file - ignoring Error..."
+upload_image "${BUILD_DIR}/au-generator.zip"
 
 # Create a named symlink.
 LINK_NAME="${FLAGS_output_root}/${BOARD}/${FLAGS_symlink}"

build_library/release_util.sh

@@ -0,0 +1,80 @@
+# Copyright (c) 2013 The CoreOS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+GSUTIL_OPTS=
+UPLOAD_ROOT="gs://storage.core-os.net/coreos"
+UPLOAD_DEFAULT=${FLAGS_FALSE}
+if [[ ${COREOS_OFFICIAL:-0} -eq 1 ]]; then
+  UPLOAD_DEFAULT=${FLAGS_TRUE}
+fi
+
+IMAGE_ZIPPER="lbzip2 --compress --keep"
+IMAGE_ZIPEXT=".bz2"
+
+DEFINE_boolean parallel ${FLAGS_TRUE} \
+  "Enable parallelism in gsutil."
+DEFINE_boolean upload ${UPLOAD_DEFAULT} \
+  "Upload all packages/images via gsutil."
+
+check_gsutil_opts() {
+    [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0
+
+    if [[ ${FLAGS_parallel} -eq ${FLAGS_TRUE} ]]; then
+        GSUTIL_OPTS="-m"
+    fi
+
+    if [[ ! -f "$HOME/.boto" ]]; then
+        die_notrace "Please run gsutil config to create ~/.boto"
+    fi
+}
+
+upload_packages() {
+    [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0
+    [[ -n "${BOARD}" ]] || die "board_options.sh must be sourced first"
+
+    local BOARD_PACKAGES="${1:-"${BOARD_ROOT}/packages"}"
+    local UPLOAD_PATH="${UPLOAD_ROOT}/${BOARD}/${COREOS_VERSION_STRING}/pkgs/"
+    info "Uploading packages"
+    gsutil ${GSUTIL_OPTS} cp -R "${BOARD_PACKAGES}"/* "${UPLOAD_PATH}"
+}
+
+make_digests() {
+    local dirname=$(dirname "$1")
+    local filename=$(basename "$1")
+    cd "${dirname}"
+    info "Computing DIGESTS for ${filename}"
+    echo -n > "${filename}.DIGESTS"
+    for hash in md5 sha1 sha512; do
+        echo "# $hash HASH" | tr "a-z" "A-Z" >> "${filename}.DIGESTS"
+        ${hash}sum "${filename}" >> "${filename}.DIGESTS"
+    done
+    cd -
+}
+
+upload_image() {
+    [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0
+    [[ -n "${BOARD}" ]] || die "board_options.sh must be sourced first"
+
+    local BUILT_IMAGE="$1"
+    local UPLOAD_PATH="${UPLOAD_ROOT}/${BOARD}/${COREOS_VERSION_STRING}/"
+
+    if [[ ! -f "${BUILT_IMAGE}" ]]; then
+        die "Image '${BUILT_IMAGE}' does not exist!"
+    fi
+
+    # Compress raw images
+    if [[ "${BUILT_IMAGE}" =~ \.(img|bin)$ ]]; then
+        info "Compressing ${BUILT_IMAGE##*/}"
+        $IMAGE_ZIPPER "${BUILT_IMAGE}"
+        BUILT_IMAGE="${BUILT_IMAGE}${IMAGE_ZIPEXT}"
+    fi
+
+    # For consistency generate a .DIGESTS file similar to the one catalyst
+    # produces for the SDK tarballs and up upload it too.
+    make_digests "${BUILT_IMAGE}"
+
+    info "Uploading ${BUILT_IMAGE##*/}"
+    gsutil ${GSUTIL_OPTS} cp "${BUILT_IMAGE}" \
+        "${BUILT_IMAGE}.DIGESTS" "${UPLOAD_PATH}"
+}

build_packages

@@ -11,12 +11,6 @@ restart_in_chroot_if_needed "$@"
 
 assert_not_root_user
 
-UPLOAD_ROOT="gs://storage.core-os.net/coreos"
-UPLOAD_DEFAULT=${FLAGS_FALSE}
-if [[ ${COREOS_OFFICIAL:-0} -eq 1 ]]; then
-  UPLOAD_DEFAULT=${FLAGS_TRUE}
-fi
-
 # Developer-visible flags.
 DEFINE_string board "${DEFAULT_BOARD}" \
   "The board to build packages for."
@@ -32,10 +26,9 @@ DEFINE_boolean withautotest "${FLAGS_TRUE}" \
   "Build autotest client code."
 DEFINE_boolean fetchonly "${FLAGS_FALSE}" \
   "Don't build anything, instead only fetch what is needed."
-DEFINE_boolean parallel ${FLAGS_TRUE} \
+
-  "Enable parallelism in gsutil."
+# include upload options
-DEFINE_boolean upload ${UPLOAD_DEFAULT} \
+. "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
-  "Upload all packages via gsutil."
 
 FLAGS_HELP="usage: $(basename $0) [flags] [packages]
 
@@ -95,9 +88,7 @@ if [[ -z "${FLAGS_board}" ]]; then
   exit 1
 fi
 
-if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} && ! -f "$HOME/.boto" ]]; then
+check_gsutil_opts
-  die_notrace "Please run gsutil config to create ~/.boto"
-fi
 
 CHROMITE_BIN="${GCLIENT_ROOT}/chromite/bin"
 
@@ -246,17 +237,8 @@ info "Merging board packages now"
     tee "${tmpfile}"
 )
 
-if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]]; then
+# upload packages if enabled
-  GSUTIL_OPTS=
+upload_packages
-  if [[ ${FLAGS_parallel} -eq ${FLAGS_TRUE} ]]; then
-      GSUTIL_OPTS="-m"
-  fi
-
-  BOARD_PACKAGES="${BOARD_ROOT}/packages"
-  UPLOAD_PATH="${UPLOAD_ROOT}/${BOARD}/${COREOS_VERSION_STRING}/pkgs/"
-  info "Uploading packages"
-  gsutil ${GSUTIL_OPTS} cp -R "${BOARD_PACKAGES}"/* "${UPLOAD_PATH}"
-fi
 
 # Extract total package count from emerge output.
 package_count=$(awk '$0 ~ /^Total: [0-9]+ packages/ { print $2 }' "${tmpfile}")

common.sh

@@ -82,7 +82,7 @@ _dump_trace() {
       ! (( p -= ${BASH_ARGC[${n} - 1]} ))
     fi
     if [[ ${n} == ${#FUNCNAME[@]} ]]; then
-      error "script called: ${0##/*} ${args}"
+      error "script called: ${0##*/} ${args}"
       error "Backtrace:  (most recent call is last)"
     else
       error "$(printf '  file %s, line %s, called: %s %s' \