Merge pull request #135 from marineam/tarball

SDK tarball library

bootstrap_sdk

@@ -34,8 +34,6 @@ SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
 . "${SCRIPT_ROOT}/common.sh" || exit 1
 
 TYPE="coreos-sdk"
-ARCH=$(portageq envvar ARCH)
-DEFAULT_PROFILE="coreos:default/linux/${ARCH}/10.0"
 
 . "${BUILD_LIBRARY_DIR}/catalyst.sh" || exit 1
 
@@ -78,17 +76,7 @@ if [[ "$STAGES" =~ stage4 ]]; then
         "$BUILDS/${build_name}.DIGESTS" > "$BUILDS/${release_name}.DIGESTS"
 
     # Validate we didn't break the DIGESTS with sed
-    for hash_type in md5 sha1 sha512; do
-        info "Validating ${hash_type} DIGESTS"
-        # shash is what's used to generate these multi-hash digests but it
-        # doesn't exit with non-zero on failure. I mean seriously...
-        #shash -c "$BUILDS/${release_name}.DIGESTS" -a "${hash_type}"
-        # So we do it the hard way...
-        grep -qi "^# ${hash_type} HASH$" "$BUILDS/${release_name}.DIGESTS"
-        (cd "$BUILDS" && grep -A1 -i "^# ${hash_type} HASH$" \
-            "${release_name}.DIGESTS" | grep -v '^--$' | \
-            ${hash_type}sum -c - --strict)
-    done
+    verify_digests "$BUILDS/${release_name}" "$BUILDS/${release_name}.CONTENTS"
 
     info "SDK ready: $BUILDS/${release_name}"
 

build_container

@@ -8,9 +8,6 @@ SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
 . "${SCRIPT_ROOT}/common.sh" || exit 1
 
 TYPE="coreos-buildbot"
-ARCH=$(portageq envvar ARCH)
-DEFAULT_SEED="builds/coreos-sdk/stage4-${ARCH}-latest.tar.bz2"
-DEFAULT_PROFILE="coreos:default/linux/${ARCH}/10.0"
 FORCE_STAGES="stage4"
 
 . "${BUILD_LIBRARY_DIR}/catalyst.sh" || exit 1

build_library/build_image_util.sh

@@ -131,6 +131,8 @@ generate_au_zip () {
   local largs="-o ${BUILD_DIR}"
   test ! -d "${BUILD_DIR}" && mkdir -p "${BUILD_DIR}"
   info "Running ${lgenerateauzip} ${largs} for generating AU updater zip file"
+  # Make sure some vars this script needs are exported
+  export REPO_MANIFESTS_DIR SCRIPTS_DIR
   $lgenerateauzip $largs
 }
 

build_library/catalyst.sh

@@ -5,20 +5,17 @@
 
 # common.sh should be sourced first
 [[ -n "${DEFAULT_BUILD_ROOT}" ]] || exit 1
+. "${SCRIPTS_DIR}/sdk_lib/sdk_util.sh"
 
 # Default option values, may be provided before including this file
 : ${TYPE:="coreos-sdk"}
 : ${ARCH:=$(portageq envvar ARCH)}
 : ${DEFAULT_CATALYST_ROOT:="${DEFAULT_BUILD_ROOT}/catalyst"}
-: ${DEFAULT_SEED:="builds/${TYPE}/stage4-${ARCH}-latest.tar.bz2"}
+: ${DEFAULT_SEED:=${COREOS_SDK_TARBALL_PATH}}
 : ${DEFAULT_PROFILE:="coreos:default/linux/${ARCH}/10.0"}
 # Set to something like "stage4" to restrict what to build
 # FORCE_STAGES=
 
-if [[ "${DEFAULT_SEED}" != /* ]]; then
-    DEFAULT_SEED="${DEFAULT_CATALYST_ROOT}/${DEFAULT_SEED}"
-fi
-
 # Values set in catalyst_init, don't use till after calling it
 CATALYST_ROOT=
 DEBUG=
@@ -28,11 +25,6 @@ DISTDIR=
 TEMPDIR=
 STAGES=
 
-# For searching for alternatives when DEFAULT_SEED doesn't exist
-# unset SDK_SEARCH=1 to disable this fallback
-SDK_TARBALL="coreos-sdk-${ARCH}-${COREOS_SDK_VERSION}.tar.bz2"
-SDK_SEARCH=1
-
 DEFINE_string catalyst_root "${DEFAULT_CATALYST_ROOT}" \
     "Path to directory for all catalyst images and other files."
 DEFINE_string portage_stable "${SRC_ROOT}/third_party/portage-stable" \
@@ -182,8 +174,10 @@ catalyst_init() {
     TEMPDIR="$CATALYST_ROOT/tmp/$TYPE"
     DISTDIR="$CATALYST_ROOT/distfiles"
 
-    # possibly search for existing seeds
-    search_for_sdk_seed
+    # automatically download the current SDK if it is the seed tarball.
+    if [[ "$FLAGS_seed_tarball" == "${COREOS_SDK_TARBALL_PATH}" ]]; then
+        sdk_download_tarball
+    fi
 
     # confirm seed exists
     if [[ ! -f "$FLAGS_seed_tarball" ]]; then
@@ -211,32 +205,6 @@ catalyst_init() {
     fi
 }
 
-# search_for_sdk_seed
-# As a fallback search around for an existing SDK tarball we
-# can use as a seed when the default doesn't exist.
-search_for_sdk_seed() {
-    # Search disabled
-    [[ "${SDK_SEARCH}" != 1 ]] && return
-    # Seed already exists
-    [[ -f "${FLAGS_seed_tarball}" ]] && return
-    # User set the option so we shouldn't change it
-    [[ "${FLAGS_seed_tarball}" != "${DEFAULT_SEED}" ]] && return
-
-    local check_path
-    for check_path in \
-        "${CATALYST_ROOT}/builds/coreos-sdk/${SDK_TARBALL}" \
-        "${CATALYST_ROOT}/builds/seeds/${SDK_TARBALL}" \
-        "/var/cache/chromeos-cache/sdks/${SDK_TARBALL}" \
-        "/mnt/host/source/.cache/sdks/${SDK_TARBALL}"
-    do
-        if [[ -f "${check_path}" ]]; then
-            info "Using SDK for seed: ${check_path}"
-            FLAGS_seed_tarball="${check_path}"
-            return
-        fi
-    done
-}
-
 write_configs() {
     # No catalyst config option, so defined via environment
     export CCACHE_DIR="$TEMPDIR/ccache"

build_library/generate_au_zip.py

@@ -16,13 +16,16 @@ import subprocess
 import sys
 import tempfile
 
+REPO_MANIFESTS_DIR = os.environ['REPO_MANIFESTS_DIR']
+SCRIPTS_DIR = os.environ['SCRIPTS_DIR']
+
 # GLOBALS
 STATIC_FILES = ['/usr/bin/old_bins/cgpt',
                 '/usr/bin/cros_generate_update_payload',
-                '~/trunk/.repo/manifests/version.txt',
-                '~/trunk/src/scripts/chromeos-common.sh',
-                '~/trunk/src/scripts/common.sh',
-                '~/trunk/src/scripts/core_upload_update',
+                '%s/version.txt' % REPO_MANIFESTS_DIR,
+                '%s/chromeos-common.sh' % SCRIPTS_DIR,
+                '%s/common.sh' % SCRIPTS_DIR,
+                '%s/core_upload_update' % SCRIPTS_DIR,
                 ]
 
 DYNAMIC_EXECUTABLES = ['/usr/bin/delta_generator',

build_library/release_util.sh

@@ -85,23 +85,6 @@ upload_packages() {
     upload_files packages ${def_upload_path} "pkgs/" "${board_packages}"/*
 }
 
-make_digests() {
-    local dirname=$(dirname "$1")
-    local basename=$(basename "$1")
-
-    cd "${dirname}"
-    echo -n > "${basename}.DIGESTS"
-    for filename in "$@"; do
-        filename=$(basename "$filename")
-        info "Computing DIGESTS for ${filename}"
-        for hash in md5 sha1 sha512; do
-            echo "# $hash HASH" | tr "a-z" "A-Z" >> "${basename}.DIGESTS"
-            ${hash}sum "${filename}" >> "${basename}.DIGESTS"
-        done
-    done
-    cd -
-}
-
 # Upload a image along with optional supporting files
 # The image file must be the first argument
 upload_image() {

build_toolchains

@@ -8,9 +8,6 @@ SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
 . "${SCRIPT_ROOT}/common.sh" || exit 1
 
 TYPE="coreos-toolchains"
-ARCH=$(portageq envvar ARCH)
-DEFAULT_SEED="builds/coreos-sdk/stage4-${ARCH}-latest.tar.bz2"
-DEFAULT_PROFILE="coreos:default/linux/${ARCH}/10.0"
 FORCE_STAGES="stage4"
 
 . "${BUILD_LIBRARY_DIR}/catalyst.sh" || exit 1

common.sh

@@ -292,15 +292,20 @@ get_gclient_root
 # Note that 'realpath' is equivalent to 'readlink -f'.
 SCRIPT_LOCATION=$(readlink -f "${SCRIPT_LOCATION}")
 GCLIENT_ROOT=$(readlink -f "${GCLIENT_ROOT}")
+# TODO(marineam): I'm tempted to deprecate GCLIENT_ROOT, this isn't Google
+# and even if it was the source is managed by 'repo', not 'gclient'
+REPO_ROOT="${GCLIENT_ROOT}"
 
 # Other directories should always be pathed down from GCLIENT_ROOT.
 SRC_ROOT="${GCLIENT_ROOT}/src"
 SRC_INTERNAL="${GCLIENT_ROOT}/src-internal"
 SCRIPTS_DIR="${SRC_ROOT}/scripts"
 BUILD_LIBRARY_DIR="${SCRIPTS_DIR}/build_library"
+REPO_CACHE_DIR="${REPO_ROOT}/.cache"
+REPO_MANIFESTS_DIR="${REPO_ROOT}/.repo/manifests"
 
 # Source COREOS_* from manifest for version information.
-COREOS_VERSION_FILE="${GCLIENT_ROOT}/.repo/manifests/version.txt"
+COREOS_VERSION_FILE="${REPO_MANIFESTS_DIR}/version.txt"
 if [[ ! -f "${COREOS_VERSION_FILE}" ]]; then
     COREOS_VERSION_FILE="${SCRIPT_LOCATION}/version.txt"
 fi
@@ -741,6 +746,50 @@ enable_rw_mount() {
             conv=notrunc count=1 bs=1
 }
 
+# Generate a DIGESTS file, as normally used by Gentoo.
+# This is an alternative to shash which doesn't know how to report errors.
+# Usage: make_digests file1 [file2...]
+# Output: file1.DIGESTS
+# Any extra files be hashed and listed in file1.DIGESTS
+_digest_types="md5 sha1 sha512"
+make_digests() {
+    local dirname=$(dirname "$1")
+    local basename=$(basename "$1")
+
+    pushd "${dirname}" >/dev/null
+    echo -n > "${basename}.DIGESTS"
+    for filename in "$@"; do
+        filename=$(basename "$filename")
+        info "Computing DIGESTS for ${filename}"
+        for hash_type in $_digest_types; do
+            echo "# $hash_type HASH" | tr "a-z" "A-Z" >> "${basename}.DIGESTS"
+            ${hash_type}sum "${filename}" >> "${basename}.DIGESTS"
+        done
+    done
+    popd >/dev/null
+}
+
+# Validate a DIGESTS file. Essentially the inverse of make_digests.
+# Usage: verify_digests file1 [file2...]
+# Checks the hash of all given files using file1.DIGESTS
+verify_digests() {
+    local dirname=$(dirname "$1")
+    local basename=$(basename "$1")
+
+    pushd "${dirname}" >/dev/null
+    for filename in "$@"; do
+        filename=$(basename "$filename")
+        info "Validating DIGESTS for ${filename}"
+        for hash_type in $_digest_types; do
+            grep -A1 -i "^# ${hash_type} HASH$" "${basename}.DIGESTS" | \
+                grep "$filename$" | ${hash_type}sum -c - --strict || return 1
+            # Also check that none of the greps failed in the above pipeline
+            [[ -z ${PIPESTATUS[*]#0} ]] || return 1
+        done
+    done
+    popd >/dev/null
+}
+
 # Get current timestamp. Assumes common.sh runs at startup.
 start_time=$(date +%s)
 

sdk_lib/sdk_util.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# Copyright (c) 2013 The CoreOS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# common.sh must be properly sourced before this file.
+[[ -n "${COREOS_SDK_VERSION}" ]] || exit 1
+
+COREOS_SDK_ARCH="amd64" # We are unlikely to support anything else.
+COREOS_SDK_TARBALL="coreos-sdk-${COREOS_SDK_ARCH}-${COREOS_SDK_VERSION}.tar.bz2"
+COREOS_SDK_TARBALL_CACHE="${REPO_CACHE_DIR}/sdks"
+COREOS_SDK_TARBALL_PATH="${COREOS_SDK_TARBALL_CACHE}/${COREOS_SDK_TARBALL}"
+COREOS_SDK_URL_PREFIX="http://storage.core-os.net/coreos/sdk"
+COREOS_SDK_URL="${COREOS_SDK_URL_PREFIX}/${COREOS_SDK_ARCH}/${COREOS_SDK_VERSION}/${COREOS_SDK_TARBALL}"
+
+# Download the current SDK tarball (if required) and verify digests/sig
+sdk_download_tarball() {
+    if sdk_verify_digests; then
+        return 0
+    fi
+
+    info "Downloading ${COREOS_SDK_TARBALL}"
+    info "URL: ${COREOS_SDK_URL}"
+    local suffix
+    for suffix in "" ".DIGESTS"; do # TODO(marineam): download .asc
+        wget --tries=3 --timeout=30 --continue \
+            -O  "${COREOS_SDK_TARBALL_PATH}${suffix}" \
+            "${COREOS_SDK_URL}${suffix}" \
+            || die_notrace "SDK download failed!"
+    done
+
+    sdk_verify_digests || die_notrace "SDK digest verification failed!"
+    sdk_clean_cache
+}
+
+sdk_verify_digests() {
+    if [[ ! -f "${COREOS_SDK_TARBALL_PATH}" || \
+          ! -f "${COREOS_SDK_TARBALL_PATH}.DIGESTS" ]]; then
+        return 1
+    fi
+
+    # TODO(marineam): Add gpg signature verification too.
+
+    verify_digests "${COREOS_SDK_TARBALL_PATH}" || return 1
+}
+
+sdk_clean_cache() {
+    pushd "${COREOS_SDK_TARBALL_CACHE}" >/dev/null
+    local filename
+    for filename in *; do
+        if [[ "${filename}" == "${COREOS_SDK_TARBALL}"* ]]; then
+            continue
+        fi
+        info "Cleaning up ${filename}"
+        # Not a big deal if this fails
+        rm -f "${filename}" || true
+    done
+    popd >/dev/null
+}

tag_release

@@ -27,7 +27,7 @@ switch_to_strict_mode
 BRANCH_NAME="build-${FLAGS_build}"
 TAG_NAME="v${FLAGS_build}.${FLAGS_branch}.${FLAGS_patch}"
 
-cd "${GCLIENT_ROOT}/.repo/manifests"
+cd "${REPO_MANIFESTS_DIR}"
 
 # Clean up existing branch manifest(s) excluding:
 # - the current branch if the file already exists.