diff --git a/bootstrap_sdk b/bootstrap_sdk index 747ddeb1c3..d9e36d6ac4 100755 --- a/bootstrap_sdk +++ b/bootstrap_sdk @@ -34,8 +34,6 @@ SCRIPT_ROOT=$(dirname $(readlink -f "$0")) . "${SCRIPT_ROOT}/common.sh" || exit 1 TYPE="coreos-sdk" -ARCH=$(portageq envvar ARCH) -DEFAULT_PROFILE="coreos:default/linux/${ARCH}/10.0" . "${BUILD_LIBRARY_DIR}/catalyst.sh" || exit 1 @@ -78,17 +76,7 @@ if [[ "$STAGES" =~ stage4 ]]; then "$BUILDS/${build_name}.DIGESTS" > "$BUILDS/${release_name}.DIGESTS" # Validate we didn't break the DIGESTS with sed - for hash_type in md5 sha1 sha512; do - info "Validating ${hash_type} DIGESTS" - # shash is what's used to generate these multi-hash digests but it - # doesn't exit with non-zero on failure. I mean seriously... - #shash -c "$BUILDS/${release_name}.DIGESTS" -a "${hash_type}" - # So we do it the hard way... - grep -qi "^# ${hash_type} HASH$" "$BUILDS/${release_name}.DIGESTS" - (cd "$BUILDS" && grep -A1 -i "^# ${hash_type} HASH$" \ - "${release_name}.DIGESTS" | grep -v '^--$' | \ - ${hash_type}sum -c - --strict) - done + verify_digests "$BUILDS/${release_name}" "$BUILDS/${release_name}.CONTENTS" info "SDK ready: $BUILDS/${release_name}" diff --git a/build_container b/build_container index 9f76ae86af..afb5a23c56 100755 --- a/build_container +++ b/build_container @@ -8,9 +8,6 @@ SCRIPT_ROOT=$(dirname $(readlink -f "$0")) . "${SCRIPT_ROOT}/common.sh" || exit 1 TYPE="coreos-buildbot" -ARCH=$(portageq envvar ARCH) -DEFAULT_SEED="builds/coreos-sdk/stage4-${ARCH}-latest.tar.bz2" -DEFAULT_PROFILE="coreos:default/linux/${ARCH}/10.0" FORCE_STAGES="stage4" . "${BUILD_LIBRARY_DIR}/catalyst.sh" || exit 1 diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 9885a08e01..eb84a2e05d 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -131,6 +131,8 @@ generate_au_zip () { local largs="-o ${BUILD_DIR}" test ! -d "${BUILD_DIR}" && mkdir -p "${BUILD_DIR}" info "Running ${lgenerateauzip} ${largs} for generating AU updater zip file" + # Make sure some vars this script needs are exported + export REPO_MANIFESTS_DIR SCRIPTS_DIR $lgenerateauzip $largs } diff --git a/build_library/catalyst.sh b/build_library/catalyst.sh index 8d53a23863..2a6ab7fb1c 100644 --- a/build_library/catalyst.sh +++ b/build_library/catalyst.sh @@ -5,20 +5,17 @@ # common.sh should be sourced first [[ -n "${DEFAULT_BUILD_ROOT}" ]] || exit 1 +. "${SCRIPTS_DIR}/sdk_lib/sdk_util.sh" # Default option values, may be provided before including this file : ${TYPE:="coreos-sdk"} : ${ARCH:=$(portageq envvar ARCH)} : ${DEFAULT_CATALYST_ROOT:="${DEFAULT_BUILD_ROOT}/catalyst"} -: ${DEFAULT_SEED:="builds/${TYPE}/stage4-${ARCH}-latest.tar.bz2"} +: ${DEFAULT_SEED:=${COREOS_SDK_TARBALL_PATH}} : ${DEFAULT_PROFILE:="coreos:default/linux/${ARCH}/10.0"} # Set to something like "stage4" to restrict what to build # FORCE_STAGES= -if [[ "${DEFAULT_SEED}" != /* ]]; then - DEFAULT_SEED="${DEFAULT_CATALYST_ROOT}/${DEFAULT_SEED}" -fi - # Values set in catalyst_init, don't use till after calling it CATALYST_ROOT= DEBUG= @@ -28,11 +25,6 @@ DISTDIR= TEMPDIR= STAGES= -# For searching for alternatives when DEFAULT_SEED doesn't exist -# unset SDK_SEARCH=1 to disable this fallback -SDK_TARBALL="coreos-sdk-${ARCH}-${COREOS_SDK_VERSION}.tar.bz2" -SDK_SEARCH=1 - DEFINE_string catalyst_root "${DEFAULT_CATALYST_ROOT}" \ "Path to directory for all catalyst images and other files." DEFINE_string portage_stable "${SRC_ROOT}/third_party/portage-stable" \ @@ -182,8 +174,10 @@ catalyst_init() { TEMPDIR="$CATALYST_ROOT/tmp/$TYPE" DISTDIR="$CATALYST_ROOT/distfiles" - # possibly search for existing seeds - search_for_sdk_seed + # automatically download the current SDK if it is the seed tarball. + if [[ "$FLAGS_seed_tarball" == "${COREOS_SDK_TARBALL_PATH}" ]]; then + sdk_download_tarball + fi # confirm seed exists if [[ ! -f "$FLAGS_seed_tarball" ]]; then @@ -211,32 +205,6 @@ catalyst_init() { fi } -# search_for_sdk_seed -# As a fallback search around for an existing SDK tarball we -# can use as a seed when the default doesn't exist. -search_for_sdk_seed() { - # Search disabled - [[ "${SDK_SEARCH}" != 1 ]] && return - # Seed already exists - [[ -f "${FLAGS_seed_tarball}" ]] && return - # User set the option so we shouldn't change it - [[ "${FLAGS_seed_tarball}" != "${DEFAULT_SEED}" ]] && return - - local check_path - for check_path in \ - "${CATALYST_ROOT}/builds/coreos-sdk/${SDK_TARBALL}" \ - "${CATALYST_ROOT}/builds/seeds/${SDK_TARBALL}" \ - "/var/cache/chromeos-cache/sdks/${SDK_TARBALL}" \ - "/mnt/host/source/.cache/sdks/${SDK_TARBALL}" - do - if [[ -f "${check_path}" ]]; then - info "Using SDK for seed: ${check_path}" - FLAGS_seed_tarball="${check_path}" - return - fi - done -} - write_configs() { # No catalyst config option, so defined via environment export CCACHE_DIR="$TEMPDIR/ccache" diff --git a/build_library/generate_au_zip.py b/build_library/generate_au_zip.py index 9dc24d1fb4..249cda43ff 100755 --- a/build_library/generate_au_zip.py +++ b/build_library/generate_au_zip.py @@ -16,13 +16,16 @@ import subprocess import sys import tempfile +REPO_MANIFESTS_DIR = os.environ['REPO_MANIFESTS_DIR'] +SCRIPTS_DIR = os.environ['SCRIPTS_DIR'] + # GLOBALS STATIC_FILES = ['/usr/bin/old_bins/cgpt', '/usr/bin/cros_generate_update_payload', - '~/trunk/.repo/manifests/version.txt', - '~/trunk/src/scripts/chromeos-common.sh', - '~/trunk/src/scripts/common.sh', - '~/trunk/src/scripts/core_upload_update', + '%s/version.txt' % REPO_MANIFESTS_DIR, + '%s/chromeos-common.sh' % SCRIPTS_DIR, + '%s/common.sh' % SCRIPTS_DIR, + '%s/core_upload_update' % SCRIPTS_DIR, ] DYNAMIC_EXECUTABLES = ['/usr/bin/delta_generator', diff --git a/build_library/release_util.sh b/build_library/release_util.sh index 8ca5fd7589..fad1936cfd 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -85,23 +85,6 @@ upload_packages() { upload_files packages ${def_upload_path} "pkgs/" "${board_packages}"/* } -make_digests() { - local dirname=$(dirname "$1") - local basename=$(basename "$1") - - cd "${dirname}" - echo -n > "${basename}.DIGESTS" - for filename in "$@"; do - filename=$(basename "$filename") - info "Computing DIGESTS for ${filename}" - for hash in md5 sha1 sha512; do - echo "# $hash HASH" | tr "a-z" "A-Z" >> "${basename}.DIGESTS" - ${hash}sum "${filename}" >> "${basename}.DIGESTS" - done - done - cd - -} - # Upload a image along with optional supporting files # The image file must be the first argument upload_image() { diff --git a/build_toolchains b/build_toolchains index 0b283b1b06..70bcda0b28 100755 --- a/build_toolchains +++ b/build_toolchains @@ -8,9 +8,6 @@ SCRIPT_ROOT=$(dirname $(readlink -f "$0")) . "${SCRIPT_ROOT}/common.sh" || exit 1 TYPE="coreos-toolchains" -ARCH=$(portageq envvar ARCH) -DEFAULT_SEED="builds/coreos-sdk/stage4-${ARCH}-latest.tar.bz2" -DEFAULT_PROFILE="coreos:default/linux/${ARCH}/10.0" FORCE_STAGES="stage4" . "${BUILD_LIBRARY_DIR}/catalyst.sh" || exit 1 diff --git a/common.sh b/common.sh index b1cba1c95f..e398350011 100644 --- a/common.sh +++ b/common.sh @@ -292,15 +292,20 @@ get_gclient_root # Note that 'realpath' is equivalent to 'readlink -f'. SCRIPT_LOCATION=$(readlink -f "${SCRIPT_LOCATION}") GCLIENT_ROOT=$(readlink -f "${GCLIENT_ROOT}") +# TODO(marineam): I'm tempted to deprecate GCLIENT_ROOT, this isn't Google +# and even if it was the source is managed by 'repo', not 'gclient' +REPO_ROOT="${GCLIENT_ROOT}" # Other directories should always be pathed down from GCLIENT_ROOT. SRC_ROOT="${GCLIENT_ROOT}/src" SRC_INTERNAL="${GCLIENT_ROOT}/src-internal" SCRIPTS_DIR="${SRC_ROOT}/scripts" BUILD_LIBRARY_DIR="${SCRIPTS_DIR}/build_library" +REPO_CACHE_DIR="${REPO_ROOT}/.cache" +REPO_MANIFESTS_DIR="${REPO_ROOT}/.repo/manifests" # Source COREOS_* from manifest for version information. -COREOS_VERSION_FILE="${GCLIENT_ROOT}/.repo/manifests/version.txt" +COREOS_VERSION_FILE="${REPO_MANIFESTS_DIR}/version.txt" if [[ ! -f "${COREOS_VERSION_FILE}" ]]; then COREOS_VERSION_FILE="${SCRIPT_LOCATION}/version.txt" fi @@ -741,6 +746,50 @@ enable_rw_mount() { conv=notrunc count=1 bs=1 } +# Generate a DIGESTS file, as normally used by Gentoo. +# This is an alternative to shash which doesn't know how to report errors. +# Usage: make_digests file1 [file2...] +# Output: file1.DIGESTS +# Any extra files be hashed and listed in file1.DIGESTS +_digest_types="md5 sha1 sha512" +make_digests() { + local dirname=$(dirname "$1") + local basename=$(basename "$1") + + pushd "${dirname}" >/dev/null + echo -n > "${basename}.DIGESTS" + for filename in "$@"; do + filename=$(basename "$filename") + info "Computing DIGESTS for ${filename}" + for hash_type in $_digest_types; do + echo "# $hash_type HASH" | tr "a-z" "A-Z" >> "${basename}.DIGESTS" + ${hash_type}sum "${filename}" >> "${basename}.DIGESTS" + done + done + popd >/dev/null +} + +# Validate a DIGESTS file. Essentially the inverse of make_digests. +# Usage: verify_digests file1 [file2...] +# Checks the hash of all given files using file1.DIGESTS +verify_digests() { + local dirname=$(dirname "$1") + local basename=$(basename "$1") + + pushd "${dirname}" >/dev/null + for filename in "$@"; do + filename=$(basename "$filename") + info "Validating DIGESTS for ${filename}" + for hash_type in $_digest_types; do + grep -A1 -i "^# ${hash_type} HASH$" "${basename}.DIGESTS" | \ + grep "$filename$" | ${hash_type}sum -c - --strict || return 1 + # Also check that none of the greps failed in the above pipeline + [[ -z ${PIPESTATUS[*]#0} ]] || return 1 + done + done + popd >/dev/null +} + # Get current timestamp. Assumes common.sh runs at startup. start_time=$(date +%s) diff --git a/sdk_lib/sdk_util.sh b/sdk_lib/sdk_util.sh new file mode 100644 index 0000000000..401b818d52 --- /dev/null +++ b/sdk_lib/sdk_util.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# Copyright (c) 2013 The CoreOS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# common.sh must be properly sourced before this file. +[[ -n "${COREOS_SDK_VERSION}" ]] || exit 1 + +COREOS_SDK_ARCH="amd64" # We are unlikely to support anything else. +COREOS_SDK_TARBALL="coreos-sdk-${COREOS_SDK_ARCH}-${COREOS_SDK_VERSION}.tar.bz2" +COREOS_SDK_TARBALL_CACHE="${REPO_CACHE_DIR}/sdks" +COREOS_SDK_TARBALL_PATH="${COREOS_SDK_TARBALL_CACHE}/${COREOS_SDK_TARBALL}" +COREOS_SDK_URL_PREFIX="http://storage.core-os.net/coreos/sdk" +COREOS_SDK_URL="${COREOS_SDK_URL_PREFIX}/${COREOS_SDK_ARCH}/${COREOS_SDK_VERSION}/${COREOS_SDK_TARBALL}" + +# Download the current SDK tarball (if required) and verify digests/sig +sdk_download_tarball() { + if sdk_verify_digests; then + return 0 + fi + + info "Downloading ${COREOS_SDK_TARBALL}" + info "URL: ${COREOS_SDK_URL}" + local suffix + for suffix in "" ".DIGESTS"; do # TODO(marineam): download .asc + wget --tries=3 --timeout=30 --continue \ + -O "${COREOS_SDK_TARBALL_PATH}${suffix}" \ + "${COREOS_SDK_URL}${suffix}" \ + || die_notrace "SDK download failed!" + done + + sdk_verify_digests || die_notrace "SDK digest verification failed!" + sdk_clean_cache +} + +sdk_verify_digests() { + if [[ ! -f "${COREOS_SDK_TARBALL_PATH}" || \ + ! -f "${COREOS_SDK_TARBALL_PATH}.DIGESTS" ]]; then + return 1 + fi + + # TODO(marineam): Add gpg signature verification too. + + verify_digests "${COREOS_SDK_TARBALL_PATH}" || return 1 +} + +sdk_clean_cache() { + pushd "${COREOS_SDK_TARBALL_CACHE}" >/dev/null + local filename + for filename in *; do + if [[ "${filename}" == "${COREOS_SDK_TARBALL}"* ]]; then + continue + fi + info "Cleaning up ${filename}" + # Not a big deal if this fails + rm -f "${filename}" || true + done + popd >/dev/null +} diff --git a/tag_release b/tag_release index 8552704008..4fd5ec6b83 100755 --- a/tag_release +++ b/tag_release @@ -27,7 +27,7 @@ switch_to_strict_mode BRANCH_NAME="build-${FLAGS_build}" TAG_NAME="v${FLAGS_build}.${FLAGS_branch}.${FLAGS_patch}" -cd "${GCLIENT_ROOT}/.repo/manifests" +cd "${REPO_MANIFESTS_DIR}" # Clean up existing branch manifest(s) excluding: # - the current branch if the file already exists.