From c5c9ea0b9be23887cc56b951fef5b416d1ee2898 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Fri, 22 Nov 2013 16:11:43 -0800 Subject: [PATCH 1/4] fix(common.sh): New variables for .cache and .repo/manfests A number of places refer to these paths and that number is going to grow. Since the standard pattern is to use environment variables for commonly used paths it is time to add ones for these: REPO_CACHE_DIR REPO_MANIFESTS_DIR --- build_library/catalyst.sh | 3 +-- build_library/generate_au_zip.py | 11 +++++++---- common.sh | 7 ++++++- tag_release | 2 +- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/build_library/catalyst.sh b/build_library/catalyst.sh index 8d53a23863..600a9eff26 100644 --- a/build_library/catalyst.sh +++ b/build_library/catalyst.sh @@ -226,8 +226,7 @@ search_for_sdk_seed() { for check_path in \ "${CATALYST_ROOT}/builds/coreos-sdk/${SDK_TARBALL}" \ "${CATALYST_ROOT}/builds/seeds/${SDK_TARBALL}" \ - "/var/cache/chromeos-cache/sdks/${SDK_TARBALL}" \ - "/mnt/host/source/.cache/sdks/${SDK_TARBALL}" + "${REPO_CACHE_DIR}/sdks/${SDK_TARBALL}" \ do if [[ -f "${check_path}" ]]; then info "Using SDK for seed: ${check_path}" diff --git a/build_library/generate_au_zip.py b/build_library/generate_au_zip.py index 9dc24d1fb4..249cda43ff 100755 --- a/build_library/generate_au_zip.py +++ b/build_library/generate_au_zip.py @@ -16,13 +16,16 @@ import subprocess import sys import tempfile +REPO_MANIFESTS_DIR = os.environ['REPO_MANIFESTS_DIR'] +SCRIPTS_DIR = os.environ['SCRIPTS_DIR'] + # GLOBALS STATIC_FILES = ['/usr/bin/old_bins/cgpt', '/usr/bin/cros_generate_update_payload', - '~/trunk/.repo/manifests/version.txt', - '~/trunk/src/scripts/chromeos-common.sh', - '~/trunk/src/scripts/common.sh', - '~/trunk/src/scripts/core_upload_update', + '%s/version.txt' % REPO_MANIFESTS_DIR, + '%s/chromeos-common.sh' % SCRIPTS_DIR, + '%s/common.sh' % SCRIPTS_DIR, + '%s/core_upload_update' % SCRIPTS_DIR, ] DYNAMIC_EXECUTABLES = ['/usr/bin/delta_generator', diff --git a/common.sh b/common.sh index b1cba1c95f..0e6242a72a 100644 --- a/common.sh +++ b/common.sh @@ -292,15 +292,20 @@ get_gclient_root # Note that 'realpath' is equivalent to 'readlink -f'. SCRIPT_LOCATION=$(readlink -f "${SCRIPT_LOCATION}") GCLIENT_ROOT=$(readlink -f "${GCLIENT_ROOT}") +# TODO(marineam): I'm tempted to deprecate GCLIENT_ROOT, this isn't Google +# and even if it was the source is managed by 'repo', not 'gclient' +REPO_ROOT="${GCLIENT_ROOT}" # Other directories should always be pathed down from GCLIENT_ROOT. SRC_ROOT="${GCLIENT_ROOT}/src" SRC_INTERNAL="${GCLIENT_ROOT}/src-internal" SCRIPTS_DIR="${SRC_ROOT}/scripts" BUILD_LIBRARY_DIR="${SCRIPTS_DIR}/build_library" +REPO_CACHE_DIR="${REPO_ROOT}/.cache" +REPO_MANIFESTS_DIR="${REPO_ROOT}/.repo/manifests" # Source COREOS_* from manifest for version information. -COREOS_VERSION_FILE="${GCLIENT_ROOT}/.repo/manifests/version.txt" +COREOS_VERSION_FILE="${REPO_MANIFESTS_DIR}/version.txt" if [[ ! -f "${COREOS_VERSION_FILE}" ]]; then COREOS_VERSION_FILE="${SCRIPT_LOCATION}/version.txt" fi diff --git a/tag_release b/tag_release index 8552704008..4fd5ec6b83 100755 --- a/tag_release +++ b/tag_release @@ -27,7 +27,7 @@ switch_to_strict_mode BRANCH_NAME="build-${FLAGS_build}" TAG_NAME="v${FLAGS_build}.${FLAGS_branch}.${FLAGS_patch}" -cd "${GCLIENT_ROOT}/.repo/manifests" +cd "${REPO_MANIFESTS_DIR}" # Clean up existing branch manifest(s) excluding: # - the current branch if the file already exists. From 763b27bb7fc53825b3189ce095b4ed501946cf1b Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Fri, 22 Nov 2013 17:14:39 -0800 Subject: [PATCH 2/4] fix(common.sh): Move DIGESTS handling code to common functions. This code would be useful to use when downloading SDK tarballs in addition to uploading them. :) --- bootstrap_sdk | 12 +--------- build_library/release_util.sh | 17 -------------- common.sh | 44 +++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 28 deletions(-) diff --git a/bootstrap_sdk b/bootstrap_sdk index 747ddeb1c3..197ad47105 100755 --- a/bootstrap_sdk +++ b/bootstrap_sdk @@ -78,17 +78,7 @@ if [[ "$STAGES" =~ stage4 ]]; then "$BUILDS/${build_name}.DIGESTS" > "$BUILDS/${release_name}.DIGESTS" # Validate we didn't break the DIGESTS with sed - for hash_type in md5 sha1 sha512; do - info "Validating ${hash_type} DIGESTS" - # shash is what's used to generate these multi-hash digests but it - # doesn't exit with non-zero on failure. I mean seriously... - #shash -c "$BUILDS/${release_name}.DIGESTS" -a "${hash_type}" - # So we do it the hard way... - grep -qi "^# ${hash_type} HASH$" "$BUILDS/${release_name}.DIGESTS" - (cd "$BUILDS" && grep -A1 -i "^# ${hash_type} HASH$" \ - "${release_name}.DIGESTS" | grep -v '^--$' | \ - ${hash_type}sum -c - --strict) - done + verify_digests "$BUILDS/${release_name}" "$BUILDS/${release_name}.CONTENTS" info "SDK ready: $BUILDS/${release_name}" diff --git a/build_library/release_util.sh b/build_library/release_util.sh index 8ca5fd7589..fad1936cfd 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -85,23 +85,6 @@ upload_packages() { upload_files packages ${def_upload_path} "pkgs/" "${board_packages}"/* } -make_digests() { - local dirname=$(dirname "$1") - local basename=$(basename "$1") - - cd "${dirname}" - echo -n > "${basename}.DIGESTS" - for filename in "$@"; do - filename=$(basename "$filename") - info "Computing DIGESTS for ${filename}" - for hash in md5 sha1 sha512; do - echo "# $hash HASH" | tr "a-z" "A-Z" >> "${basename}.DIGESTS" - ${hash}sum "${filename}" >> "${basename}.DIGESTS" - done - done - cd - -} - # Upload a image along with optional supporting files # The image file must be the first argument upload_image() { diff --git a/common.sh b/common.sh index 0e6242a72a..e398350011 100644 --- a/common.sh +++ b/common.sh @@ -746,6 +746,50 @@ enable_rw_mount() { conv=notrunc count=1 bs=1 } +# Generate a DIGESTS file, as normally used by Gentoo. +# This is an alternative to shash which doesn't know how to report errors. +# Usage: make_digests file1 [file2...] +# Output: file1.DIGESTS +# Any extra files be hashed and listed in file1.DIGESTS +_digest_types="md5 sha1 sha512" +make_digests() { + local dirname=$(dirname "$1") + local basename=$(basename "$1") + + pushd "${dirname}" >/dev/null + echo -n > "${basename}.DIGESTS" + for filename in "$@"; do + filename=$(basename "$filename") + info "Computing DIGESTS for ${filename}" + for hash_type in $_digest_types; do + echo "# $hash_type HASH" | tr "a-z" "A-Z" >> "${basename}.DIGESTS" + ${hash_type}sum "${filename}" >> "${basename}.DIGESTS" + done + done + popd >/dev/null +} + +# Validate a DIGESTS file. Essentially the inverse of make_digests. +# Usage: verify_digests file1 [file2...] +# Checks the hash of all given files using file1.DIGESTS +verify_digests() { + local dirname=$(dirname "$1") + local basename=$(basename "$1") + + pushd "${dirname}" >/dev/null + for filename in "$@"; do + filename=$(basename "$filename") + info "Validating DIGESTS for ${filename}" + for hash_type in $_digest_types; do + grep -A1 -i "^# ${hash_type} HASH$" "${basename}.DIGESTS" | \ + grep "$filename$" | ${hash_type}sum -c - --strict || return 1 + # Also check that none of the greps failed in the above pipeline + [[ -z ${PIPESTATUS[*]#0} ]] || return 1 + done + done + popd >/dev/null +} + # Get current timestamp. Assumes common.sh runs at startup. start_time=$(date +%s) From 59ef0901d355c52ce2b7d1370bfff24d2cfe0c32 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sat, 23 Nov 2013 20:31:46 -0800 Subject: [PATCH 3/4] add(sdk_util.sh): Port basic SDK download logic to a shell library. The current logic for downloading SDK tarballs is in cros_sdk and written in python which isn't super convenient for re-using in the rest of our shell scripts. This is a start of rewriting that logic into a re-usable library but does not yet replace the functionality in cros_sdk. --- sdk_lib/sdk_util.sh | 60 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 sdk_lib/sdk_util.sh diff --git a/sdk_lib/sdk_util.sh b/sdk_lib/sdk_util.sh new file mode 100644 index 0000000000..401b818d52 --- /dev/null +++ b/sdk_lib/sdk_util.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# Copyright (c) 2013 The CoreOS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# common.sh must be properly sourced before this file. +[[ -n "${COREOS_SDK_VERSION}" ]] || exit 1 + +COREOS_SDK_ARCH="amd64" # We are unlikely to support anything else. +COREOS_SDK_TARBALL="coreos-sdk-${COREOS_SDK_ARCH}-${COREOS_SDK_VERSION}.tar.bz2" +COREOS_SDK_TARBALL_CACHE="${REPO_CACHE_DIR}/sdks" +COREOS_SDK_TARBALL_PATH="${COREOS_SDK_TARBALL_CACHE}/${COREOS_SDK_TARBALL}" +COREOS_SDK_URL_PREFIX="http://storage.core-os.net/coreos/sdk" +COREOS_SDK_URL="${COREOS_SDK_URL_PREFIX}/${COREOS_SDK_ARCH}/${COREOS_SDK_VERSION}/${COREOS_SDK_TARBALL}" + +# Download the current SDK tarball (if required) and verify digests/sig +sdk_download_tarball() { + if sdk_verify_digests; then + return 0 + fi + + info "Downloading ${COREOS_SDK_TARBALL}" + info "URL: ${COREOS_SDK_URL}" + local suffix + for suffix in "" ".DIGESTS"; do # TODO(marineam): download .asc + wget --tries=3 --timeout=30 --continue \ + -O "${COREOS_SDK_TARBALL_PATH}${suffix}" \ + "${COREOS_SDK_URL}${suffix}" \ + || die_notrace "SDK download failed!" + done + + sdk_verify_digests || die_notrace "SDK digest verification failed!" + sdk_clean_cache +} + +sdk_verify_digests() { + if [[ ! -f "${COREOS_SDK_TARBALL_PATH}" || \ + ! -f "${COREOS_SDK_TARBALL_PATH}.DIGESTS" ]]; then + return 1 + fi + + # TODO(marineam): Add gpg signature verification too. + + verify_digests "${COREOS_SDK_TARBALL_PATH}" || return 1 +} + +sdk_clean_cache() { + pushd "${COREOS_SDK_TARBALL_CACHE}" >/dev/null + local filename + for filename in *; do + if [[ "${filename}" == "${COREOS_SDK_TARBALL}"* ]]; then + continue + fi + info "Cleaning up ${filename}" + # Not a big deal if this fails + rm -f "${filename}" || true + done + popd >/dev/null +} From 75972cd9917684f74f7a12bbdfa62f88833a7dde Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Sat, 23 Nov 2013 20:32:55 -0800 Subject: [PATCH 4/4] fix(catalyst): Always use the current SDK tarball as seed. Right now there is some funky logic to either use a previous build as a seed or the current SDK tarball if it happens to have been downloaded. This is a bit confusing and doesn't work reliably since it is reasonable for there to be neither a previous build or the current SDK available if the SDK chroot was created some time ago. Fix this by using the new SDK library and always use the latest SDK, downloading it if needed. --- bootstrap_sdk | 2 -- build_container | 3 --- build_library/build_image_util.sh | 2 ++ build_library/catalyst.sh | 43 +++++-------------------------- build_toolchains | 3 --- 5 files changed, 8 insertions(+), 45 deletions(-) diff --git a/bootstrap_sdk b/bootstrap_sdk index 197ad47105..d9e36d6ac4 100755 --- a/bootstrap_sdk +++ b/bootstrap_sdk @@ -34,8 +34,6 @@ SCRIPT_ROOT=$(dirname $(readlink -f "$0")) . "${SCRIPT_ROOT}/common.sh" || exit 1 TYPE="coreos-sdk" -ARCH=$(portageq envvar ARCH) -DEFAULT_PROFILE="coreos:default/linux/${ARCH}/10.0" . "${BUILD_LIBRARY_DIR}/catalyst.sh" || exit 1 diff --git a/build_container b/build_container index 9f76ae86af..afb5a23c56 100755 --- a/build_container +++ b/build_container @@ -8,9 +8,6 @@ SCRIPT_ROOT=$(dirname $(readlink -f "$0")) . "${SCRIPT_ROOT}/common.sh" || exit 1 TYPE="coreos-buildbot" -ARCH=$(portageq envvar ARCH) -DEFAULT_SEED="builds/coreos-sdk/stage4-${ARCH}-latest.tar.bz2" -DEFAULT_PROFILE="coreos:default/linux/${ARCH}/10.0" FORCE_STAGES="stage4" . "${BUILD_LIBRARY_DIR}/catalyst.sh" || exit 1 diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 9885a08e01..eb84a2e05d 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -131,6 +131,8 @@ generate_au_zip () { local largs="-o ${BUILD_DIR}" test ! -d "${BUILD_DIR}" && mkdir -p "${BUILD_DIR}" info "Running ${lgenerateauzip} ${largs} for generating AU updater zip file" + # Make sure some vars this script needs are exported + export REPO_MANIFESTS_DIR SCRIPTS_DIR $lgenerateauzip $largs } diff --git a/build_library/catalyst.sh b/build_library/catalyst.sh index 600a9eff26..2a6ab7fb1c 100644 --- a/build_library/catalyst.sh +++ b/build_library/catalyst.sh @@ -5,20 +5,17 @@ # common.sh should be sourced first [[ -n "${DEFAULT_BUILD_ROOT}" ]] || exit 1 +. "${SCRIPTS_DIR}/sdk_lib/sdk_util.sh" # Default option values, may be provided before including this file : ${TYPE:="coreos-sdk"} : ${ARCH:=$(portageq envvar ARCH)} : ${DEFAULT_CATALYST_ROOT:="${DEFAULT_BUILD_ROOT}/catalyst"} -: ${DEFAULT_SEED:="builds/${TYPE}/stage4-${ARCH}-latest.tar.bz2"} +: ${DEFAULT_SEED:=${COREOS_SDK_TARBALL_PATH}} : ${DEFAULT_PROFILE:="coreos:default/linux/${ARCH}/10.0"} # Set to something like "stage4" to restrict what to build # FORCE_STAGES= -if [[ "${DEFAULT_SEED}" != /* ]]; then - DEFAULT_SEED="${DEFAULT_CATALYST_ROOT}/${DEFAULT_SEED}" -fi - # Values set in catalyst_init, don't use till after calling it CATALYST_ROOT= DEBUG= @@ -28,11 +25,6 @@ DISTDIR= TEMPDIR= STAGES= -# For searching for alternatives when DEFAULT_SEED doesn't exist -# unset SDK_SEARCH=1 to disable this fallback -SDK_TARBALL="coreos-sdk-${ARCH}-${COREOS_SDK_VERSION}.tar.bz2" -SDK_SEARCH=1 - DEFINE_string catalyst_root "${DEFAULT_CATALYST_ROOT}" \ "Path to directory for all catalyst images and other files." DEFINE_string portage_stable "${SRC_ROOT}/third_party/portage-stable" \ @@ -182,8 +174,10 @@ catalyst_init() { TEMPDIR="$CATALYST_ROOT/tmp/$TYPE" DISTDIR="$CATALYST_ROOT/distfiles" - # possibly search for existing seeds - search_for_sdk_seed + # automatically download the current SDK if it is the seed tarball. + if [[ "$FLAGS_seed_tarball" == "${COREOS_SDK_TARBALL_PATH}" ]]; then + sdk_download_tarball + fi # confirm seed exists if [[ ! -f "$FLAGS_seed_tarball" ]]; then @@ -211,31 +205,6 @@ catalyst_init() { fi } -# search_for_sdk_seed -# As a fallback search around for an existing SDK tarball we -# can use as a seed when the default doesn't exist. -search_for_sdk_seed() { - # Search disabled - [[ "${SDK_SEARCH}" != 1 ]] && return - # Seed already exists - [[ -f "${FLAGS_seed_tarball}" ]] && return - # User set the option so we shouldn't change it - [[ "${FLAGS_seed_tarball}" != "${DEFAULT_SEED}" ]] && return - - local check_path - for check_path in \ - "${CATALYST_ROOT}/builds/coreos-sdk/${SDK_TARBALL}" \ - "${CATALYST_ROOT}/builds/seeds/${SDK_TARBALL}" \ - "${REPO_CACHE_DIR}/sdks/${SDK_TARBALL}" \ - do - if [[ -f "${check_path}" ]]; then - info "Using SDK for seed: ${check_path}" - FLAGS_seed_tarball="${check_path}" - return - fi - done -} - write_configs() { # No catalyst config option, so defined via environment export CCACHE_DIR="$TEMPDIR/ccache" diff --git a/build_toolchains b/build_toolchains index 0b283b1b06..70bcda0b28 100755 --- a/build_toolchains +++ b/build_toolchains @@ -8,9 +8,6 @@ SCRIPT_ROOT=$(dirname $(readlink -f "$0")) . "${SCRIPT_ROOT}/common.sh" || exit 1 TYPE="coreos-toolchains" -ARCH=$(portageq envvar ARCH) -DEFAULT_SEED="builds/coreos-sdk/stage4-${ARCH}-latest.tar.bz2" -DEFAULT_PROFILE="coreos:default/linux/${ARCH}/10.0" FORCE_STAGES="stage4" . "${BUILD_LIBRARY_DIR}/catalyst.sh" || exit 1