From 96186e3c47e0bc3a255c4599a8693b2bf01de4ec Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Wed, 26 Jun 2013 10:51:42 -0400 Subject: [PATCH 1/7] fix(update_distfiles): Add --download option to for incremental updates Temporary workaround until we have a version of emirrordist that can operate directly on remote storage. Download the entire mirror (if it isn't already local) so emirrordist can make an incremental update. Not great but it will do for now. --- update_distfiles | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/update_distfiles b/update_distfiles index a61ba63c02..8b2fc316a8 100755 --- a/update_distfiles +++ b/update_distfiles @@ -10,6 +10,11 @@ SCRIPT_ROOT=$(dirname $(readlink -f "$0")) DEFINE_boolean dry_run ${FLAGS_FALSE} "Trial run, makes no changes." DEFINE_boolean upload ${FLAGS_FALSE} "Upload distfile mirror via gsutil." +# FIXME(marineam): We need to add gs support to emirrordist so it +# doesn't have to operate on a local copy of the complete mirror. +DEFINE_boolean download ${FLAGS_FALSE} \ + "Download the current mirror before making updates to it." + MIRROR_ROOT="${DEFAULT_BUILD_ROOT}/mirror" UPLOAD_ROOT="gs://storage.core-os.net/mirror" @@ -62,6 +67,28 @@ upload_mirror() { gsutil cp -a project-private \ "${local_mirror}/info/*" "${remote_mirror}/info" } +download_mirror() { + local repo_name="$1" + local local_mirror="${MIRROR_ROOT}/$repo_name" + local remote_mirror="${UPLOAD_ROOT}/$repo_name" + + info "Downloading public distfiles for $repo_name" + mkdir -p "${local_mirror}/"{distfiles,info} + gsutil -m cp -n \ + "${remote_mirror}/distfiles/*" "${local_mirror}/distfiles" + + info "Downloading private metadata for $repo_name" + gsutil cp "${remote_mirror}/info/*" "${local_mirror}/info" +} + +if [[ ${FLAGS_download} -eq ${FLAGS_TRUE} ]]; then + if [[ ! -f "$HOME/.boto" ]]; then + die_notrace "Please run gsutil config to create ~/.boto" + fi + for repo in "$@"; do + download_mirror "$repo" + done +fi for repo in "$@"; do if ! portageq get_repo_path / "$repo" >/dev/null; then @@ -70,6 +97,11 @@ for repo in "$@"; do update_local_mirror "$repo" done +if [[ ${FLAGS_dry_run} == ${FLAGS_TRUE} ]]; then + info "Dry-run complete." + exit +fi + if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]]; then if [[ ! -f "$HOME/.boto" ]]; then die_notrace "Please run gsutil config to create ~/.boto" From 76ecff816998464a5906c383071f6312773bbacf Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Wed, 26 Jun 2013 10:58:04 -0400 Subject: [PATCH 2/7] fix(bootstrap_sdk): Rename final output tarball The name "coreos-sdk-amd64-..." makes much more sense for general distribution than "stage4..." so after catalyst is done rename the final tarball and fixup the DIGESTS file to refer to the new name. --- bootstrap_sdk | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/bootstrap_sdk b/bootstrap_sdk index 5f20dd929b..d064457e49 100755 --- a/bootstrap_sdk +++ b/bootstrap_sdk @@ -52,3 +52,26 @@ catalyst_stage_default catalyst_init "$@" catalyst_build + +info "Build complete! Changing output name to something more sensible." +build_name="stage4-${ARCH}-${FLAGS_version}.tar.bz2" +release_name="${TYPE}-${ARCH}-${FLAGS_version}.tar.bz2" +ln -f "$BUILDS/${build_name}" "$BUILDS/${release_name}" +ln -f "$BUILDS/${build_name}.CONTENTS" "$BUILDS/${release_name}.CONTENTS" +sed -e "s/${build_name}/${release_name}/" \ + "$BUILDS/${build_name}.DIGESTS" > "$BUILDS/${release_name}.DIGESTS" + +# Validate we didn't break the DIGESTS with sed +for hash_type in md5 sha1 sha512; do + info "Validating ${hash_type} DIGESTS" + # shash is what's used to generate these multi-hash digests but it + # doesn't exit with non-zero on failure. I mean seriously... + #shash -c "$BUILDS/${release_name}.DIGESTS" -a "${hash_type}" + # So we do it the hard way... + grep -qi "^# ${hash_type} HASH$" "$BUILDS/${release_name}.DIGESTS" + (cd "$BUILDS" && grep -A1 -i "^# ${hash_type} HASH$" \ + "${release_name}.DIGESTS" | grep -v '^--$' | \ + ${hash_type}sum -c - --strict) +done + +info "SDK ready: $BUILDS/${release_name}" From a25f12bbc4886c93d423ca3fa3e74907c6d6e4a6 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Wed, 26 Jun 2013 11:44:16 -0400 Subject: [PATCH 3/7] fix(update_distfiles): Add option to control gsutil's parallelism gsutil can be hard to follow when parallel upload/downloads are enabled. "I see it is transferring something, but what?" So this provides an option to disable that for debugging purposes. --- update_distfiles | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/update_distfiles b/update_distfiles index 8b2fc316a8..cda9e7f535 100755 --- a/update_distfiles +++ b/update_distfiles @@ -8,6 +8,7 @@ SCRIPT_ROOT=$(dirname $(readlink -f "$0")) . "${SCRIPT_ROOT}/common.sh" || exit 1 DEFINE_boolean dry_run ${FLAGS_FALSE} "Trial run, makes no changes." +DEFINE_boolean parallel ${FLAGS_TRUE} "Enable parallelism in gsutil." DEFINE_boolean upload ${FLAGS_FALSE} "Upload distfile mirror via gsutil." # FIXME(marineam): We need to add gs support to emirrordist so it @@ -27,6 +28,12 @@ if [[ $# -eq 0 ]]; then eval set -- portage-stable coreos fi +GSUTIL_OPTS= +if [[ ${FLAGS_parallel} -eq ${FLAGS_TRUE} ]]; then + GSUTIL_OPTS="-m" +fi + + update_local_mirror() { local repo_name="$1" local repo_mirror="${MIRROR_ROOT}/$repo_name" @@ -60,11 +67,11 @@ upload_mirror() { local remote_mirror="${UPLOAD_ROOT}/$repo_name" info "Uploading public distfiles for $repo_name" - gsutil -m cp -n \ + gsutil ${GSUTIL_OPTS} cp -n \ "${local_mirror}/distfiles/*" "${remote_mirror}/distfiles" info "Uploading private metadata for $repo_name" - gsutil cp -a project-private \ + gsutil ${GSUTIL_OPTS} cp -a project-private \ "${local_mirror}/info/*" "${remote_mirror}/info" } download_mirror() { @@ -74,11 +81,11 @@ download_mirror() { info "Downloading public distfiles for $repo_name" mkdir -p "${local_mirror}/"{distfiles,info} - gsutil -m cp -n \ + gsutil ${GSUTIL_OPTS} cp -n \ "${remote_mirror}/distfiles/*" "${local_mirror}/distfiles" info "Downloading private metadata for $repo_name" - gsutil cp "${remote_mirror}/info/*" "${local_mirror}/info" + gsutil ${GSUTIL_OPTS} cp "${remote_mirror}/info/*" "${local_mirror}/info" } if [[ ${FLAGS_download} -eq ${FLAGS_TRUE} ]]; then From deb3824deecaa7943fc5f4a5ddf42988dc4871a3 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Wed, 26 Jun 2013 12:01:47 -0400 Subject: [PATCH 4/7] fix(update_distfiles): Set temporary directory Provide an alternate directory for in-progress downloads to avoid uploading any by accident. --- update_distfiles | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/update_distfiles b/update_distfiles index cda9e7f535..488950e6b6 100755 --- a/update_distfiles +++ b/update_distfiles @@ -46,7 +46,7 @@ update_local_mirror() { info "Starting distfiles update for $repo_name" fi - mkdir -p "${repo_mirror}/"{distfiles,info,log} + mkdir -p "${repo_mirror}/"{distfiles,info,log,tmp} emirrordist --mirror --verbose $extra_flags \ --jobs=${NUM_JOBS} --repo="${repo_name}" \ --distfiles="${repo_mirror}/distfiles" \ @@ -59,6 +59,7 @@ update_local_mirror() { --distfiles-db="${repo_mirror}/info/distfiles.db" \ --deletion-delay=$((86400 * 14)) \ --restrict-mirror-exemptions="gentoo" \ + --temp-dir="${repo_mirror}/tmp" \ --verify-existing-digest } upload_mirror() { From 22de22b7eaf0e1723c223e3212ce35280ee1f6f1 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Wed, 26 Jun 2013 16:18:43 -0400 Subject: [PATCH 5/7] fix(bootstrap_sdk): Use the latest SDK tarball as a fallback seed. When running bootstrap_sdk for the first time on a host the default of using a previous catalyst run as a seed won't work but this may be a fresh SDK install so that tarball is probably around somewhere and will work as a default seed for most things. --- lib/catalyst.sh | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/lib/catalyst.sh b/lib/catalyst.sh index 73c195e256..a8e536cfd5 100644 --- a/lib/catalyst.sh +++ b/lib/catalyst.sh @@ -22,6 +22,12 @@ DISTDIR= TEMPDIR= STAGES= +# For searching for alternatives when DEFAULT_SEED doesn't exist +# unset SDK_SEARCH=1 to disable this fallback +SDK_VERSION_FILE="coreos/binhost/host/sdk_version.conf" +SDK_TARBALL_FMT="coreos-sdk-${ARCH}-%s.tar.bz2" +SDK_SEARCH=1 + DEFINE_string catalyst_root "${DEFAULT_CATALYST_ROOT}" \ "Path to directory for all catalyst images and other files." DEFINE_string portage_stable "${SRC_ROOT}/third_party/portage-stable" \ @@ -166,7 +172,10 @@ catalyst_init() { TEMPDIR="$CATALYST_ROOT/tmp/$TYPE" DISTDIR="$CATALYST_ROOT/distfiles" - # check for recent seed + # possibly search for existing seeds + search_for_sdk_seed + + # confirm seed exists if [[ ! -f "$FLAGS_seed_tarball" ]]; then die_notrace "Seed tarball not found: $FLAGS_seed_tarball" fi @@ -192,6 +201,36 @@ catalyst_init() { fi } +# search_for_sdk_seed +# As a fallback search around for an existing SDK tarball we +# can use as a seed when the default doesn't exist. +search_for_sdk_seed() { + # Search disabled + [[ "${SDK_SEARCH}" != 1 ]] && return + # Seed already exists + [[ -f "${FLAGS_seed_tarball}" ]] && return + # User set the option so we shouldn't change it + [[ "${FLAGS_seed_tarball}" != "${DEFAULT_SEED}" ]] && return + + local SDK_LATEST_VERSION SDK_TARBALL check_path + eval $(grep "^SDK_LATEST_VERSION=" \ + "${FLAGS_coreos_overlay}/${SDK_VERSION_FILE}") + SDK_TARBALL=$(printf "${SDK_TARBALL_FMT}" "${SDK_LATEST_VERSION}") + + for check_path in \ + "${CATALYST_ROOT}/builds/coreos-sdk/${SDK_TARBALL}" \ + "${CATALYST_ROOT}/builds/seeds/${SDK_TARBALL}" \ + "/var/cache/chromeos-cache/sdks/${SDK_TARBALL}" \ + "/mnt/host/source/.cache/sdks/${SDK_TARBALL}" + do + if [[ -f "${check_path}" ]]; then + info "Using SDK for seed: ${check_path}" + FLAGS_seed_tarball="${check_path}" + return + fi + done +} + write_configs() { # No catalyst config option, so defined via environment export CCACHE_DIR="$TEMPDIR/ccache" From 82b7949c5f97f16784a3d3326875e101a86a3180 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Wed, 26 Jun 2013 17:03:57 -0400 Subject: [PATCH 6/7] fix(bootstrap_sdk): Add option to upload built SDK and packages This automates the process of uploading the fresh SDK release. --- bootstrap_sdk | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/bootstrap_sdk b/bootstrap_sdk index d064457e49..189cd689b3 100755 --- a/bootstrap_sdk +++ b/bootstrap_sdk @@ -36,9 +36,14 @@ SCRIPT_ROOT=$(dirname $(readlink -f "$0")) TYPE="coreos-sdk" ARCH=$(portageq envvar ARCH) DEFAULT_PROFILE="coreos:default/linux/${ARCH}/10.0" +UPLOAD_ROOT="gs://storage.core-os.net/coreos/sdk/${ARCH}" . "${SCRIPT_ROOT}/lib/catalyst.sh" || exit 1 +DEFINE_boolean parallel ${FLAGS_TRUE} "Enable parallelism in gsutil." +DEFINE_boolean upload ${FLAGS_FALSE} \ + "Upload final tarball and all packages via gsutil." + ## Define the stage4 config template catalyst_stage4() { cat < Date: Wed, 26 Jun 2013 18:41:52 -0400 Subject: [PATCH 7/7] fix(enter_chroot): Use .boto from the user's HOME We don't have any private overlays but users will likely have their own credentials in ~/.boto so make it available in the chroot. --- sdk_lib/enter_chroot.sh | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/sdk_lib/enter_chroot.sh b/sdk_lib/enter_chroot.sh index 128fdeba0c..d4d421be44 100755 --- a/sdk_lib/enter_chroot.sh +++ b/sdk_lib/enter_chroot.sh @@ -83,6 +83,8 @@ FILES_TO_COPY_TO_CHROOT=( .gdata_token # Auth token for Google Docs on chromium.org .disable_build_stats_upload # Presence of file disables command stats upload .netrc # May contain required source fetching credentials + .boto # Auth information for gsutil + .boto-key.p12 # Service account key for gsutil ) INNER_CHROME_ROOT=$FLAGS_chrome_root_mount # inside chroot @@ -434,18 +436,6 @@ setup_env() { # semaphores. chmod -R 777 "${FLAGS_chroot}/dev/shm" - # If the private overlays are installed, gsutil can use those credentials. - # We're also installing credentials for use by sudoed invocations. - boto='src/private-overlays/coreos-overlay/googlestorage_account.boto' - if [ -s "${FLAGS_trunk}/${boto}" ]; then - if [ ! -L "${FLAGS_chroot}/home/${SUDO_USER}/.boto" ]; then - user_symlink "trunk/${boto}" "${FLAGS_chroot}/home/${SUDO_USER}/.boto" - fi - if [ ! -L "${FLAGS_chroot}/root/.boto" ]; then - ln -sf "${CHROOT_TRUNK_DIR}/${boto}" "${FLAGS_chroot}/root/.boto" - fi - fi - # Have found a few chroots where ~/.gsutil is owned by root:root, probably # as a result of old gsutil or tools. This causes permission errors when # gsutil cp tries to create its cache files, so ensure the user can