#!/bin/bash # Copyright (c) 2013 The CoreOS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. SCRIPT_ROOT=$(dirname $(readlink -f "$0")) . "${SCRIPT_ROOT}/common.sh" || exit 1 DEFINE_boolean dry_run ${FLAGS_FALSE} "Trial run, makes no changes." DEFINE_boolean parallel ${FLAGS_TRUE} "Enable parallelism in gsutil." DEFINE_boolean upload ${FLAGS_FALSE} "Upload distfile mirror via gsutil." # FIXME(marineam): We need to add gs support to emirrordist so it # doesn't have to operate on a local copy of the complete mirror. DEFINE_boolean download ${FLAGS_FALSE} \ "Download the current mirror before making updates to it." MIRROR_ROOT="${DEFAULT_BUILD_ROOT}/mirror" UPLOAD_ROOT="gs://storage.core-os.net/mirror" SECOND_ROOT="gs://coreos-net-storage/mirror" # Parse flags FLAGS "$@" || exit 1 eval set -- "${FLAGS_ARGV}" switch_to_strict_mode if [[ $# -eq 0 ]]; then eval set -- portage-stable coreos fi GSUTIL_OPTS= if [[ ${FLAGS_parallel} -eq ${FLAGS_TRUE} ]]; then GSUTIL_OPTS="-m" fi EXIT_CODE=0 update_local_mirror() { local repo_name="$1" local repo_mirror="${MIRROR_ROOT}/$repo_name" local extra_flags="" if [[ ${FLAGS_dry_run} == ${FLAGS_TRUE} ]]; then info "Pretend distfiles update for $repo_name" extra_flags+=" --dry-run " else info "Starting distfiles update for $repo_name" fi rm -rf "${repo_mirror}/log" # clear old logs mkdir -p "${repo_mirror}/"{distfiles,info,log,tmp} emirrordist --mirror --verbose $extra_flags \ --jobs=${NUM_JOBS} --repo="${repo_name}" \ --distfiles="${repo_mirror}/distfiles" \ --distfiles-local="$(portageq envvar DISTDIR)" \ --fetch-log-dir="${repo_mirror}/log" \ --failure-log="${repo_mirror}/log/failure.log" \ --success-log="${repo_mirror}/log/success.log" \ --distfiles-db="${repo_mirror}/info/distfiles.db" \ --restrict-mirror-exemptions="gentoo" \ --temp-dir="${repo_mirror}/tmp" \ --verify-existing-digest if [[ ! -s "${repo_mirror}/log/failure.log" ]]; then info "Completed distfiles update for $repo_name without error" return fi # report what went wrong :( local lastpkg pkg file error while read pkg file error; do local log="${repo_mirror}/log/${file}.log" if [[ "${pkg}" != "${lastpkg}" ]]; then error "${pkg} failed:" lastpkg="${pkg}" fi error " ${file} ${error}" [[ -s "${log}" ]] && cat "${log}" done <"${repo_mirror}/log/failure.log" EXIT_CODE=1 } upload_mirror() { local repo_name="$1" local local_mirror="${MIRROR_ROOT}/$repo_name" local remote_mirror="$2/$repo_name" info "Uploading public distfiles for $repo_name" gsutil ${GSUTIL_OPTS} rsync -c \ "${local_mirror}/distfiles/" "${remote_mirror}/distfiles" info "Uploading private metadata for $repo_name" # uses cp instead of rsync in order to provide acl gsutil ${GSUTIL_OPTS} cp -a project-private \ "${local_mirror}/info/*" "${remote_mirror}/info" } download_mirror() { local repo_name="$1" local local_mirror="${MIRROR_ROOT}/$repo_name" local remote_mirror="${UPLOAD_ROOT}/$repo_name" info "Downloading public distfiles for $repo_name" mkdir -p "${local_mirror}/"{distfiles,info} gsutil ${GSUTIL_OPTS} rsync -c -d \ "${remote_mirror}/distfiles/" "${local_mirror}/distfiles" info "Downloading private metadata for $repo_name" gsutil ${GSUTIL_OPTS} rsync -c -d \ "${remote_mirror}/info/" "${local_mirror}/info" } if [[ ${FLAGS_download} -eq ${FLAGS_TRUE} ]]; then for repo in "$@"; do download_mirror "$repo" done fi for repo in "$@"; do if ! portageq get_repo_path / "$repo" >/dev/null; then die_notrace "Unknown repo name '$repo'" fi update_local_mirror "$repo" done if [[ ${FLAGS_dry_run} == ${FLAGS_TRUE} ]]; then info "Dry-run complete." exit fi if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]]; then for repo in "$@"; do upload_mirror "$repo" "$UPLOAD_ROOT" upload_mirror "$repo" "$SECOND_ROOT" done fi command_completed exit $EXIT_CODE