flatcar-scripts/update_distfiles

#!/bin/bash

# Copyright (c) 2013 The CoreOS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
. "${SCRIPT_ROOT}/common.sh" || exit 1

DEFINE_boolean dry_run ${FLAGS_FALSE} "Trial run, makes no changes."
DEFINE_boolean parallel ${FLAGS_TRUE} "Enable parallelism in gsutil."
DEFINE_boolean upload ${FLAGS_FALSE} "Upload distfile mirror via gsutil."

# FIXME(marineam): We need to add gs support to emirrordist so it
# doesn't have to operate on a local copy of the complete mirror.
DEFINE_boolean download ${FLAGS_FALSE} \
    "Download the current mirror before making updates to it."

MIRROR_ROOT="${DEFAULT_BUILD_ROOT}/mirror"
UPLOAD_ROOT="gs://storage.core-os.net/mirror"
SECOND_ROOT="gs://coreos-net-storage/mirror"

# Parse flags
FLAGS "$@" || exit 1
eval set -- "${FLAGS_ARGV}"
switch_to_strict_mode

if [[ $# -eq 0 ]]; then
    eval set -- portage-stable coreos-overlay
fi

GSUTIL_OPTS=
if [[ ${FLAGS_parallel} -eq ${FLAGS_TRUE} ]]; then
    GSUTIL_OPTS="-m"
fi

EXIT_CODE=0


update_local_mirror() {
    local repo_name="$1"
    local repo_mirror="${MIRROR_ROOT}/$repo_name"
    local extra_flags=""

    if [[ ${FLAGS_dry_run} == ${FLAGS_TRUE} ]]; then
        info "Pretend distfiles update for $repo_name"
        extra_flags+=" --dry-run "
    else
        info "Starting distfiles update for $repo_name"
    fi

    rm -rf "${repo_mirror}/log" # clear old logs
    mkdir -p "${repo_mirror}/"{distfiles,info,log,tmp}
    emirrordist --mirror --verbose $extra_flags \
        --jobs=${NUM_JOBS} --repo="${repo_name}" \
        --distfiles="${repo_mirror}/distfiles" \
        --distfiles-local="$(portageq envvar DISTDIR)" \
        --fetch-log-dir="${repo_mirror}/log" \
        --failure-log="${repo_mirror}/log/failure.log" \
        --success-log="${repo_mirror}/log/success.log" \
        --distfiles-db="${repo_mirror}/info/distfiles.db" \
        --restrict-mirror-exemptions="gentoo" \
        --temp-dir="${repo_mirror}/tmp" \
        --verify-existing-digest

    if [[ ! -s "${repo_mirror}/log/failure.log" ]]; then
        info "Completed distfiles update for $repo_name without error"
        return
    fi

    # report what went wrong :(
    local lastpkg pkg file error
    while read pkg file error; do
        local log="${repo_mirror}/log/${file}.log"
        if [[ "${pkg}" != "${lastpkg}" ]]; then
            error "${pkg} failed:"
            lastpkg="${pkg}"
        fi
        error " ${file} ${error}"
        [[ -s "${log}" ]] && cat "${log}"
    done <"${repo_mirror}/log/failure.log"

    EXIT_CODE=1
}
upload_mirror() {
    local repo_name="$1"
    local local_mirror="${MIRROR_ROOT}/$repo_name"
    local remote_mirror="$2/$repo_name"

    info "Uploading public distfiles for $repo_name"
    gsutil ${GSUTIL_OPTS} rsync -c \
        "${local_mirror}/distfiles/" "${remote_mirror}/distfiles"

    info "Uploading private metadata for $repo_name"
    # uses cp instead of rsync in order to provide acl
    gsutil ${GSUTIL_OPTS} cp -a project-private \
        "${local_mirror}/info/*" "${remote_mirror}/info"
}
download_mirror() {
    local repo_name="$1"
    local local_mirror="${MIRROR_ROOT}/$repo_name"
    local remote_mirror="${UPLOAD_ROOT}/$repo_name"

    info "Downloading public distfiles for $repo_name"
    mkdir -p "${local_mirror}/"{distfiles,info}
    gsutil ${GSUTIL_OPTS} rsync -c -d \
        "${remote_mirror}/distfiles/" "${local_mirror}/distfiles"

    info "Downloading private metadata for $repo_name"
    gsutil ${GSUTIL_OPTS} rsync -c -d \
        "${remote_mirror}/info/" "${local_mirror}/info"
}

if [[ ${FLAGS_download} -eq ${FLAGS_TRUE} ]]; then
    for repo in "$@"; do
        download_mirror "$repo"
    done
fi

for repo in "$@"; do
    if ! portageq get_repo_path / "$repo" >/dev/null; then
        die_notrace "Unknown repo name '$repo'"
    fi
    update_local_mirror "$repo"
done

if [[ ${FLAGS_dry_run} == ${FLAGS_TRUE} ]]; then
    info "Dry-run complete."
    exit
fi

if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]]; then
    for repo in "$@"; do
        upload_mirror "$repo" "$UPLOAD_ROOT"
        upload_mirror "$repo" "$SECOND_ROOT"
    done
fi

command_completed
exit $EXIT_CODE