mirror of
https://github.com/flatcar/scripts.git
synced 2025-09-23 06:31:18 +02:00
441ced7498
If files cannot be mirrored due to problems fetching or missing digests report that loudly so we are actually aware of it and can fix it.
139 lines
4.1 KiB
Bash
Executable File
139 lines
4.1 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Copyright (c) 2013 The CoreOS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
|
|
. "${SCRIPT_ROOT}/common.sh" || exit 1
|
|
|
|
DEFINE_boolean dry_run ${FLAGS_FALSE} "Trial run, makes no changes."
|
|
DEFINE_boolean parallel ${FLAGS_TRUE} "Enable parallelism in gsutil."
|
|
DEFINE_boolean upload ${FLAGS_FALSE} "Upload distfile mirror via gsutil."
|
|
|
|
# FIXME(marineam): We need to add gs support to emirrordist so it
|
|
# doesn't have to operate on a local copy of the complete mirror.
|
|
DEFINE_boolean download ${FLAGS_FALSE} \
|
|
"Download the current mirror before making updates to it."
|
|
|
|
MIRROR_ROOT="${DEFAULT_BUILD_ROOT}/mirror"
|
|
UPLOAD_ROOT="gs://storage.core-os.net/mirror"
|
|
|
|
# Parse flags
|
|
FLAGS "$@" || exit 1
|
|
eval set -- "${FLAGS_ARGV}"
|
|
switch_to_strict_mode
|
|
|
|
if [[ $# -eq 0 ]]; then
|
|
eval set -- portage-stable coreos
|
|
fi
|
|
|
|
GSUTIL_OPTS=
|
|
if [[ ${FLAGS_parallel} -eq ${FLAGS_TRUE} ]]; then
|
|
GSUTIL_OPTS="-m"
|
|
fi
|
|
|
|
EXIT_CODE=0
|
|
|
|
|
|
update_local_mirror() {
|
|
local repo_name="$1"
|
|
local repo_mirror="${MIRROR_ROOT}/$repo_name"
|
|
local extra_flags=""
|
|
|
|
if [[ ${FLAGS_dry_run} == ${FLAGS_TRUE} ]]; then
|
|
info "Pretend distfiles update for $repo_name"
|
|
extra_flags+=" --dry-run "
|
|
else
|
|
info "Starting distfiles update for $repo_name"
|
|
fi
|
|
|
|
rm -rf "${repo_mirror}/log" # clear old logs
|
|
mkdir -p "${repo_mirror}/"{distfiles,info,log,tmp}
|
|
emirrordist --mirror --verbose $extra_flags \
|
|
--jobs=${NUM_JOBS} --repo="${repo_name}" \
|
|
--distfiles="${repo_mirror}/distfiles" \
|
|
--distfiles-local="$(portageq envvar DISTDIR)" \
|
|
--fetch-log-dir="${repo_mirror}/log" \
|
|
--failure-log="${repo_mirror}/log/failure.log" \
|
|
--success-log="${repo_mirror}/log/success.log" \
|
|
--distfiles-db="${repo_mirror}/info/distfiles.db" \
|
|
--restrict-mirror-exemptions="gentoo" \
|
|
--temp-dir="${repo_mirror}/tmp" \
|
|
--verify-existing-digest
|
|
|
|
if [[ ! -s "${repo_mirror}/log/failure.log" ]]; then
|
|
info "Completed distfiles update for $repo_name without error"
|
|
return
|
|
fi
|
|
|
|
# report what went wrong :(
|
|
local lastpkg pkg file error
|
|
while read pkg file error; do
|
|
local log="${repo_mirror}/log/${file}.log"
|
|
if [[ "${pkg}" != "${lastpkg}" ]]; then
|
|
error "${pkg} failed:"
|
|
lastpkg="${pkg}"
|
|
fi
|
|
error " ${file} ${error}"
|
|
[[ -s "${log}" ]] && cat "${log}"
|
|
done <"${repo_mirror}/log/failure.log"
|
|
|
|
EXIT_CODE=1
|
|
}
|
|
upload_mirror() {
|
|
local repo_name="$1"
|
|
local local_mirror="${MIRROR_ROOT}/$repo_name"
|
|
local remote_mirror="${UPLOAD_ROOT}/$repo_name"
|
|
|
|
info "Uploading public distfiles for $repo_name"
|
|
gsutil ${GSUTIL_OPTS} rsync -c \
|
|
"${local_mirror}/distfiles/" "${remote_mirror}/distfiles"
|
|
|
|
info "Uploading private metadata for $repo_name"
|
|
# uses cp instead of rsync in order to provide acl
|
|
gsutil ${GSUTIL_OPTS} cp -a project-private \
|
|
"${local_mirror}/info/*" "${remote_mirror}/info"
|
|
}
|
|
download_mirror() {
|
|
local repo_name="$1"
|
|
local local_mirror="${MIRROR_ROOT}/$repo_name"
|
|
local remote_mirror="${UPLOAD_ROOT}/$repo_name"
|
|
|
|
info "Downloading public distfiles for $repo_name"
|
|
mkdir -p "${local_mirror}/"{distfiles,info}
|
|
gsutil ${GSUTIL_OPTS} rsync -c -d \
|
|
"${remote_mirror}/distfiles/" "${local_mirror}/distfiles"
|
|
|
|
info "Downloading private metadata for $repo_name"
|
|
gsutil ${GSUTIL_OPTS} rsync -c -d \
|
|
"${remote_mirror}/info/" "${local_mirror}/info"
|
|
}
|
|
|
|
if [[ ${FLAGS_download} -eq ${FLAGS_TRUE} ]]; then
|
|
for repo in "$@"; do
|
|
download_mirror "$repo"
|
|
done
|
|
fi
|
|
|
|
for repo in "$@"; do
|
|
if ! portageq get_repo_path / "$repo" >/dev/null; then
|
|
die_notrace "Unknown repo name '$repo'"
|
|
fi
|
|
update_local_mirror "$repo"
|
|
done
|
|
|
|
if [[ ${FLAGS_dry_run} == ${FLAGS_TRUE} ]]; then
|
|
info "Dry-run complete."
|
|
exit
|
|
fi
|
|
|
|
if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]]; then
|
|
for repo in "$@"; do
|
|
upload_mirror "$repo"
|
|
done
|
|
fi
|
|
|
|
command_completed
|
|
exit $EXIT_CODE
|