mirror of
https://github.com/flatcar/scripts.git
synced 2025-08-06 20:47:00 +02:00
141 lines
4.2 KiB
Bash
Executable File
141 lines
4.2 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Copyright (c) 2013 The CoreOS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
|
|
. "${SCRIPT_ROOT}/common.sh" || exit 1
|
|
|
|
DEFINE_boolean dry_run ${FLAGS_FALSE} "Trial run, makes no changes."
|
|
DEFINE_boolean parallel ${FLAGS_TRUE} "Enable parallelism in gsutil."
|
|
DEFINE_boolean upload ${FLAGS_FALSE} "Upload distfile mirror via gsutil."
|
|
|
|
# FIXME(marineam): We need to add gs support to emirrordist so it
|
|
# doesn't have to operate on a local copy of the complete mirror.
|
|
DEFINE_boolean download ${FLAGS_FALSE} \
|
|
"Download the current mirror before making updates to it."
|
|
|
|
MIRROR_ROOT="${DEFAULT_BUILD_ROOT}/mirror"
|
|
UPLOAD_ROOT="gs://storage.core-os.net/mirror"
|
|
SECOND_ROOT="gs://coreos-net-storage/mirror"
|
|
|
|
# Parse flags
|
|
FLAGS "$@" || exit 1
|
|
eval set -- "${FLAGS_ARGV}"
|
|
switch_to_strict_mode
|
|
|
|
if [[ $# -eq 0 ]]; then
|
|
eval set -- portage-stable coreos
|
|
fi
|
|
|
|
GSUTIL_OPTS=
|
|
if [[ ${FLAGS_parallel} -eq ${FLAGS_TRUE} ]]; then
|
|
GSUTIL_OPTS="-m"
|
|
fi
|
|
|
|
EXIT_CODE=0
|
|
|
|
|
|
update_local_mirror() {
|
|
local repo_name="$1"
|
|
local repo_mirror="${MIRROR_ROOT}/$repo_name"
|
|
local extra_flags=""
|
|
|
|
if [[ ${FLAGS_dry_run} == ${FLAGS_TRUE} ]]; then
|
|
info "Pretend distfiles update for $repo_name"
|
|
extra_flags+=" --dry-run "
|
|
else
|
|
info "Starting distfiles update for $repo_name"
|
|
fi
|
|
|
|
rm -rf "${repo_mirror}/log" # clear old logs
|
|
mkdir -p "${repo_mirror}/"{distfiles,info,log,tmp}
|
|
emirrordist --mirror --verbose $extra_flags \
|
|
--jobs=${NUM_JOBS} --repo="${repo_name}" \
|
|
--distfiles="${repo_mirror}/distfiles" \
|
|
--distfiles-local="$(portageq envvar DISTDIR)" \
|
|
--fetch-log-dir="${repo_mirror}/log" \
|
|
--failure-log="${repo_mirror}/log/failure.log" \
|
|
--success-log="${repo_mirror}/log/success.log" \
|
|
--distfiles-db="${repo_mirror}/info/distfiles.db" \
|
|
--restrict-mirror-exemptions="gentoo" \
|
|
--temp-dir="${repo_mirror}/tmp" \
|
|
--verify-existing-digest
|
|
|
|
if [[ ! -s "${repo_mirror}/log/failure.log" ]]; then
|
|
info "Completed distfiles update for $repo_name without error"
|
|
return
|
|
fi
|
|
|
|
# report what went wrong :(
|
|
local lastpkg pkg file error
|
|
while read pkg file error; do
|
|
local log="${repo_mirror}/log/${file}.log"
|
|
if [[ "${pkg}" != "${lastpkg}" ]]; then
|
|
error "${pkg} failed:"
|
|
lastpkg="${pkg}"
|
|
fi
|
|
error " ${file} ${error}"
|
|
[[ -s "${log}" ]] && cat "${log}"
|
|
done <"${repo_mirror}/log/failure.log"
|
|
|
|
EXIT_CODE=1
|
|
}
|
|
upload_mirror() {
|
|
local repo_name="$1"
|
|
local local_mirror="${MIRROR_ROOT}/$repo_name"
|
|
local remote_mirror="$2/$repo_name"
|
|
|
|
info "Uploading public distfiles for $repo_name"
|
|
gsutil ${GSUTIL_OPTS} rsync -c \
|
|
"${local_mirror}/distfiles/" "${remote_mirror}/distfiles"
|
|
|
|
info "Uploading private metadata for $repo_name"
|
|
# uses cp instead of rsync in order to provide acl
|
|
gsutil ${GSUTIL_OPTS} cp -a project-private \
|
|
"${local_mirror}/info/*" "${remote_mirror}/info"
|
|
}
|
|
download_mirror() {
|
|
local repo_name="$1"
|
|
local local_mirror="${MIRROR_ROOT}/$repo_name"
|
|
local remote_mirror="${UPLOAD_ROOT}/$repo_name"
|
|
|
|
info "Downloading public distfiles for $repo_name"
|
|
mkdir -p "${local_mirror}/"{distfiles,info}
|
|
gsutil ${GSUTIL_OPTS} rsync -c -d \
|
|
"${remote_mirror}/distfiles/" "${local_mirror}/distfiles"
|
|
|
|
info "Downloading private metadata for $repo_name"
|
|
gsutil ${GSUTIL_OPTS} rsync -c -d \
|
|
"${remote_mirror}/info/" "${local_mirror}/info"
|
|
}
|
|
|
|
if [[ ${FLAGS_download} -eq ${FLAGS_TRUE} ]]; then
|
|
for repo in "$@"; do
|
|
download_mirror "$repo"
|
|
done
|
|
fi
|
|
|
|
for repo in "$@"; do
|
|
if ! portageq get_repo_path / "$repo" >/dev/null; then
|
|
die_notrace "Unknown repo name '$repo'"
|
|
fi
|
|
update_local_mirror "$repo"
|
|
done
|
|
|
|
if [[ ${FLAGS_dry_run} == ${FLAGS_TRUE} ]]; then
|
|
info "Dry-run complete."
|
|
exit
|
|
fi
|
|
|
|
if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]]; then
|
|
for repo in "$@"; do
|
|
upload_mirror "$repo" "$UPLOAD_ROOT"
|
|
upload_mirror "$repo" "$SECOND_ROOT"
|
|
done
|
|
fi
|
|
|
|
command_completed
|
|
exit $EXIT_CODE
|