Merge pull request #510 from marineam/dist

Properly report errors from update_distfiles and other cleanups
This commit is contained in:
Michael Marineau 2016-03-09 20:25:41 -08:00
commit 4744e3f96c

View File

@ -33,6 +33,8 @@ if [[ ${FLAGS_parallel} -eq ${FLAGS_TRUE} ]]; then
GSUTIL_OPTS="-m"
fi
EXIT_CODE=0
update_local_mirror() {
local repo_name="$1"
@ -46,6 +48,7 @@ update_local_mirror() {
info "Starting distfiles update for $repo_name"
fi
rm -rf "${repo_mirror}/log" # clear old logs
mkdir -p "${repo_mirror}/"{distfiles,info,log,tmp}
emirrordist --mirror --verbose $extra_flags \
--jobs=${NUM_JOBS} --repo="${repo_name}" \
@ -54,13 +57,29 @@ update_local_mirror() {
--fetch-log-dir="${repo_mirror}/log" \
--failure-log="${repo_mirror}/log/failure.log" \
--success-log="${repo_mirror}/log/success.log" \
--scheduled-deletion-log="${repo_mirror}/log/deletion.log" \
--deletion-db="${repo_mirror}/info/deletion.db" \
--distfiles-db="${repo_mirror}/info/distfiles.db" \
--deletion-delay=$((86400 * 14)) \
--restrict-mirror-exemptions="gentoo" \
--temp-dir="${repo_mirror}/tmp" \
--verify-existing-digest
if [[ ! -s "${repo_mirror}/log/failure.log" ]]; then
info "Completed distfiles update for $repo_name without error"
return
fi
# report what went wrong :(
local lastpkg pkg file error
while read pkg file error; do
local log="${repo_mirror}/log/${file}.log"
if [[ "${pkg}" != "${lastpkg}" ]]; then
error "${pkg} failed:"
lastpkg="${pkg}"
fi
error " ${file} ${error}"
[[ -s "${log}" ]] && cat "${log}"
done <"${repo_mirror}/log/failure.log"
EXIT_CODE=1
}
upload_mirror() {
local repo_name="$1"
@ -68,10 +87,11 @@ upload_mirror() {
local remote_mirror="${UPLOAD_ROOT}/$repo_name"
info "Uploading public distfiles for $repo_name"
gsutil ${GSUTIL_OPTS} cp -n \
"${local_mirror}/distfiles/*" "${remote_mirror}/distfiles"
gsutil ${GSUTIL_OPTS} rsync -c \
"${local_mirror}/distfiles/" "${remote_mirror}/distfiles"
info "Uploading private metadata for $repo_name"
# uses cp instead of rsync in order to provide acl
gsutil ${GSUTIL_OPTS} cp -a project-private \
"${local_mirror}/info/*" "${remote_mirror}/info"
}
@ -82,17 +102,15 @@ download_mirror() {
info "Downloading public distfiles for $repo_name"
mkdir -p "${local_mirror}/"{distfiles,info}
gsutil ${GSUTIL_OPTS} cp -n \
"${remote_mirror}/distfiles/*" "${local_mirror}/distfiles"
gsutil ${GSUTIL_OPTS} rsync -c -d \
"${remote_mirror}/distfiles/" "${local_mirror}/distfiles"
info "Downloading private metadata for $repo_name"
gsutil ${GSUTIL_OPTS} cp "${remote_mirror}/info/*" "${local_mirror}/info"
gsutil ${GSUTIL_OPTS} rsync -c -d \
"${remote_mirror}/info/" "${local_mirror}/info"
}
if [[ ${FLAGS_download} -eq ${FLAGS_TRUE} ]]; then
if [[ ! -f "$HOME/.boto" ]]; then
die_notrace "Please run gsutil config to create ~/.boto"
fi
for repo in "$@"; do
download_mirror "$repo"
done
@ -111,12 +129,10 @@ if [[ ${FLAGS_dry_run} == ${FLAGS_TRUE} ]]; then
fi
if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]]; then
if [[ ! -f "$HOME/.boto" ]]; then
die_notrace "Please run gsutil config to create ~/.boto"
fi
for repo in "$@"; do
upload_mirror "$repo"
done
fi
info "Done!"
command_completed
exit $EXIT_CODE