diff --git a/update_distfiles b/update_distfiles index 0eceea8b92..1ce8ad5310 100755 --- a/update_distfiles +++ b/update_distfiles @@ -33,6 +33,8 @@ if [[ ${FLAGS_parallel} -eq ${FLAGS_TRUE} ]]; then GSUTIL_OPTS="-m" fi +EXIT_CODE=0 + update_local_mirror() { local repo_name="$1" @@ -46,6 +48,7 @@ update_local_mirror() { info "Starting distfiles update for $repo_name" fi + rm -rf "${repo_mirror}/log" # clear old logs mkdir -p "${repo_mirror}/"{distfiles,info,log,tmp} emirrordist --mirror --verbose $extra_flags \ --jobs=${NUM_JOBS} --repo="${repo_name}" \ @@ -54,13 +57,29 @@ update_local_mirror() { --fetch-log-dir="${repo_mirror}/log" \ --failure-log="${repo_mirror}/log/failure.log" \ --success-log="${repo_mirror}/log/success.log" \ - --scheduled-deletion-log="${repo_mirror}/log/deletion.log" \ - --deletion-db="${repo_mirror}/info/deletion.db" \ --distfiles-db="${repo_mirror}/info/distfiles.db" \ - --deletion-delay=$((86400 * 14)) \ --restrict-mirror-exemptions="gentoo" \ --temp-dir="${repo_mirror}/tmp" \ --verify-existing-digest + + if [[ ! -s "${repo_mirror}/log/failure.log" ]]; then + info "Completed distfiles update for $repo_name without error" + return + fi + + # report what went wrong :( + local lastpkg pkg file error + while read pkg file error; do + local log="${repo_mirror}/log/${file}.log" + if [[ "${pkg}" != "${lastpkg}" ]]; then + error "${pkg} failed:" + lastpkg="${pkg}" + fi + error " ${file} ${error}" + [[ -s "${log}" ]] && cat "${log}" + done <"${repo_mirror}/log/failure.log" + + EXIT_CODE=1 } upload_mirror() { local repo_name="$1" @@ -68,10 +87,11 @@ upload_mirror() { local remote_mirror="${UPLOAD_ROOT}/$repo_name" info "Uploading public distfiles for $repo_name" - gsutil ${GSUTIL_OPTS} cp -n \ - "${local_mirror}/distfiles/*" "${remote_mirror}/distfiles" + gsutil ${GSUTIL_OPTS} rsync -c \ + "${local_mirror}/distfiles/" "${remote_mirror}/distfiles" info "Uploading private metadata for $repo_name" + # uses cp instead of rsync in order to provide acl gsutil ${GSUTIL_OPTS} cp -a project-private \ "${local_mirror}/info/*" "${remote_mirror}/info" } @@ -82,17 +102,15 @@ download_mirror() { info "Downloading public distfiles for $repo_name" mkdir -p "${local_mirror}/"{distfiles,info} - gsutil ${GSUTIL_OPTS} cp -n \ - "${remote_mirror}/distfiles/*" "${local_mirror}/distfiles" + gsutil ${GSUTIL_OPTS} rsync -c -d \ + "${remote_mirror}/distfiles/" "${local_mirror}/distfiles" info "Downloading private metadata for $repo_name" - gsutil ${GSUTIL_OPTS} cp "${remote_mirror}/info/*" "${local_mirror}/info" + gsutil ${GSUTIL_OPTS} rsync -c -d \ + "${remote_mirror}/info/" "${local_mirror}/info" } if [[ ${FLAGS_download} -eq ${FLAGS_TRUE} ]]; then - if [[ ! -f "$HOME/.boto" ]]; then - die_notrace "Please run gsutil config to create ~/.boto" - fi for repo in "$@"; do download_mirror "$repo" done @@ -111,12 +129,10 @@ if [[ ${FLAGS_dry_run} == ${FLAGS_TRUE} ]]; then fi if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]]; then - if [[ ! -f "$HOME/.boto" ]]; then - die_notrace "Please run gsutil config to create ~/.boto" - fi for repo in "$@"; do upload_mirror "$repo" done fi -info "Done!" +command_completed +exit $EXIT_CODE