From ff29c10f7ac7a1398ecb7bbdc7f48d0a47a9c13b Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Wed, 9 Mar 2016 13:10:55 -0800 Subject: [PATCH 1/4] update_distfiles: remove old .boto file check The config may be located other places these days. --- update_distfiles | 6 ------ 1 file changed, 6 deletions(-) diff --git a/update_distfiles b/update_distfiles index 0eceea8b92..4bcc56e487 100755 --- a/update_distfiles +++ b/update_distfiles @@ -90,9 +90,6 @@ download_mirror() { } if [[ ${FLAGS_download} -eq ${FLAGS_TRUE} ]]; then - if [[ ! -f "$HOME/.boto" ]]; then - die_notrace "Please run gsutil config to create ~/.boto" - fi for repo in "$@"; do download_mirror "$repo" done @@ -111,9 +108,6 @@ if [[ ${FLAGS_dry_run} == ${FLAGS_TRUE} ]]; then fi if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]]; then - if [[ ! -f "$HOME/.boto" ]]; then - die_notrace "Please run gsutil config to create ~/.boto" - fi for repo in "$@"; do upload_mirror "$repo" done From 3a66dc83ce480439cbd4d77da191465cbd3fc11c Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Wed, 9 Mar 2016 13:17:47 -0800 Subject: [PATCH 2/4] update_distfiles: use gsutil rsync instead of cp --- update_distfiles | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/update_distfiles b/update_distfiles index 4bcc56e487..d497fda49c 100755 --- a/update_distfiles +++ b/update_distfiles @@ -68,10 +68,11 @@ upload_mirror() { local remote_mirror="${UPLOAD_ROOT}/$repo_name" info "Uploading public distfiles for $repo_name" - gsutil ${GSUTIL_OPTS} cp -n \ - "${local_mirror}/distfiles/*" "${remote_mirror}/distfiles" + gsutil ${GSUTIL_OPTS} rsync -c \ + "${local_mirror}/distfiles/" "${remote_mirror}/distfiles" info "Uploading private metadata for $repo_name" + # uses cp instead of rsync in order to provide acl gsutil ${GSUTIL_OPTS} cp -a project-private \ "${local_mirror}/info/*" "${remote_mirror}/info" } @@ -82,11 +83,12 @@ download_mirror() { info "Downloading public distfiles for $repo_name" mkdir -p "${local_mirror}/"{distfiles,info} - gsutil ${GSUTIL_OPTS} cp -n \ - "${remote_mirror}/distfiles/*" "${local_mirror}/distfiles" + gsutil ${GSUTIL_OPTS} rsync -c -d \ + "${remote_mirror}/distfiles/" "${local_mirror}/distfiles" info "Downloading private metadata for $repo_name" - gsutil ${GSUTIL_OPTS} cp "${remote_mirror}/info/*" "${local_mirror}/info" + gsutil ${GSUTIL_OPTS} rsync -c -d \ + "${remote_mirror}/info/" "${local_mirror}/info" } if [[ ${FLAGS_download} -eq ${FLAGS_TRUE} ]]; then From accd254c85dfe52fa513c5d4f15fede79c155106 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Wed, 9 Mar 2016 16:27:29 -0800 Subject: [PATCH 3/4] update_distfiles: remove deletion options We have never actually deleted anything from our mirror since gsutil cp didn't really have a way to do that and the for some unknown reason emirrordist never tried to delete anything anyway. Additionally we actually don't want to delete anything from our mirror to ensure that the source remains available for old releases. --- update_distfiles | 3 --- 1 file changed, 3 deletions(-) diff --git a/update_distfiles b/update_distfiles index d497fda49c..8ae2160fef 100755 --- a/update_distfiles +++ b/update_distfiles @@ -54,10 +54,7 @@ update_local_mirror() { --fetch-log-dir="${repo_mirror}/log" \ --failure-log="${repo_mirror}/log/failure.log" \ --success-log="${repo_mirror}/log/success.log" \ - --scheduled-deletion-log="${repo_mirror}/log/deletion.log" \ - --deletion-db="${repo_mirror}/info/deletion.db" \ --distfiles-db="${repo_mirror}/info/distfiles.db" \ - --deletion-delay=$((86400 * 14)) \ --restrict-mirror-exemptions="gentoo" \ --temp-dir="${repo_mirror}/tmp" \ --verify-existing-digest From 441ced749830d6f8a9be494cafe6d663d0d7f9cb Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Wed, 9 Mar 2016 15:37:46 -0800 Subject: [PATCH 4/4] update_distfiles: report errors properly If files cannot be mirrored due to problems fetching or missing digests report that loudly so we are actually aware of it and can fix it. --- update_distfiles | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/update_distfiles b/update_distfiles index 8ae2160fef..1ce8ad5310 100755 --- a/update_distfiles +++ b/update_distfiles @@ -33,6 +33,8 @@ if [[ ${FLAGS_parallel} -eq ${FLAGS_TRUE} ]]; then GSUTIL_OPTS="-m" fi +EXIT_CODE=0 + update_local_mirror() { local repo_name="$1" @@ -46,6 +48,7 @@ update_local_mirror() { info "Starting distfiles update for $repo_name" fi + rm -rf "${repo_mirror}/log" # clear old logs mkdir -p "${repo_mirror}/"{distfiles,info,log,tmp} emirrordist --mirror --verbose $extra_flags \ --jobs=${NUM_JOBS} --repo="${repo_name}" \ @@ -58,6 +61,25 @@ update_local_mirror() { --restrict-mirror-exemptions="gentoo" \ --temp-dir="${repo_mirror}/tmp" \ --verify-existing-digest + + if [[ ! -s "${repo_mirror}/log/failure.log" ]]; then + info "Completed distfiles update for $repo_name without error" + return + fi + + # report what went wrong :( + local lastpkg pkg file error + while read pkg file error; do + local log="${repo_mirror}/log/${file}.log" + if [[ "${pkg}" != "${lastpkg}" ]]; then + error "${pkg} failed:" + lastpkg="${pkg}" + fi + error " ${file} ${error}" + [[ -s "${log}" ]] && cat "${log}" + done <"${repo_mirror}/log/failure.log" + + EXIT_CODE=1 } upload_mirror() { local repo_name="$1" @@ -112,4 +134,5 @@ if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]]; then done fi -info "Done!" +command_completed +exit $EXIT_CODE