Merge pull request #510 from marineam/dist

Properly report errors from update_distfiles and other cleanups

update_distfiles

@@ -33,6 +33,8 @@ if [[ ${FLAGS_parallel} -eq ${FLAGS_TRUE} ]]; then
     GSUTIL_OPTS="-m"
 fi
 
+EXIT_CODE=0
+
 
 update_local_mirror() {
     local repo_name="$1"
@@ -46,6 +48,7 @@ update_local_mirror() {
         info "Starting distfiles update for $repo_name"
     fi
 
+    rm -rf "${repo_mirror}/log" # clear old logs
     mkdir -p "${repo_mirror}/"{distfiles,info,log,tmp}
     emirrordist --mirror --verbose $extra_flags \
         --jobs=${NUM_JOBS} --repo="${repo_name}" \
@@ -54,13 +57,29 @@ update_local_mirror() {
         --fetch-log-dir="${repo_mirror}/log" \
         --failure-log="${repo_mirror}/log/failure.log" \
         --success-log="${repo_mirror}/log/success.log" \
-        --scheduled-deletion-log="${repo_mirror}/log/deletion.log" \
-        --deletion-db="${repo_mirror}/info/deletion.db" \
         --distfiles-db="${repo_mirror}/info/distfiles.db" \
-        --deletion-delay=$((86400 * 14)) \
         --restrict-mirror-exemptions="gentoo" \
         --temp-dir="${repo_mirror}/tmp" \
         --verify-existing-digest
+
+    if [[ ! -s "${repo_mirror}/log/failure.log" ]]; then
+        info "Completed distfiles update for $repo_name without error"
+        return
+    fi
+
+    # report what went wrong :(
+    local lastpkg pkg file error
+    while read pkg file error; do
+        local log="${repo_mirror}/log/${file}.log"
+        if [[ "${pkg}" != "${lastpkg}" ]]; then
+            error "${pkg} failed:"
+            lastpkg="${pkg}"
+        fi
+        error " ${file} ${error}"
+        [[ -s "${log}" ]] && cat "${log}"
+    done <"${repo_mirror}/log/failure.log"
+
+    EXIT_CODE=1
 }
 upload_mirror() {
     local repo_name="$1"
@@ -68,10 +87,11 @@ upload_mirror() {
     local remote_mirror="${UPLOAD_ROOT}/$repo_name"
 
     info "Uploading public distfiles for $repo_name"
-    gsutil ${GSUTIL_OPTS} cp -n \
+    gsutil ${GSUTIL_OPTS} rsync -c \
-        "${local_mirror}/distfiles/*" "${remote_mirror}/distfiles"
+        "${local_mirror}/distfiles/" "${remote_mirror}/distfiles"
 
     info "Uploading private metadata for $repo_name"
+    # uses cp instead of rsync in order to provide acl
     gsutil ${GSUTIL_OPTS} cp -a project-private \
         "${local_mirror}/info/*" "${remote_mirror}/info"
 }
@@ -82,17 +102,15 @@ download_mirror() {
 
     info "Downloading public distfiles for $repo_name"
     mkdir -p "${local_mirror}/"{distfiles,info}
-    gsutil ${GSUTIL_OPTS} cp -n \
+    gsutil ${GSUTIL_OPTS} rsync -c -d \
-        "${remote_mirror}/distfiles/*" "${local_mirror}/distfiles"
+        "${remote_mirror}/distfiles/" "${local_mirror}/distfiles"
 
     info "Downloading private metadata for $repo_name"
-    gsutil ${GSUTIL_OPTS} cp "${remote_mirror}/info/*" "${local_mirror}/info"
+    gsutil ${GSUTIL_OPTS} rsync -c -d \
+        "${remote_mirror}/info/" "${local_mirror}/info"
 }
 
 if [[ ${FLAGS_download} -eq ${FLAGS_TRUE} ]]; then
-    if [[ ! -f "$HOME/.boto" ]]; then
-        die_notrace "Please run gsutil config to create ~/.boto"
-    fi
     for repo in "$@"; do
         download_mirror "$repo"
     done
@@ -111,12 +129,10 @@ if [[ ${FLAGS_dry_run} == ${FLAGS_TRUE} ]]; then
 fi
 
 if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]]; then
-    if [[ ! -f "$HOME/.boto" ]]; then
-        die_notrace "Please run gsutil config to create ~/.boto"
-    fi
     for repo in "$@"; do
         upload_mirror "$repo"
     done
 fi
 
-info "Done!"
+command_completed
+exit $EXIT_CODE