mirrors/flatcar-scripts
1
0
Fork 0
mirror of https://github.com/flatcar/scripts.git synced 2025-09-22 22:21:10 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(update_distfiles): Add option to control gsutil's parallelism

Browse Source 
gsutil can be hard to follow when parallel upload/downloads are enabled.
"I see it is transferring something, but what?" So this provides an
option to disable that for debugging purposes.
This commit is contained in:
Michael Marineau 2013-06-26 11:44:16 -04:00
parent 76ecff8169
commit a25f12bbc4
1 changed files with 11 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
update_distfiles
View File

@ -8,6 +8,7 @@ SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
. "${SCRIPT_ROOT}/common.sh" || exit 1 . "${SCRIPT_ROOT}/common.sh" || exit 1
DEFINE_boolean dry_run ${FLAGS_FALSE} "Trial run, makes no changes." DEFINE_boolean dry_run ${FLAGS_FALSE} "Trial run, makes no changes."
DEFINE_boolean parallel ${FLAGS_TRUE} "Enable parallelism in gsutil."
DEFINE_boolean upload ${FLAGS_FALSE} "Upload distfile mirror via gsutil." DEFINE_boolean upload ${FLAGS_FALSE} "Upload distfile mirror via gsutil."
# FIXME(marineam): We need to add gs support to emirrordist so it # FIXME(marineam): We need to add gs support to emirrordist so it
@ -27,6 +28,12 @@ if [[ $# -eq 0 ]]; then
eval set -- portage-stable coreos eval set -- portage-stable coreos
fi fi
GSUTIL_OPTS=
if [[ ${FLAGS_parallel} -eq ${FLAGS_TRUE} ]]; then
GSUTIL_OPTS="-m"
fi
update_local_mirror() { update_local_mirror() {
local repo_name="$1" local repo_name="$1"
local repo_mirror="${MIRROR_ROOT}/$repo_name" local repo_mirror="${MIRROR_ROOT}/$repo_name"
@ -60,11 +67,11 @@ upload_mirror() {
local remote_mirror="${UPLOAD_ROOT}/$repo_name" local remote_mirror="${UPLOAD_ROOT}/$repo_name"
info "Uploading public distfiles for $repo_name" info "Uploading public distfiles for $repo_name"
gsutil -m cp -n \ gsutil ${GSUTIL_OPTS} cp -n \
"${local_mirror}/distfiles/*" "${remote_mirror}/distfiles" "${local_mirror}/distfiles/*" "${remote_mirror}/distfiles"
info "Uploading private metadata for $repo_name" info "Uploading private metadata for $repo_name"
gsutil cp -a project-private \ gsutil ${GSUTIL_OPTS} cp -a project-private \
"${local_mirror}/info/*" "${remote_mirror}/info" "${local_mirror}/info/*" "${remote_mirror}/info"
} }
download_mirror() { download_mirror() {
@ -74,11 +81,11 @@ download_mirror() {
info "Downloading public distfiles for $repo_name" info "Downloading public distfiles for $repo_name"
mkdir -p "${local_mirror}/"{distfiles,info} mkdir -p "${local_mirror}/"{distfiles,info}
gsutil -m cp -n \ gsutil ${GSUTIL_OPTS} cp -n \
"${remote_mirror}/distfiles/*" "${local_mirror}/distfiles" "${remote_mirror}/distfiles/*" "${local_mirror}/distfiles"
info "Downloading private metadata for $repo_name" info "Downloading private metadata for $repo_name"
gsutil cp "${remote_mirror}/info/*" "${local_mirror}/info" gsutil ${GSUTIL_OPTS} cp "${remote_mirror}/info/*" "${local_mirror}/info"
} }
if [[ ${FLAGS_download} -eq ${FLAGS_TRUE} ]]; then if [[ ${FLAGS_download} -eq ${FLAGS_TRUE} ]]; then