diff --git a/jenkins/images.sh b/jenkins/images.sh index faced8cd66..90de620254 100755 --- a/jenkins/images.sh +++ b/jenkins/images.sh @@ -14,7 +14,12 @@ gpg --import verify.asc # key imports fail, let's create it here as a workaround mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/" -DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}" +SCRIPT_LOCATION="$(dirname "$(readlink -f "$0")")" +if [ -f "${SCRIPT_LOCATION}/override-vars.env" ] ; then + source "${SCRIPT_LOCATION}/override-vars.env" +else + DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}" +fi SCRIPTS_PATCH_ARG="" OVERLAY_PATCH_ARG="" @@ -29,8 +34,11 @@ if [ "$(cat portage.patch | wc -l)" != 0 ]; then PORTAGE_PATCH_ARG="--portage-patch portage.patch" fi +sdk_host="${DOWNLOAD_ROOT_SDK#*://}" +sdk_host="${sdk_host%/*}" bin/cork update \ --create --downgrade-replace --verify --verify-signature --verbose \ + --sdk-url "${sdk_host}" \ --sdk-url-path "${SDK_URL_PATH}" \ --force-sync \ ${SCRIPTS_PATCH_ARG} ${OVERLAY_PATCH_ARG} ${PORTAGE_PATCH_ARG} \ @@ -51,13 +59,20 @@ enter() { sudo ln -f verify.asc chroot/etc/portage/gangue.asc && verify_key=--verify-key=/etc/portage/gangue.asc sudo ln -f "${GS_DEVEL_CREDS}" chroot/etc/portage/gangue.json - bin/cork enter --bind-gpg-agent=false -- env \ - FLATCAR_DEV_BUILDS="${DOWNLOAD_ROOT}" \ - FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \ - {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \ ---json-key=/etc/portage/gangue.json $verify_key \ -"'"${URI}" "${DISTDIR}/${FILE}"' \ - "$@" + if [[ "${DOWNLOAD_ROOT}" = 'gs://'* ]]; then + bin/cork enter --bind-gpg-agent=false -- env \ + FLATCAR_DEV_BUILDS="${DOWNLOAD_ROOT}" \ + FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \ + {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \ + --json-key=/etc/portage/gangue.json $verify_key \ + "'"${URI}" "${DISTDIR}/${FILE}"' \ + "$@" + else + bin/cork enter --bind-gpg-agent=false -- env \ + FLATCAR_DEV_BUILDS="${DOWNLOAD_ROOT}" \ + FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \ + "$@" + fi ) } diff --git a/jenkins/override-vars.env b/jenkins/override-vars.env new file mode 100644 index 0000000000..3b8d0e71f2 --- /dev/null +++ b/jenkins/override-vars.env @@ -0,0 +1,40 @@ +RELEASEHOST_BASE_URL="https://mirror.release.flatcar-linux.net" +BINHOST_BASE_URL="https://bincache.flatcar-linux.net" +BINHOST_UPLOAD_BASE_URL="rsync://bincache@bincache.flatcar-linux.net:/srv/bincache" + +# overrides section +if [ -n "${SDK_URL_PATH}" ] ; then + SDK_URL_PATH="${SDK_URL_PATH/flatcar-jenkins\//}" +fi +DOWNLOAD_ROOT_SDK="${BINHOST_BASE_URL}${SDK_URL_PATH}" + + +if [ "${COREOS_OFFICIAL}" != "1" ] ; then + BINHOST_BASE_URL="${BINHOST_BASE_URL}/developer" + BINHOST_UPLOAD_BASE_URL="${BINHOST_UPLOAD_BASE_URL}/developer" +fi + +TECTONIC_TORCX_DOWNLOAD_ROOT="${BINHOST_BASE_URL}/torcx" +TORCX_PKG_DOWNLOAD_ROOT="${BINHOST_BASE_URL}/torcx" +TORCX_PKG_UPLOAD_ROOT="${BINHOST_UPLOAD_BASE_URL}/torcx" +DOWNLOAD_ROOT="${BINHOST_BASE_URL}" +UPLOAD_ROOT="${BINHOST_UPLOAD_BASE_URL}" + +function ssh_versionfile() { + local upload_root="$1" + local filename="$2" + local contents="$3" + + local rsync_upload_path="${upload_root#rsync://}" + local sshcmd="ssh -o BatchMode=yes " + sshcmd="$sshcmd -o StrictHostKeyChecking=no" + sshcmd="$sshcmd -o UserKnownHostsFile=/dev/null" + sshcmd="$sshcmd -o NumberOfPasswordPrompts=0" + local sshuserhost="${rsync_upload_path%:*}" + local path_prefix="${rsync_upload_path#*:}" + local destfile="${path_prefix}${filename}" + + ${sshcmd} "${sshuserhost}" "mkdir -p $(dirname "${destfile}")" + ${sshcmd} "${sshuserhost}" "echo '${contents}' > ${destfile}" + +} diff --git a/jenkins/packages.sh b/jenkins/packages.sh index 670f83aa0b..6251ba8d05 100755 --- a/jenkins/packages.sh +++ b/jenkins/packages.sh @@ -22,6 +22,16 @@ gpg --import verify.asc # key imports fail, let's create it here as a workaround mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/" +SCRIPT_LOCATION="$(dirname "$(readlink -f "$0")")" +if [ -f "${SCRIPT_LOCATION}/override-vars.env" ] ; then + source "${SCRIPT_LOCATION}/override-vars.env" + # we're actually uploading... + TORCX_PKG_DOWNLOAD_ROOT="${TORCX_PKG_UPLOAD_ROOT}" +else + DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}" +fi + +SCRIPTS_PATCH_ARG="" SCRIPTS_PATCH_ARG="" OVERLAY_PATCH_ARG="" PORTAGE_PATCH_ARG="" @@ -35,8 +45,11 @@ if [ "$(cat portage.patch | wc -l)" != 0 ]; then PORTAGE_PATCH_ARG="--portage-patch portage.patch" fi +sdk_host="${DOWNLOAD_ROOT_SDK#*://}" +sdk_host="${sdk_host%/*}" bin/cork update \ --create --downgrade-replace --verify --verify-signature --verbose \ + --sdk-url "${sdk_host}" \ --sdk-url-path "${SDK_URL_PATH}" \ --force-sync \ ${SCRIPTS_PATCH_ARG} ${OVERLAY_PATCH_ARG} ${PORTAGE_PATCH_ARG} \ @@ -56,13 +69,20 @@ enter() { verify_key=--verify-key=/etc/portage/gangue.asc sudo ln -f "${GOOGLE_APPLICATION_CREDENTIALS}" \ chroot/etc/portage/gangue.json - bin/cork enter --bind-gpg-agent=false -- env \ - FLATCAR_DEV_BUILDS="${DOWNLOAD_ROOT}" \ - FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \ - {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \ ---json-key=/etc/portage/gangue.json $verify_key \ -"'"${URI}" "${DISTDIR}/${FILE}"' \ - "$@" + if [[ "${DOWNLOAD_ROOT}" = 'gs://'* ]]; then + bin/cork enter --bind-gpg-agent=false -- env \ + FLATCAR_DEV_BUILDS="${DOWNLOAD_ROOT}" \ + FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \ + {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \ + --json-key=/etc/portage/gangue.json $verify_key \ + "'"${URI}" "${DISTDIR}/${FILE}"' \ + "$@" + else + bin/cork enter --bind-gpg-agent=false -- env \ + FLATCAR_DEV_BUILDS="${DOWNLOAD_ROOT}" \ + FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \ + "$@" + fi ) } @@ -112,5 +132,9 @@ if [[ "${FLATCAR_BUILD_ID}" == *-*-nightly-* ]] then # Extract the nightly name like "flatcar-MAJOR-nightly" from "dev-flatcar-MAJOR-nightly-NUMBER" NAME=$(echo "${FLATCAR_BUILD_ID}" | grep -o "dev-.*-nightly" | cut -d - -f 2-) - echo "${FLATCAR_VERSION}" | bin/cork enter --bind-gpg-agent=false -- gsutil cp - "${UPLOAD_ROOT}/boards/${BOARD}/${NAME}.txt" + if [[ "${UPLOAD_ROOT}" = 'rsync://'* ]]; then + ssh_versionfile "${UPLOAD_ROOT}" "${FLATCAR_VERSION}" "${UPLOAD_ROOT}/boards/${BOARD}/${NAME}.txt" + else + echo "${FLATCAR_VERSION}" | bin/cork enter --bind-gpg-agent=false -- gsutil cp - "${UPLOAD_ROOT}/boards/${BOARD}/${NAME}.txt" + fi fi diff --git a/jenkins/sdk.sh b/jenkins/sdk.sh index 2b2520b49a..a310c2a2c7 100755 --- a/jenkins/sdk.sh +++ b/jenkins/sdk.sh @@ -27,7 +27,10 @@ then fi fi -DOWNLOAD_ROOT_SDK=https://storage.googleapis.com/flatcar-jenkins/sdk +SCRIPT_LOCATION="$(dirname "$(readlink -f "$0")")" +if [ -f "${SCRIPT_LOCATION}/override-vars.env" ] ; then + source "${SCRIPT_LOCATION}/override-vars.env" +fi # We do not use a nightly SDK as seed for bootstrapping because the next major Alpha SDK release would also have to use the last published Alpha release SDK as seed. # Also, we don't want compiler bugs to propagate from one nightly SDK to the next even though the commit in question was reverted. @@ -36,9 +39,11 @@ DOWNLOAD_ROOT_SDK=https://storage.googleapis.com/flatcar-jenkins/sdk # In rare cases this will mean that a huge compiler update has to be split because first a released SDK with a newer compiler is needed to compile an even newer compiler # (or linker, libc etc). For experiments one can download the nightly/developer SDK and start the bootstrap from it locally but exposing this functionality in Jenkins would # cause more confusion than helping to understand what the requirements are to get SDK changes to a releasable state. +DOWNLOAD_ROOT_SDK="$RELEASEHOST_BASE_URL/sdk" bin/cork update \ --create --downgrade-replace --verify --verify-signature --verbose \ + --sdk-url "${DOWNLOAD_ROOT_SDK}" \ --sdk-version "${SEED_SDK_VERSION}" \ --force-sync \ --manifest-branch "refs/tags/${MANIFEST_TAG}" \ @@ -78,5 +83,10 @@ if [[ "${FLATCAR_BUILD_ID}" == *-*-nightly-* ]] then # Extract the nightly name like "flatcar-MAJOR-nightly" from "dev-flatcar-MAJOR-nightly-NUMBER" NAME=$(echo "${FLATCAR_BUILD_ID}" | grep -o "dev-.*-nightly" | cut -d - -f 2-) - echo "${FLATCAR_VERSION}" | enter gsutil cp - "${UPLOAD_ROOT}/sdk/amd64/sdk-${NAME}.txt" + + if [[ "${UPLOAD_ROOT}" = 'rsync://'* ]]; then + ssh_versionfile "${UPLOAD_ROOT}" "${FLATCAR_VERSION}" "/sdk/amd64/sdk-${NAME}.txt" + else + echo "${FLATCAR_VERSION}" | enter gsutil cp - "${UPLOAD_ROOT}/sdk/amd64/sdk-${NAME}.txt" + fi fi diff --git a/jenkins/toolchains.sh b/jenkins/toolchains.sh index 236094499b..e8312257a5 100755 --- a/jenkins/toolchains.sh +++ b/jenkins/toolchains.sh @@ -17,7 +17,12 @@ gpg --import verify.asc # key imports fail, let's create it here as a workaround mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/" -DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}" +SCRIPT_LOCATION="$(dirname "$(readlink -f "$0")")" +if [ -f "${SCRIPT_LOCATION}/override-vars.env" ] ; then + source "${SCRIPT_LOCATION}/override-vars.env" +else + DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}" +fi SCRIPTS_PATCH_ARG="" OVERLAY_PATCH_ARG="" @@ -32,8 +37,11 @@ if [ "$(cat portage.patch | wc -l)" != 0 ]; then PORTAGE_PATCH_ARG="--portage-patch portage.patch" fi +sdk_host="${DOWNLOAD_ROOT_SDK#*://}" +sdk_host="${sdk_host%/*}" bin/cork update \ --create --downgrade-replace --verify --verify-signature --verbose \ + --sdk-url "${sdk_host}" \ --sdk-url-path "${SDK_URL_PATH}" \ --force-sync \ ${SCRIPTS_PATCH_ARG} ${OVERLAY_PATCH_ARG} ${PORTAGE_PATCH_ARG} \