jenkins/: override CI vars via override-vars.env

jenkins/images.sh

@@ -14,7 +14,12 @@ gpg --import verify.asc
 # key imports fail, let's create it here as a workaround
 mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/"
 
-DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}"
+SCRIPT_LOCATION="$(dirname "$(readlink -f "$0")")"
+if [ -f "${SCRIPT_LOCATION}/override-vars.env" ] ; then
+    source "${SCRIPT_LOCATION}/override-vars.env"
+else
+    DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}"
+fi
 
 SCRIPTS_PATCH_ARG=""
 OVERLAY_PATCH_ARG=""
@@ -29,8 +34,11 @@ if [ "$(cat portage.patch | wc -l)" != 0 ]; then
   PORTAGE_PATCH_ARG="--portage-patch portage.patch"
 fi
 
+sdk_host="${DOWNLOAD_ROOT_SDK#*://}" 
+sdk_host="${sdk_host%/*}"
 bin/cork update \
     --create --downgrade-replace --verify --verify-signature --verbose \
+    --sdk-url "${sdk_host}" \
     --sdk-url-path "${SDK_URL_PATH}" \
     --force-sync \
     ${SCRIPTS_PATCH_ARG} ${OVERLAY_PATCH_ARG} ${PORTAGE_PATCH_ARG} \
@@ -51,13 +59,20 @@ enter() {
         sudo ln -f verify.asc chroot/etc/portage/gangue.asc &&
         verify_key=--verify-key=/etc/portage/gangue.asc
         sudo ln -f "${GS_DEVEL_CREDS}" chroot/etc/portage/gangue.json
-        bin/cork enter --bind-gpg-agent=false -- env \
-            FLATCAR_DEV_BUILDS="${DOWNLOAD_ROOT}" \
-            FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \
-            {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \
---json-key=/etc/portage/gangue.json $verify_key \
-"'"${URI}" "${DISTDIR}/${FILE}"' \
-            "$@"
+        if [[ "${DOWNLOAD_ROOT}" = 'gs://'* ]]; then
+            bin/cork enter --bind-gpg-agent=false -- env \
+                FLATCAR_DEV_BUILDS="${DOWNLOAD_ROOT}" \
+                FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \
+                {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \
+    --json-key=/etc/portage/gangue.json $verify_key \
+    "'"${URI}" "${DISTDIR}/${FILE}"' \
+                "$@"
+        else
+            bin/cork enter --bind-gpg-agent=false -- env \
+                FLATCAR_DEV_BUILDS="${DOWNLOAD_ROOT}" \
+                FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \
+                "$@"
+        fi
         )
 }
 

jenkins/override-vars.env

@@ -0,0 +1,40 @@
+RELEASEHOST_BASE_URL="https://mirror.release.flatcar-linux.net"
+BINHOST_BASE_URL="https://bincache.flatcar-linux.net"
+BINHOST_UPLOAD_BASE_URL="rsync://bincache@bincache.flatcar-linux.net:/srv/bincache"
+
+# overrides section
+if [ -n "${SDK_URL_PATH}" ] ; then
+    SDK_URL_PATH="${SDK_URL_PATH/flatcar-jenkins\//}"
+fi
+DOWNLOAD_ROOT_SDK="${BINHOST_BASE_URL}${SDK_URL_PATH}"
+
+
+if [ "${COREOS_OFFICIAL}" != "1" ] ; then
+    BINHOST_BASE_URL="${BINHOST_BASE_URL}/developer"
+    BINHOST_UPLOAD_BASE_URL="${BINHOST_UPLOAD_BASE_URL}/developer"
+fi
+
+TECTONIC_TORCX_DOWNLOAD_ROOT="${BINHOST_BASE_URL}/torcx"
+TORCX_PKG_DOWNLOAD_ROOT="${BINHOST_BASE_URL}/torcx"
+TORCX_PKG_UPLOAD_ROOT="${BINHOST_UPLOAD_BASE_URL}/torcx"
+DOWNLOAD_ROOT="${BINHOST_BASE_URL}"
+UPLOAD_ROOT="${BINHOST_UPLOAD_BASE_URL}"
+
+function ssh_versionfile() {
+    local upload_root="$1"
+    local filename="$2"
+    local contents="$3"
+
+    local rsync_upload_path="${upload_root#rsync://}"
+    local sshcmd="ssh -o BatchMode=yes "
+          sshcmd="$sshcmd -o StrictHostKeyChecking=no"
+          sshcmd="$sshcmd -o UserKnownHostsFile=/dev/null"
+          sshcmd="$sshcmd -o NumberOfPasswordPrompts=0"
+    local sshuserhost="${rsync_upload_path%:*}"
+    local path_prefix="${rsync_upload_path#*:}"
+    local destfile="${path_prefix}${filename}"
+
+    ${sshcmd} "${sshuserhost}" "mkdir -p $(dirname "${destfile}")"
+    ${sshcmd} "${sshuserhost}" "echo '${contents}' > ${destfile}"
+
+}

jenkins/packages.sh

@@ -22,6 +22,16 @@ gpg --import verify.asc
 # key imports fail, let's create it here as a workaround
 mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/"
 
+SCRIPT_LOCATION="$(dirname "$(readlink -f "$0")")"
+if [ -f "${SCRIPT_LOCATION}/override-vars.env" ] ; then
+    source "${SCRIPT_LOCATION}/override-vars.env"
+    # we're actually uploading...
+    TORCX_PKG_DOWNLOAD_ROOT="${TORCX_PKG_UPLOAD_ROOT}"
+else
+    DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}"
+fi
+
+SCRIPTS_PATCH_ARG=""
 SCRIPTS_PATCH_ARG=""
 OVERLAY_PATCH_ARG=""
 PORTAGE_PATCH_ARG=""
@@ -35,8 +45,11 @@ if [ "$(cat portage.patch | wc -l)" != 0 ]; then
   PORTAGE_PATCH_ARG="--portage-patch portage.patch"
 fi
 
+sdk_host="${DOWNLOAD_ROOT_SDK#*://}" 
+sdk_host="${sdk_host%/*}"
 bin/cork update \
     --create --downgrade-replace --verify --verify-signature --verbose \
+    --sdk-url "${sdk_host}" \
     --sdk-url-path "${SDK_URL_PATH}" \
     --force-sync \
     ${SCRIPTS_PATCH_ARG} ${OVERLAY_PATCH_ARG} ${PORTAGE_PATCH_ARG} \
@@ -56,13 +69,20 @@ enter() {
         verify_key=--verify-key=/etc/portage/gangue.asc
         sudo ln -f "${GOOGLE_APPLICATION_CREDENTIALS}" \
             chroot/etc/portage/gangue.json
-        bin/cork enter --bind-gpg-agent=false -- env \
-            FLATCAR_DEV_BUILDS="${DOWNLOAD_ROOT}" \
-            FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \
-            {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \
---json-key=/etc/portage/gangue.json $verify_key \
-"'"${URI}" "${DISTDIR}/${FILE}"' \
-            "$@"
+        if [[ "${DOWNLOAD_ROOT}" = 'gs://'* ]]; then
+            bin/cork enter --bind-gpg-agent=false -- env \
+                FLATCAR_DEV_BUILDS="${DOWNLOAD_ROOT}" \
+                FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \
+                {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \
+    --json-key=/etc/portage/gangue.json $verify_key \
+    "'"${URI}" "${DISTDIR}/${FILE}"' \
+                "$@"
+        else
+            bin/cork enter --bind-gpg-agent=false -- env \
+                FLATCAR_DEV_BUILDS="${DOWNLOAD_ROOT}" \
+                FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \
+                "$@"
+        fi
         )
 }
 
@@ -112,5 +132,9 @@ if [[ "${FLATCAR_BUILD_ID}" == *-*-nightly-* ]]
 then
   # Extract the nightly name like "flatcar-MAJOR-nightly" from "dev-flatcar-MAJOR-nightly-NUMBER"
   NAME=$(echo "${FLATCAR_BUILD_ID}" | grep -o "dev-.*-nightly" | cut -d - -f 2-)
-  echo "${FLATCAR_VERSION}" | bin/cork enter --bind-gpg-agent=false -- gsutil cp - "${UPLOAD_ROOT}/boards/${BOARD}/${NAME}.txt"
+  if [[ "${UPLOAD_ROOT}" = 'rsync://'* ]]; then
+    ssh_versionfile "${UPLOAD_ROOT}" "${FLATCAR_VERSION}" "${UPLOAD_ROOT}/boards/${BOARD}/${NAME}.txt"
+  else
+    echo "${FLATCAR_VERSION}" | bin/cork enter --bind-gpg-agent=false -- gsutil cp - "${UPLOAD_ROOT}/boards/${BOARD}/${NAME}.txt"
+  fi
 fi

jenkins/sdk.sh

@@ -27,7 +27,10 @@ then
 	fi
 fi
 
-DOWNLOAD_ROOT_SDK=https://storage.googleapis.com/flatcar-jenkins/sdk
+SCRIPT_LOCATION="$(dirname "$(readlink -f "$0")")"
+if [ -f "${SCRIPT_LOCATION}/override-vars.env" ] ; then
+    source "${SCRIPT_LOCATION}/override-vars.env"
+fi
 
 # We do not use a nightly SDK as seed for bootstrapping because the next major Alpha SDK release would also have to use the last published Alpha release SDK as seed.
 # Also, we don't want compiler bugs to propagate from one nightly SDK to the next even though the commit in question was reverted.
@@ -36,9 +39,11 @@ DOWNLOAD_ROOT_SDK=https://storage.googleapis.com/flatcar-jenkins/sdk
 # In rare cases this will mean that a huge compiler update has to be split because first a released SDK with a newer compiler is needed to compile an even newer compiler
 # (or linker, libc etc). For experiments one can download the nightly/developer SDK and start the bootstrap from it locally but exposing this functionality in Jenkins would
 # cause more confusion than helping to understand what the requirements are to get SDK changes to a releasable state.
+DOWNLOAD_ROOT_SDK="$RELEASEHOST_BASE_URL/sdk"
 
 bin/cork update \
     --create --downgrade-replace --verify --verify-signature --verbose \
+    --sdk-url "${DOWNLOAD_ROOT_SDK}" \
     --sdk-version "${SEED_SDK_VERSION}" \
     --force-sync \
     --manifest-branch "refs/tags/${MANIFEST_TAG}" \
@@ -78,5 +83,10 @@ if [[ "${FLATCAR_BUILD_ID}" == *-*-nightly-* ]]
 then
   # Extract the nightly name like "flatcar-MAJOR-nightly" from "dev-flatcar-MAJOR-nightly-NUMBER"
   NAME=$(echo "${FLATCAR_BUILD_ID}" | grep -o "dev-.*-nightly" | cut -d - -f 2-)
-  echo "${FLATCAR_VERSION}" | enter gsutil cp - "${UPLOAD_ROOT}/sdk/amd64/sdk-${NAME}.txt"
+
+  if [[ "${UPLOAD_ROOT}" = 'rsync://'* ]]; then
+     ssh_versionfile "${UPLOAD_ROOT}" "${FLATCAR_VERSION}" "/sdk/amd64/sdk-${NAME}.txt"
+  else
+     echo "${FLATCAR_VERSION}" | enter gsutil cp - "${UPLOAD_ROOT}/sdk/amd64/sdk-${NAME}.txt"
+  fi
 fi

jenkins/toolchains.sh

@@ -17,7 +17,12 @@ gpg --import verify.asc
 # key imports fail, let's create it here as a workaround
 mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/"
 
-DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}"
+SCRIPT_LOCATION="$(dirname "$(readlink -f "$0")")"
+if [ -f "${SCRIPT_LOCATION}/override-vars.env" ] ; then
+    source "${SCRIPT_LOCATION}/override-vars.env"
+else
+    DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}"
+fi
 
 SCRIPTS_PATCH_ARG=""
 OVERLAY_PATCH_ARG=""
@@ -32,8 +37,11 @@ if [ "$(cat portage.patch | wc -l)" != 0 ]; then
   PORTAGE_PATCH_ARG="--portage-patch portage.patch"
 fi
 
+sdk_host="${DOWNLOAD_ROOT_SDK#*://}" 
+sdk_host="${sdk_host%/*}"
 bin/cork update \
     --create --downgrade-replace --verify --verify-signature --verbose \
+    --sdk-url "${sdk_host}" \
     --sdk-url-path "${SDK_URL_PATH}" \
     --force-sync \
     ${SCRIPTS_PATCH_ARG} ${OVERLAY_PATCH_ARG} ${PORTAGE_PATCH_ARG} \