Merge pull request #163 from flatcar-linux/tormath1-jepio/ci-authenticated

jenkins: use private bucket with authentication for fetching binaries.
2026-05-04 11:51:14 +02:00 · 2021-10-12 13:33:22 +02:00 · 2021-10-12 13:33:22 +02:00 · 24128594e0
commit 24128594e0
parent dcccdb8815 d9542cec7d
6 changed files with 62 additions and 31 deletions
--- a/jenkins/images.sh
+++ b/jenkins/images.sh
@ -14,7 +14,7 @@ gpg --import verify.asc
 # key imports fail, let's create it here as a workaround
 mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/"

-DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}"
+DOWNLOAD_ROOT_SDK="${DOWNLOAD_ROOT}/sdk"

 SCRIPTS_PATCH_ARG=""
 OVERLAY_PATCH_ARG=""
@ -29,16 +29,15 @@ if [ "$(cat portage.patch | wc -l)" != 0 ]; then
  PORTAGE_PATCH_ARG="--portage-patch portage.patch"
 fi

-bin/cork update \
-    --create --downgrade-replace --verify --verify-signature --verbose \
+bin/cork create \
+    --verify --verify-signature --replace \
    --sdk-url-path "${SDK_URL_PATH}" \
    --json-key "${GS_DEVEL_CREDS}" \
-    --force-sync \
    ${SCRIPTS_PATCH_ARG} ${OVERLAY_PATCH_ARG} ${PORTAGE_PATCH_ARG} \
    --manifest-branch "refs/tags/${MANIFEST_TAG}" \
    --manifest-name "${MANIFEST_NAME}" \
-    --sdk-url storage.googleapis.com \
-    --manifest-url "${MANIFEST_URL}" -- --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}"
+    --manifest-url "${MANIFEST_URL}" \
+    --sdk-url=storage.googleapis.com

 # Clear out old images.
 sudo rm -rf chroot/build src/build torcx
@ -73,6 +72,9 @@ export FLATCAR_BUILD_ID
 # Set up GPG for signing uploads.
 gpg --import "${GPG_SECRET_KEY_FILE}"

+script update_chroot \
+    --toolchain_boards="${BOARD}" --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}"
+
 script setup_board \
    --board="${BOARD}" \
    --getbinpkgver="${FLATCAR_VERSION}" \
--- a/jenkins/kola/qemu_common.sh
+++ b/jenkins/kola/qemu_common.sh
@ -42,6 +42,10 @@ else
  }
 fi

+script() {
+  enter "/mnt/host/source/src/scripts/$@"
+}
+
 # Set up GPG for verifying tags.
 export GNUPGHOME="${PWD}/.gnupg"
 rm -rf "${GNUPGHOME}"
@ -52,7 +56,7 @@ gpg --import verify.asc
 # key imports fail, let's create it here as a workaround
 mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/"

-DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}"
+DOWNLOAD_ROOT_SDK="${DOWNLOAD_ROOT}/sdk"

 if native_arm64 ; then
  mkdir -p .repo/
@ -64,19 +68,23 @@ if native_arm64 ; then
  git -C .repo/manifests tag -v "${MANIFEST_TAG}"
  git -C .repo/manifests checkout "${MANIFEST_TAG}"
 else
-  bin/cork update \
-      --create --downgrade-replace --verify --verify-signature --verbose \
+  bin/cork create \
+      --verify --verify-signature --replace \
      --sdk-url-path "${SDK_URL_PATH}" \
-      --force-sync \
      --json-key "${GOOGLE_APPLICATION_CREDENTIALS}" \
      --manifest-branch "refs/tags/${MANIFEST_TAG}" \
      --manifest-name "${MANIFEST_NAME}" \
-      --manifest-url "${MANIFEST_URL}" -- --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}"
+      --sdk-url storage.googleapis.com \
+      --manifest-url "${MANIFEST_URL}"
 fi
+
 source .repo/manifests/version.txt

 [ -s verify.asc ] && verify_key=--verify-key=verify.asc || verify_key=

+script update_chroot \
+    --toolchain_boards="${BOARD}" --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}"
+
 mkdir -p tmp
 bin/cork download-image \
    --cache-dir=tmp \
@ -101,7 +109,6 @@ rm -f flatcar_test_update.gz
 bin/gangue get \
    --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \
    --verify=true $verify_key \
-    --sdk-url=storage.googleapis.com \
    "${DOWNLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}/flatcar_test_update.gz"
 mv flatcar_test_update.gz tmp/

--- a/jenkins/packages.sh
+++ b/jenkins/packages.sh
@ -10,7 +10,7 @@ set -ex
 if [[ "${RELEASE_BASE_IS_DEV}" = "false" && "${GROUP}" = "developer" && "${RELEASE_BASE}" != "" ]]; then
    DOWNLOAD_ROOT=$(echo ${DOWNLOAD_ROOT} | sed 's,/developer,,');
 fi
-DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}"
+DOWNLOAD_ROOT_SDK="${DOWNLOAD_ROOT}/sdk"

 # Set up GPG for verifying tags.
 export GNUPGHOME="${PWD}/.gnupg"
@ -35,17 +35,15 @@ if [ "$(cat portage.patch | wc -l)" != 0 ]; then
  PORTAGE_PATCH_ARG="--portage-patch portage.patch"
 fi

-bin/cork update \
-    --create --downgrade-replace --verify --verify-signature --verbose \
+bin/cork create \
+    --verify --verify-signature --replace \
    --sdk-url-path "${SDK_URL_PATH}" \
-    --force-sync \
    --json-key "${GOOGLE_APPLICATION_CREDENTIALS}" \
    ${SCRIPTS_PATCH_ARG} ${OVERLAY_PATCH_ARG} ${PORTAGE_PATCH_ARG} \
    --manifest-branch "refs/tags/${MANIFEST_TAG}" \
    --manifest-name "${MANIFEST_NAME}" \
    --manifest-url "${MANIFEST_URL}" \
-    --sdk-url=storage.googleapis.com \
-    -- --toolchain_boards="${BOARD}" --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}"
+    --sdk-url=storage.googleapis.com

 enter() {
        local verify_key=
@ -78,6 +76,9 @@ export FLATCAR_BUILD_ID
 # Set up GPG for signing uploads.
 gpg --import "${GPG_SECRET_KEY_FILE}"

+script update_chroot \
+    --toolchain_boards="${BOARD}" --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}"
+
 script setup_board \
    --board="${BOARD}" \
    --getbinpkgver=${RELEASE_BASE:-"${FLATCAR_VERSION}" --toolchainpkgonly} \
--- a/jenkins/sdk.sh
+++ b/jenkins/sdk.sh
@ -27,7 +27,7 @@ then
 	fi
 fi

-DOWNLOAD_ROOT_SDK=https://storage.googleapis.com/flatcar-jenkins/sdk
+DOWNLOAD_ROOT_SDK="${DOWNLOAD_ROOT}/sdk"

 # We do not use a nightly SDK as seed for bootstrapping because the next major Alpha SDK release would also have to use the last published Alpha release SDK as seed.
 # Also, we don't want compiler bugs to propagate from one nightly SDK to the next even though the commit in question was reverted.
@ -44,6 +44,7 @@ bin/cork update \
    --json-key "${GOOGLE_APPLICATION_CREDENTIALS}" \
    --manifest-branch "refs/tags/${MANIFEST_TAG}" \
    --sdk-url storage.googleapis.com \
+    --sdk-url-path "/flatcar-jenkins/sdk" \
    --manifest-name "${MANIFEST_NAME}" \
    --manifest-url "${MANIFEST_URL}"  -- --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}"

--- a/jenkins/toolchains.sh
+++ b/jenkins/toolchains.sh
@ -17,7 +17,8 @@ gpg --import verify.asc
 # key imports fail, let's create it here as a workaround
 mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/"

-DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}"
+DOWNLOAD_ROOT=${DOWNLOAD_ROOT:-"${UPLOAD_ROOT}"}
+DOWNLOAD_ROOT_SDK="${DOWNLOAD_ROOT}/sdk"

 SCRIPTS_PATCH_ARG=""
 OVERLAY_PATCH_ARG=""
@ -32,24 +33,41 @@ if [ "$(cat portage.patch | wc -l)" != 0 ]; then
  PORTAGE_PATCH_ARG="--portage-patch portage.patch"
 fi

-bin/cork update \
-    --create --downgrade-replace --verify --verify-signature --verbose \
+bin/cork create \
+    --verify --verify-signature --replace \
    --sdk-url-path "${SDK_URL_PATH}" \
    --json-key "${GOOGLE_APPLICATION_CREDENTIALS}" \
    --sdk-url storage.googleapis.com \
-    --force-sync \
    ${SCRIPTS_PATCH_ARG} ${OVERLAY_PATCH_ARG} ${PORTAGE_PATCH_ARG} \
    --manifest-branch "refs/tags/${MANIFEST_TAG}" \
    --manifest-name "${MANIFEST_NAME}" \
-    --manifest-url "${MANIFEST_URL}" -- --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}"
+    --manifest-url "${MANIFEST_URL}"

 enter() {
-        bin/cork enter --bind-gpg-agent=false -- "$@"
+  sudo ln -f "${GOOGLE_APPLICATION_CREDENTIALS}" \
+      chroot/etc/portage/gangue.json
+  bin/cork enter --bind-gpg-agent=false -- env \
+      FLATCAR_DEV_BUILDS="${DOWNLOAD_ROOT}" \
+      FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \
+      {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \
+--json-key=/etc/portage/gangue.json $verify_key \
+"'"${URI}" "${DISTDIR}/${FILE}"' \
+      "$@"
+}
+
+script() {
+  enter "/mnt/host/source/src/scripts/$@"
 }

 source .repo/manifests/version.txt
 export FLATCAR_BUILD_ID

+# Fetch DIGEST to prevent re-downloading the same SDK tarball
+enter gangue get --json-key /etc/portage/gangue.json "${DOWNLOAD_ROOT_SDK}/amd64/${FLATCAR_SDK_VERSION}/flatcar-sdk-amd64-${FLATCAR_SDK_VERSION}.tar.bz2.DIGESTS" /mnt/host/source/.cache/sdks/
+
+script update_chroot \
+    --toolchain_boards="${BOARD}" --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}"
+
 # Set up GPG for signing uploads.
 gpg --import "${GPG_SECRET_KEY_FILE}"

--- a/jenkins/vms.sh
+++ b/jenkins/vms.sh
@ -14,7 +14,7 @@ gpg --import verify.asc
 # key imports fail, let's create it here as a workaround
 mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/"

-DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}"
+DOWNLOAD_ROOT_SDK="${DOWNLOAD_ROOT}/sdk"

 SCRIPTS_PATCH_ARG=""
 OVERLAY_PATCH_ARG=""
@ -29,15 +29,15 @@ if [ "$(cat portage.patch | wc -l)" != 0 ]; then
  PORTAGE_PATCH_ARG="--portage-patch portage.patch"
 fi

-bin/cork update \
-    --create --downgrade-replace --verify --verify-signature --verbose \
+bin/cork create \
+    --replace --verify --verify-signature --verbose \
    --sdk-url-path "${SDK_URL_PATH}" \
-    --force-sync \
    --json-key "${GS_DEVEL_CREDS}" \
    ${SCRIPTS_PATCH_ARG} ${OVERLAY_PATCH_ARG} ${PORTAGE_PATCH_ARG} \
    --manifest-branch "refs/tags/${MANIFEST_TAG}" \
    --manifest-name "${MANIFEST_NAME}" \
-    --manifest-url "${MANIFEST_URL}" -- --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}"
+    --manifest-url "${MANIFEST_URL}" \
+    --sdk-url=storage.googleapis.com

 # Clear out old images.
 sudo rm -rf chroot/build tmp
@ -69,6 +69,9 @@ script() {
 source .repo/manifests/version.txt
 export FLATCAR_BUILD_ID

+script update_chroot \
+    --toolchain_boards="${BOARD}" --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}"
+
 # Set up GPG for signing uploads.
 gpg --import "${GPG_SECRET_KEY_FILE}"

@ -79,7 +82,6 @@ bin/cork download-image \
    --root="${UPLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}" \
    --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \
    --cache-dir=./src \
-    --sdk-url=storage.googleapis.com \
    --platform=qemu \
    --verify=true $verify_key