From 93cf5aa5c3c3b5ab734d4eb87e71d450378ea6f9 Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Mon, 11 Oct 2021 15:23:48 +0200 Subject: [PATCH 1/4] jenkins: use gs:// uri for DOWNLOAD_ROOT_SDK so that we can apply authentication --- jenkins/images.sh | 2 +- jenkins/packages.sh | 2 +- jenkins/sdk.sh | 2 +- jenkins/toolchains.sh | 2 +- jenkins/vms.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/jenkins/images.sh b/jenkins/images.sh index 0ca888eb86..00c2dcc2a3 100755 --- a/jenkins/images.sh +++ b/jenkins/images.sh @@ -14,7 +14,7 @@ gpg --import verify.asc # key imports fail, let's create it here as a workaround mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/" -DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}" +DOWNLOAD_ROOT_SDK="${DOWNLOAD_ROOT}/sdk" SCRIPTS_PATCH_ARG="" OVERLAY_PATCH_ARG="" diff --git a/jenkins/packages.sh b/jenkins/packages.sh index b138851285..bf93f665bb 100755 --- a/jenkins/packages.sh +++ b/jenkins/packages.sh @@ -10,7 +10,7 @@ set -ex if [[ "${RELEASE_BASE_IS_DEV}" = "false" && "${GROUP}" = "developer" && "${RELEASE_BASE}" != "" ]]; then DOWNLOAD_ROOT=$(echo ${DOWNLOAD_ROOT} | sed 's,/developer,,'); fi -DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}" +DOWNLOAD_ROOT_SDK="${DOWNLOAD_ROOT}/sdk" # Set up GPG for verifying tags. export GNUPGHOME="${PWD}/.gnupg" diff --git a/jenkins/sdk.sh b/jenkins/sdk.sh index ef1dee1e36..4083339747 100755 --- a/jenkins/sdk.sh +++ b/jenkins/sdk.sh @@ -27,7 +27,7 @@ then fi fi -DOWNLOAD_ROOT_SDK=https://storage.googleapis.com/flatcar-jenkins/sdk +DOWNLOAD_ROOT_SDK="${DOWNLOAD_ROOT}/sdk" # We do not use a nightly SDK as seed for bootstrapping because the next major Alpha SDK release would also have to use the last published Alpha release SDK as seed. # Also, we don't want compiler bugs to propagate from one nightly SDK to the next even though the commit in question was reverted. diff --git a/jenkins/toolchains.sh b/jenkins/toolchains.sh index 5293cc7c6d..46090c5a5b 100755 --- a/jenkins/toolchains.sh +++ b/jenkins/toolchains.sh @@ -17,7 +17,7 @@ gpg --import verify.asc # key imports fail, let's create it here as a workaround mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/" -DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}" +DOWNLOAD_ROOT_SDK="${DOWNLOAD_ROOT}/sdk" SCRIPTS_PATCH_ARG="" OVERLAY_PATCH_ARG="" diff --git a/jenkins/vms.sh b/jenkins/vms.sh index 6665bbd1c9..82400a749d 100755 --- a/jenkins/vms.sh +++ b/jenkins/vms.sh @@ -14,7 +14,7 @@ gpg --import verify.asc # key imports fail, let's create it here as a workaround mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/" -DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}" +DOWNLOAD_ROOT_SDK="${DOWNLOAD_ROOT}/sdk" SCRIPTS_PATCH_ARG="" OVERLAY_PATCH_ARG="" From cbf003e617fafe82bc01d80daa359f59068e99ad Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Tue, 12 Oct 2021 13:31:43 +0200 Subject: [PATCH 2/4] jenkins: use 'cork create' instead of 'cork update' because we need to pass google credentials to update_chroot, and 'cork update' doesn't support that. Add --sdk-url-path to sdk.sh for new cork default. --- jenkins/images.sh | 12 +++++++----- jenkins/kola/qemu_common.sh | 19 +++++++++++++------ jenkins/packages.sh | 11 ++++++----- jenkins/sdk.sh | 1 + jenkins/toolchains.sh | 25 ++++++++++++++++++++----- jenkins/vms.sh | 11 +++++++---- 6 files changed, 54 insertions(+), 25 deletions(-) diff --git a/jenkins/images.sh b/jenkins/images.sh index 00c2dcc2a3..7c90108b88 100755 --- a/jenkins/images.sh +++ b/jenkins/images.sh @@ -29,16 +29,15 @@ if [ "$(cat portage.patch | wc -l)" != 0 ]; then PORTAGE_PATCH_ARG="--portage-patch portage.patch" fi -bin/cork update \ - --create --downgrade-replace --verify --verify-signature --verbose \ +bin/cork create \ + --verify --verify-signature --replace \ --sdk-url-path "${SDK_URL_PATH}" \ --json-key "${GS_DEVEL_CREDS}" \ - --force-sync \ ${SCRIPTS_PATCH_ARG} ${OVERLAY_PATCH_ARG} ${PORTAGE_PATCH_ARG} \ --manifest-branch "refs/tags/${MANIFEST_TAG}" \ --manifest-name "${MANIFEST_NAME}" \ - --sdk-url storage.googleapis.com \ - --manifest-url "${MANIFEST_URL}" -- --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" + --manifest-url "${MANIFEST_URL}" \ + --sdk-url=storage.googleapis.com # Clear out old images. sudo rm -rf chroot/build src/build torcx @@ -73,6 +72,9 @@ export FLATCAR_BUILD_ID # Set up GPG for signing uploads. gpg --import "${GPG_SECRET_KEY_FILE}" +script update_chroot \ + --toolchain_boards="${BOARD}" --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" + script setup_board \ --board="${BOARD}" \ --getbinpkgver="${FLATCAR_VERSION}" \ diff --git a/jenkins/kola/qemu_common.sh b/jenkins/kola/qemu_common.sh index 01e5179ae1..ebe42f95ae 100755 --- a/jenkins/kola/qemu_common.sh +++ b/jenkins/kola/qemu_common.sh @@ -42,6 +42,10 @@ else } fi +script() { + enter "/mnt/host/source/src/scripts/$@" +} + # Set up GPG for verifying tags. export GNUPGHOME="${PWD}/.gnupg" rm -rf "${GNUPGHOME}" @@ -52,7 +56,7 @@ gpg --import verify.asc # key imports fail, let's create it here as a workaround mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/" -DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}" +DOWNLOAD_ROOT_SDK="${DOWNLOAD_ROOT}/sdk" if native_arm64 ; then mkdir -p .repo/ @@ -64,19 +68,23 @@ if native_arm64 ; then git -C .repo/manifests tag -v "${MANIFEST_TAG}" git -C .repo/manifests checkout "${MANIFEST_TAG}" else - bin/cork update \ - --create --downgrade-replace --verify --verify-signature --verbose \ + bin/cork create \ + --verify --verify-signature --replace \ --sdk-url-path "${SDK_URL_PATH}" \ - --force-sync \ --json-key "${GOOGLE_APPLICATION_CREDENTIALS}" \ --manifest-branch "refs/tags/${MANIFEST_TAG}" \ --manifest-name "${MANIFEST_NAME}" \ - --manifest-url "${MANIFEST_URL}" -- --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" + --sdk-url storage.googleapis.com \ + --manifest-url "${MANIFEST_URL}" fi + source .repo/manifests/version.txt [ -s verify.asc ] && verify_key=--verify-key=verify.asc || verify_key= +script update_chroot \ + --toolchain_boards="${BOARD}" --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" + mkdir -p tmp bin/cork download-image \ --cache-dir=tmp \ @@ -101,7 +109,6 @@ rm -f flatcar_test_update.gz bin/gangue get \ --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \ --verify=true $verify_key \ - --sdk-url=storage.googleapis.com \ "${DOWNLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}/flatcar_test_update.gz" mv flatcar_test_update.gz tmp/ diff --git a/jenkins/packages.sh b/jenkins/packages.sh index bf93f665bb..29187cbf81 100755 --- a/jenkins/packages.sh +++ b/jenkins/packages.sh @@ -35,17 +35,15 @@ if [ "$(cat portage.patch | wc -l)" != 0 ]; then PORTAGE_PATCH_ARG="--portage-patch portage.patch" fi -bin/cork update \ - --create --downgrade-replace --verify --verify-signature --verbose \ +bin/cork create \ + --verify --verify-signature --replace \ --sdk-url-path "${SDK_URL_PATH}" \ - --force-sync \ --json-key "${GOOGLE_APPLICATION_CREDENTIALS}" \ ${SCRIPTS_PATCH_ARG} ${OVERLAY_PATCH_ARG} ${PORTAGE_PATCH_ARG} \ --manifest-branch "refs/tags/${MANIFEST_TAG}" \ --manifest-name "${MANIFEST_NAME}" \ --manifest-url "${MANIFEST_URL}" \ - --sdk-url=storage.googleapis.com \ - -- --toolchain_boards="${BOARD}" --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" + --sdk-url=storage.googleapis.com enter() { local verify_key= @@ -78,6 +76,9 @@ export FLATCAR_BUILD_ID # Set up GPG for signing uploads. gpg --import "${GPG_SECRET_KEY_FILE}" +script update_chroot \ + --toolchain_boards="${BOARD}" --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" + script setup_board \ --board="${BOARD}" \ --getbinpkgver=${RELEASE_BASE:-"${FLATCAR_VERSION}" --toolchainpkgonly} \ diff --git a/jenkins/sdk.sh b/jenkins/sdk.sh index 4083339747..a2d4872e46 100755 --- a/jenkins/sdk.sh +++ b/jenkins/sdk.sh @@ -44,6 +44,7 @@ bin/cork update \ --json-key "${GOOGLE_APPLICATION_CREDENTIALS}" \ --manifest-branch "refs/tags/${MANIFEST_TAG}" \ --sdk-url storage.googleapis.com \ + --sdk-url-path "/flatcar-jenkins/sdk" \ --manifest-name "${MANIFEST_NAME}" \ --manifest-url "${MANIFEST_URL}" -- --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" diff --git a/jenkins/toolchains.sh b/jenkins/toolchains.sh index 46090c5a5b..e9246d848e 100755 --- a/jenkins/toolchains.sh +++ b/jenkins/toolchains.sh @@ -17,6 +17,7 @@ gpg --import verify.asc # key imports fail, let's create it here as a workaround mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/" +DOWNLOAD_ROOT=${DOWNLOAD_ROOT:-"${UPLOAD_ROOT}"} DOWNLOAD_ROOT_SDK="${DOWNLOAD_ROOT}/sdk" SCRIPTS_PATCH_ARG="" @@ -32,24 +33,38 @@ if [ "$(cat portage.patch | wc -l)" != 0 ]; then PORTAGE_PATCH_ARG="--portage-patch portage.patch" fi -bin/cork update \ - --create --downgrade-replace --verify --verify-signature --verbose \ +bin/cork create \ + --verify --verify-signature --replace \ --sdk-url-path "${SDK_URL_PATH}" \ --json-key "${GOOGLE_APPLICATION_CREDENTIALS}" \ --sdk-url storage.googleapis.com \ - --force-sync \ ${SCRIPTS_PATCH_ARG} ${OVERLAY_PATCH_ARG} ${PORTAGE_PATCH_ARG} \ --manifest-branch "refs/tags/${MANIFEST_TAG}" \ --manifest-name "${MANIFEST_NAME}" \ - --manifest-url "${MANIFEST_URL}" -- --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" + --manifest-url "${MANIFEST_URL}" enter() { - bin/cork enter --bind-gpg-agent=false -- "$@" + sudo ln -f "${GOOGLE_APPLICATION_CREDENTIALS}" \ + chroot/etc/portage/gangue.json + bin/cork enter --bind-gpg-agent=false -- env \ + FLATCAR_DEV_BUILDS="${DOWNLOAD_ROOT}" \ + FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \ + {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \ +--json-key=/etc/portage/gangue.json $verify_key \ +"'"${URI}" "${DISTDIR}/${FILE}"' \ + "$@" +} + +script() { + enter "/mnt/host/source/src/scripts/$@" } source .repo/manifests/version.txt export FLATCAR_BUILD_ID +script update_chroot \ + --toolchain_boards="${BOARD}" --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" + # Set up GPG for signing uploads. gpg --import "${GPG_SECRET_KEY_FILE}" diff --git a/jenkins/vms.sh b/jenkins/vms.sh index 82400a749d..45dcb9d3f4 100755 --- a/jenkins/vms.sh +++ b/jenkins/vms.sh @@ -29,15 +29,15 @@ if [ "$(cat portage.patch | wc -l)" != 0 ]; then PORTAGE_PATCH_ARG="--portage-patch portage.patch" fi -bin/cork update \ - --create --downgrade-replace --verify --verify-signature --verbose \ +bin/cork create \ + --replace --verify --verify-signature --verbose \ --sdk-url-path "${SDK_URL_PATH}" \ - --force-sync \ --json-key "${GS_DEVEL_CREDS}" \ ${SCRIPTS_PATCH_ARG} ${OVERLAY_PATCH_ARG} ${PORTAGE_PATCH_ARG} \ --manifest-branch "refs/tags/${MANIFEST_TAG}" \ --manifest-name "${MANIFEST_NAME}" \ - --manifest-url "${MANIFEST_URL}" -- --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" + --manifest-url "${MANIFEST_URL}" \ + --sdk-url=storage.googleapis.com # Clear out old images. sudo rm -rf chroot/build tmp @@ -69,6 +69,9 @@ script() { source .repo/manifests/version.txt export FLATCAR_BUILD_ID +script update_chroot \ + --toolchain_boards="${BOARD}" --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" + # Set up GPG for signing uploads. gpg --import "${GPG_SECRET_KEY_FILE}" From 46ac63fdeef1c508e451c01ddb7d1f293210604f Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Tue, 12 Oct 2021 09:08:59 +0200 Subject: [PATCH 3/4] jenkins/vm: remove --sdk-url `download-image` already holds the remote host with the --root flag. Signed-off-by: Mathieu Tortuyaux --- jenkins/vms.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/jenkins/vms.sh b/jenkins/vms.sh index 45dcb9d3f4..ce51e989c4 100755 --- a/jenkins/vms.sh +++ b/jenkins/vms.sh @@ -82,7 +82,6 @@ bin/cork download-image \ --root="${UPLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}" \ --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \ --cache-dir=./src \ - --sdk-url=storage.googleapis.com \ --platform=qemu \ --verify=true $verify_key From d9542cec7ddd8b22c955e31b5228cd4c4a8fa54b Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Tue, 12 Oct 2021 11:08:12 +0200 Subject: [PATCH 4/4] jenkins/toolchains: fetch DIGESTS file to allow reusing downloaded SDK as seed The catalyst build uses the same SDK version as seed as the current SDK, but will only reuse the cached tarball if a DIGESTS file exists and is correct. Prefetch this file to prevent the build from trying to access google storage anonymously. Signed-off-by: Jeremi Piotrowski --- jenkins/toolchains.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jenkins/toolchains.sh b/jenkins/toolchains.sh index e9246d848e..73719fff7f 100755 --- a/jenkins/toolchains.sh +++ b/jenkins/toolchains.sh @@ -62,6 +62,9 @@ script() { source .repo/manifests/version.txt export FLATCAR_BUILD_ID +# Fetch DIGEST to prevent re-downloading the same SDK tarball +enter gangue get --json-key /etc/portage/gangue.json "${DOWNLOAD_ROOT_SDK}/amd64/${FLATCAR_SDK_VERSION}/flatcar-sdk-amd64-${FLATCAR_SDK_VERSION}.tar.bz2.DIGESTS" /mnt/host/source/.cache/sdks/ + script update_chroot \ --toolchain_boards="${BOARD}" --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}"