diff --git a/jenkins/images.sh b/jenkins/images.sh index 0ca888eb86..7c90108b88 100755 --- a/jenkins/images.sh +++ b/jenkins/images.sh @@ -14,7 +14,7 @@ gpg --import verify.asc # key imports fail, let's create it here as a workaround mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/" -DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}" +DOWNLOAD_ROOT_SDK="${DOWNLOAD_ROOT}/sdk" SCRIPTS_PATCH_ARG="" OVERLAY_PATCH_ARG="" @@ -29,16 +29,15 @@ if [ "$(cat portage.patch | wc -l)" != 0 ]; then PORTAGE_PATCH_ARG="--portage-patch portage.patch" fi -bin/cork update \ - --create --downgrade-replace --verify --verify-signature --verbose \ +bin/cork create \ + --verify --verify-signature --replace \ --sdk-url-path "${SDK_URL_PATH}" \ --json-key "${GS_DEVEL_CREDS}" \ - --force-sync \ ${SCRIPTS_PATCH_ARG} ${OVERLAY_PATCH_ARG} ${PORTAGE_PATCH_ARG} \ --manifest-branch "refs/tags/${MANIFEST_TAG}" \ --manifest-name "${MANIFEST_NAME}" \ - --sdk-url storage.googleapis.com \ - --manifest-url "${MANIFEST_URL}" -- --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" + --manifest-url "${MANIFEST_URL}" \ + --sdk-url=storage.googleapis.com # Clear out old images. sudo rm -rf chroot/build src/build torcx @@ -73,6 +72,9 @@ export FLATCAR_BUILD_ID # Set up GPG for signing uploads. gpg --import "${GPG_SECRET_KEY_FILE}" +script update_chroot \ + --toolchain_boards="${BOARD}" --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" + script setup_board \ --board="${BOARD}" \ --getbinpkgver="${FLATCAR_VERSION}" \ diff --git a/jenkins/kola/qemu_common.sh b/jenkins/kola/qemu_common.sh index 01e5179ae1..ebe42f95ae 100755 --- a/jenkins/kola/qemu_common.sh +++ b/jenkins/kola/qemu_common.sh @@ -42,6 +42,10 @@ else } fi +script() { + enter "/mnt/host/source/src/scripts/$@" +} + # Set up GPG for verifying tags. export GNUPGHOME="${PWD}/.gnupg" rm -rf "${GNUPGHOME}" @@ -52,7 +56,7 @@ gpg --import verify.asc # key imports fail, let's create it here as a workaround mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/" -DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}" +DOWNLOAD_ROOT_SDK="${DOWNLOAD_ROOT}/sdk" if native_arm64 ; then mkdir -p .repo/ @@ -64,19 +68,23 @@ if native_arm64 ; then git -C .repo/manifests tag -v "${MANIFEST_TAG}" git -C .repo/manifests checkout "${MANIFEST_TAG}" else - bin/cork update \ - --create --downgrade-replace --verify --verify-signature --verbose \ + bin/cork create \ + --verify --verify-signature --replace \ --sdk-url-path "${SDK_URL_PATH}" \ - --force-sync \ --json-key "${GOOGLE_APPLICATION_CREDENTIALS}" \ --manifest-branch "refs/tags/${MANIFEST_TAG}" \ --manifest-name "${MANIFEST_NAME}" \ - --manifest-url "${MANIFEST_URL}" -- --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" + --sdk-url storage.googleapis.com \ + --manifest-url "${MANIFEST_URL}" fi + source .repo/manifests/version.txt [ -s verify.asc ] && verify_key=--verify-key=verify.asc || verify_key= +script update_chroot \ + --toolchain_boards="${BOARD}" --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" + mkdir -p tmp bin/cork download-image \ --cache-dir=tmp \ @@ -101,7 +109,6 @@ rm -f flatcar_test_update.gz bin/gangue get \ --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \ --verify=true $verify_key \ - --sdk-url=storage.googleapis.com \ "${DOWNLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}/flatcar_test_update.gz" mv flatcar_test_update.gz tmp/ diff --git a/jenkins/packages.sh b/jenkins/packages.sh index b138851285..29187cbf81 100755 --- a/jenkins/packages.sh +++ b/jenkins/packages.sh @@ -10,7 +10,7 @@ set -ex if [[ "${RELEASE_BASE_IS_DEV}" = "false" && "${GROUP}" = "developer" && "${RELEASE_BASE}" != "" ]]; then DOWNLOAD_ROOT=$(echo ${DOWNLOAD_ROOT} | sed 's,/developer,,'); fi -DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}" +DOWNLOAD_ROOT_SDK="${DOWNLOAD_ROOT}/sdk" # Set up GPG for verifying tags. export GNUPGHOME="${PWD}/.gnupg" @@ -35,17 +35,15 @@ if [ "$(cat portage.patch | wc -l)" != 0 ]; then PORTAGE_PATCH_ARG="--portage-patch portage.patch" fi -bin/cork update \ - --create --downgrade-replace --verify --verify-signature --verbose \ +bin/cork create \ + --verify --verify-signature --replace \ --sdk-url-path "${SDK_URL_PATH}" \ - --force-sync \ --json-key "${GOOGLE_APPLICATION_CREDENTIALS}" \ ${SCRIPTS_PATCH_ARG} ${OVERLAY_PATCH_ARG} ${PORTAGE_PATCH_ARG} \ --manifest-branch "refs/tags/${MANIFEST_TAG}" \ --manifest-name "${MANIFEST_NAME}" \ --manifest-url "${MANIFEST_URL}" \ - --sdk-url=storage.googleapis.com \ - -- --toolchain_boards="${BOARD}" --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" + --sdk-url=storage.googleapis.com enter() { local verify_key= @@ -78,6 +76,9 @@ export FLATCAR_BUILD_ID # Set up GPG for signing uploads. gpg --import "${GPG_SECRET_KEY_FILE}" +script update_chroot \ + --toolchain_boards="${BOARD}" --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" + script setup_board \ --board="${BOARD}" \ --getbinpkgver=${RELEASE_BASE:-"${FLATCAR_VERSION}" --toolchainpkgonly} \ diff --git a/jenkins/sdk.sh b/jenkins/sdk.sh index ef1dee1e36..a2d4872e46 100755 --- a/jenkins/sdk.sh +++ b/jenkins/sdk.sh @@ -27,7 +27,7 @@ then fi fi -DOWNLOAD_ROOT_SDK=https://storage.googleapis.com/flatcar-jenkins/sdk +DOWNLOAD_ROOT_SDK="${DOWNLOAD_ROOT}/sdk" # We do not use a nightly SDK as seed for bootstrapping because the next major Alpha SDK release would also have to use the last published Alpha release SDK as seed. # Also, we don't want compiler bugs to propagate from one nightly SDK to the next even though the commit in question was reverted. @@ -44,6 +44,7 @@ bin/cork update \ --json-key "${GOOGLE_APPLICATION_CREDENTIALS}" \ --manifest-branch "refs/tags/${MANIFEST_TAG}" \ --sdk-url storage.googleapis.com \ + --sdk-url-path "/flatcar-jenkins/sdk" \ --manifest-name "${MANIFEST_NAME}" \ --manifest-url "${MANIFEST_URL}" -- --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" diff --git a/jenkins/toolchains.sh b/jenkins/toolchains.sh index 5293cc7c6d..73719fff7f 100755 --- a/jenkins/toolchains.sh +++ b/jenkins/toolchains.sh @@ -17,7 +17,8 @@ gpg --import verify.asc # key imports fail, let's create it here as a workaround mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/" -DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}" +DOWNLOAD_ROOT=${DOWNLOAD_ROOT:-"${UPLOAD_ROOT}"} +DOWNLOAD_ROOT_SDK="${DOWNLOAD_ROOT}/sdk" SCRIPTS_PATCH_ARG="" OVERLAY_PATCH_ARG="" @@ -32,24 +33,41 @@ if [ "$(cat portage.patch | wc -l)" != 0 ]; then PORTAGE_PATCH_ARG="--portage-patch portage.patch" fi -bin/cork update \ - --create --downgrade-replace --verify --verify-signature --verbose \ +bin/cork create \ + --verify --verify-signature --replace \ --sdk-url-path "${SDK_URL_PATH}" \ --json-key "${GOOGLE_APPLICATION_CREDENTIALS}" \ --sdk-url storage.googleapis.com \ - --force-sync \ ${SCRIPTS_PATCH_ARG} ${OVERLAY_PATCH_ARG} ${PORTAGE_PATCH_ARG} \ --manifest-branch "refs/tags/${MANIFEST_TAG}" \ --manifest-name "${MANIFEST_NAME}" \ - --manifest-url "${MANIFEST_URL}" -- --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" + --manifest-url "${MANIFEST_URL}" enter() { - bin/cork enter --bind-gpg-agent=false -- "$@" + sudo ln -f "${GOOGLE_APPLICATION_CREDENTIALS}" \ + chroot/etc/portage/gangue.json + bin/cork enter --bind-gpg-agent=false -- env \ + FLATCAR_DEV_BUILDS="${DOWNLOAD_ROOT}" \ + FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \ + {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \ +--json-key=/etc/portage/gangue.json $verify_key \ +"'"${URI}" "${DISTDIR}/${FILE}"' \ + "$@" +} + +script() { + enter "/mnt/host/source/src/scripts/$@" } source .repo/manifests/version.txt export FLATCAR_BUILD_ID +# Fetch DIGEST to prevent re-downloading the same SDK tarball +enter gangue get --json-key /etc/portage/gangue.json "${DOWNLOAD_ROOT_SDK}/amd64/${FLATCAR_SDK_VERSION}/flatcar-sdk-amd64-${FLATCAR_SDK_VERSION}.tar.bz2.DIGESTS" /mnt/host/source/.cache/sdks/ + +script update_chroot \ + --toolchain_boards="${BOARD}" --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" + # Set up GPG for signing uploads. gpg --import "${GPG_SECRET_KEY_FILE}" diff --git a/jenkins/vms.sh b/jenkins/vms.sh index 6665bbd1c9..ce51e989c4 100755 --- a/jenkins/vms.sh +++ b/jenkins/vms.sh @@ -14,7 +14,7 @@ gpg --import verify.asc # key imports fail, let's create it here as a workaround mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/" -DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}" +DOWNLOAD_ROOT_SDK="${DOWNLOAD_ROOT}/sdk" SCRIPTS_PATCH_ARG="" OVERLAY_PATCH_ARG="" @@ -29,15 +29,15 @@ if [ "$(cat portage.patch | wc -l)" != 0 ]; then PORTAGE_PATCH_ARG="--portage-patch portage.patch" fi -bin/cork update \ - --create --downgrade-replace --verify --verify-signature --verbose \ +bin/cork create \ + --replace --verify --verify-signature --verbose \ --sdk-url-path "${SDK_URL_PATH}" \ - --force-sync \ --json-key "${GS_DEVEL_CREDS}" \ ${SCRIPTS_PATCH_ARG} ${OVERLAY_PATCH_ARG} ${PORTAGE_PATCH_ARG} \ --manifest-branch "refs/tags/${MANIFEST_TAG}" \ --manifest-name "${MANIFEST_NAME}" \ - --manifest-url "${MANIFEST_URL}" -- --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" + --manifest-url "${MANIFEST_URL}" \ + --sdk-url=storage.googleapis.com # Clear out old images. sudo rm -rf chroot/build tmp @@ -69,6 +69,9 @@ script() { source .repo/manifests/version.txt export FLATCAR_BUILD_ID +script update_chroot \ + --toolchain_boards="${BOARD}" --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" + # Set up GPG for signing uploads. gpg --import "${GPG_SECRET_KEY_FILE}" @@ -79,7 +82,6 @@ bin/cork download-image \ --root="${UPLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}" \ --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \ --cache-dir=./src \ - --sdk-url=storage.googleapis.com \ --platform=qemu \ --verify=true $verify_key