From 363f2321ab0bfacf90ae585f3227bd726007cefc Mon Sep 17 00:00:00 2001 From: James Le Cuirot Date: Fri, 22 Nov 2024 11:29:25 +0000 Subject: [PATCH] Delete (almost) all of the jenkins directory This has been superseded by code in the ci-automation directory. Signed-off-by: James Le Cuirot --- jenkins/README.md | 1 - jenkins/formats-amd64-usr.txt | 34 ----- jenkins/formats-arm64-usr.txt | 7 - jenkins/images.sh | 169 ----------------------- jenkins/kola/Dockerfile.kola-test-runner | 4 - jenkins/kola/aws.sh | 93 ------------- jenkins/kola/azure.sh | 55 -------- jenkins/kola/dev-container.sh | 47 ------- jenkins/kola/do.sh | 47 ------- jenkins/kola/gce.sh | 52 ------- jenkins/kola/packet.sh | 95 ------------- jenkins/kola/qemu.sh | 5 - jenkins/kola/qemu_common.sh | 151 -------------------- jenkins/kola/qemu_uefi.sh | 9 -- jenkins/kola/qemu_uefi_arm64.sh | 40 ------ jenkins/kola/vmware-esx.sh | 40 ------ jenkins/manifest.sh | 149 -------------------- jenkins/packages.sh | 120 ---------------- jenkins/post-github-status.sh | 53 ------- jenkins/prerelease/aws.sh | 23 --- jenkins/prerelease/azure.sh | 33 ----- jenkins/release.sh | 21 --- jenkins/sdk.sh | 94 ------------- jenkins/toolchains.sh | 84 ----------- jenkins/vms.sh | 126 ----------------- 25 files changed, 1552 deletions(-) delete mode 100644 jenkins/README.md delete mode 100644 jenkins/formats-amd64-usr.txt delete mode 100644 jenkins/formats-arm64-usr.txt delete mode 100755 jenkins/images.sh delete mode 100644 jenkins/kola/Dockerfile.kola-test-runner delete mode 100755 jenkins/kola/aws.sh delete mode 100755 jenkins/kola/azure.sh delete mode 100755 jenkins/kola/dev-container.sh delete mode 100755 jenkins/kola/do.sh delete mode 100755 jenkins/kola/gce.sh delete mode 100755 jenkins/kola/packet.sh delete mode 100755 jenkins/kola/qemu.sh delete mode 100755 jenkins/kola/qemu_common.sh delete mode 100755 jenkins/kola/qemu_uefi.sh delete mode 100755 jenkins/kola/qemu_uefi_arm64.sh delete mode 100755 jenkins/kola/vmware-esx.sh delete mode 100755 jenkins/manifest.sh delete mode 100755 jenkins/packages.sh delete mode 100755 jenkins/post-github-status.sh delete mode 100755 jenkins/prerelease/aws.sh delete mode 100755 jenkins/prerelease/azure.sh delete mode 100755 jenkins/release.sh delete mode 100755 jenkins/sdk.sh delete mode 100755 jenkins/toolchains.sh delete mode 100755 jenkins/vms.sh diff --git a/jenkins/README.md b/jenkins/README.md deleted file mode 100644 index 827d4706fe..0000000000 --- a/jenkins/README.md +++ /dev/null @@ -1 +0,0 @@ -This folder is unused. diff --git a/jenkins/formats-amd64-usr.txt b/jenkins/formats-amd64-usr.txt deleted file mode 100644 index 444bdf6542..0000000000 --- a/jenkins/formats-amd64-usr.txt +++ /dev/null @@ -1,34 +0,0 @@ -ami -ami_vmdk -azure -azure_gen2 -gce -iso -pxe -qemu -qemu_uefi -brightbox -cloudsigma -cloudstack -cloudstack_vhd -digitalocean -exoscale -hyperv -niftycloud -openstack -openstack_mini -packet -parallels -rackspace -rackspace_onmetal -rackspace_vhd -vagrant -vagrant_parallels -vagrant_virtualbox -vagrant_vmware_fusion -virtualbox -vmware -vmware_insecure -vmware_ova -vmware_raw -xen diff --git a/jenkins/formats-arm64-usr.txt b/jenkins/formats-arm64-usr.txt deleted file mode 100644 index f4ac1db9e3..0000000000 --- a/jenkins/formats-arm64-usr.txt +++ /dev/null @@ -1,7 +0,0 @@ -ami_vmdk -azure_gen2 -openstack -openstack_mini -packet -pxe -qemu_uefi diff --git a/jenkins/images.sh b/jenkins/images.sh deleted file mode 100755 index 29f1208216..0000000000 --- a/jenkins/images.sh +++ /dev/null @@ -1,169 +0,0 @@ -#!/bin/bash -set -ex - -# The build may not be started without a tag value. -[ -n "${MANIFEST_TAG}" ] - -# Set up GPG for verifying tags. -export GNUPGHOME="${PWD}/.gnupg" -rm -rf "${GNUPGHOME}" -trap 'rm -rf "${GNUPGHOME}"' EXIT -mkdir --mode=0700 "${GNUPGHOME}" -gpg --import verify.asc -# Sometimes this directory is not created automatically making further private -# key imports fail, let's create it here as a workaround -mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/" - -# since /flatcar-jenkins/developer/sdk starts with a / we only use one -DOWNLOAD_ROOT_SDK="gs:/${SDK_URL_PATH}" - -SCRIPTS_PATCH_ARG="" -OVERLAY_PATCH_ARG="" -PORTAGE_PATCH_ARG="" -if [ "$(cat scripts.patch | wc -l)" != 0 ]; then - SCRIPTS_PATCH_ARG="--scripts-patch scripts.patch" -fi -if [ "$(cat overlay.patch | wc -l)" != 0 ]; then - OVERLAY_PATCH_ARG="--overlay-patch overlay.patch" -fi -if [ "$(cat portage.patch | wc -l)" != 0 ]; then - PORTAGE_PATCH_ARG="--portage-patch portage.patch" -fi - -bin/cork create \ - --verify --verify-signature --replace \ - --sdk-url-path "${SDK_URL_PATH}" \ - --json-key "${GS_DEVEL_CREDS}" \ - ${SCRIPTS_PATCH_ARG} ${OVERLAY_PATCH_ARG} ${PORTAGE_PATCH_ARG} \ - --manifest-branch "refs/tags/${MANIFEST_TAG}" \ - --manifest-name "${MANIFEST_NAME}" \ - --manifest-url "${MANIFEST_URL}" \ - --sdk-url=storage.googleapis.com - -# Clear out old images. -sudo rm -rf chroot/build src/build torcx - -enter() { - local verify_key= - # Run in a subshell to clean some gangue files on exit without - # possibly clobbering the global EXIT trap. - ( - trap 'sudo rm -f chroot/etc/portage/gangue.*' EXIT - [ -s verify.asc ] && - sudo ln -f verify.asc chroot/etc/portage/gangue.asc && - verify_key=--verify-key=/etc/portage/gangue.asc - sudo ln -f "${GS_DEVEL_CREDS}" chroot/etc/portage/gangue.json - bin/cork enter --bind-gpg-agent=false -- env \ - FLATCAR_DEV_BUILDS="${DOWNLOAD_ROOT}" \ - FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \ - {FETCH,RESUME}COMMAND_GS="/mnt/host/source/bin/gangue get \ ---json-key=/etc/portage/gangue.json $verify_key \ -"'"${URI}" "${DISTDIR}/${FILE}"' \ - "$@" - ) -} - -script() { - enter "/mnt/host/source/src/scripts/$@" -} - -source .repo/manifests/version.txt -export FLATCAR_BUILD_ID - -# Set up GPG for signing uploads. -gpg --import "${GPG_SECRET_KEY_FILE}" - -script update_chroot \ - --toolchain_boards="${BOARD}" --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" - -script setup_board \ - --board="${BOARD}" \ - --getbinpkgver="${FLATCAR_VERSION}" \ - --regen_configs_only - -if [ "x${COREOS_OFFICIAL}" == x1 ] -then - script set_official --board="${BOARD}" --official -else - script set_official --board="${BOARD}" --noofficial -fi - -# Retrieve this version's torcx manifest -mkdir -p torcx/pkgs -enter gsutil cp -r \ - "${DOWNLOAD_ROOT}/torcx/manifests/${BOARD}/${FLATCAR_VERSION}/torcx_manifest.json"{,.sig} \ - /mnt/host/source/torcx/ -gpg --verify torcx/torcx_manifest.json.sig - -BASH_SYNTAX_ERROR_WORKAROUND=$(mktemp) -exec {keep_open}<>"${BASH_SYNTAX_ERROR_WORKAROUND}" -rm "${BASH_SYNTAX_ERROR_WORKAROUND}" -jq -r '.value.packages[] | . as $p | .name as $n | $p.versions[] | [.casDigest, .hash] | join(" ") | [$n, .] | join(" ")' "torcx/torcx_manifest.json" > "/proc/$$/fd/${keep_open}" -# Download all cas references from the manifest and verify their checksums -# TODO: technically we can skip ones that don't have a 'path' since they're not -# included in the image. -while read name digest hash -do - mkdir -p "torcx/pkgs/${BOARD}/${name}/${digest}" - enter gsutil cp -r "${TORCX_PKG_DOWNLOAD_ROOT}/pkgs/${BOARD}/${name}/${digest}" \ - "/mnt/host/source/torcx/pkgs/${BOARD}/${name}/" - downloaded_hash=$(sha512sum "torcx/pkgs/${BOARD}/${name}/${digest}/"*.torcx.tgz | awk '{print $1}') - if [[ "sha512-${downloaded_hash}" != "${hash}" ]] - then - echo "Torcx package had wrong hash: ${downloaded_hash} instead of ${hash}" - exit 1 - fi -done < "/proc/$$/fd/${keep_open}" -# This was "done < <(jq ...)" but it suddenly gave a syntax error with bash 4 when run with systemd-run-wrap.sh - -script build_image \ - --board="${BOARD}" \ - --group="${GROUP}" \ - --getbinpkg \ - --getbinpkgver="${FLATCAR_VERSION}" \ - --sign="${SIGNING_USER}" \ - --sign_digests="${SIGNING_USER}" \ - --torcx_manifest=/mnt/host/source/torcx/torcx_manifest.json \ - --torcx_root=/mnt/host/source/torcx/ \ - --upload_root="${UPLOAD_ROOT}" \ - --upload prodtar container - -set +x -# Don't fail the whole job -set +e -echo "===================================================================" -echo -export BOARD_A="${BOARD}" -export BOARD_B="${BOARD}" -if [ "${GROUP}" != "developer" ]; then - export CHANNEL_A="${GROUP}" -else - export CHANNEL_A="${CHANNEL_BASE}" -fi - -export VERSION_A=$(curl -s -S -f -L "https://${CHANNEL_A}.release.flatcar-linux.net/${BOARD}/current/version.txt" | grep -m 1 "FLATCAR_VERSION=" | cut -d "=" -f 2) - -if [ "${GROUP}" = "developer" ]; then - export CHANNEL_B="developer" - export MODE_B="/developer/" -else - export CHANNEL_B="${GROUP}" -fi -echo "Image differences compared to ${CHANNEL_A} ${VERSION_A}:" -rm -f package-diff -curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 "https://raw.githubusercontent.com/flatcar-linux/flatcar-build-scripts/master/package-diff" -chmod +x package-diff -echo "Package updates, compared to ${CHANNEL_A} ${VERSION_A}:" -FILE=flatcar_production_image_packages.txt ./package-diff "${VERSION_A}" "${FLATCAR_VERSION}" -echo -echo "Image file changes, compared to ${CHANNEL_A} ${VERSION_A}:" -FILE=flatcar_production_image_contents.txt FILESONLY=1 CUTKERNEL=1 ./package-diff "${VERSION_A}" "${FLATCAR_VERSION}" -echo -echo "Image kernel config changes, compared to ${CHANNEL_A} ${VERSION_A}:" -FILE=flatcar_production_image_kernel_config.txt ./package-diff "${VERSION_A}" "${FLATCAR_VERSION}" -echo -echo "Image file size change (includes /boot, /usr and the default rootfs partitions), compared to ${CHANNEL_A} ${VERSION_A}:" -FILE=flatcar_production_image_contents.txt CALCSIZE=1 ./package-diff "${VERSION_A}" "${FLATCAR_VERSION}" -echo -BASE_PATH="https://bucket.release.flatcar-linux.net/$(echo $UPLOAD_ROOT | sed 's|gs://||g')/boards/${BOARD}/${FLATCAR_VERSION}" -echo "Image URL: ${BASE_PATH}/flatcar_production_image.bin.bz2" diff --git a/jenkins/kola/Dockerfile.kola-test-runner b/jenkins/kola/Dockerfile.kola-test-runner deleted file mode 100644 index cae0d4ba7a..0000000000 --- a/jenkins/kola/Dockerfile.kola-test-runner +++ /dev/null @@ -1,4 +0,0 @@ -FROM debian:11 - -RUN apt-get update && \ - apt-get install -y qemu-system-aarch64 qemu-efi-aarch64 lbzip2 sudo dnsmasq gnupg2 git curl iptables diff --git a/jenkins/kola/aws.sh b/jenkins/kola/aws.sh deleted file mode 100755 index fcb417e9df..0000000000 --- a/jenkins/kola/aws.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash -set -ex - -rm -rf *.tap _kola_temp* - -NAME="jenkins-${JOB_NAME##*/}-${BUILD_NUMBER}" - -if [[ "${AWS_INSTANCE_TYPE}" != "" ]]; then - instance_type="${AWS_INSTANCE_TYPE}" -elif [[ "${BOARD}" == "arm64-usr" ]]; then - instance_type="a1.large" -elif [[ "${BOARD}" == "amd64-usr" ]]; then - instance_type="t3.small" -fi - -# If the OFFER is empty, it should be treated as the basic offering. -if [[ "${OFFER}" == "" ]]; then - OFFER="basic" -fi - -# Append the offer as oem suffix. -if [[ "${OFFER}" != "basic" ]]; then - OEM_SUFFIX="_${OFFER}" -fi - -if [[ "${KOLA_TESTS}" == "" ]]; then - KOLA_TESTS="*" -fi - -if [[ "${AWS_AMI_ID}" == "" ]]; then - [ -s verify.asc ] && verify_key=--verify-key=verify.asc || verify_key= - mkdir -p tmp - bin/cork download-image \ - --cache-dir=tmp \ - --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \ - --platform="aws${OEM_SUFFIX}" \ - --root="${DOWNLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}" \ - --sanity-check=false --verify=true $verify_key - bunzip2 "tmp/flatcar_production_ami_vmdk${OEM_SUFFIX}_image.vmdk.bz2" - BUCKET="flatcar-kola-ami-import-${AWS_REGION}" - trap 'bin/ore -d aws delete --region="${AWS_REGION}" --name="${NAME}" --ami-name="${NAME}" --file="tmp/flatcar_production_ami_vmdk${OEM_SUFFIX}_image.vmdk" --bucket "s3://${BUCKET}/${BOARD}/"; rm -r tmp/' EXIT - bin/ore aws initialize --region="${AWS_REGION}" --bucket "${BUCKET}" - AWS_AMI_ID=$(bin/ore aws upload --force --region="${AWS_REGION}" --name=${NAME} --ami-name="${NAME}" --ami-description="Flatcar Test ${NAME}" --file="tmp/flatcar_production_ami_vmdk${OEM_SUFFIX}_image.vmdk" --bucket "s3://${BUCKET}/${BOARD}/" | jq -r .HVM) - echo "Created new AMI ${AWS_AMI_ID} (will be removed after testing)" -fi - -# Run the cl.internet test on multiple machine types only if it should run in general -cl_internet_included="$(set -o noglob; bin/kola list --platform=aws --filter ${KOLA_TESTS} | { grep cl.internet || true ; } )" -if [[ "${BOARD}" == "amd64-usr" ]] && [[ "${cl_internet_included}" != "" ]]; then - for INSTANCE in m4.2xlarge; do - ( - set +x - OUTPUT=$(timeout --signal=SIGQUIT 6h bin/kola run \ - --parallel=8 \ - --basename="${NAME}" \ - --board="${BOARD}" \ - --aws-ami="${AWS_AMI_ID}" \ - --aws-region="${AWS_REGION}" \ - --aws-type="${INSTANCE}" \ - --aws-iam-profile="${AWS_IAM_PROFILE}" \ - --platform=aws \ - --channel="${GROUP}" \ - --offering="${OFFER}" \ - --tapfile="${JOB_NAME##*/}_validate_${INSTANCE}.tap" \ - --torcx-manifest=torcx_manifest.json \ - cl.internet 2>&1 || true) - echo "=== START $INSTANCE ===" - echo "${OUTPUT}" | sed "s/^/${INSTANCE}: /g" - echo "=== END $INSTANCE ===" - ) & - done -fi - -# Do not expand the kola test patterns globs -set -o noglob -timeout --signal=SIGQUIT 6h bin/kola run \ - --parallel=8 \ - --basename="${NAME}" \ - --board="${BOARD}" \ - --aws-ami="${AWS_AMI_ID}" \ - --aws-region="${AWS_REGION}" \ - --aws-type="${instance_type}" \ - --aws-iam-profile="${AWS_IAM_PROFILE}" \ - --platform=aws \ - --channel="${GROUP}" \ - --offering="${OFFER}" \ - --tapfile="${JOB_NAME##*/}.tap" \ - --torcx-manifest=torcx_manifest.json \ - ${KOLA_TESTS} -set +o noglob - -# wait for the cl.internet test results -wait diff --git a/jenkins/kola/azure.sh b/jenkins/kola/azure.sh deleted file mode 100755 index 25b1dcdf8f..0000000000 --- a/jenkins/kola/azure.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -set -ex - -rm -rf *.tap _kola_temp* - -NAME="jenkins-${JOB_NAME##*/}-${BUILD_NUMBER}" - -if [[ "${BOARD}" == "arm64-usr" ]]; then - if [[ "${AZURE_HYPER_V_GENERATION}" != "V2" ]]; then - echo "Unsupported combination" - exit 1 - fi - AZURE_USE_GALLERY="--azure-use-gallery" -fi - -if [[ "${KOLA_TESTS}" == "" ]]; then - KOLA_TESTS="*" -fi - -if [[ "${AZURE_MACHINE_SIZE}" != "" ]]; then - AZURE_MACHINE_SIZE_OPT="--azure-size=${AZURE_MACHINE_SIZE}" -fi - -# If the OFFER is empty, it should be treated as the basic offering. -if [[ "${OFFER}" == "" ]]; then - OFFER="basic" -fi - -if [ "${BLOB_URL}" = "" ]; then - exit 0 -fi - -# Do not expand the kola test patterns globs -set -o noglob -# Align timeout with ore azure gc --duration parameter -timeout --signal=SIGQUIT 6h bin/kola run \ - --parallel="${PARALLEL}" \ - --basename="${NAME}" \ - --board="${BOARD}" \ - --channel="${GROUP}" \ - --platform=azure \ - --offering="${OFFER}" \ - --azure-blob-url="${BLOB_URL}" \ - --azure-location="${LOCATION}" \ - --azure-profile="${AZURE_CREDENTIALS}" \ - --azure-auth="${AZURE_AUTH_CREDENTIALS}" \ - --tapfile="${JOB_NAME##*/}.tap" \ - --torcx-manifest=torcx_manifest.json \ - ${AZURE_USE_GALLERY} \ - ${AZURE_MACHINE_SIZE_OPT} \ - ${AZURE_HYPER_V_GENERATION:+--azure-hyper-v-generation=${AZURE_HYPER_V_GENERATION}} \ - ${AZURE_VNET_SUBNET_NAME:+--azure-vnet-subnet-name=${AZURE_VNET_SUBNET_NAME}} \ - ${AZURE_USE_PRIVATE_IPS:+--azure-use-private-ips=${AZURE_USE_PRIVATE_IPS}} \ - ${KOLA_TESTS} -set +o noglob diff --git a/jenkins/kola/dev-container.sh b/jenkins/kola/dev-container.sh deleted file mode 100755 index 74869c19eb..0000000000 --- a/jenkins/kola/dev-container.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -set -ex - -sudo rm -f flatcar_developer_container.bin* -trap 'sudo rm -f flatcar_developer_container.bin*' EXIT - -[ -s verify.asc ] && verify_key=--verify-key=verify.asc || verify_key= - -bin/gangue get \ - --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \ - --verify=true $verify_key \ - "${DOWNLOAD_ROOT}/boards/${BOARD}/${VERSION}/flatcar_production_image_kernel_config.txt" - -bin/gangue get \ - --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \ - --verify=true $verify_key \ - "${DOWNLOAD_ROOT}/boards/${BOARD}/${VERSION}/flatcar_developer_container.bin.bz2" -bunzip2 flatcar_developer_container.bin.bz2 - -if [[ "$(systemd-nspawn --version | grep 'systemd 241')" = "" ]] -then - PIPEARG="--pipe" -else - # TODO: Remove this case once Flatcar >=2592 is used on all nodes - PIPEARG="" -fi - -sudo systemd-nspawn $PIPEARG \ - --setenv=PORTAGE_BINHOST="${PORTAGE_BINHOST}" \ - --bind-ro=/lib/modules \ - --bind-ro="$PWD/flatcar_production_image_kernel_config.txt:/boot/config" \ - --bind-ro="${GOOGLE_APPLICATION_CREDENTIALS}:/opt/credentials.json" \ - --bind-ro="$PWD/verify.asc:/opt/verify.asc" \ - --bind-ro="$PWD/bin/gangue:/opt/bin/gangue" \ - --image=flatcar_developer_container.bin \ - --machine=flatcar-developer-container-$(uuidgen) \ - --tmpfs=/usr/src \ - --tmpfs=/var/tmp \ - /bin/bash -eux << 'EOF' -export PORTAGE_BINHOST="${PORTAGE_BINHOST}" -export {FETCH,RESUME}COMMAND_GS="/opt/bin/gangue get --json-key=/opt/credentials.json --verify=true /opt/verify.asc \"\${URI}\" \"\${DISTDIR}/\${FILE}\"" -emerge-gitclone -. /usr/share/coreos/release -emerge -gv coreos-sources -ln -fns /boot/config /usr/src/linux/.config -exec make -C /usr/src/linux -j"$(nproc)" modules_prepare V=1 -EOF diff --git a/jenkins/kola/do.sh b/jenkins/kola/do.sh deleted file mode 100755 index 8287f5c75b..0000000000 --- a/jenkins/kola/do.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -set -ex - -# JOB_NAME will not fit within the character limit -NAME="jenkins-${BUILD_NUMBER}" - -set -o pipefail - -if [[ "${DOWNLOAD_ROOT}" == gs://flatcar-jenkins-private/* ]]; then - echo "Fetching google/cloud-sdk" - docker pull google/cloud-sdk > /dev/null - BUCKET_PATH="${DOWNLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}/flatcar_production_digitalocean_image.bin.bz2" - IMAGE_URL="$(docker run --rm --net=host -v "${GOOGLE_APPLICATION_CREDENTIALS}:${GOOGLE_APPLICATION_CREDENTIALS}" google/cloud-sdk sh -c "python3 -m pip install pyopenssl > /dev/null; gsutil signurl -d 7d -r us ${GOOGLE_APPLICATION_CREDENTIALS} ${BUCKET_PATH} | grep -o 'https.*'")" -else - BASE_URL="https://bucket.release.flatcar-linux.net/$(echo $DOWNLOAD_ROOT | sed 's|gs://||g')/boards/${BOARD}/${FLATCAR_VERSION}" - IMAGE_URL="${BASE_URL}/flatcar_production_digitalocean_image.bin.bz2" -fi - -bin/ore do create-image \ - --config-file="${DIGITALOCEAN_CREDS}" \ - --region="${DO_REGION}" \ - --name="${NAME}" \ - --url="${IMAGE_URL}" - -trap 'bin/ore do delete-image \ - --name="${NAME}" \ - --config-file="${DIGITALOCEAN_CREDS}"' EXIT - -if [[ "${KOLA_TESTS}" == "" ]]; then - KOLA_TESTS="*" -fi - -# Do not expand the kola test patterns globs -set -o noglob -timeout --signal=SIGQUIT 4h bin/kola run \ - --do-size=${DO_MACHINE_SIZE} \ - --do-region=${DO_REGION} \ - --basename="${NAME}" \ - --do-config-file="${DIGITALOCEAN_CREDS}" \ - --do-image="${NAME}" \ - --parallel=8 \ - --platform=do \ - --channel="${GROUP}" \ - --tapfile="${JOB_NAME##*/}.tap" \ - --torcx-manifest=torcx_manifest.json \ - ${KOLA_TESTS} -set +o noglob diff --git a/jenkins/kola/gce.sh b/jenkins/kola/gce.sh deleted file mode 100755 index f5fce48483..0000000000 --- a/jenkins/kola/gce.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -set -ex - -rm -rf *.tap _kola_temp* - -# If the OFFER is empty, it should be treated as the basic offering. -if [[ "${OFFER}" == "" ]]; then - OFFER="basic" -fi - -# Append the offer as oem suffix. -if [[ "${OFFER}" != "basic" ]]; then - OEM_SUFFIX="_${OFFER}" -fi - -# Create a name that includes the OFFER, -# but replace _ with -, as gcloud doesn't like it otherwise. -OEMNAME="${OFFER}-${BUILD_NUMBER}" -NAME=${OEMNAME//_/-} - -bin/ore gcloud create-image \ - --board="${BOARD}" \ - --family="${NAME}" \ - --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \ - --source-root="${DOWNLOAD_ROOT}/boards" \ - --source-name=flatcar_production_gce${OEM_SUFFIX}.tar.gz \ - --version="${FLATCAR_VERSION}" - -GCE_NAME="${NAME//[+.]/-}-${FLATCAR_VERSION//[+.]/-}" - -trap 'bin/ore gcloud delete-images \ - --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \ - "${GCE_NAME}"' EXIT - -if [[ "${KOLA_TESTS}" == "" ]]; then - KOLA_TESTS="*" -fi - -# Do not expand the kola test patterns globs -set -o noglob -timeout --signal=SIGQUIT 6h bin/kola run \ - --basename="${NAME}" \ - --gce-image="${GCE_NAME}" \ - --gce-json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \ - --gce-machinetype="${GCE_MACHINE_TYPE}" \ - --parallel=4 \ - --platform=gce \ - --channel="${GROUP}" \ - --tapfile="${JOB_NAME##*/}.tap" \ - --torcx-manifest=torcx_manifest.json \ - ${KOLA_TESTS} -set +o noglob diff --git a/jenkins/kola/packet.sh b/jenkins/kola/packet.sh deleted file mode 100755 index e95005b400..0000000000 --- a/jenkins/kola/packet.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/bash -set -ex - -# JOB_NAME will not fit within the character limit -NAME="jenkins-${BUILD_NUMBER}" - -# same as the GC timeout -timeout=6h - -set -o pipefail - -# Construct the URLs of the image to be used during tests. -if [[ "${DOWNLOAD_ROOT}" == gs://flatcar-jenkins-private/* ]]; then - echo "Fetching google/cloud-sdk" - docker pull google/cloud-sdk > /dev/null - BUCKET_PATH="${DOWNLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}" - IMAGE_URL="$(docker run --rm --net=host -v "${GOOGLE_APPLICATION_CREDENTIALS}:${GOOGLE_APPLICATION_CREDENTIALS}" google/cloud-sdk sh -c "python3 -m pip install pyopenssl > /dev/null; gsutil signurl -d 7d -r us ${GOOGLE_APPLICATION_CREDENTIALS} ${BUCKET_PATH}/flatcar_production_packet_image.bin.bz2 | grep -o 'https.*'")" - KERNEL_URL="$(docker run --rm --net=host -v "${GOOGLE_APPLICATION_CREDENTIALS}:${GOOGLE_APPLICATION_CREDENTIALS}" google/cloud-sdk sh -c "python3 -m pip install pyopenssl > /dev/null; gsutil signurl -d 7d -r us ${GOOGLE_APPLICATION_CREDENTIALS} ${BUCKET_PATH}/flatcar_production_pxe.vmlinuz | grep -o 'https.*'")" - CPIO_URL="$(docker run --rm --net=host -v "${GOOGLE_APPLICATION_CREDENTIALS}:${GOOGLE_APPLICATION_CREDENTIALS}" google/cloud-sdk sh -c "python3 -m pip install pyopenssl > /dev/null; gsutil signurl -d 7d -r us ${GOOGLE_APPLICATION_CREDENTIALS} ${BUCKET_PATH}/flatcar_production_pxe_image.cpio.gz | grep -o 'https.*'")" -else - BASE_PATH="bucket.release.flatcar-linux.net/$(echo $DOWNLOAD_ROOT | sed 's|gs://||g')/boards/${BOARD}/${FLATCAR_VERSION}" - IMAGE_URL="https://${BASE_PATH}/flatcar_production_packet_image.bin.bz2" - KERNEL_URL="https://${BASE_PATH}/flatcar_production_pxe.vmlinuz" - CPIO_URL="https://${BASE_PATH}/flatcar_production_pxe_image.cpio.gz" -fi - -if [[ "${KOLA_TESTS}" == "" ]]; then - KOLA_TESTS="*" -fi - -# Equinix Metal ARM server are not yet hourly available in the default `sv15` region -# so we override the `PACKET_REGION` to `DC`. ARM servers are available in metro -# either DA (Dallas) or DC (Washington), but DC has more servers available. -# See also https://metal.equinix.com/developers/docs/locations/capacity/. -# We do not override `PACKET_REGION` for both board on top level because we need to keep proximity -# for PXE booting. -if [[ "${BOARD}" == "arm64-usr" ]]; then - PACKET_REGION="DC" -fi - -# Run the cl.internet test on multiple machine types only if it should run in general -cl_internet_included="$(set -o noglob; bin/kola list --platform=packet --filter ${KOLA_TESTS} | { grep cl.internet || true ; } )" -if [[ "${BOARD}" == "amd64-usr" ]] && [[ "${cl_internet_included}" != "" ]]; then - for INSTANCE in m3.small.x86 c3.medium.x86 m3.large.x86 s3.xlarge.x86 n2.xlarge.x86; do - ( - set +x - OUTPUT=$(timeout --signal=SIGQUIT "${timeout}" bin/kola run \ - --basename="${NAME}" \ - --board="${BOARD}" \ - --channel="${GROUP}" \ - --gce-json-key="${UPLOAD_CREDS}" \ - --packet-api-key="${PACKET_API_KEY}" \ - --packet-image-url="${IMAGE_URL}" \ - --packet-installer-image-kernel-url="${KERNEL_URL}" \ - --packet-installer-image-cpio-url="${CPIO_URL}" \ - --packet-project="${PACKET_PROJECT}" \ - --packet-storage-url="${UPLOAD_ROOT}/mantle/packet" \ - --packet-plan="${INSTANCE}" \ - --equinixmetal-metro="${PACKET_REGION}" \ - --parallel="${PARALLEL_TESTS}" \ - --platform=packet \ - --tapfile="${JOB_NAME##*/}_validate_${INSTANCE}.tap" \ - --torcx-manifest=torcx_manifest.json \ - cl.internet 2>&1 || true) - echo "=== START $INSTANCE ===" - echo "${OUTPUT}" | sed "s/^/${INSTANCE}: /g" - echo "=== END $INSTANCE ===" - ) & - done -fi - -# Do not expand the kola test patterns globs -set -o noglob -timeout --signal=SIGQUIT "${timeout}" bin/kola run \ - --basename="${NAME}" \ - --board="${BOARD}" \ - --channel="${GROUP}" \ - --gce-json-key="${UPLOAD_CREDS}" \ - --packet-api-key="${PACKET_API_KEY}" \ - --packet-image-url="${IMAGE_URL}" \ - --packet-installer-image-kernel-url="${KERNEL_URL}" \ - --packet-installer-image-cpio-url="${CPIO_URL}" \ - --packet-project="${PACKET_PROJECT}" \ - --packet-storage-url="${UPLOAD_ROOT}/mantle/packet" \ - --packet-plan="${PACKET_MACHINE_TYPE}" \ - --equinixmetal-metro="${PACKET_REGION}" \ - --parallel="${PARALLEL_TESTS}" \ - --platform=packet \ - --tapfile="${JOB_NAME##*/}.tap" \ - --torcx-manifest=torcx_manifest.json \ - ${KOLA_TESTS} -set +o noglob - -# wait for the cl.internet test results -wait diff --git a/jenkins/kola/qemu.sh b/jenkins/kola/qemu.sh deleted file mode 100755 index 1496d50edf..0000000000 --- a/jenkins/kola/qemu.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -set -ex - -SCRIPTFOLDER="$(dirname "$(readlink -f "$0")")" -"${SCRIPTFOLDER}/qemu_common.sh" qemu diff --git a/jenkins/kola/qemu_common.sh b/jenkins/kola/qemu_common.sh deleted file mode 100755 index fb6bbbbe22..0000000000 --- a/jenkins/kola/qemu_common.sh +++ /dev/null @@ -1,151 +0,0 @@ -#!/bin/bash -set -ex - -PLATFORM="$1" -if [ "${PLATFORM}" = qemu ]; then - TIMEOUT="12h" - BIOS="bios-256k.bin" -elif [ "${PLATFORM}" = qemu_uefi ]; then - TIMEOUT="14h" - BIOS="/mnt/host/source/tmp/flatcar_production_qemu_uefi_efi_code.qcow2" -else - echo "Unknown platform: \"${PLATFORM}\"" -fi - -native_arm64() { - [[ "${NATIVE_ARM64}" == true ]] -} - -sudo rm -rf *.tap src/scripts/_kola_temp tmp _kola_temp* _tmp - -if native_arm64 ; then - # for kola reflinking - sudo rm -rf /var/tmp - mkdir -p _tmp - chmod 1777 _tmp - ln -s "$PWD/_tmp" /var/tmp - # use arm64 mantle bins - rm -rf bin - mv bin.arm64 bin - # simulate SDK folder structure - mkdir -p src - ln -s .. src/scripts - sudo rm -f chroot - ln -s / chroot - - enter() { - "$@" - } -else - enter() { - bin/cork enter --bind-gpg-agent=false -- "$@" - } -fi - -script() { - enter "/mnt/host/source/src/scripts/$@" -} - -# Set up GPG for verifying tags. -export GNUPGHOME="${PWD}/.gnupg" -rm -rf "${GNUPGHOME}" -trap 'rm -rf "${GNUPGHOME}"' EXIT -mkdir --mode=0700 "${GNUPGHOME}" -gpg --import verify.asc -# Sometimes this directory is not created automatically making further private -# key imports fail, let's create it here as a workaround -mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/" - -# since /flatcar-jenkins/developer/sdk starts with a / we only use one -DOWNLOAD_ROOT_SDK="gs:/${SDK_URL_PATH}" - -if native_arm64 ; then - mkdir -p .repo/ - if [ ! -e .repo/manifests ]; then - mkdir -p ~/.ssh - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts - git clone "${MANIFEST_URL}" .repo/manifests - fi - git -C .repo/manifests tag -v "${MANIFEST_TAG}" - git -C .repo/manifests checkout "${MANIFEST_TAG}" -else - bin/cork create \ - --verify --verify-signature --replace \ - --sdk-url-path "${SDK_URL_PATH}" \ - --json-key "${GOOGLE_APPLICATION_CREDENTIALS}" \ - --manifest-branch "refs/tags/${MANIFEST_TAG}" \ - --manifest-name "${MANIFEST_NAME}" \ - --sdk-url storage.googleapis.com \ - --manifest-url "${MANIFEST_URL}" -fi - -source .repo/manifests/version.txt - -[ -s verify.asc ] && verify_key=--verify-key=verify.asc || verify_key= - -if ! native_arm64; then - script update_chroot \ - --toolchain_boards="${BOARD}" --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" -fi - -mkdir -p tmp -bin/cork download-image \ - --cache-dir=tmp \ - --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \ - --platform="${PLATFORM}" \ - --root="${DOWNLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}" \ - --verify=true $verify_key -enter lbunzip2 -k -f /mnt/host/source/tmp/flatcar_production_image.bin.bz2 - -# create folder to handle case where arm64 is missing -sudo mkdir -p chroot/usr/lib/kola/{arm64,amd64} -# copy all of the latest mantle binaries into the chroot -sudo cp -t chroot/usr/lib/kola/arm64 bin/arm64/* -sudo cp -t chroot/usr/lib/kola/amd64 bin/amd64/* -sudo cp -t chroot/usr/bin bin/[b-z]* - -if [[ "${KOLA_TESTS}" == "" ]]; then - KOLA_TESTS="*" -fi - -rm -f flatcar_test_update.gz -bin/gangue get \ - --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \ - --verify=true $verify_key \ - "${DOWNLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}/flatcar_test_update.gz" -mv flatcar_test_update.gz tmp/ - -if [ "${KOLA_TESTS}" = "*" ] || [ "$(echo "${KOLA_TESTS}" | grep 'cl.update.payload')" != "" ]; then - # First test to update from the previous release, this is done before running the real kola suite so that the qemu-latest symlink still points to the full run - rm -f flatcar_production_image.bin.bz2 - curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 "https://${GROUP}.release.flatcar-linux.net/${BOARD}/current/flatcar_production_image.bin.bz2" - mv flatcar_production_image.bin.bz2 tmp/flatcar_production_image_previous.bin.bz2 - enter lbunzip2 -k -f /mnt/host/source/tmp/flatcar_production_image_previous.bin.bz2 - enter sudo timeout --signal=SIGQUIT "${TIMEOUT}" kola run \ - --board="${BOARD}" \ - --channel="${GROUP}" \ - --parallel="${PARALLEL}" \ - --platform=qemu \ - --qemu-bios="${BIOS}" \ - --qemu-image=/mnt/host/source/tmp/flatcar_production_image_previous.bin \ - --tapfile="/mnt/host/source/${JOB_NAME##*/}_update_from_previous_release.tap" \ - --torcx-manifest=/mnt/host/source/torcx_manifest.json \ - --update-payload=/mnt/host/source/tmp/flatcar_test_update.gz \ - cl.update.payload || true -fi - -# Do not expand the kola test patterns globs -set -o noglob -enter sudo timeout --signal=SIGQUIT "${TIMEOUT}" kola run \ - --board="${BOARD}" \ - --channel="${GROUP}" \ - --parallel="${PARALLEL}" \ - --platform=qemu \ - --qemu-bios="${BIOS}" \ - --qemu-image=/mnt/host/source/tmp/flatcar_production_image.bin \ - --tapfile="/mnt/host/source/${JOB_NAME##*/}.tap" \ - --torcx-manifest=/mnt/host/source/torcx_manifest.json \ - ${KOLA_TESTS} -set +o noglob - -sudo rm -rf tmp diff --git a/jenkins/kola/qemu_uefi.sh b/jenkins/kola/qemu_uefi.sh deleted file mode 100755 index ee10ec8429..0000000000 --- a/jenkins/kola/qemu_uefi.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -set -ex - -SCRIPTFOLDER="$(dirname "$(readlink -f "$0")")" -if [[ "$NATIVE_ARM64" == true ]]; then - "${SCRIPTFOLDER}/qemu_uefi_arm64.sh" qemu_uefi -else - "${SCRIPTFOLDER}/qemu_common.sh" qemu_uefi -fi diff --git a/jenkins/kola/qemu_uefi_arm64.sh b/jenkins/kola/qemu_uefi_arm64.sh deleted file mode 100755 index 94e3223a79..0000000000 --- a/jenkins/kola/qemu_uefi_arm64.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -set -ex - -SCRIPTFOLDER="$(dirname "$(readlink -f "$0")")" -# strip $PWD prefix so that we can access the path relative to the container working directory -SCRIPTFOLDER=${SCRIPTFOLDER#$PWD/} - -DOCKER_IMG=ghcr.io/kinvolk/kola-test-runner:latest - -envarg=() -envflags=( - SSH_AUTH_SOCK - BOARD - MANIFEST_URL - SDK_URL_PATH - CHANNEL_BASE - GROUP - KOLA_TESTS - MANIFEST_TAG - DOWNLOAD_ROOT - PARALLEL - GOOGLE_APPLICATION_CREDENTIALS - NATIVE_ARM64 -) -for envvar in ${envflags[@]}; do - envarg+=( -e "${envvar}=${!envvar}" ) -done - -docker pull ${DOCKER_IMG} -exec docker run --privileged \ - --rm \ - -v /dev:/dev \ - -w /mnt/host/source \ - -v ${PWD}:/mnt/host/source \ - -v ${GOOGLE_APPLICATION_CREDENTIALS}:${GOOGLE_APPLICATION_CREDENTIALS} \ - ${SSH_AUTH_SOCK:+-v ${SSH_AUTH_SOCK}:${SSH_AUTH_SOCK}} \ - "${envarg[@]}" \ - ${DOCKER_IMG} \ - "${SCRIPTFOLDER}/qemu_common.sh" qemu_uefi diff --git a/jenkins/kola/vmware-esx.sh b/jenkins/kola/vmware-esx.sh deleted file mode 100755 index a4951ff71d..0000000000 --- a/jenkins/kola/vmware-esx.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -set -ex - -# JOB_NAME will not fit within the character limit -NAME="jenkins-${BUILD_NUMBER}" - -[ -s verify.asc ] && verify_key=--verify-key=verify.asc || verify_key= - -mkdir -p tmp -bin/cork download-image \ - --cache-dir=tmp \ - --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \ - --platform=esx \ - --root="${DOWNLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}" \ - --verify=true $verify_key - -trap 'bin/ore esx --esx-config-file "${VMWARE_ESX_CREDS}" remove-vms \ - --pattern "${NAME}*" || true' EXIT - -if [[ "${KOLA_TESTS}" == "" ]]; then - KOLA_TESTS="*" -fi - -# Delete every VM that is running because we'll use all available spots -bin/ore esx --esx-config-file "${VMWARE_ESX_CREDS}" remove-vms || true - -# Do not expand the kola test patterns globs -set -o noglob -timeout --signal=SIGQUIT 2h bin/kola run \ - --basename="${NAME}" \ - --esx-config-file "${VMWARE_ESX_CREDS}" \ - --esx-ova-path tmp/flatcar_production_vmware_ova.ova \ - --parallel=4 \ - --platform=esx \ - --channel="${GROUP}" \ - --tapfile="${JOB_NAME##*/}.tap" \ - --torcx-manifest=torcx_manifest.json \ - ${KOLA_TESTS} -set +o noglob -sudo rm -rf tmp diff --git a/jenkins/manifest.sh b/jenkins/manifest.sh deleted file mode 100755 index 94573f8966..0000000000 --- a/jenkins/manifest.sh +++ /dev/null @@ -1,149 +0,0 @@ -#!/bin/bash -set -ex -BASE=$(dirname $(readlink -f "$0")) -git -C manifest config user.name "${GIT_AUTHOR_NAME}" -git -C manifest config user.email "${GIT_AUTHOR_EMAIL}" - -COREOS_OFFICIAL=0 - -finish() { - local tag="$1" - git -C manifest tag -v "${tag}" - git -C manifest push "${BUILDS_PUSH_URL}" "refs/tags/${tag}:refs/tags/${tag}" - tee manifest.properties << EOF -MANIFEST_URL = ${BUILDS_CLONE_URL} -MANIFEST_REF = refs/tags/${tag} -MANIFEST_NAME = release.xml -COREOS_OFFICIAL = ${COREOS_OFFICIAL:-0} -EOF -} - -# Set up GPG for verifying tags. -export GNUPGHOME="${PWD}/.gnupg" -rm -rf "${GNUPGHOME}" -trap 'rm -rf "${GNUPGHOME}"' EXIT -mkdir --mode=0700 "${GNUPGHOME}" -gpg --import verify.asc -# Sometimes this directory is not created automatically making further private -# key imports fail, let's create it here as a workaround -mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/" - -# Branches are of the form remote-name/branch-name. Tags are just tag-name. -# If we have a release tag use it, for branches we need to make a tag. -if [[ "${GIT_BRANCH}" != */* ]] -then - COREOS_OFFICIAL=1 - finish "${GIT_BRANCH}" - exit -fi - -MANIFEST_BRANCH="${GIT_BRANCH##*/}" -MANIFEST_ID="${MANIFEST_BRANCH}" -# Nightly builds use the "default" manifest from flatcar-master and have the same scripts/overlay/portage branches without a "user/" prefix. -# No further exclusions are made because nothing bad happens if other branches were used. -if [[ "${MANIFEST_NAME}" = default ]] && [[ "${MANIFEST_BRANCH}" = flatcar-master ]] && \ - [[ "${SCRIPTS_REF}" = "${OVERLAY_REF}" ]] && [[ "${OVERLAY_REF}" = "${PORTAGE_REF}" ]] && \ - [[ "${SCRIPTS_REF}" != */* ]] && [[ "${SCRIPTS_REF}" != "" ]] -then - # Use SCRIPTS_REF but others also work since they have the same value - MANIFEST_ID="${SCRIPTS_REF}-nightly" -fi - -MANIFEST_NAME="${MANIFEST_NAME}.xml" -[[ -f "manifest/${MANIFEST_NAME}" ]] - -source manifest/version.txt - -if [[ "${SDK_VERSION}" == sdk-*-nightly ]] -then - # Get the SDK version from GCS - we use gsutil to get access to the bucket since it's private. - SDK_VERSION=$(docker run --rm -v "${GOOGLE_APPLICATION_CREDENTIALS}:/opt/release.json:ro" google/cloud-sdk:alpine bash -c "gcloud auth activate-service-account --key-file /opt/release.json && gsutil cat gs://flatcar-jenkins/developer/sdk/amd64/${SDK_VERSION}.txt" | tee /dev/stderr) - if [[ -z "${SDK_VERSION}" ]] - then - echo "No SDK found, retrigger the manifest job with default SDK_VERSION and SDK_URL_PATH values." - exit 1 - fi -fi - -export FLATCAR_BUILD_ID="${BUILD_ID_PREFIX}${MANIFEST_ID}-${BUILD_NUMBER}" -# Nightlies and dev builds have the current date as Flatcar version -if [[ "${MANIFEST_BRANCH}" = flatcar-master ]] -then - FLATCAR_VERSION_ID="$(date '+%Y.%m.%d')" -fi - -if [[ "${SDK_VERSION}" = sdk-new ]] -then - # Use the version of the current developer build for DOWNSTREAM=all(-full), requires a seed SDK to be set - # (releases use git tags where all this code here is not executed because the manifest - # and version.txt should not be modified, the Alpha release version.txt has to refer to - # the release to be build for its SDK version) - SDK_VERSION="${FLATCAR_VERSION_ID}+${FLATCAR_BUILD_ID}" -fi - -if [[ -n "${SDK_VERSION}" ]] -then - export FLATCAR_SDK_VERSION="${SDK_VERSION}" -fi - -# Ensure that each XML tag occupies exactly one line each by first removing all line breaks and then adding -# a line break after each tag. -# This way set_manifest_ref can find the right tag by matching for "/$reponame". -cat manifest/"${MANIFEST_NAME}" | tr '\n' ' ' | sed 's#/>#/>\n#g' > "manifest/${FLATCAR_BUILD_ID}.xml" - -set_manifest_ref() { - local reponame="$1" - local reference="$2" - # Select lines with "/$reponame" (kept as first group) and "revision" (kept as second group) and replace the value - # of "revision" (third group, not kept) with the new reference. - sed -i -E "s#(/$reponame.*)(revision=\")([^\"]*)#\1\2$reference#g" "manifest/${FLATCAR_BUILD_ID}.xml" -} - -setup_manifest_ref() { - local reponame="${1}" - local ref="${2}" - local full_ref="refs/heads/${ref}" - - if [[ -z "${ref//[0-9]}" ]]; then - full_ref="refs/pull/${ref}/head" - fi - set_manifest_ref "${reponame}" "${full_ref}" - "${BASE}/post-github-status.sh" --repo "flatcar-linux/${reponame}" --ref "${full_ref}" --status pending -} - -if [[ -n "${SCRIPTS_REF}" ]] -then - setup_manifest_ref scripts "${SCRIPTS_REF}" -fi -if [[ -n "${OVERLAY_REF}" ]] -then - setup_manifest_ref coreos-overlay "${OVERLAY_REF}" -fi -if [[ -n "${PORTAGE_REF}" ]] -then - setup_manifest_ref portage-stable "${PORTAGE_REF}" -fi - -ln -fns "${FLATCAR_BUILD_ID}.xml" manifest/default.xml -ln -fns "${FLATCAR_BUILD_ID}.xml" manifest/release.xml - -tee manifest/version.txt << EOF -FLATCAR_VERSION=${FLATCAR_VERSION_ID}+${FLATCAR_BUILD_ID} -FLATCAR_VERSION_ID=${FLATCAR_VERSION_ID} -FLATCAR_BUILD_ID=${FLATCAR_BUILD_ID} -FLATCAR_SDK_VERSION=${FLATCAR_SDK_VERSION} -EOF -# Note: You have to keep FLATCAR_VERSION in sync with the value used in the "sdk-new" case. - -# Set up GPG for signing tags. -gpg --import "${GPG_SECRET_KEY_FILE}" - -# Tag a development build manifest. -git -C manifest add "${FLATCAR_BUILD_ID}.xml" default.xml release.xml version.txt -git -C manifest commit \ - -m "${FLATCAR_BUILD_ID}: add build manifest" \ - -m "Based on ${GIT_URL} branch ${MANIFEST_BRANCH}" \ - -m "${BUILD_URL}" -git -C manifest tag -u "${SIGNING_USER}" -m "${FLATCAR_BUILD_ID}" "${FLATCAR_BUILD_ID}" - -finish "${FLATCAR_BUILD_ID}" diff --git a/jenkins/packages.sh b/jenkins/packages.sh deleted file mode 100755 index 830aafe8a6..0000000000 --- a/jenkins/packages.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/bin/bash -set -ex - -# The build may not be started without a tag value. -[ -n "${MANIFEST_TAG}" ] - -# For developer builds that are based on a non-developer release, -# we need the DOWNLOAD_ROOT variable to be the base path, keeping the -# UPLOAD_ROOT variable as the developer path. -if [[ "${RELEASE_BASE_IS_DEV}" = "false" && "${GROUP}" = "developer" && "${RELEASE_BASE}" != "" ]]; then - DOWNLOAD_ROOT=$(echo ${DOWNLOAD_ROOT} | sed 's,/developer,,'); -fi -# since /flatcar-jenkins/developer/sdk starts with a / we only use one -DOWNLOAD_ROOT_SDK="gs:/${SDK_URL_PATH}" - -# Set up GPG for verifying tags. -export GNUPGHOME="${PWD}/.gnupg" -rm -rf "${GNUPGHOME}" -trap 'rm -rf "${GNUPGHOME}"' EXIT -mkdir --mode=0700 "${GNUPGHOME}" -gpg --import verify.asc -# Sometimes this directory is not created automatically making further private -# key imports fail, let's create it here as a workaround -mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/" - -SCRIPTS_PATCH_ARG="" -OVERLAY_PATCH_ARG="" -PORTAGE_PATCH_ARG="" -if [ "$(cat scripts.patch | wc -l)" != 0 ]; then - SCRIPTS_PATCH_ARG="--scripts-patch scripts.patch" -fi -if [ "$(cat overlay.patch | wc -l)" != 0 ]; then - OVERLAY_PATCH_ARG="--overlay-patch overlay.patch" -fi -if [ "$(cat portage.patch | wc -l)" != 0 ]; then - PORTAGE_PATCH_ARG="--portage-patch portage.patch" -fi - -bin/cork create \ - --verify --verify-signature --replace \ - --sdk-url-path "${SDK_URL_PATH}" \ - --json-key "${GOOGLE_APPLICATION_CREDENTIALS}" \ - ${SCRIPTS_PATCH_ARG} ${OVERLAY_PATCH_ARG} ${PORTAGE_PATCH_ARG} \ - --manifest-branch "refs/tags/${MANIFEST_TAG}" \ - --manifest-name "${MANIFEST_NAME}" \ - --manifest-url "${MANIFEST_URL}" \ - --sdk-url=storage.googleapis.com - -enter() { - local verify_key= - # Run in a subshell to clean some gangue files on exit without - # possibly clobbering the global EXIT trap. - ( - trap 'sudo rm -f chroot/etc/portage/gangue.*' EXIT - [ -s verify.asc ] && - sudo ln -f verify.asc chroot/etc/portage/gangue.asc && - verify_key=--verify-key=/etc/portage/gangue.asc - sudo ln -f "${GOOGLE_APPLICATION_CREDENTIALS}" \ - chroot/etc/portage/gangue.json - bin/cork enter --bind-gpg-agent=false -- env \ - FLATCAR_DEV_BUILDS="${DOWNLOAD_ROOT}" \ - FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \ - {FETCH,RESUME}COMMAND_GS="/mnt/host/source/bin/gangue get \ ---json-key=/etc/portage/gangue.json $verify_key \ -"'"${URI}" "${DISTDIR}/${FILE}"' \ - "$@" - ) -} - -script() { - enter "/mnt/host/source/src/scripts/$@" -} - -source .repo/manifests/version.txt -export FLATCAR_BUILD_ID - -# Set up GPG for signing uploads. -gpg --import "${GPG_SECRET_KEY_FILE}" - -script update_chroot \ - --toolchain_boards="${BOARD}" --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}" - -script setup_board \ - --board="${BOARD}" \ - --getbinpkgver=${RELEASE_BASE:-"${FLATCAR_VERSION}" --toolchainpkgonly} \ - --skip_chroot_upgrade \ - --force - -script build_packages \ - --board="${BOARD}" \ - --getbinpkgver=${RELEASE_BASE:-"${FLATCAR_VERSION}" --toolchainpkgonly} \ - --usepkg_exclude="${BINARY_PACKAGES_TO_EXCLUDE}" \ - --skip_chroot_upgrade \ - --skip_torcx_store \ - --sign="${SIGNING_USER}" \ - --sign_digests="${SIGNING_USER}" \ - --upload_root="${UPLOAD_ROOT}" \ - --upload - -script build_torcx_store \ - --board="${BOARD}" \ - --sign="${SIGNING_USER}" \ - --sign_digests="${SIGNING_USER}" \ - --upload_root="${UPLOAD_ROOT}" \ - --torcx_upload_root="${TORCX_PKG_DOWNLOAD_ROOT}" \ - --tectonic_torcx_download_root="${TECTONIC_TORCX_DOWNLOAD_ROOT}" \ - --upload - -if [[ "${GROUP}" = "developer" ]] -then - GROUP="${CHANNEL_BASE}" -fi - -# Update entry for latest nightly build reference (there are no symlinks in GCS and it is also good to keep it deterministic) -if [[ "${FLATCAR_BUILD_ID}" == *-*-nightly-* ]] -then - # Extract the nightly name like "flatcar-MAJOR-nightly" from "dev-flatcar-MAJOR-nightly-NUMBER" - NAME=$(echo "${FLATCAR_BUILD_ID}" | grep -o "dev-.*-nightly" | cut -d - -f 2-) - echo "${FLATCAR_VERSION}" | bin/cork enter --bind-gpg-agent=false -- gsutil cp - "${UPLOAD_ROOT}/boards/${BOARD}/${NAME}.txt" -fi diff --git a/jenkins/post-github-status.sh b/jenkins/post-github-status.sh deleted file mode 100755 index 804873c7f8..0000000000 --- a/jenkins/post-github-status.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -SHFLAGS=$(dirname $(readlink -f "$0"))/../lib/shflags/shflags -. "${SHFLAGS}" || exit 1 - -DEFINE_string repo "" "Name of the repository to which to post status" -DEFINE_string ref "" "Reference from which to figure out commit" -DEFINE_string github_token "${GITHUB_TOKEN}" "Github Personal Access Token used to submit the commit status" -DEFINE_string status "pending" "Status to submit for commit. [failure,pending,success,error]" -DEFINE_string context "ci/jenkins" "Context to use for commit status." -DEFINE_boolean verbose "${FLAGS_FALSE}" "Show curl output" - -# Parse command line -FLAGS "$@" || exit 1 -eval set -- "${FLAGS_ARGV}" - -if [ -z "${FLAGS_repo}" ]; then - echo >&2 "Error: --repo is required" - exit 1 -fi -if [ -z "${FLAGS_ref}" ]; then - echo >&2 "Error: --ref is required" - exit 1 -fi -if [ -z "${FLAGS_github_token}" ]; then - echo >&2 "Error: --github_token is required" - exit 1 -fi - -CURLOPTS="-sS" -if [[ "${FLAGS_verbose}" -eq "${FLAGS_true}" ]]; then - CURLOPTS="" -fi - -GITHUB_API="https://api.github.com" -# BUILD_URL = JENKINS_URL + JOB_NAME + BUILD_NUMBER -target_url="${BUILD_URL}cldsv" -commit=$(git ls-remote "https://github.com/${FLAGS_repo}" "${FLAGS_ref}"| cut -f1) -if [ -z "${commit}" ]; then - echo >&2 "Can't figure out commit for repo ${FLAGS_repo} ref ${FLAGS_ref}" - exit 2 -fi - -curl ${CURLOPTS} "${GITHUB_API}/repos/${FLAGS_repo}/statuses/${commit}" \ - -H "Content-Type: application/json" \ - -H "Authorization: token ${FLAGS_github_token}" \ - -X POST -d @- <