From 1b70f59cd0519683f3e54ad1580b8dfd6b72c4ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kai=20L=C3=BCke?= <kailuke@microsoft.com>
Date: Mon, 26 Jul 2021 15:01:24 +0200
Subject: [PATCH 1/2] jenkins/kola: share a single qemu script file

---
 jenkins/kola/qemu.sh        | 74 +------------------------------
 jenkins/kola/qemu_common.sh | 86 +++++++++++++++++++++++++++++++++++++
 jenkins/kola/qemu_uefi.sh   | 74 +------------------------------
 3 files changed, 90 insertions(+), 144 deletions(-)
 create mode 100755 jenkins/kola/qemu_common.sh

diff --git a/jenkins/kola/qemu.sh b/jenkins/kola/qemu.sh
index 82ae81edcd..1496d50edf 100755
--- a/jenkins/kola/qemu.sh
+++ b/jenkins/kola/qemu.sh
@@ -1,75 +1,5 @@
 #!/bin/bash
 set -ex
 
-sudo rm -rf *.tap src/scripts/_kola_temp tmp _kola_temp*
-
-enter() {
-  bin/cork enter --bind-gpg-agent=false -- "$@"
-}
-
-# Set up GPG for verifying tags.
-export GNUPGHOME="${PWD}/.gnupg"
-rm -rf "${GNUPGHOME}"
-trap 'rm -rf "${GNUPGHOME}"' EXIT
-mkdir --mode=0700 "${GNUPGHOME}"
-gpg --import verify.asc
-# Sometimes this directory is not created automatically making further private
-# key imports fail, let's create it here as a workaround
-mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/"
-
-DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}"
-
-bin/cork update \
-    --create --downgrade-replace --verify --verify-signature --verbose \
-    --sdk-url-path "${SDK_URL_PATH}" \
-    --force-sync \
-    --manifest-branch "refs/tags/${MANIFEST_TAG}" \
-    --manifest-name "${MANIFEST_NAME}" \
-    --manifest-url "${MANIFEST_URL}" -- --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}"
-source .repo/manifests/version.txt
-
-[ -s verify.asc ] && verify_key=--verify-key=verify.asc || verify_key=
-
-mkdir -p tmp
-bin/cork download-image \
-    --cache-dir=tmp \
-    --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \
-    --platform=qemu \
-    --root="${DOWNLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}" \
-    --verify=true $verify_key
-enter lbunzip2 -k -f /mnt/host/source/tmp/flatcar_production_image.bin.bz2
-
-# create folder to handle case where arm64 is missing
-sudo mkdir -p chroot/usr/lib/kola/arm64
-# copy all of the latest mantle binaries into the chroot
-sudo cp -t chroot/usr/lib/kola/arm64 bin/arm64/*
-sudo cp -t chroot/usr/lib/kola/amd64 bin/amd64/*
-sudo cp -t chroot/usr/bin bin/[b-z]*
-
-if [[ "${KOLA_TESTS}" == "" ]]; then
-  KOLA_TESTS="*"
-fi
-
-rm -f flatcar_test_update.gz
-bin/gangue get \
-    --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \
-    --verify=true $verify_key \
-    "${DOWNLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}/flatcar_test_update.gz"
-mv flatcar_test_update.gz tmp/
-
-# Do not expand the kola test patterns globs
-set -o noglob
-enter sudo timeout --signal=SIGQUIT 12h kola run \
-    --board="${BOARD}" \
-    --channel="${GROUP}" \
-    --parallel="${PARALLEL}" \
-    --platform=qemu \
-    --qemu-bios=bios-256k.bin \
-    --qemu-image=/mnt/host/source/tmp/flatcar_production_image.bin \
-    --tapfile="/mnt/host/source/${JOB_NAME##*/}.tap" \
-    --torcx-manifest=/mnt/host/source/torcx_manifest.json \
-    --update-payload=/mnt/host/source/tmp/flatcar_test_update.gz \
-    ${KOLA_TESTS}
-set +o noglob
-
-sudo rm -rf tmp
+SCRIPTFOLDER="$(dirname "$(readlink -f "$0")")"
+"${SCRIPTFOLDER}/qemu_common.sh" qemu
diff --git a/jenkins/kola/qemu_common.sh b/jenkins/kola/qemu_common.sh
new file mode 100755
index 0000000000..cfffd64a2e
--- /dev/null
+++ b/jenkins/kola/qemu_common.sh
@@ -0,0 +1,86 @@
+#!/bin/bash
+set -ex
+
+PLATFORM="$1"
+if [ "${PLATFORM}" = qemu ]; then
+  TIMEOUT="12h"
+  BIOS="bios-256k.bin"
+elif [ "${PLATFORM}" = qemu_uefi ]; then
+  TIMEOUT="14h"
+  BIOS="/mnt/host/source/tmp/flatcar_production_qemu_uefi_efi_code.fd"
+else
+  echo "Unknown platform: \"${PLATFORM}\""
+fi
+
+sudo rm -rf *.tap src/scripts/_kola_temp tmp _kola_temp*
+
+enter() {
+  bin/cork enter --bind-gpg-agent=false -- "$@"
+}
+
+# Set up GPG for verifying tags.
+export GNUPGHOME="${PWD}/.gnupg"
+rm -rf "${GNUPGHOME}"
+trap 'rm -rf "${GNUPGHOME}"' EXIT
+mkdir --mode=0700 "${GNUPGHOME}"
+gpg --import verify.asc
+# Sometimes this directory is not created automatically making further private
+# key imports fail, let's create it here as a workaround
+mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/"
+
+DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}"
+
+bin/cork update \
+    --create --downgrade-replace --verify --verify-signature --verbose \
+    --sdk-url-path "${SDK_URL_PATH}" \
+    --force-sync \
+    --manifest-branch "refs/tags/${MANIFEST_TAG}" \
+    --manifest-name "${MANIFEST_NAME}" \
+    --manifest-url "${MANIFEST_URL}" -- --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}"
+source .repo/manifests/version.txt
+
+[ -s verify.asc ] && verify_key=--verify-key=verify.asc || verify_key=
+
+mkdir -p tmp
+bin/cork download-image \
+    --cache-dir=tmp \
+    --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \
+    --platform="${PLATFORM}" \
+    --root="${DOWNLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}" \
+    --verify=true $verify_key
+enter lbunzip2 -k -f /mnt/host/source/tmp/flatcar_production_image.bin.bz2
+
+# create folder to handle case where arm64 is missing
+sudo mkdir -p chroot/usr/lib/kola/arm64
+# copy all of the latest mantle binaries into the chroot
+sudo cp -t chroot/usr/lib/kola/arm64 bin/arm64/*
+sudo cp -t chroot/usr/lib/kola/amd64 bin/amd64/*
+sudo cp -t chroot/usr/bin bin/[b-z]*
+
+if [[ "${KOLA_TESTS}" == "" ]]; then
+  KOLA_TESTS="*"
+fi
+
+rm -f flatcar_test_update.gz
+bin/gangue get \
+    --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \
+    --verify=true $verify_key \
+    "${DOWNLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}/flatcar_test_update.gz"
+mv flatcar_test_update.gz tmp/
+
+# Do not expand the kola test patterns globs
+set -o noglob
+enter sudo timeout --signal=SIGQUIT "${TIMEOUT}" kola run \
+    --board="${BOARD}" \
+    --channel="${GROUP}" \
+    --parallel="${PARALLEL}" \
+    --platform=qemu \
+    --qemu-bios="${BIOS}" \
+    --qemu-image=/mnt/host/source/tmp/flatcar_production_image.bin \
+    --tapfile="/mnt/host/source/${JOB_NAME##*/}.tap" \
+    --torcx-manifest=/mnt/host/source/torcx_manifest.json \
+    --update-payload=/mnt/host/source/tmp/flatcar_test_update.gz \
+    ${KOLA_TESTS}
+set +o noglob
+
+sudo rm -rf tmp
diff --git a/jenkins/kola/qemu_uefi.sh b/jenkins/kola/qemu_uefi.sh
index 59b3af5e38..5f28b82254 100755
--- a/jenkins/kola/qemu_uefi.sh
+++ b/jenkins/kola/qemu_uefi.sh
@@ -1,75 +1,5 @@
 #!/bin/bash
 set -ex
 
-sudo rm -rf *.tap src/scripts/_kola_temp tmp _kola_temp*
-
-enter() {
-  bin/cork enter --bind-gpg-agent=false -- "$@"
-}
-
-# Set up GPG for verifying tags.
-export GNUPGHOME="${PWD}/.gnupg"
-rm -rf "${GNUPGHOME}"
-trap 'rm -rf "${GNUPGHOME}"' EXIT
-mkdir --mode=0700 "${GNUPGHOME}"
-gpg --import verify.asc
-# Sometimes this directory is not created automatically making further private
-# key imports fail, let's create it here as a workaround
-mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/"
-
-DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}"
-
-bin/cork update \
-    --create --downgrade-replace --verify --verify-signature --verbose \
-    --sdk-url-path "${SDK_URL_PATH}" \
-    --force-sync \
-    --manifest-branch "refs/tags/${MANIFEST_TAG}" \
-    --manifest-name "${MANIFEST_NAME}" \
-    --manifest-url "${MANIFEST_URL}" -- --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}"
-source .repo/manifests/version.txt
-
-[ -s verify.asc ] && verify_key=--verify-key=verify.asc || verify_key=
-
-mkdir -p tmp
-bin/cork download-image \
-    --cache-dir=tmp \
-    --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \
-    --platform=qemu_uefi \
-    --root="${DOWNLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}" \
-    --verify=true $verify_key
-enter lbunzip2 -k -f /mnt/host/source/tmp/flatcar_production_image.bin.bz2
-
-# create folder to handle case where arm64 is missing
-sudo mkdir -p chroot/usr/lib/kola/arm64
-# copy all of the latest mantle binaries into the chroot
-sudo cp -t chroot/usr/lib/kola/arm64 bin/arm64/*
-sudo cp -t chroot/usr/lib/kola/amd64 bin/amd64/*
-sudo cp -t chroot/usr/bin bin/[b-z]*
-
-if [[ "${KOLA_TESTS}" == "" ]]; then
-  KOLA_TESTS="*"
-fi
-
-rm -f flatcar_test_update.gz
-bin/gangue get \
-    --json-key="${GOOGLE_APPLICATION_CREDENTIALS}" \
-    --verify=true $verify_key \
-    "${DOWNLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}/flatcar_test_update.gz"
-mv flatcar_test_update.gz tmp/
-
-# Do not expand the kola test patterns globs
-set -o noglob
-enter sudo timeout --signal=SIGQUIT 14h kola run \
-    --board="${BOARD}" \
-    --channel="${GROUP}" \
-    --parallel="${PARALLEL}" \
-    --platform=qemu \
-    --qemu-bios=/mnt/host/source/tmp/flatcar_production_qemu_uefi_efi_code.fd \
-    --qemu-image=/mnt/host/source/tmp/flatcar_production_image.bin \
-    --tapfile="/mnt/host/source/${JOB_NAME##*/}.tap" \
-    --torcx-manifest=/mnt/host/source/torcx_manifest.json \
-    --update-payload=/mnt/host/source/tmp/flatcar_test_update.gz \
-    ${KOLA_TESTS}
-set +o noglob
-
-sudo rm -rf tmp
+SCRIPTFOLDER="$(dirname "$(readlink -f "$0")")"
+"${SCRIPTFOLDER}/qemu_common.sh" qemu_uefi

From e96c1c5e6d56ef10be19cd512e1c1fbc93bb6069 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kai=20L=C3=BCke?= <kailuke@microsoft.com>
Date: Mon, 26 Jul 2021 15:23:53 +0200
Subject: [PATCH 2/2] jenkins/kola/qemu(_uefi): run update test from previous
 release

The newly enabled update test performs an update from the built image
to itself. This is useful to test that the update mechanism didn't
break but it doesn't say if the built image will be accepted as update
from the previous official release.

Introduce an additional kola run that begins from the previous official
release and tests to update to the built image. Since the test does two
updates it also covers the case of updating from the built image to the
built image. Thus, we can skip the test in the normal run.
This new kola run is done first to keep the qemu-latest symlink valid
for the main test suite.
---
 jenkins/kola/qemu_common.sh | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/jenkins/kola/qemu_common.sh b/jenkins/kola/qemu_common.sh
index cfffd64a2e..d447dca6cc 100755
--- a/jenkins/kola/qemu_common.sh
+++ b/jenkins/kola/qemu_common.sh
@@ -68,6 +68,25 @@ bin/gangue get \
     "${DOWNLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}/flatcar_test_update.gz"
 mv flatcar_test_update.gz tmp/
 
+if [ "${KOLA_TESTS}" = "*" ] || [ "$(echo "${KOLA_TESTS}" | grep 'cl.update.payload')" != "" ]; then
+  # First test to update from the previous release, this is done before running the real kola suite so that the qemu-latest symlink still points to the full run
+  rm -f flatcar_production_image.bin.bz2
+  curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 "https://${GROUP}.release.flatcar-linux.net/${BOARD}/current/flatcar_production_image.bin.bz2"
+  mv flatcar_production_image.bin.bz2 tmp/flatcar_production_image_previous.bin.bz2
+  enter lbunzip2 -k -f /mnt/host/source/tmp/flatcar_production_image_previous.bin.bz2
+  enter sudo timeout --signal=SIGQUIT "${TIMEOUT}" kola run \
+    --board="${BOARD}" \
+    --channel="${GROUP}" \
+    --parallel="${PARALLEL}" \
+    --platform=qemu \
+    --qemu-bios="${BIOS}" \
+    --qemu-image=/mnt/host/source/tmp/flatcar_production_image_previous.bin \
+    --tapfile="/mnt/host/source/${JOB_NAME##*/}_update_from_previous_release.tap" \
+    --torcx-manifest=/mnt/host/source/torcx_manifest.json \
+    --update-payload=/mnt/host/source/tmp/flatcar_test_update.gz \
+    cl.update.payload
+fi
+
 # Do not expand the kola test patterns globs
 set -o noglob
 enter sudo timeout --signal=SIGQUIT "${TIMEOUT}" kola run \
@@ -79,7 +98,6 @@ enter sudo timeout --signal=SIGQUIT "${TIMEOUT}" kola run \
     --qemu-image=/mnt/host/source/tmp/flatcar_production_image.bin \
     --tapfile="/mnt/host/source/${JOB_NAME##*/}.tap" \
     --torcx-manifest=/mnt/host/source/torcx_manifest.json \
-    --update-payload=/mnt/host/source/tmp/flatcar_test_update.gz \
     ${KOLA_TESTS}
 set +o noglob