From c8dd87c095f1488c09576fc94c3facb79d247076 Mon Sep 17 00:00:00 2001
From: Jeremi Piotrowski <jpiotrowski@microsoft.com>
Date: Fri, 3 Sep 2021 16:01:11 +0200
Subject: [PATCH] jenkins: add script to run kola arm64 tests under docker

Included is a dockerfile that installs system deps of kola in an debian:11
image. For the test script, the control flow is:

qemu_uefi.sh
  qemu_uefi_arm64.sh
    (docker)
      qemu_common.sh

qemu_common uses the 'NATIVE_ARM64' variable passed by the jenkins job to control the behavior.
The differences are:

* use git directly to fetch (and verify) the manifest
* setup some symlinks so that /var/tmp is on the same BTRFS partition as $PWD/tmp
* setup symlinks so that we don't have to fixup installation of mantle to chroot
* run things directly instead of in chroot through cork

The whole script is executed as root, because kola requires root privileges
anyway and making kvm and sudo work with an arbitrary host user inside the
container would require a custom entrypoint to setup groups.

Signed-off-by: Jeremi Piotrowski <jpiotrowski@microsoft.com>
---
 jenkins/kola/Dockerfile.kola-test-runner |  4 ++
 jenkins/kola/qemu_common.sh              | 59 +++++++++++++++++++-----
 jenkins/kola/qemu_uefi.sh                |  6 ++-
 jenkins/kola/qemu_uefi_arm64.sh          | 40 ++++++++++++++++
 4 files changed, 96 insertions(+), 13 deletions(-)
 create mode 100644 jenkins/kola/Dockerfile.kola-test-runner
 create mode 100755 jenkins/kola/qemu_uefi_arm64.sh

diff --git a/jenkins/kola/Dockerfile.kola-test-runner b/jenkins/kola/Dockerfile.kola-test-runner
new file mode 100644
index 0000000000..cae0d4ba7a
--- /dev/null
+++ b/jenkins/kola/Dockerfile.kola-test-runner
@@ -0,0 +1,4 @@
+FROM debian:11
+
+RUN apt-get update && \
+  apt-get install -y qemu-system-aarch64 qemu-efi-aarch64 lbzip2 sudo dnsmasq gnupg2 git curl iptables
diff --git a/jenkins/kola/qemu_common.sh b/jenkins/kola/qemu_common.sh
index f6f0a3946b..73711f5ce0 100755
--- a/jenkins/kola/qemu_common.sh
+++ b/jenkins/kola/qemu_common.sh
@@ -12,12 +12,36 @@ else
   echo "Unknown platform: \"${PLATFORM}\""
 fi
 
-sudo rm -rf *.tap src/scripts/_kola_temp tmp _kola_temp*
-
-enter() {
-  bin/cork enter --bind-gpg-agent=false -- "$@"
+native_arm64() {
+  [[ "${NATIVE_ARM64}" == true ]]
 }
 
+sudo rm -rf *.tap src/scripts/_kola_temp tmp _kola_temp* _tmp
+
+if native_arm64 ; then
+  # for kola reflinking
+  sudo rm -rf /var/tmp
+  mkdir -p _tmp
+  chmod 1777 _tmp
+  ln -s "$PWD/_tmp" /var/tmp
+  # use arm64 mantle bins
+  rm -rf bin
+  mv bin.arm64 bin
+  # simulate SDK folder structure
+  mkdir -p src
+  ln -s .. src/scripts
+  sudo rm -f chroot
+  ln -s / chroot
+
+  enter() {
+    "$@"
+  }
+else
+  enter() {
+    bin/cork enter --bind-gpg-agent=false -- "$@"
+  }
+fi
+
 # Set up GPG for verifying tags.
 export GNUPGHOME="${PWD}/.gnupg"
 rm -rf "${GNUPGHOME}"
@@ -30,13 +54,24 @@ mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/"
 
 DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}"
 
-bin/cork update \
-    --create --downgrade-replace --verify --verify-signature --verbose \
-    --sdk-url-path "${SDK_URL_PATH}" \
-    --force-sync \
-    --manifest-branch "refs/tags/${MANIFEST_TAG}" \
-    --manifest-name "${MANIFEST_NAME}" \
-    --manifest-url "${MANIFEST_URL}" -- --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}"
+if native_arm64 ; then
+  mkdir -p .repo/
+  if [ ! -e .repo/manifests ]; then
+    mkdir -p ~/.ssh
+    ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts
+    git clone "${MANIFEST_URL}" .repo/manifests
+  fi
+  git -C .repo/manifests tag -v "${MANIFEST_TAG}"
+  git -C .repo/manifests checkout "${MANIFEST_TAG}"
+else
+  bin/cork update \
+      --create --downgrade-replace --verify --verify-signature --verbose \
+      --sdk-url-path "${SDK_URL_PATH}" \
+      --force-sync \
+      --manifest-branch "refs/tags/${MANIFEST_TAG}" \
+      --manifest-name "${MANIFEST_NAME}" \
+      --manifest-url "${MANIFEST_URL}" -- --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}"
+fi
 source .repo/manifests/version.txt
 
 [ -s verify.asc ] && verify_key=--verify-key=verify.asc || verify_key=
@@ -51,7 +86,7 @@ bin/cork download-image \
 enter lbunzip2 -k -f /mnt/host/source/tmp/flatcar_production_image.bin.bz2
 
 # create folder to handle case where arm64 is missing
-sudo mkdir -p chroot/usr/lib/kola/arm64
+sudo mkdir -p chroot/usr/lib/kola/{arm64,amd64}
 # copy all of the latest mantle binaries into the chroot
 sudo cp -t chroot/usr/lib/kola/arm64 bin/arm64/*
 sudo cp -t chroot/usr/lib/kola/amd64 bin/amd64/*
diff --git a/jenkins/kola/qemu_uefi.sh b/jenkins/kola/qemu_uefi.sh
index 5f28b82254..ee10ec8429 100755
--- a/jenkins/kola/qemu_uefi.sh
+++ b/jenkins/kola/qemu_uefi.sh
@@ -2,4 +2,8 @@
 set -ex
 
 SCRIPTFOLDER="$(dirname "$(readlink -f "$0")")"
-"${SCRIPTFOLDER}/qemu_common.sh" qemu_uefi
+if [[ "$NATIVE_ARM64" == true ]]; then
+  "${SCRIPTFOLDER}/qemu_uefi_arm64.sh" qemu_uefi
+else
+  "${SCRIPTFOLDER}/qemu_common.sh" qemu_uefi
+fi
diff --git a/jenkins/kola/qemu_uefi_arm64.sh b/jenkins/kola/qemu_uefi_arm64.sh
new file mode 100755
index 0000000000..94e3223a79
--- /dev/null
+++ b/jenkins/kola/qemu_uefi_arm64.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -ex
+
+SCRIPTFOLDER="$(dirname "$(readlink -f "$0")")"
+# strip $PWD prefix so that we can access the path relative to the container working directory
+SCRIPTFOLDER=${SCRIPTFOLDER#$PWD/}
+
+DOCKER_IMG=ghcr.io/kinvolk/kola-test-runner:latest
+
+envarg=()
+envflags=(
+  SSH_AUTH_SOCK
+  BOARD
+  MANIFEST_URL
+  SDK_URL_PATH
+  CHANNEL_BASE
+  GROUP
+  KOLA_TESTS
+  MANIFEST_TAG
+  DOWNLOAD_ROOT
+  PARALLEL
+  GOOGLE_APPLICATION_CREDENTIALS
+  NATIVE_ARM64
+)
+for envvar in ${envflags[@]}; do
+  envarg+=( -e "${envvar}=${!envvar}" )
+done
+
+docker pull ${DOCKER_IMG}
+exec docker run --privileged \
+  --rm \
+  -v /dev:/dev \
+  -w /mnt/host/source \
+  -v ${PWD}:/mnt/host/source \
+  -v ${GOOGLE_APPLICATION_CREDENTIALS}:${GOOGLE_APPLICATION_CREDENTIALS} \
+  ${SSH_AUTH_SOCK:+-v ${SSH_AUTH_SOCK}:${SSH_AUTH_SOCK}} \
+  "${envarg[@]}" \
+  ${DOCKER_IMG} \
+  "${SCRIPTFOLDER}/qemu_common.sh" qemu_uefi