jenkins: add script to run kola arm64 tests under docker

Included is a dockerfile that installs system deps of kola in an debian:11
image. For the test script, the control flow is:

qemu_uefi.sh
  qemu_uefi_arm64.sh
    (docker)
      qemu_common.sh

qemu_common uses the 'NATIVE_ARM64' variable passed by the jenkins job to control the behavior.
The differences are:

* use git directly to fetch (and verify) the manifest
* setup some symlinks so that /var/tmp is on the same BTRFS partition as $PWD/tmp
* setup symlinks so that we don't have to fixup installation of mantle to chroot
* run things directly instead of in chroot through cork

The whole script is executed as root, because kola requires root privileges
anyway and making kvm and sudo work with an arbitrary host user inside the
container would require a custom entrypoint to setup groups.

Signed-off-by: Jeremi Piotrowski <jpiotrowski@microsoft.com>

jenkins/kola/Dockerfile.kola-test-runner

@@ -0,0 +1,4 @@
+FROM debian:11
+
+RUN apt-get update && \
+  apt-get install -y qemu-system-aarch64 qemu-efi-aarch64 lbzip2 sudo dnsmasq gnupg2 git curl iptables

jenkins/kola/qemu_common.sh

@@ -12,12 +12,36 @@ else
   echo "Unknown platform: \"${PLATFORM}\""
 fi
 
-sudo rm -rf *.tap src/scripts/_kola_temp tmp _kola_temp*
-
-enter() {
-  bin/cork enter --bind-gpg-agent=false -- "$@"
+native_arm64() {
+  [[ "${NATIVE_ARM64}" == true ]]
 }
 
+sudo rm -rf *.tap src/scripts/_kola_temp tmp _kola_temp* _tmp
+
+if native_arm64 ; then
+  # for kola reflinking
+  sudo rm -rf /var/tmp
+  mkdir -p _tmp
+  chmod 1777 _tmp
+  ln -s "$PWD/_tmp" /var/tmp
+  # use arm64 mantle bins
+  rm -rf bin
+  mv bin.arm64 bin
+  # simulate SDK folder structure
+  mkdir -p src
+  ln -s .. src/scripts
+  sudo rm -f chroot
+  ln -s / chroot
+
+  enter() {
+    "$@"
+  }
+else
+  enter() {
+    bin/cork enter --bind-gpg-agent=false -- "$@"
+  }
+fi
+
 # Set up GPG for verifying tags.
 export GNUPGHOME="${PWD}/.gnupg"
 rm -rf "${GNUPGHOME}"
@@ -30,13 +54,24 @@ mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/"
 
 DOWNLOAD_ROOT_SDK="https://storage.googleapis.com${SDK_URL_PATH}"
 
-bin/cork update \
-    --create --downgrade-replace --verify --verify-signature --verbose \
-    --sdk-url-path "${SDK_URL_PATH}" \
-    --force-sync \
-    --manifest-branch "refs/tags/${MANIFEST_TAG}" \
-    --manifest-name "${MANIFEST_NAME}" \
-    --manifest-url "${MANIFEST_URL}" -- --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}"
+if native_arm64 ; then
+  mkdir -p .repo/
+  if [ ! -e .repo/manifests ]; then
+    mkdir -p ~/.ssh
+    ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts
+    git clone "${MANIFEST_URL}" .repo/manifests
+  fi
+  git -C .repo/manifests tag -v "${MANIFEST_TAG}"
+  git -C .repo/manifests checkout "${MANIFEST_TAG}"
+else
+  bin/cork update \
+      --create --downgrade-replace --verify --verify-signature --verbose \
+      --sdk-url-path "${SDK_URL_PATH}" \
+      --force-sync \
+      --manifest-branch "refs/tags/${MANIFEST_TAG}" \
+      --manifest-name "${MANIFEST_NAME}" \
+      --manifest-url "${MANIFEST_URL}" -- --dev_builds_sdk="${DOWNLOAD_ROOT_SDK}"
+fi
 source .repo/manifests/version.txt
 
 [ -s verify.asc ] && verify_key=--verify-key=verify.asc || verify_key=
@@ -51,7 +86,7 @@ bin/cork download-image \
 enter lbunzip2 -k -f /mnt/host/source/tmp/flatcar_production_image.bin.bz2
 
 # create folder to handle case where arm64 is missing
-sudo mkdir -p chroot/usr/lib/kola/arm64
+sudo mkdir -p chroot/usr/lib/kola/{arm64,amd64}
 # copy all of the latest mantle binaries into the chroot
 sudo cp -t chroot/usr/lib/kola/arm64 bin/arm64/*
 sudo cp -t chroot/usr/lib/kola/amd64 bin/amd64/*

jenkins/kola/qemu_uefi.sh

@@ -2,4 +2,8 @@
 set -ex
 
 SCRIPTFOLDER="$(dirname "$(readlink -f "$0")")"
-"${SCRIPTFOLDER}/qemu_common.sh" qemu_uefi
+if [[ "$NATIVE_ARM64" == true ]]; then
+  "${SCRIPTFOLDER}/qemu_uefi_arm64.sh" qemu_uefi
+else
+  "${SCRIPTFOLDER}/qemu_common.sh" qemu_uefi
+fi

jenkins/kola/qemu_uefi_arm64.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -ex
+
+SCRIPTFOLDER="$(dirname "$(readlink -f "$0")")"
+# strip $PWD prefix so that we can access the path relative to the container working directory
+SCRIPTFOLDER=${SCRIPTFOLDER#$PWD/}
+
+DOCKER_IMG=ghcr.io/kinvolk/kola-test-runner:latest
+
+envarg=()
+envflags=(
+  SSH_AUTH_SOCK
+  BOARD
+  MANIFEST_URL
+  SDK_URL_PATH
+  CHANNEL_BASE
+  GROUP
+  KOLA_TESTS
+  MANIFEST_TAG
+  DOWNLOAD_ROOT
+  PARALLEL
+  GOOGLE_APPLICATION_CREDENTIALS
+  NATIVE_ARM64
+)
+for envvar in ${envflags[@]}; do
+  envarg+=( -e "${envvar}=${!envvar}" )
+done
+
+docker pull ${DOCKER_IMG}
+exec docker run --privileged \
+  --rm \
+  -v /dev:/dev \
+  -w /mnt/host/source \
+  -v ${PWD}:/mnt/host/source \
+  -v ${GOOGLE_APPLICATION_CREDENTIALS}:${GOOGLE_APPLICATION_CREDENTIALS} \
+  ${SSH_AUTH_SOCK:+-v ${SSH_AUTH_SOCK}:${SSH_AUTH_SOCK}} \
+  "${envarg[@]}" \
+  ${DOCKER_IMG} \
+  "${SCRIPTFOLDER}/qemu_common.sh" qemu_uefi