Merge pull request #288 from flatcar-linux/krnowak/digital-ocean

ci-automation: Initial test script for Digital Ocean

ci-automation/ci-config.env

@@ -17,6 +17,8 @@ CONTAINER_REGISTRY="ghcr.io/flatcar-linux"
 
 GC_BUCKET="flatcar-linux"
 
+DEFAULT_HTTP_IMAGE_URL_TEMPLATE="@PROTO@://${BUILDCACHE_SERVER}/images/@ARCH@/@VERNUM@"
+
 if ! command -v pigz > /dev/null; then
   # No PIGZ on Flatcar
   PIGZ="docker run --rm -i ghcr.io/flatcar-linux/pigz --fast"
@@ -82,3 +84,14 @@ GCE_IMAGE_NAME="flatcar_production_gce.tar.gz"
 GCE_GCS_IMAGE_UPLOAD="gs://flatcar-jenkins/developer/gce-ci"
 GCE_MACHINE_TYPE="${GCE_MACHINE_TYPE:-n1-standard-2}"
 GCE_PARALLEL="${PARALLEL_TESTS:-4}"
+
+# -- Digital Ocean --
+
+# Use the "@PROTO@" "@ARCH@", "@CHANNEL@" and "@VERNUM@" placeholders. They will
+# be replaced.
+: ${DIGITALOCEAN_IMAGE_URL_TEMPLATE:="${DEFAULT_HTTP_IMAGE_URL_TEMPLATE}/flatcar_production_digitalocean_image.bin.bz2"}
+: ${DIGITALOCEAN_REGION:='sfo3'}
+: ${DIGITALOCEAN_MACHINE_SIZE:='s-2vcpu-2gb'}
+DIGITALOCEAN_PARALLEL="${PARALLEL_TESTS:-8}"
+# DIGITALOCEAN_TOKEN_JSON env var is used for credentials, and should
+# come from sdk_container/.env. It must be base64-encoded.

ci-automation/ci_automation_common.sh

@@ -10,6 +10,8 @@ source ci-automation/ci-config.env
 : ${PIGZ:=pigz}
 : ${docker:=docker}
 
+: ${TEST_WORK_DIR:='__TESTS__'}
+
 function init_submodules() {
     git submodule init
     git submodule update
@@ -224,3 +226,76 @@ function docker_image_from_registry_or_buildcache() {
     docker_image_from_buildcache "${image}" "${version}"
 }
 # --
+
+# Called by vendor test in case of complete failure not eligible for
+# reruns (like trying to run tests on unsupported architecture).
+function break_retest_cycle() {
+    local work_dir=$(dirname "${PWD}")
+    local dir=$(basename "${work_dir}")
+
+    if [[ "${dir}" != "${TEST_WORK_DIR}" ]]; then
+        echo "Not breaking retest cycle, expected test work dir to be a parent directory" >&2
+        return
+    fi
+    touch "${work_dir}/break_retests"
+}
+# --
+
+# Called by test runner to see if the retest cycle should be broken.
+function retest_cycle_broken() {
+    # Using the reverse boolean logic here!
+    local broken=1
+    if [[ -f "${TEST_WORK_DIR}/break_retests" ]]; then
+        broken=0
+        rm -f "${TEST_WORK_DIR}/break_retests"
+    fi
+    return ${broken}
+}
+# --
+
+# Substitutes fields in the passed template and prints the
+# result. Followed by the template, the parameters used for
+# replacement are in alphabetical order: arch, channel, proto and
+# vernum.
+function url_from_template() {
+    local template="${1}"; shift
+    local arch="${1}"; shift
+    local channel="${1}"; shift
+    local proto="${1}"; shift
+    local vernum="${1}"; shift
+    local url="${template}"
+
+    url="${url//@ARCH@/${arch}}"
+    url="${url//@CHANNEL@/${channel}}"
+    url="${url//@PROTO@/${proto}}"
+    url="${url//@VERNUM@/${vernum}}"
+
+    echo "${url}"
+}
+# --
+
+# Puts a secret into a file, while trying for the secret to not end up
+# on a filesystem at all. A path to the file with the secret in /proc
+# in put into the chosen variable. The secret is assumed to be
+# base64-encoded.
+#
+# Typical use:
+#   secret_file=''
+#   secret_to_file secret_file "${some_secret}"
+#
+# Parameters:
+# 1 - name of the variable where the path is stored
+# 2 - the secret to store in the file
+function secret_to_file() {
+    local config_var_name="${1}"; shift
+    local secret="${1}"; shift
+    local tmpfile=$(mktemp)
+    local -n config_ref="${config_var_name}"
+    local fd
+
+    exec {fd}<>"${tmpfile}"
+    rm -f "${tmpfile}"
+    echo "${secret}" | base64 --decode >&${fd}
+    config_ref="/proc/${$}/fd/${fd}"
+}
+# --

ci-automation/test.sh

@@ -122,7 +122,7 @@ function test_run() {
     local docker_vernum
     docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
 
-    local work_dir="__TESTS__"
+    local work_dir="${TEST_WORK_DIR}"
     local tests_dir="${work_dir}/${image}"
     mkdir -p "${tests_dir}"
 
@@ -182,6 +182,16 @@ function test_run() {
             break
         fi
 
+        if retest_cycle_broken; then
+            echo "########### Test cycle requested to break ###########"
+            echo "Failed tests: $failed_tests"
+            echo "-----------"
+            # not really a success, but don't print a message about
+            # exhaused reruns and giving up
+            success=true
+            break
+        fi
+
         echo "########### Some tests failed and will be re-run (${retry} / ${retries}). ###########"
         echo "Failed tests: $failed_tests"
         echo "-----------"

ci-automation/vendor-testing/digitalocean.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+# Copyright (c) 2022 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -euo pipefail
+
+# Test execution script for the Digital Ocean vendor image.
+# This script is supposed to run in the mantle container.
+
+source ci-automation/vendor_test.sh
+
+# We never ran Digital Ocean on arm64, so for now fail it as an
+# unsupported option.
+if [[ "${CIA_ARCH}" == "arm64" ]]; then
+    echo "1..1" > "${CIA_TAPFILE}"
+    echo "not ok - all Digital Ocean tests" >> "${CIA_TAPFILE}"
+    echo "  ---" >> "${CIA_TAPFILE}"
+    echo "  ERROR: ARM64 tests not supported on Digital Ocean." | tee -a "${CIA_TAPFILE}"
+    echo "  ..." >> "${CIA_TAPFILE}"
+    break_retest_cycle
+    exit 1
+fi
+
+image_name="ci-${CIA_VERNUM//+/-}"
+image_url="$(url_from_template "${DIGITALOCEAN_IMAGE_URL_TEMPLATE}" "${CIA_ARCH}" "${CIA_CHANNEL}" 'https' "${CIA_VERNUM}")"
+
+config_file=''
+secret_to_file config_file "${DIGITALOCEAN_TOKEN_JSON}"
+
+ore do create-image \
+    --config-file="${config_file}" \
+    --region="${DIGITALOCEAN_REGION}" \
+    --name="${image_name}" \
+    --url="${image_url}"
+
+trap 'ore do delete-image \
+    --name="${image_name}" \
+    --config-file="${config_file}"' EXIT
+
+set -x
+
+timeout --signal=SIGQUIT 4h\
+    kola run \
+    --do-size="${DIGITALOCEAN_MACHINE_SIZE}" \
+    --do-region="${DIGITALOCEAN_REGION}" \
+    --basename="${image_name}" \
+    --do-config-file="${config_file}" \
+    --do-image="${image_name}" \
+    --parallel="${DIGITALOCEAN_PARALLEL}" \
+    --platform=do \
+    --channel="${CIA_CHANNEL}" \
+    --tapfile="${CIA_TAPFILE}" \
+    --torcx-manifest="${CIA_TORCX_MANIFEST}" \
+    "${@}"
+
+set +x