diff --git a/ci-automation/ci-config.env b/ci-automation/ci-config.env index 20e2bf4f01..2c2672758f 100644 --- a/ci-automation/ci-config.env +++ b/ci-automation/ci-config.env @@ -17,6 +17,8 @@ CONTAINER_REGISTRY="ghcr.io/flatcar-linux" GC_BUCKET="flatcar-linux" +DEFAULT_HTTP_IMAGE_URL_TEMPLATE="@PROTO@://${BUILDCACHE_SERVER}/images/@ARCH@/@VERNUM@" + if ! command -v pigz > /dev/null; then # No PIGZ on Flatcar PIGZ="docker run --rm -i ghcr.io/flatcar-linux/pigz --fast" @@ -82,3 +84,14 @@ GCE_IMAGE_NAME="flatcar_production_gce.tar.gz" GCE_GCS_IMAGE_UPLOAD="gs://flatcar-jenkins/developer/gce-ci" GCE_MACHINE_TYPE="${GCE_MACHINE_TYPE:-n1-standard-2}" GCE_PARALLEL="${PARALLEL_TESTS:-4}" + +# -- Digital Ocean -- + +# Use the "@PROTO@" "@ARCH@", "@CHANNEL@" and "@VERNUM@" placeholders. They will +# be replaced. +: ${DIGITALOCEAN_IMAGE_URL_TEMPLATE:="${DEFAULT_HTTP_IMAGE_URL_TEMPLATE}/flatcar_production_digitalocean_image.bin.bz2"} +: ${DIGITALOCEAN_REGION:='sfo3'} +: ${DIGITALOCEAN_MACHINE_SIZE:='s-2vcpu-2gb'} +DIGITALOCEAN_PARALLEL="${PARALLEL_TESTS:-8}" +# DIGITALOCEAN_TOKEN_JSON env var is used for credentials, and should +# come from sdk_container/.env. It must be base64-encoded. diff --git a/ci-automation/ci_automation_common.sh b/ci-automation/ci_automation_common.sh index 82cf997d2f..ce01d55765 100644 --- a/ci-automation/ci_automation_common.sh +++ b/ci-automation/ci_automation_common.sh @@ -10,6 +10,8 @@ source ci-automation/ci-config.env : ${PIGZ:=pigz} : ${docker:=docker} +: ${TEST_WORK_DIR:='__TESTS__'} + function init_submodules() { git submodule init git submodule update @@ -224,3 +226,76 @@ function docker_image_from_registry_or_buildcache() { docker_image_from_buildcache "${image}" "${version}" } # -- + +# Called by vendor test in case of complete failure not eligible for +# reruns (like trying to run tests on unsupported architecture). +function break_retest_cycle() { + local work_dir=$(dirname "${PWD}") + local dir=$(basename "${work_dir}") + + if [[ "${dir}" != "${TEST_WORK_DIR}" ]]; then + echo "Not breaking retest cycle, expected test work dir to be a parent directory" >&2 + return + fi + touch "${work_dir}/break_retests" +} +# -- + +# Called by test runner to see if the retest cycle should be broken. +function retest_cycle_broken() { + # Using the reverse boolean logic here! + local broken=1 + if [[ -f "${TEST_WORK_DIR}/break_retests" ]]; then + broken=0 + rm -f "${TEST_WORK_DIR}/break_retests" + fi + return ${broken} +} +# -- + +# Substitutes fields in the passed template and prints the +# result. Followed by the template, the parameters used for +# replacement are in alphabetical order: arch, channel, proto and +# vernum. +function url_from_template() { + local template="${1}"; shift + local arch="${1}"; shift + local channel="${1}"; shift + local proto="${1}"; shift + local vernum="${1}"; shift + local url="${template}" + + url="${url//@ARCH@/${arch}}" + url="${url//@CHANNEL@/${channel}}" + url="${url//@PROTO@/${proto}}" + url="${url//@VERNUM@/${vernum}}" + + echo "${url}" +} +# -- + +# Puts a secret into a file, while trying for the secret to not end up +# on a filesystem at all. A path to the file with the secret in /proc +# in put into the chosen variable. The secret is assumed to be +# base64-encoded. +# +# Typical use: +# secret_file='' +# secret_to_file secret_file "${some_secret}" +# +# Parameters: +# 1 - name of the variable where the path is stored +# 2 - the secret to store in the file +function secret_to_file() { + local config_var_name="${1}"; shift + local secret="${1}"; shift + local tmpfile=$(mktemp) + local -n config_ref="${config_var_name}" + local fd + + exec {fd}<>"${tmpfile}" + rm -f "${tmpfile}" + echo "${secret}" | base64 --decode >&${fd} + config_ref="/proc/${$}/fd/${fd}" +} +# -- diff --git a/ci-automation/test.sh b/ci-automation/test.sh index fa7c6c3720..c66fffeb09 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -122,7 +122,7 @@ function test_run() { local docker_vernum docker_vernum="$(vernum_to_docker_image_version "${vernum}")" - local work_dir="__TESTS__" + local work_dir="${TEST_WORK_DIR}" local tests_dir="${work_dir}/${image}" mkdir -p "${tests_dir}" @@ -182,6 +182,16 @@ function test_run() { break fi + if retest_cycle_broken; then + echo "########### Test cycle requested to break ###########" + echo "Failed tests: $failed_tests" + echo "-----------" + # not really a success, but don't print a message about + # exhaused reruns and giving up + success=true + break + fi + echo "########### Some tests failed and will be re-run (${retry} / ${retries}). ###########" echo "Failed tests: $failed_tests" echo "-----------" diff --git a/ci-automation/vendor-testing/digitalocean.sh b/ci-automation/vendor-testing/digitalocean.sh new file mode 100755 index 0000000000..3ca82ac2e4 --- /dev/null +++ b/ci-automation/vendor-testing/digitalocean.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# Copyright (c) 2022 The Flatcar Maintainers. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +set -euo pipefail + +# Test execution script for the Digital Ocean vendor image. +# This script is supposed to run in the mantle container. + +source ci-automation/vendor_test.sh + +# We never ran Digital Ocean on arm64, so for now fail it as an +# unsupported option. +if [[ "${CIA_ARCH}" == "arm64" ]]; then + echo "1..1" > "${CIA_TAPFILE}" + echo "not ok - all Digital Ocean tests" >> "${CIA_TAPFILE}" + echo " ---" >> "${CIA_TAPFILE}" + echo " ERROR: ARM64 tests not supported on Digital Ocean." | tee -a "${CIA_TAPFILE}" + echo " ..." >> "${CIA_TAPFILE}" + break_retest_cycle + exit 1 +fi + +image_name="ci-${CIA_VERNUM//+/-}" +image_url="$(url_from_template "${DIGITALOCEAN_IMAGE_URL_TEMPLATE}" "${CIA_ARCH}" "${CIA_CHANNEL}" 'https' "${CIA_VERNUM}")" + +config_file='' +secret_to_file config_file "${DIGITALOCEAN_TOKEN_JSON}" + +ore do create-image \ + --config-file="${config_file}" \ + --region="${DIGITALOCEAN_REGION}" \ + --name="${image_name}" \ + --url="${image_url}" + +trap 'ore do delete-image \ + --name="${image_name}" \ + --config-file="${config_file}"' EXIT + +set -x + +timeout --signal=SIGQUIT 4h\ + kola run \ + --do-size="${DIGITALOCEAN_MACHINE_SIZE}" \ + --do-region="${DIGITALOCEAN_REGION}" \ + --basename="${image_name}" \ + --do-config-file="${config_file}" \ + --do-image="${image_name}" \ + --parallel="${DIGITALOCEAN_PARALLEL}" \ + --platform=do \ + --channel="${CIA_CHANNEL}" \ + --tapfile="${CIA_TAPFILE}" \ + --torcx-manifest="${CIA_TORCX_MANIFEST}" \ + "${@}" + +set +x