mirror of
https://github.com/flatcar/scripts.git
synced 2025-08-07 21:16:57 +02:00
527bd2237b
It uses the SIGNER environment variable to decide whether the signatures should be created or not. It expect the key of the SIGNER to exist in GPGHOME, and that's what gpg_setup.sh is already doing. In some places we need to recursively change the owner of the directory that contains artifacts to be signed, otherwise we won't be able to create new files with signatures there. This is because some of the artifacts are either created inside the SDK container (so the created files belong to root outside the container) or are created with `sudo`.
350 lines
9.3 KiB
Bash
350 lines
9.3 KiB
Bash
#!/bin/bash
|
|
#
|
|
# Copyright (c) 2021 The Flatcar Maintainers.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
# CI automation common functions.
|
|
|
|
source ci-automation/ci-config.env
|
|
: ${PIGZ:=pigz}
|
|
: ${docker:=docker}
|
|
|
|
: ${TEST_WORK_DIR:='__TESTS__'}
|
|
|
|
function init_submodules() {
|
|
git submodule init
|
|
git submodule update
|
|
}
|
|
# --
|
|
|
|
function update_submodule() {
|
|
local submodule="$1"
|
|
local commit_ish="$2"
|
|
|
|
cd "sdk_container/src/third_party/${submodule}"
|
|
git fetch --all --tags
|
|
git checkout "${commit_ish}"
|
|
cd -
|
|
}
|
|
# --
|
|
|
|
function check_version_string() {
|
|
local version="$1"
|
|
|
|
if ! echo "${version}" | grep -qE '^(main-|alpha-|beta-|stable-|lts-)' ; then
|
|
echo "ERROR: invalid version '${version}', must start with 'main-', 'alpha-', 'beta-', 'stable-', or 'lts-'"
|
|
exit 1
|
|
fi
|
|
}
|
|
# --
|
|
|
|
function update_submodules() {
|
|
local coreos_git="$1"
|
|
local portage_git="$2"
|
|
|
|
init_submodules
|
|
update_submodule "coreos-overlay" "${coreos_git}"
|
|
update_submodule "portage-stable" "${portage_git}"
|
|
}
|
|
# --
|
|
|
|
function update_and_push_version() {
|
|
local version="$1"
|
|
local push_to_branch="${2:-false}"
|
|
|
|
# set up author and email so git does not complain when tagging
|
|
if ! git config --get user.name >/dev/null 2>&1 ; then
|
|
git -C . config user.name "${CI_GIT_AUTHOR}"
|
|
fi
|
|
if ! git config --get user.email >/dev/null 2>&1 ; then
|
|
git -C . config user.email "${CI_GIT_EMAIL}"
|
|
fi
|
|
|
|
# Add and commit local changes
|
|
git add "sdk_container/src/third_party/coreos-overlay"
|
|
git add "sdk_container/src/third_party/portage-stable"
|
|
git add "sdk_container/.repo/manifests/version.txt"
|
|
|
|
git commit --allow-empty -m "New version: ${version}"
|
|
|
|
git fetch --all --tags --force
|
|
local ret=0
|
|
git diff --exit-code "${version}" || ret=$?
|
|
# This will return != 0 if
|
|
# - the remote tag does not exist (rc: 127)
|
|
# - the tag does not exist locally (rc: 128)
|
|
# - the remote tag has changes compared to the local tree (rc: 1)
|
|
if [ "$ret" = "0" ]; then
|
|
echo "Reusing existing tag" >&2
|
|
git checkout -f --recurse-submodules "${version}"
|
|
return
|
|
elif [ "$ret" = "1" ]; then
|
|
echo "Remote tag exists already and is not equal" >&2
|
|
return 1
|
|
elif [ "$ret" != "127" ] && [ "$ret" != "128" ]; then
|
|
echo "Error: Unexpected git diff return code ($ret)" >&2
|
|
return 1
|
|
fi
|
|
|
|
local -a TAG_ARGS
|
|
if [ "${SIGN-0}" = 1 ]; then
|
|
TAG_ARGS=("-s" "-m" "${version}")
|
|
fi
|
|
|
|
git tag -f "${TAG_ARGS[@]}" "${version}"
|
|
|
|
if [ "${push_to_branch}" = "true" ]; then
|
|
local branch="$(git rev-parse --abbrev-ref HEAD)"
|
|
git push origin "${branch}"
|
|
fi
|
|
|
|
git push origin "${version}"
|
|
}
|
|
# --
|
|
|
|
function copy_from_buildcache() {
|
|
local what="$1"
|
|
local where_to="$2"
|
|
|
|
mkdir -p "$where_to"
|
|
curl --fail --silent --show-error --location --retry-delay 1 --retry 60 \
|
|
--retry-connrefused --retry-max-time 60 --connect-timeout 20 \
|
|
--remote-name --output-dir "${where_to}" "https://${BUILDCACHE_SERVER}/${what}"
|
|
}
|
|
# --
|
|
|
|
function gen_sshcmd() {
|
|
echo -n "ssh -o BatchMode=yes"
|
|
echo -n " -o StrictHostKeyChecking=no"
|
|
echo -n " -o UserKnownHostsFile=/dev/null"
|
|
echo " -o NumberOfPasswordPrompts=0"
|
|
}
|
|
# --
|
|
|
|
function copy_to_buildcache() {
|
|
local remote_path="${BUILDCACHE_PATH_PREFIX}/$1"
|
|
shift
|
|
|
|
local sshcmd="$(gen_sshcmd)"
|
|
|
|
$sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \
|
|
"mkdir -p ${remote_path}"
|
|
|
|
rsync --partial -a -e "${sshcmd}" "$@" \
|
|
"${BUILDCACHE_USER}@${BUILDCACHE_SERVER}:${remote_path}"
|
|
}
|
|
# --
|
|
|
|
function image_exists_locally() {
|
|
local name="$1"
|
|
local version="$2"
|
|
local image="${name}:${version}"
|
|
|
|
local image_exists="$($docker images "${image}" \
|
|
--no-trunc --format '{{.Repository}}:{{.Tag}}')"
|
|
|
|
[ "${image}" = "${image_exists}" ]
|
|
}
|
|
# --
|
|
|
|
# Derive docker-safe image version string from vernum.
|
|
#
|
|
function vernum_to_docker_image_version() {
|
|
local vernum="$1"
|
|
echo "$vernum" | sed 's/[+]/-/g'
|
|
}
|
|
# --
|
|
|
|
# Return the full name (repo+name+tag) of an image. Useful for SDK images
|
|
# pulled from the registry (which have the registry pre-pended)
|
|
function docker_image_fullname() {
|
|
local image="$1"
|
|
local version="$2"
|
|
|
|
$docker images --no-trunc --format '{{.Repository}}:{{.Tag}}' \
|
|
| grep -E "^(${CONTAINER_REGISTRY}/)*${image}:${version}$"
|
|
}
|
|
# --
|
|
|
|
function docker_image_to_buildcache() {
|
|
local image="$1"
|
|
local version="$2"
|
|
|
|
# strip potential container registry prefix
|
|
local tarball="$(basename "$image")-${version}.tar.gz"
|
|
|
|
$docker save "${image}":"${version}" | $PIGZ -c > "${tarball}"
|
|
sign_artifacts "${SIGNER:-}" "${tarball}"
|
|
copy_to_buildcache "containers/${version}" "${tarball}"*
|
|
}
|
|
# --
|
|
|
|
function docker_commit_to_buildcache() {
|
|
local container="$1"
|
|
local image_name="$2"
|
|
local image_version="$3"
|
|
|
|
$docker commit "${container}" "${image_name}:${image_version}"
|
|
docker_image_to_buildcache "${image_name}" "${image_version}"
|
|
}
|
|
# --
|
|
|
|
function docker_image_from_buildcache() {
|
|
local name="$1"
|
|
local version="$2"
|
|
local tgz="${name}-${version}.tar.gz"
|
|
|
|
if image_exists_locally "${name}" "${version}" ; then
|
|
return
|
|
fi
|
|
|
|
local url="https://${BUILDCACHE_SERVER}/containers/${version}/${tgz}"
|
|
|
|
curl --fail --silent --show-error --location --retry-delay 1 --retry 60 \
|
|
--retry-connrefused --retry-max-time 60 --connect-timeout 20 \
|
|
--remote-name "${url}"
|
|
|
|
cat "${tgz}" | $PIGZ -d -c | $docker load
|
|
|
|
rm "${tgz}"
|
|
}
|
|
# --
|
|
|
|
function docker_image_from_registry_or_buildcache() {
|
|
local image="$1"
|
|
local version="$2"
|
|
|
|
if image_exists_locally "${CONTAINER_REGISTRY}/${image}" "${version}" ; then
|
|
return
|
|
fi
|
|
|
|
if $docker pull "${CONTAINER_REGISTRY}/${image}:${version}" ; then
|
|
return
|
|
fi
|
|
|
|
echo "Falling back to tar ball download..." >&2
|
|
docker_image_from_buildcache "${image}" "${version}"
|
|
}
|
|
# --
|
|
|
|
# Called by vendor test in case of complete failure not eligible for
|
|
# reruns (like trying to run tests on unsupported architecture).
|
|
function break_retest_cycle() {
|
|
local work_dir=$(dirname "${PWD}")
|
|
local dir=$(basename "${work_dir}")
|
|
|
|
if [[ "${dir}" != "${TEST_WORK_DIR}" ]]; then
|
|
echo "Not breaking retest cycle, expected test work dir to be a parent directory" >&2
|
|
return
|
|
fi
|
|
touch "${work_dir}/break_retests"
|
|
}
|
|
# --
|
|
|
|
# Called by test runner to see if the retest cycle should be broken.
|
|
function retest_cycle_broken() {
|
|
# Using the reverse boolean logic here!
|
|
local broken=1
|
|
if [[ -f "${TEST_WORK_DIR}/break_retests" ]]; then
|
|
broken=0
|
|
rm -f "${TEST_WORK_DIR}/break_retests"
|
|
fi
|
|
return ${broken}
|
|
}
|
|
# --
|
|
|
|
# Substitutes fields in the passed template and prints the
|
|
# result. Followed by the template, the parameters used for
|
|
# replacement are in alphabetical order: arch, channel, proto and
|
|
# vernum.
|
|
function url_from_template() {
|
|
local template="${1}"; shift
|
|
local arch="${1}"; shift
|
|
local channel="${1}"; shift
|
|
local proto="${1}"; shift
|
|
local vernum="${1}"; shift
|
|
local url="${template}"
|
|
|
|
url="${url//@ARCH@/${arch}}"
|
|
url="${url//@CHANNEL@/${channel}}"
|
|
url="${url//@PROTO@/${proto}}"
|
|
url="${url//@VERNUM@/${vernum}}"
|
|
|
|
echo "${url}"
|
|
}
|
|
# --
|
|
|
|
# Puts a secret into a file, while trying for the secret to not end up
|
|
# on a filesystem at all. A path to the file with the secret in /proc
|
|
# in put into the chosen variable. The secret is assumed to be
|
|
# base64-encoded.
|
|
#
|
|
# Typical use:
|
|
# secret_file=''
|
|
# secret_to_file secret_file "${some_secret}"
|
|
#
|
|
# Parameters:
|
|
# 1 - name of the variable where the path is stored
|
|
# 2 - the secret to store in the file
|
|
function secret_to_file() {
|
|
local config_var_name="${1}"; shift
|
|
local secret="${1}"; shift
|
|
local tmpfile=$(mktemp)
|
|
local -n config_ref="${config_var_name}"
|
|
local fd
|
|
|
|
exec {fd}<>"${tmpfile}"
|
|
rm -f "${tmpfile}"
|
|
echo "${secret}" | base64 --decode >&${fd}
|
|
config_ref="/proc/${$}/fd/${fd}"
|
|
}
|
|
# --
|
|
|
|
# Creates signatures for the passed files and directories. In case of
|
|
# directory, all files inside are signed. Files ending with .asc or
|
|
# .sig or .gpg are ignored, though. This function is a noop if signer
|
|
# is empty.
|
|
#
|
|
# Typical use:
|
|
# sign_artifacts "${SIGNER}" artifact.tar.gz
|
|
# copy_to_buildcache "artifacts/directory" artifact.tar.gz*
|
|
#
|
|
# Parameters:
|
|
#
|
|
# 1 - signer whose key is expected to be already imported into the
|
|
# keyring
|
|
# @ - files and directories to sign
|
|
function sign_artifacts() {
|
|
local signer="${1}"; shift
|
|
# rest of the parameters are directories/files to sign
|
|
local to_sign=()
|
|
local file
|
|
local files
|
|
|
|
if [[ -z "${signer}" ]]; then
|
|
return
|
|
fi
|
|
|
|
for file; do
|
|
files=()
|
|
if [[ -d "${file}" ]]; then
|
|
readarray -d '' files < <(find "${file}" ! -type d -print0)
|
|
elif [[ -e "${file}" ]]; then
|
|
files+=( "${file}" )
|
|
fi
|
|
for file in "${files[@]}"; do
|
|
if [[ "${file}" =~ \.(asc|gpg|sig)$ ]]; then
|
|
continue
|
|
fi
|
|
to_sign+=( "${file}" )
|
|
done
|
|
done
|
|
for file in "${to_sign[@]}"; do
|
|
gpg --batch --local-user "${signer}" \
|
|
--output "${file}.sig" \
|
|
--detach-sign "${file}"
|
|
done
|
|
}
|
|
# --
|