mirrors/flatcar-scripts
1
0
Fork 0
mirror of https://github.com/flatcar/scripts.git synced 2025-08-07 13:06:59 +02:00
Code Issues Packages Projects Releases Wiki Activity

ci-automation: Update sdk-bootstrap, packages-tag and upload functions

Browse Source 
- Tighten the patterns used for nightly tags detection.
- Compare hashes instead of names to figure out if we are on top of a
  branch (fixes the issue of no nightly tags reachable from the
  release branches). Jenkins is doing `git fetch origin "${branch}";
  git checkout FETCH_HEAD` and this was confusing the `git rev-parse
  --abbrev-ref HEAD` code (it returned `HEAD` instead of `${branch}`).
- Account for possible multiple tags in a single commit.
- Made the tagging fail in dubious situations.
- Reindent the code, modernize a bit.
This commit is contained in:
Krzesimir Nowak 2025-07-08 16:32:01 +02:00
parent a1047bbd0c
commit 782bb560e6
3 changed files with 138 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
ci-automation/ci_automation_common.sh
View File

@ -22,15 +22,15 @@ function check_version_string() {
# --
function update_and_push_version() {
local version="$1"
local target_branch="${2:-}"
local version=${1}
local target_branch=${2:-}
# set up author and email so git does not complain when tagging
if ! git config --get user.name >/dev/null 2>&1 ; then
git -C . config user.name "${CI_GIT_AUTHOR}"
git config user.name "${CI_GIT_AUTHOR}"
fi
if ! git config --get user.email >/dev/null 2>&1 ; then
git -C . config user.email "${CI_GIT_EMAIL}"
git config user.email "${CI_GIT_EMAIL}"
fi
# Add and commit local changes
@ -39,32 +39,33 @@ function update_and_push_version() {
git commit --allow-empty -m "New version: ${version}"
git fetch --all --tags --force
local ret=0
git diff --exit-code "${version}" || ret=$?
local -i ret=0
git diff --quiet --exit-code "${version}" 2>/dev/null || ret=${?}
# This will return != 0 if
# - the remote tag does not exist (rc: 127)
# - the tag does not exist locally (rc: 128)
# - the remote tag has changes compared to the local tree (rc: 1)
if [ "$ret" = "0" ]; then
if [[ ret -eq 0 ]]; then
# this means that we created an empty commit above, reusing the tag gets rid of it
echo "Reusing existing tag" >&2
git checkout -f "${version}"
git checkout --force "${version}"
return
elif [ "$ret" = "1" ]; then
elif [[ ret -eq 1 ]]; then
echo "Remote tag exists already and is not equal" >&2
return 1
elif [ "$ret" != "127" ] && [ "$ret" != "128" ]; then
echo "Error: Unexpected git diff return code ($ret)" >&2
elif [[ ret -ne 127 && ret -ne 128 ]]; then
echo "Error: Unexpected git diff return code (${ret})" >&2
return 1
fi
local -a TAG_ARGS
if [ "${SIGN-0}" = 1 ]; then
TAG_ARGS=("-s" "-m" "${version}")
local -a TAG_ARGS=()
if [[ ${SIGN-0} = 1 ]]; then
TAG_ARGS=("--sign" "--message=${version}")
fi
git tag -f "${TAG_ARGS[@]}" "${version}"
git tag --force "${TAG_ARGS[@]}" "${version}"
if [[ -n "${target_branch}" ]]; then
if [[ -n ${target_branch} ]]; then
git push origin "HEAD:${target_branch}"
fi

66
ci-automation/packages-tag.sh
View File

@ -56,7 +56,7 @@ function packages_tag() {
# --
function _packages_tag_impl() {
local version="$1"
local version=${1}
source ci-automation/ci_automation_common.sh
source ci-automation/gpg_setup.sh
@ -64,7 +64,7 @@ function _packages_tag_impl() {
check_version_string "${version}"
source sdk_container/.repo/manifests/version.txt
local sdk_version="${FLATCAR_SDK_VERSION}"
local sdk_version=${FLATCAR_SDK_VERSION}
if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then
set -x
@ -77,43 +77,71 @@ function _packages_tag_impl() {
# builds without messing with actual release branches.
local flatcar_branch_prefix=${CIA_DEBUGFLATCARBRANCHPREFIX:-flatcar}
local nightly=${CIA_DEBUGNIGHTLY:-nightly}
# Patterns used below.
local nightly_pattern_1='^(stable|alpha|beta|lts)-[0-9.]+-'"${nightly}"'-[-0-9]+$'
local nightly_pattern_2='^(stable|alpha|beta|lts)-[0-9.]+(|-'"${nightly}"'-[-0-9]+)$'
local flatcar_pattern='^'"${flatcar_branch_prefix}"'-[0-9]+$'
if [[ "${version}" =~ ${nightly_pattern_1} ]] \
&& [[ "$(git rev-parse --abbrev-ref HEAD)" =~ ${flatcar_pattern} ]] ; then
target_branch="$(git rev-parse --abbrev-ref HEAD)"
local existing_tag=""
# Matches the usual nightly tag name (stable-1234.2.3-nightly-yyyymmdd-hhmm)
local nightly_pattern='^(stable|alpha|beta|lts)-([0-9]+)(\.[0-9]+){2}-'"${nightly}"'-[0-9]{8}-[0-9]{4}$'
local -i major_version=0
local branch_name=''
local branch_hash=''
if [[ ${version} =~ ${nightly_pattern} ]]; then
major_version=${BASH_REMATCH[2]}
branch_name=${flatcar_branch_prefix}-${major_version}
branch_hash=$(git rev-parse "origin/${branch_name}")
fi
local -a existing_tags=()
if [[ -n ${branch_hash} ]]; then
if [[ $(git rev-parse HEAD) != "${branch_hash}" ]]; then
echo "We are doing a nightly build but we are not on top of the ${branch_name} branch. This is wrong and would result in the nightly tag not being a part of the branch." >&2
exit 1
fi
target_branch=${branch_name}
# Check for the existing tag only when we allow shortcutting
# the builds. That way we can skip the checks for build
# shortcutting.
if bool_is_true "${AVOID_NIGHTLY_BUILD_SHORTCUTS}"; then
echo "Continuing the build because AVOID_NIGHTLY_BUILD_SHORTCUTS is bool true (${AVOID_NIGHTLY_BUILD_SHORTCUTS})" >&2
else
existing_tag=$(git tag --points-at HEAD) # exit code is always 0, output may be empty
git fetch --all --tags --force
# exit code of git tag is always 0; output may be empty,
# but may also have multiple tags
mapfile -t existing_tags < <(git tag --points-at HEAD)
fi
# If the found tag is a release or nightly tag, we stop this build if there are no changes
if [[ "${existing_tag}" =~ ${nightly_pattern_2} ]]; then
local ret=0
git diff --exit-code "${existing_tag}" || ret=$?
fi
local nightly_or_release_tag=''
if [[ major_version -gt 0 && ${#existing_tags[@]} -gt 0 ]]; then
local nightly_or_release_pattern='^(stable|alpha|beta|lts)-'"${major_version}"'(\.[0-9]+){2}(-'"${nightly}"'-[0-9]{8}-[0-9]{4})?$'
local tag
for tag in "${existing_tags[@]}"; do
if [[ ${tag} =~ ${nightly_or_release_pattern} ]]; then
nightly_or_release_tag=${tag}
break
fi
done
fi
# If the found tag is a release or nightly tag, we stop this build
# if there are no changes and the relevant images can be found in
# bincache.
if [[ -n ${nightly_or_release_tag} ]]; then
local -i ret=0
git diff --exit-code --quiet "${nightly_or_release_tag}" || ret=$?
if [[ ret -eq 0 ]]; then
# no changes in the code, but check if images exist (they
# could be missing if build failed)
if check_bincache_images_existence \
"https://${BUILDCACHE_SERVER}/images/amd64/${FLATCAR_VERSION}/flatcar_production_image.bin.bz2" \
"https://${BUILDCACHE_SERVER}/images/arm64/${FLATCAR_VERSION}/flatcar_production_image.bin.bz2"; then
touch ./skip-build
echo "Creating ./skip-build flag file, indicating that the build must not to continue because no new tag got created as there are no changes since tag ${existing_tag} and the Flatcar images exist" >&2
echo "Creating ./skip-build flag file, indicating that the build must not to continue because no new tag got created as there are no changes since tag ${nightly_or_release_tag} and the Flatcar images exist" >&2
return 0
fi
echo "No changes but continuing build because Flatcar images do not exist"
elif [[ ret -eq 1 ]]; then
echo "Found changes since last tag ${existing_tag}" >&2
echo "HEAD is tagged with a nightly tag and yet there a differences? This is fishy and needs to be investigated. Maybe you forgot to commit your changes?" >&2
exit 1
else
echo "Error: Unexpected git diff return code (${ret})" >&2
return 1
fi
fi
fi
if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then
set +x
fi
@ -121,7 +149,7 @@ function _packages_tag_impl() {
# Create version file
(
source sdk_lib/sdk_container_common.sh
create_versionfile "$sdk_version" "$version"
create_versionfile "${sdk_version}" "${version}"
)
if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then
set -x

79
ci-automation/sdk_bootstrap.sh
View File

@ -65,8 +65,8 @@ function sdk_bootstrap() {
# --
function _sdk_bootstrap_impl() {
local seed_version="$1"
local version="$2"
local seed_version=${1}
local version=${2}
: ${ARCH:="amd64"}
source ci-automation/ci_automation_common.sh
@ -86,27 +86,48 @@ function _sdk_bootstrap_impl() {
# builds without messing with actual release branches.
local main_branch=${CIA_DEBUGMAINBRANCH:-main}
local nightly=${CIA_DEBUGNIGHTLY:-nightly}
# Patterns used below.
local nightly_pattern_1='^main-[0-9.]+-'"${nightly}"'-[-0-9]+(-INTERMEDIATE)?$'
local nightly_pattern_2='^main-[0-9.]+-'"${nightly}"'-[-0-9]+$'
if [[ "${version}" =~ ${nightly_pattern_1} ]] \
&& [ "$(git rev-parse HEAD)" = "$(git rev-parse "origin/${main_branch}")" ] ; then
# Matches the usual nightly tag name, optionally with an
# intermediate suffix of the build ID too
# (main-1234.2.3-nightly-yyyymmdd-hhmm-INTERMEDIATE).
local nightly_pattern_1='^main-[0-9]+(\.[0-9]+){2}-'"${nightly}"'-[0-9]{8}-[0-9]{4}(-INTERMEDIATE)?$'
local main_branch_hash=''
if [[ ${version} =~ ${nightly_pattern_1} ]]; then
main_branch_hash=$(git rev-parse "origin/${main_branch}")
fi
local -a existing_tags=()
if [[ -n ${main_branch_hash} ]]; then
if [[ $(git rev-parse HEAD) != "${main_branch_hash}" ]] ; then
echo "We are doing a nightly build but we are not on top of the ${main_branch} branch. This is wrong and would result in the nightly tag not being a part of the branch." >&2
exit 1
fi
target_branch=${main_branch}
local existing_tag=""
# Check for the existing tag only when we allow shortcutting
# the builds. That way we can skip the checks for build
# shortcutting.
# Check for the existing tag only when we allow
# shortcutting the builds. That way we can skip the checks
# for build shortcutting.
if bool_is_true "${AVOID_NIGHTLY_BUILD_SHORTCUTS}"; then
echo "Continuing the build because AVOID_NIGHTLY_BUILD_SHORTCUTS is bool true (${AVOID_NIGHTLY_BUILD_SHORTCUTS})" >&2
else
existing_tag=$(git tag --points-at HEAD) # exit code is always 0, output may be empty
git fetch --all --tags --force
# exit code is always 0, output may be empty
mapfile -t existing_tags < <(git tag --points-at HEAD)
fi
# If the found tag is a nightly tag, we stop this build if there are no changes
if [[ "${existing_tag}" =~ ${nightly_pattern_2} ]]; then
local ret=0
git diff --exit-code "${existing_tag}" || ret=$?
if [ "$ret" = "0" ]; then
local versions=(
fi
local nightly_pattern_2='^main-[0-9]+(\.[0-9]+){2}-'"${nightly}"'-[0-9]{8}-[0-9]{4}$'
local tag nightly_tag=''
for tag in "${existing_tags[@]}"; do
if [[ ${tag} =~ ${nightly_pattern_2} ]]; then
nightly_tag=${tag}
break
fi
done
# If the found tag is a nightly tag, we stop this build if there
# are no changes and the relevant images can be found in the
# bincache.
if [[ -n ${nightly_tag} ]]; then
local -i ret=0
git diff --exit-code --quiet "${nightly_tag}" || ret=$?
if [[ ret -eq 0 ]]; then
local -a versions=(
$(
source sdk_lib/sdk_container_common.sh
source "${sdk_container_common_versionfile}"
@ -114,32 +135,32 @@ function _sdk_bootstrap_impl() {
echo "${FLATCAR_VERSION}"
)
)
local flatcar_sdk_version="${versions[0]}"
local flatcar_version="${versions[1]}"
local flatcar_sdk_version=${versions[0]}
local flatcar_version=${versions[1]}
local sdk_docker_vernum=""
sdk_docker_vernum=$(vernum_to_docker_image_version "${flatcar_sdk_version}")
if check_bincache_images_existence \
"https://${BUILDCACHE_SERVER}/containers/${sdk_docker_vernum}/flatcar-sdk-all-${sdk_docker_vernum}.tar.zst" \
"https://${BUILDCACHE_SERVER}/images/amd64/${flatcar_version}/flatcar_production_image.bin.bz2" \
"https://${BUILDCACHE_SERVER}/images/arm64/${flatcar_version}/flatcar_production_image.bin.bz2"; then
echo "Stopping build because there are no changes since tag ${existing_tag}, the SDK container tar ball and the Flatcar images exist" >&2
echo "Stopping build because there are no changes since tag ${nightly_tag}, the SDK container tar ball and the Flatcar images exist" >&2
return 0
fi
echo "No changes but continuing build because SDK container tar ball and/or the Flatcar images do not exist" >&2
elif [ "$ret" = "1" ]; then
echo "Found changes since last tag ${existing_tag}" >&2
elif [[ ret -eq 1 ]]; then
echo "HEAD is tagged with a nightly tag and yet there a differences? This is fishy and needs to be investigated. Maybe you forgot to commit your changes?" >&2
exit 1
else
echo "Error: Unexpected git diff return code (${ret})" >&2
return 1
fi
fi
fi
if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then
set +x
fi
local vernum="${version#*-}" # remove alpha-,beta-,stable-,lts- version tag
local git_vernum="${vernum}"
local vernum=${version#*-} # remove alpha-,beta-,stable-,lts- version tag
local git_vernum=${vernum}
# Update FLATCAR_VERSION[_ID], BUILD_ID, and SDK in versionfile
(
@ -167,8 +188,10 @@ function _sdk_bootstrap_impl() {
# to ourselves, otherwise we could fail to sign the artifacts as
# we lacked write permissions in the directory of the signed
# artifact
local uid=$(id --user)
local gid=$(id --group)
local uid
local gid
uid=$(id --user)
gid=$(id --group)
sudo chown --recursive "${uid}:${gid}" __build__
(
cd "__build__/images/catalyst/builds/flatcar-sdk"