mirrors/flatcar-scripts
1
0
Fork 0
mirror of https://github.com/flatcar/scripts.git synced 2025-08-07 13:06:59 +02:00
Code Issues Packages Projects Releases Wiki Activity

ci-automation: Update sdk-bootstrap, packages-tag and upload functions

Browse Source 
- Tighten the patterns used for nightly tags detection.
- Compare hashes instead of names to figure out if we are on top of a
  branch (fixes the issue of no nightly tags reachable from the
  release branches). Jenkins is doing `git fetch origin "${branch}";
  git checkout FETCH_HEAD` and this was confusing the `git rev-parse
  --abbrev-ref HEAD` code (it returned `HEAD` instead of `${branch}`).
- Account for possible multiple tags in a single commit.
- Made the tagging fail in dubious situations.
- Reindent the code, modernize a bit.
This commit is contained in:
Krzesimir Nowak 2025-07-08 16:32:01 +02:00
parent a1047bbd0c
commit 782bb560e6
3 changed files with 138 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
ci-automation/ci_automation_common.sh
View File

@ -22,15 +22,15 @@ function check_version_string() {
# -- # --
function update_and_push_version() { function update_and_push_version() {
local version="$1" local version=${1}
local target_branch="${2:-}" local target_branch=${2:-}
# set up author and email so git does not complain when tagging # set up author and email so git does not complain when tagging
if ! git config --get user.name >/dev/null 2>&1 ; then if ! git config --get user.name >/dev/null 2>&1 ; then
git -C . config user.name "${CI_GIT_AUTHOR}" git config user.name "${CI_GIT_AUTHOR}"
fi fi
if ! git config --get user.email >/dev/null 2>&1 ; then if ! git config --get user.email >/dev/null 2>&1 ; then
git -C . config user.email "${CI_GIT_EMAIL}" git config user.email "${CI_GIT_EMAIL}"
fi fi
# Add and commit local changes # Add and commit local changes
@ -39,33 +39,34 @@ function update_and_push_version() {
git commit --allow-empty -m "New version: ${version}" git commit --allow-empty -m "New version: ${version}"
git fetch --all --tags --force git fetch --all --tags --force
local ret=0 local -i ret=0
git diff --exit-code "${version}" || ret=$? git diff --quiet --exit-code "${version}" 2>/dev/null || ret=${?}
# This will return != 0 if # This will return != 0 if
# - the remote tag does not exist (rc: 127) # - the remote tag does not exist (rc: 127)
# - the tag does not exist locally (rc: 128) # - the tag does not exist locally (rc: 128)
# - the remote tag has changes compared to the local tree (rc: 1) # - the remote tag has changes compared to the local tree (rc: 1)
if [ "$ret" = "0" ]; then if [[ ret -eq 0 ]]; then
echo "Reusing existing tag" >&2 # this means that we created an empty commit above, reusing the tag gets rid of it
git checkout -f "${version}" echo "Reusing existing tag" >&2
return git checkout --force "${version}"
elif [ "$ret" = "1" ]; then return
echo "Remote tag exists already and is not equal" >&2 elif [[ ret -eq 1 ]]; then
return 1 echo "Remote tag exists already and is not equal" >&2
elif [ "$ret" != "127" ] && [ "$ret" != "128" ]; then return 1
echo "Error: Unexpected git diff return code ($ret)" >&2 elif [[ ret -ne 127 && ret -ne 128 ]]; then
return 1 echo "Error: Unexpected git diff return code (${ret})" >&2
return 1
fi fi
local -a TAG_ARGS local -a TAG_ARGS=()
if [ "${SIGN-0}" = 1 ]; then if [[ ${SIGN-0} = 1 ]]; then
TAG_ARGS=("-s" "-m" "${version}") TAG_ARGS=("--sign" "--message=${version}")
fi fi
git tag -f "${TAG_ARGS[@]}" "${version}" git tag --force "${TAG_ARGS[@]}" "${version}"
if [[ -n "${target_branch}" ]]; then if [[ -n ${target_branch} ]]; then
git push origin "HEAD:${target_branch}" git push origin "HEAD:${target_branch}"
fi fi
git push origin "${version}" git push origin "${version}"

74
ci-automation/packages-tag.sh
View File

@ -56,7 +56,7 @@ function packages_tag() {
# -- # --
function _packages_tag_impl() { function _packages_tag_impl() {
local version="$1" local version=${1}
source ci-automation/ci_automation_common.sh source ci-automation/ci_automation_common.sh
source ci-automation/gpg_setup.sh source ci-automation/gpg_setup.sh
@ -64,7 +64,7 @@ function _packages_tag_impl() {
check_version_string "${version}" check_version_string "${version}"
source sdk_container/.repo/manifests/version.txt source sdk_container/.repo/manifests/version.txt
local sdk_version="${FLATCAR_SDK_VERSION}" local sdk_version=${FLATCAR_SDK_VERSION}
if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then
set -x set -x
@ -77,41 +77,69 @@ function _packages_tag_impl() {
# builds without messing with actual release branches. # builds without messing with actual release branches.
local flatcar_branch_prefix=${CIA_DEBUGFLATCARBRANCHPREFIX:-flatcar} local flatcar_branch_prefix=${CIA_DEBUGFLATCARBRANCHPREFIX:-flatcar}
local nightly=${CIA_DEBUGNIGHTLY:-nightly} local nightly=${CIA_DEBUGNIGHTLY:-nightly}
# Patterns used below. # Matches the usual nightly tag name (stable-1234.2.3-nightly-yyyymmdd-hhmm)
local nightly_pattern_1='^(stable|alpha|beta|lts)-[0-9.]+-'"${nightly}"'-[-0-9]+$' local nightly_pattern='^(stable|alpha|beta|lts)-([0-9]+)(\.[0-9]+){2}-'"${nightly}"'-[0-9]{8}-[0-9]{4}$'
local nightly_pattern_2='^(stable|alpha|beta|lts)-[0-9.]+(|-'"${nightly}"'-[-0-9]+)$' local -i major_version=0
local flatcar_pattern='^'"${flatcar_branch_prefix}"'-[0-9]+$' local branch_name=''
if [[ "${version}" =~ ${nightly_pattern_1} ]] \ local branch_hash=''
&& [[ "$(git rev-parse --abbrev-ref HEAD)" =~ ${flatcar_pattern} ]] ; then if [[ ${version} =~ ${nightly_pattern} ]]; then
target_branch="$(git rev-parse --abbrev-ref HEAD)" major_version=${BASH_REMATCH[2]}
local existing_tag="" branch_name=${flatcar_branch_prefix}-${major_version}
branch_hash=$(git rev-parse "origin/${branch_name}")
fi
local -a existing_tags=()
if [[ -n ${branch_hash} ]]; then
if [[ $(git rev-parse HEAD) != "${branch_hash}" ]]; then
echo "We are doing a nightly build but we are not on top of the ${branch_name} branch. This is wrong and would result in the nightly tag not being a part of the branch." >&2
exit 1
fi
target_branch=${branch_name}
# Check for the existing tag only when we allow shortcutting # Check for the existing tag only when we allow shortcutting
# the builds. That way we can skip the checks for build # the builds. That way we can skip the checks for build
# shortcutting. # shortcutting.
if bool_is_true "${AVOID_NIGHTLY_BUILD_SHORTCUTS}"; then if bool_is_true "${AVOID_NIGHTLY_BUILD_SHORTCUTS}"; then
echo "Continuing the build because AVOID_NIGHTLY_BUILD_SHORTCUTS is bool true (${AVOID_NIGHTLY_BUILD_SHORTCUTS})" >&2 echo "Continuing the build because AVOID_NIGHTLY_BUILD_SHORTCUTS is bool true (${AVOID_NIGHTLY_BUILD_SHORTCUTS})" >&2
else else
existing_tag=$(git tag --points-at HEAD) # exit code is always 0, output may be empty git fetch --all --tags --force
# exit code of git tag is always 0; output may be empty,
# but may also have multiple tags
mapfile -t existing_tags < <(git tag --points-at HEAD)
fi fi
# If the found tag is a release or nightly tag, we stop this build if there are no changes fi
if [[ "${existing_tag}" =~ ${nightly_pattern_2} ]]; then local nightly_or_release_tag=''
local ret=0 if [[ major_version -gt 0 && ${#existing_tags[@]} -gt 0 ]]; then
git diff --exit-code "${existing_tag}" || ret=$? local nightly_or_release_pattern='^(stable|alpha|beta|lts)-'"${major_version}"'(\.[0-9]+){2}(-'"${nightly}"'-[0-9]{8}-[0-9]{4})?$'
if [[ ret -eq 0 ]]; then local tag
for tag in "${existing_tags[@]}"; do
if [[ ${tag} =~ ${nightly_or_release_pattern} ]]; then
nightly_or_release_tag=${tag}
break
fi
done
fi
# If the found tag is a release or nightly tag, we stop this build
# if there are no changes and the relevant images can be found in
# bincache.
if [[ -n ${nightly_or_release_tag} ]]; then
local -i ret=0
git diff --exit-code --quiet "${nightly_or_release_tag}" || ret=$?
if [[ ret -eq 0 ]]; then
# no changes in the code, but check if images exist (they
# could be missing if build failed)
if check_bincache_images_existence \ if check_bincache_images_existence \
"https://${BUILDCACHE_SERVER}/images/amd64/${FLATCAR_VERSION}/flatcar_production_image.bin.bz2" \ "https://${BUILDCACHE_SERVER}/images/amd64/${FLATCAR_VERSION}/flatcar_production_image.bin.bz2" \
"https://${BUILDCACHE_SERVER}/images/arm64/${FLATCAR_VERSION}/flatcar_production_image.bin.bz2"; then "https://${BUILDCACHE_SERVER}/images/arm64/${FLATCAR_VERSION}/flatcar_production_image.bin.bz2"; then
touch ./skip-build touch ./skip-build
echo "Creating ./skip-build flag file, indicating that the build must not to continue because no new tag got created as there are no changes since tag ${existing_tag} and the Flatcar images exist" >&2 echo "Creating ./skip-build flag file, indicating that the build must not to continue because no new tag got created as there are no changes since tag ${nightly_or_release_tag} and the Flatcar images exist" >&2
return 0 return 0
fi fi
echo "No changes but continuing build because Flatcar images do not exist" echo "No changes but continuing build because Flatcar images do not exist"
elif [[ ret -eq 1 ]]; then elif [[ ret -eq 1 ]]; then
echo "Found changes since last tag ${existing_tag}" >&2 echo "HEAD is tagged with a nightly tag and yet there a differences? This is fishy and needs to be investigated. Maybe you forgot to commit your changes?" >&2
else exit 1
else
echo "Error: Unexpected git diff return code (${ret})" >&2 echo "Error: Unexpected git diff return code (${ret})" >&2
return 1 return 1
fi
fi fi
fi fi
if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then
@ -120,8 +148,8 @@ function _packages_tag_impl() {
# Create version file # Create version file
( (
source sdk_lib/sdk_container_common.sh source sdk_lib/sdk_container_common.sh
create_versionfile "$sdk_version" "$version" create_versionfile "${sdk_version}" "${version}"
) )
if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then
set -x set -x

105
ci-automation/sdk_bootstrap.sh
View File

@ -65,8 +65,8 @@ function sdk_bootstrap() {
# -- # --
function _sdk_bootstrap_impl() { function _sdk_bootstrap_impl() {
local seed_version="$1" local seed_version=${1}
local version="$2" local version=${2}
: ${ARCH:="amd64"} : ${ARCH:="amd64"}
source ci-automation/ci_automation_common.sh source ci-automation/ci_automation_common.sh
@ -86,65 +86,86 @@ function _sdk_bootstrap_impl() {
# builds without messing with actual release branches. # builds without messing with actual release branches.
local main_branch=${CIA_DEBUGMAINBRANCH:-main} local main_branch=${CIA_DEBUGMAINBRANCH:-main}
local nightly=${CIA_DEBUGNIGHTLY:-nightly} local nightly=${CIA_DEBUGNIGHTLY:-nightly}
# Patterns used below. # Matches the usual nightly tag name, optionally with an
local nightly_pattern_1='^main-[0-9.]+-'"${nightly}"'-[-0-9]+(-INTERMEDIATE)?$' # intermediate suffix of the build ID too
local nightly_pattern_2='^main-[0-9.]+-'"${nightly}"'-[-0-9]+$' # (main-1234.2.3-nightly-yyyymmdd-hhmm-INTERMEDIATE).
if [[ "${version}" =~ ${nightly_pattern_1} ]] \ local nightly_pattern_1='^main-[0-9]+(\.[0-9]+){2}-'"${nightly}"'-[0-9]{8}-[0-9]{4}(-INTERMEDIATE)?$'
&& [ "$(git rev-parse HEAD)" = "$(git rev-parse "origin/${main_branch}")" ] ; then local main_branch_hash=''
if [[ ${version} =~ ${nightly_pattern_1} ]]; then
main_branch_hash=$(git rev-parse "origin/${main_branch}")
fi
local -a existing_tags=()
if [[ -n ${main_branch_hash} ]]; then
if [[ $(git rev-parse HEAD) != "${main_branch_hash}" ]] ; then
echo "We are doing a nightly build but we are not on top of the ${main_branch} branch. This is wrong and would result in the nightly tag not being a part of the branch." >&2
exit 1
fi
target_branch=${main_branch} target_branch=${main_branch}
local existing_tag="" # Check for the existing tag only when we allow
# Check for the existing tag only when we allow shortcutting # shortcutting the builds. That way we can skip the checks
# the builds. That way we can skip the checks for build # for build shortcutting.
# shortcutting.
if bool_is_true "${AVOID_NIGHTLY_BUILD_SHORTCUTS}"; then if bool_is_true "${AVOID_NIGHTLY_BUILD_SHORTCUTS}"; then
echo "Continuing the build because AVOID_NIGHTLY_BUILD_SHORTCUTS is bool true (${AVOID_NIGHTLY_BUILD_SHORTCUTS})" >&2 echo "Continuing the build because AVOID_NIGHTLY_BUILD_SHORTCUTS is bool true (${AVOID_NIGHTLY_BUILD_SHORTCUTS})" >&2
else else
existing_tag=$(git tag --points-at HEAD) # exit code is always 0, output may be empty git fetch --all --tags --force
# exit code is always 0, output may be empty
mapfile -t existing_tags < <(git tag --points-at HEAD)
fi fi
# If the found tag is a nightly tag, we stop this build if there are no changes fi
if [[ "${existing_tag}" =~ ${nightly_pattern_2} ]]; then local nightly_pattern_2='^main-[0-9]+(\.[0-9]+){2}-'"${nightly}"'-[0-9]{8}-[0-9]{4}$'
local ret=0 local tag nightly_tag=''
git diff --exit-code "${existing_tag}" || ret=$? for tag in "${existing_tags[@]}"; do
if [ "$ret" = "0" ]; then if [[ ${tag} =~ ${nightly_pattern_2} ]]; then
local versions=( nightly_tag=${tag}
$( break
source sdk_lib/sdk_container_common.sh fi
source "${sdk_container_common_versionfile}" done
echo "${FLATCAR_SDK_VERSION}" # If the found tag is a nightly tag, we stop this build if there
echo "${FLATCAR_VERSION}" # are no changes and the relevant images can be found in the
) # bincache.
if [[ -n ${nightly_tag} ]]; then
local -i ret=0
git diff --exit-code --quiet "${nightly_tag}" || ret=$?
if [[ ret -eq 0 ]]; then
local -a versions=(
$(
source sdk_lib/sdk_container_common.sh
source "${sdk_container_common_versionfile}"
echo "${FLATCAR_SDK_VERSION}"
echo "${FLATCAR_VERSION}"
)
) )
local flatcar_sdk_version="${versions[0]}" local flatcar_sdk_version=${versions[0]}
local flatcar_version="${versions[1]}" local flatcar_version=${versions[1]}
local sdk_docker_vernum="" local sdk_docker_vernum=""
sdk_docker_vernum=$(vernum_to_docker_image_version "${flatcar_sdk_version}") sdk_docker_vernum=$(vernum_to_docker_image_version "${flatcar_sdk_version}")
if check_bincache_images_existence \ if check_bincache_images_existence \
"https://${BUILDCACHE_SERVER}/containers/${sdk_docker_vernum}/flatcar-sdk-all-${sdk_docker_vernum}.tar.zst" \ "https://${BUILDCACHE_SERVER}/containers/${sdk_docker_vernum}/flatcar-sdk-all-${sdk_docker_vernum}.tar.zst" \
"https://${BUILDCACHE_SERVER}/images/amd64/${flatcar_version}/flatcar_production_image.bin.bz2" \ "https://${BUILDCACHE_SERVER}/images/amd64/${flatcar_version}/flatcar_production_image.bin.bz2" \
"https://${BUILDCACHE_SERVER}/images/arm64/${flatcar_version}/flatcar_production_image.bin.bz2"; then "https://${BUILDCACHE_SERVER}/images/arm64/${flatcar_version}/flatcar_production_image.bin.bz2"; then
echo "Stopping build because there are no changes since tag ${existing_tag}, the SDK container tar ball and the Flatcar images exist" >&2 echo "Stopping build because there are no changes since tag ${nightly_tag}, the SDK container tar ball and the Flatcar images exist" >&2
return 0 return 0
fi fi
echo "No changes but continuing build because SDK container tar ball and/or the Flatcar images do not exist" >&2 echo "No changes but continuing build because SDK container tar ball and/or the Flatcar images do not exist" >&2
elif [ "$ret" = "1" ]; then elif [[ ret -eq 1 ]]; then
echo "Found changes since last tag ${existing_tag}" >&2 echo "HEAD is tagged with a nightly tag and yet there a differences? This is fishy and needs to be investigated. Maybe you forgot to commit your changes?" >&2
else exit 1
else
echo "Error: Unexpected git diff return code (${ret})" >&2 echo "Error: Unexpected git diff return code (${ret})" >&2
return 1 return 1
fi
fi fi
fi fi
if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then
set +x set +x
fi fi
local vernum="${version#*-}" # remove alpha-,beta-,stable-,lts- version tag local vernum=${version#*-} # remove alpha-,beta-,stable-,lts- version tag
local git_vernum="${vernum}" local git_vernum=${vernum}
# Update FLATCAR_VERSION[_ID], BUILD_ID, and SDK in versionfile # Update FLATCAR_VERSION[_ID], BUILD_ID, and SDK in versionfile
( (
source sdk_lib/sdk_container_common.sh source sdk_lib/sdk_container_common.sh
create_versionfile "${vernum}" create_versionfile "${vernum}"
) )
if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then
set -x set -x
@ -167,14 +188,16 @@ function _sdk_bootstrap_impl() {
# to ourselves, otherwise we could fail to sign the artifacts as # to ourselves, otherwise we could fail to sign the artifacts as
# we lacked write permissions in the directory of the signed # we lacked write permissions in the directory of the signed
# artifact # artifact
local uid=$(id --user) local uid
local gid=$(id --group) local gid
uid=$(id --user)
gid=$(id --group)
sudo chown --recursive "${uid}:${gid}" __build__ sudo chown --recursive "${uid}:${gid}" __build__
( (
cd "__build__/images/catalyst/builds/flatcar-sdk" cd "__build__/images/catalyst/builds/flatcar-sdk"
create_digests "${SIGNER}" "${dest_tarball}" create_digests "${SIGNER}" "${dest_tarball}"
sign_artifacts "${SIGNER}" "${dest_tarball}"* sign_artifacts "${SIGNER}" "${dest_tarball}"*
copy_to_buildcache "sdk/${ARCH}/${FLATCAR_SDK_VERSION}" "${dest_tarball}"* copy_to_buildcache "sdk/${ARCH}/${FLATCAR_SDK_VERSION}" "${dest_tarball}"*
) )
} }
# -- # --