mirrors/flatcar-scripts
1
0
Fork 0
mirror of https://github.com/flatcar/scripts.git synced 2025-08-07 21:16:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

ci-automation: use a single git tag and skip nightlies with no changes

Browse Source 
The pipeline created two tags if an SDK was built, one for the SDK and
one for the OS build (which was a free-standing tag or a local state
that was equivalent to the existing tag of the same name). The
nightlies created update commits on the main branch, even if no change
was done, and on the release branches we lacked these commits.

Create the release tag in the nightly SDK bootstrap already and reuse
it for the nightly OS build. Instead of local state, checkout the
existing tags explicitly. Extend the nightly update commit logic to
cover release branches and detect if we can skip building because no
changes were done.
This commit is contained in:
Kai Lueke 2022-03-31 14:05:51 +02:00
parent 3d761ed8e8
commit bd970357c8
5 changed files with 107 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
ci-automation/README.md
View File

@ -46,23 +46,23 @@ image_build amd64
The resulting image will come in "amd64", "arm64", and "all" flavours, with support for respective OS target architectures. This step builds the Flatcar SDK container images published at ghcr.io/flatcar-linux.
```
.---------. .------------. .--------.
| scripts | | CI | | Build |
| repo | | automation | | cache |
`---------´ `------------´ `--------´
| | |
| "alpha-3449.0.0-dev23" |
| | |
| _______v_______ |
+------- clone -----> ( SDK bootstrap ) |
| `-------------´ |
|<- tag: sdk-3499.0.0-dev23 -´|`--- sdk tarball --->|
| | |
| _______v_______ |
+-- clone -> ( SDK container ) |
| sdk-3499.0.0-dev23 `-------------´ |
| |`- sdk container --->|
v v image
.---------. .------------. .--------.
| scripts | | CI | | Build |
| repo | | automation | | cache |
`---------´ `------------´ `--------´
| | |
| "alpha-3449.0.0-dev23" |
| | |
| _______v_______ |
+-------- clone -------> ( SDK bootstrap ) |
| `-------------´ |
|<- tag: alpha-3499.0.0-dev23 --´|`--- sdk tarball --->|
| | |
| _______v_______ |
+-------- clone -------> ( SDK container ) |
| alpha-3499.0.0-dev23 `-------------´ |
| |`- sdk container --->|
v v image
continue to OS
image build
|

38
ci-automation/ci_automation_common.sh
View File

@ -66,6 +66,25 @@ function update_and_push_version() {
git commit --allow-empty -m "New version: ${version}"
git fetch --all --tags --force
local ret=0
git diff --exit-code "${version}" || ret=$?
# This will return != 0 if
# - the remote tag does not exist (rc: 127)
# - the tag does not exist locally (rc: 128)
# - the remote tag has changes compared to the local tree (rc: 1)
if [ "$ret" = "0" ]; then
echo "Reusing existing tag" >&2
git checkout -f --recurse-submodules "${version}"
return
elif [ "$ret" = "1" ]; then
echo "Remote tag exists already and is not equal" >&2
return 1
elif [ "$ret" != "127" ] && [ "$ret" != "128" ]; then
echo "Error: Unexpected git diff return code ($ret)" >&2
return 1
fi
local -a TAG_ARGS
if [ "${SIGN-0}" = 1 ]; then
TAG_ARGS=("-s" "-m" "${version}")
@ -78,24 +97,7 @@ function update_and_push_version() {
git push origin "${branch}"
fi
if git push origin "${version}" ; then
return
fi
# Push (above) may fail because a tag already exists.
# We check for tag presence, and for the difference
# between local and remote, and bail
# only if the remote / local contents differ.
# Remove local tag, (re-)fetch remote tags
git tag -d "${version}"
# refresh tree, let origin overwrite local tags
git fetch --all --tags --force
# This will return != 0 if
# - the remote tag does not exist (rc: 127)
# - the remote tag has changes compared to the local tree (rc: 1)
git diff --exit-code "${version}"
git push origin "${version}"
}
# --

32
ci-automation/garbage_collect.sh
View File

@ -32,7 +32,7 @@ function garbage_collect() {
local purge_versions="${PURGE_VERSIONS:-}"
local versions_detected="$(git tag -l --sort=-committerdate \
| grep -E '(main|alpha|beta|stable|lts|sdk)-[0-9]+\.[0-9]+\.[0-9]+\-.*' \
| grep -E '(main|alpha|beta|stable|lts)-[0-9]+\.[0-9]+\.[0-9]+\-.*' \
| grep -vE '(-pro)$')"
echo "######## Full list of version(s) found ########"
@ -45,7 +45,7 @@ function garbage_collect() {
else
# make sure we only accept dev versions
purge_versions="$(echo "${purge_versions}" | sed 's/ /\n/g' \
| grep -E '(main|alpha|beta|stable|lts|sdk)-[0-9]+\.[0-9]+\.[0-9]+\-.*' \
| grep -E '(main|alpha|beta|stable|lts)-[0-9]+\.[0-9]+\.[0-9]+\-.*' \
| grep -vE '(-pro)$')"
fi
@ -74,26 +74,20 @@ function garbage_collect() {
git checkout "${version}" -- sdk_container/.repo/manifests/version.txt
source sdk_container/.repo/manifests/version.txt
# Assuming that the SDK build version also has the same OS version
local os_vernum="${FLATCAR_VERSION}"
local os_docker_vernum="$(vernum_to_docker_image_version "${FLATCAR_VERSION}")"
local sdk_vernum="${FLATCAR_SDK_VERSION}"
local sdk_docker_vernum="$(vernum_to_docker_image_version "${FLATCAR_SDK_VERSION}")"
# Remove container image tarballs and SDK tarball (if applicable)
#
local rmpat=""
if [[ "${version}" = 'sdk-'* ]] ; then
echo "## ${version} is an SDK version. ##"
rmpat="${BUILDCACHE_PATH_PREFIX}/sdk/*/${sdk_vernum}/"
rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${sdk_docker_vernum}/flatcar-sdk-*"
else
echo "## ${version} is an OS image version. ##"
rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-packages-*"
rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/boards/*/${os_vernum}/"
rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-images-*"
rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/images/*/${os_vernum}/"
rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/testing/${os_vernum}/"
fi
rmpat="${BUILDCACHE_PATH_PREFIX}/sdk/*/${os_vernum}/"
rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-sdk-*"
rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-packages-*"
rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/boards/*/${os_vernum}/"
rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-images-*"
rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/images/*/${os_vernum}/"
rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/testing/${os_vernum}/"
echo "## The following files will be removed ##"
$sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \
@ -110,11 +104,7 @@ function garbage_collect() {
# Remove container image directory if empty
#
if [[ "${version}" = 'sdk-'* ]] ; then
rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${sdk_docker_vernum}/"
else
rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/"
fi
rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/"
echo "## Checking if container directory is empty and can be removed (it's OK if this fails) ##"
echo "## The following directory will be removed if below output is empty: '${rmpat}' ##"

30
ci-automation/packages.sh
View File

@ -80,6 +80,34 @@ function packages_build() {
update_submodule "portage-stable" "${portage_git}"
fi
# Create new tag in scripts repo w/ updated versionfile + submodules.
# Also push the changes to the branch ONLY IF we're doing a nightly
# build of the 'main'/'flatcar-MAJOR' branch AND we're definitely ON the respective branch
# (`scripts` and submodules).
local push_branch="false"
if [[ "${version}" =~ ^(stable|alpha|beta|lts)-[0-9.]+-nightly-[-0-9]+$ ]] \
&& [[ "$(git rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] \
&& [[ "$(git -C sdk_container/src/third_party/coreos-overlay/ rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] \
&& [[ "$(git -C sdk_container/src/third_party/portage-stable/ rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] ; then
push_branch="true"
local existing_tag=""
existing_tag=$(git tag --points-at HEAD) # exit code is always 0, output may be empty
# If the found tag is a release or nightly tag, we stop this build if there are no changes
if [[ "${existing_tag}" =~ ^(stable|alpha|beta|lts)-[0-9.]+(|-nightly-[-0-9]+)$ ]]; then
local ret=0
git diff --exit-code "${existing_tag}" || ret=$?
if [ "$ret" = "0" ]; then
echo "Stopping build because there are no changes since tag ${existing_tag}" >&2
return 0
elif [ "$ret" = "1" ]; then
echo "Found changes since last tag ${existing_tag}" >&2
else
echo "Error: Unexpected git diff return code (${ret})" >&2
return 1
fi
fi
fi
# Get SDK from either the registry or import from build cache
# This is a NOP if the image is present locally.
local sdk_name="flatcar-sdk-${arch}"
@ -127,6 +155,6 @@ function packages_build() {
copy_to_buildcache "images/${arch}/${vernum}/torcx" \
"${torcx_tmp}/torcx/pkgs/${arch}-usr/docker/"*/*.torcx.tgz
update_and_push_version "${version}"
update_and_push_version "${version}" "${push_branch}"
}
# --

43
ci-automation/sdk_bootstrap.sh
View File

@ -22,7 +22,7 @@
# 2. Version of the TARGET SDK to build (string).
# The version pattern 'MMMM.m.p' (e.g. '3051.0.0') denotes a "official" build, i.e. a release build to be published.
# Use any version diverging from the pattern (e.g. '3051.0.0-nightly-4302') for development / CI builds.
# A tag "sdk-[VERSION]" will be created in the scripts repo and pushed upstream.
# A free-standing tagged commit will be created in the scripts repo and pushed upstream.
#
# OPTIONAL INPUT:
#
@ -69,6 +69,34 @@ function sdk_bootstrap() {
update_submodule "portage-stable" "${portage_git}"
fi
# Create new tag in scripts repo w/ updated versionfile + submodules.
# Also push the changes to the branch ONLY IF we're doing a nightly
# build of the 'main' branch AND we're definitely ON the main branch
# (`scripts` and submodules).
local push_branch="false"
if [[ "${version}" =~ ^main-[0-9.]+-nightly-[-0-9]+$ ]] \
&& [ "$(git rev-parse --abbrev-ref HEAD)" = "main" ] \
&& [ "$(git -C sdk_container/src/third_party/coreos-overlay/ rev-parse --abbrev-ref HEAD)" = "main" ] \
&& [ "$(git -C sdk_container/src/third_party/portage-stable/ rev-parse --abbrev-ref HEAD)" = "main" ] ; then
push_branch="true"
local existing_tag=""
existing_tag=$(git tag --points-at HEAD) # exit code is always 0, output may be empty
# If the found tag is a nightly tag, we stop this build if there are no changes
if [[ "${existing_tag}" =~ ^main-[0-9.]+-nightly-[-0-9]+$ ]]; then
local ret=0
git diff --exit-code "${existing_tag}" || ret=$?
if [ "$ret" = "0" ]; then
echo "Stopping build because there are no changes since tag ${existing_tag}" >&2
return 0
elif [ "$ret" = "1" ]; then
echo "Found changes since last tag ${existing_tag}" >&2
else
echo "Error: Unexpected git diff return code (${ret})" >&2
return 1
fi
fi
fi
local vernum="${version#*-}" # remove alpha-,beta-,stable-,lts- version tag
local git_vernum="${vernum}"
@ -84,17 +112,6 @@ function sdk_bootstrap() {
copy_to_buildcache "sdk/${ARCH}/${vernum}" "${dest_tarball}"*
cd -
# Create new tag in scripts repo w/ updated versionfile + submodules.
# Also push the changes to the branch ONLY IF we're doing a nightly
# build of the 'main' branch AND we're definitely ON the main branch
# (`scripts` and submodules).
local push_branch="false"
if [[ "${version}" =~ ^main-[0-9.]+-nightly-[-0-9]+$ ]] \
&& [ "$(git rev-parse --abbrev-ref HEAD)" = "main" ] \
&& [ "$(git -C sdk_container/src/third_party/coreos-overlay/ rev-parse --abbrev-ref HEAD)" = "main" ] \
&& [ "$(git -C sdk_container/src/third_party/portage-stable/ rev-parse --abbrev-ref HEAD)" = "main" ] ; then
push_branch="true"
fi
update_and_push_version "sdk-${git_vernum}" "${push_branch}"
update_and_push_version "${version}" "${push_branch}"
}
# --