diff --git a/ci-automation/ci_automation_common.sh b/ci-automation/ci_automation_common.sh index 64f675226e..87dca40b99 100644 --- a/ci-automation/ci_automation_common.sh +++ b/ci-automation/ci_automation_common.sh @@ -22,15 +22,15 @@ function check_version_string() { # -- function update_and_push_version() { - local version="$1" - local target_branch="${2:-}" + local version=${1} + local target_branch=${2:-} # set up author and email so git does not complain when tagging if ! git config --get user.name >/dev/null 2>&1 ; then - git -C . config user.name "${CI_GIT_AUTHOR}" + git config user.name "${CI_GIT_AUTHOR}" fi if ! git config --get user.email >/dev/null 2>&1 ; then - git -C . config user.email "${CI_GIT_EMAIL}" + git config user.email "${CI_GIT_EMAIL}" fi # Add and commit local changes @@ -39,33 +39,34 @@ function update_and_push_version() { git commit --allow-empty -m "New version: ${version}" git fetch --all --tags --force - local ret=0 - git diff --exit-code "${version}" || ret=$? + local -i ret=0 + git diff --quiet --exit-code "${version}" 2>/dev/null || ret=${?} # This will return != 0 if # - the remote tag does not exist (rc: 127) # - the tag does not exist locally (rc: 128) # - the remote tag has changes compared to the local tree (rc: 1) - if [ "$ret" = "0" ]; then - echo "Reusing existing tag" >&2 - git checkout -f "${version}" - return - elif [ "$ret" = "1" ]; then - echo "Remote tag exists already and is not equal" >&2 - return 1 - elif [ "$ret" != "127" ] && [ "$ret" != "128" ]; then - echo "Error: Unexpected git diff return code ($ret)" >&2 - return 1 + if [[ ret -eq 0 ]]; then + # this means that we created an empty commit above, reusing the tag gets rid of it + echo "Reusing existing tag" >&2 + git checkout --force "${version}" + return + elif [[ ret -eq 1 ]]; then + echo "Remote tag exists already and is not equal" >&2 + return 1 + elif [[ ret -ne 127 && ret -ne 128 ]]; then + echo "Error: Unexpected git diff return code (${ret})" >&2 + return 1 fi - local -a TAG_ARGS - if [ "${SIGN-0}" = 1 ]; then - TAG_ARGS=("-s" "-m" "${version}") + local -a TAG_ARGS=() + if [[ ${SIGN-0} = 1 ]]; then + TAG_ARGS=("--sign" "--message=${version}") fi - git tag -f "${TAG_ARGS[@]}" "${version}" + git tag --force "${TAG_ARGS[@]}" "${version}" - if [[ -n "${target_branch}" ]]; then - git push origin "HEAD:${target_branch}" + if [[ -n ${target_branch} ]]; then + git push origin "HEAD:${target_branch}" fi git push origin "${version}" diff --git a/ci-automation/packages-tag.sh b/ci-automation/packages-tag.sh index 8a6fe93769..475490b49a 100644 --- a/ci-automation/packages-tag.sh +++ b/ci-automation/packages-tag.sh @@ -56,7 +56,7 @@ function packages_tag() { # -- function _packages_tag_impl() { - local version="$1" + local version=${1} source ci-automation/ci_automation_common.sh source ci-automation/gpg_setup.sh @@ -64,7 +64,7 @@ function _packages_tag_impl() { check_version_string "${version}" source sdk_container/.repo/manifests/version.txt - local sdk_version="${FLATCAR_SDK_VERSION}" + local sdk_version=${FLATCAR_SDK_VERSION} if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then set -x @@ -77,41 +77,69 @@ function _packages_tag_impl() { # builds without messing with actual release branches. local flatcar_branch_prefix=${CIA_DEBUGFLATCARBRANCHPREFIX:-flatcar} local nightly=${CIA_DEBUGNIGHTLY:-nightly} - # Patterns used below. - local nightly_pattern_1='^(stable|alpha|beta|lts)-[0-9.]+-'"${nightly}"'-[-0-9]+$' - local nightly_pattern_2='^(stable|alpha|beta|lts)-[0-9.]+(|-'"${nightly}"'-[-0-9]+)$' - local flatcar_pattern='^'"${flatcar_branch_prefix}"'-[0-9]+$' - if [[ "${version}" =~ ${nightly_pattern_1} ]] \ - && [[ "$(git rev-parse --abbrev-ref HEAD)" =~ ${flatcar_pattern} ]] ; then - target_branch="$(git rev-parse --abbrev-ref HEAD)" - local existing_tag="" + # Matches the usual nightly tag name (stable-1234.2.3-nightly-yyyymmdd-hhmm) + local nightly_pattern='^(stable|alpha|beta|lts)-([0-9]+)(\.[0-9]+){2}-'"${nightly}"'-[0-9]{8}-[0-9]{4}$' + local -i major_version=0 + local branch_name='' + local branch_hash='' + if [[ ${version} =~ ${nightly_pattern} ]]; then + major_version=${BASH_REMATCH[2]} + branch_name=${flatcar_branch_prefix}-${major_version} + branch_hash=$(git rev-parse "origin/${branch_name}") + fi + local -a existing_tags=() + if [[ -n ${branch_hash} ]]; then + if [[ $(git rev-parse HEAD) != "${branch_hash}" ]]; then + echo "We are doing a nightly build but we are not on top of the ${branch_name} branch. This is wrong and would result in the nightly tag not being a part of the branch." >&2 + exit 1 + fi + target_branch=${branch_name} # Check for the existing tag only when we allow shortcutting # the builds. That way we can skip the checks for build # shortcutting. if bool_is_true "${AVOID_NIGHTLY_BUILD_SHORTCUTS}"; then echo "Continuing the build because AVOID_NIGHTLY_BUILD_SHORTCUTS is bool true (${AVOID_NIGHTLY_BUILD_SHORTCUTS})" >&2 else - existing_tag=$(git tag --points-at HEAD) # exit code is always 0, output may be empty + git fetch --all --tags --force + # exit code of git tag is always 0; output may be empty, + # but may also have multiple tags + mapfile -t existing_tags < <(git tag --points-at HEAD) fi - # If the found tag is a release or nightly tag, we stop this build if there are no changes - if [[ "${existing_tag}" =~ ${nightly_pattern_2} ]]; then - local ret=0 - git diff --exit-code "${existing_tag}" || ret=$? - if [[ ret -eq 0 ]]; then + fi + local nightly_or_release_tag='' + if [[ major_version -gt 0 && ${#existing_tags[@]} -gt 0 ]]; then + local nightly_or_release_pattern='^(stable|alpha|beta|lts)-'"${major_version}"'(\.[0-9]+){2}(-'"${nightly}"'-[0-9]{8}-[0-9]{4})?$' + local tag + for tag in "${existing_tags[@]}"; do + if [[ ${tag} =~ ${nightly_or_release_pattern} ]]; then + nightly_or_release_tag=${tag} + break + fi + done + fi + # If the found tag is a release or nightly tag, we stop this build + # if there are no changes and the relevant images can be found in + # bincache. + if [[ -n ${nightly_or_release_tag} ]]; then + local -i ret=0 + git diff --exit-code --quiet "${nightly_or_release_tag}" || ret=$? + if [[ ret -eq 0 ]]; then + # no changes in the code, but check if images exist (they + # could be missing if build failed) if check_bincache_images_existence \ "https://${BUILDCACHE_SERVER}/images/amd64/${FLATCAR_VERSION}/flatcar_production_image.bin.bz2" \ "https://${BUILDCACHE_SERVER}/images/arm64/${FLATCAR_VERSION}/flatcar_production_image.bin.bz2"; then touch ./skip-build - echo "Creating ./skip-build flag file, indicating that the build must not to continue because no new tag got created as there are no changes since tag ${existing_tag} and the Flatcar images exist" >&2 + echo "Creating ./skip-build flag file, indicating that the build must not to continue because no new tag got created as there are no changes since tag ${nightly_or_release_tag} and the Flatcar images exist" >&2 return 0 fi echo "No changes but continuing build because Flatcar images do not exist" - elif [[ ret -eq 1 ]]; then - echo "Found changes since last tag ${existing_tag}" >&2 - else + elif [[ ret -eq 1 ]]; then + echo "HEAD is tagged with a nightly tag and yet there a differences? This is fishy and needs to be investigated. Maybe you forgot to commit your changes?" >&2 + exit 1 + else echo "Error: Unexpected git diff return code (${ret})" >&2 return 1 - fi fi fi if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then @@ -120,8 +148,8 @@ function _packages_tag_impl() { # Create version file ( - source sdk_lib/sdk_container_common.sh - create_versionfile "$sdk_version" "$version" + source sdk_lib/sdk_container_common.sh + create_versionfile "${sdk_version}" "${version}" ) if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then set -x diff --git a/ci-automation/sdk_bootstrap.sh b/ci-automation/sdk_bootstrap.sh index 708e5927b2..89d5b4bdc6 100644 --- a/ci-automation/sdk_bootstrap.sh +++ b/ci-automation/sdk_bootstrap.sh @@ -65,8 +65,8 @@ function sdk_bootstrap() { # -- function _sdk_bootstrap_impl() { - local seed_version="$1" - local version="$2" + local seed_version=${1} + local version=${2} : ${ARCH:="amd64"} source ci-automation/ci_automation_common.sh @@ -86,65 +86,86 @@ function _sdk_bootstrap_impl() { # builds without messing with actual release branches. local main_branch=${CIA_DEBUGMAINBRANCH:-main} local nightly=${CIA_DEBUGNIGHTLY:-nightly} - # Patterns used below. - local nightly_pattern_1='^main-[0-9.]+-'"${nightly}"'-[-0-9]+(-INTERMEDIATE)?$' - local nightly_pattern_2='^main-[0-9.]+-'"${nightly}"'-[-0-9]+$' - if [[ "${version}" =~ ${nightly_pattern_1} ]] \ - && [ "$(git rev-parse HEAD)" = "$(git rev-parse "origin/${main_branch}")" ] ; then + # Matches the usual nightly tag name, optionally with an + # intermediate suffix of the build ID too + # (main-1234.2.3-nightly-yyyymmdd-hhmm-INTERMEDIATE). + local nightly_pattern_1='^main-[0-9]+(\.[0-9]+){2}-'"${nightly}"'-[0-9]{8}-[0-9]{4}(-INTERMEDIATE)?$' + local main_branch_hash='' + if [[ ${version} =~ ${nightly_pattern_1} ]]; then + main_branch_hash=$(git rev-parse "origin/${main_branch}") + fi + local -a existing_tags=() + if [[ -n ${main_branch_hash} ]]; then + if [[ $(git rev-parse HEAD) != "${main_branch_hash}" ]] ; then + echo "We are doing a nightly build but we are not on top of the ${main_branch} branch. This is wrong and would result in the nightly tag not being a part of the branch." >&2 + exit 1 + fi target_branch=${main_branch} - local existing_tag="" - # Check for the existing tag only when we allow shortcutting - # the builds. That way we can skip the checks for build - # shortcutting. + # Check for the existing tag only when we allow + # shortcutting the builds. That way we can skip the checks + # for build shortcutting. if bool_is_true "${AVOID_NIGHTLY_BUILD_SHORTCUTS}"; then echo "Continuing the build because AVOID_NIGHTLY_BUILD_SHORTCUTS is bool true (${AVOID_NIGHTLY_BUILD_SHORTCUTS})" >&2 else - existing_tag=$(git tag --points-at HEAD) # exit code is always 0, output may be empty + git fetch --all --tags --force + # exit code is always 0, output may be empty + mapfile -t existing_tags < <(git tag --points-at HEAD) fi - # If the found tag is a nightly tag, we stop this build if there are no changes - if [[ "${existing_tag}" =~ ${nightly_pattern_2} ]]; then - local ret=0 - git diff --exit-code "${existing_tag}" || ret=$? - if [ "$ret" = "0" ]; then - local versions=( - $( - source sdk_lib/sdk_container_common.sh - source "${sdk_container_common_versionfile}" - echo "${FLATCAR_SDK_VERSION}" - echo "${FLATCAR_VERSION}" - ) + fi + local nightly_pattern_2='^main-[0-9]+(\.[0-9]+){2}-'"${nightly}"'-[0-9]{8}-[0-9]{4}$' + local tag nightly_tag='' + for tag in "${existing_tags[@]}"; do + if [[ ${tag} =~ ${nightly_pattern_2} ]]; then + nightly_tag=${tag} + break + fi + done + # If the found tag is a nightly tag, we stop this build if there + # are no changes and the relevant images can be found in the + # bincache. + if [[ -n ${nightly_tag} ]]; then + local -i ret=0 + git diff --exit-code --quiet "${nightly_tag}" || ret=$? + if [[ ret -eq 0 ]]; then + local -a versions=( + $( + source sdk_lib/sdk_container_common.sh + source "${sdk_container_common_versionfile}" + echo "${FLATCAR_SDK_VERSION}" + echo "${FLATCAR_VERSION}" + ) ) - local flatcar_sdk_version="${versions[0]}" - local flatcar_version="${versions[1]}" + local flatcar_sdk_version=${versions[0]} + local flatcar_version=${versions[1]} local sdk_docker_vernum="" sdk_docker_vernum=$(vernum_to_docker_image_version "${flatcar_sdk_version}") if check_bincache_images_existence \ "https://${BUILDCACHE_SERVER}/containers/${sdk_docker_vernum}/flatcar-sdk-all-${sdk_docker_vernum}.tar.zst" \ "https://${BUILDCACHE_SERVER}/images/amd64/${flatcar_version}/flatcar_production_image.bin.bz2" \ "https://${BUILDCACHE_SERVER}/images/arm64/${flatcar_version}/flatcar_production_image.bin.bz2"; then - echo "Stopping build because there are no changes since tag ${existing_tag}, the SDK container tar ball and the Flatcar images exist" >&2 + echo "Stopping build because there are no changes since tag ${nightly_tag}, the SDK container tar ball and the Flatcar images exist" >&2 return 0 fi echo "No changes but continuing build because SDK container tar ball and/or the Flatcar images do not exist" >&2 - elif [ "$ret" = "1" ]; then - echo "Found changes since last tag ${existing_tag}" >&2 - else + elif [[ ret -eq 1 ]]; then + echo "HEAD is tagged with a nightly tag and yet there a differences? This is fishy and needs to be investigated. Maybe you forgot to commit your changes?" >&2 + exit 1 + else echo "Error: Unexpected git diff return code (${ret})" >&2 return 1 - fi fi fi if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then set +x fi - local vernum="${version#*-}" # remove alpha-,beta-,stable-,lts- version tag - local git_vernum="${vernum}" + local vernum=${version#*-} # remove alpha-,beta-,stable-,lts- version tag + local git_vernum=${vernum} # Update FLATCAR_VERSION[_ID], BUILD_ID, and SDK in versionfile ( - source sdk_lib/sdk_container_common.sh - create_versionfile "${vernum}" + source sdk_lib/sdk_container_common.sh + create_versionfile "${vernum}" ) if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then set -x @@ -167,14 +188,16 @@ function _sdk_bootstrap_impl() { # to ourselves, otherwise we could fail to sign the artifacts as # we lacked write permissions in the directory of the signed # artifact - local uid=$(id --user) - local gid=$(id --group) + local uid + local gid + uid=$(id --user) + gid=$(id --group) sudo chown --recursive "${uid}:${gid}" __build__ ( - cd "__build__/images/catalyst/builds/flatcar-sdk" - create_digests "${SIGNER}" "${dest_tarball}" - sign_artifacts "${SIGNER}" "${dest_tarball}"* - copy_to_buildcache "sdk/${ARCH}/${FLATCAR_SDK_VERSION}" "${dest_tarball}"* + cd "__build__/images/catalyst/builds/flatcar-sdk" + create_digests "${SIGNER}" "${dest_tarball}" + sign_artifacts "${SIGNER}" "${dest_tarball}"* + copy_to_buildcache "sdk/${ARCH}/${FLATCAR_SDK_VERSION}" "${dest_tarball}"* ) } # --