diff --git a/ci-automation/ci_automation_common.sh b/ci-automation/ci_automation_common.sh index a0745b25ec..87dca40b99 100644 --- a/ci-automation/ci_automation_common.sh +++ b/ci-automation/ci_automation_common.sh @@ -22,15 +22,15 @@ function check_version_string() { # -- function update_and_push_version() { - local version="$1" - local target_branch="${2:-}" + local version=${1} + local target_branch=${2:-} # set up author and email so git does not complain when tagging if ! git config --get user.name >/dev/null 2>&1 ; then - git -C . config user.name "${CI_GIT_AUTHOR}" + git config user.name "${CI_GIT_AUTHOR}" fi if ! git config --get user.email >/dev/null 2>&1 ; then - git -C . config user.email "${CI_GIT_EMAIL}" + git config user.email "${CI_GIT_EMAIL}" fi # Add and commit local changes @@ -39,39 +39,50 @@ function update_and_push_version() { git commit --allow-empty -m "New version: ${version}" git fetch --all --tags --force - local ret=0 - git diff --exit-code "${version}" || ret=$? + local -i ret=0 + git diff --quiet --exit-code "${version}" 2>/dev/null || ret=${?} # This will return != 0 if # - the remote tag does not exist (rc: 127) # - the tag does not exist locally (rc: 128) # - the remote tag has changes compared to the local tree (rc: 1) - if [ "$ret" = "0" ]; then - echo "Reusing existing tag" >&2 - git checkout -f "${version}" - return - elif [ "$ret" = "1" ]; then - echo "Remote tag exists already and is not equal" >&2 - return 1 - elif [ "$ret" != "127" ] && [ "$ret" != "128" ]; then - echo "Error: Unexpected git diff return code ($ret)" >&2 - return 1 + if [[ ret -eq 0 ]]; then + # this means that we created an empty commit above, reusing the tag gets rid of it + echo "Reusing existing tag" >&2 + git checkout --force "${version}" + return + elif [[ ret -eq 1 ]]; then + echo "Remote tag exists already and is not equal" >&2 + return 1 + elif [[ ret -ne 127 && ret -ne 128 ]]; then + echo "Error: Unexpected git diff return code (${ret})" >&2 + return 1 fi - local -a TAG_ARGS - if [ "${SIGN-0}" = 1 ]; then - TAG_ARGS=("-s" "-m" "${version}") + local -a TAG_ARGS=() + if [[ ${SIGN-0} = 1 ]]; then + TAG_ARGS=("--sign" "--message=${version}") fi - git tag -f "${TAG_ARGS[@]}" "${version}" + git tag --force "${TAG_ARGS[@]}" "${version}" - if [[ -n "${target_branch}" ]]; then - git push origin "HEAD:${target_branch}" + if [[ -n ${target_branch} ]]; then + git push origin "HEAD:${target_branch}" fi git push origin "${version}" } # -- +function check_bincache_images_existence() { + case ${CIA_DEBUGIMAGESEXIST:-} in + 'yes') return 0;; + 'no' ) return 1;; + '') curl --head --fail --fail-early --silent --show-error --location "${@}" || return 1;; + *) echo "Invalid CIA_DEBUGIMAGESEXIST value (${CIA_DEBUGIMAGESEXIST@Q})" >&2; exit 1;; + esac +} +# -- + function copy_from_buildcache() { local what="$1" local where_to="$2" diff --git a/ci-automation/packages-tag.sh b/ci-automation/packages-tag.sh index 053f893546..475490b49a 100644 --- a/ci-automation/packages-tag.sh +++ b/ci-automation/packages-tag.sh @@ -56,7 +56,7 @@ function packages_tag() { # -- function _packages_tag_impl() { - local version="$1" + local version=${1} source ci-automation/ci_automation_common.sh source ci-automation/gpg_setup.sh @@ -64,59 +64,100 @@ function _packages_tag_impl() { check_version_string "${version}" source sdk_container/.repo/manifests/version.txt - local sdk_version="${FLATCAR_SDK_VERSION}" + local sdk_version=${FLATCAR_SDK_VERSION} + if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then + set -x + fi # Create new tag in scripts repo w/ updated versionfile # Also push the changes to the branch ONLY IF we're doing a nightly # build of the 'flatcar-MAJOR' branch AND we're definitely ON the respective branch local target_branch='' # These variables are here to make it easier to test nightly # builds without messing with actual release branches. - local flatcar_branch_prefix='flatcar' - local nightly='nightly' - # Patterns used below. - local nightly_pattern_1='^(stable|alpha|beta|lts)-[0-9.]+-'"${nightly}"'-[-0-9]+$' - local nightly_pattern_2='^(stable|alpha|beta|lts)-[0-9.]+(|-'"${nightly}"'-[-0-9]+)$' - local flatcar_pattern='^'"${flatcar_branch_prefix}"'-[0-9]+$' - if [[ "${version}" =~ ${nightly_pattern_1} ]] \ - && [[ "$(git rev-parse --abbrev-ref HEAD)" =~ ${flatcar_pattern} ]] ; then - target_branch="$(git rev-parse --abbrev-ref HEAD)" - local existing_tag="" + local flatcar_branch_prefix=${CIA_DEBUGFLATCARBRANCHPREFIX:-flatcar} + local nightly=${CIA_DEBUGNIGHTLY:-nightly} + # Matches the usual nightly tag name (stable-1234.2.3-nightly-yyyymmdd-hhmm) + local nightly_pattern='^(stable|alpha|beta|lts)-([0-9]+)(\.[0-9]+){2}-'"${nightly}"'-[0-9]{8}-[0-9]{4}$' + local -i major_version=0 + local branch_name='' + local branch_hash='' + if [[ ${version} =~ ${nightly_pattern} ]]; then + major_version=${BASH_REMATCH[2]} + branch_name=${flatcar_branch_prefix}-${major_version} + branch_hash=$(git rev-parse "origin/${branch_name}") + fi + local -a existing_tags=() + if [[ -n ${branch_hash} ]]; then + if [[ $(git rev-parse HEAD) != "${branch_hash}" ]]; then + echo "We are doing a nightly build but we are not on top of the ${branch_name} branch. This is wrong and would result in the nightly tag not being a part of the branch." >&2 + exit 1 + fi + target_branch=${branch_name} # Check for the existing tag only when we allow shortcutting # the builds. That way we can skip the checks for build # shortcutting. if bool_is_true "${AVOID_NIGHTLY_BUILD_SHORTCUTS}"; then echo "Continuing the build because AVOID_NIGHTLY_BUILD_SHORTCUTS is bool true (${AVOID_NIGHTLY_BUILD_SHORTCUTS})" >&2 else - existing_tag=$(git tag --points-at HEAD) # exit code is always 0, output may be empty + git fetch --all --tags --force + # exit code of git tag is always 0; output may be empty, + # but may also have multiple tags + mapfile -t existing_tags < <(git tag --points-at HEAD) fi - # If the found tag is a release or nightly tag, we stop this build if there are no changes - if [[ "${existing_tag}" =~ ${nightly_pattern_2} ]]; then - local ret=0 - git diff --exit-code "${existing_tag}" || ret=$? - if [[ ret -eq 0 ]]; then - if curl --head --fail --silent --show-error --location "https://${BUILDCACHE_SERVER}/images/amd64/${FLATCAR_VERSION}/flatcar_production_image.bin.bz2" \ - && curl --head --fail --silent --show-error --location "https://${BUILDCACHE_SERVER}/images/arm64/${FLATCAR_VERSION}/flatcar_production_image.bin.bz2"; then + fi + local nightly_or_release_tag='' + if [[ major_version -gt 0 && ${#existing_tags[@]} -gt 0 ]]; then + local nightly_or_release_pattern='^(stable|alpha|beta|lts)-'"${major_version}"'(\.[0-9]+){2}(-'"${nightly}"'-[0-9]{8}-[0-9]{4})?$' + local tag + for tag in "${existing_tags[@]}"; do + if [[ ${tag} =~ ${nightly_or_release_pattern} ]]; then + nightly_or_release_tag=${tag} + break + fi + done + fi + # If the found tag is a release or nightly tag, we stop this build + # if there are no changes and the relevant images can be found in + # bincache. + if [[ -n ${nightly_or_release_tag} ]]; then + local -i ret=0 + git diff --exit-code --quiet "${nightly_or_release_tag}" || ret=$? + if [[ ret -eq 0 ]]; then + # no changes in the code, but check if images exist (they + # could be missing if build failed) + if check_bincache_images_existence \ + "https://${BUILDCACHE_SERVER}/images/amd64/${FLATCAR_VERSION}/flatcar_production_image.bin.bz2" \ + "https://${BUILDCACHE_SERVER}/images/arm64/${FLATCAR_VERSION}/flatcar_production_image.bin.bz2"; then touch ./skip-build - echo "Creating ./skip-build flag file, indicating that the build must not to continue because no new tag got created as there are no changes since tag ${existing_tag} and the Flatcar images exist" >&2 + echo "Creating ./skip-build flag file, indicating that the build must not to continue because no new tag got created as there are no changes since tag ${nightly_or_release_tag} and the Flatcar images exist" >&2 return 0 fi echo "No changes but continuing build because Flatcar images do not exist" - elif [[ ret -eq 1 ]]; then - echo "Found changes since last tag ${existing_tag}" >&2 - else + elif [[ ret -eq 1 ]]; then + echo "HEAD is tagged with a nightly tag and yet there a differences? This is fishy and needs to be investigated. Maybe you forgot to commit your changes?" >&2 + exit 1 + else echo "Error: Unexpected git diff return code (${ret})" >&2 return 1 - fi fi fi + if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then + set +x + fi # Create version file ( - source sdk_lib/sdk_container_common.sh - create_versionfile "$sdk_version" "$version" + source sdk_lib/sdk_container_common.sh + create_versionfile "${sdk_version}" "${version}" ) + if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then + set -x + fi update_and_push_version "${version}" "${target_branch}" + if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then + exit 0 + fi apply_local_patches } # -- diff --git a/ci-automation/sdk_bootstrap.sh b/ci-automation/sdk_bootstrap.sh index 3540ec9023..89d5b4bdc6 100644 --- a/ci-automation/sdk_bootstrap.sh +++ b/ci-automation/sdk_bootstrap.sh @@ -65,8 +65,8 @@ function sdk_bootstrap() { # -- function _sdk_bootstrap_impl() { - local seed_version="$1" - local version="$2" + local seed_version=${1} + local version=${2} : ${ARCH:="amd64"} source ci-automation/ci_automation_common.sh @@ -74,6 +74,9 @@ function _sdk_bootstrap_impl() { check_version_string "${version}" + if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then + set -x + fi # Create new tag in scripts repo w/ updated versionfile. # Also push the changes to the branch ONLY IF we're doing a nightly # build of the 'main' branch AND we're definitely ON the main branch. @@ -81,65 +84,96 @@ function _sdk_bootstrap_impl() { local target_branch='' # These variables are here to make it easier to test nightly # builds without messing with actual release branches. - local main_branch='main' - local nightly='nightly' - # Patterns used below. - local nightly_pattern_1='^main-[0-9.]+-'"${nightly}"'-[-0-9]+(-INTERMEDIATE)?$' - local nightly_pattern_2='^main-[0-9.]+-'"${nightly}"'-[-0-9]+$' - if [[ "${version}" =~ ${nightly_pattern_1} ]] \ - && [ "$(git rev-parse HEAD)" = "$(git rev-parse "origin/${main_branch}")" ] ; then + local main_branch=${CIA_DEBUGMAINBRANCH:-main} + local nightly=${CIA_DEBUGNIGHTLY:-nightly} + # Matches the usual nightly tag name, optionally with an + # intermediate suffix of the build ID too + # (main-1234.2.3-nightly-yyyymmdd-hhmm-INTERMEDIATE). + local nightly_pattern_1='^main-[0-9]+(\.[0-9]+){2}-'"${nightly}"'-[0-9]{8}-[0-9]{4}(-INTERMEDIATE)?$' + local main_branch_hash='' + if [[ ${version} =~ ${nightly_pattern_1} ]]; then + main_branch_hash=$(git rev-parse "origin/${main_branch}") + fi + local -a existing_tags=() + if [[ -n ${main_branch_hash} ]]; then + if [[ $(git rev-parse HEAD) != "${main_branch_hash}" ]] ; then + echo "We are doing a nightly build but we are not on top of the ${main_branch} branch. This is wrong and would result in the nightly tag not being a part of the branch." >&2 + exit 1 + fi target_branch=${main_branch} - local existing_tag="" - # Check for the existing tag only when we allow shortcutting - # the builds. That way we can skip the checks for build - # shortcutting. + # Check for the existing tag only when we allow + # shortcutting the builds. That way we can skip the checks + # for build shortcutting. if bool_is_true "${AVOID_NIGHTLY_BUILD_SHORTCUTS}"; then echo "Continuing the build because AVOID_NIGHTLY_BUILD_SHORTCUTS is bool true (${AVOID_NIGHTLY_BUILD_SHORTCUTS})" >&2 else - existing_tag=$(git tag --points-at HEAD) # exit code is always 0, output may be empty + git fetch --all --tags --force + # exit code is always 0, output may be empty + mapfile -t existing_tags < <(git tag --points-at HEAD) fi - # If the found tag is a nightly tag, we stop this build if there are no changes - if [[ "${existing_tag}" =~ ${nightly_pattern_2} ]]; then - local ret=0 - git diff --exit-code "${existing_tag}" || ret=$? - if [ "$ret" = "0" ]; then - local versions=( - $( - source sdk_lib/sdk_container_common.sh - source "${sdk_container_common_versionfile}" - echo "${FLATCAR_SDK_VERSION}" - echo "${FLATCAR_VERSION}" - ) + fi + local nightly_pattern_2='^main-[0-9]+(\.[0-9]+){2}-'"${nightly}"'-[0-9]{8}-[0-9]{4}$' + local tag nightly_tag='' + for tag in "${existing_tags[@]}"; do + if [[ ${tag} =~ ${nightly_pattern_2} ]]; then + nightly_tag=${tag} + break + fi + done + # If the found tag is a nightly tag, we stop this build if there + # are no changes and the relevant images can be found in the + # bincache. + if [[ -n ${nightly_tag} ]]; then + local -i ret=0 + git diff --exit-code --quiet "${nightly_tag}" || ret=$? + if [[ ret -eq 0 ]]; then + local -a versions=( + $( + source sdk_lib/sdk_container_common.sh + source "${sdk_container_common_versionfile}" + echo "${FLATCAR_SDK_VERSION}" + echo "${FLATCAR_VERSION}" + ) ) - local flatcar_sdk_version="${versions[0]}" - local flatcar_version="${versions[1]}" + local flatcar_sdk_version=${versions[0]} + local flatcar_version=${versions[1]} local sdk_docker_vernum="" sdk_docker_vernum=$(vernum_to_docker_image_version "${flatcar_sdk_version}") - if curl --head --fail --silent --show-error --location "https://${BUILDCACHE_SERVER}/containers/${sdk_docker_vernum}/flatcar-sdk-all-${sdk_docker_vernum}.tar.zst" \ - && curl --head --fail --silent --show-error --location "https://${BUILDCACHE_SERVER}/images/amd64/${flatcar_version}/flatcar_production_image.bin.bz2" \ - && curl --head --fail --silent --show-error --location "https://${BUILDCACHE_SERVER}/images/arm64/${flatcar_version}/flatcar_production_image.bin.bz2"; then - echo "Stopping build because there are no changes since tag ${existing_tag}, the SDK container tar ball and the Flatcar images exist" >&2 + if check_bincache_images_existence \ + "https://${BUILDCACHE_SERVER}/containers/${sdk_docker_vernum}/flatcar-sdk-all-${sdk_docker_vernum}.tar.zst" \ + "https://${BUILDCACHE_SERVER}/images/amd64/${flatcar_version}/flatcar_production_image.bin.bz2" \ + "https://${BUILDCACHE_SERVER}/images/arm64/${flatcar_version}/flatcar_production_image.bin.bz2"; then + echo "Stopping build because there are no changes since tag ${nightly_tag}, the SDK container tar ball and the Flatcar images exist" >&2 return 0 fi echo "No changes but continuing build because SDK container tar ball and/or the Flatcar images do not exist" >&2 - elif [ "$ret" = "1" ]; then - echo "Found changes since last tag ${existing_tag}" >&2 - else + elif [[ ret -eq 1 ]]; then + echo "HEAD is tagged with a nightly tag and yet there a differences? This is fishy and needs to be investigated. Maybe you forgot to commit your changes?" >&2 + exit 1 + else echo "Error: Unexpected git diff return code (${ret})" >&2 return 1 - fi fi fi + if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then + set +x + fi - local vernum="${version#*-}" # remove alpha-,beta-,stable-,lts- version tag - local git_vernum="${vernum}" + local vernum=${version#*-} # remove alpha-,beta-,stable-,lts- version tag + local git_vernum=${vernum} # Update FLATCAR_VERSION[_ID], BUILD_ID, and SDK in versionfile ( - source sdk_lib/sdk_container_common.sh - create_versionfile "${vernum}" + source sdk_lib/sdk_container_common.sh + create_versionfile "${vernum}" ) + if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then + set -x + fi update_and_push_version "${version}" "${target_branch}" + if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then + exit 0 + fi apply_local_patches ./bootstrap_sdk_container -x ./ci-cleanup.sh "${seed_version}" "${vernum}" @@ -154,14 +188,16 @@ function _sdk_bootstrap_impl() { # to ourselves, otherwise we could fail to sign the artifacts as # we lacked write permissions in the directory of the signed # artifact - local uid=$(id --user) - local gid=$(id --group) + local uid + local gid + uid=$(id --user) + gid=$(id --group) sudo chown --recursive "${uid}:${gid}" __build__ ( - cd "__build__/images/catalyst/builds/flatcar-sdk" - create_digests "${SIGNER}" "${dest_tarball}" - sign_artifacts "${SIGNER}" "${dest_tarball}"* - copy_to_buildcache "sdk/${ARCH}/${FLATCAR_SDK_VERSION}" "${dest_tarball}"* + cd "__build__/images/catalyst/builds/flatcar-sdk" + create_digests "${SIGNER}" "${dest_tarball}" + sign_artifacts "${SIGNER}" "${dest_tarball}"* + copy_to_buildcache "sdk/${ARCH}/${FLATCAR_SDK_VERSION}" "${dest_tarball}"* ) } # -- diff --git a/test-nightly-tagging.sh b/test-nightly-tagging.sh new file mode 100755 index 0000000000..52e4710a23 --- /dev/null +++ b/test-nightly-tagging.sh @@ -0,0 +1,635 @@ +#!/bin/bash + +set -euo pipefail + +declare -ga BUILD_TYPES=( + sdk + packages +) + +declare -ga TESTCASES=( + not_a_nightly # tag pushed + nightly_not_HEAD # fail + nightly_HEAD_no_tags # tag pushed, commit in branch + nightly_HEAD_no_nightly_tag # tag pushed, commit in branch + nightly_HEAD_nightly_tag_with_diff # fail + nightly_HEAD_nightly_tag_images_exist # nothing to do + nightly_HEAD_nightly_tag_images_not_exist # tag pushed, commit in branch +) + +declare -ga CHECKOUT_TYPES=( + sane # basically a branch checkout + fetch_head # git fetch origin ${branch}; git checkout FETCH_HEAD +) + +function fail() { + printf '%s\n' "${*}" >&2 + exit 1 +} + +declare -g LOGDIR=logs +declare -g START_AT_TC=${TESTCASES[0]} START_AT_BT=${BUILD_TYPES[0]} START_AT_CT=${CHECKOUT_TYPES[0]} + +RANDOM=${SRANDOM}${SRANDOM} + +while [[ ${#} -gt 0 ]]; do + case ${1} in + -l|--log-dir) + LOGDIR=${2} + shift 2 + ;; + -st|--start-at-test) + tc_ok='' + for tc in "${TESTCASES[@]}"; do + if [[ ${tc} = "${2}" ]]; then + START_AT_TC=${2} + tc_ok=x + break + fi + done + if [[ -z ${tc_ok} ]]; then + fail "wrong testcase name to start at, possible testcase names are ${TESTCASES[*]}" + fi + unset tc_ok tc + shift 2 + ;; + -sb|--start-at-build) + bt_ok='' + for bt in "${BUILD_TYPES[@]}"; do + if [[ ${bt} = "${2}" ]]; then + START_AT_BT=${2} + bt_ok=x + break + fi + done + if [[ -z ${bt_ok} ]]; then + fail "wrong build type to start at, possible build types are ${BUILD_TYPES[*]}" + fi + unset bt_ok bt + shift 2 + ;; + -sc|--start-at-checkout) + ct_ok='' + for ct in "${CHECKOUT_TYPES[@]}"; do + if [[ ${ct} = "${2}" ]]; then + START_AT_CT=${2} + ct_ok=x + break + fi + done + if [[ -z ${ct_ok} ]]; then + fail "wrong checkout type to start at, possible checkout types are ${CHECKOUT_TYPES[*]}" + fi + unset ct_ok ct + shift 2 + ;; + --) + shift + break + ;; + -*) + fail "unknown flag ${1@Q}" + ;; + esac +done + +declare -g DOT_GIT_PARENT_DIR= + +function find_git() { + local -n git_parent_dir_ref=${1}; shift + local dir=. + + while [[ $(realpath "${dir}") != '/' ]]; do + if [[ -e ${dir}/.git ]]; then + git_parent_dir_ref=${dir} + return + fi + dir+=/.. + done + fail "need to be running within a git repo" +} +find_git DOT_GIT_PARENT_DIR +unset -f find_git + +function log_test() { + printf ' %s\n' "${*}" >&3 +} + +function fail_test() { + log_test "FAIL: ${*}" + exit 1 +} + +function vc() { # verbose command + printf '%s\n' "${*}" + "${@}" +} + +function setup_cleanup_trap() { + declare -g cleanups_to_execute=':' + trap "${cleanups_to_execute}" EXIT +} + +function add_cleanup() { + cleanups_to_execute="${1} ; ${cleanups_to_execute}" + trap "set +e ; log_test 'doing cleanups'; if [[ \${DID_PUSHD} ]]; then popd; fi; ${cleanups_to_execute}" EXIT +} + +# avoid zeros - leading zeros are messing things up into octals and +# stuff, meh +declare -ga DIGITS=( {1..9} ) +declare -ga ALNUMS=( {1..9} {a..z} {A..Z} ) + +function random_from_array() { + local -i count=${1}; shift + local -n array_ref=${1}; shift + local -n str_ref=${1}; shift + + local -i i len=${#array_ref[@]} + local str='' + for ((i=0; i < count; ++i)); do + str+=${array_ref[$(( RANDOM % len ))]} + done + + str_ref=${str} +} + +function random_digits() { + random_from_array "${1}" DIGITS "${2}" +} + +function random_alnums() { + random_from_array "${1}" ALNUMS "${2}" +} + +declare -g DID_PUSHD='' + +function git_wt_setup_open() { + local build_type=${1}; shift + local tag_type=${1}; shift + local images_exist=${1}; shift + local -n branch_ref=${1}; shift + local -n test_script_ref=${1}; shift + local -n tag_ref=${1}; shift + + local work_dir random_work_dir_part + random_alnums 8 random_work_dir_part + local test_name=${FUNCNAME[1]} + work_dir=${DOT_GIT_PARENT_DIR}/../${test_name}_${build_type}_${random_work_dir_part} + + local random_branch_part random_version_part + random_alnums 8 random_branch_part + random_digits 4 random_version_part + + local branch channel version source_file + local -a function_call export_lines + case ${build_type} in + sdk) + branch=testmaintest${random_branch_part} + channel=main + version=${random_version_part}.0.0 + source_file='./ci-automation/sdk_bootstrap.sh' + # first argument is a seed version, not important at all + # for testing + function_call=( 'sdk_bootstrap' '1234.0.0' ) + export_lines=( "CIA_DEBUGMAINBRANCH=${branch}" ) + ;; + packages) + branch=testflatcartest${random_branch_part}-${random_version_part} + channel=stable + version=${random_version_part}.2.0 + source_file='./ci-automation/packages-tag.sh' + function_call=( 'packages_tag' ) + export_lines=( "CIA_DEBUGFLATCARBRANCHPREFIX=testflatcartest${random_branch_part}" ) + ;; + *) fail_test "wrong build type ${build_type@Q}, expected sdk or packages";; + esac + + local tag random_tag_part + case ${tag_type} in + nightly) + tag=${channel}-${version}-testnightlytest + random_digits 8 random_tag_part + tag+="-${random_tag_part}" + random_digits 4 random_tag_part + tag+="-${random_tag_part}" + ;; + devel) + random_alnums 8 random_tag_part + tag=${channel}-${version}-some-devel-stuff-${random_tag_part} + ;; + *) fail_test "wrong tag type ${tag_type@Q}, expected nightly or devel";; + esac + log_test "will use tag ${tag}" + + log_test "adding worktree at ${work_dir} with branch ${branch}" + git worktree add -b "${branch}" "${work_dir}" + add_cleanup "log_test 'removing local branch '${branch@Q}; git branch -D ${branch@Q}" + add_cleanup "log_test 'removing worktree at '${work_dir@Q}; git worktree remove --force ${work_dir@Q}" + + local test_script=$(mktemp --tmpdir ts-XXXXXXXX.sh) + log_test "creating test script at ${test_script}" + add_cleanup "log_test 'removing test script '${test_script@Q}; rm -f ${test_script@Q}" + printf '%s\n' \ + '#!/bin/bash' \ + 'set -euo pipefail' \ + "${export_lines[@]/#/export }" \ + 'export CIA_DEBUGNIGHTLY="testnightlytest"' \ + "export CIA_DEBUGIMAGESEXIST=${images_exist@Q}" \ + 'export CIA_DEBUGTESTRUN=x' \ + "source ${source_file@Q}" \ + "${function_call[*]@Q}"' "${@}"' >"${test_script}" + chmod a+x "${test_script}" + pushd "${work_dir}" + DID_PUSHD=x + + branch_ref=${branch} + test_script_ref=${test_script} + tag_ref=${tag} +} + +function git_wt_setup_close_and_run() { + local branch=${1}; shift + local test_script=${1}; shift + local tag=${1}; shift + local checkout_type=${1}; shift + local -n ret_ref=${1}; shift + local -n tag_hash_ref=${1}; shift + local -n branch_hash_ref=${1}; shift + + case ${checkout_type} in + sane) :;; + fetch_head) + git fetch origin "${branch}" + git checkout FETCH_HEAD + ;; + *) fail_test "wrong checkout type (${checkout_type@Q}), expected sane or fetch_head" + esac + + log_test "running the test script" + local -i ret=0 + "${test_script}" "${tag}" || ret=${?} + add_cleanup "if [[ -n \$(git tag -l ${tag@Q}) ]]; then log_test 'removing local tag '${tag@Q}; git tag -d ${tag@Q}; else log_test 'no local tag '${tag@Q}' to remove'; fi" + add_cleanup "if [[ -n \$(git ls-remote origin ${tag@Q}) ]]; then log_test 'removing remote tag '${tag@Q}; git push --delete origin ${tag@Q}; else log_test 'no remote tag '${tag@Q}' to remove'; fi" + popd + DID_PUSHD='' + + local hash ref tag_hash='' branch_hash='' + while read hash ref; do + case ${ref} in + "refs/tags/${tag}") + tag_hash=${hash} + ;; + "refs/heads/${branch}") + branch_hash=${hash} + ;; + esac + done < <(git ls-remote origin) + + ret_ref=${ret} + tag_hash_ref=${tag_hash} + branch_hash_ref=${branch_hash} +} + +function not_a_nightly() ( + local build_type=${1}; shift + local checkout_type=${1}; shift + + setup_cleanup_trap + + local tc_branch tc_script tc_tag + + git_wt_setup_open "${build_type}" 'devel' 'fail-if-reached' tc_branch tc_script tc_tag + + local tmpfile=$(mktemp testfile-XXXXXXXX) + log_test "creating a commit and pushing it to remote" + echo 'foo' >"${tmpfile}" + git add "${tmpfile}" + git commit -m 'stuff' + git push --set-upstream origin "${tc_branch}" + add_cleanup "log_test 'removing remote branch '${tc_branch@Q}; git push --delete origin ${tc_branch@Q}" + + local -i tc_ret=0 + local tc_tag_hash tc_branch_hash + git_wt_setup_close_and_run "${tc_branch}" "${tc_script}" "${tc_tag}" "${checkout_type}" tc_ret tc_tag_hash tc_branch_hash + + if [[ tc_ret -ne 0 ]]; then + fail_test "the script finished with exit status ${tc_ret}, expected 0" + fi + if [[ -z ${tc_tag_hash} ]]; then + fail_test "tag ${tc_tag@Q} not found on origin" + fi + if [[ -z ${tc_branch_hash} ]]; then + fail_test "branch ${tc_branch@Q} not found on origin" + fi + GIT_PAGER= vc git show "${tc_tag_hash}" + GIT_PAGER= vc git show "${tc_branch_hash}" + if [[ ${tc_tag_hash} = "${tc_branch_hash}" ]]; then + fail_test "tag pushed to the branch, while it should not be" + fi + log_test "OK" +) + +function nightly_not_HEAD() ( + local build_type=${1}; shift + local checkout_type=${1}; shift + + if [[ ${checkout_type} = 'fetch_head' ]]; then + log_test 'skipping, fetch_head checkout type sidesteps the issue' + return + fi + + setup_cleanup_trap + + local tc_branch tc_script tc_tag + + git_wt_setup_open "${build_type}" 'nightly' 'fail-if-reached' tc_branch tc_script tc_tag + + log_test "creating a commit, pushing it to remote and resetting to HEAD^ locally" + local tmpfile=$(mktemp testfile-XXXXXXXX) + echo 'foo' >"${tmpfile}" + git add "${tmpfile}" + git commit -m 'stuff' + git push --set-upstream origin "${tc_branch}" + add_cleanup "log_test 'removing remote branch '${tc_branch@Q}; git push --delete origin ${tc_branch@Q}" + git reset --hard HEAD^ + + local -i tc_ret=0 + local tc_tag_hash tc_branch_hash + git_wt_setup_close_and_run "${tc_branch}" "${tc_script}" "${tc_tag}" "${checkout_type}" tc_ret tc_tag_hash tc_branch_hash + + if [[ tc_ret -eq 0 ]]; then + fail_test 'the script finished with exit status 0, expected non-zero' + fi + if [[ -n ${tc_tag_hash} ]]; then + fail_test "tag ${tc_tag@Q} found on origin, but should not be" + fi + if [[ -z ${tc_branch_hash} ]]; then + fail_test "branch ${tc_branch@Q} not found on origin" + fi + log_test "OK" +) + +function nightly_HEAD_no_tags() ( + local build_type=${1}; shift + local checkout_type=${1}; shift + + setup_cleanup_trap + + local tc_branch tc_script tc_tag + + git_wt_setup_open "${build_type}" 'nightly' 'fail-if-reached' tc_branch tc_script tc_tag + + log_test "creating a commit and pushing it to remote" + local tmpfile=$(mktemp testfile-XXXXXXXX) + echo 'foo' >"${tmpfile}" + git add "${tmpfile}" + git commit -m 'stuff' + git push --set-upstream origin "${tc_branch}" + add_cleanup "log_test 'removing remote branch '${tc_branch@Q}; git push --delete origin ${tc_branch@Q}" + + local -i tc_ret=0 + local tc_tag_hash tc_branch_hash + git_wt_setup_close_and_run "${tc_branch}" "${tc_script}" "${tc_tag}" "${checkout_type}" tc_ret tc_tag_hash tc_branch_hash + + if [[ tc_ret -ne 0 ]]; then + fail_test "the script finished with exit status ${tc_ret}, expected 0" + fi + if [[ -z ${tc_tag_hash} ]]; then + fail_test "tag ${tc_tag@Q} not found on origin" + fi + if [[ -z ${tc_branch_hash} ]]; then + fail_test "branch ${tc_branch@Q} not found on origin" + fi + GIT_PAGER= vc git show "${tc_tag_hash}" + GIT_PAGER= vc git show "${tc_branch_hash}" + if [[ ${tc_tag_hash} != "${tc_branch_hash}" ]]; then + fail_test "tag not pushed to the branch, while it should be" + fi + log_test "OK" +) + +function nightly_HEAD_no_nightly_tag() ( + local build_type=${1}; shift + local checkout_type=${1}; shift + + setup_cleanup_trap + + local tc_branch tc_script tc_tag + + git_wt_setup_open "${build_type}" 'nightly' 'fail-if-reached' tc_branch tc_script tc_tag + + log_test "creating a commit, tagging it with a non-nightly tag and pushing them to remote" + local tmpfile=$(mktemp testfile-XXXXXXXX) + local channel=${tc_tag##-*} rest=${tc_tag%*-} + local version=${rest##-*} + local custom_tag=${channel}-${version}-some-custom-stuff + echo 'foo' >"${tmpfile}" + git add "${tmpfile}" + git commit -m 'stuff' + git tag "${custom_tag}" + add_cleanup "log_test 'removing local tag '${custom_tag@Q}; git tag -d ${custom_tag@Q}" + git push --set-upstream origin "${tc_branch}" + add_cleanup "log_test 'removing remote branch '${tc_branch@Q}; git push --delete origin ${tc_branch@Q}" + git push origin "${custom_tag}" + add_cleanup "log_test 'removing remote tag '${custom_tag@Q}; git push --delete origin ${custom_tag@Q}" + + local -i tc_ret=0 + local tc_tag_hash tc_branch_hash + git_wt_setup_close_and_run "${tc_branch}" "${tc_script}" "${tc_tag}" "${checkout_type}" tc_ret tc_tag_hash tc_branch_hash + + if [[ tc_ret -ne 0 ]]; then + fail_test "the script finished with exit status ${tc_ret}, expected 0" + fi + if [[ -z ${tc_tag_hash} ]]; then + fail_test "tag ${tc_tag@Q} not found on origin" + fi + if [[ -z ${tc_branch_hash} ]]; then + fail_test "branch ${tc_branch@Q} not found on origin" + fi + GIT_PAGER= vc git show "${tc_tag_hash}" + GIT_PAGER= vc git show "${tc_branch_hash}" + if [[ ${tc_tag_hash} != "${tc_branch_hash}" ]]; then + fail_test "tag not pushed to the branch, while it should be" + fi + log_test "OK" +) + +function another_nightly_tag() { + local tag=${1}; shift + local -n tag_ref=${1}; shift + + local new_tag_prefix=${tag} + new_tag_prefix=${new_tag_prefix%-*} # cut off -hhmm + new_tag_prefix=${new_tag_prefix%-*} # cut off -yyyymmdd + + local -i random_part + local new_tag + while :; do + new_tag=${new_tag_prefix} + random_digits 8 random_part + new_tag+=-${random_part} + random_digits 4 random_part + new_tag+=-${random_part} + if [[ ${new_tag} != ${tag} ]]; then + break + fi + done + tag_ref=${new_tag} +} + +function nightly_HEAD_nightly_tag_with_diff() ( + local build_type=${1}; shift + local checkout_type=${1}; shift + + setup_cleanup_trap + + local tc_branch tc_script tc_tag + + git_wt_setup_open "${build_type}" 'nightly' 'fail-if-reached' tc_branch tc_script tc_tag + + log_test "creating a commit, tagging it with a nightly tag and pushing them to remote, making local uncommited changes" + local tmpfile=$(mktemp testfile-XXXXXXXX) + local prev_nightly_tag + another_nightly_tag "${tc_tag}" prev_nightly_tag + echo 'foo' >"${tmpfile}" + git add "${tmpfile}" + git commit -m 'stuff' + git tag "${prev_nightly_tag}" + add_cleanup "log_test 'removing local tag '${prev_nightly_tag@Q}; git tag -d ${prev_nightly_tag@Q}" + git push --set-upstream origin "${tc_branch}" + add_cleanup "log_test 'removing remote branch '${tc_branch@Q}; git push --delete origin ${tc_branch@Q}" + git push origin "${prev_nightly_tag}" + add_cleanup "log_test 'removing remote tag '${prev_nightly_tag@Q}; git push --delete origin ${prev_nightly_tag@Q}" + echo 'bar' >"${tmpfile}" + + local -i tc_ret=0 + local tc_tag_hash tc_branch_hash + git_wt_setup_close_and_run "${tc_branch}" "${tc_script}" "${tc_tag}" "${checkout_type}" tc_ret tc_tag_hash tc_branch_hash + + if [[ tc_ret -eq 0 ]]; then + fail_test 'the script finished with exit status 0, expected non-zero' + fi + if [[ -n ${tc_tag_hash} ]]; then + fail_test "tag ${tc_tag@Q} found on origin, but should not be" + fi + if [[ -z ${tc_branch_hash} ]]; then + fail_test "branch ${tc_branch@Q} not found on origin" + fi + log_test "OK" +) + +function nightly_HEAD_nightly_tag_images_exist() ( + local build_type=${1}; shift + local checkout_type=${1}; shift + + setup_cleanup_trap + + local tc_branch tc_script tc_tag + + git_wt_setup_open "${build_type}" 'nightly' 'yes' tc_branch tc_script tc_tag + + log_test "creating a commit, tagging it with a nightly tag and pushing them to remote, assuming the images for the tag exist" + local tmpfile=$(mktemp testfile-XXXXXXXX) + local prev_nightly_tag + another_nightly_tag "${tc_tag}" prev_nightly_tag + echo 'foo' >"${tmpfile}" + git add "${tmpfile}" + git commit -m 'stuff' + git tag "${prev_nightly_tag}" + add_cleanup "log_test 'removing local tag '${prev_nightly_tag@Q}; git tag -d ${prev_nightly_tag@Q}" + git push --set-upstream origin "${tc_branch}" + add_cleanup "log_test 'removing remote branch '${tc_branch@Q}; git push --delete origin ${tc_branch@Q}" + git push origin "${prev_nightly_tag}" + add_cleanup "log_test 'removing remote tag '${prev_nightly_tag@Q}; git push --delete origin ${prev_nightly_tag@Q}" + + local old_branch_hash + old_branch_hash=$(git rev-parse "${tc_branch}") + + local -i tc_ret=0 + local tc_tag_hash tc_branch_hash + git_wt_setup_close_and_run "${tc_branch}" "${tc_script}" "${tc_tag}" "${checkout_type}" tc_ret tc_tag_hash tc_branch_hash + + if [[ tc_ret -ne 0 ]]; then + fail_test "the script finished with exit status ${tc_ret}, expected 0" + fi + if [[ -n ${tc_tag_hash} ]]; then + fail_test "tag ${tc_tag@Q} found on origin, while it should not be" + fi + if [[ -z ${tc_branch_hash} ]]; then + fail_test "branch ${tc_branch@Q} not found on origin" + fi + if [[ ${old_branch_hash} != "${tc_branch_hash}" ]]; then + fail_test "the branch has changed, while it should not" + fi + log_test "OK" +) + +function nightly_HEAD_nightly_tag_images_not_exist() ( + local build_type=${1}; shift + local checkout_type=${1}; shift + + setup_cleanup_trap + + local tc_branch tc_script tc_tag + + git_wt_setup_open "${build_type}" 'nightly' 'no' tc_branch tc_script tc_tag + + log_test "creating a commit, tagging it with a nightly tag and pushing them to remote, assuming the images for the tag do not exist" + local tmpfile=$(mktemp testfile-XXXXXXXX) + local prev_nightly_tag + another_nightly_tag "${tc_tag}" prev_nightly_tag + echo 'foo' >"${tmpfile}" + git add "${tmpfile}" + git commit -m 'stuff' + git tag "${prev_nightly_tag}" + add_cleanup "log_test 'removing local tag '${prev_nightly_tag@Q}; git tag -d ${prev_nightly_tag@Q}" + git push --set-upstream origin "${tc_branch}" + add_cleanup "log_test 'removing remote branch '${tc_branch@Q}; git push --delete origin ${tc_branch@Q}" + git push origin "${prev_nightly_tag}" + add_cleanup "log_test 'removing remote tag '${prev_nightly_tag@Q}; git push --delete origin ${prev_nightly_tag@Q}" + + local -i tc_ret=0 + local tc_tag_hash tc_branch_hash + git_wt_setup_close_and_run "${tc_branch}" "${tc_script}" "${tc_tag}" "${checkout_type}" tc_ret tc_tag_hash tc_branch_hash + + if [[ tc_ret -ne 0 ]]; then + fail_test "the script finished with exit status ${tc_ret}, expected 0" + fi + if [[ -z ${tc_tag_hash} ]]; then + fail_test "tag ${tc_tag@Q} not found on origin, while it should be" + fi + if [[ -z ${tc_branch_hash} ]]; then + fail_test "branch ${tc_branch@Q} not found on origin" + fi + if [[ ${tc_tag_hash} != "${tc_branch_hash}" ]]; then + fail_test "tag not pushed to the branch, while it should be" + fi + log_test "OK" +) + +mkdir -p "${LOGDIR}" + +declare -g START_AT_TC_NAME=test_${START_AT_TC}__build_${START_AT_BT}__checkout_${START_AT_CT} + +for tc in "${TESTCASES[@]}"; do + for bt in "${BUILD_TYPES[@]}"; do + for ct in "${CHECKOUT_TYPES[@]}"; do + tc_name=test_${tc}__build_${bt}__checkout_${ct} + if [[ -n ${START_AT_TC_NAME} ]]; then + if [[ ${START_AT_TC_NAME} != "${tc_name}" ]]; then + continue + fi + START_AT_TC_NAME='' + fi + # redirect testcase's stdout and stderr to log file, + # testcase's fd 3 to our stdout + echo "testcase: ${tc} build type: ${bt} checkout type: ${ct}" + "${tc}" "${bt}" "${ct}" 3>&1 1>"${LOGDIR}/${tc_name}" 2>&1 + unset tc_name + done + done +done +unset tc bt ct + +echo 'ALL DONE'