Merge pull request #266 from flatcar-linux/kai/sdk-from-release-tag

ci-automation: use a single git tag and skip nightlies with no changes

ci-automation/README.md

@@ -54,13 +54,13 @@ image_build amd64
               |                      "alpha-3449.0.0-dev23"          |
               |                                |                     |
               |                         _______v_______              |
-              +------- clone -----> ( SDK bootstrap )             |
+              +-------- clone -------> ( SDK bootstrap )             |
               |                         `-------------´              |
-              |<- tag: sdk-3499.0.0-dev23 -´|`--- sdk tarball --->|
+              |<- tag: alpha-3499.0.0-dev23 --´|`--- sdk tarball --->|
               |                                |                     |
               |                         _______v_______              |
-              +--      clone     -> ( SDK container )             |
+              +-------- clone -------> ( SDK container )             |
-              | sdk-3499.0.0-dev23   `-------------´              |
+              | alpha-3499.0.0-dev23    `-------------´              |
               |                                |`- sdk container --->|
               v                                v        image
                       continue to OS

ci-automation/ci_automation_common.sh

@@ -66,6 +66,25 @@ function update_and_push_version() {
 
     git commit --allow-empty -m "New version: ${version}"
 
+    git fetch --all --tags --force
+    local ret=0
+    git diff --exit-code "${version}" || ret=$?
+    # This will return != 0 if
+    #  - the remote tag does not exist (rc: 127)
+    #  - the tag does not exist locally (rc: 128)
+    #  - the remote tag has changes compared to the local tree (rc: 1)
+    if [ "$ret" = "0" ]; then
+      echo "Reusing existing tag" >&2
+      git checkout -f --recurse-submodules "${version}"
+      return
+    elif [ "$ret" = "1" ]; then
+      echo "Remote tag exists already and is not equal" >&2
+      return 1
+    elif [ "$ret" != "127" ] && [ "$ret" != "128" ]; then
+      echo "Error: Unexpected git diff return code ($ret)" >&2
+      return 1
+    fi
+
     local -a TAG_ARGS
     if [ "${SIGN-0}" = 1 ]; then
       TAG_ARGS=("-s" "-m" "${version}")
@@ -78,24 +97,7 @@ function update_and_push_version() {
       git push origin "${branch}"
     fi
 
-    if git push origin "${version}" ; then
+    git push origin "${version}"
-        return
-    fi
-    # Push (above) may fail because a tag already exists.
-    #  We check for tag presence, and for the difference
-    #  between local and remote, and bail
-    #  only if the remote / local contents differ.
-
-    # Remove local tag, (re-)fetch remote tags
-    git tag -d "${version}"
-
-    # refresh tree, let origin overwrite local tags
-    git fetch --all --tags --force
-
-    # This will return != 0 if
-    #  - the remote tag does not exist (rc: 127)
-    #  - the remote tag has changes compared to the local tree (rc: 1)
-    git diff --exit-code "${version}"
 }
 # --
 

ci-automation/garbage_collect.sh

@@ -32,7 +32,7 @@ function garbage_collect() {
     local purge_versions="${PURGE_VERSIONS:-}"
 
     local versions_detected="$(git tag -l --sort=-committerdate \
-                | grep -E '(main|alpha|beta|stable|lts|sdk)-[0-9]+\.[0-9]+\.[0-9]+\-.*' \
+                | grep -E '(main|alpha|beta|stable|lts)-[0-9]+\.[0-9]+\.[0-9]+\-.*' \
                 | grep -vE '(-pro)$')"
 
     echo "######## Full list of version(s) found ########"
@@ -45,7 +45,7 @@ function garbage_collect() {
     else
         # make sure we only accept dev versions
         purge_versions="$(echo "${purge_versions}" | sed 's/ /\n/g' \
-                            | grep -E '(main|alpha|beta|stable|lts|sdk)-[0-9]+\.[0-9]+\.[0-9]+\-.*' \
+                            | grep -E '(main|alpha|beta|stable|lts)-[0-9]+\.[0-9]+\.[0-9]+\-.*' \
                             | grep -vE '(-pro)$')"
     fi
 
@@ -74,26 +74,20 @@ function garbage_collect() {
         git checkout "${version}" -- sdk_container/.repo/manifests/version.txt
         source sdk_container/.repo/manifests/version.txt
 
+        # Assuming that the SDK build version also has the same OS version
         local os_vernum="${FLATCAR_VERSION}"
         local os_docker_vernum="$(vernum_to_docker_image_version "${FLATCAR_VERSION}")"
-        local sdk_vernum="${FLATCAR_SDK_VERSION}"
-        local sdk_docker_vernum="$(vernum_to_docker_image_version "${FLATCAR_SDK_VERSION}")"
 
         # Remove container image tarballs and SDK tarball (if applicable)
         #
         local rmpat=""
-        if [[ "${version}" = 'sdk-'* ]] ; then
+        rmpat="${BUILDCACHE_PATH_PREFIX}/sdk/*/${os_vernum}/"
-            echo "## ${version} is an SDK version. ##"
+        rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-sdk-*"
-            rmpat="${BUILDCACHE_PATH_PREFIX}/sdk/*/${sdk_vernum}/"
+        rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-packages-*"
-            rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${sdk_docker_vernum}/flatcar-sdk-*"
-        else
-            echo "## ${version} is an OS image version. ##"
-            rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-packages-*"
         rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/boards/*/${os_vernum}/"
         rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-images-*"
         rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/images/*/${os_vernum}/"
         rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/testing/${os_vernum}/"
-        fi
 
         echo "## The following files will be removed ##"
         $sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \
@@ -110,11 +104,7 @@ function garbage_collect() {
 
         # Remove container image directory if empty
         #
-        if [[ "${version}" = 'sdk-'* ]] ; then
-            rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${sdk_docker_vernum}/"
-        else
         rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/"
-        fi
 
         echo "## Checking if container directory is empty and can be removed (it's OK if this fails) ##"
         echo "## The following directory will be removed if below output is empty: '${rmpat}' ##"

ci-automation/packages.sh

@@ -80,6 +80,34 @@ function packages_build() {
         update_submodule "portage-stable" "${portage_git}"
     fi
 
+    # Create new tag in scripts repo w/ updated versionfile + submodules.
+    # Also push the changes to the branch ONLY IF we're doing a nightly
+    #   build of the 'main'/'flatcar-MAJOR' branch AND we're definitely ON the respective branch
+    #   (`scripts` and submodules).
+    local push_branch="false"
+    if   [[ "${version}" =~ ^(stable|alpha|beta|lts)-[0-9.]+-nightly-[-0-9]+$ ]] \
+       && [[ "$(git rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] \
+       && [[ "$(git -C sdk_container/src/third_party/coreos-overlay/ rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] \
+       && [[ "$(git -C sdk_container/src/third_party/portage-stable/ rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] ; then
+        push_branch="true"
+        local existing_tag=""
+        existing_tag=$(git tag --points-at HEAD) # exit code is always 0, output may be empty
+        # If the found tag is a release or nightly tag, we stop this build if there are no changes
+        if [[ "${existing_tag}" =~ ^(stable|alpha|beta|lts)-[0-9.]+(|-nightly-[-0-9]+)$ ]]; then
+          local ret=0
+          git diff --exit-code "${existing_tag}" || ret=$?
+          if [ "$ret" = "0" ]; then
+            echo "Stopping build because there are no changes since tag ${existing_tag}" >&2
+            return 0
+          elif [ "$ret" = "1" ]; then
+            echo "Found changes since last tag ${existing_tag}" >&2
+          else
+            echo "Error: Unexpected git diff return code (${ret})" >&2
+            return 1
+          fi
+        fi
+    fi
+
     # Get SDK from either the registry or import from build cache
     # This is a NOP if the image is present locally.
     local sdk_name="flatcar-sdk-${arch}"
@@ -127,6 +155,6 @@ function packages_build() {
     copy_to_buildcache "images/${arch}/${vernum}/torcx" \
         "${torcx_tmp}/torcx/pkgs/${arch}-usr/docker/"*/*.torcx.tgz
 
-    update_and_push_version "${version}"
+    update_and_push_version "${version}" "${push_branch}"
 }
 # --

ci-automation/sdk_bootstrap.sh

@@ -22,7 +22,7 @@
 #   2. Version of the TARGET SDK to build (string).
 #       The version pattern 'MMMM.m.p' (e.g. '3051.0.0') denotes a "official" build, i.e. a release build to be published.
 #       Use any version diverging from the pattern (e.g. '3051.0.0-nightly-4302') for development / CI builds.
-#       A tag "sdk-[VERSION]" will be created in the scripts repo and pushed upstream.
+#       A free-standing tagged commit will be created in the scripts repo and pushed upstream.
 #
 # OPTIONAL INPUT:
 #
@@ -69,6 +69,34 @@ function sdk_bootstrap() {
         update_submodule "portage-stable" "${portage_git}"
     fi
 
+    # Create new tag in scripts repo w/ updated versionfile + submodules.
+    # Also push the changes to the branch ONLY IF we're doing a nightly
+    #   build of the 'main' branch AND we're definitely ON the main branch
+    #   (`scripts` and submodules).
+    local push_branch="false"
+    if   [[ "${version}" =~ ^main-[0-9.]+-nightly-[-0-9]+$ ]] \
+       && [ "$(git rev-parse --abbrev-ref HEAD)" = "main"  ] \
+       && [ "$(git -C sdk_container/src/third_party/coreos-overlay/ rev-parse --abbrev-ref HEAD)" = "main"  ] \
+       && [ "$(git -C sdk_container/src/third_party/portage-stable/ rev-parse --abbrev-ref HEAD)" = "main"  ] ; then
+        push_branch="true"
+        local existing_tag=""
+        existing_tag=$(git tag --points-at HEAD) # exit code is always 0, output may be empty
+        # If the found tag is a nightly tag, we stop this build if there are no changes
+        if [[ "${existing_tag}" =~ ^main-[0-9.]+-nightly-[-0-9]+$ ]]; then
+          local ret=0
+          git diff --exit-code "${existing_tag}" || ret=$?
+          if [ "$ret" = "0" ]; then
+            echo "Stopping build because there are no changes since tag ${existing_tag}" >&2
+            return 0
+          elif [ "$ret" = "1" ]; then
+            echo "Found changes since last tag ${existing_tag}" >&2
+          else
+            echo "Error: Unexpected git diff return code (${ret})" >&2
+            return 1
+          fi
+        fi
+    fi
+
     local vernum="${version#*-}" # remove alpha-,beta-,stable-,lts- version tag
     local git_vernum="${vernum}"
 
@@ -84,17 +112,6 @@ function sdk_bootstrap() {
     copy_to_buildcache "sdk/${ARCH}/${vernum}" "${dest_tarball}"*
     cd -
 
-    # Create new tag in scripts repo w/ updated versionfile + submodules.
+    update_and_push_version "${version}" "${push_branch}"
-    # Also push the changes to the branch ONLY IF we're doing a nightly
-    #   build of the 'main' branch AND we're definitely ON the main branch
-    #   (`scripts` and submodules).
-    local push_branch="false"
-    if   [[ "${version}" =~ ^main-[0-9.]+-nightly-[-0-9]+$ ]] \
-       && [ "$(git rev-parse --abbrev-ref HEAD)" = "main"  ] \
-       && [ "$(git -C sdk_container/src/third_party/coreos-overlay/ rev-parse --abbrev-ref HEAD)" = "main"  ] \
-       && [ "$(git -C sdk_container/src/third_party/portage-stable/ rev-parse --abbrev-ref HEAD)" = "main"  ] ; then
-        push_branch="true"
-    fi
-    update_and_push_version "sdk-${git_vernum}" "${push_branch}"
 }
 # --