Merge pull request #386 from flatcar-linux/kai/new-packages-top-job

ci-automation: Move git tagging into own script

ci-automation/README.md

@@ -71,13 +71,14 @@ image_build amd64
 
 ### OS image build
 
-3. Packages build (`packages.sh`): using the SDK container version recorded in the versionfile, build OS image packages and generate a new container image (containing both SDK and packages).
+3. Packages tag (`packages-tag.sh`): Creates git tag if needed with the SDK container version recorded in the versionfile and the OS version from the tag.
-   This step updates the versionfile, recording the Flatcar OS image version just built.
+   Creates `skip-build` flag file if no changes since last nightly tag are seen.
-   It will generate and push a new version tag to the scripts repo.
+   Pushes the new version tag to the scripts repo as free-standing tag, and for nightlies also to the branch.
-4. Packages are published and the generic OS image is built.
+4. Packages build (`packages.sh`): Build OS image packages and generate a new container image (containing both SDK and packages).
+5. Packages are published and the generic OS image is built.
    1. Binary packages are published (`push_pkgs.sh`) to the build cache, making them available to developers who base their work on the main branch.
    2. Image build (`image.sh`): Using the container from 3., build an OS image and torcx store, and generate a new container image with everything in it.
-5. VMs build (`vms.sh`). Using the packages+torcx+image container from 4., build vendor images. Results are vendor-specific OS images.
+6. VMs build (`vms.sh`). Using the packages+torcx+image container from 4., build vendor images. Results are vendor-specific OS images.
 
 ```
        .---------.                     .------------.             .--------.
@@ -87,10 +88,14 @@ image_build amd64
             |                                 |                        |
             |                       "alpha-3449.0.0-dev23"             |
             |                                 |                        |
+            |                           ______v_______                 |
+            +---------- clone ------>  ( packages-tag )                |
+            |                           `------------´                 |
+            |<-- tag: alpha-3499.0.0-dev23 --´|                        |
             |                             ____v_____                   |
-            +---------- clone -------->  ( packages )                  |
+            +-----       clone      ---> ( packages )                  |
-            |                             `--------´                   |
+            |    alpha-3499.0.0-dev23     `--------´                   |
-            |<-- tag: alpha-3499.0.0-dev23 --´|`- sdk + OS packages -->|
+            |                                 |`- sdk + OS packages -->|
             |                                 |    container image     |
             |                                 |    torcx manifest      |
             |                           ______v_______                 |

ci-automation/packages-tag.sh

@@ -0,0 +1,120 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
+#
+# packages_tag() should be called w/ the positional INPUT parameters below.
+
+# build tag automation stub.
+#   This script will update the versionfile with the OS packages version to build,
+#    and will add a version tag (see INPUT) to the scripts repo.
+#
+# PREREQUISITES:
+#
+#   1. SDK version is recorded in sdk_container/.repo/manifests/version.txt
+#   2. SDK container is either
+#       - available via ghcr.io/flatcar-linux/flatcar-sdk-[ARCH]:[VERSION] (official SDK release)
+#       OR
+#       - available via build cache server "/containers/[VERSION]/flatcar-sdk-[ARCH]-[VERSION].tar.gz"
+#         (dev SDK)
+#
+# INPUT:
+#
+#   1. Version of the TARGET OS image to build (string).
+#       The version pattern '(alpha|beta|stable|lts)-MMMM.m.p' (e.g. 'alpha-3051.0.0')
+#         denotes a "official" build, i.e. a release build to be published.
+#       Use any version diverging from the pattern (e.g. 'alpha-3051.0.0-nightly-4302') for development / CI builds.
+#       A tag of this version will be created in the scripts repo and pushed upstream.
+#
+#
+# OPTIONAL INPUT:
+#
+#   2. coreos-overlay repository tag to use (commit-ish).
+#       Optional - use scripts repo sub-modules as-is if not set.
+#       This version will be checked out / pulled from remote in the coreos-overlay git submodule.
+#       The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed.
+#
+#   3. portage-stable repository tag to use (commit-ish).
+#       Optional - use scripts repo sub-modules as-is if not set.
+#       This version will be checked out / pulled from remote in the portage-stable git submodule.
+#       The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed.
+#
+# OUTPUT:
+#
+#   1. Updated scripts repository
+#        - version tag w/ submodules
+#        - sdk_container/.repo/manifests/version.txt denotes new FLATCAR OS version
+#   2. "./skip-build" as flag file to signal that the build should stop
+
+function packages_tag() {
+    # Run a subshell, so the traps, environment changes and global
+    # variables are not spilled into the caller.
+    (
+        set -euo pipefail
+
+        _packages_tag_impl "${@}"
+    )
+}
+# --
+
+function _packages_tag_impl() {
+    local version="$1"
+    local coreos_git="${2:-}"
+    local portage_git="${3:-}"
+
+    source ci-automation/ci_automation_common.sh
+    source ci-automation/gpg_setup.sh
+    init_submodules
+
+    check_version_string "${version}"
+
+    source sdk_container/.repo/manifests/version.txt
+    local sdk_version="${FLATCAR_SDK_VERSION}"
+
+    if [ -n "${coreos_git}" ] ; then
+        update_submodule "coreos-overlay" "${coreos_git}"
+    fi
+    if [ -n "${portage_git}" ] ; then
+        update_submodule "portage-stable" "${portage_git}"
+    fi
+
+    # Create new tag in scripts repo w/ updated versionfile + submodules.
+    # Also push the changes to the branch ONLY IF we're doing a nightly
+    #   build of the 'main'/'flatcar-MAJOR' branch AND we're definitely ON the respective branch
+    #   (`scripts` and submodules).
+    local push_branch="false"
+    if    [[ "${version}" =~ ^(stable|alpha|beta|lts)-[0-9.]+-nightly-[-0-9]+$ ]] \
+       && [[ "$(git rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] \
+       && [[ "$(git -C sdk_container/src/third_party/coreos-overlay/ rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] \
+       && [[ "$(git -C sdk_container/src/third_party/portage-stable/ rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] ; then
+        push_branch="true"
+        local existing_tag=""
+        existing_tag=$(git tag --points-at HEAD) # exit code is always 0, output may be empty
+        # If the found tag is a release or nightly tag, we stop this build if there are no changes
+        if [[ "${existing_tag}" =~ ^(stable|alpha|beta|lts)-[0-9.]+(|-nightly-[-0-9]+)$ ]]; then
+          local ret=0
+          git diff --exit-code "${existing_tag}" || ret=$?
+          if [[ ret -eq 0 ]]; then
+            touch ./skip-build
+            echo "Creating ./skip-build flag file, indicating that the build must not to continue because no new tag got created as there are no changes since tag ${existing_tag}" >&2
+            return 0
+          elif [[ ret -eq 1 ]]; then
+            echo "Found changes since last tag ${existing_tag}" >&2
+          else
+            echo "Error: Unexpected git diff return code (${ret})" >&2
+            return 1
+          fi
+        fi
+    fi
+
+    # Create version file
+    (
+      source sdk_lib/sdk_container_common.sh
+      create_versionfile "$sdk_version" "$version"
+    )
+    update_and_push_version "${version}" "${push_branch}"
+}
+# --

ci-automation/packages.sh

@@ -10,13 +10,12 @@
 
 # OS image binary packages build automation stub.
 #   This script will use an SDK container to build packages for an OS image.
-#   It will update the versionfile with the OS packages version built,
-#    and will add a version tag (see INPUT) to the scripts repo. 
 #
 # PREREQUISITES:
 #
-#   1. SDK version is recorded in sdk_container/.repo/manifests/version.txt
+#   1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt
-#   2. SDK container is either
+#   2. Scripts repo version tag of OS image version to be built is available and checked out.
+#   3. SDK container is either
 #       - available via ghcr.io/flatcar-linux/flatcar-sdk-[ARCH]:[VERSION] (official SDK release)
 #       OR
 #       - available via build cache server "/containers/[VERSION]/flatcar-sdk-[ARCH]-[VERSION].tar.gz"
@@ -24,32 +23,16 @@
 #
 # INPUT:
 #
-#   1. Version of the TARGET OS image to build (string).
+#   1. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64").
-#       The version pattern '(alpha|beta|stable|lts)-MMMM.m.p' (e.g. 'alpha-3051.0.0')
-#         denotes a "official" build, i.e. a release build to be published.
-#       Use any version diverging from the pattern (e.g. 'alpha-3051.0.0-nightly-4302') for development / CI builds.
-#       A tag of this version will be created in the scripts repo and pushed upstream.
-#
-#   2. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64").
 #
 #
 # OPTIONAL INPUT:
 #
-#   3. coreos-overlay repository tag to use (commit-ish).
+#   2. SIGNER. Environment variable. Name of the owner of the artifact signing key.
-#       Optional - use scripts repo sub-modules as-is if not set.
-#       This version will be checked out / pulled from remote in the coreos-overlay git submodule.
-#       The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed.
-#
-#   4. portage-stable repository tag to use (commit-ish).
-#       Optional - use scripts repo sub-modules as-is if not set.
-#       This version will be checked out / pulled from remote in the portage-stable git submodule.
-#       The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed.
-#
-#   5. SIGNER. Environment variable. Name of the owner of the artifact signing key.
 #        Defaults to nothing if not set - in such case, artifacts will not be signed.
 #        If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored.
 #
-#   6. SIGNING_KEY. Environment variable. The artifact signing key.
+#   3. SIGNING_KEY. Environment variable. The artifact signing key.
 #        Defaults to nothing if not set - in such case, artifacts will not be signed.
 #        If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored.
 #
@@ -58,13 +41,10 @@
 #   1. Exported container image "flatcar-packages-[ARCH]-[VERSION].tar.gz" with binary packages
 #       pushed to buildcache, and torcx_manifest.json pushed to "images/${arch}/${vernum}/"
 #       (for use with tests).
-#   2. Updated scripts repository
+#   2. "./ci-cleanup.sh" with commands to clean up temporary build resources,
-#        - version tag w/ submodules
-#        - sdk_container/.repo/manifests/version.txt denotes new FLATCAR OS version
-#   3. "./ci-cleanup.sh" with commands to clean up temporary build resources,
 #        to be run after this step finishes / when this step is aborted.
-#   4. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache.
+#   3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache.
-#   5. DIGESTS of the artifacts from point 1, pushed to buildcache. If signer key was passed, armored ASCII files of the generated DIGESTS files too, pushed to buildcache.
+#   4. DIGESTS of the artifacts from point 1, pushed to buildcache. If signer key was passed, armored ASCII files of the generated DIGESTS files too, pushed to buildcache.
 
 function packages_build() {
     # Run a subshell, so the traps, environment changes and global
@@ -78,55 +58,15 @@ function packages_build() {
 # --
 
 function _packages_build_impl() {
-    local version="$1"
+    local arch="$1"
-    local arch="$2"
-    local coreos_git="${3:-}"
-    local portage_git="${4:-}"
 
     source ci-automation/ci_automation_common.sh
     source ci-automation/gpg_setup.sh
     init_submodules
 
-    check_version_string "${version}"
-
     source sdk_container/.repo/manifests/version.txt
     local sdk_version="${FLATCAR_SDK_VERSION}"
 
-    if [ -n "${coreos_git}" ] ; then
-        update_submodule "coreos-overlay" "${coreos_git}"
-    fi
-    if [ -n "${portage_git}" ] ; then
-        update_submodule "portage-stable" "${portage_git}"
-    fi
-
-    # Create new tag in scripts repo w/ updated versionfile + submodules.
-    # Also push the changes to the branch ONLY IF we're doing a nightly
-    #   build of the 'main'/'flatcar-MAJOR' branch AND we're definitely ON the respective branch
-    #   (`scripts` and submodules).
-    local push_branch="false"
-    if   [[ "${version}" =~ ^(stable|alpha|beta|lts)-[0-9.]+-nightly-[-0-9]+$ ]] \
-       && [[ "$(git rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] \
-       && [[ "$(git -C sdk_container/src/third_party/coreos-overlay/ rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] \
-       && [[ "$(git -C sdk_container/src/third_party/portage-stable/ rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] ; then
-        push_branch="true"
-        local existing_tag=""
-        existing_tag=$(git tag --points-at HEAD) # exit code is always 0, output may be empty
-        # If the found tag is a release or nightly tag, we stop this build if there are no changes
-        if [[ "${existing_tag}" =~ ^(stable|alpha|beta|lts)-[0-9.]+(|-nightly-[-0-9]+)$ ]]; then
-          local ret=0
-          git diff --exit-code "${existing_tag}" || ret=$?
-          if [ "$ret" = "0" ]; then
-            echo "Stopping build because there are no changes since tag ${existing_tag}" >&2
-            return 0
-          elif [ "$ret" = "1" ]; then
-            echo "Found changes since last tag ${existing_tag}" >&2
-          else
-            echo "Error: Unexpected git diff return code (${ret})" >&2
-            return 1
-          fi
-        fi
-    fi
-
     # Get SDK from either the registry or import from build cache
     # This is a NOP if the image is present locally.
     local sdk_name="flatcar-sdk-${arch}"
@@ -137,17 +77,10 @@ function _packages_build_impl() {
     echo "docker image rm -f '${sdk_image}'" >> ./ci-cleanup.sh
 
     # Set name of the packages container for later rename / export
-    local vernum="${version#*-}" # remove main-,alpha-,beta-,stable-,lts- version tag
+    local vernum="${FLATCAR_VERSION}"
     local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
     local packages_container="flatcar-packages-${arch}-${docker_vernum}"
 
-    # Create version file
-    (
-      source sdk_lib/sdk_container_common.sh
-      create_versionfile "$sdk_version" "$version"
-    )
-    update_and_push_version "${version}" "${push_branch}"
-
     # Build packages; store packages and torcx output in container
     ./run_sdk_container -x ./ci-cleanup.sh -n "${packages_container}" -v "${version}" \
         -C "${sdk_image}" \