diff --git a/ci-automation/README.md b/ci-automation/README.md index 14aa764ed7..7877ebe4e2 100644 --- a/ci-automation/README.md +++ b/ci-automation/README.md @@ -71,13 +71,14 @@ image_build amd64 ### OS image build -3. Packages build (`packages.sh`): using the SDK container version recorded in the versionfile, build OS image packages and generate a new container image (containing both SDK and packages). - This step updates the versionfile, recording the Flatcar OS image version just built. - It will generate and push a new version tag to the scripts repo. -4. Packages are published and the generic OS image is built. +3. Packages tag (`packages-tag.sh`): Creates git tag if needed with the SDK container version recorded in the versionfile and the OS version from the tag. + Creates `skip-build` flag file if no changes since last nightly tag are seen. + Pushes the new version tag to the scripts repo as free-standing tag, and for nightlies also to the branch. +4. Packages build (`packages.sh`): Build OS image packages and generate a new container image (containing both SDK and packages). +5. Packages are published and the generic OS image is built. 1. Binary packages are published (`push_pkgs.sh`) to the build cache, making them available to developers who base their work on the main branch. 2. Image build (`image.sh`): Using the container from 3., build an OS image and torcx store, and generate a new container image with everything in it. -5. VMs build (`vms.sh`). Using the packages+torcx+image container from 4., build vendor images. Results are vendor-specific OS images. +6. VMs build (`vms.sh`). Using the packages+torcx+image container from 4., build vendor images. Results are vendor-specific OS images. ``` .---------. .------------. .--------. @@ -87,10 +88,14 @@ image_build amd64 | | | | "alpha-3449.0.0-dev23" | | | | + | ______v_______ | + +---------- clone ------> ( packages-tag ) | + | `------------´ | + |<-- tag: alpha-3499.0.0-dev23 --´| | | ____v_____ | - +---------- clone --------> ( packages ) | - | `--------´ | - |<-- tag: alpha-3499.0.0-dev23 --´|`- sdk + OS packages -->| + +----- clone ---> ( packages ) | + | alpha-3499.0.0-dev23 `--------´ | + | |`- sdk + OS packages -->| | | container image | | | torcx manifest | | ______v_______ | diff --git a/ci-automation/packages-tag.sh b/ci-automation/packages-tag.sh new file mode 100644 index 0000000000..f874f8846f --- /dev/null +++ b/ci-automation/packages-tag.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# +# Copyright (c) 2021 The Flatcar Maintainers. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# >>> This file is supposed to be SOURCED from the repository ROOT. <<< +# +# packages_tag() should be called w/ the positional INPUT parameters below. + +# build tag automation stub. +# This script will update the versionfile with the OS packages version to build, +# and will add a version tag (see INPUT) to the scripts repo. +# +# PREREQUISITES: +# +# 1. SDK version is recorded in sdk_container/.repo/manifests/version.txt +# 2. SDK container is either +# - available via ghcr.io/flatcar-linux/flatcar-sdk-[ARCH]:[VERSION] (official SDK release) +# OR +# - available via build cache server "/containers/[VERSION]/flatcar-sdk-[ARCH]-[VERSION].tar.gz" +# (dev SDK) +# +# INPUT: +# +# 1. Version of the TARGET OS image to build (string). +# The version pattern '(alpha|beta|stable|lts)-MMMM.m.p' (e.g. 'alpha-3051.0.0') +# denotes a "official" build, i.e. a release build to be published. +# Use any version diverging from the pattern (e.g. 'alpha-3051.0.0-nightly-4302') for development / CI builds. +# A tag of this version will be created in the scripts repo and pushed upstream. +# +# +# OPTIONAL INPUT: +# +# 2. coreos-overlay repository tag to use (commit-ish). +# Optional - use scripts repo sub-modules as-is if not set. +# This version will be checked out / pulled from remote in the coreos-overlay git submodule. +# The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed. +# +# 3. portage-stable repository tag to use (commit-ish). +# Optional - use scripts repo sub-modules as-is if not set. +# This version will be checked out / pulled from remote in the portage-stable git submodule. +# The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed. +# +# OUTPUT: +# +# 1. Updated scripts repository +# - version tag w/ submodules +# - sdk_container/.repo/manifests/version.txt denotes new FLATCAR OS version +# 2. "./skip-build" as flag file to signal that the build should stop + +function packages_tag() { + # Run a subshell, so the traps, environment changes and global + # variables are not spilled into the caller. + ( + set -euo pipefail + + _packages_tag_impl "${@}" + ) +} +# -- + +function _packages_tag_impl() { + local version="$1" + local coreos_git="${2:-}" + local portage_git="${3:-}" + + source ci-automation/ci_automation_common.sh + source ci-automation/gpg_setup.sh + init_submodules + + check_version_string "${version}" + + source sdk_container/.repo/manifests/version.txt + local sdk_version="${FLATCAR_SDK_VERSION}" + + if [ -n "${coreos_git}" ] ; then + update_submodule "coreos-overlay" "${coreos_git}" + fi + if [ -n "${portage_git}" ] ; then + update_submodule "portage-stable" "${portage_git}" + fi + + # Create new tag in scripts repo w/ updated versionfile + submodules. + # Also push the changes to the branch ONLY IF we're doing a nightly + # build of the 'main'/'flatcar-MAJOR' branch AND we're definitely ON the respective branch + # (`scripts` and submodules). + local push_branch="false" + if [[ "${version}" =~ ^(stable|alpha|beta|lts)-[0-9.]+-nightly-[-0-9]+$ ]] \ + && [[ "$(git rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] \ + && [[ "$(git -C sdk_container/src/third_party/coreos-overlay/ rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] \ + && [[ "$(git -C sdk_container/src/third_party/portage-stable/ rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] ; then + push_branch="true" + local existing_tag="" + existing_tag=$(git tag --points-at HEAD) # exit code is always 0, output may be empty + # If the found tag is a release or nightly tag, we stop this build if there are no changes + if [[ "${existing_tag}" =~ ^(stable|alpha|beta|lts)-[0-9.]+(|-nightly-[-0-9]+)$ ]]; then + local ret=0 + git diff --exit-code "${existing_tag}" || ret=$? + if [[ ret -eq 0 ]]; then + touch ./skip-build + echo "Creating ./skip-build flag file, indicating that the build must not to continue because no new tag got created as there are no changes since tag ${existing_tag}" >&2 + return 0 + elif [[ ret -eq 1 ]]; then + echo "Found changes since last tag ${existing_tag}" >&2 + else + echo "Error: Unexpected git diff return code (${ret})" >&2 + return 1 + fi + fi + fi + + # Create version file + ( + source sdk_lib/sdk_container_common.sh + create_versionfile "$sdk_version" "$version" + ) + update_and_push_version "${version}" "${push_branch}" +} +# -- diff --git a/ci-automation/packages.sh b/ci-automation/packages.sh index 77ab19aa28..5389b04aaf 100644 --- a/ci-automation/packages.sh +++ b/ci-automation/packages.sh @@ -10,13 +10,12 @@ # OS image binary packages build automation stub. # This script will use an SDK container to build packages for an OS image. -# It will update the versionfile with the OS packages version built, -# and will add a version tag (see INPUT) to the scripts repo. # # PREREQUISITES: # -# 1. SDK version is recorded in sdk_container/.repo/manifests/version.txt -# 2. SDK container is either +# 1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt +# 2. Scripts repo version tag of OS image version to be built is available and checked out. +# 3. SDK container is either # - available via ghcr.io/flatcar-linux/flatcar-sdk-[ARCH]:[VERSION] (official SDK release) # OR # - available via build cache server "/containers/[VERSION]/flatcar-sdk-[ARCH]-[VERSION].tar.gz" @@ -24,32 +23,16 @@ # # INPUT: # -# 1. Version of the TARGET OS image to build (string). -# The version pattern '(alpha|beta|stable|lts)-MMMM.m.p' (e.g. 'alpha-3051.0.0') -# denotes a "official" build, i.e. a release build to be published. -# Use any version diverging from the pattern (e.g. 'alpha-3051.0.0-nightly-4302') for development / CI builds. -# A tag of this version will be created in the scripts repo and pushed upstream. -# -# 2. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64"). +# 1. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64"). # # # OPTIONAL INPUT: # -# 3. coreos-overlay repository tag to use (commit-ish). -# Optional - use scripts repo sub-modules as-is if not set. -# This version will be checked out / pulled from remote in the coreos-overlay git submodule. -# The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed. -# -# 4. portage-stable repository tag to use (commit-ish). -# Optional - use scripts repo sub-modules as-is if not set. -# This version will be checked out / pulled from remote in the portage-stable git submodule. -# The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed. -# -# 5. SIGNER. Environment variable. Name of the owner of the artifact signing key. +# 2. SIGNER. Environment variable. Name of the owner of the artifact signing key. # Defaults to nothing if not set - in such case, artifacts will not be signed. # If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored. # -# 6. SIGNING_KEY. Environment variable. The artifact signing key. +# 3. SIGNING_KEY. Environment variable. The artifact signing key. # Defaults to nothing if not set - in such case, artifacts will not be signed. # If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored. # @@ -58,13 +41,10 @@ # 1. Exported container image "flatcar-packages-[ARCH]-[VERSION].tar.gz" with binary packages # pushed to buildcache, and torcx_manifest.json pushed to "images/${arch}/${vernum}/" # (for use with tests). -# 2. Updated scripts repository -# - version tag w/ submodules -# - sdk_container/.repo/manifests/version.txt denotes new FLATCAR OS version -# 3. "./ci-cleanup.sh" with commands to clean up temporary build resources, +# 2. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. -# 4. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache. -# 5. DIGESTS of the artifacts from point 1, pushed to buildcache. If signer key was passed, armored ASCII files of the generated DIGESTS files too, pushed to buildcache. +# 3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache. +# 4. DIGESTS of the artifacts from point 1, pushed to buildcache. If signer key was passed, armored ASCII files of the generated DIGESTS files too, pushed to buildcache. function packages_build() { # Run a subshell, so the traps, environment changes and global @@ -78,55 +58,15 @@ function packages_build() { # -- function _packages_build_impl() { - local version="$1" - local arch="$2" - local coreos_git="${3:-}" - local portage_git="${4:-}" + local arch="$1" source ci-automation/ci_automation_common.sh source ci-automation/gpg_setup.sh init_submodules - check_version_string "${version}" - source sdk_container/.repo/manifests/version.txt local sdk_version="${FLATCAR_SDK_VERSION}" - if [ -n "${coreos_git}" ] ; then - update_submodule "coreos-overlay" "${coreos_git}" - fi - if [ -n "${portage_git}" ] ; then - update_submodule "portage-stable" "${portage_git}" - fi - - # Create new tag in scripts repo w/ updated versionfile + submodules. - # Also push the changes to the branch ONLY IF we're doing a nightly - # build of the 'main'/'flatcar-MAJOR' branch AND we're definitely ON the respective branch - # (`scripts` and submodules). - local push_branch="false" - if [[ "${version}" =~ ^(stable|alpha|beta|lts)-[0-9.]+-nightly-[-0-9]+$ ]] \ - && [[ "$(git rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] \ - && [[ "$(git -C sdk_container/src/third_party/coreos-overlay/ rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] \ - && [[ "$(git -C sdk_container/src/third_party/portage-stable/ rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] ; then - push_branch="true" - local existing_tag="" - existing_tag=$(git tag --points-at HEAD) # exit code is always 0, output may be empty - # If the found tag is a release or nightly tag, we stop this build if there are no changes - if [[ "${existing_tag}" =~ ^(stable|alpha|beta|lts)-[0-9.]+(|-nightly-[-0-9]+)$ ]]; then - local ret=0 - git diff --exit-code "${existing_tag}" || ret=$? - if [ "$ret" = "0" ]; then - echo "Stopping build because there are no changes since tag ${existing_tag}" >&2 - return 0 - elif [ "$ret" = "1" ]; then - echo "Found changes since last tag ${existing_tag}" >&2 - else - echo "Error: Unexpected git diff return code (${ret})" >&2 - return 1 - fi - fi - fi - # Get SDK from either the registry or import from build cache # This is a NOP if the image is present locally. local sdk_name="flatcar-sdk-${arch}" @@ -137,17 +77,10 @@ function _packages_build_impl() { echo "docker image rm -f '${sdk_image}'" >> ./ci-cleanup.sh # Set name of the packages container for later rename / export - local vernum="${version#*-}" # remove main-,alpha-,beta-,stable-,lts- version tag + local vernum="${FLATCAR_VERSION}" local docker_vernum="$(vernum_to_docker_image_version "${vernum}")" local packages_container="flatcar-packages-${arch}-${docker_vernum}" - # Create version file - ( - source sdk_lib/sdk_container_common.sh - create_versionfile "$sdk_version" "$version" - ) - update_and_push_version "${version}" "${push_branch}" - # Build packages; store packages and torcx output in container ./run_sdk_container -x ./ci-cleanup.sh -n "${packages_container}" -v "${version}" \ -C "${sdk_image}" \