#!/bin/bash # # Copyright (c) 2021 The Flatcar Maintainers. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # >>> This file is supposed to be SOURCED from the repository ROOT. <<< # # sdk_bootstrap() should be called w/ the positional INPUT parameters below. # Bootstrap SDK build automation stub. # This script will use a seed SDK container + tarball to bootstrap a # new SDK tarball. # # INPUT: # # 1. Version of the SEED SDK to use (string). # The seed SDK tarball must be available on https://mirror.release.flatcar-linux.net/sdk/ ... # The seed SDK container must be available from https://github.com/orgs/flatcar/packages # (via ghcr.io/flatcar/flatcar-sdk-all:[VERSION]). # # 2. Version of the TARGET SDK to build (string). # The version pattern 'MMMM.m.p' (e.g. '3051.0.0') denotes a "official" build, i.e. a release build to be published. # Use any version diverging from the pattern (e.g. '3051.0.0-nightly-4302') for development / CI builds. # A free-standing tagged commit will be created in the scripts repo and pushed upstream. # # OPTIONAL INPUT: # # 3. ARCH. Environment variable. Target architecture for the SDK to run on. # Either "amd64" or "arm64"; defaults to "amd64" if not set. # # 4. SIGNER. Environment variable. Name of the owner of the artifact signing key. # Defaults to nothing if not set - in such case, artifacts will not be signed. # If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored. # # 5. SIGNING_KEY. Environment variable. The artifact signing key. # Defaults to nothing if not set - in such case, artifacts will not be signed. # If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored. # # 6. A file ../scripts.patch to apply with "git am -3" for the scripts repo. # # 7. AVOID_NIGHTLY_BUILD_SHORTCUTS. Environment variable. Tells the script to build the SDK even if nothing has changed since last nightly build. # See the description in ci-config.env. # # OUTPUT: # # 1. SDK tarball (gentoo catalyst output) of the new SDK, pushed to buildcache. # 2. Updated scripts repository # - version tag # - sdk_container/.repo/manifests/version.txt denotes new SDK version # 3. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. # 4. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache. # 5. DIGESTS of the artifacts from point 1, pushed to buildcache. If signer key was passed, armored ASCII files of the generated DIGESTS files too, pushed to buildcache. function sdk_bootstrap() { # Run a subshell, so the traps, environment changes and global # variables are not spilled into the caller. ( set -euo pipefail _sdk_bootstrap_impl "${@}" ) } # -- function _sdk_bootstrap_impl() { local seed_version=${1} local version=${2} : ${ARCH:="amd64"} source ci-automation/ci_automation_common.sh source ci-automation/gpg_setup.sh check_version_string "${version}" if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then set -x fi # Create new tag in scripts repo w/ updated versionfile. # Also push the changes to the branch ONLY IF we're doing a nightly # build of the 'main' branch AND we're definitely ON the main branch. # This includes intermediate SDKs when doing 2-phase nightly builds. local target_branch='' # These variables are here to make it easier to test nightly # builds without messing with actual release branches. local main_branch=${CIA_DEBUGMAINBRANCH:-main} local nightly=${CIA_DEBUGNIGHTLY:-nightly} # Matches the usual nightly tag name, optionally with an # intermediate suffix of the build ID too # (main-1234.2.3-nightly-yyyymmdd-hhmm-INTERMEDIATE). local nightly_pattern_1='^main-[0-9]+(\.[0-9]+){2}-'"${nightly}"'-[0-9]{8}-[0-9]{4}(-INTERMEDIATE)?$' local main_branch_hash='' if [[ ${version} =~ ${nightly_pattern_1} ]]; then main_branch_hash=$(git rev-parse "origin/${main_branch}") fi local -a existing_tags=() if [[ -n ${main_branch_hash} ]]; then if [[ $(git rev-parse HEAD) != "${main_branch_hash}" ]] ; then echo "We are doing a nightly build but we are not on top of the ${main_branch} branch. This is wrong and would result in the nightly tag not being a part of the branch." >&2 exit 1 fi target_branch=${main_branch} # Check for the existing tag only when we allow # shortcutting the builds. That way we can skip the checks # for build shortcutting. if bool_is_true "${AVOID_NIGHTLY_BUILD_SHORTCUTS}"; then echo "Continuing the build because AVOID_NIGHTLY_BUILD_SHORTCUTS is bool true (${AVOID_NIGHTLY_BUILD_SHORTCUTS})" >&2 else git fetch --all --tags --force # exit code is always 0, output may be empty mapfile -t existing_tags < <(git tag --points-at HEAD) fi fi local nightly_pattern_2='^main-[0-9]+(\.[0-9]+){2}-'"${nightly}"'-[0-9]{8}-[0-9]{4}$' local tag nightly_tag='' for tag in "${existing_tags[@]}"; do if [[ ${tag} =~ ${nightly_pattern_2} ]]; then nightly_tag=${tag} break fi done # If the found tag is a nightly tag, we stop this build if there # are no changes and the relevant images can be found in the # bincache. if [[ -n ${nightly_tag} ]]; then local -i ret=0 git diff --exit-code --quiet "${nightly_tag}" || ret=$? if [[ ret -eq 0 ]]; then local -a versions=( $( source sdk_lib/sdk_container_common.sh source "${sdk_container_common_versionfile}" echo "${FLATCAR_SDK_VERSION}" echo "${FLATCAR_VERSION}" ) ) local flatcar_sdk_version=${versions[0]} local flatcar_version=${versions[1]} local sdk_docker_vernum="" sdk_docker_vernum=$(vernum_to_docker_image_version "${flatcar_sdk_version}") if check_bincache_images_existence \ "https://${BUILDCACHE_SERVER}/containers/${sdk_docker_vernum}/flatcar-sdk-all-${sdk_docker_vernum}.tar.zst" \ "https://${BUILDCACHE_SERVER}/images/amd64/${flatcar_version}/flatcar_production_image.bin.bz2" \ "https://${BUILDCACHE_SERVER}/images/arm64/${flatcar_version}/flatcar_production_image.bin.bz2"; then echo "Stopping build because there are no changes since tag ${nightly_tag}, the SDK container tar ball and the Flatcar images exist" >&2 return 0 fi echo "No changes but continuing build because SDK container tar ball and/or the Flatcar images do not exist" >&2 elif [[ ret -eq 1 ]]; then echo "HEAD is tagged with a nightly tag and yet there a differences? This is fishy and needs to be investigated. Maybe you forgot to commit your changes?" >&2 exit 1 else echo "Error: Unexpected git diff return code (${ret})" >&2 return 1 fi fi if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then set +x fi local vernum=${version#*-} # remove alpha-,beta-,stable-,lts- version tag local git_vernum=${vernum} # Update FLATCAR_VERSION[_ID], BUILD_ID, and SDK in versionfile ( source sdk_lib/sdk_container_common.sh create_versionfile "${vernum}" ) if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then set -x fi update_and_push_version "${version}" "${target_branch}" if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then exit 0 fi apply_local_patches ./bootstrap_sdk_container -x ./ci-cleanup.sh "${seed_version}" "${vernum}" # push SDK tarball to buildcache # Get Flatcar version number format (separator is '+' instead of '-', # equal to $(strip_version_prefix "$version") source sdk_container/.repo/manifests/version.txt local dest_tarball="flatcar-sdk-${ARCH}-${FLATCAR_SDK_VERSION}.tar.bz2" # change the owner of the files and directories in __build__ back # to ourselves, otherwise we could fail to sign the artifacts as # we lacked write permissions in the directory of the signed # artifact local uid local gid uid=$(id --user) gid=$(id --group) sudo chown --recursive "${uid}:${gid}" __build__ ( cd "__build__/images/catalyst/builds/flatcar-sdk" create_digests "${SIGNER}" "${dest_tarball}" sign_artifacts "${SIGNER}" "${dest_tarball}"* copy_to_buildcache "sdk/${ARCH}/${FLATCAR_SDK_VERSION}" "${dest_tarball}"* ) } # --