diff --git a/ci-automation/image.sh b/ci-automation/image.sh index f858a91f10..9a69de92c8 100644 --- a/ci-automation/image.sh +++ b/ci-automation/image.sh @@ -109,73 +109,5 @@ function _image_build_impl() { create_digests "${SIGNER}" "images/latest/"* sign_artifacts "${SIGNER}" "images/latest/"* copy_to_buildcache "images/${arch}/${vernum}/" "images/latest/"* - - ( - set +x - # Don't fail the whole job - set +e - echo "===================================================================" - export BOARD_A="${arch}-usr" - export FROM_A="release" - if [ "${channel}" = "developer" ]; then - NEW_CHANNEL="alpha" - else - NEW_CHANNEL="${channel}" - fi - NEW_CHANNEL_VERSION_A=$(curl -fsSL --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 "https://${NEW_CHANNEL}.release.flatcar-linux.net/${BOARD_A}/current/version.txt" | grep -m 1 FLATCAR_VERSION= | cut -d = -f 2) - MAJOR_A=$(echo "${NEW_CHANNEL_VERSION_A}" | cut -d . -f 1) - MAJOR_B=$(echo "${FLATCAR_VERSION}" | cut -d . -f 1) - # When the major version for the new channel is different, a transition has happened and we can find the previous release in the old channel - if [ "${MAJOR_A}" != "${MAJOR_B}" ]; then - case "${NEW_CHANNEL}" in - lts) - CHANNEL_A=stable - ;; - stable) - CHANNEL_A=beta - ;; - *) - CHANNEL_A=alpha - ;; - esac - VERSION_A=$(curl -fsSL --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 "https://${CHANNEL_A}.release.flatcar-linux.net/${BOARD_A}/current/version.txt" | grep -m 1 FLATCAR_VERSION= | cut -d = -f 2) - else - CHANNEL_A="${NEW_CHANNEL}" - VERSION_A="${NEW_CHANNEL_VERSION_A}" - fi - export VERSION_A - export CHANNEL_A - export FROM_B="file://${PWD}/images/latest" - # Use the directory directly (and BOARD_B and CHANNEL_B are unused) - export VERSION_B="." - echo "== Image differences compared to ${CHANNEL_A} ${VERSION_A} ==" - NEW_VERSION=$(git tag --points-at HEAD) - cd .. - rm -rf flatcar-build-scripts - git clone "https://github.com/flatcar-linux/flatcar-build-scripts" - echo "Package updates, compared to ${CHANNEL_A} ${VERSION_A}:" - FILE=flatcar_production_image_packages.txt flatcar-build-scripts/package-diff "${VERSION_A}" "${VERSION_B}" - echo - echo "Image file changes, compared to ${CHANNEL_A} ${VERSION_A}:" - FILE=flatcar_production_image_contents.txt FILESONLY=1 CUTKERNEL=1 flatcar-build-scripts/package-diff "${VERSION_A}" "${VERSION_B}" - echo - echo "Image kernel config changes, compared to ${CHANNEL_A} ${VERSION_A}:" - FILE=flatcar_production_image_kernel_config.txt flatcar-build-scripts/package-diff "${VERSION_A}" "${VERSION_B}" - echo - echo "Image file size change (includes /boot, /usr and the default rootfs partitions), compared to ${CHANNEL_A} ${VERSION_A}:" - FILE=flatcar_production_image_contents.txt CALCSIZE=1 flatcar-build-scripts/package-diff "${VERSION_A}" "${VERSION_B}" - echo - BASE_URL="http://${BUILDCACHE_SERVER}/images/${arch}/${vernum}" - echo "Image URL: ${BASE_URL}/flatcar_production_image.bin.bz2" - echo - # Provide a python3 command for the CVE DB parsing - export PATH="$PATH:$PWD/scripts/ci-automation/python-bin" - # The first changelog we print is always against the previous version of the new channel (is only same as CHANNEL_A VERSION_A without a transition) - flatcar-build-scripts/show-changes "${NEW_CHANNEL}-${NEW_CHANNEL_VERSION_A}" "${NEW_VERSION}" - # See if a channel transition happened and print the changelog against CHANNEL_A VERSION_A which is the previous release - if [ "${CHANNEL_A}" != "${NEW_CHANNEL}" ]; then - flatcar-build-scripts/show-changes "${CHANNEL_A}-${VERSION_A}" "${NEW_VERSION}" - fi - ) } # -- diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh new file mode 100644 index 0000000000..90a8e63271 --- /dev/null +++ b/ci-automation/image_changes.sh @@ -0,0 +1,122 @@ +#!/bin/bash +# +# Copyright (c) 2021 The Flatcar Maintainers. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# >>> This file is supposed to be SOURCED from the repository ROOT. <<< +# +# image_changes() should be called w/ the positional INPUT parameters below. + +# OS image differences display stub. +# This script will display the differences between the last released image and the currently built one. +# +# PREREQUISITES: +# +# 1. Artifacts describing the built image (kernel config, contents, packages, etc.) must be present in build cache server. +# 2. Scripts repo version tag of OS image version to be built is available and checked out. +# +# INPUT: +# +# 1. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64"). +# +# OPTIONAL INPUT: +# +# (none) +# +# OUTPUT: +# +# 1. Currently the script prints the image differences compared to the last release and the changelog for the release notes but doesn't store it yet in the buildcache. + +function image_changes() { + # Run a subshell, so the traps, environment changes and global + # variables are not spilled into the caller. + ( + set -euo pipefail + + _image_changes_impl "${@}" + ) +} +# -- + +function _image_changes_impl() { + local arch="$1" + + source sdk_lib/sdk_container_common.sh + local channel="" + channel="$(get_git_channel)" + source ci-automation/ci_automation_common.sh + source ci-automation/gpg_setup.sh + init_submodules + + source sdk_container/.repo/manifests/version.txt + local vernum="${FLATCAR_VERSION}" + + echo "===================================================================" + export BOARD_A="${arch}-usr" + export FROM_A="release" + if [ "${channel}" = "developer" ]; then + NEW_CHANNEL="alpha" + else + NEW_CHANNEL="${channel}" + fi + NEW_CHANNEL_VERSION_A=$(curl -fsSL --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 "https://${NEW_CHANNEL}.release.flatcar-linux.net/${BOARD_A}/current/version.txt" | grep -m 1 FLATCAR_VERSION= | cut -d = -f 2) + MAJOR_A=$(echo "${NEW_CHANNEL_VERSION_A}" | cut -d . -f 1) + MAJOR_B=$(echo "${FLATCAR_VERSION}" | cut -d . -f 1) + # When the major version for the new channel is different, a transition has happened and we can find the previous release in the old channel + if [ "${MAJOR_A}" != "${MAJOR_B}" ]; then + case "${NEW_CHANNEL}" in + lts) + CHANNEL_A=stable + ;; + stable) + CHANNEL_A=beta + ;; + *) + CHANNEL_A=alpha + ;; + esac + VERSION_A=$(curl -fsSL --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 "https://${CHANNEL_A}.release.flatcar-linux.net/${BOARD_A}/current/version.txt" | grep -m 1 FLATCAR_VERSION= | cut -d = -f 2) + else + CHANNEL_A="${NEW_CHANNEL}" + VERSION_A="${NEW_CHANNEL_VERSION_A}" + fi + export VERSION_A + export CHANNEL_A + export FROM_B="bincache" + export VERSION_B="${vernum}" + export BOARD_B="${arch}-usr" + # CHANNEL_B is unused + echo "== Image differences compared to ${CHANNEL_A} ${VERSION_A} ==" + NEW_VERSION=$(git tag --points-at HEAD) + cd .. + rm -rf flatcar-build-scripts + git clone "https://github.com/flatcar-linux/flatcar-build-scripts" + # Don't fail the job + set +e + echo "Package updates, compared to ${CHANNEL_A} ${VERSION_A}:" + FILE=flatcar_production_image_packages.txt flatcar-build-scripts/package-diff "${VERSION_A}" "${VERSION_B}" + echo + echo "Image file changes, compared to ${CHANNEL_A} ${VERSION_A}:" + FILE=flatcar_production_image_contents.txt FILESONLY=1 CUTKERNEL=1 flatcar-build-scripts/package-diff "${VERSION_A}" "${VERSION_B}" + echo + echo "Image kernel config changes, compared to ${CHANNEL_A} ${VERSION_A}:" + FILE=flatcar_production_image_kernel_config.txt flatcar-build-scripts/package-diff "${VERSION_A}" "${VERSION_B}" + echo + echo "Image file size change (includes /boot, /usr and the default rootfs partitions), compared to ${CHANNEL_A} ${VERSION_A}:" + FILE=flatcar_production_image_contents.txt CALCSIZE=1 flatcar-build-scripts/package-diff "${VERSION_A}" "${VERSION_B}" + echo + BASE_URL="http://${BUILDCACHE_SERVER}/images/${arch}/${vernum}" + echo "Image URL: ${BASE_URL}/flatcar_production_image.bin.bz2" + echo + # Provide a python3 command for the CVE DB parsing + export PATH="$PATH:$PWD/scripts/ci-automation/python-bin" + # The first changelog we print is always against the previous version of the new channel (is only same as CHANNEL_A VERSION_A without a transition) + flatcar-build-scripts/show-changes "${NEW_CHANNEL}-${NEW_CHANNEL_VERSION_A}" "${NEW_VERSION}" + # See if a channel transition happened and print the changelog against CHANNEL_A VERSION_A which is the previous release + if [ "${CHANNEL_A}" != "${NEW_CHANNEL}" ]; then + flatcar-build-scripts/show-changes "${CHANNEL_A}-${VERSION_A}" "${NEW_VERSION}" + fi + set -e +} +# -- diff --git a/ci-automation/release.sh b/ci-automation/release.sh new file mode 100644 index 0000000000..0ed3b6739c --- /dev/null +++ b/ci-automation/release.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +# Copyright (c) 2022 The Flatcar Maintainers. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# >>> This file is supposed to be SOURCED from the repository ROOT. <<< +# +# release_build() is currently called with no positional INPUT parameters but uses the signing env vars. + +# Release build automation stub. +# This script will release the image build from bincache to the cloud offers. +# +# PREREQUISITES: +# +# 1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt +# 2. Scripts repo version tag of OS image version to be built is available and checked out. +# 3. Mantle container docker image reference is stored in sdk_container/.repo/manifests/mantle-container. +# 4. Vendor image and torcx docker tarball + manifest to run tests for are available on buildcache +# ( images/[ARCH]/[FLATCAR_VERSION]/ ) +# 5. SDK container is either +# - available via ghcr.io/flatcar-linux/flatcar-sdk-[ARCH]:[VERSION] (official SDK release) +# OR +# - available via build cache server "/containers/[VERSION]/flatcar-sdk-[ARCH]-[VERSION].tar.gz" +# (dev SDK) +# +# INPUT: +# +# (none) +# +# OPTIONAL INPUT: +# +# 1. SIGNER. Environment variable. Name of the owner of the artifact signing key. +# Defaults to nothing if not set - in such case, artifacts will not be signed. +# If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored. +# +# 2. SIGNING_KEY. Environment variable. The artifact signing key. +# Defaults to nothing if not set - in such case, artifacts will not be signed. +# If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored. +# +# OUTPUT: +# +# 1. The cloud images are published with mantle's plume and ore tools +# 2. The AWS AMI text files are pushed to buildcache ( images/[ARCH]/[FLATCAR_VERSION]/ ) +# 3. "./ci-cleanup.sh" with commands to clean up temporary build resources, +# to be run after this step finishes / when this step is aborted. +# 4. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache. +# 5. DIGESTS of the artifacts from point 1, pushed to buildcache. If signer key was passed, armored ASCII files of the generated DIGESTS files too, pushed to buildcache. + +function release_build() { + # Run a subshell, so the traps, environment changes and global + # variables are not spilled into the caller. + ( + set -euo pipefail + + _release_build_impl "${@}" + ) +} + +function _inside_mantle() { + # Run a subshell for the same reasons as above + ( + set -euo pipefail + + source ci-automation/ci_automation_common.sh + source sdk_container/.repo/manifests/version.txt + + # TODO: set up credentials + # TODO: run mantle pre-release and release for all platforms + # (needs changes in mantle to consume from buildcache via https) + # TODO: run ore for AWS marketplace upload + ) +} + +function _release_build_impl() { + source ci-automation/ci_automation_common.sh + source ci-automation/gpg_setup.sh + init_submodules + + source sdk_container/.repo/manifests/version.txt + local sdk_version="${FLATCAR_SDK_VERSION}" + local docker_sdk_vernum="$(vernum_to_docker_image_version "${sdk_version}")" + local vernum="${FLATCAR_VERSION}" + local docker_vernum="$(vernum_to_docker_image_version "${vernum}")" + + local container_name="flatcar-publish-${docker_vernum}" + local mantle_ref + mantle_ref=$(cat sdk_container/.repo/manifests/mantle-container) + # A job on each worker prunes old mantle images (docker image prune), no need to do it here + echo "docker rm -f '${container_name}'" >> ./ci-cleanup.sh + + touch sdk_container/.env # This file should already contain the required credentials as env vars + docker run --pull always --rm --name="${container_name}" --net host \ + -w /work -v "$PWD":/work "${mantle_ref}" bash -c "source ci-automation/release.sh; _inside_mantle" + # TODO: sign and copy resulting AMI text file to buildcache + # TODO: run CF template update + # TODO: publish SDK container image if not published yet (i.e., on new majors) + echo "====" + echo "Done, now you can copy the images to Origin" + echo "====" + # Future: trigger copy to Origin in a secure way + # Future: trigger update payload signing + # Future: trigger website update + # Future: trigger release email sending + # Future: trigger push to nebraska + # Future: trigger Origin symlink switch +}