diff --git a/build_image b/build_image index 5ac5e1cc72..8fe2e9e26a 100755 --- a/build_image +++ b/build_image @@ -36,7 +36,7 @@ DEFINE_string base_dev_pkg "coreos-base/coreos-dev" \ DEFINE_string torcx_manifest "${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest/torcx_manifest.json" \ "The torcx manifest describing torcx packages for this image (or blank for none)" DEFINE_string torcx_root "${DEFAULT_BUILD_ROOT}/torcx" \ - "Directory in which torcx packages can be found" + "Directory in which torcx packages can be found. Will update the default --torcx_manifest if set." DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \ "Directory in which to place image result directories (named by version)" DEFINE_string disk_layout "" \ @@ -93,7 +93,7 @@ check_gsutil_opts # Patch around default values not being able to depend on other flags. if [ "x${FLAGS_torcx_manifest}" = "x${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest/torcx_manifest.json" ]; then - FLAGS_torcx_manifest="${DEFAULT_BUILD_ROOT}/torcx/${FLAGS_board}/latest/torcx_manifest.json" + FLAGS_torcx_manifest="${FLAGS_torcx_root}/${FLAGS_board}/latest/torcx_manifest.json" fi # If downloading packages is enabled ensure the board is configured properly. diff --git a/build_packages b/build_packages index 5620565dee..0c70bc8dad 100755 --- a/build_packages +++ b/build_packages @@ -36,6 +36,8 @@ DEFINE_boolean skip_toolchain_update "${FLAGS_FALSE}" \ "Don't update toolchain automatically." DEFINE_boolean skip_chroot_upgrade "${FLAGS_FALSE}" \ "Don't run the chroot upgrade automatically; use with care." +DEFINE_string torcx_output_root "${DEFAULT_BUILD_ROOT}" \ + "Directory in which to place torcx stores and manifests (named by board/version)" DEFINE_boolean skip_torcx_store "${FLAGS_FALSE}" \ "Don't build a new torcx store from the updated sysroot." @@ -301,7 +303,7 @@ upload_packages # Build a new torcx store with the updated packages, passing flags through. if [ "${FLAGS_skip_torcx_store}" -eq "${FLAGS_FALSE}" ]; then - "${SCRIPTS_DIR}"/build_torcx_store --board="${BOARD}" + "${SCRIPTS_DIR}"/build_torcx_store --board="${BOARD}" --output_root="${FLAGS_torcx_output_root}" fi info "Builds complete" diff --git a/ci-automation/README.md b/ci-automation/README.md new file mode 100644 index 0000000000..ddabf76248 --- /dev/null +++ b/ci-automation/README.md @@ -0,0 +1,77 @@ +# CI automation glue logic scripts + +Scripts in this directory aim to ease automation of Flatcar builds in continuous integration systems. + +Each script represents a distinct build step; each step ingests the container image of the previous step and produces a new container image for the next step. +Notable exceptions are "SDK Bootstrap" (`sdk.sh`) which only creates an SDK tarball, and "VMs build` which does not output a container but only VM (vendor) images. + +Please refer to the individual scripts for prerequisites, input parameters, and outputs. + +All steps make use of a "build cache" server for pulling (https) and pushing (rsync) build inputs and artefacts. + +## Build steps + +The build pipeline can be used to build everything from scratch, including the SDK (starting from 1. below) orto build a new OS image (starting from 3.). + +### SDK bootstrap build + +1. SDK Bootstrap (`sdk.sh`): Use a seed SDK tarball and seed SDK container image to build a new SDK tarball. + The resulting SDK tarball will use packages and versions pinned in the coreos-overlay and portage-stable submodules. + This step updates the versionfile, recording the SDK container version just built. + It will generate and push a new version tag to the scripts repo. +2. SDK container build (`sdk_container.sh`) : use SDK tarball and to build an SDK container image. + The resulting image will come in "amd64", "arm64", and "all" flavours, with support for respective OS target architectures. This step builds the Flatcar SDK container images published at ghcr.io/flatcar-linux. + +``` + .---------. .------------. + | scripts | | CI | + | repo | | automation | + `---------´ `------------´ + | | + | "alpha-3449.0.0-dev23" + | | + | _______v_______ + +------- clone -----> ( SDK bootstrap ) + | `-------------´ + |<- tag: sdk-3499.0.0-dev23 -´| + | | + | _______v_______ + +-- clone -> ( SDK container ) + | sdk-3499.0.0-dev23 `-------------´ + | | + v v + continue to OS + image build + | + v +``` + +### OS image build + +3. Packages build (`packages.sh`): using the SDK container version recorded in the versionfile, build OS image packages and generate a new container image (containing both SDK and packages). + This step updates the versionfile, recording the Flatcar OS image version just built. + It will generate and push a new version tag to the scripts repo. +4. Image build (`image.sh`): Using the container from 3., build an OS image and torcx store, and generate a new container image with everything in it. +5. VMs build (`vms.sh`). Using the packages+torcx+image container from 4., build vendor images. Result are vendor-specific OS images. + +``` + .---------. .------------. + | scripts | | CI | + | repo | | automation | + `---------´ `------------´ + | | + | "alpha-3449.0.0-dev23" + | | + | ____v_____ + +---------- clone --------> ( packages ) + | `--------´ + |<-- tag: alpha-3499.0.0-dev23 --´| + | | + | ___v___ + +----- clone ---> ( image ) + | alpha-3499.0.0-dev23 `-----´ + | | + | __v__ + +----- clone ---> ( vms ) + alpha-3499.0.0-dev23 `---´ +``` diff --git a/ci-automation/ci-config.env b/ci-automation/ci-config.env new file mode 100644 index 0000000000..14f6f3803f --- /dev/null +++ b/ci-automation/ci-config.env @@ -0,0 +1,24 @@ +# Copyright (c) 2021 The Flatcar Maintainers. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Flatcar CI static configuration + +BUILDCACHE_SERVER="bincache.flatcar-linux.net" +BUILDCACHE_PATH_PREFIX="/srv/bincache" +BUILDCACHE_USER="bincache" + +RELEASES_SERVER="mirror.release.flatcar-linux.net" +CONTAINER_REGISTRY="ghcr.io/flatcar-linux" + +GC_BUCKET="flatcar-linux" + +# No PIGZ on Flatcar +PIGZ="docker run --rm -i ghcr.io/flatcar-linux/pigz --fast" + +CI_GIT_AUTHOR="flatcar-ci" +CI_GIT_EMAIL="infra+ci@flatcar-linux.org" + +# build artefacts go here (in container) +CONTAINER_TORCX_ROOT="/home/sdk/build/torcx" +CONTAINER_IMAGE_ROOT="/home/sdk/build/images" diff --git a/ci-automation/ci_automation_common.sh b/ci-automation/ci_automation_common.sh new file mode 100644 index 0000000000..e5bf83c674 --- /dev/null +++ b/ci-automation/ci_automation_common.sh @@ -0,0 +1,208 @@ +#!/bin/bash +# +# Copyright (c) 2021 The Flatcar Maintainers. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# CI automation common functions. + +source ci-automation/ci-config.env +: ${PIGZ:=pigz} + +# set up author and email so git does not complain when tagging +git -C . config user.name "${CI_GIT_AUTHOR}" +git -C . config user.email "${CI_GIT_EMAIL}" + +function init_submodules() { + git submodule init + git submodule update +} +# -- + +function update_submodule() { + local submodule="$1" + local commit_ish="$2" + + cd "sdk_container/src/third_party/${submodule}" + git fetch --all --tags + git checkout "${commit_ish}" + cd - + +} +# -- + +function check_version_string() { + local version="$1" + + if ! echo "${version}" | grep -qE '^(alpha-|beta-|stable-|lts-)' ; then + echo "ERROR: invalid version '${version}', must start with 'alpha-', 'beta-', 'stable-', 'lts-'" + exit 1 + fi +} +# -- + +function update_submodules() { + local coreos_git="$1" + local portage_git="$2" + + init_submodules + update_submodule "coreos-overlay" "${coreos_git}" + update_submodule "portage-stable" "${portage_git}" +} +# -- + +function update_and_push_version() { + local version="$1" + + # Add and commit local changes + git add "sdk_container/src/third_party/coreos-overlay" + git add "sdk_container/src/third_party/portage-stable" + git add "sdk_container/.repo/manifests/version.txt" + + git commit --allow-empty -m "New version: ${version}" + + git tag -f "${version}" + + if git push origin "${version}" ; then + return + fi + # Push (above) may fail because a tag already exists. + # We check for tag presence, and for the difference + # between local and remote, and bail + # only if the remote / local contents differ. + + # Remove local tag, (re-)fetch remote tags + git tag -d "${version}" + + # refresh tree, let origin overwrite local tags + git fetch --all --tags --force + + # This will return != 0 if + # - the remote tag does not exist (rc: 127) + # - the remote tag has changes compared to the local tree (rc: 1) + git diff --exit-code "${version}" +} +# -- + +function copy_from_buildcache() { + local what="$1" + local where_to="$2" + + mkdir -p "$where_to" + curl --verbose --fail --silent --show-error --location --retry-delay 1 --retry 60 \ + --retry-connrefused --retry-max-time 60 --connect-timeout 20 \ + --remote-name --output-dir "${where_to}" "https://${BUILDCACHE_SERVER}/${what}" +} +# -- + +function gen_sshcmd() { + echo -n "ssh -o BatchMode=yes" + echo -n " -o StrictHostKeyChecking=no" + echo -n " -o UserKnownHostsFile=/dev/null" + echo " -o NumberOfPasswordPrompts=0" +} +# -- + +function copy_to_buildcache() { + local remote_path="${BUILDCACHE_PATH_PREFIX}/$1" + shift + + local sshcmd="$(gen_sshcmd)" + + $sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \ + "mkdir -p ${remote_path}" + + rsync -Pav -e "${sshcmd}" "$@" \ + "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}:${remote_path}" +} +# -- + +function image_exists_locally() { + local name="$1" + local version="$2" + local image="${name}:${version}" + + local image_exists="$(docker images "${image}" \ + --no-trunc --format '{{.Repository}}:{{.Tag}}')" + + [ "${image}" = "${image_exists}" ] +} +# -- + +# Derive docker-safe image version string from vernum. +# +function vernum_to_docker_image_version() { + local vernum="$1" + echo "$vernum" | sed 's/[+]/-/g' +} +# -- + +# Return the full name (repo+name+tag) of an image. Useful for SDK images +# pulled from the registry (which have the registry pre-pended) +function docker_image_fullname() { + local image="$1" + local version="$2" + + docker images --no-trunc --format '{{.Repository}}:{{.Tag}}' \ + | grep -E "^(${CONTAINER_REGISTRY}/)*${image}:${version}$" +} +# -- + +function docker_image_to_buildcache() { + local image="$1" + local version="$2" + + # strip potential container registry prefix + local tarball="$(basename "$image")-${version}.tar.gz" + + docker save "${image}":"${version}" | $PIGZ -c > "${tarball}" + copy_to_buildcache "containers/${version}" "${tarball}" +} +# -- + +function docker_commit_to_buildcache() { + local container="$1" + local image_name="$2" + local image_version="$3" + + docker commit "${container}" "${image_name}:${image_version}" + docker_image_to_buildcache "${image_name}" "${image_version}" +} +# -- + +function docker_image_from_buildcache() { + local name="$1" + local version="$2" + local tgz="${name}-${version}.tar.gz" + + if image_exists_locally "${name}" "${version}" ; then + return + fi + + local url="https://${BUILDCACHE_SERVER}/containers/${version}/${tgz}" + + curl --verbose --fail --silent --show-error --location --retry-delay 1 --retry 60 \ + --retry-connrefused --retry-max-time 60 --connect-timeout 20 \ + --remote-name "${url}" + + cat "${tgz}" | $PIGZ -d -c | docker load + + rm "${tgz}" +} +# -- + +function docker_image_from_registry_or_buildcache() { + local image="$1" + local version="$2" + + if image_exists_locally "${CONTAINER_REGISTRY}/${image}" "${version}" ; then + return + fi + + if docker pull "${CONTAINER_REGISTRY}/${image}:${version}" ; then + return + fi + + docker_image_from_buildcache "${image}" "${version}" +} +# -- diff --git a/ci-automation/garbage_collect.sh b/ci-automation/garbage_collect.sh new file mode 100644 index 0000000000..d747c79dab --- /dev/null +++ b/ci-automation/garbage_collect.sh @@ -0,0 +1,127 @@ +#!/bin/bash +# +# Copyright (c) 2021 The Flatcar Maintainers. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# >>> This file is supposed to be SOURCED from the repository ROOT. <<< +# +# garbage_collect() should be called after sourcing. +# +# OPTIONAL INPUT +# - Number of (recent) versions to keep. Defaults to 50. +# - DRY_RUN (Env variable). Set to "y" to just list what would be done but not +# actually purge anything. + +# Flatcar CI automation garbage collector. +# This script removes development (non-official) build artefacts: +# - SDK tarballs, build step containers, and vendor images on buildcache +# - tags from the scripts repository +# +# Garbage collection is based on development (non-official) version tags +# in the scripts repo. The newest 50 builds will be retained, +# all older builds will be purged (50 is the default, see OPTIONAL INPUT above). + +set -eu + +function garbage_collect() { + local keep="${1:-50}" + local dry_run="${DRY_RUN:-}" + + keep="$((keep + 1))" # for tail -n+... + local purge_versions="$(git tag -l --sort=-committerdate \ + | grep -E '(alpha|beta|stable|lts|sdk)-[0-9]+\.[0-9]+\.[0-9]+\-.*' \ + | grep -vE '(-pro)$' \ + | tail -n+"${keep}")" + + source ci-automation/ci_automation_common.sh + + local sshcmd="$(gen_sshcmd)" + + echo + echo "######## The following versions (starting with ${keep}-oldest) will be purged ########" + if [ "$dry_run" = "y" ] ; then + echo + echo "(NOTE this is just a dry run since DRY_RUN=y)" + echo + fi + echo "${purge_versions}" | sed 's/ /\n/g' + echo + echo + + local version + for version in ${purge_versions}; do + echo "--------------------------------------------" + echo + echo "#### Processing version '${version}' ####" + echo + + git checkout "${version}" -- sdk_container/.repo/manifests/version.txt + source sdk_container/.repo/manifests/version.txt + + local os_docker_vernum="$(vernum_to_docker_image_version "${FLATCAR_VERSION}")" + local sdk_vernum="${FLATCAR_SDK_VERSION}" + local sdk_docker_vernum="$(vernum_to_docker_image_version "${FLATCAR_SDK_VERSION}")" + + # Remove container image tarballs and SDK tarball (if applicable) + # + local rmpat="" + if [[ "${version}" = 'sdk-'* ]] ; then + echo "## ${version} is an SDK version. ##" + rmpat="${BUILDCACHE_PATH_PREFIX}/sdk/*/${sdk_vernum}/" + rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${sdk_docker_vernum}/flatcar-sdk-*" + else + echo "## ${version} is an OS image version. ##" + rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-packages-*" + rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-image-*" + rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/images/*/${os_docker_vernum}/" + fi + + echo "## The following files will be removed ##" + $sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \ + "ls -la ${rmpat} || true" + + if [ "$dry_run" != "y" ] ; then + set -x + $sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \ + "rm -rf ${rmpat}" + set +x + else + echo "## (DRY_RUN=y so not doing anything) ##" + fi + + # Remove container image directory if empty + # + if [[ "${version}" = 'sdk-'* ]] ; then + rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${sdk_docker_vernum}/" + else + rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/" + fi + + echo "## Checking if container directory is empty and can be removed (it's OK if this fails) ##" + echo "## The following directory will be removed if below output is empty: '${rmpat}' ##" + $sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \ + "ls -la ${rmpat} || true" + + if [ "$dry_run" != "y" ] ; then + set -x + $sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \ + "rmdir ${rmpat} || true" + set +x + else + echo "## (DRY_RUN=y so not doing anything) ##" + fi + + # Remove git tag (local and remote) + # + echo "## The following TAG will be deleted: '${version}' ##" + if [ "$dry_run" != "y" ] ; then + set -x + git tag -d "${version}" + git push --delete origin "${version}" + set +x + else + echo "## (DRY_RUN=y so not doing anything) ##" + fi + done +} diff --git a/ci-automation/image.sh b/ci-automation/image.sh new file mode 100644 index 0000000000..69c4ca40cf --- /dev/null +++ b/ci-automation/image.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# +# Copyright (c) 2021 The Flatcar Maintainers. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# >>> This file is supposed to be SOURCED from the repository ROOT. <<< +# +# packages_build() should be called w/ the positional INPUT parameters below. + +# Binary OS image build automation stub. +# This script will build the OS image from a pre-built packages container. +# +# PREREQUISITES: +# +# 1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt +# 2. Scripts repo version tag of OS image version to be built is available and checked out. +# 3. Flatcar packages container is available via build cache server +# from "/containers/[VERSION]/flatcar-packages-[ARCH]-[FLATCAR_VERSION].tar.gz" +# or present locally. Container must contain binary packages and torcx artefacts. +# +# INPUT: +# +# 1. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64"). +# +# OUTPUT: +# +# 1. Exported container image with OS image, dev container, and related artefacts at +# /home/sdk/image/[ARCH], torcx packages at /home/sdk/torcx +# named "flatcar-images-[ARCH]-[FLATCAR_VERSION].tar.gz" +# pushed to buildcache. +# 2. "./ci-cleanup.sh" with commands to clean up temporary build resources, +# to be run after this step finishes / when this step is aborted. + +set -eu + +function image_build() { + local arch="$1" + + source ci-automation/ci_automation_common.sh + init_submodules + + source sdk_container/.repo/manifests/version.txt + local vernum="${FLATCAR_VERSION}" + local docker_vernum="$(vernum_to_docker_image_version "${vernum}")" + + local packages="flatcar-packages-${arch}" + local packages_image="${packages}:${docker_vernum}" + + docker_image_from_buildcache "${packages}" "${docker_vernum}" + + local image="flatcar-images-${arch}" + local image_container="${image}-${docker_vernum}" + + # build image and store it in the container + ./run_sdk_container -x ./ci-cleanup.sh -n "${image_container}" -C "${packages_image}" \ + -v "${vernum}" \ + mkdir -p "${CONTAINER_IMAGE_ROOT}" + ./run_sdk_container -n "${image_container}" -C "${packages_image}" \ + -v "${vernum}" \ + ./build_image --board="${arch}-usr" \ + --output_root="${CONTAINER_IMAGE_ROOT}" \ + --torcx_root="${CONTAINER_TORCX_ROOT}" prodtar container + + # rename container and push to build cache + docker_commit_to_buildcache "${image_container}" "${image}" "${docker_vernum}" +} +# -- diff --git a/ci-automation/packages.sh b/ci-automation/packages.sh new file mode 100644 index 0000000000..64ba904fd4 --- /dev/null +++ b/ci-automation/packages.sh @@ -0,0 +1,116 @@ +#!/bin/bash +# +# Copyright (c) 2021 The Flatcar Maintainers. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# >>> This file is supposed to be SOURCED from the repository ROOT. <<< +# +# packages_build() should be called w/ the positional INPUT parameters below. + +# OS image binary packages build automation stub. +# This script will use an SDK container to build packages for an OS image. +# It will update the versionfile with the OS packages version built, +# and will add a version tag (see INPUT) to the scripts repo. +# +# PREREQUISITES: +# +# 1. SDK version is recorded in sdk_container/.repo/manifests/version.txt +# 2. SDK container is either +# - available via ghcr.io/flatcar-linux/flatcar-sdk-[ARCH]:[VERSION] (official SDK release) +# OR +# - available via build cache server "/containers/[VERSION]/flatcar-sdk-[ARCH]-[VERSION].tar.gz" +# (dev SDK) +# +# INPUT: +# +# 1. Version of the TARGET OS image to build (string). +# The version pattern '(alpha|beta|stable|lts)-MMMM.m.p' (e.g. 'alpha-3051.0.0') +# denotes a "official" build, i.e. a release build to be published. +# Use any version diverging from the pattern (e.g. 'alpha-3051.0.0-nightly-4302') for development / CI builds. +# A tag of this version will be created in the scripts repo and pushed upstream. +# +# 2. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64"). +# +# +# OPTIONAL INPUT: +# +# 3. coreos-overlay repository tag to use (commit-ish). +# Optional - use scripts repo sub-modules as-is if not set. +# This version will be checked out / pulled from remote in the coreos-overlay git submodule. +# The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed. +# +# 4. portage-stable repository tag to use (commit-ish). +# Optional - use scripts repo sub-modules as-is if not set. +# This version will be checked out / pulled from remote in the portage-stable git submodule. +# The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed. +# +# OUTPUT: +# +# 1. Exported container image "flatcar-packages-[ARCH]-[VERSION].tar.gz" with binary packages +# pushed to buildcache. +# 2. Updated scripts repository +# - version tag w/ submodules +# - sdk_container/.repo/manifests/version.txt denotes new FLATCAR OS version +# 3. "./ci-cleanup.sh" with commands to clean up temporary build resources, +# to be run after this step finishes / when this step is aborted. + + +set -eu + +function packages_build() { + local version="$1" + local arch="$2" + local coreos_git="${3:-}" + local portage_git="${4:-}" + + source ci-automation/ci_automation_common.sh + init_submodules + + check_version_string "${version}" + + source sdk_container/.repo/manifests/version.txt + local sdk_version="${FLATCAR_SDK_VERSION}" + + if [ -n "${coreos_git}" ] ; then + update_submodule "coreos-overlay" "${coreos_git}" + fi + if [ -n "${portage_git}" ] ; then + update_submodule "portage-stable" "${portage_git}" + fi + + # Get SDK from either the registry or import from build cache + # This is a NOP if the image is present locally. + local sdk_name="flatcar-sdk-${arch}" + local docker_sdk_vernum="$(vernum_to_docker_image_version "${sdk_version}")" + + docker_image_from_registry_or_buildcache "${sdk_name}" "${docker_sdk_vernum}" + local sdk_image="$(docker_image_fullname "${sdk_name}" "${docker_sdk_vernum}")" + echo "docker image rm -f '${sdk_image}'" >> ./ci-cleanup.sh + + # Set name of the packages container for later rename / export + local vernum="${version#*-}" # remove alpha-,beta-,stable-,lts- version tag + local docker_vernum="$(vernum_to_docker_image_version "${vernum}")" + local packages_container="flatcar-packages-${arch}-${docker_vernum}" + + # Build packages; store packages and torcx output in container + ./run_sdk_container -x ./ci-cleanup.sh -n "${packages_container}" -v "${version}" \ + -C "${sdk_image}" \ + mkdir -p "${CONTAINER_TORCX_ROOT}" + ./run_sdk_container -n "${packages_container}" -v "${version}" \ + -C "${sdk_image}" \ + ./build_packages --board="${arch}-usr" \ + --torcx_output_root="${CONTAINER_TORCX_ROOT}" + + # run_sdk_container updates the version file, use that version from here on + source sdk_container/.repo/manifests/version.txt + local vernum="${FLATCAR_VERSION}" + local docker_vernum="$(vernum_to_docker_image_version "${vernum}")" + local packages_image="flatcar-packages-${arch}" + + # generate image + push to build cache + docker_commit_to_buildcache "${packages_container}" "${packages_image}" "${docker_vernum}" + + update_and_push_version "${version}" +} +# -- diff --git a/ci-automation/sdk_bootstrap.sh b/ci-automation/sdk_bootstrap.sh new file mode 100644 index 0000000000..fcf1826ead --- /dev/null +++ b/ci-automation/sdk_bootstrap.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# +# Copyright (c) 2021 The Flatcar Maintainers. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# >>> This file is supposed to be SOURCED from the repository ROOT. <<< +# +# sdk_bootstrap() should be called w/ the positional INPUT parameters below. + +# Boostrap SDK build automation stub. +# This script will use a seed SDK container + tarball to bootstrap a +# new SDK tarball. +# +# INPUT: +# +# 1. Version of the SEED SDK to use (string). +# The seed SDK tarball must be available on https://mirror.release.flatcar-linux.net/sdk/ ... +# The seed SDK container must be available from https://github.com/orgs/flatcar-linux/packages +# (via ghcr.io/flatcar-linux/flatcar-sdk-all:[VERSION]). +# +# 2. Version of the TARGET SDK to build (string). +# The version pattern 'MMMM.m.p' (e.g. '3051.0.0') denotes a "official" build, i.e. a release build to be published. +# Use any version diverging from the pattern (e.g. '3051.0.0-nightly-4302') for development / CI builds. +# A tag "sdk-[VERSION]" will be created in the scripts repo and pushed upstream. +# +# OPTIONAL INPUT: +# +# 3. coreos-overlay repository tag to use (commit-ish). +# This version will be checked out / pulled from remote in the coreos-overlay git submodule. +# The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed. +# Leave empty to use coreos-overlay as-is. +# +# 4. portage-stable repository tag to use (commit-ish). +# This version will be checked out / pulled from remote in the portage-stable git submodule. +# The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed. +# Leave empty to use portage-stable as-is. +# +# 5. ARCH. Environment variable. Target architecture for the SDK to run on. +# Either "amd64" or "arm64"; defaults to "amd64" if not set. +# +# OUTPUT: +# +# 1. SDK tarball (gentoo catalyst output) of the new SDK, pushed to buildcache. +# 2. Updated scripts repository +# - version tag w/ submodules +# - sdk_container/.repo/manifests/version.txt denotes new SDK version +# 3. "./ci-cleanup.sh" with commands to clean up temporary build resources, +# to be run after this step finishes / when this step is aborted. + +set -eu + +function sdk_bootstrap() { + local seed_version="$1" + local version="$2" + local coreos_git="$3" + local portage_git="$4" + : ${ARCH:="amd64"} + + source ci-automation/ci_automation_common.sh + init_submodules + + check_version_string "${version}" + + if [ -n "${coreos_git}" ] ; then + update_submodule "coreos-overlay" "${coreos_git}" + fi + if [ -n "${portage_git}" ] ; then + update_submodule "portage-stable" "${portage_git}" + fi + + local vernum="${version#*-}" # remove alpha-,beta-,stable-,lts- version tag + local git_vernum="${vernum}" + + # This will update FLATCAR_VERSION[_ID] and BUILD_ID in versionfile + ./bootstrap_sdk_container -x ./ci-cleanup.sh "${seed_version}" "${vernum}" + + # push SDK tarball to buildcache + source sdk_container/.repo/manifests/version.txt + local vernum="${FLATCAR_SDK_VERSION}" + local dest_tarball="flatcar-sdk-${ARCH}-${vernum}.tar.bz2" + + cd "__build__/images/catalyst/builds/flatcar-sdk" + copy_to_buildcache "sdk/${ARCH}/${vernum}" "${dest_tarball}"* + cd - + + # Create new tag in scripts repo w/ updated versionfile + submodules + update_and_push_version "sdk-${git_vernum}" +} +# -- diff --git a/ci-automation/sdk_container.sh b/ci-automation/sdk_container.sh new file mode 100644 index 0000000000..29e021a767 --- /dev/null +++ b/ci-automation/sdk_container.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# +# Copyright (c) 2021 The Flatcar Maintainers. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# >>> This file is supposed to be SOURCED from the repository ROOT. <<< +# +# sdk_container_build() should be called w/ the positional INPUT parameters below. + +# SDK container builder automation stub. +# This script will build an SDK container w/ board support from an SDK tarball. +# NOTE that SDK tarball and scripts repo must have the same version or building +# the SDK container will fail. +# +# PREREQUISITES: +# +# 1. SDK version is recorded in sdk_container/.repo/manifests/version.txt and a matching +# SDK tarball is available on BUILDCACHE/sdk/[ARCH]/[VERSION]/flatcar-sdk-[ARCH]-[VERSION].tar.bz2 +# +# OPTIONAL INPUT: + +# 2. ARCH. Environment variable. Target architecture for the SDK to run on. +# Either "amd64" or "arm64"; defaults to "amd64" if not set. +# +# OUTPUT: +# +# 1. SDK container image of the new SDK, published to buildcache. +# 2. "./ci-cleanup.sh" with commands to clean up temporary build resources, +# to be run after this step finishes / when this step is aborted. + +set -eu + +function sdk_container_build() { + : ${ARCH:="amd64"} + + source ci-automation/ci_automation_common.sh + + init_submodules + + source sdk_container/.repo/manifests/version.txt + local vernum="${FLATCAR_SDK_VERSION}" + local sdk_tarball="flatcar-sdk-${ARCH}-${vernum}.tar.bz2" + + # __build__ is in .dockerignore, so the tarball is excluded from build context + mkdir -p __build__ + copy_from_buildcache "sdk/${ARCH}/${vernum}/${sdk_tarball}" "./__build__" + + + # This will update the SDK_VERSION in versionfile + ./build_sdk_container_image -x ./ci-cleanup.sh ./__build__/"${sdk_tarball}" + + # push artefacts to build cache + local docker_vernum="$(vernum_to_docker_image_version "${vernum}")" + docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-all" "${docker_vernum}" + docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-amd64" "${docker_vernum}" + docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-arm64" "${docker_vernum}" +} +# -- diff --git a/ci-automation/util/Dockerfile.pigz b/ci-automation/util/Dockerfile.pigz new file mode 100644 index 0000000000..33bcfedcaa --- /dev/null +++ b/ci-automation/util/Dockerfile.pigz @@ -0,0 +1,4 @@ +FROM alpine +RUN apk add pigz +ENTRYPOINT [ "/usr/bin/pigz" ] +CMD [ "-h" ] diff --git a/ci-automation/vms.sh b/ci-automation/vms.sh new file mode 100644 index 0000000000..6548421fc1 --- /dev/null +++ b/ci-automation/vms.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# +# Copyright (c) 2021 The Flatcar Maintainers. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# >>> This file is supposed to be SOURCED from the repository ROOT. <<< +# +# packages_build() should be called w/ the positional INPUT parameters below. + +# Vendor images build automation stub. +# This script will build one or more vendor images ("vm") using a pre-built image container. +# +# PREREQUISITES: +# +# 1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt +# 2. Scripts repo version tag of OS image version to be built is available and checked out. +# 3. Flatcar image container is available via build cache server +# from "/containers/[VERSION]/flatcar-images-[ARCH]-[FLATCAR_VERSION].tar.gz" +# or present locally. Must contain packages and image. +# +# INPUT: +# +# 1. Architecture (ARCH) of the TARGET vm images ("arm64", "amd64"). +# 2. Image formats to be built. Can be multiple, separated by spaces. +# Run ./image_to_vm.sh -h in the SDK to get a list of supported images. +# +# OUTPUT: +# +# 1. Exported VM image(s), pushed to buildcache ( images/[ARCH]/[FLATCAR_VERSION]/ ) +# 2. "./ci-cleanup.sh" with commands to clean up temporary build resources, +# to be run after this step finishes / when this step is aborted. + +set -eu + +function vm_build() { + local arch="$1" + shift + # $@ now contains image formats to build + + source ci-automation/ci_automation_common.sh + init_submodules + + source sdk_container/.repo/manifests/version.txt + local vernum="${FLATCAR_VERSION}" + local docker_vernum="$(vernum_to_docker_image_version "${vernum}")" + + local image="flatcar-images-${arch}" + local image_image="${image}:${docker_vernum}" + local vms_container="flatcar-vms-${docker_vernum}" + + docker_image_from_buildcache "${image}" "${docker_vernum}" + + # clean up dangling containers from previous builds + docker container rm -f "${vms_container}" || true + + local images_out="images/" + rm -rf "${images_out}" + + echo "docker container rm -f '${vms_container}'" >> ci-cleanup.sh + + for format in "${@}"; do + echo " ################### VENDOR '${format}' ################### " + ./run_sdk_container -n "${vms_container}" -C "${image_image}" \ + -v "${vernum}" \ + ./image_to_vm.sh --format "${format}" --board="${arch}-usr" \ + --from "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest" + done + + # copy resulting images + push to buildcache + ./run_sdk_container -n "${vms_container}" \ + -v "${vernum}" \ + cp -R "${CONTAINER_IMAGE_ROOT}/${arch}-usr/" "./${images_out}/" + + cd "images/latest" + copy_to_buildcache "images/${arch}/${vernum}/" * +} +# --