ci-automation: simplified CI automation unsing containers

ci-automation builds on the SDK container and simplifies CI automation
build tasks (SDK bootstrap, SDK container, packages, image, VMs).

See ci-automation/README.md for a brief introduction.

Signed-off-by: Thilo Fromm <thilo@kinvolk.io>

build_image

@@ -36,7 +36,7 @@ DEFINE_string base_dev_pkg "coreos-base/coreos-dev" \
 DEFINE_string torcx_manifest "${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest/torcx_manifest.json" \
   "The torcx manifest describing torcx packages for this image (or blank for none)"
 DEFINE_string torcx_root "${DEFAULT_BUILD_ROOT}/torcx" \
-  "Directory in which torcx packages can be found"
+  "Directory in which torcx packages can be found. Will update the default --torcx_manifest if set."
 DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \
   "Directory in which to place image result directories (named by version)"
 DEFINE_string disk_layout "" \
@@ -93,7 +93,7 @@ check_gsutil_opts
 
 # Patch around default values not being able to depend on other flags.
 if [ "x${FLAGS_torcx_manifest}" = "x${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest/torcx_manifest.json" ]; then
-  FLAGS_torcx_manifest="${DEFAULT_BUILD_ROOT}/torcx/${FLAGS_board}/latest/torcx_manifest.json"
+  FLAGS_torcx_manifest="${FLAGS_torcx_root}/${FLAGS_board}/latest/torcx_manifest.json"
 fi
 
 # If downloading packages is enabled ensure the board is configured properly.

build_packages

@@ -36,6 +36,8 @@ DEFINE_boolean skip_toolchain_update "${FLAGS_FALSE}" \
   "Don't update toolchain automatically."
 DEFINE_boolean skip_chroot_upgrade "${FLAGS_FALSE}" \
   "Don't run the chroot upgrade automatically; use with care."
+DEFINE_string torcx_output_root "${DEFAULT_BUILD_ROOT}" \
+  "Directory in which to place torcx stores and manifests (named by board/version)"
 DEFINE_boolean skip_torcx_store "${FLAGS_FALSE}" \
   "Don't build a new torcx store from the updated sysroot."
 
@@ -301,7 +303,7 @@ upload_packages
 
 # Build a new torcx store with the updated packages, passing flags through.
 if [ "${FLAGS_skip_torcx_store}" -eq "${FLAGS_FALSE}" ]; then
-  "${SCRIPTS_DIR}"/build_torcx_store --board="${BOARD}"
+  "${SCRIPTS_DIR}"/build_torcx_store --board="${BOARD}" --output_root="${FLAGS_torcx_output_root}"
 fi
 
 info "Builds complete"

ci-automation/README.md

@@ -0,0 +1,77 @@
+# CI automation glue logic scripts
+
+Scripts in this directory aim to ease automation of Flatcar builds in continuous integration systems.
+
+Each script represents a distinct build step; each step ingests the container image of the previous step and produces a new container image for the next step.
+Notable exceptions are "SDK Bootstrap" (`sdk.sh`) which only creates an SDK tarball, and "VMs build` which does not output a container but only VM (vendor) images.
+
+Please refer to the individual scripts for prerequisites, input parameters, and outputs.
+
+All steps make use of a "build cache" server for pulling (https) and pushing (rsync) build inputs and artefacts.
+
+## Build steps
+
+The build pipeline can be used to build everything from scratch, including the SDK (starting from 1. below) orto build a new OS image (starting from 3.).
+
+### SDK bootstrap build
+
+1. SDK Bootstrap (`sdk.sh`): Use a seed SDK tarball and seed SDK container image to build a new SDK tarball.
+   The resulting SDK tarball will use packages and versions pinned in the coreos-overlay and portage-stable submodules.
+   This step updates the versionfile, recording the SDK container version just built.
+   It will generate and push a new version tag to the scripts repo.
+2. SDK container build (`sdk_container.sh`) : use SDK tarball and to build an SDK container image.
+   The resulting image will come in "amd64", "arm64", and "all" flavours, with support for respective OS target architectures. This step builds the Flatcar SDK container images published at ghcr.io/flatcar-linux.
+
+```
+         .---------.                 .------------.
+         | scripts |                 |     CI     |
+         |  repo   |                 | automation |
+         `---------´                 `------------´
+              |                             |
+              |                   "alpha-3449.0.0-dev23"
+              |                             |
+              |                      _______v_______
+              +------- clone -----> ( SDK bootstrap )
+              |                      `-------------´
+              |<- tag: sdk-3499.0.0-dev23 -´|
+              |                             |
+              |                      _______v_______
+              +--      clone     -> ( SDK container )
+              | sdk-3499.0.0-dev23   `-------------´
+              |                             |
+              v                             v
+                      continue to OS
+                       image build
+                            |
+                            v 
+```
+
+### OS image build
+
+3. Packages build (`packages.sh`): using the SDK container version recorded in the versionfile, build OS image packages and generate a new container image (containing both SDK and packages).
+   This step updates the versionfile, recording the Flatcar OS image version just built.
+   It will generate and push a new version tag to the scripts repo.
+4. Image build (`image.sh`): Using the container from 3., build an OS image and torcx store, and generate a new container image with everything in it.
+5. VMs build (`vms.sh`). Using the packages+torcx+image container from 4., build vendor images. Result are vendor-specific OS images.
+
+```
+       .---------.                     .------------.
+       | scripts |                     |     CI     |
+       |  repo   |                     | automation |
+       `---------´                     `------------´
+            |                                 |
+            |                       "alpha-3449.0.0-dev23"
+            |                                 |
+            |                             ____v_____
+            +---------- clone -------->  ( packages )
+            |                             `--------´
+            |<-- tag: alpha-3499.0.0-dev23 --´|
+            |                                 |
+            |                              ___v___
+            +-----       clone      --->  ( image )
+            |    alpha-3499.0.0-dev23      `-----´
+            |                                 |
+            |                               __v__
+            +-----       clone      --->   ( vms )
+                 alpha-3499.0.0-dev23       `---´
+```

ci-automation/ci-config.env

@@ -0,0 +1,24 @@
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Flatcar CI static configuration
+
+BUILDCACHE_SERVER="bincache.flatcar-linux.net"
+BUILDCACHE_PATH_PREFIX="/srv/bincache"
+BUILDCACHE_USER="bincache"
+
+RELEASES_SERVER="mirror.release.flatcar-linux.net"
+CONTAINER_REGISTRY="ghcr.io/flatcar-linux"
+
+GC_BUCKET="flatcar-linux"
+
+# No PIGZ on Flatcar
+PIGZ="docker run --rm -i ghcr.io/flatcar-linux/pigz --fast"
+
+CI_GIT_AUTHOR="flatcar-ci"
+CI_GIT_EMAIL="infra+ci@flatcar-linux.org"
+
+# build artefacts go here (in container)
+CONTAINER_TORCX_ROOT="/home/sdk/build/torcx"
+CONTAINER_IMAGE_ROOT="/home/sdk/build/images"

ci-automation/ci_automation_common.sh

@@ -0,0 +1,208 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# CI automation common functions.
+
+source ci-automation/ci-config.env
+: ${PIGZ:=pigz}
+
+# set up author and email so git does not complain when tagging
+git -C . config user.name "${CI_GIT_AUTHOR}"  
+git -C . config user.email "${CI_GIT_EMAIL}"
+
+function init_submodules() {
+    git submodule init
+    git submodule update
+}
+# --
+
+function update_submodule() {
+    local submodule="$1"
+    local commit_ish="$2"
+
+    cd "sdk_container/src/third_party/${submodule}"
+    git fetch --all --tags
+    git checkout "${commit_ish}"
+    cd -
+
+}
+# --
+
+function check_version_string() {
+    local version="$1"
+
+    if ! echo "${version}" | grep -qE '^(alpha-|beta-|stable-|lts-)' ; then
+        echo "ERROR: invalid version '${version}', must start with 'alpha-', 'beta-', 'stable-', 'lts-'"
+        exit 1
+    fi
+}
+# --
+
+function update_submodules() {
+    local coreos_git="$1"
+    local portage_git="$2"
+
+    init_submodules
+    update_submodule "coreos-overlay" "${coreos_git}"
+    update_submodule "portage-stable" "${portage_git}"
+}
+# --
+
+function update_and_push_version() {
+    local version="$1"
+
+    # Add and commit local changes
+    git add "sdk_container/src/third_party/coreos-overlay"
+    git add "sdk_container/src/third_party/portage-stable"
+    git add "sdk_container/.repo/manifests/version.txt"
+
+    git commit --allow-empty -m "New version: ${version}"
+
+    git tag -f "${version}"
+
+    if git push origin "${version}" ; then
+        return
+    fi
+    # Push (above) may fail because a tag already exists.
+    #  We check for tag presence, and for the difference
+    #  between local and remote, and bail
+    #  only if the remote / local contents differ.
+
+    # Remove local tag, (re-)fetch remote tags
+    git tag -d "${version}"
+
+    # refresh tree, let origin overwrite local tags
+    git fetch --all --tags --force
+
+    # This will return != 0 if
+    #  - the remote tag does not exist (rc: 127)
+    #  - the remote tag has changes compared to the local tree (rc: 1)
+    git diff --exit-code "${version}"
+}
+# --
+
+function copy_from_buildcache() {
+    local what="$1"
+    local where_to="$2"
+
+    mkdir -p "$where_to"
+    curl --verbose --fail --silent --show-error --location --retry-delay 1 --retry 60 \
+        --retry-connrefused --retry-max-time 60 --connect-timeout 20 \
+        --remote-name --output-dir "${where_to}" "https://${BUILDCACHE_SERVER}/${what}" 
+}
+# --
+
+function gen_sshcmd() {
+    echo -n "ssh -o BatchMode=yes"
+    echo -n " -o StrictHostKeyChecking=no"
+    echo -n " -o UserKnownHostsFile=/dev/null"
+    echo    " -o NumberOfPasswordPrompts=0"
+}
+# --
+
+function copy_to_buildcache() {
+    local remote_path="${BUILDCACHE_PATH_PREFIX}/$1"
+    shift
+
+    local sshcmd="$(gen_sshcmd)"
+
+    $sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \
+        "mkdir -p ${remote_path}"
+
+    rsync -Pav -e "${sshcmd}" "$@" \
+        "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}:${remote_path}"
+}
+# --
+
+function image_exists_locally() {
+    local name="$1"
+    local version="$2"
+    local image="${name}:${version}"
+
+    local image_exists="$(docker images "${image}" \
+                            --no-trunc --format '{{.Repository}}:{{.Tag}}')"
+
+    [ "${image}" = "${image_exists}" ]
+}
+# --
+
+# Derive docker-safe image version string from vernum.
+#
+function vernum_to_docker_image_version() {
+    local vernum="$1"
+    echo "$vernum" | sed 's/[+]/-/g'
+}
+# --
+
+# Return the full name (repo+name+tag) of an image. Useful for SDK images
+#  pulled from the registry (which have the registry pre-pended)
+function docker_image_fullname() {
+    local image="$1"
+    local version="$2"
+
+    docker images --no-trunc --format '{{.Repository}}:{{.Tag}}' \
+        | grep -E "^(${CONTAINER_REGISTRY}/)*${image}:${version}$"
+}
+# --
+
+function docker_image_to_buildcache() {
+    local image="$1"
+    local version="$2"
+
+    # strip potential container registry prefix
+    local tarball="$(basename "$image")-${version}.tar.gz"
+
+    docker save "${image}":"${version}" | $PIGZ -c > "${tarball}"
+    copy_to_buildcache "containers/${version}" "${tarball}"
+}
+# --
+
+function docker_commit_to_buildcache() {
+    local container="$1"
+    local image_name="$2"
+    local image_version="$3"
+
+    docker commit "${container}" "${image_name}:${image_version}"
+    docker_image_to_buildcache "${image_name}" "${image_version}"
+}
+# --
+
+function docker_image_from_buildcache() {
+    local name="$1"
+    local version="$2"
+    local tgz="${name}-${version}.tar.gz"
+
+    if image_exists_locally "${name}" "${version}" ; then
+        return
+    fi
+
+    local url="https://${BUILDCACHE_SERVER}/containers/${version}/${tgz}"
+
+    curl --verbose --fail --silent --show-error --location --retry-delay 1 --retry 60 \
+        --retry-connrefused --retry-max-time 60 --connect-timeout 20 \
+        --remote-name "${url}"
+
+    cat "${tgz}" | $PIGZ -d -c | docker load
+
+    rm "${tgz}"
+}
+# --
+
+function docker_image_from_registry_or_buildcache() {
+    local image="$1"
+    local version="$2"
+
+    if image_exists_locally "${CONTAINER_REGISTRY}/${image}" "${version}" ; then
+        return
+    fi
+
+    if docker pull "${CONTAINER_REGISTRY}/${image}:${version}" ; then
+        return
+    fi
+
+    docker_image_from_buildcache "${image}" "${version}"
+}
+# --

ci-automation/garbage_collect.sh

@@ -0,0 +1,127 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
+#
+# garbage_collect() should be called after sourcing.
+#
+#  OPTIONAL INPUT
+#  - Number of (recent) versions to keep. Defaults to 50.
+#  - DRY_RUN (Env variable). Set to "y" to just list what would be done but not
+#            actually purge anything.
+
+# Flatcar CI automation garbage collector.
+#  This script removes development (non-official) build artefacts:
+#   - SDK tarballs, build step containers, and vendor images on buildcache
+#   - tags from the scripts repository
+#
+#  Garbage collection is based on development (non-official) version tags
+#   in the scripts repo. The newest 50 builds will be retained,
+#   all older builds will be purged (50 is the default, see OPTIONAL INPUT above).
+
+set -eu
+
+function garbage_collect() {
+    local keep="${1:-50}" 
+    local dry_run="${DRY_RUN:-}"
+
+    keep="$((keep + 1))" # for tail -n+...
+    local purge_versions="$(git tag -l --sort=-committerdate \
+            | grep -E '(alpha|beta|stable|lts|sdk)-[0-9]+\.[0-9]+\.[0-9]+\-.*' \
+            | grep -vE '(-pro)$' \
+            | tail -n+"${keep}")"
+
+    source ci-automation/ci_automation_common.sh
+
+    local sshcmd="$(gen_sshcmd)"
+
+    echo 
+    echo "######## The following versions (starting with ${keep}-oldest) will be purged ########"
+    if [ "$dry_run" = "y" ] ; then
+        echo
+        echo "(NOTE this is just a dry run since DRY_RUN=y)"
+        echo
+    fi
+    echo "${purge_versions}" | sed 's/ /\n/g'
+    echo 
+    echo 
+
+    local version
+    for version in ${purge_versions}; do
+        echo "--------------------------------------------"
+        echo 
+        echo "#### Processing version '${version}' ####"
+        echo
+
+        git checkout "${version}" -- sdk_container/.repo/manifests/version.txt
+        source sdk_container/.repo/manifests/version.txt
+
+        local os_docker_vernum="$(vernum_to_docker_image_version "${FLATCAR_VERSION}")"
+        local sdk_vernum="${FLATCAR_SDK_VERSION}"
+        local sdk_docker_vernum="$(vernum_to_docker_image_version "${FLATCAR_SDK_VERSION}")"
+
+        # Remove container image tarballs and SDK tarball (if applicable)
+        #
+        local rmpat=""
+        if [[ "${version}" = 'sdk-'* ]] ; then
+            echo "## ${version} is an SDK version. ##"
+            rmpat="${BUILDCACHE_PATH_PREFIX}/sdk/*/${sdk_vernum}/"
+            rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${sdk_docker_vernum}/flatcar-sdk-*"
+        else
+            echo "## ${version} is an OS image version. ##"
+            rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-packages-*"
+            rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-image-*"
+            rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/images/*/${os_docker_vernum}/"
+        fi
+
+        echo "## The following files will be removed ##"
+        $sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \
+            "ls -la ${rmpat} || true"
+
+        if [ "$dry_run" != "y" ] ; then
+            set -x
+            $sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \
+                "rm -rf ${rmpat}"
+            set +x
+        else
+            echo "## (DRY_RUN=y so not doing anything) ##"
+        fi
+
+        # Remove container image directory if empty
+        #
+        if [[ "${version}" = 'sdk-'* ]] ; then
+            rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${sdk_docker_vernum}/"
+        else
+            rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/"
+        fi
+
+        echo "## Checking if container directory is empty and can be removed (it's OK if this fails) ##"
+        echo "## The following directory will be removed if below output is empty: '${rmpat}' ##"
+        $sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \
+            "ls -la ${rmpat} || true"
+
+        if [ "$dry_run" != "y" ] ; then
+            set -x
+            $sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \
+                "rmdir ${rmpat} || true"
+            set +x
+        else
+            echo "## (DRY_RUN=y so not doing anything) ##"
+        fi
+
+        # Remove git tag (local and remote)
+        #
+        echo "## The following TAG will be deleted: '${version}' ##"
+        if [ "$dry_run" != "y" ] ; then
+            set -x
+            git tag -d "${version}"
+            git push --delete origin "${version}"
+            set +x
+        else
+            echo "## (DRY_RUN=y so not doing anything) ##"
+        fi
+    done
+}

ci-automation/image.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
+#
+# packages_build() should be called w/ the positional INPUT parameters below.
+
+# Binary OS image build automation stub.
+#   This script will build the OS image from a pre-built packages container.
+#
+# PREREQUISITES:
+#
+#   1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt
+#   2. Scripts repo version tag of OS image version to be built is available and checked out.
+#   3. Flatcar packages container is available via build cache server
+#       from "/containers/[VERSION]/flatcar-packages-[ARCH]-[FLATCAR_VERSION].tar.gz"
+#       or present locally. Container must contain binary packages and torcx artefacts.
+#
+# INPUT:
+#
+#   1. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64").
+#
+# OUTPUT:
+#
+#   1. Exported container image with OS image, dev container, and related artefacts at
+#        /home/sdk/image/[ARCH], torcx packages at /home/sdk/torcx
+#        named "flatcar-images-[ARCH]-[FLATCAR_VERSION].tar.gz"
+#        pushed to buildcache.
+#   2. "./ci-cleanup.sh" with commands to clean up temporary build resources,
+#        to be run after this step finishes / when this step is aborted.
+
+set -eu
+
+function image_build() {
+    local arch="$1"
+
+    source ci-automation/ci_automation_common.sh
+    init_submodules
+
+    source sdk_container/.repo/manifests/version.txt
+    local vernum="${FLATCAR_VERSION}"
+    local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
+
+    local packages="flatcar-packages-${arch}"
+    local packages_image="${packages}:${docker_vernum}"
+
+    docker_image_from_buildcache "${packages}" "${docker_vernum}"
+
+    local image="flatcar-images-${arch}"
+    local image_container="${image}-${docker_vernum}"
+
+    # build image and store it in the container
+    ./run_sdk_container -x ./ci-cleanup.sh -n "${image_container}" -C "${packages_image}" \
+            -v "${vernum}" \
+            mkdir -p "${CONTAINER_IMAGE_ROOT}"
+    ./run_sdk_container -n "${image_container}" -C "${packages_image}" \
+            -v "${vernum}" \
+            ./build_image --board="${arch}-usr" \
+                          --output_root="${CONTAINER_IMAGE_ROOT}" \
+                          --torcx_root="${CONTAINER_TORCX_ROOT}" prodtar container
+
+    # rename container and push to build cache
+    docker_commit_to_buildcache "${image_container}" "${image}" "${docker_vernum}"
+}
+# --

ci-automation/packages.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
+#
+# packages_build() should be called w/ the positional INPUT parameters below.
+
+# OS image binary packages build automation stub.
+#   This script will use an SDK container to build packages for an OS image.
+#   It will update the versionfile with the OS packages version built,
+#    and will add a version tag (see INPUT) to the scripts repo. 
+#
+# PREREQUISITES:
+#
+#   1. SDK version is recorded in sdk_container/.repo/manifests/version.txt
+#   2. SDK container is either
+#       - available via ghcr.io/flatcar-linux/flatcar-sdk-[ARCH]:[VERSION] (official SDK release)
+#       OR
+#       - available via build cache server "/containers/[VERSION]/flatcar-sdk-[ARCH]-[VERSION].tar.gz"
+#         (dev SDK)
+#
+# INPUT:
+#
+#   1. Version of the TARGET OS image to build (string).
+#       The version pattern '(alpha|beta|stable|lts)-MMMM.m.p' (e.g. 'alpha-3051.0.0')
+#         denotes a "official" build, i.e. a release build to be published.
+#       Use any version diverging from the pattern (e.g. 'alpha-3051.0.0-nightly-4302') for development / CI builds.
+#       A tag of this version will be created in the scripts repo and pushed upstream.
+#
+#   2. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64").
+#
+#
+# OPTIONAL INPUT:
+#
+#   3. coreos-overlay repository tag to use (commit-ish).
+#       Optional - use scripts repo sub-modules as-is if not set.
+#       This version will be checked out / pulled from remote in the coreos-overlay git submodule.
+#       The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed.
+#
+#   4. portage-stable repository tag to use (commit-ish).
+#       Optional - use scripts repo sub-modules as-is if not set.
+#       This version will be checked out / pulled from remote in the portage-stable git submodule.
+#       The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed.
+#
+# OUTPUT:
+#
+#   1. Exported container image "flatcar-packages-[ARCH]-[VERSION].tar.gz" with binary packages
+#       pushed to buildcache.
+#   2. Updated scripts repository
+#        - version tag w/ submodules
+#        - sdk_container/.repo/manifests/version.txt denotes new FLATCAR OS version
+#   3. "./ci-cleanup.sh" with commands to clean up temporary build resources,
+#        to be run after this step finishes / when this step is aborted.
+
+
+set -eu
+
+function packages_build() {
+    local version="$1"
+    local arch="$2"
+    local coreos_git="${3:-}"
+    local portage_git="${4:-}"
+
+    source ci-automation/ci_automation_common.sh
+    init_submodules
+
+    check_version_string "${version}"
+
+    source sdk_container/.repo/manifests/version.txt
+    local sdk_version="${FLATCAR_SDK_VERSION}"
+
+    if [ -n "${coreos_git}" ] ; then
+        update_submodule "coreos-overlay" "${coreos_git}"
+    fi
+    if [ -n "${portage_git}" ] ; then
+        update_submodule "portage-stable" "${portage_git}"
+    fi
+
+    # Get SDK from either the registry or import from build cache
+    # This is a NOP if the image is present locally.
+    local sdk_name="flatcar-sdk-${arch}"
+    local docker_sdk_vernum="$(vernum_to_docker_image_version "${sdk_version}")"
+
+    docker_image_from_registry_or_buildcache "${sdk_name}" "${docker_sdk_vernum}"
+    local sdk_image="$(docker_image_fullname "${sdk_name}" "${docker_sdk_vernum}")"
+    echo "docker image rm -f '${sdk_image}'" >> ./ci-cleanup.sh
+
+    # Set name of the packages container for later rename / export
+    local vernum="${version#*-}" # remove alpha-,beta-,stable-,lts- version tag
+    local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
+    local packages_container="flatcar-packages-${arch}-${docker_vernum}"
+
+    # Build packages; store packages and torcx output in container
+    ./run_sdk_container -x ./ci-cleanup.sh -n "${packages_container}" -v "${version}" \
+        -C "${sdk_image}" \
+        mkdir -p "${CONTAINER_TORCX_ROOT}"
+    ./run_sdk_container -n "${packages_container}" -v "${version}" \
+        -C "${sdk_image}" \
+        ./build_packages --board="${arch}-usr" \
+            --torcx_output_root="${CONTAINER_TORCX_ROOT}"
+
+    # run_sdk_container updates the version file, use that version from here on
+    source sdk_container/.repo/manifests/version.txt
+    local vernum="${FLATCAR_VERSION}"
+    local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
+    local packages_image="flatcar-packages-${arch}"
+
+    # generate image + push to build cache
+    docker_commit_to_buildcache "${packages_container}" "${packages_image}" "${docker_vernum}"
+
+    update_and_push_version "${version}"
+}
+# --

ci-automation/sdk_bootstrap.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
+#
+# sdk_bootstrap() should be called w/ the positional INPUT parameters below.
+
+# Boostrap SDK build automation stub.
+#  This script will use a seed SDK container + tarball to bootstrap a 
+#   new SDK tarball.
+#
+# INPUT:
+#
+#   1. Version of the SEED SDK to use (string).
+#       The seed SDK tarball must be available on https://mirror.release.flatcar-linux.net/sdk/ ...
+#       The seed SDK container must be available from https://github.com/orgs/flatcar-linux/packages
+#          (via ghcr.io/flatcar-linux/flatcar-sdk-all:[VERSION]).
+#
+#   2. Version of the TARGET SDK to build (string).
+#       The version pattern 'MMMM.m.p' (e.g. '3051.0.0') denotes a "official" build, i.e. a release build to be published.
+#       Use any version diverging from the pattern (e.g. '3051.0.0-nightly-4302') for development / CI builds.
+#       A tag "sdk-[VERSION]" will be created in the scripts repo and pushed upstream.
+#
+# OPTIONAL INPUT:
+#
+#   3. coreos-overlay repository tag to use (commit-ish).
+#       This version will be checked out / pulled from remote in the coreos-overlay git submodule.
+#       The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed.
+#       Leave empty to use coreos-overlay as-is.
+#
+#   4. portage-stable repository tag to use (commit-ish).
+#       This version will be checked out / pulled from remote in the portage-stable git submodule.
+#       The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed.
+#       Leave empty to use portage-stable as-is.
+#
+#   5. ARCH. Environment variable. Target architecture for the SDK to run on.
+#        Either "amd64" or "arm64"; defaults to "amd64" if not set.
+#
+# OUTPUT:
+#
+#   1. SDK tarball (gentoo catalyst output) of the new SDK, pushed to buildcache.
+#   2. Updated scripts repository
+#        - version tag w/ submodules
+#        - sdk_container/.repo/manifests/version.txt denotes new SDK version
+#   3. "./ci-cleanup.sh" with commands to clean up temporary build resources,
+#        to be run after this step finishes / when this step is aborted.
+
+set -eu
+
+function sdk_bootstrap() {
+    local seed_version="$1"
+    local version="$2"
+    local coreos_git="$3"
+    local portage_git="$4"
+    : ${ARCH:="amd64"}
+
+    source ci-automation/ci_automation_common.sh
+    init_submodules
+
+    check_version_string "${version}"
+
+    if [ -n "${coreos_git}" ] ; then
+        update_submodule "coreos-overlay" "${coreos_git}"
+    fi
+    if [ -n "${portage_git}" ] ; then
+        update_submodule "portage-stable" "${portage_git}"
+    fi
+
+    local vernum="${version#*-}" # remove alpha-,beta-,stable-,lts- version tag
+    local git_vernum="${vernum}"
+
+    # This will update FLATCAR_VERSION[_ID] and BUILD_ID in versionfile
+    ./bootstrap_sdk_container -x ./ci-cleanup.sh "${seed_version}" "${vernum}"
+
+    # push SDK tarball to buildcache
+    source sdk_container/.repo/manifests/version.txt
+    local vernum="${FLATCAR_SDK_VERSION}"
+    local dest_tarball="flatcar-sdk-${ARCH}-${vernum}.tar.bz2"
+
+    cd "__build__/images/catalyst/builds/flatcar-sdk"
+    copy_to_buildcache "sdk/${ARCH}/${vernum}" "${dest_tarball}"*
+    cd -
+
+    # Create new tag in scripts repo w/ updated versionfile + submodules
+    update_and_push_version "sdk-${git_vernum}"
+}
+# --

ci-automation/sdk_container.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
+#
+# sdk_container_build() should be called w/ the positional INPUT parameters below.
+
+# SDK container builder automation stub.
+#  This script will build an SDK container w/ board support from an SDK tarball.
+#  NOTE that SDK tarball and scripts repo must have the same version or building
+#  the SDK container will fail.
+#
+# PREREQUISITES:
+#
+#   1. SDK version is recorded in sdk_container/.repo/manifests/version.txt and a matching
+#       SDK tarball is available on BUILDCACHE/sdk/[ARCH]/[VERSION]/flatcar-sdk-[ARCH]-[VERSION].tar.bz2
+#
+# OPTIONAL INPUT:
+
+#   2. ARCH. Environment variable. Target architecture for the SDK to run on.
+#        Either "amd64" or "arm64"; defaults to "amd64" if not set.
+#
+# OUTPUT:
+#
+#   1. SDK container image of the new SDK, published to buildcache.
+#   2. "./ci-cleanup.sh" with commands to clean up temporary build resources,
+#        to be run after this step finishes / when this step is aborted.
+
+set -eu
+
+function sdk_container_build() {
+    : ${ARCH:="amd64"}
+
+    source ci-automation/ci_automation_common.sh
+
+    init_submodules
+
+    source sdk_container/.repo/manifests/version.txt
+    local vernum="${FLATCAR_SDK_VERSION}"
+    local sdk_tarball="flatcar-sdk-${ARCH}-${vernum}.tar.bz2"
+
+    # __build__ is in .dockerignore, so the tarball is excluded from build context
+    mkdir -p __build__
+    copy_from_buildcache "sdk/${ARCH}/${vernum}/${sdk_tarball}" "./__build__"
+
+
+    # This will update the SDK_VERSION in versionfile
+    ./build_sdk_container_image -x ./ci-cleanup.sh ./__build__/"${sdk_tarball}"
+
+    # push artefacts to build cache
+    local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
+    docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-all" "${docker_vernum}"
+    docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-amd64" "${docker_vernum}"
+    docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-arm64" "${docker_vernum}"
+}
+# --

ci-automation/util/Dockerfile.pigz

@@ -0,0 +1,4 @@
+FROM alpine
+RUN apk add pigz
+ENTRYPOINT [ "/usr/bin/pigz" ]
+CMD [ "-h" ]

ci-automation/vms.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
+#
+# packages_build() should be called w/ the positional INPUT parameters below.
+
+# Vendor images build automation stub.
+#   This script will build one or more vendor images ("vm") using a pre-built image container.
+#
+# PREREQUISITES:
+#
+#   1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt
+#   2. Scripts repo version tag of OS image version to be built is available and checked out.
+#   3. Flatcar image container is available via build cache server
+#       from "/containers/[VERSION]/flatcar-images-[ARCH]-[FLATCAR_VERSION].tar.gz"
+#       or present locally. Must contain packages and image.
+#
+# INPUT:
+#
+#   1. Architecture (ARCH) of the TARGET vm images ("arm64", "amd64").
+#   2. Image formats to be built. Can be multiple, separated by spaces.
+#      Run ./image_to_vm.sh -h in the SDK to get a list of supported images.
+#
+# OUTPUT:
+#
+#   1. Exported VM image(s), pushed to buildcache ( images/[ARCH]/[FLATCAR_VERSION]/ )
+#   2. "./ci-cleanup.sh" with commands to clean up temporary build resources,
+#        to be run after this step finishes / when this step is aborted.
+
+set -eu
+
+function vm_build() {
+    local arch="$1"
+    shift
+    # $@ now contains image formats to build
+
+    source ci-automation/ci_automation_common.sh
+    init_submodules
+
+    source sdk_container/.repo/manifests/version.txt
+    local vernum="${FLATCAR_VERSION}"
+    local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
+
+    local image="flatcar-images-${arch}"
+    local image_image="${image}:${docker_vernum}"
+    local vms_container="flatcar-vms-${docker_vernum}"
+
+    docker_image_from_buildcache "${image}" "${docker_vernum}"
+
+    # clean up dangling containers from previous builds
+    docker container rm -f "${vms_container}" || true
+
+    local images_out="images/"
+    rm -rf "${images_out}"
+
+    echo "docker container rm -f '${vms_container}'" >> ci-cleanup.sh
+
+    for format in "${@}"; do
+        echo " ###################  VENDOR '${format}' ################### "
+        ./run_sdk_container -n "${vms_container}" -C "${image_image}" \
+            -v "${vernum}" \
+            ./image_to_vm.sh --format "${format}" --board="${arch}-usr" \
+                --from "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest"
+    done
+
+    # copy resulting images + push to buildcache
+    ./run_sdk_container -n "${vms_container}" \
+        -v "${vernum}" \
+        cp -R "${CONTAINER_IMAGE_ROOT}/${arch}-usr/" "./${images_out}/"
+
+    cd "images/latest"
+    copy_to_buildcache "images/${arch}/${vernum}/" *
+}
+# --