From 350b884067f155f8903a8a4336c2789205fd021e Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Tue, 5 Jul 2022 17:56:05 +0200
Subject: [PATCH] Copy new pipeline stuff from main branch

Synced to commit 25883519d971d3670f1e188b608da0d13a5e4a84.
---
 .dockerignore                                 |   2 +
 .gitmodules                                   |   6 +
 bootstrap_sdk_container                       |  79 +++
 build_library/build_image_util.sh             |  38 +-
 build_library/dev_container_util.sh           |  16 +-
 build_library/modify_image_util.sh            |  16 +-
 build_library/prod_image_util.sh              |  14 +-
 build_library/release_util.sh                 | 121 ++++-
 build_library/toolchain_util.sh               |   4 +
 build_library/vm_image_util.sh                |  65 +--
 build_packages                                |   2 +-
 build_sdk_container_image                     | 251 ++++++++++
 checkout                                      | 178 +++++++
 ci-automation/README.md                       | 165 +++++++
 ci-automation/ci-config.env                   | 131 +++++
 ci-automation/ci_automation_common.sh         | 456 ++++++++++++++++++
 ci-automation/garbage_collect.sh              | 156 ++++++
 ci-automation/garbage_collect_cloud.sh        |  19 +
 ci-automation/gpg_setup.sh                    |  31 ++
 ci-automation/image.sh                        | 113 +++++
 ci-automation/image_changes.sh                | 122 +++++
 ci-automation/packages-tag.sh                 | 120 +++++
 ci-automation/packages.sh                     | 136 ++++++
 ci-automation/push_pkgs.sh                    |  99 ++++
 ci-automation/python-bin/python3              |   6 +
 ci-automation/release.sh                      | 405 ++++++++++++++++
 ci-automation/sdk_bootstrap.sh                | 152 ++++++
 ci-automation/sdk_container.sh                |  79 +++
 ci-automation/tapfile_helper_lib.sh           | 308 ++++++++++++
 ci-automation/test.sh                         | 252 ++++++++++
 ci-automation/test_update_reruns.sh           |  26 +
 ci-automation/util/Dockerfile.pigz            |   4 +
 ci-automation/util/fetch_image.sh             |  73 +++
 ci-automation/vendor-testing/aws.sh           |  82 ++++
 ci-automation/vendor-testing/azure.sh         |  90 ++++
 ci-automation/vendor-testing/digitalocean.sh  |  57 +++
 ci-automation/vendor-testing/equinix_metal.sh |  65 +++
 ci-automation/vendor-testing/gce.sh           |  82 ++++
 ci-automation/vendor-testing/qemu.sh          |  58 +++
 ci-automation/vendor-testing/qemu_uefi.sh     |   1 +
 ci-automation/vendor-testing/qemu_update.sh   |  64 +++
 ci-automation/vendor-testing/vmware.sh        |  68 +++
 ci-automation/vendor_test.sh                  | 391 +++++++++++++++
 ci-automation/vms.sh                          | 139 ++++++
 retag-for-jenkins                             |  40 ++
 run_sdk_container                             | 150 ++++++
 .../.repo/manifests/mantle-container          |   1 +
 sdk_container/.repo/manifests/os-release      |   6 +
 sdk_container/.repo/manifests/version.txt     |   4 +
 .../git-override/.git-coreos-overlay          |   1 +
 .../git-override/.git-portage-stable          |   1 +
 sdk_container/git-override/README.md          |   5 +
 sdk_container/src/third_party/coreos-overlay  |   1 +
 sdk_container/src/third_party/portage-stable  |   1 +
 sdk_lib/90_env_keep                           |   9 +
 sdk_lib/Dockerfile.lean-arch                  |  29 ++
 sdk_lib/Dockerfile.sdk-build                  |  17 +
 sdk_lib/Dockerfile.sdk-import                 |  58 +++
 sdk_lib/Dockerfile.sdk-update                 |  21 +
 sdk_lib/sdk_container_common.sh               | 291 +++++++++++
 sdk_lib/sdk_entry.sh                          |  73 +++
 sdk_lib/sdk_init_selfcontained.sh             |  36 ++
 settings.env                                  |   8 +
 update_sdk_container_image                    | 100 ++++
 64 files changed, 5542 insertions(+), 52 deletions(-)
 create mode 100644 .dockerignore
 create mode 100644 .gitmodules
 create mode 100755 bootstrap_sdk_container
 create mode 100755 build_sdk_container_image
 create mode 100755 checkout
 create mode 100644 ci-automation/README.md
 create mode 100644 ci-automation/ci-config.env
 create mode 100644 ci-automation/ci_automation_common.sh
 create mode 100644 ci-automation/garbage_collect.sh
 create mode 100755 ci-automation/garbage_collect_cloud.sh
 create mode 100644 ci-automation/gpg_setup.sh
 create mode 100644 ci-automation/image.sh
 create mode 100644 ci-automation/image_changes.sh
 create mode 100644 ci-automation/packages-tag.sh
 create mode 100644 ci-automation/packages.sh
 create mode 100644 ci-automation/push_pkgs.sh
 create mode 100755 ci-automation/python-bin/python3
 create mode 100644 ci-automation/release.sh
 create mode 100644 ci-automation/sdk_bootstrap.sh
 create mode 100644 ci-automation/sdk_container.sh
 create mode 100644 ci-automation/tapfile_helper_lib.sh
 create mode 100644 ci-automation/test.sh
 create mode 100755 ci-automation/test_update_reruns.sh
 create mode 100644 ci-automation/util/Dockerfile.pigz
 create mode 100755 ci-automation/util/fetch_image.sh
 create mode 100755 ci-automation/vendor-testing/aws.sh
 create mode 100755 ci-automation/vendor-testing/azure.sh
 create mode 100755 ci-automation/vendor-testing/digitalocean.sh
 create mode 100755 ci-automation/vendor-testing/equinix_metal.sh
 create mode 100755 ci-automation/vendor-testing/gce.sh
 create mode 100755 ci-automation/vendor-testing/qemu.sh
 create mode 120000 ci-automation/vendor-testing/qemu_uefi.sh
 create mode 100755 ci-automation/vendor-testing/qemu_update.sh
 create mode 100755 ci-automation/vendor-testing/vmware.sh
 create mode 100644 ci-automation/vendor_test.sh
 create mode 100644 ci-automation/vms.sh
 create mode 100755 retag-for-jenkins
 create mode 100755 run_sdk_container
 create mode 100644 sdk_container/.repo/manifests/mantle-container
 create mode 100644 sdk_container/.repo/manifests/os-release
 create mode 100644 sdk_container/.repo/manifests/version.txt
 create mode 100644 sdk_container/git-override/.git-coreos-overlay
 create mode 100644 sdk_container/git-override/.git-portage-stable
 create mode 100644 sdk_container/git-override/README.md
 create mode 160000 sdk_container/src/third_party/coreos-overlay
 create mode 160000 sdk_container/src/third_party/portage-stable
 create mode 100644 sdk_lib/90_env_keep
 create mode 100644 sdk_lib/Dockerfile.lean-arch
 create mode 100644 sdk_lib/Dockerfile.sdk-build
 create mode 100644 sdk_lib/Dockerfile.sdk-import
 create mode 100644 sdk_lib/Dockerfile.sdk-update
 create mode 100644 sdk_lib/sdk_container_common.sh
 create mode 100755 sdk_lib/sdk_entry.sh
 create mode 100755 sdk_lib/sdk_init_selfcontained.sh
 create mode 100644 settings.env
 create mode 100755 update_sdk_container_image

diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000000..d7d526ae8e
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,2 @@
+__build__
+sdk_container/.cache
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000000..e026cc5d93
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,6 @@
+[submodule "sdk_container/src/third_party/coreos-overlay"]
+	path = sdk_container/src/third_party/coreos-overlay
+	url = https://github.com/flatcar/coreos-overlay.git
+[submodule "sdk_container/src/third_party/portage-stable"]
+	path = sdk_container/src/third_party/portage-stable
+	url = https://github.com/flatcar/portage-stable.git
diff --git a/bootstrap_sdk_container b/bootstrap_sdk_container
new file mode 100755
index 0000000000..6384e69e49
--- /dev/null
+++ b/bootstrap_sdk_container
@@ -0,0 +1,79 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -eu
+
+cd $(dirname "$0")
+source sdk_lib/sdk_container_common.sh
+
+seed_version=""
+target_version=""
+
+declare -a cleanup
+
+# --
+
+usage() {
+    echo "  Usage:"
+    echo "  $0 <seed-sdk-version> <new-sdk-version> [-x <cleanup-script>]"
+    echo
+    echo "      This script will bootstrap a new SDK tarball using an SDK container."
+    echo "      '$sdk_container_common_versionfile' will be updated to the target version."
+    echo
+    echo "       <seed-sdk-vernum>  - SDK version number (e.g. '3005.0.0') to use for bootstrapping."
+    echo "                             The SDK container will be pulled and the tarball"
+    echo "                             downloaded if necessary."
+    echo "       <new-sdk-vernum>   - SDK version number (e.g. '3027.0.0') of the new SDK."
+    echo "      -x <cleanup-script> - For each resource generated during build (container etc.)"
+    echo "                             add a cleanup line to <script> which, when run, will free"
+    echo "                             the resource. Useful for CI."
+    echo "      -h                  - Print this help."
+    echo
+}
+# --
+
+while [ 0 -lt $# ] ; do
+    case "$1" in
+    -h) usage; exit 0;;
+    -x) cleanup=("-x" "$2"); shift; shift;;
+    *)  if [ -z "$seed_version" ] ; then
+            seed_version="$1"
+        elif [ -z "$target_version" ] ; then
+            target_version="$1"
+        else
+            echo "ERROR: Spurious positional parameter '$1'"
+            usage; exit 1;
+        fi
+        shift;;
+    esac
+done
+
+if [ -z "$seed_version" -o -z "$target_version" ] ; then
+    echo "ERROR: Missing seed and /or target SDK version."
+    usage
+    exit 1
+fi
+# --
+
+vernum="$(strip_version_prefix "$target_version")"
+if is_official "$vernum" ; then
+    official="true"
+else
+    official="false"
+fi
+
+yell "\n######\n###### Bootstrapping SDK version $target_version from seed ($seed_version)"
+
+if $official; then
+    export COREOS_OFFICIAL=1
+fi
+
+# bootstrap_sdk needs FLATCAR_SDK_VERSION set to the seed version
+./run_sdk_container "${cleanup[@]}" -V "$seed_version" -v "$target_version" \
+          sudo -E ./bootstrap_sdk
+
+# Update versionfile to the actual SDK version
+create_versionfile "${target_version}"
diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh
index af6ace6787..6d7cd4836d 100755
--- a/build_library/build_image_util.sh
+++ b/build_library/build_image_util.sh
@@ -58,10 +58,33 @@ extract_update() {
   local image_name="$1"
   local disk_layout="$2"
   local update_path="${BUILD_DIR}/${image_name%_image.bin}_update.bin"
+  local digest_path="${update_path}.DIGESTS"
 
   "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout="${disk_layout}" \
     extract "${BUILD_DIR}/${image_name}" "USR-A" "${update_path}"
-  upload_image "${update_path}"
+
+  # Compress image
+  files_to_evaluate+=( "${update_path}" )
+  declare -a compressed_images
+  declare -a extra_files
+  compress_disk_images files_to_evaluate compressed_images extra_files
+
+  # Upload compressed image
+  upload_image -d "${digest_path}" "${compressed_images[@]}" "${extra_files[@]}"
+
+  # Upload legacy digests
+  upload_legacy_digests "${digest_path}" compressed_images
+
+  # For production as well as dev builds we generate a dev-key-signed update
+  # payload for running tests (the signature won't be accepted by production systems).
+  local update_test="${BUILD_DIR}/flatcar_test_update.gz"
+  delta_generator \
+      -private_key "/usr/share/update_engine/update-payload-key.key.pem" \
+      -new_image "${update_path}" \
+      -new_kernel "${BUILD_DIR}/${image_name%.bin}.vmlinuz" \
+      -out_file "${update_test}"
+
+  upload_image "${update_test}"
 }
 
 zip_update_tools() {
@@ -94,7 +117,18 @@ generate_update() {
       -new_kernel "${image_kernel}" \
       -out_file "${update}.gz"
 
-  upload_image -d "${update}.DIGESTS" "${update}".{bin,gz,zip}
+  # Compress image
+  declare -a files_to_evaluate
+  declare -a compressed_images
+  declare -a extra_files
+  files_to_evaluate+=( "${update}.bin" )
+  compress_disk_images files_to_evaluate compressed_images extra_files
+
+  # Upload images
+  upload_image -d "${update}.DIGESTS" "${update}".{gz,zip} "${compressed_images[@]}" "${extra_files[@]}"
+
+  # Upload legacy digests
+  upload_legacy_digests "${update}.DIGESTS" compressed_images
 }
 
 # ldconfig cannot generate caches for non-native arches.
diff --git a/build_library/dev_container_util.sh b/build_library/dev_container_util.sh
index 9e8f9d8346..34a028baef 100755
--- a/build_library/dev_container_util.sh
+++ b/build_library/dev_container_util.sh
@@ -108,9 +108,21 @@ create_dev_container() {
   systemd_enable "${root_fs_dir}" "multi-user.target" "remount-usr.service"
 
   finish_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${image_contents}"
-  upload_image -d "${BUILD_DIR}/${image_name}.bz2.DIGESTS" \
+
+  declare -a files_to_evaluate
+  declare -a compressed_images
+  declare -a extra_files
+
+  files_to_evaluate+=( "${BUILD_DIR}/${image_name}" )
+  compress_disk_images files_to_evaluate compressed_images extra_files
+
+  upload_image -d "${BUILD_DIR}/${image_name}.DIGESTS" \
       "${BUILD_DIR}/${image_contents}" \
       "${BUILD_DIR}/${image_packages}" \
       "${BUILD_DIR}/${image_licenses}" \
-      "${BUILD_DIR}/${image_name}"
+      "${compressed_images[@]}" \
+      "${extra_files[@]}"
+
+  # Upload legacy digests
+  upload_legacy_digests "${BUILD_DIR}/${image_name}.DIGESTS" compressed_images
 }
diff --git a/build_library/modify_image_util.sh b/build_library/modify_image_util.sh
index f8ed535762..53fa0cc81e 100755
--- a/build_library/modify_image_util.sh
+++ b/build_library/modify_image_util.sh
@@ -83,7 +83,21 @@ finish_modify_image() {
     cleanup_mounts "${ROOT_FS_DIR}"
     trap - EXIT
 
-    upload_image "${DST_IMAGE}"
+
+    declare -a files_to_evaluate
+    declare -a compressed_images
+    declare -a extra_files
+
+    files_to_evaluate+=( "${DST_IMAGE}" )
+    compress_disk_images files_to_evaluate compressed_images extra_files
+
+    upload_image -d "${DST_IMAGE}.DIGESTS" \
+        "${compressed_images[@]}" \
+        "${extra_files[@]}"
+
+    # Upload legacy digests
+    upload_legacy_digests "${DST_IMAGE}.DIGESTS" compressed_images
+
     for filename in "${EXTRA_FILES[@]}"; do
         if [[ -e "${BUILD_DIR}/${filename}" ]]; then
             upload_image "${BUILD_DIR}/${filename}"
diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh
index 509e350aa4..935c9aaf49 100755
--- a/build_library/prod_image_util.sh
+++ b/build_library/prod_image_util.sh
@@ -138,17 +138,27 @@ EOF
     "${BUILD_DIR}/${image_contents}"
     "${BUILD_DIR}/${image_packages}"
     "${BUILD_DIR}/${image_licenses}"
-    "${BUILD_DIR}/${image_name}"
     "${BUILD_DIR}/${image_kernel}"
     "${BUILD_DIR}/${image_pcr_policy}"
     "${BUILD_DIR}/${image_grub}"
     "${BUILD_DIR}/${image_kconfig}"
   )
+
+  local files_to_evaluate=( "${BUILD_DIR}/${image_name}" )
+  declare -a compressed_images
+  declare -a extra_files
+  compress_disk_images files_to_evaluate compressed_images extra_files
+  to_upload+=( "${compressed_images[@]}" )
+  to_upload+=( "${extra_files[@]}" )
+
   # FIXME(bgilbert): no shim on arm64
   if [[ -f "${BUILD_DIR}/${image_shim}" ]]; then
     to_upload+=("${BUILD_DIR}/${image_shim}")
   fi
-  upload_image -d "${BUILD_DIR}/${image_name}.bz2.DIGESTS" "${to_upload[@]}"
+  upload_image -d "${BUILD_DIR}/${image_name}.DIGESTS" "${to_upload[@]}"
+
+  # Upload legacy digests
+  upload_legacy_digests "${BUILD_DIR}/${image_name}.DIGESTS" compressed_images
 }
 
 create_prod_tar() {
diff --git a/build_library/release_util.sh b/build_library/release_util.sh
index 47964a8f3d..7cb054ba0f 100644
--- a/build_library/release_util.sh
+++ b/build_library/release_util.sh
@@ -7,6 +7,7 @@ UPLOAD_ROOT=
 UPLOAD_PATH=
 TORCX_UPLOAD_ROOT=
 UPLOAD_DEFAULT=${FLAGS_FALSE}
+DEFAULT_IMAGE_COMPRESSION_FORMAT="bz2"
 
 # Default upload root can be overridden from the environment.
 _user="${USER}"
@@ -15,9 +16,6 @@ _user="${USER}"
 : ${FLATCAR_TORCX_UPLOAD_ROOT:=${FLATCAR_UPLOAD_ROOT}/torcx}
 unset _user
 
-IMAGE_ZIPPER="lbzip2 --compress --keep"
-IMAGE_ZIPEXT=".bz2"
-
 DEFINE_boolean parallel ${FLAGS_TRUE} \
   "Enable parallelism in gsutil."
 DEFINE_boolean upload ${UPLOAD_DEFAULT} \
@@ -42,6 +40,113 @@ DEFINE_string sign "" \
   "Sign all files to be uploaded with the given GPG key."
 DEFINE_string sign_digests "" \
   "Sign image DIGESTS files with the given GPG key."
+DEFINE_string image_compression_formats "${DEFAULT_IMAGE_COMPRESSION_FORMAT}" \
+  "Compress the resulting images using thise formats. This option acceps a list of comma separated values. Options are: none, bz2, gz, zip, zst"
+DEFINE_boolean only_store_compressed ${FLAGS_TRUE} \
+  "Delete input file when compressing, except when 'none' is part of the compression formats or the generic image is the input"
+
+
+compress_file() {
+    local filepath="$1"
+    local compression_format="$2"
+
+    [ ! -f "${filepath}" ] && die "Image file ${filepath} does not exist"
+    [ -z "${compression_format}" ] && die "compression format parameter is mandatory"
+
+    case "${compression_format}" in
+    "none"|"")
+        echo -n "${filepath}"
+        return 0
+        ;;
+    "bz2")
+        IMAGE_ZIPPER="lbzip2 --compress --keep"
+        ;;
+    "gz")
+        IMAGE_ZIPPER="pigz --keep"
+        ;;
+    "zip")
+        IMAGE_ZIPPER="pigz --keep --zip"
+        ;;
+    "zst")
+       IMAGE_ZIPPER="zstd --format=zstd -k -q --no-progress"
+       ;;
+    *)
+        die "Unsupported compression format ${compression_format}"
+        ;;
+    esac
+
+    ${IMAGE_ZIPPER} -f "${filepath}" 2>&1 >/dev/null || die "failed to compress ${filepath}"
+
+    echo -n "${filepath}.${compression_format}"
+}
+
+compress_disk_images() {
+    # An array of files that are to be evaluated and possibly compressed if images are
+    # among them.
+    local -n local_files_to_evaluate="$1"
+
+    # An array that will hold the path on disk to the resulting disk image archives.
+    # Multiple compression formats may be requested, so this array may hold
+    # multiple archives for the same image.
+    local -n local_resulting_archives="$2"
+
+    # Files that did not match the filter for disk images.
+    local -n local_extra_files="$3"
+
+    info "Compressing images"
+    # We want to compress images, but we also want to remove the uncompressed files
+    # from the list of uploadable files.
+    for filename in "${local_files_to_evaluate[@]}"; do
+        if [[ "${filename}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then
+            # Parse the formats as an array. This will yield an extra empty
+            # array element at the end.
+            readarray -td, FORMATS<<<"${FLAGS_image_compression_formats},"
+            # unset the last element
+            unset 'FORMATS[-1]'
+
+            # An associative array we set an element on whenever we process a format.
+            # This way we don't process the same format twice. A unique for array elements.
+            declare -A processed_format
+            for format in "${FORMATS[@]}";do
+                if [ -z "${processed_format[${format}]}" ]; then
+                    info "Compressing ${filename##*/} to ${format}"
+                    COMPRESSED_FILENAME=$(compress_file "${filename}" "${format}")
+                    local_resulting_archives+=( "$COMPRESSED_FILENAME" )
+                    processed_format["${format}"]=1
+                fi
+            done
+            # If requested, delete the input file after compression (only if 'none' is not part of the formats)
+            # Exclude the generic image and update payload because they are needed for generating other formats
+            if [ "${FLAGS_only_store_compressed}" -eq "${FLAGS_TRUE}" ] &&
+               [ "${filename##*/}" != "flatcar_production_image.bin" ] &&
+               [ "${filename##*/}" != "flatcar_production_update.bin" ] &&
+               ! echo "${FORMATS[@]}" | grep -q "none"; then
+                rm "${filename}"
+            fi
+        else
+            local_extra_files+=( "${filename}" )            
+        fi
+    done
+}
+
+upload_legacy_digests() {
+    [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0
+
+    local local_digest_file="$1"
+    local -n local_compressed_files="$2"
+
+    [[ "${#local_compressed_files[@]}" -gt 0 ]] || return 0
+
+    # Upload legacy digests
+    declare -a digests_to_upload
+    for file in "${local_compressed_files[@]}";do
+        legacy_digest_file="${file}.DIGESTS"
+        cp "${local_digest_file}" "${legacy_digest_file}"
+        digests_to_upload+=( "${legacy_digest_file}" )
+    done
+    local def_upload_path="${UPLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}"
+    upload_files "digests" "${def_upload_path}" "" "${digests_to_upload[@]}"
+}
 
 check_gsutil_opts() {
     [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0
@@ -209,15 +314,7 @@ upload_image() {
         if [[ ! -f "${filename}" ]]; then
             die "File '${filename}' does not exist!"
         fi
-
-        # Compress disk images
-        if [[ "${filename}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then
-            info "Compressing ${filename##*/}"
-            $IMAGE_ZIPPER -f "${filename}"
-            uploads+=( "${filename}${IMAGE_ZIPEXT}" )
-        else
-            uploads+=( "${filename}" )
-        fi
+        uploads+=( "${filename}" )
     done
 
     if [[ -z "${digests}" ]]; then
diff --git a/build_library/toolchain_util.sh b/build_library/toolchain_util.sh
index d48f1c4f6c..8023b844f2 100644
--- a/build_library/toolchain_util.sh
+++ b/build_library/toolchain_util.sh
@@ -167,6 +167,10 @@ get_sdk_libdir() {
     portageq envvar "LIBDIR_$(get_sdk_arch)"
 }
 
+get_sdk_symlink_lib() {
+    portageq envvar "SYMLINK_LIB"
+}
+
 # Usage: get_sdk_binhost [version...]
 # If no versions are specified the current and SDK versions are used.
 get_sdk_binhost() {
diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh
index 67c9b785e3..91e4680d4b 100644
--- a/build_library/vm_image_util.sh
+++ b/build_library/vm_image_util.sh
@@ -1193,45 +1193,46 @@ vm_cleanup() {
 }
 
 vm_upload() {
+
+    declare -a legacy_uploads
+    declare -a uploadable_files
+    declare -a compressed_images
+    declare -a image_files
+    declare -a digest_uploads
+
+    compress_disk_images VM_GENERATED_FILES compressed_images uploadable_files
+
+    if [ "${#compressed_images[@]}" -gt 0 ]; then
+        uploadable_files+=( "${compressed_images[@]}" )
+        legacy_uploads+=( "${compressed_images[@]}" )
+    fi
+
     local digests="$(_dst_dir)/$(_dst_name .DIGESTS)"
-    upload_image -d "${digests}" "${VM_GENERATED_FILES[@]}"
+    upload_image -d "${digests}" "${uploadable_files[@]}"
 
     [[ -e "${digests}" ]] || return 0
 
-    # FIXME(marineam): Temporary alternate name for .DIGESTS
-    # This used to be derived from the first file listed in
-    # ${VM_GENERATED_FILES[@]}", usually $VM_DST_IMG or similar.
-    # Since not everything actually uploads $VM_DST_IMG this was not very
-    # consistent and relying on ordering was breakable.
-    # Now the common prefix, output by $(_dst_name) is used above.
-    # Some download/install scripts may still refer to the old name.
-    local uploaded legacy_uploaded
-    for uploaded in "${VM_GENERATED_FILES[@]}"; do
-        if [[ "${uploaded}" == "${VM_DST_IMG}" ]]; then
-            legacy_uploaded="$(_dst_dir)/$(basename ${VM_DST_IMG})"
-            break
+    # Since depending on the ordering of $VM_GENERATED_FILES is brittle only
+    # use it if $VM_DST_IMG isn't included in the uploaded files.
+    if [ "${#legacy_uploads[@]}" -eq 0 ];then
+        legacy_uploads+=( "${VM_GENERATED_FILES[0]}" )
+    fi
+
+    for legacy_upload in "${legacy_uploads[@]}";do
+        local legacy_digest_file="${legacy_upload}.DIGESTS"
+        [[ "${legacy_digest_file}" == "${digests}" ]] && continue
+
+        cp "${digests}" "${legacy_digest_file}"
+        digest_uploads+=( "${legacy_digest_file}" )
+
+        if [[ -e "${digests}.asc" ]]; then
+            digest_uploads+=( "${legacy_digest_file}.asc" )
+            cp "${digests}.asc" "${legacy_digest_file}.asc"
         fi
     done
 
-    # Since depending on the ordering of $VM_GENERATED_FILES is brittle only
-    # use it if $VM_DST_IMG isn't included in the uploaded files.
-    if [[ -z "${legacy_uploaded}" ]]; then
-        legacy_uploaded="${VM_GENERATED_FILES[0]}"
-    fi
-
-    # If upload_images compressed $legacy_uploaded be sure to add .bz2
-    if [[ "${legacy_uploaded}" =~ \.(img|bin|vdi|vhd|vmdk)$ ]]; then
-        legacy_uploaded+="${IMAGE_ZIPEXT}"
-    fi
-
-    local legacy_digests="${legacy_uploaded}.DIGESTS"
-    [[ "${legacy_digests}" != "${digests}" ]] || return 0
-
-    local legacy_uploads=( "${legacy_digests}" )
-    cp "${digests}" "${legacy_digests}"
-    if [[ -e "${digests}.asc" ]]; then
-        legacy_uploads+=( "${legacy_digests}.asc" )
-        cp "${digests}.asc" "${legacy_digests}.asc"
+    if [ "${#digest_uploads[@]}" -gt 0 ];then
+        legacy_uploads+=( "${digest_uploads[@]}" )
     fi
 
     local def_upload_path="${UPLOAD_ROOT}/boards/${BOARD}/${FLATCAR_VERSION}"
diff --git a/build_packages b/build_packages
index df822efbd3..aaeca62a64 100755
--- a/build_packages
+++ b/build_packages
@@ -173,7 +173,7 @@ fi
 # Build cros_workon packages when they are changed.
 CROS_WORKON_PKGS=()
 if [ "${FLAGS_workon}" -eq "${FLAGS_TRUE}" ]; then
-  CROS_WORKON_PKGS+=( $(cros_workon list --board=${FLAGS_board}) )
+  CROS_WORKON_PKGS+=( $("${SRC_ROOT}/scripts/cros_workon" list --board=${FLAGS_board}) )
 fi
 
 if [[ ${#CROS_WORKON_PKGS[@]} -gt 0 ]]; then
diff --git a/build_sdk_container_image b/build_sdk_container_image
new file mode 100755
index 0000000000..d1e4fce88d
--- /dev/null
+++ b/build_sdk_container_image
@@ -0,0 +1,251 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This script will generate an SDK container image from an SDK tarball
+#  (which in turn is generated by bootstrap_sdk[_container]).
+#
+# It uses a number of intermediate build steps:
+#   1. Import the SDK tarball into a "tarball" container image.
+#   2. Build a "plain" SDK container image which creates the "sdk" user
+#      and runs "update_chroot" to initialise the x86 and aarch64 SDK cross env.
+#      This step uses sdk_lib/Dockerfile.sdk-import.
+#   3. Using the "plain" SDK image, start a temporary "toolchain" container
+#      to build toolchain binary packages.
+#   4. Inheriting from the "plain" SDK image and using the toolchain packages,
+#      build a "full" SDK container image, with board support for both amd64-usr and arm64-usr.
+#      A temporary HTTP server on the Docker interface IP is spun up in this step
+#      to serve the toolchain binpkgs to the build container.
+#      This step uses sdk_lib/Dockerfile.sdk-build.
+#   5. In a last step, all unnecessary binaries are removed from the "full" image and 3
+#      output SDK images are produced:
+#      - "all", with both amd64 and arm64 target support, and
+#      - "amd64" and "arm64, with only amd64 or arm64 target support respectively.
+#      This step uses sdk_lib/Dockerfile.lean-arch.
+
+set -eu
+
+cd $(dirname "$0")
+source sdk_lib/sdk_container_common.sh
+
+arch="amd64"
+official="0"
+tarball=""
+os_version=""
+
+keep="false"
+cleanup=""
+
+usage() {
+    echo "  $0 - Create SDK container image from an SDK tarball"
+    echo "       This script will set up a new SDK container from a tarball."
+    echo "       The resulting container comes in 3 flavours:"
+    echo "        1.    flatcar-sdk-all - includes both ARM64 and AMD64 support"
+    echo "        2.+3. flatcar-sdk-(amd64|arm64) - only includes support for one target."
+    echo "  Usage:"
+    echo "  $0 <tarball>] [-k] [-v <os-version>] [-x <script>]"
+    echo
+    echo "      <tarball>    - Local tarball to build SDK from."
+    echo "                     The tarball must follow the format"
+    echo "                     flatcar-sdk-(amd64|arm64)-<version>.tar.bz2."
+    echo "      -v <version> - Use custom OS version (defaults to tarball's SDK version)."
+    echo "      -k           - Keep intermediate build containers (sdk-import / sdk-tarball)."
+    echo "                     (Useful for sdk container build script development.)"
+    echo "      -x <script>  - For each resource generated during build (container etc.)"
+    echo "                     add a cleanup line to <script> which, when run, will free"
+    echo "                     the resource. Useful for CI."
+    echo "      -h           - Print this help."
+    echo
+}
+
+# --
+
+while [ 0 -lt $# ] ; do
+    case "$1" in
+    -h) usage; exit 0;;
+    -k) keep="true";      shift;;
+    -v) os_version="$2";  shift; shift;;
+    -x) cleanup="$2";     shift; shift;;
+    *)  if [ -z "$tarball" ] ;  then
+            tarball="$1"; shift
+        else
+            echo "ERROR: spurious positional parameter '$@'."
+            usage
+            exit 1
+        fi;;
+    esac
+done
+
+if [ -z "$tarball" -o ! -s "$tarball" ] ; then
+    echo "ERROR: missing / invalid SDK tarball argument"
+    exit 1
+fi
+# --
+
+# Grok version / arch from tarball name, set official if version is a release version
+version="$(echo "$tarball" | sed -n 's/.*flatcar-sdk-\(arm64\|amd64\)-\(.\+\)\.tar\.bz2/\2/p')"
+arch="$(echo "$tarball" | sed -n 's/.*flatcar-sdk-\(arm64\|amd64\)-.*\.tar\.bz2/\1/p')"
+if [ -z "$version" -o -z "$arch" ]; then
+    echo "ERROR: Unable to determine version / arch from '$tarball'"
+    exit 1
+fi
+
+if [ -z "${os_version}" ] ; then
+    os_version="${version}"
+fi
+
+if is_official "$version" && [ "${version}" = "${os_version}" ] ; then
+    official="1"
+else
+    official="0"
+fi
+
+# --
+
+# import tarball
+#
+yell "\n######\n###### Building SDK container for version $version from '$tarball'"
+create_versionfile "$version" "${os_version}"
+
+docker_vernum="$(vernum_to_docker_image_version "${version}")"
+import_tarball="flatcar-sdk-tarball:${docker_vernum}"
+image_present="$($docker image ls "$import_tarball" --format '{{.Repository}}:{{.Tag}}')"
+
+if [ "${image_present}" = "${import_tarball}" ] ; then
+    yell "Using existing SDK tarball image '${import_tarball}'"
+else
+    yell "Importing SDK tarball"
+    if [ -n "$cleanup" ] ; then
+        echo "$docker image rm -f '${import_tarball}'" >> "$cleanup"
+    fi
+    $docker import "${tarball}" "${import_tarball}"
+fi
+
+# --
+
+# build plain SDK container w/o board support
+#
+import_image="flatcar-sdk-import:${docker_vernum}"
+image_present="$($docker image ls "${import_image}" --format '{{.Repository}}:{{.Tag}}')"
+
+if [ "$image_present" = "${import_image}" ] ; then
+    yell "Using existing SDK import image '${import_image}'"
+else
+    yell "Building plain SDK import image"
+    if [ -n "$cleanup" ] ; then
+        echo "$docker image rm -f '${import_image}'" >> "$cleanup"
+    fi
+    $docker build -t "$import_image" \
+                 --build-arg VERSION="${docker_vernum}" \
+                 -f sdk_lib/Dockerfile.sdk-import \
+                 .
+fi
+
+# --
+
+# build full SDK container w/ board support.
+#  This uses the SDK import container to first build toolchain binpkgs.
+#  Then, the import container and toolchain packages are used
+#  to build a full SDK container w/ amd64 and arm64 board support.
+#
+sdk_build_image="flatcar-sdk-build:${docker_vernum}"
+image_present="$($docker image ls "${sdk_build_image}" --format '{{.Repository}}:{{.Tag}}')"
+if [ "$image_present" = "${sdk_build_image}"  ] ; then
+    yell "Using existing SDK build image '${sdk_build_image}'"
+else
+    # --- Toolchains build ---
+    yell "Building toolchains in a temporary container."
+    # We need to use run_sdk_container instead of building from a Dockerfile
+    # since toolchains build uses catalyst which requires privileged access.
+    tarball_copied=""
+    if [ "$(basename "${tarball}")" != "${tarball}" ] ; then
+        cp --reflink=auto "${tarball}" ./
+        tarball="$(basename "${tarball}")"
+        tarball_copied="${tarball}"
+    fi
+
+    toolchains_container="flatcar-sdk-toolchains-build-${docker_vernum}"
+    if [ -n "$cleanup" ] ; then
+        echo "$docker container rm -f '${toolchains_container}'" >> "$cleanup"
+    fi
+    ./run_sdk_container -C "${import_image}" -n "${toolchains_container}" \
+            sudo ./build_toolchains --seed_tarball="./${tarball}"
+
+    # remove sdk tarball from scripts root so it's not part of the SDK container build context
+    if [ -f "${tarball_copied}" ] ; then
+        rm "${tarball_copied}"
+    fi
+
+    $docker container rm -f "${toolchains_container}"
+
+    docker_interface="docker0"
+    if "${is_podman}"; then
+      # Make a dummy run without "--net host" here for the interface to be created
+      $docker run --rm alpine
+      docker_interface="cni-podman0"
+    fi
+    host_ip="$(ip addr show "${docker_interface}" | grep -Po 'inet \K[\d.]+')"
+    binhost_port="$((1000 + (RANDOM % 55000) ))"
+    binhost="${host_ip}:${binhost_port}"
+    binhost_container="${toolchains_container}-binhost-${binhost_port}"
+    yell "Building SDK container + board support, toolchain packages served at http://${binhost} by ${binhost_container}"
+
+    # Spin up temporary toolchains package binhost
+    if [ -n "$cleanup" ] ; then
+        echo "$docker container rm -f '${binhost_container}'" >> "$cleanup"
+    fi
+    $docker run --rm -d -p "${binhost}":80 \
+        --name ${binhost_container} \
+        -v "$(pwd)/__build__/images/catalyst/packages/coreos-toolchains/target":/usr/share/caddy \
+        docker.io/library/caddy caddy file-server \
+        --root /usr/share/caddy --browse
+
+    # --- Full SDK container build ---
+
+    yell "Initialising the SDK container and building board packages"
+    if [ -n "$cleanup" ] ; then
+        echo "$docker image rm -f '${sdk_build_image}'" >> "$cleanup"
+    fi
+    $docker build -t "${sdk_build_image}" \
+                 --build-arg VERSION="${docker_vernum}" \
+                 --build-arg BINHOST="http://${binhost}" \
+                 --build-arg OFFICIAL="${official}" \
+                 -f sdk_lib/Dockerfile.sdk-build \
+                 .
+
+    $docker stop "${binhost_container}"
+fi
+
+# --
+
+# Derive "lean" SDK containers from full build. Main purpose
+#  of this step is to remove "white-outs", i.e. files which have been
+#  deleted in the full image but are still present in an intermediate layer.
+#
+for a in all arm64 amd64; do
+    yell "Creating '$a' arch SDK image"
+    rmarch=""; rmcross=""
+    case $a in
+        arm64) rmarch="amd64-usr"; rmcross="x86_64-cros-linux-gnu";;
+        amd64) rmarch="arm64-usr"; rmcross="aarch64-cros-linux-gnu";;
+    esac
+    $docker build -t "$sdk_container_common_registry/flatcar-sdk-${a}:${docker_vernum}" \
+                 --build-arg VERSION="${docker_vernum}" \
+                 --build-arg RMARCH="${rmarch}" \
+                 --build-arg RMCROSS="${rmcross}" \
+                 -f sdk_lib/Dockerfile.lean-arch \
+                 .
+done
+
+# --
+
+# Cleanup
+#
+if ! $keep; then
+    yell "Cleaning up intermediate containers"
+    $docker rmi flatcar-sdk-build:"${docker_vernum}"
+    $docker rmi flatcar-sdk-import:"${docker_vernum}"
+    $docker rmi flatcar-sdk-tarball:"${docker_vernum}"
+fi
diff --git a/checkout b/checkout
new file mode 100755
index 0000000000..5a18a255b6
--- /dev/null
+++ b/checkout
@@ -0,0 +1,178 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -eu
+
+force_tag="false"
+force_branch="false"
+
+update_strategy="fast-forward"
+
+name=""
+
+usage() {
+    echo "  Usage:"
+    echo "  $0 [-t|-b] [-n] [-u <strategy>] <tag-or-branch>"
+    echo "       Check our a tag or branch and synchronise git submodules"
+    echo "          coreos-overlay and portage-stable."
+    echo "       By default, $0 tries to auto-detect whether to handle a tag or a branch,"
+    echo "         with tags being prioritised over branches of the same name."
+    echo
+    echo "       <tag-or-branch> Name of the tag or branch to check out."
+    echo "                       BRANCH: Check out the branch in scripts, coreos-overlay"
+    echo "                                and portage-stable, and fast-forward to the"
+    echo "                                latest changes by default (but see '-u' below)."
+    echo "                       TAG:    Check out the tag in scripts, and update submodules."
+    echo
+    echo "       -t            force TAG mode. Branches of the given name are ignored."
+    echo "                      Mutually exclusive with '-b'."
+    echo "       -b            force BRANCH mode. Tags of the given name are ignored."
+    echo "                      Mutually exclusive with '-t'."
+    echo
+    echo "       -u <strategy> Only applies to BRANCH checkouts."
+    echo "                      When checking out submodule branches, use <strategy>."
+    echo "                      instead of fast-forward. Strategy is one of:"
+    echo "                      'fast-forward' - fast-forward to upstream branch tip. Default."
+    echo "                      'rebase' - rebase local changes on upstream changes."
+    echo "                                 Useful for keepling local changes in submodules."
+    echo "                      'omit'   - check out branches, but do not update."
+    echo "                      Defaults to '$update_strategy'"
+    echo
+    echo "       -h            Print this help."
+    echo
+}
+# --
+
+while [ 0 -lt $# ] ; do
+    case "$1" in
+    -h) usage; exit 0;;
+    -t) force_tag="true";     shift;;
+    -b) force_branch="true";  shift;;
+    -u) update_strategy="$2"; shift; shift;;
+    *) if [ -n "$name" ] ; then
+            echo
+            echo "ERROR: only ONE tag-or-branch can be specified."
+            echo
+            usage
+            exit 1
+       fi
+       name="$1";      shift;;
+    esac
+done
+
+if [ -z "$name" ] ; then
+    usage
+    exit 0
+fi
+
+if $force_branch && $force_tag; then
+    echo
+    echo "ERROR: '-t' and '-b' are mutually exclusive. Please make up your mind."
+    echo
+    usage
+    exit 1
+fi
+
+case "$update_strategy" in
+    fast-forward) update_strategy="--ff-only";;
+    rebase)       update_strategy="--rebase";;
+    omit)         update_strategy="";;
+    *)  echo
+        echo "ERROR: unsupported branch update strategy '$update_strategy'."
+        echo
+        usage
+        exit 1;;
+esac
+# --
+
+# make sure submodules are initialised
+git submodule init
+for dir in sdk_container/src/third_party/coreos-overlay \
+           sdk_container/src/third_party/portage-stable ; do
+    if [ ! -f "$dir"/.git ] ; then
+        git submodule update -N "$dir"
+    fi
+done
+
+function check_all() {
+    local gitcmd="$1"
+    local name="$2"
+
+    local scripts="$(git $gitcmd \
+                        | sed -e 's/^[[:space:]]*//' -e 's:remotes/[^/]\+/::' \
+                        | grep -m1 -E "^$name\$")"
+
+    # tag has submodules pinned, no need to check
+    if [[ "${gitcmd}" =~ ^tag\ .* ]] ; then
+        echo "${scripts}"
+        return
+    fi
+
+    local overlay="$(git -C sdk_container/src/third_party/coreos-overlay $gitcmd \
+                        | sed -e 's/^[[:space:]]*//' -e 's:remotes/[^/]\+/::' \
+                        | grep -m1 -E "^$name\$")"
+    local portage="$(git -C sdk_container/src/third_party/portage-stable $gitcmd \
+                        | sed -e 's/^[[:space:]]*//' -e 's:remotes/[^/]\+/::' \
+                        | grep -m1 -E "^$name\$")"
+
+    if [ -n "$scripts" -a -n "$overlay" -a -n "$portage" ] ; then
+        echo "$scripts"
+    fi
+}
+# --
+
+#
+# TAG
+#
+if ! $force_branch; then
+    for dir in . \
+        sdk_container/src/third_party/coreos-overlay \
+        sdk_container/src/third_party/portage-stable ; do
+        git -C "$dir" fetch --tags --force --prune --prune-tags
+    done
+    tag="$(check_all 'tag -l' "$name")"
+
+    if [ -n "$tag" ] ; then
+        echo
+        echo "Checking out TAG '$tag'"
+        echo "----------------------------------"
+        git checkout "$tag"
+        git submodule update
+        exit
+    fi
+fi
+
+echo "No tag by name '$name' in repo + submodules."
+
+if $force_tag; then
+    echo "Tag-only mode forced, exiting."
+    exit 1
+fi
+
+#
+# BRANCH
+#
+
+branch="$(check_all "branch -a -l" "$name")"
+if [ -z "$branch" ]; then
+    echo "No branch by name '$name' in repo + submodules."
+    exit 1
+fi
+
+echo
+echo "Checking out BRANCH '$branch'"
+echo "----------------------------------"
+for dir in . \
+    sdk_container/src/third_party/coreos-overlay \
+    sdk_container/src/third_party/portage-stable ; do
+
+    git -C "$dir" checkout "$branch"
+    if [ -n "$update_strategy" ] ; then
+        echo "updating branch in '$dir' /'$update_strategy')"
+        git -C "$dir" pull "$update_strategy"
+    fi
+done
+
diff --git a/ci-automation/README.md b/ci-automation/README.md
new file mode 100644
index 0000000000..13187e7afb
--- /dev/null
+++ b/ci-automation/README.md
@@ -0,0 +1,165 @@
+# CI automation glue logic scripts
+
+Scripts in this directory aim to ease automation of Flatcar builds in continuous integration systems.
+
+Design goal of the automation scripts is to provide self-contained, context-aware automation with a low integration overhead.
+Each step takes its context from the repository (version to build etc.) and from the artifact of the previous build, with the aim of reducing the number of arguments to an absolute minimum.
+
+Each script represents a distinct build step; each step ingests the container image of the previous step and produces a new container image for the next step.
+Notable exceptions are "SDK Bootstrap" (`sdk.sh`) which only creates an SDK tarball, and "VMs build" which does not output a container but only VM (vendor) images.
+The container images are self-contained and aim for ease of reproducibility.
+All steps make use of a "build cache" server for pulling (https) build inputs and for pushing (rsync) artifacts.
+
+Test automation is provided alongside build automation, following the same design principles.
+
+Please refer to the individual scripts for prerequisites, input parameters, and outputs.
+
+
+## Build steps
+
+The build pipeline can be used to build everything from scratch, including the SDK (starting from 1. below) or to build a new OS image (starting from 3.).
+"From scratch" builds (i.e. builds which include a new SDK) are usually only done for the `main` branch (`main` can be considered `alpha-next`).
+Release / maintenance branches in the majority of cases do note build a new SDK but start with the OS image build.
+Release branches usually use the SDK introduced when the new major version was branched off `main` throughout the lifetime of the major version; i.e. release `stable-MMMM.mm.pp` would use `SDK-MMMM.0.0`.
+
+To reproduce any given build step, follow this pattern:
+```
+./checkout <build-tag> # Build tag from either SDK bootstrap pr Packages step
+source ci-automation/<step-script>.sh
+<step_function> <parameters>
+```
+
+For example, to rebuild the AMD64 OS image of build `main-3145.0.0-nightly-20220209-0139`, do
+```
+./checkout main-3145.0.0-nightly-20220209-0139
+source ci-automation/image.sh
+image_build amd64
+```
+
+### SDK bootstrap build
+
+1. SDK Bootstrap (`sdk.sh`): Use a seed SDK tarball and seed SDK container image to build a new SDK tarball.
+   The resulting SDK tarball will use packages and versions pinned in the coreos-overlay and portage-stable submodules.
+   This step updates the versionfile, recording the SDK container version just built.
+   It will generate and push a new version tag to the scripts repo.
+2. SDK container build (`sdk_container.sh`) : use SDK tarball to build an SDK container image.
+   The resulting image will come in "amd64", "arm64", and "all" flavours, with support for respective OS target architectures. This step builds the Flatcar SDK container images published at ghcr.io/flatcar.
+
+```
+         .---------.                    .------------.          .--------.
+         | scripts |                    |     CI     |          |  Build |
+         |  repo   |                    | automation |          |  cache |
+         `---------´                    `------------´          `--------´
+              |                                |                     |
+              |                      "alpha-3449.0.0-dev23"          |
+              |                                |                     |
+              |                         _______v_______              |
+              +-------- clone -------> ( SDK bootstrap )             |
+              |                         `-------------´              |
+              |<- tag: alpha-3499.0.0-dev23 --´|`--- sdk tarball --->|
+              |                                |                     |
+              |                         _______v_______              |
+              +-------- clone -------> ( SDK container )             |
+              | alpha-3499.0.0-dev23    `-------------´              |
+              |                                |`- sdk container --->|
+              v                                v        image
+                      continue to OS
+                       image build
+                            |
+                            v 
+```
+
+### OS image build
+
+3. Packages tag (`packages-tag.sh`): Creates git tag if needed with the SDK container version recorded in the versionfile and the OS version from the tag.
+   Creates `skip-build` flag file if no changes since last nightly tag are seen.
+   Pushes the new version tag to the scripts repo as free-standing tag, and for nightlies also to the branch.
+4. Packages build (`packages.sh`): Build OS image packages and generate a new container image (containing both SDK and packages).
+5. Packages are published and the generic OS image is built.
+   1. Binary packages are published (`push_pkgs.sh`) to the build cache, making them available to developers who base their work on the main branch.
+   2. Image build (`image.sh`): Using the container from 3., build an OS image and torcx store, and generate a new container image with everything in it.
+6. VMs build (`vms.sh`). Using the packages+torcx+image container from 4., build vendor images. Results are vendor-specific OS images.
+
+```
+       .---------.                     .------------.             .--------.
+       | scripts |                     |     CI     |             |  Build |
+       |  repo   |                     | automation |             |  cache |
+       `---------´                     `------------´             `--------´
+            |                                 |                        |
+            |                       "alpha-3449.0.0-dev23"             |
+            |                                 |                        |
+            |                           ______v_______                 |
+            +---------- clone ------>  ( packages-tag )                |
+            |                           `------------´                 |
+            |<-- tag: alpha-3499.0.0-dev23 --´|                        |
+            |                             ____v_____                   |
+            +-----       clone      ---> ( packages )                  |
+            |    alpha-3499.0.0-dev23     `--------´                   |
+            |                                 |`- sdk + OS packages -->|
+            |                                 |    container image     |
+            |                                 |    torcx manifest      |
+            |                           ______v_______                 |
+            |                          ( publish pkgs )                |
+            |                           `------------´                 |
+            |                                 |`-- binary packages --->|
+            |                              ___v__                      |
+            +-----       clone      --->  ( image )                    |
+            |    alpha-3499.0.0-dev23      `-----´                     |
+            |                                 |`-- sdk + packages + -->|
+            |                               __v__  OS image cnt img    |
+            +-----       clone      --->   ( vms )                     |
+                 alpha-3499.0.0-dev23       `---´                      |
+                                              `- vendor OS images ---->|
+```
+
+## Testing
+
+Testing follows the same design principles build automation adheres to - it's self-contained and context-aware, reducing required parameters to a minimum.
+The `test.sh` script needs exactly two parameters: the architecture, and the image type to be tested.
+Optionally, patterns matching a group of tests can be supplied (or simply a list of tests); this defaults to "all tests" of a given vendor / image.
+`test.sh` also supports re-running failed tests automatically to reduce the need for human interaction on flaky tests.
+
+Testing is implemented in two layers:
+1. `ci-automation/test.sh` is a generic test wrapper / stub to be called from CI.
+2. `ci-automation/vendor-testing/` contains low-level vendor-specific test wrappers around [`kola`](https://github.com/flatcar/mantle/tree/flatcar-master/kola/), our test scenario orchestrator.
+
+Testing relies on the SDK container and will use tools / test suites from the SDK.
+The low-level vendor / image specific script (layer 2. in the list above) runs inside the SDK.
+Testing will use the vendor image published by `vms.sh` from buildcache, and the torcx manifest published by `packages.sh`.
+
+Additionally, a script library is provided (at `ci-automation/tapfile_helper_lib.sh`) to help handling `.tap` test result files produced by test runs.
+Library functions may be used to merge the result of multiple test runs (e.g. for multiple image types / vendors) into a single test result report.
+The test runs are considered successful only if all tests succeeded for all vendors / images at least once.
+
+
+**Usage**
+```
+./checkout <version-to-test>
+source ci-automation/test.sh
+test_run <arch> <image-type>
+```
+
+E.g. for running qemu / amd64 tests on `main-3145.0.0-nightly-20220209-0139`:
+```
+./checkout main-3145.0.0-nightly-20220209-0139
+source ci-automation/test.sh
+test_run amd64 qemu
+```
+
+### QEmu test
+
+`ci-automation/vendor-testing/qemu.sh` implements a `kola` wrapper for testing the `qemu` image.
+The wrapper is a straightforward call to `kola` and does not have any additional requirements.
+
+**NOTE** that the generic image (`flatcar_production_image.bin`) is used for the test instead of the QEmu vendor image.
+
+**NOTE on host firewalling** The test automation uses bridged networking and will handle forwarding and NAT.
+However, we experienced test failures from lack of internet access with several firewall implementations.
+It is recommended to stop firewalling on the host the tests are run on (for example, use `systemctl stop firewalld` if the host used `firewalld`).
+
+**Settings**
+
+* `QEMU_IMAGE_NAME` - file name of the QEmu image to fetch from bincache.
+* `QEMU_PARALLEL` - Number of parallel test cases to run.
+                  Note that test cases may involve launching mutliple QEmu VMs (network testing etc.).
+                  Tests are memory bound, not CPU bound; e.g. `20` is a sensible value for a 6 core / 12 threads systwem w/ 32 GB RAM.
diff --git a/ci-automation/ci-config.env b/ci-automation/ci-config.env
new file mode 100644
index 0000000000..1c2e5b4735
--- /dev/null
+++ b/ci-automation/ci-config.env
@@ -0,0 +1,131 @@
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Flatcar CI static configuration
+
+# Build cache server for build artifacts.
+#  Required services:
+#  - http and https (WITHOUT auto-redirect)
+#  - ssh for BUILDCACHE_USER
+BUILDCACHE_SERVER="${BUILDCACHE_SERVER:-bincache.flatcar-linux.net}"
+BUILDCACHE_PATH_PREFIX="/srv/bincache"
+BUILDCACHE_USER="bincache"
+
+RELEASES_SERVER="mirror.release.flatcar-linux.net"
+CONTAINER_REGISTRY="ghcr.io/flatcar"
+
+GC_BUCKET="flatcar-linux"
+
+DEFAULT_HTTP_IMAGE_URL_TEMPLATE="@PROTO@://${BUILDCACHE_SERVER}/images/@ARCH@/@VERNUM@"
+
+if ! command -v pigz > /dev/null; then
+  # No PIGZ on Flatcar
+  PIGZ="docker run --rm -i ghcr.io/flatcar/pigz --fast"
+fi
+
+CI_GIT_AUTHOR="flatcar-ci"
+CI_GIT_EMAIL="infra+ci@flatcar-linux.org"
+
+# build artifacts go here (in container)
+CONTAINER_TORCX_ROOT="/home/sdk/build/torcx"
+CONTAINER_IMAGE_ROOT="/home/sdk/build/images"
+
+#
+# Image / vendor tests settings
+#
+
+# NOTE that these settings are evaluated by the vendor-tests script inside the
+#  SDK container. To override, new values must be passed into the container.
+#  Use something like
+#    echo "export [VAR]=\"${[VALUE]}\\"" > sdk_container/.env
+#  in your CI to override, e.g.
+#    echo "export PARALLEL_TESTS=\"5\"" > sdk_container/.env
+#  to override the number of test cases to be run in parallel.
+
+# -- QEMU --
+
+QEMU_IMAGE_NAME=${QEMU_IMAGE_NAME:-flatcar_production_image.bin}
+QEMU_PARALLEL="${PARALLEL_TESTS:-20}"
+
+# BIOS path within the SDK
+QEMU_BIOS="/usr/share/qemu/bios-256k.bin"
+
+# UEFI bios filename on build cache.
+# Published by vms.sh as part of the qemu vendor build.
+QEMU_UEFI_BIOS="${QEMU_UEFI_BIOS:-flatcar_production_qemu_uefi_efi_code.fd}"
+
+
+# -- Equinix Metal --
+EQUINIXMETAL_PARALLEL="${PARALLEL_TESTS:-4}"
+# Metro is a set of Equinix Metal regions
+EQUINIXMETAL_amd64_METRO="${EQUINIXMETAL_amd64_METRO:-SV}"
+EQUINIXMETAL_arm64_METRO="${EQUINIXMETAL_arm64_METRO:-DC}"
+# Name of the Equinix Metal image
+EQUINIXMETAL_IMAGE_NAME="flatcar_production_packet_image.bin.bz2"
+# Storage URL required to store user-data
+EQUINIXMETAL_STORAGE_URL="${EQUINIXMETAL_STORAGE_URL:-gs://flatcar-jenkins/mantle/packet}"
+# Equinix Metal default AMD64 instance type
+EQUINIXMETAL_amd64_INSTANCE_TYPE="${EQUINIXMETAL_amd64_INSTANCE_TYPE:-c3.small.x86}"
+# Space separated list of instance types. On those instances the
+# cl.internet kola test will be run if this test is selected to run.
+EQUINIXMETAL_amd64_MORE_INSTANCE_TYPES="m3.small.x86 c3.medium.x86 m3.large.x86 s3.xlarge.x86 n2.xlarge.x86"
+# Equinix Metal default ARM64 instance type
+EQUINIXMETAL_arm64_INSTANCE_TYPE="c3.large.arm"
+# Space separated list of instance types. On those instances the
+# cl.internet kola test will be run if this test is selected to run.
+EQUINIXMETAL_arm64_MORE_INSTANCE_TYPES=""
+
+# -- PXE --
+PXE_KERNEL_NAME="flatcar_production_pxe.vmlinuz"
+PXE_IMAGE_NAME="flatcar_production_pxe_image.cpio.gz"
+
+GCE_IMAGE_NAME="flatcar_production_gce.tar.gz"
+GCE_GCS_IMAGE_UPLOAD="gs://flatcar-jenkins/developer/gce-ci"
+GCE_MACHINE_TYPE="${GCE_MACHINE_TYPE:-n1-standard-2}"
+GCE_PARALLEL="${PARALLEL_TESTS:-4}"
+
+# -- Digital Ocean --
+
+# Use the "@PROTO@" "@ARCH@", "@CHANNEL@" and "@VERNUM@" placeholders. They will
+# be replaced.
+: ${DIGITALOCEAN_IMAGE_URL_TEMPLATE:="${DEFAULT_HTTP_IMAGE_URL_TEMPLATE}/flatcar_production_digitalocean_image.bin.bz2"}
+: ${DIGITALOCEAN_REGION:='sfo3'}
+: ${DIGITALOCEAN_MACHINE_SIZE:='s-2vcpu-2gb'}
+DIGITALOCEAN_PARALLEL="${PARALLEL_TESTS:-8}"
+# DIGITALOCEAN_TOKEN_JSON env var is used for credentials, and should
+# come from sdk_container/.env. It must be base64-encoded.
+
+# -- VMware ESX --
+
+: ${VMWARE_ESX_IMAGE_NAME:='flatcar_production_vmware_ova.ova'}
+VMWARE_ESX_PARALLEL="${PARALLEL_TESTS:-4}"
+# VMWARE_ESX_CREDS should come from sdk_container/.env and must be
+# base64-encoded.
+
+# -- AWS --
+
+: ${AWS_amd64_INSTANCE_TYPE:="t3.small"}
+# Space separated list of instance types. On those instances the
+# cl.internet kola test will be run if this test is selected to run.
+: ${AWS_amd64_MORE_INSTANCE_TYPES:="m4.2xlarge"}
+: ${AWS_arm64_INSTANCE_TYPE:="a1.large"}
+# Space separated list of instance types. On those instances the
+# cl.internet kola test will be run if this test is selected to run.
+: ${AWS_arm64_MORE_INSTANCE_TYPES:=""}
+: ${AWS_IAM_PROFILE:="ciauto-test"}
+: ${AWS_REGION:="us-east-1"}
+: ${AWS_AMI_ID:=""}
+AWS_PARALLEL="${PARALLEL_TESTS:-8}"
+# AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY should come from
+# sdk_container/.env
+
+# -- Azure --
+: ${AZURE_IMAGE_NAME:="flatcar_production_azure_image.vhd"}
+: ${AZURE_amd64_MACHINE_SIZE:="Standard_D2s_v4"}
+: ${AZURE_arm64_MACHINE_SIZE:="Standard_D2pls_v5"}
+: ${AZURE_USE_GALLERY:=""}
+: ${AZURE_USE_PRIVATE_IPS:=true}
+: ${AZURE_VNET_SUBNET_NAME:="jenkins-vnet-westeurope"}
+AZURE_PARALLEL="${PARALLEL_TESTS:-20}"
+AZURE_LOCATION="${AZURE_LOCATION:-westeurope}"
diff --git a/ci-automation/ci_automation_common.sh b/ci-automation/ci_automation_common.sh
new file mode 100644
index 0000000000..488b4d6a1c
--- /dev/null
+++ b/ci-automation/ci_automation_common.sh
@@ -0,0 +1,456 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# CI automation common functions.
+
+source ci-automation/ci-config.env
+: ${PIGZ:=pigz}
+: ${docker:=docker}
+
+: ${TEST_WORK_DIR:='__TESTS__'}
+
+function init_submodules() {
+    git submodule init
+    git submodule update
+}
+# --
+
+function update_submodule() {
+    local submodule="$1"
+    local commit_ish="$2"
+
+    cd "sdk_container/src/third_party/${submodule}"
+    git fetch --all --tags
+    git checkout "${commit_ish}"
+    cd -
+}
+# --
+
+function check_version_string() {
+    local version="$1"
+
+    if [[ ! "${version}" =~ ^(main|alpha|beta|stable|lts)-[0-9]+\.[0-9]+\.[0-9]+(-.+)?$ ]]; then
+        echo "ERROR: invalid version '${version}', must start with 'main', 'alpha', 'beta', 'stable' or 'lts', followed by a dash and three dot-separated numbers, optionally followed by a dash and a non-empty build ID"
+        exit 1
+    fi
+}
+# --
+
+function update_submodules() {
+    local coreos_git="$1"
+    local portage_git="$2"
+
+    init_submodules
+    update_submodule "coreos-overlay" "${coreos_git}"
+    update_submodule "portage-stable" "${portage_git}"
+}
+# --
+
+function update_and_push_version() {
+    local version="$1"
+    local push_to_branch="${2:-false}"
+
+    # set up author and email so git does not complain when tagging
+    if ! git config --get user.name >/dev/null 2>&1 ; then
+        git -C . config user.name "${CI_GIT_AUTHOR}"
+    fi
+    if ! git config --get user.email >/dev/null 2>&1 ; then
+        git -C . config user.email "${CI_GIT_EMAIL}"
+    fi
+
+    # Add and commit local changes
+    git add "sdk_container/src/third_party/coreos-overlay"
+    git add "sdk_container/src/third_party/portage-stable"
+    git add "sdk_container/.repo/manifests/version.txt"
+
+    git commit --allow-empty -m "New version: ${version}"
+
+    git fetch --all --tags --force
+    local ret=0
+    git diff --exit-code "${version}" || ret=$?
+    # This will return != 0 if
+    #  - the remote tag does not exist (rc: 127)
+    #  - the tag does not exist locally (rc: 128)
+    #  - the remote tag has changes compared to the local tree (rc: 1)
+    if [ "$ret" = "0" ]; then
+      echo "Reusing existing tag" >&2
+      git checkout -f --recurse-submodules "${version}"
+      return
+    elif [ "$ret" = "1" ]; then
+      echo "Remote tag exists already and is not equal" >&2
+      return 1
+    elif [ "$ret" != "127" ] && [ "$ret" != "128" ]; then
+      echo "Error: Unexpected git diff return code ($ret)" >&2
+      return 1
+    fi
+
+    local -a TAG_ARGS
+    if [ "${SIGN-0}" = 1 ]; then
+      TAG_ARGS=("-s" "-m" "${version}")
+    fi
+
+    git tag -f "${TAG_ARGS[@]}" "${version}"
+
+    if [ "${push_to_branch}" = "true" ]; then
+      local branch="$(git rev-parse --abbrev-ref HEAD)"
+      git push origin "${branch}"
+    fi
+
+    git push origin "${version}"
+}
+# --
+
+function copy_from_buildcache() {
+    local what="$1"
+    local where_to="$2"
+
+    mkdir -p "$where_to"
+    curl --fail --silent --show-error --location --retry-delay 1 --retry 60 \
+        --retry-connrefused --retry-max-time 60 --connect-timeout 20 \
+        --remote-name --output-dir "${where_to}" "https://${BUILDCACHE_SERVER}/${what}" 
+}
+# --
+
+function gen_sshcmd() {
+    echo -n "ssh -o BatchMode=yes"
+    echo -n " -o StrictHostKeyChecking=no"
+    echo -n " -o UserKnownHostsFile=/dev/null"
+    echo    " -o NumberOfPasswordPrompts=0"
+}
+# --
+
+function copy_dir_from_buildcache() {
+    local remote_path="${BUILDCACHE_PATH_PREFIX}/$1"
+    local local_path="$2"
+
+    local sshcmd="$(gen_sshcmd)"
+    mkdir -p "${local_path}"
+    rsync --partial -a -e "${sshcmd}" "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}:${remote_path}" \
+        "${local_path}"
+}
+
+# --
+
+function copy_to_buildcache() {
+    local remote_path="${BUILDCACHE_PATH_PREFIX}/$1"
+    shift
+
+    local sshcmd="$(gen_sshcmd)"
+
+    $sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \
+        "mkdir -p ${remote_path}"
+
+    rsync --partial -a -e "${sshcmd}" "$@" \
+        "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}:${remote_path}"
+}
+# --
+
+function image_exists_locally() {
+    local name="$1"
+    local version="$2"
+    local image="${name}:${version}"
+
+    local image_exists="$($docker images "${image}" \
+                            --no-trunc --format '{{.Repository}}:{{.Tag}}')"
+
+    [ "${image}" = "${image_exists}" ]
+}
+# --
+
+# Derive docker-safe image version string from vernum.
+#
+function vernum_to_docker_image_version() {
+    local vernum="$1"
+    echo "$vernum" | sed 's/[+]/-/g'
+}
+# --
+
+# Return the full name (repo+name+tag) of an image. Useful for SDK images
+#  pulled from the registry (which have the registry pre-pended)
+function docker_image_fullname() {
+    local image="$1"
+    local version="$2"
+
+    $docker images --no-trunc --format '{{.Repository}}:{{.Tag}}' \
+        | grep -E "^(${CONTAINER_REGISTRY}/)*${image}:${version}$"
+}
+# --
+
+function docker_image_to_buildcache() {
+    local image="$1"
+    local version="$2"
+
+    # strip potential container registry prefix
+    local tarball="$(basename "$image")-${version}.tar.gz"
+
+    $docker save "${image}":"${version}" | $PIGZ -c > "${tarball}"
+    create_digests "${SIGNER:-}" "${tarball}"
+    sign_artifacts "${SIGNER:-}" "${tarball}"*
+    copy_to_buildcache "containers/${version}" "${tarball}"*
+}
+# --
+
+function docker_commit_to_buildcache() {
+    local container="$1"
+    local image_name="$2"
+    local image_version="$3"
+
+    $docker commit "${container}" "${image_name}:${image_version}"
+    docker_image_to_buildcache "${image_name}" "${image_version}"
+}
+# --
+
+function docker_image_from_buildcache() {
+    local name="$1"
+    local version="$2"
+    local tgz="${name}-${version}.tar.gz"
+
+    if image_exists_locally "${name}" "${version}" ; then
+        return
+    fi
+
+    local url="https://${BUILDCACHE_SERVER}/containers/${version}/${tgz}"
+
+    curl --fail --silent --show-error --location --retry-delay 1 --retry 60 \
+        --retry-connrefused --retry-max-time 60 --connect-timeout 20 \
+        --remote-name "${url}"
+
+    cat "${tgz}" | $PIGZ -d -c | $docker load
+
+    rm "${tgz}"
+}
+# --
+
+function docker_image_from_registry_or_buildcache() {
+    local image="$1"
+    local version="$2"
+
+    if image_exists_locally "${CONTAINER_REGISTRY}/${image}" "${version}" ; then
+        return
+    fi
+
+    if $docker pull "${CONTAINER_REGISTRY}/${image}:${version}" ; then
+        return
+    fi
+
+    echo "Falling back to tar ball download..." >&2
+    docker_image_from_buildcache "${image}" "${version}"
+}
+# --
+
+# Called by vendor test in case of complete failure not eligible for
+# reruns (like trying to run tests on unsupported architecture).
+function break_retest_cycle() {
+    local work_dir=$(dirname "${PWD}")
+    local dir=$(basename "${work_dir}")
+
+    if [[ "${dir}" != "${TEST_WORK_DIR}" ]]; then
+        echo "Not breaking retest cycle, expected test work dir to be a parent directory" >&2
+        return
+    fi
+    touch "${work_dir}/break_retests"
+}
+# --
+
+# Called by test runner to see if the retest cycle should be broken.
+function retest_cycle_broken() {
+    # Using the reverse boolean logic here!
+    local broken=1
+    if [[ -f "${TEST_WORK_DIR}/break_retests" ]]; then
+        broken=0
+        rm -f "${TEST_WORK_DIR}/break_retests"
+    fi
+    return ${broken}
+}
+# --
+
+# Substitutes fields in the passed template and prints the
+# result. Followed by the template, the parameters used for
+# replacement are in alphabetical order: arch, channel, proto and
+# vernum.
+function url_from_template() {
+    local template="${1}"; shift
+    local arch="${1}"; shift
+    local channel="${1}"; shift
+    local proto="${1}"; shift
+    local vernum="${1}"; shift
+    local url="${template}"
+
+    url="${url//@ARCH@/${arch}}"
+    url="${url//@CHANNEL@/${channel}}"
+    url="${url//@PROTO@/${proto}}"
+    url="${url//@VERNUM@/${vernum}}"
+
+    echo "${url}"
+}
+# --
+
+# Puts a secret into a file, while trying for the secret to not end up
+# on a filesystem at all. A path to the file with the secret in /proc
+# in put into the chosen variable. The secret is assumed to be
+# base64-encoded.
+#
+# Typical use:
+#   secret_file=''
+#   secret_to_file secret_file "${some_secret}"
+#
+# Parameters:
+# 1 - name of the variable where the path is stored
+# 2 - the secret to store in the file
+function secret_to_file() {
+    local config_var_name="${1}"; shift
+    local secret="${1}"; shift
+    local tmpfile=$(mktemp)
+    local -n config_ref="${config_var_name}"
+    local fd
+
+    exec {fd}<>"${tmpfile}"
+    rm -f "${tmpfile}"
+    echo "${secret}" | base64 --decode >&${fd}
+    # Use BASHPID because we may be in a subshell but $$ is only the main shell's PID
+    config_ref="/proc/${BASHPID}/fd/${fd}"
+}
+# --
+
+# Creates signatures for the passed files and directories. In case of
+# directory, all files inside are signed. Files ending with .asc or
+# .sig or .gpg are ignored, though. This function is a noop if signer
+# is empty.
+#
+# Typical use:
+#   sign_artifacts "${SIGNER}" artifact.tar.gz
+#   copy_to_buildcache "artifacts/directory" artifact.tar.gz*
+#
+# Parameters:
+#
+# 1 - signer whose key is expected to be already imported into the
+#       keyring
+# @ - files and directories to sign
+function sign_artifacts() {
+    local signer="${1}"; shift
+    # rest of the parameters are directories/files to sign
+    local to_sign=()
+    local file
+
+    if [[ -z "${signer}" ]]; then
+        return
+    fi
+
+    list_files to_sign 'asc,gpg,sig' "${@}"
+
+    for file in "${to_sign[@]}"; do
+        gpg --batch --local-user "${signer}" \
+            --output "${file}.sig" \
+            --detach-sign "${file}"
+    done
+}
+# --
+
+# Creates digests files and armored ASCII files out of them for the
+# passed files and directories. In case of directory, all files inside
+# it are processed. No new digests file is created if there is one
+# already for the processed file. Same for armored ASCII file. Files
+# ending with .asc or .sig or .gpg or .DIGESTS are not processed. The
+# armored ASCII files won't be created if the signer is empty.
+#
+# Typical use:
+#   create_digests "${SIGNER}" artifact.tar.gz
+#   sign_artifacts "${SIGNER}" artifact.tar.gz*
+#   copy_to_buildcache "artifacts/directory" artifact.tar.gz*
+#
+# Parameters:
+#
+# 1 - signer whose key is expected to be already imported into the
+#       keyring
+# @ - files and directories to create digests for
+function create_digests() {
+    local signer="${1}"; shift
+    # rest of the parameters are files or directories to create
+    # digests for
+    local to_digest=()
+    local file
+    local df
+    local fbn
+    local hash_type
+    local output
+    local af
+
+    list_files to_digest 'asc,gpg,sig,DIGESTS' "${@}"
+
+    for file in "${to_digest[@]}"; do
+        df="${file}.DIGESTS"
+        if [[ ! -e "${df}" ]]; then
+            touch "${df}"
+            fbn=$(basename "${file}")
+            # TODO: modernize - drop md5 and sha1, add b2
+            for hash_type in md5 sha1 sha512; do
+                echo "# ${hash_type} HASH" | tr "a-z" "A-Z" >>"${df}"
+                output=$("${hash_type}sum" "${file}")
+                echo "${output%% *}  ${fbn}" >>"${df}"
+            done
+        fi
+        if [[ -z "${signer}" ]]; then
+            continue
+        fi
+        af="${df}.asc"
+        if [[ ! -e "${af}" ]]; then
+            gpg --batch --local-user "${signer}" \
+                --output "${af}" \
+                --clearsign "${df}"
+        fi
+    done
+}
+# --
+
+# Puts a filtered list of files from the passed files and directories
+# in the passed variable. The filtering is done by ignoring files that
+# end with the passed extensions. The extensions list should not
+# contain the leading dot.
+#
+# Typical use:
+#   local all_files=()
+#   local ignored_extensions='sh,py,pl' # ignore the shell, python and perl scripts
+#   list_files all_files "${ignored_extensions}" "${directories_and_files[@]}"
+#
+# Parameters:
+#
+# 1 - name of an array variable where the filtered files will be stored
+# 2 - comma-separated list of extensions that will be used for filtering files
+# @ - files and directories to scan for files
+function list_files() {
+    local files_variable_name="${1}"; shift
+    local ignored_extensions="${1}"; shift
+    # rest of the parameters are files or directories to list
+    local -n files="${files_variable_name}"
+    local file
+    local tmp_files
+    local pattern=''
+
+    if [[ -n "${ignored_extensions}" ]]; then
+        pattern='\.('"${ignored_extensions//,/|}"')$'
+    fi
+
+    files=()
+    for file; do
+        tmp_files=()
+        if [[ -d "${file}" ]]; then
+            readarray -d '' tmp_files < <(find "${file}" ! -type d -print0)
+        elif [[ -e "${file}" ]]; then
+            tmp_files+=( "${file}" )
+        fi
+        if [[ -z "${pattern}" ]]; then
+            files+=( "${tmp_files[@]}" )
+            continue
+        fi
+        for file in "${tmp_files[@]}"; do
+            if [[ "${file}" =~ ${pattern} ]]; then
+                continue
+            fi
+            files+=( "${file}" )
+        done
+    done
+}
+# --
diff --git a/ci-automation/garbage_collect.sh b/ci-automation/garbage_collect.sh
new file mode 100644
index 0000000000..cff5d441a5
--- /dev/null
+++ b/ci-automation/garbage_collect.sh
@@ -0,0 +1,156 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
+#
+# garbage_collect() should be called after sourcing.
+#
+#  OPTIONAL INPUT
+#  - Number of (recent) versions to keep. Defaults to 50.
+#  - PURGE_VERSIONS (Env variable). Space-separated list of versions to purge
+#            instead of all but the 50 most recent ones.
+#  - DRY_RUN (Env variable). Set to "y" to just list what would be done but not
+#            actually purge anything.
+
+# Flatcar CI automation garbage collector.
+#  This script removes development (non-official) build artifacts:
+#   - SDK tarballs, build step containers, and vendor images on buildcache
+#   - tags from the scripts repository
+#
+#  Garbage collection is based on development (non-official) version tags
+#   in the scripts repo. The newest 50 builds will be retained,
+#   all older builds will be purged (50 is the default, see OPTIONAL INPUT above).
+
+function garbage_collect() {
+    # Run a subshell, so the traps, environment changes and global
+    # variables are not spilled into the caller.
+    (
+        set -euo pipefail
+
+        _garbage_collect_impl "${@}"
+    )
+}
+# --
+
+function _garbage_collect_impl() {
+    local keep="${1:-50}"
+    local dry_run="${DRY_RUN:-}"
+    local purge_versions="${PURGE_VERSIONS:-}"
+
+    local versions_detected="$(git tag -l --sort=-committerdate \
+                | grep -E '(main|alpha|beta|stable|lts)-[0-9]+\.[0-9]+\.[0-9]+\-.*' \
+                | grep -vE '(-pro)$')"
+
+    echo "######## Full list of version(s) found ########"
+    echo "${versions_detected}" | awk '{printf "%5d %s\n", NR, $0}'
+
+    if [ -z "${purge_versions}" ] ; then
+        keep="$((keep + 1))" # for tail -n+...
+        purge_versions="$(echo "${versions_detected}" \
+                            | tail -n+"${keep}")"
+    else
+        # make sure we only accept dev versions
+        purge_versions="$(echo "${purge_versions}" | sed 's/ /\n/g' \
+                            | grep -E '(main|alpha|beta|stable|lts)-[0-9]+\.[0-9]+\.[0-9]+\-.*' \
+                            | grep -vE '(-pro)$')"
+    fi
+
+    source ci-automation/ci_automation_common.sh
+
+    local sshcmd="$(gen_sshcmd)"
+
+    echo
+    echo "######## The following version(s) will be purged ########"
+    if [ "$dry_run" = "y" ] ; then
+        echo
+        echo "(NOTE this is just a dry run since DRY_RUN=y)"
+        echo
+    fi
+    echo "${purge_versions}" | awk -v keep="${keep}" '{if ($0 == "") next; printf "%5d %s\n", NR + keep - 1, $0}'
+    echo
+    echo
+
+    local version
+    for version in ${purge_versions}; do
+        echo "--------------------------------------------"
+        echo
+        echo "#### Processing version '${version}' ####"
+        echo
+
+        git checkout "${version}" -- sdk_container/.repo/manifests/version.txt
+        source sdk_container/.repo/manifests/version.txt
+
+        # Assuming that the SDK build version also has the same OS version
+        local os_vernum="${FLATCAR_VERSION}"
+        local os_docker_vernum="$(vernum_to_docker_image_version "${FLATCAR_VERSION}")"
+
+        # Remove container image tarballs and SDK tarball (if applicable)
+        #
+        local rmpat=""
+        rmpat="${BUILDCACHE_PATH_PREFIX}/sdk/*/${os_vernum}/"
+        rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-sdk-*"
+        rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-packages-*"
+        rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/boards/*/${os_vernum}/"
+        rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-images-*"
+        rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/images/*/${os_vernum}/"
+        rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/testing/${os_vernum}/"
+
+        echo "## The following files will be removed ##"
+        $sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \
+            "ls -la ${rmpat} || true"
+
+        if [ "$dry_run" != "y" ] ; then
+            set -x
+            $sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \
+                "rm -rf ${rmpat}"
+            set +x
+        else
+            echo "## (DRY_RUN=y so not doing anything) ##"
+        fi
+
+        # Remove container image directory if empty
+        #
+        rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/"
+
+        echo "## Checking if container directory is empty and can be removed (it's OK if this fails) ##"
+        echo "## The following directory will be removed if below output is empty: '${rmpat}' ##"
+        $sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \
+            "ls -la ${rmpat} || true"
+
+        if [ "$dry_run" != "y" ] ; then
+            set -x
+            $sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \
+                "rmdir ${rmpat} || true"
+            set +x
+        else
+            echo "## (DRY_RUN=y so not doing anything) ##"
+        fi
+
+        # Remove git tag (local and remote)
+        #
+        echo "## The following TAG will be deleted: '${version}' ##"
+        if [ "$dry_run" != "y" ] ; then
+            set -x
+            git tag -d "${version}"
+            git push --delete origin "${version}"
+            set +x
+        else
+            echo "## (DRY_RUN=y so not doing anything) ##"
+        fi
+    done
+
+    local mantle_ref
+    mantle_ref=$(cat sdk_container/.repo/manifests/mantle-container)
+    docker run --pull always --rm --net host \
+      --env AZURE_AUTH_CREDENTIALS --env AZURE_PROFILE \
+      --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY \
+      --env DIGITALOCEAN_TOKEN_JSON \
+      --env EQUINIXMETAL_KEY --env EQUINIXMETAL_PROJECT \
+      --env GCP_JSON_KEY \
+      --env VMWARE_ESX_CREDS \
+      -w /work -v "$PWD":/work "${mantle_ref}" /work/ci-automation/garbage_collect_cloud.sh
+}
+# --
diff --git a/ci-automation/garbage_collect_cloud.sh b/ci-automation/garbage_collect_cloud.sh
new file mode 100755
index 0000000000..33d145a7d5
--- /dev/null
+++ b/ci-automation/garbage_collect_cloud.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+set -euo pipefail
+timeout --signal=SIGQUIT 60m ore aws gc --access-id "${AWS_ACCESS_KEY_ID}" --secret-key "${AWS_SECRET_ACCESS_KEY}"
+timeout --signal=SIGQUIT 60m ore do gc --config-file=<(echo "${DIGITALOCEAN_TOKEN_JSON}" | base64 --decode)
+timeout --signal=SIGQUIT 60m ore gcloud gc --json-key <(echo "${GCP_JSON_KEY}" | base64 --decode)
+# Because the Azure file gets read multiple times it can't be passed like <(cmd) because bash backs this FD
+# by a pipe meaning the data is gone after reading. We can create an FD (the FD number is assigned to
+# variable through exec {NAME}) manually and use a file under /tmp to back it instead, allowing multiple
+# reads.
+echo "${AZURE_PROFILE}" | base64 --decode > /tmp/azure_profile
+exec {azure_profile}</tmp/azure_profile
+rm /tmp/azure_profile
+echo "${AZURE_AUTH_CREDENTIALS}" | base64 --decode > /tmp/azure_auth
+exec {azure_auth}</tmp/azure_auth
+rm /tmp/azure_auth
+timeout --signal=SIGQUIT 60m ore azure gc --duration 6h \
+  --azure-profile="/proc/$$/fd/${azure_profile}" --azure-auth="/proc/$$/fd/${azure_auth}"
+timeout --signal=SIGQUIT 60m ore equinixmetal gc --duration 6h \
+  --project="${EQUINIXMETAL_PROJECT}" --gs-json-key=<(echo "${GCP_JSON_KEY}" | base64 --decode) --api-key="${EQUINIXMETAL_KEY}"
diff --git a/ci-automation/gpg_setup.sh b/ci-automation/gpg_setup.sh
new file mode 100644
index 0000000000..d88eea2334
--- /dev/null
+++ b/ci-automation/gpg_setup.sh
@@ -0,0 +1,31 @@
+# Common gpg setup code to be sourced by other scripts in this
+# directory. It will set up GnuPG home directory, possibly with a key
+# from SIGNING_KEY environment variable.
+#
+# After this file is sourced, SIGNER is always defined and exported,
+# even if empty. SIGNING_KEY is clobbered.
+
+: ${SIGNING_KEY:=''}
+: ${SIGNER:=''}
+
+if [[ "${HOME}/.gnupg" -ef "${PWD}/.gnupg" ]]; then
+    echo 'Do not source ${BASH_SOURCE} directly in your home directory - it will clobber your GnuPG directory!' >&2
+    exit 1
+fi
+
+export GNUPGHOME="${PWD}/.gnupg"
+rm -rf "${GNUPGHOME}"
+trap 'rm -rf "${GNUPGHOME}"' EXIT
+mkdir --mode=0700 "${GNUPGHOME}"
+# Sometimes this directory is not automatically created thus making
+# further private key imports to fail. Let's create it here as a
+# workaround.
+mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/"
+if [[ -n "${SIGNING_KEY}" ]] && [[ -n "${SIGNER}" ]]; then
+    gpg --import "${SIGNING_KEY}"
+else
+    SIGNER=''
+fi
+export SIGNER
+# Clobber signing key variable, we don't need it any more.
+export SIGNING_KEY=''
diff --git a/ci-automation/image.sh b/ci-automation/image.sh
new file mode 100644
index 0000000000..9a69de92c8
--- /dev/null
+++ b/ci-automation/image.sh
@@ -0,0 +1,113 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
+#
+# image_build() should be called w/ the positional INPUT parameters below.
+
+# Binary OS image build automation stub.
+#   This script will build the OS image from a pre-built packages container.
+#
+# PREREQUISITES:
+#
+#   1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt
+#   2. Scripts repo version tag of OS image version to be built is available and checked out.
+#   3. Flatcar packages container is available via build cache server
+#       from "/containers/[VERSION]/flatcar-packages-[ARCH]-[FLATCAR_VERSION].tar.gz"
+#       or present locally. Container must contain binary packages and torcx artefacts.
+#
+# INPUT:
+#
+#   1. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64").
+#
+# OPTIONAL INPUT:
+#
+#   1. SIGNER. Environment variable. Name of the owner of the artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored.
+#
+#   2. SIGNING_KEY. Environment variable. The artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored.
+#
+# OUTPUT:
+#
+#   1. OS image, dev container, related artifacts, and torcx packages pushed to buildcache.
+#   2. "./ci-cleanup.sh" with commands to clean up temporary build resources,
+#        to be run after this step finishes / when this step is aborted.
+#   3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache.
+#   4. DIGESTS of the artifacts from point 1, pushed to buildcache. If signer key was passed, armored ASCII files of the generated DIGESTS files too, pushed to buildcache.
+
+function image_build() {
+    # Run a subshell, so the traps, environment changes and global
+    # variables are not spilled into the caller.
+    (
+        set -euo pipefail
+
+        _image_build_impl "${@}"
+    )
+}
+# --
+
+function _image_build_impl() {
+    local arch="$1"
+
+    source sdk_lib/sdk_container_common.sh
+    local channel=""
+    channel="$(get_git_channel)"
+    source ci-automation/ci_automation_common.sh
+    source ci-automation/gpg_setup.sh
+    init_submodules
+
+    source sdk_container/.repo/manifests/version.txt
+    local vernum="${FLATCAR_VERSION}"
+    local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
+
+    local packages="flatcar-packages-${arch}"
+    local packages_image="${packages}:${docker_vernum}"
+
+    docker_image_from_buildcache "${packages}" "${docker_vernum}"
+
+    local image="flatcar-images-${arch}"
+    local image_container="${image}-${docker_vernum}"
+
+    local official_arg=""
+    if is_official "${vernum}"; then
+            export COREOS_OFFICIAL=1
+            official_arg="--official"
+    else
+            export COREOS_OFFICIAL=0
+            official_arg="--noofficial"
+    fi
+
+    # build image and related artifacts
+    ./run_sdk_container -x ./ci-cleanup.sh -n "${image_container}" -C "${packages_image}" \
+            -v "${vernum}" \
+            mkdir -p "${CONTAINER_IMAGE_ROOT}"
+    ./run_sdk_container -n "${image_container}" -C "${packages_image}" \
+            -v "${vernum}" \
+            ./set_official --board="${arch}-usr" "${official_arg}"
+    ./run_sdk_container -n "${image_container}" -C "${packages_image}" \
+            -v "${vernum}" \
+            ./build_image --board="${arch}-usr" --group="${channel}" \
+                          --output_root="${CONTAINER_IMAGE_ROOT}" \
+                          --only_store_compressed \
+                          --torcx_root="${CONTAINER_TORCX_ROOT}" prodtar container
+
+    # copy resulting images + push to buildcache
+    local images_out="images/"
+    rm -rf "${images_out}"
+    ./run_sdk_container -n "${image_container}" -C "${packages_image}" \
+            -v "${vernum}" \
+            mv "${CONTAINER_IMAGE_ROOT}/${arch}-usr/" "./${images_out}/"
+
+    # Delete uncompressed generic image before signing and upload
+    rm "images/latest/flatcar_production_image.bin" "images/latest/flatcar_production_update.bin"
+    create_digests "${SIGNER}" "images/latest/"*
+    sign_artifacts "${SIGNER}" "images/latest/"*
+    copy_to_buildcache "images/${arch}/${vernum}/" "images/latest/"*
+}
+# --
diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
new file mode 100644
index 0000000000..e17eb10a0b
--- /dev/null
+++ b/ci-automation/image_changes.sh
@@ -0,0 +1,122 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
+#
+# image_changes() should be called w/ the positional INPUT parameters below.
+
+# OS image differences display stub.
+#   This script will display the differences between the last released image and the currently built one.
+#
+# PREREQUISITES:
+#
+#   1. Artifacts describing the built image (kernel config, contents, packages, etc.) must be present in build cache server.
+#   2. Scripts repo version tag of OS image version to be built is available and checked out.
+#
+# INPUT:
+#
+#   1. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64").
+#
+# OPTIONAL INPUT:
+#
+#   (none)
+#
+# OUTPUT:
+#
+#   1. Currently the script prints the image differences compared to the last release and the changelog for the release notes but doesn't store it yet in the buildcache.
+
+function image_changes() {
+    # Run a subshell, so the traps, environment changes and global
+    # variables are not spilled into the caller.
+    (
+        set -euo pipefail
+
+        _image_changes_impl "${@}"
+    )
+}
+# --
+
+function _image_changes_impl() {
+    local arch="$1"
+
+    source sdk_lib/sdk_container_common.sh
+    local channel=""
+    channel="$(get_git_channel)"
+    source ci-automation/ci_automation_common.sh
+    source ci-automation/gpg_setup.sh
+    init_submodules
+
+    source sdk_container/.repo/manifests/version.txt
+    local vernum="${FLATCAR_VERSION}"
+
+    echo "==================================================================="
+    export BOARD_A="${arch}-usr"
+    export FROM_A="release"
+    if [ "${channel}" = "developer" ]; then
+            NEW_CHANNEL="alpha"
+    else
+            NEW_CHANNEL="${channel}"
+    fi
+    NEW_CHANNEL_VERSION_A=$(curl -fsSL --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 "https://${NEW_CHANNEL}.release.flatcar-linux.net/${BOARD_A}/current/version.txt" | grep -m 1 FLATCAR_VERSION= | cut -d = -f 2)
+    MAJOR_A=$(echo "${NEW_CHANNEL_VERSION_A}" | cut -d . -f 1)
+    MAJOR_B=$(echo "${FLATCAR_VERSION}" | cut -d . -f 1)
+    # When the major version for the new channel is different, a transition has happened and we can find the previous release in the old channel
+    if [ "${MAJOR_A}" != "${MAJOR_B}" ]; then
+        case "${NEW_CHANNEL}" in
+          lts)
+            CHANNEL_A=stable
+            ;;
+          stable)
+            CHANNEL_A=beta
+            ;;
+          *)
+            CHANNEL_A=alpha
+            ;;
+        esac
+        VERSION_A=$(curl -fsSL --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 "https://${CHANNEL_A}.release.flatcar-linux.net/${BOARD_A}/current/version.txt" | grep -m 1 FLATCAR_VERSION= | cut -d = -f 2)
+    else
+        CHANNEL_A="${NEW_CHANNEL}"
+        VERSION_A="${NEW_CHANNEL_VERSION_A}"
+    fi
+    export VERSION_A
+    export CHANNEL_A
+    export FROM_B="bincache"
+    export VERSION_B="${vernum}"
+    export BOARD_B="${arch}-usr"
+    # CHANNEL_B is unused
+    echo "== Image differences compared to ${CHANNEL_A} ${VERSION_A} =="
+    NEW_VERSION=$(git tag --points-at HEAD)
+    cd ..
+    rm -rf flatcar-build-scripts
+    git clone "https://github.com/flatcar/flatcar-build-scripts"
+    # Don't fail the job
+    set +e
+    echo "Package updates, compared to ${CHANNEL_A} ${VERSION_A}:"
+    FILE=flatcar_production_image_packages.txt flatcar-build-scripts/package-diff "${VERSION_A}" "${VERSION_B}"
+    echo
+    echo "Image file changes, compared to ${CHANNEL_A} ${VERSION_A}:"
+    FILE=flatcar_production_image_contents.txt FILESONLY=1 CUTKERNEL=1 flatcar-build-scripts/package-diff "${VERSION_A}" "${VERSION_B}"
+    echo
+    echo "Image kernel config changes, compared to ${CHANNEL_A} ${VERSION_A}:"
+    FILE=flatcar_production_image_kernel_config.txt flatcar-build-scripts/package-diff "${VERSION_A}" "${VERSION_B}"
+    echo
+    echo "Image file size change (includes /boot, /usr and the default rootfs partitions), compared to ${CHANNEL_A} ${VERSION_A}:"
+    FILE=flatcar_production_image_contents.txt CALCSIZE=1 flatcar-build-scripts/package-diff "${VERSION_A}" "${VERSION_B}"
+    echo
+    BASE_URL="http://${BUILDCACHE_SERVER}/images/${arch}/${vernum}"
+    echo "Image URL: ${BASE_URL}/flatcar_production_image.bin.bz2"
+    echo
+    # Provide a python3 command for the CVE DB parsing
+    export PATH="$PATH:$PWD/scripts/ci-automation/python-bin"
+    # The first changelog we print is always against the previous version of the new channel (is only same as CHANNEL_A VERSION_A without a transition)
+    flatcar-build-scripts/show-changes "${NEW_CHANNEL}-${NEW_CHANNEL_VERSION_A}" "${NEW_VERSION}"
+    # See if a channel transition happened and print the changelog against CHANNEL_A VERSION_A which is the previous release
+    if [ "${CHANNEL_A}" != "${NEW_CHANNEL}" ]; then
+      flatcar-build-scripts/show-changes "${CHANNEL_A}-${VERSION_A}" "${NEW_VERSION}"
+    fi
+    set -e
+}
+# --
diff --git a/ci-automation/packages-tag.sh b/ci-automation/packages-tag.sh
new file mode 100644
index 0000000000..7ae886866f
--- /dev/null
+++ b/ci-automation/packages-tag.sh
@@ -0,0 +1,120 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
+#
+# packages_tag() should be called w/ the positional INPUT parameters below.
+
+# build tag automation stub.
+#   This script will update the versionfile with the OS packages version to build,
+#    and will add a version tag (see INPUT) to the scripts repo.
+#
+# PREREQUISITES:
+#
+#   1. SDK version is recorded in sdk_container/.repo/manifests/version.txt
+#   2. SDK container is either
+#       - available via ghcr.io/flatcar/flatcar-sdk-[ARCH]:[VERSION] (official SDK release)
+#       OR
+#       - available via build cache server "/containers/[VERSION]/flatcar-sdk-[ARCH]-[VERSION].tar.gz"
+#         (dev SDK)
+#
+# INPUT:
+#
+#   1. Version of the TARGET OS image to build (string).
+#       The version pattern '(alpha|beta|stable|lts)-MMMM.m.p' (e.g. 'alpha-3051.0.0')
+#         denotes a "official" build, i.e. a release build to be published.
+#       Use any version diverging from the pattern (e.g. 'alpha-3051.0.0-nightly-4302') for development / CI builds.
+#       A tag of this version will be created in the scripts repo and pushed upstream.
+#
+#
+# OPTIONAL INPUT:
+#
+#   2. coreos-overlay repository tag to use (commit-ish).
+#       Optional - use scripts repo sub-modules as-is if not set.
+#       This version will be checked out / pulled from remote in the coreos-overlay git submodule.
+#       The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed.
+#
+#   3. portage-stable repository tag to use (commit-ish).
+#       Optional - use scripts repo sub-modules as-is if not set.
+#       This version will be checked out / pulled from remote in the portage-stable git submodule.
+#       The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed.
+#
+# OUTPUT:
+#
+#   1. Updated scripts repository
+#        - version tag w/ submodules
+#        - sdk_container/.repo/manifests/version.txt denotes new FLATCAR OS version
+#   2. "./skip-build" as flag file to signal that the build should stop
+
+function packages_tag() {
+    # Run a subshell, so the traps, environment changes and global
+    # variables are not spilled into the caller.
+    (
+        set -euo pipefail
+
+        _packages_tag_impl "${@}"
+    )
+}
+# --
+
+function _packages_tag_impl() {
+    local version="$1"
+    local coreos_git="${2:-}"
+    local portage_git="${3:-}"
+
+    source ci-automation/ci_automation_common.sh
+    source ci-automation/gpg_setup.sh
+    init_submodules
+
+    check_version_string "${version}"
+
+    source sdk_container/.repo/manifests/version.txt
+    local sdk_version="${FLATCAR_SDK_VERSION}"
+
+    if [ -n "${coreos_git}" ] ; then
+        update_submodule "coreos-overlay" "${coreos_git}"
+    fi
+    if [ -n "${portage_git}" ] ; then
+        update_submodule "portage-stable" "${portage_git}"
+    fi
+
+    # Create new tag in scripts repo w/ updated versionfile + submodules.
+    # Also push the changes to the branch ONLY IF we're doing a nightly
+    #   build of the 'main'/'flatcar-MAJOR' branch AND we're definitely ON the respective branch
+    #   (`scripts` and submodules).
+    local push_branch="false"
+    if    [[ "${version}" =~ ^(stable|alpha|beta|lts)-[0-9.]+-nightly-[-0-9]+$ ]] \
+       && [[ "$(git rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] \
+       && [[ "$(git -C sdk_container/src/third_party/coreos-overlay/ rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] \
+       && [[ "$(git -C sdk_container/src/third_party/portage-stable/ rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] ; then
+        push_branch="true"
+        local existing_tag=""
+        existing_tag=$(git tag --points-at HEAD) # exit code is always 0, output may be empty
+        # If the found tag is a release or nightly tag, we stop this build if there are no changes
+        if [[ "${existing_tag}" =~ ^(stable|alpha|beta|lts)-[0-9.]+(|-nightly-[-0-9]+)$ ]]; then
+          local ret=0
+          git diff --exit-code "${existing_tag}" || ret=$?
+          if [[ ret -eq 0 ]]; then
+            touch ./skip-build
+            echo "Creating ./skip-build flag file, indicating that the build must not to continue because no new tag got created as there are no changes since tag ${existing_tag}" >&2
+            return 0
+          elif [[ ret -eq 1 ]]; then
+            echo "Found changes since last tag ${existing_tag}" >&2
+          else
+            echo "Error: Unexpected git diff return code (${ret})" >&2
+            return 1
+          fi
+        fi
+    fi
+
+    # Create version file
+    (
+      source sdk_lib/sdk_container_common.sh
+      create_versionfile "$sdk_version" "$version"
+    )
+    update_and_push_version "${version}" "${push_branch}"
+}
+# --
diff --git a/ci-automation/packages.sh b/ci-automation/packages.sh
new file mode 100644
index 0000000000..465eb4d58e
--- /dev/null
+++ b/ci-automation/packages.sh
@@ -0,0 +1,136 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
+#
+# packages_build() should be called w/ the positional INPUT parameters below.
+
+# OS image binary packages build automation stub.
+#   This script will use an SDK container to build packages for an OS image.
+#
+# PREREQUISITES:
+#
+#   1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt
+#   2. Scripts repo version tag of OS image version to be built is available and checked out.
+#   3. SDK container is either
+#       - available via ghcr.io/flatcar/flatcar-sdk-[ARCH]:[VERSION] (official SDK release)
+#       OR
+#       - available via build cache server "/containers/[VERSION]/flatcar-sdk-[ARCH]-[VERSION].tar.gz"
+#         (dev SDK)
+#
+# INPUT:
+#
+#   1. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64").
+#
+#
+# OPTIONAL INPUT:
+#
+#   2. SIGNER. Environment variable. Name of the owner of the artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored.
+#
+#   3. SIGNING_KEY. Environment variable. The artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored.
+#
+# OUTPUT:
+#
+#   1. Exported container image "flatcar-packages-[ARCH]-[VERSION].tar.gz" with binary packages
+#       pushed to buildcache, and torcx_manifest.json pushed to "images/${arch}/${vernum}/"
+#       (for use with tests).
+#   2. "./ci-cleanup.sh" with commands to clean up temporary build resources,
+#        to be run after this step finishes / when this step is aborted.
+#   3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache.
+#   4. DIGESTS of the artifacts from point 1, pushed to buildcache. If signer key was passed, armored ASCII files of the generated DIGESTS files too, pushed to buildcache.
+
+function packages_build() {
+    # Run a subshell, so the traps, environment changes and global
+    # variables are not spilled into the caller.
+    (
+        set -euo pipefail
+
+        _packages_build_impl "${@}"
+    )
+}
+# --
+
+function _packages_build_impl() {
+    local arch="$1"
+
+    source ci-automation/ci_automation_common.sh
+    source ci-automation/gpg_setup.sh
+    init_submodules
+
+    source sdk_container/.repo/manifests/version.txt
+    local sdk_version="${FLATCAR_SDK_VERSION}"
+
+    # Get SDK from either the registry or import from build cache
+    # This is a NOP if the image is present locally.
+    local sdk_name="flatcar-sdk-${arch}"
+    local docker_sdk_vernum="$(vernum_to_docker_image_version "${sdk_version}")"
+
+    docker_image_from_registry_or_buildcache "${sdk_name}" "${docker_sdk_vernum}"
+    local sdk_image="$(docker_image_fullname "${sdk_name}" "${docker_sdk_vernum}")"
+    echo "docker image rm -f '${sdk_image}'" >> ./ci-cleanup.sh
+
+    # Set name of the packages container for later rename / export
+    local vernum="${FLATCAR_VERSION}"
+    local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
+    local packages_container="flatcar-packages-${arch}-${docker_vernum}"
+    local torcx_pkg_url="https://${BUILDCACHE_SERVER}/images/${arch}/${vernum}/torcx"
+
+    source sdk_lib/sdk_container_common.sh
+
+    if is_official "${vernum}"; then
+        # A channel returned by get_git_channel should not ever be
+        # "developer" here, because it's an official build done from
+        # one of the maintenance branches. So if the channel happens
+        # to be "developer", then you are doing it wrong (releasing
+        # from the main branch?).
+        torcx_pkg_url="https://$(get_git_channel).release.flatcar-linux.net/${arch}-usr/${vernum}/torcx"
+    fi
+
+    # Build packages; store packages and torcx output in container
+    ./run_sdk_container -x ./ci-cleanup.sh -n "${packages_container}" -v "${vernum}" \
+        -C "${sdk_image}" \
+        mkdir -p "${CONTAINER_TORCX_ROOT}"
+    ./run_sdk_container -n "${packages_container}" -v "${vernum}" \
+        -C "${sdk_image}" \
+        ./build_packages --board="${arch}-usr" \
+            --torcx_output_root="${CONTAINER_TORCX_ROOT}" \
+            --torcx_extra_pkg_url="${torcx_pkg_url}"
+
+    # copy torcx manifest and docker tarball for publishing
+    local torcx_tmp="__build__/torcx_tmp"
+    rm -rf "${torcx_tmp}"
+    mkdir "${torcx_tmp}"
+    ./run_sdk_container -n "${packages_container}" -v "${vernum}" \
+        -C "${sdk_image}" \
+        cp -r "${CONTAINER_TORCX_ROOT}/" \
+        "${torcx_tmp}"
+
+    # run_sdk_container updates the version file, use that version from here on
+    source sdk_container/.repo/manifests/version.txt
+    local vernum="${FLATCAR_VERSION}"
+    local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
+    local packages_image="flatcar-packages-${arch}"
+
+    # generate image + push to build cache
+    docker_commit_to_buildcache "${packages_container}" "${packages_image}" "${docker_vernum}"
+
+    # Publish torcx manifest and docker tarball to "images" cache so tests can pull it later.
+    create_digests "${SIGNER}" \
+        "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json" \
+        "${torcx_tmp}/torcx/pkgs/${arch}-usr/docker/"*/*.torcx.tgz
+    sign_artifacts "${SIGNER}" \
+        "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json"* \
+        "${torcx_tmp}/torcx/pkgs/${arch}-usr/docker/"*/*.torcx.tgz*
+    copy_to_buildcache "images/${arch}/${vernum}/torcx" \
+        "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json"*
+    copy_to_buildcache "images/${arch}/${vernum}/torcx" \
+        "${torcx_tmp}/torcx/pkgs/${arch}-usr/docker/"*/*.torcx.tgz*
+}
+# --
diff --git a/ci-automation/push_pkgs.sh b/ci-automation/push_pkgs.sh
new file mode 100644
index 0000000000..4b0c153148
--- /dev/null
+++ b/ci-automation/push_pkgs.sh
@@ -0,0 +1,99 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
+#
+# push_packages() should be called w/ the positional INPUT parameters below.
+
+# OS image binary packages publisher automation stub.
+#   This script will publish the packages from a pre-built packages container to
+#   the buildcache server, effectively turning the build cache into a
+#   binary packages server for the SDK.
+#
+# PREREQUISITES:
+#
+#   1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt
+#   2. Scripts repo version tag of OS image version to be built is available and checked out.
+#   3. Flatcar packages container is available via build cache server
+#       from "/containers/[VERSION]/flatcar-packages-[ARCH]-[FLATCAR_VERSION].tar.gz"
+#       or present locally. Container must contain binary packages and torcx artifacts.
+#
+# INPUT:
+#
+#   1. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64").
+#
+# OPTIONAL INPUT:
+#
+#   1. SIGNER. Environment variable. Name of the owner of the artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored.
+#
+#   2. SIGNING_KEY. Environment variable. The artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored.
+#
+# OUTPUT:
+#
+#   1. Binary packages published to buildcache at "boards/[ARCH]-usr/[VERSION]/pkgs".
+#   2. "./ci-cleanup.sh" with commands to clean up temporary build resources,
+#        to be run after this step finishes / when this step is aborted.
+#   3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache.
+
+# This function is run _inside_ the SDK container
+function image_build__copy_to_bincache() {
+    local arch="$1"
+    local version="$2"
+
+    source ci-automation/ci_automation_common.sh
+
+    # change the owner of the files and directories in __build__ back
+    # to ourselves, otherwise we could fail to sign the artifacts as
+    # we lacked write permissions in the directory of the signed
+    # artifact
+    local uid=$(id --user)
+    local gid=$(id --group)
+    cd /build/$arch-usr/var/lib/portage/pkgs/
+    sudo chown --recursive "${uid}:${gid}" .
+    sign_artifacts "${SIGNER}" *
+    copy_to_buildcache "boards/$arch-usr/$version/pkgs" *
+}
+# --
+
+function push_packages() {
+    # Run a subshell, so the traps, environment changes and global
+    # variables are not spilled into the caller.
+    (
+        set -euo pipefail
+
+        _push_packages_impl "${@}"
+    )
+}
+# --
+
+function _push_packages_impl() {
+    local arch="$1"
+
+    source ci-automation/ci_automation_common.sh
+    source ci-automation/gpg_setup.sh
+    init_submodules
+
+    source sdk_container/.repo/manifests/version.txt
+    local vernum="${FLATCAR_VERSION}"
+    local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
+
+    local packages="flatcar-packages-${arch}"
+    local packages_image="${packages}:${docker_vernum}"
+
+    docker_image_from_buildcache "${packages}" "${docker_vernum}"
+
+    local cmd="source ci-automation/push_pkgs.sh"
+    cmd="$cmd; image_build__copy_to_bincache '$arch' '$vernum'"
+
+    local my_name="flatcar-packages-publisher-${arch}-${docker_vernum}"
+    ./run_sdk_container -x ./ci-cleanup.sh -n "${my_name}" -C "${packages_image}" \
+            bash -c "$cmd"
+}
+# --
diff --git a/ci-automation/python-bin/python3 b/ci-automation/python-bin/python3
new file mode 100755
index 0000000000..ccc29501dc
--- /dev/null
+++ b/ci-automation/python-bin/python3
@@ -0,0 +1,6 @@
+#!/bin/sh
+# Expects to be invoked as interpreter through a shebang
+FOLDER="$(dirname "$(readlink -f "$1")")"
+docker pull docker.io/python:alpine 2>/dev/null >/dev/null
+# Map the current and the script folder, install the pip package needed for flatcar-build-scripts/show-fixed-kernel-cves.py
+exec docker run --rm -i -v "${FOLDER}:${FOLDER}" -v "${PWD}:${PWD}" -w "${PWD}" docker.io/python:alpine sh -c "pip install packaging 2>/dev/null >/dev/null; python3 $*"
diff --git a/ci-automation/release.sh b/ci-automation/release.sh
new file mode 100644
index 0000000000..5d8fa263a9
--- /dev/null
+++ b/ci-automation/release.sh
@@ -0,0 +1,405 @@
+#!/bin/bash
+
+# Copyright (c) 2022 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
+#
+# release_build() is currently called with no positional INPUT parameters but uses the signing env vars.
+
+# Release build automation stub.
+#   This script will release the image build from bincache to the cloud offers.
+#
+# PREREQUISITES:
+#
+#   1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt
+#   2. Scripts repo version tag of OS image version to be built is available and checked out.
+#   3. Mantle container docker image reference is stored in sdk_container/.repo/manifests/mantle-container.
+#   4. Vendor image and torcx docker tarball + manifest to run tests for are available on buildcache
+#         ( images/[ARCH]/[FLATCAR_VERSION]/ )
+#   5. SDK container is either
+#       - available via ghcr.io/flatcar/flatcar-sdk-[ARCH]:[VERSION] (official SDK release)
+#       OR
+#       - available via build cache server "/containers/[VERSION]/flatcar-sdk-[ARCH]-[VERSION].tar.gz"
+#         (dev SDK)
+#
+# INPUT:
+#
+#   (none)
+#
+# OPTIONAL INPUT:
+#
+#   1. SIGNER. Environment variable. Name of the owner of the artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored.
+#
+#   2. SIGNING_KEY. Environment variable. The artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored.
+#
+#   3. REGISTRY_USERNAME. Environment variable. The username to use for Docker registry login.
+#        Defaults to nothing if not set - in such case, SDK container will not be pushed.
+#
+#   4. REGISTRY_PASSWORD. Environment variable. The password to use for Docker registry login.
+#        Defaults to nothing if not set - in such case, SDK container will not be pushed.
+#
+#   5. Cloud credentials as secrets via the environment variables AZURE_PROFILE, AZURE_AUTH_CREDENTIALS,
+#      AWS_CREDENTIALS, AWS_MARKETPLACE_CREDENTIALS, AWS_MARKETPLACE_ARN, AWS_CLOUDFORMATION_CREDENTIALS,
+#      GCP_JSON_KEY, GOOGLE_RELEASE_CREDENTIALS.
+#
+# OUTPUT:
+#
+#   1. The cloud images are published with mantle's plume and ore tools
+#   2. The AWS AMI text files are pushed to buildcache ( images/[ARCH]/[FLATCAR_VERSION]/ )
+#   3. "./ci-cleanup.sh" with commands to clean up temporary build resources,
+#        to be run after this step finishes / when this step is aborted.
+#   4. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache.
+#   5. DIGESTS of the artifacts from point 1, pushed to buildcache. If signer key was passed, armored ASCII files of the generated DIGESTS files too, pushed to buildcache.
+
+function release_build() {
+    # Run a subshell, so the traps, environment changes and global
+    # variables are not spilled into the caller.
+    (
+        set -euo pipefail
+
+        _release_build_impl "${@}"
+    )
+}
+
+function _inside_mantle() {
+  # Run a subshell for the same reasons as above
+  (
+    set -euo pipefail
+
+    source sdk_lib/sdk_container_common.sh
+    source ci-automation/ci_automation_common.sh
+    source sdk_container/.repo/manifests/version.txt
+    # Needed because we are not the SDK container here
+    source sdk_container/.env
+    CHANNEL="$(get_git_channel)"
+    VERSION="${FLATCAR_VERSION}"
+    azure_profile_config_file=""
+    secret_to_file azure_profile_config_file "${AZURE_PROFILE}"
+    azure_auth_config_file=""
+    secret_to_file azure_auth_config_file "${AZURE_AUTH_CREDENTIALS}"
+    aws_credentials_config_file=""
+    secret_to_file aws_credentials_config_file "${AWS_CREDENTIALS}"
+    aws_marketplace_credentials_file=""
+    secret_to_file aws_marketplace_credentials_file "${AWS_MARKETPLACE_CREDENTIALS}"
+    gcp_json_key_path=""
+    secret_to_file gcp_json_key_path "${GCP_JSON_KEY}"
+    google_release_credentials_file=""
+    secret_to_file google_release_credentials_file "${GOOGLE_RELEASE_CREDENTIALS}"
+
+    for platform in aws azure; do
+      for arch in amd64 arm64; do
+        # Create a folder where plume stores flatcar_production_ami_*txt and flatcar_production_ami_*json
+        # for later push to bincache
+        rm -rf "${platform}-${arch}"
+        mkdir "${platform}-${arch}"
+        pushd "${platform}-${arch}"
+
+        # For pre-release we don't use the Google Cloud token because it's not needed
+        # and we don't want to upload the AMIs to GCS anymore
+        # (change https://github.com/flatcar/mantle/blob/bc6bc232677c45e389feb221da295cc674882f8c/cmd/plume/prerelease.go#L663-L667
+        # if you want to add GCP release code in plume pre-release instead of plume release)
+        plume pre-release --force \
+          --debug \
+          --platform="${platform}" \
+          --aws-credentials="${aws_credentials_config_file}" \
+          --azure-profile="${azure_profile_config_file}" \
+          --azure-auth="${azure_auth_config_file}" \
+          --gce-json-key=none \
+          --board="${arch}-usr" \
+          --channel="${CHANNEL}" \
+          --version="${FLATCAR_VERSION}" \
+          --write-image-list="images.json"
+        popd
+      done
+    done
+    for arch in amd64 arm64; do
+      # Create a folder where plume stores any temporarily downloaded files
+      rm -rf "release-${arch}"
+      mkdir "release-${arch}"
+      pushd "release-${arch}"
+
+      export product="${CHANNEL}-${arch}"
+      pid=$(jq -r ".[env.product]" ../product-ids.json)
+
+      # If the channel is 'stable' and the arch 'amd64', we add the stable-pro-amd64 product ID to the product IDs.
+      # The published AMI ID is the same for both offer.
+      [[ "${CHANNEL}" == "stable" ]] && [[ "${arch}" == "amd64" ]] && pid="${pid},$(jq -r '.["stable-pro-amd64"]' ../product-ids.json)"
+
+      plume release \
+        --debug \
+        --aws-credentials="${aws_credentials_config_file}" \
+        --aws-marketplace-credentials="${aws_marketplace_credentials_file}" \
+        --publish-marketplace \
+        --access-role-arn="${AWS_MARKETPLACE_ARN}" \
+        --product-ids="${pid}" \
+        --azure-profile="${azure_profile_config_file}" \
+        --azure-auth="${azure_auth_config_file}" \
+        --gce-json-key="${gcp_json_key_path}" \
+        --gce-release-key="${google_release_credentials_file}" \
+        --board="${arch}-usr" \
+        --channel="${CHANNEL}" \
+        --version="${VERSION}"
+        popd
+    done
+
+    # Future: move this to "plume release", in the past this was done in "update-cloudformation-template"
+    aws_cloudformation_credentials_file=""
+    secret_to_file aws_cloudformation_credentials_file "${AWS_CLOUDFORMATION_CREDENTIALS}"
+    export AWS_SHARED_CREDENTIALS_FILE="${aws_cloudformation_credentials_file}"
+    rm -rf cloudformation-files
+    mkdir cloudformation-files
+    for arch in amd64 arm64; do
+      generate_templates "aws-${arch}/flatcar_production_ami_all.json" "${CHANNEL}" "${arch}-usr"
+    done
+    aws s3 cp --recursive --acl public-read cloudformation-files/ "s3://flatcar-prod-ami-import-eu-central-1/dist/aws/"
+  )
+}
+
+function publish_sdk() {
+    local docker_sdk_vernum="$1"
+    local sdk_name=""
+
+    # If the registry password or the registry username is not set, we leave early.
+    [[ -z "${REGISTRY_PASSWORD}" ]] || [[ -z "${REGISTRY_USERNAME}" ]] && return
+
+    (
+      # Don't print the password to stderr when logging in
+      set +x
+      local container_registry=""
+      container_registry=$(echo "${sdk_container_common_registry}" | cut -d / -f 1)
+      echo "${REGISTRY_PASSWORD}" | docker login "${container_registry}" -u "${REGISTRY_USERNAME}" --password-stdin
+    )
+
+    # Docker images are pushed in the container registry.
+    for a in all amd64 arm64; do
+      sdk_name="flatcar-sdk-${a}"
+      docker_image_from_registry_or_buildcache "${sdk_name}" "${docker_sdk_vernum}"
+      docker push "${sdk_container_common_registry}/flatcar-sdk-${a}:${docker_sdk_vernum}"
+    done
+}
+
+function _release_build_impl() {
+    source sdk_lib/sdk_container_common.sh
+    source ci-automation/ci_automation_common.sh
+    source ci-automation/gpg_setup.sh
+    init_submodules
+
+    source sdk_container/.repo/manifests/version.txt
+    # Needed because we are not the SDK container here
+    source sdk_container/.env
+    local sdk_version="${FLATCAR_SDK_VERSION}"
+    local docker_sdk_vernum=""
+    docker_sdk_vernum="$(vernum_to_docker_image_version "${sdk_version}")"
+    local vernum="${FLATCAR_VERSION}"
+    local docker_vernum=""
+    docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
+
+    local container_name="flatcar-publish-${docker_vernum}"
+    local mantle_ref
+    mantle_ref=$(cat sdk_container/.repo/manifests/mantle-container)
+    # A job on each worker prunes old mantle images (docker image prune), no need to do it here
+    echo "docker rm -f '${container_name}'" >> ./ci-cleanup.sh
+
+    touch sdk_container/.env # This file should already contain the required credentials as env vars
+    docker run --pull always --rm --name="${container_name}" --net host \
+      -w /work -v "$PWD":/work "${mantle_ref}" bash -c "source ci-automation/release.sh; _inside_mantle"
+    # Push flatcar_production_ami_*txt and flatcar_production_ami_*json to the right bincache folder
+    for arch in amd64 arm64; do
+      sudo chown -R "$USER:$USER" "aws-${arch}"
+      create_digests "${SIGNER}" "aws-${arch}/flatcar_production_ami_"*txt "aws-${arch}/flatcar_production_ami_"*json
+      sign_artifacts "${SIGNER}" "aws-${arch}/flatcar_production_ami_"*txt "aws-${arch}/flatcar_production_ami_"*json
+      copy_to_buildcache "images/${arch}/${vernum}/" "aws-${arch}/flatcar_production_ami_"*txt* "aws-${arch}/flatcar_production_ami_"*json*
+    done
+    publish_sdk "${docker_sdk_vernum}"
+    echo "===="
+    echo "Done, now you can copy the images to Origin"
+    echo "===="
+    # Future: trigger copy to Origin in a secure way
+    # Future: trigger update payload signing
+    # Future: trigger website update
+    # Future: trigger release email sending
+    # Future: trigger push to nebraska
+    # Future: trigger Origin symlink switch
+}
+
+TEMPLATE='
+{
+  "AWSTemplateFormatVersion": "2010-09-09",
+  "Description": "Flatcar Linux on EC2: https://kinvolk.io/docs/flatcar-container-linux/latest/installing/cloud/aws-ec2/",
+  "Mappings" : {
+      "RegionMap" : {
+###AMIS###
+      }
+  },
+  "Parameters": {
+    "InstanceType" : {
+      "Description" : "EC2 HVM instance type (m3.medium, etc).",
+      "Type" : "String",
+      "Default" : "m3.medium",
+      "ConstraintDescription" : "Must be a valid EC2 HVM instance type."
+    },
+    "ClusterSize": {
+      "Default": "3",
+      "MinValue": "3",
+      "MaxValue": "12",
+      "Description": "Number of nodes in cluster (3-12).",
+      "Type": "Number"
+    },
+    "DiscoveryURL": {
+      "Description": "An unique etcd cluster discovery URL. Grab a new token from https://discovery.etcd.io/new?size=<your cluster size>",
+      "Type": "String"
+    },
+    "AdvertisedIPAddress": {
+      "Description": "Use 'private' if your etcd cluster is within one region or 'public' if it spans regions or cloud providers.",
+      "Default": "private",
+      "AllowedValues": ["private", "public"],
+      "Type": "String"
+    },
+    "AllowSSHFrom": {
+      "Description": "The net block (CIDR) that SSH is available to.",
+      "Default": "0.0.0.0/0",
+      "Type": "String"
+    },
+    "KeyPair" : {
+      "Description" : "The name of an EC2 Key Pair to allow SSH access to the instance.",
+      "Type" : "String"
+    }
+  },
+  "Resources": {
+    "FlatcarSecurityGroup": {
+      "Type": "AWS::EC2::SecurityGroup",
+      "Properties": {
+        "GroupDescription": "Flatcar Linux SecurityGroup",
+        "SecurityGroupIngress": [
+          {"IpProtocol": "tcp", "FromPort": "22", "ToPort": "22", "CidrIp": {"Ref": "AllowSSHFrom"}}
+        ]
+      }
+    },
+    "Ingress4001": {
+      "Type": "AWS::EC2::SecurityGroupIngress",
+      "Properties": {
+        "GroupName": {"Ref": "FlatcarSecurityGroup"}, "IpProtocol": "tcp", "FromPort": "4001", "ToPort": "4001", "SourceSecurityGroupId": {
+          "Fn::GetAtt" : [ "FlatcarSecurityGroup", "GroupId" ]
+        }
+      }
+    },
+    "Ingress2379": {
+      "Type": "AWS::EC2::SecurityGroupIngress",
+      "Properties": {
+        "GroupName": {"Ref": "FlatcarSecurityGroup"}, "IpProtocol": "tcp", "FromPort": "2379", "ToPort": "2379", "SourceSecurityGroupId": {
+          "Fn::GetAtt" : [ "FlatcarSecurityGroup", "GroupId" ]
+        }
+      }
+    },
+    "Ingress2380": {
+      "Type": "AWS::EC2::SecurityGroupIngress",
+      "Properties": {
+        "GroupName": {"Ref": "FlatcarSecurityGroup"}, "IpProtocol": "tcp", "FromPort": "2380", "ToPort": "2380", "SourceSecurityGroupId": {
+          "Fn::GetAtt" : [ "FlatcarSecurityGroup", "GroupId" ]
+        }
+      }
+    },
+    "FlatcarServerAutoScale": {
+      "Type": "AWS::AutoScaling::AutoScalingGroup",
+      "Properties": {
+        "AvailabilityZones": {"Fn::GetAZs": ""},
+        "LaunchConfigurationName": {"Ref": "FlatcarServerLaunchConfig"},
+        "MinSize": "3",
+        "MaxSize": "12",
+        "DesiredCapacity": {"Ref": "ClusterSize"},
+        "Tags": [
+            {"Key": "Name", "Value": { "Ref" : "AWS::StackName" }, "PropagateAtLaunch": true}
+        ]
+      }
+    },
+    "FlatcarServerLaunchConfig": {
+      "Type": "AWS::AutoScaling::LaunchConfiguration",
+      "Properties": {
+        "ImageId" : { "Fn::FindInMap" : [ "RegionMap", { "Ref" : "AWS::Region" }, "AMI" ]},
+        "InstanceType": {"Ref": "InstanceType"},
+        "KeyName": {"Ref": "KeyPair"},
+        "SecurityGroups": [{"Ref": "FlatcarSecurityGroup"}],
+        "UserData" : { "Fn::Base64":
+          { "Fn::Join": [ "", [
+            "#cloud-config\n\n",
+            "coreos:\n",
+            "  etcd2:\n",
+            "    discovery: ", { "Ref": "DiscoveryURL" }, "\n",
+            "    advertise-client-urls: http://$", { "Ref": "AdvertisedIPAddress" }, "_ipv4:2379\n",
+            "    initial-advertise-peer-urls: http://$", { "Ref": "AdvertisedIPAddress" }, "_ipv4:2380\n",
+            "    listen-client-urls: http://0.0.0.0:2379,http://0.0.0.0:4001\n",
+            "    listen-peer-urls: http://$", { "Ref": "AdvertisedIPAddress" }, "_ipv4:2380\n",
+            "  units:\n",
+            "    - name: etcd2.service\n",
+            "      command: start\n",
+            "    - name: fleet.service\n",
+            "      command: start\n"
+            ] ]
+          }
+        }
+      }
+    }
+  }
+}
+'
+function generate_templates() {
+    local IFILE="$1"
+    local CHANNEL="$2"
+    local BOARD="$3"
+    local TMPFILE=""
+    local ARCHTAG=""
+
+    local REGIONS=("eu-central-1"
+                   "ap-northeast-1"
+                   "ap-northeast-2"
+                   # "ap-northeast-3" #  Disabled for now because we do not have access
+                   "af-south-1"
+                   "ca-central-1"
+                   "ap-south-1"
+                   "sa-east-1"
+                   "ap-southeast-1"
+                   "ap-southeast-2"
+                   "ap-southeast-3"
+                   "us-east-1"
+                   "us-east-2"
+                   "us-west-2"
+                   "us-west-1"
+                   "eu-west-1"
+                   "eu-west-2"
+                   "eu-west-3"
+                   "eu-north-1"
+                   "eu-south-1"
+                   "ap-east-1"
+                   "me-south-1")
+
+    if [ "${BOARD}" = "amd64-usr" ]; then
+      ARCHTAG=""
+    elif [ "${BOARD}" = "arm64-usr" ]; then
+      ARCHTAG="-arm64"
+    else
+      echo "No architecture tag defined for board \"${BOARD}\""
+      exit 1
+    fi
+
+    TMPFILE=$(mktemp)
+
+    >${TMPFILE}
+    for region in "${REGIONS[@]}"; do
+        echo "         \"${region}\" : {" >> ${TMPFILE}
+        echo -n '             "AMI" : ' >> ${TMPFILE}
+        cat "${IFILE}" | jq ".[] | map(select(.name == \"${region}\")) | .[0] | .\"hvm\"" >> ${TMPFILE}
+        echo "         }," >> ${TMPFILE}
+    done
+
+    truncate -s-2 ${TMPFILE}
+
+    echo "${TEMPLATE}" | perl -i -0pe "s/###AMIS###/$(cat -- ${TMPFILE})/g" > "cloudformation-files/flatcar-${CHANNEL}${ARCHTAG}-hvm.template"
+
+    rm "${TMPFILE}"
+}
diff --git a/ci-automation/sdk_bootstrap.sh b/ci-automation/sdk_bootstrap.sh
new file mode 100644
index 0000000000..724acefa0d
--- /dev/null
+++ b/ci-automation/sdk_bootstrap.sh
@@ -0,0 +1,152 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
+#
+# sdk_bootstrap() should be called w/ the positional INPUT parameters below.
+
+# Bootstrap SDK build automation stub.
+#  This script will use a seed SDK container + tarball to bootstrap a 
+#   new SDK tarball.
+#
+# INPUT:
+#
+#   1. Version of the SEED SDK to use (string).
+#       The seed SDK tarball must be available on https://mirror.release.flatcar-linux.net/sdk/ ...
+#       The seed SDK container must be available from https://github.com/orgs/flatcar/packages
+#          (via ghcr.io/flatcar/flatcar-sdk-all:[VERSION]).
+#
+#   2. Version of the TARGET SDK to build (string).
+#       The version pattern 'MMMM.m.p' (e.g. '3051.0.0') denotes a "official" build, i.e. a release build to be published.
+#       Use any version diverging from the pattern (e.g. '3051.0.0-nightly-4302') for development / CI builds.
+#       A free-standing tagged commit will be created in the scripts repo and pushed upstream.
+#
+# OPTIONAL INPUT:
+#
+#   3. coreos-overlay repository tag to use (commit-ish).
+#       This version will be checked out / pulled from remote in the coreos-overlay git submodule.
+#       The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed.
+#       Leave empty to use coreos-overlay as-is.
+#
+#   4. portage-stable repository tag to use (commit-ish).
+#       This version will be checked out / pulled from remote in the portage-stable git submodule.
+#       The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed.
+#       Leave empty to use portage-stable as-is.
+#
+#   5. ARCH. Environment variable. Target architecture for the SDK to run on.
+#        Either "amd64" or "arm64"; defaults to "amd64" if not set.
+#
+#   6. SIGNER. Environment variable. Name of the owner of the artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored.
+#
+#   7. SIGNING_KEY. Environment variable. The artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored.
+#
+# OUTPUT:
+#
+#   1. SDK tarball (gentoo catalyst output) of the new SDK, pushed to buildcache.
+#   2. Updated scripts repository
+#        - version tag w/ submodules
+#        - sdk_container/.repo/manifests/version.txt denotes new SDK version
+#   3. "./ci-cleanup.sh" with commands to clean up temporary build resources,
+#        to be run after this step finishes / when this step is aborted.
+#   4. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache.
+#   5. DIGESTS of the artifacts from point 1, pushed to buildcache. If signer key was passed, armored ASCII files of the generated DIGESTS files too, pushed to buildcache.
+
+function sdk_bootstrap() {
+    # Run a subshell, so the traps, environment changes and global
+    # variables are not spilled into the caller.
+    (
+        set -euo pipefail
+
+        _sdk_bootstrap_impl "${@}"
+    )
+}
+# --
+
+function _sdk_bootstrap_impl() {
+    local seed_version="$1"
+    local version="$2"
+    local coreos_git="${3-}"
+    local portage_git="${4-}"
+    : ${ARCH:="amd64"}
+
+    source ci-automation/ci_automation_common.sh
+    source ci-automation/gpg_setup.sh
+    init_submodules
+
+    check_version_string "${version}"
+
+    if [ -n "${coreos_git}" ] ; then
+        update_submodule "coreos-overlay" "${coreos_git}"
+    fi
+    if [ -n "${portage_git}" ] ; then
+        update_submodule "portage-stable" "${portage_git}"
+    fi
+
+    # Create new tag in scripts repo w/ updated versionfile + submodules.
+    # Also push the changes to the branch ONLY IF we're doing a nightly
+    #   build of the 'main' branch AND we're definitely ON the main branch
+    #   (`scripts` and submodules).
+    local push_branch="false"
+    if   [[ "${version}" =~ ^main-[0-9.]+-nightly-[-0-9]+$ ]] \
+       && [ "$(git rev-parse --abbrev-ref HEAD)" = "main"  ] \
+       && [ "$(git -C sdk_container/src/third_party/coreos-overlay/ rev-parse --abbrev-ref HEAD)" = "main"  ] \
+       && [ "$(git -C sdk_container/src/third_party/portage-stable/ rev-parse --abbrev-ref HEAD)" = "main"  ] ; then
+        push_branch="true"
+        local existing_tag=""
+        existing_tag=$(git tag --points-at HEAD) # exit code is always 0, output may be empty
+        # If the found tag is a nightly tag, we stop this build if there are no changes
+        if [[ "${existing_tag}" =~ ^main-[0-9.]+-nightly-[-0-9]+$ ]]; then
+          local ret=0
+          git diff --exit-code "${existing_tag}" || ret=$?
+          if [ "$ret" = "0" ]; then
+            echo "Stopping build because there are no changes since tag ${existing_tag}" >&2
+            return 0
+          elif [ "$ret" = "1" ]; then
+            echo "Found changes since last tag ${existing_tag}" >&2
+          else
+            echo "Error: Unexpected git diff return code (${ret})" >&2
+            return 1
+          fi
+        fi
+    fi
+
+    local vernum="${version#*-}" # remove alpha-,beta-,stable-,lts- version tag
+    local git_vernum="${vernum}"
+
+    # Update FLATCAR_VERSION[_ID], BUILD_ID, and SDK in versionfile
+    (
+      source sdk_lib/sdk_container_common.sh
+      create_versionfile "${vernum}"
+    )
+    update_and_push_version "${version}" "${push_branch}"
+
+    ./bootstrap_sdk_container -x ./ci-cleanup.sh "${seed_version}" "${vernum}"
+
+    # push SDK tarball to buildcache
+    # Get Flatcar version number format (separator is '+' instead of '-',
+    # equal to $(strip_version_prefix "$version")
+    source sdk_container/.repo/manifests/version.txt
+    local dest_tarball="flatcar-sdk-${ARCH}-${FLATCAR_SDK_VERSION}.tar.bz2"
+
+    # change the owner of the files and directories in __build__ back
+    # to ourselves, otherwise we could fail to sign the artifacts as
+    # we lacked write permissions in the directory of the signed
+    # artifact
+    local uid=$(id --user)
+    local gid=$(id --group)
+    sudo chown --recursive "${uid}:${gid}" __build__
+    (
+      cd "__build__/images/catalyst/builds/flatcar-sdk"
+      create_digests "${SIGNER}" "${dest_tarball}"
+      sign_artifacts "${SIGNER}" "${dest_tarball}"*
+      copy_to_buildcache "sdk/${ARCH}/${FLATCAR_SDK_VERSION}" "${dest_tarball}"*
+    )
+}
+# --
diff --git a/ci-automation/sdk_container.sh b/ci-automation/sdk_container.sh
new file mode 100644
index 0000000000..fa77faa356
--- /dev/null
+++ b/ci-automation/sdk_container.sh
@@ -0,0 +1,79 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
+#
+# sdk_container_build() should be called w/ the positional INPUT parameters below.
+
+# SDK container builder automation stub.
+#  This script will build an SDK container w/ board support from an SDK tarball.
+#  NOTE that SDK tarball and scripts repo must have the same version or building
+#  the SDK container will fail.
+#
+# PREREQUISITES:
+#
+#   1. SDK version is recorded in sdk_container/.repo/manifests/version.txt and a matching
+#       SDK tarball is available on BUILDCACHE/sdk/[ARCH]/[VERSION]/flatcar-sdk-[ARCH]-[VERSION].tar.bz2
+#
+# OPTIONAL INPUT:
+#
+#   2. ARCH. Environment variable. Target architecture for the SDK to run on.
+#        Either "amd64" or "arm64"; defaults to "amd64" if not set.
+#
+#   3. SIGNER. Environment variable. Name of the owner of the artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored.
+#
+#   4. SIGNING_KEY. Environment variable. The artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored.
+#
+# OUTPUT:
+#
+#   1. SDK container image of the new SDK, published to buildcache.
+#   2. "./ci-cleanup.sh" with commands to clean up temporary build resources,
+#        to be run after this step finishes / when this step is aborted.
+#   3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache.
+#   4. DIGESTS of the artifacts from point 1, pushed to buildcache. If signer key was passed, armored ASCII files of the generated DIGESTS files too, pushed to buildcache.
+
+function sdk_container_build() {
+    # Run a subshell, so the traps, environment changes and global
+    # variables are not spilled into the caller.
+    (
+        set -euo pipefail
+
+        _sdk_container_build_impl "${@}"
+    )
+}
+# --
+
+function _sdk_container_build_impl() {
+    : ${ARCH:="amd64"}
+
+    source ci-automation/ci_automation_common.sh
+    source ci-automation/gpg_setup.sh
+
+    init_submodules
+
+    source sdk_container/.repo/manifests/version.txt
+    local vernum="${FLATCAR_SDK_VERSION}"
+    local sdk_tarball="flatcar-sdk-${ARCH}-${vernum}.tar.bz2"
+
+    # __build__ is in .dockerignore, so the tarball is excluded from build context
+    mkdir -p __build__
+    copy_from_buildcache "sdk/${ARCH}/${vernum}/${sdk_tarball}" "./__build__"
+
+
+    # This will update the SDK_VERSION in versionfile
+    ./build_sdk_container_image -x ./ci-cleanup.sh ./__build__/"${sdk_tarball}"
+
+    # push artifacts to build cache
+    local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
+    docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-all" "${docker_vernum}"
+    docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-amd64" "${docker_vernum}"
+    docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-arm64" "${docker_vernum}"
+}
+# --
diff --git a/ci-automation/tapfile_helper_lib.sh b/ci-automation/tapfile_helper_lib.sh
new file mode 100644
index 0000000000..bbb7c6c4c5
--- /dev/null
+++ b/ci-automation/tapfile_helper_lib.sh
@@ -0,0 +1,308 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Helper script for extracting information from TAP files and for merging multiple
+#  TAP files into one report.
+# The script uses a temporary SQLite DB for querying and for result generation.
+#
+# Brief usage overview (scroll down for parameters etc.):
+#   tap_ingest_tapfile - add test results from tap file to the DB 
+#   tap_list_vendors   - list all vendors TAP files have been ingested for
+#   tap_failed_tests_for_vendor - list all tests that never succeded even once, per vendor
+#   tap_generate_report - generate a merged test report
+
+
+TAPFILE_HELPER_DBNAME="results.sqlite3"
+
+# wrapper around sqlite3 w/ retries if DB is locked
+function __sqlite3_wrapper() {
+    local dbfile="${TAPFILE_HELPER_DBNAME}"
+
+    local params=""
+    while [[ "$1" == -* ]] ; do
+        params="$params $1"
+        shift
+    done
+
+    while true; do
+        sqlite3 "${dbfile}" $params "PRAGMA foreign_keys = ON;$@"
+        local ret="$?"
+        if [ "$ret" -ne 5 ] ; then
+            return $ret
+        fi
+        local sleep="$((1 + $RANDOM % 5))"
+        echo "Retrying in ${sleep} seconds." >&2
+        sleep "${sleep}"
+    done
+}
+# --
+
+# Initialise the DB if it wasn't yet.
+function __db_init() {
+
+    __sqlite3_wrapper '
+    CREATE TABLE IF NOT EXISTS "test_case" (
+        "id"    INTEGER,
+        "name"  TEXT UNIQUE,
+        PRIMARY KEY("id")
+    );
+    CREATE TABLE IF NOT EXISTS "vendor" (
+        "id"    INTEGER,
+        "name"  TEXT UNIQUE,
+        PRIMARY KEY("id")
+    );
+    CREATE TABLE IF NOT EXISTS "test_run" (
+        "id"        INTEGER NOT NULL,
+        "result"    INTEGER NOT NULL,
+        "output"    TEXT,
+        "case_id"   INTEGER NOT NULL,
+        "run"       INTEGER NOT NULL,
+        "vendor_id" INTEGER,
+        PRIMARY KEY("id"),
+        FOREIGN KEY("case_id") REFERENCES "test_case"("id"),
+        FOREIGN KEY("vendor_id") REFERENCES "vendor"("id"),
+        UNIQUE (case_id, run, vendor_id)
+    );
+'
+}
+# --
+
+# Read tapfile into temporary DB.
+# INPUT:
+# 1: <tapfile> - tapfile to ingest
+# 2: <vendor>  - vendor (qemu, azure, aws, etc...)
+# 3: <run>     - re-run iteration
+
+function tap_ingest_tapfile() {
+
+    local tapfile="${1}"
+    local vendor="${2}"
+    local run="${3}"
+
+    local result=""
+    local test_name=""
+    local in_error_message=false
+
+    if ! [ -f "${TAPFILE_HELPER_DBNAME}" ] ; then
+       __db_init
+    fi
+
+    # Wrap all SQL commands in a transaction to speed up INSERTs
+    # We will commit intermediately if there's an error message to insert so the
+    # error message file can be reused.
+    local SQL="BEGIN TRANSACTION;"
+    local has_error_message="false"
+
+
+    # run the parse loop in a subshell and clean up temporary error message file on exit
+    (
+        local error_message_file="$(mktemp)"
+        trap "rm -f '${error_message_file}'" EXIT
+        # Example TAP input:
+        # ok - coreos.auth.verify
+        # ok - coreos.locksmith.tls
+        # not ok - cl.filesystem
+        #   ---
+        #   Error: "--- FAIL: cl.filesystem/deadlinks (1.86s)\n            files.go:90: Dead symbolic links found: [/var/lib/flatcar-oem-gce/usr/lib64/python3.9/site-packages/certifi-3021.3.16-py3.9.egg-info]"
+        #   ...
+        # ok - cl.cloudinit.script
+        # ok - kubeadm.v1.22.0.flannel.base
+        while read -r line; do
+            if [[ "${line}" == "1.."* ]] ; then continue; fi
+            if [ "${line}" = "---" ] ; then  # note: read removes leading whitespaces
+                in_error_message=true
+                continue
+            fi
+
+            if $in_error_message ; then
+                if [ "${line}" = "..." ] ; then
+                    in_error_message=false
+                    has_error_message="true"
+                else
+                    # remove special characters and unicode. Jenkins TAP parser don't unicode.
+                    echo -e "$line" \
+                        | LC_ALL=C sed -e 's/^Error: "--- FAIL: /"/' -e 's/^[[:space:]]*//' \
+                              -e "s/[>\\\"']/_/g" -e 's/[[:space:]]/ /g' \
+                              -e 's/.\{200\}/&\n/g' \
+                              -e 's/[^\x1F-\x7F]/?/g' \
+                        >> "${error_message_file}"
+                    continue
+                fi
+            else
+                test_name="$(echo "${line}" | sed 's/^[^-]* - //')"
+                local result_string
+                result_string="$(echo "${line}" | sed 's/ - .*//')"
+                result=0
+                if [ "${result_string}" = "ok" ] ; then
+                    result=1
+                fi
+            fi
+
+            local test_output="/dev/null"
+            if [ "${has_error_message}" = "true" ] ; then
+                test_output="${error_message_file}"
+            fi
+            SQL="${SQL}INSERT OR IGNORE INTO test_case(name) VALUES ('${test_name}');"
+            SQL="${SQL}INSERT OR IGNORE INTO vendor(name) VALUES ('${vendor}');"
+
+            SQL="${SQL}INSERT OR REPLACE INTO test_run(run,result,output,case_id,vendor_id)
+                                 VALUES ('${run}','${result}', readfile('${test_output}'),
+                                         (SELECT id FROM test_case WHERE name='${test_name}'),
+                                         (SELECT id FROM vendor WHERE name='${vendor}'));"
+
+            if [ "${has_error_message}" = "true" ] ; then
+                SQL="${SQL}COMMIT;"
+                __sqlite3_wrapper "${SQL}"
+                truncate --size 0 "${error_message_file}"
+                has_error_message="false"
+                SQL="BEGIN TRANSACTION;"
+            fi
+
+        done < "$tapfile"
+        SQL="${SQL}COMMIT;"
+        __sqlite3_wrapper "${SQL}"
+    )
+}
+# --
+
+# Print a list of all vendors we've seen so far.
+function tap_list_vendors() {
+    __sqlite3_wrapper 'SELECT DISTINCT name from vendor;'
+}
+# --
+
+# List tests that never succeeded for a given vendor.
+# INPUT:
+# 1: <vendor> - Vendor name to check for failed test runs
+function tap_failed_tests_for_vendor() {
+    local vendor="$1"
+
+    __sqlite3_wrapper "
+		SELECT failed.name FROM test_case AS failed
+		WHERE EXISTS (
+				SELECT * FROM test_run AS t, vendor AS v, test_case AS c
+				WHERE t.vendor_id=v.id AND t.case_id=c.id               
+					AND v.name='${vendor}'
+					AND c.name=failed.name
+			)
+			AND NOT exists (
+				SELECT * FROM test_run AS t, vendor AS v, test_case AS c
+				WHERE t.vendor_id=v.id AND t.case_id=c.id               
+					AND v.name='${vendor}'
+					AND c.name=failed.name
+					AND t.result=1 );"
+}
+# --
+
+# Print the tap file from contents of the database.
+# INPUT:
+# 1: <arch>    - Architecture to be included in the first line of the report
+# 2: <version> - OS version tested, to be included in the first line of the report
+# 3: <include_transient_errors> - If set to "true" then debug output of transient test failures
+#                   is included in the result report.
+function tap_generate_report() {
+    local arch="$1"
+    local version="$2"
+    local full_error_report="${3:-false}"
+
+    local count
+    count="$(__sqlite3_wrapper 'SELECT count(name) FROM test_case;')"
+    local vendors
+    vendors="$(__sqlite3_wrapper 'SELECT name FROM vendor;' | tr '\n' ' ')"
+
+    echo "1..$((count+1))"
+    echo "ok - Version: ${version}, Architecture: ${arch}" 
+    echo "   ---"
+    echo "   Platforms tested: ${vendors}"
+    echo "   ..."
+
+    # Print result line for every test, including platforms it succeeded on
+    #  and transient failed runs.
+    __sqlite3_wrapper 'SELECT DISTINCT name from test_case;' | \
+    while read -r test_name; do
+
+        # "ok" if the test succeeded at least once for all vendors that run the test,
+        #   "not ok" otherwise.
+        local verdict
+        verdict="$(__sqlite3_wrapper "
+        SELECT failed.name FROM vendor AS failed
+        WHERE EXISTS (
+                SELECT * FROM test_run AS t, vendor AS v, test_case AS c
+                WHERE t.vendor_id=v.id AND t.case_id=c.id
+                    AND v.name=failed.name
+                    AND c.name='${test_name}'
+            )
+            AND NOT exists (
+                SELECT * FROM test_run AS t, vendor AS v, test_case AS c
+                WHERE t.vendor_id=v.id AND t.case_id=c.id
+                    AND v.name=failed.name
+                    AND c.name='${test_name}'
+                    AND t.result=1 );
+        ")"
+        if [ -n "${verdict}" ] ; then
+            verdict="not ok"
+        else
+            verdict="ok"
+        fi
+
+        # Generate a list of vendors and respective runs, in a single line.
+        function list_runs() {
+            local res="$1"
+            __sqlite3_wrapper -csv "
+                SELECT v.name, t.run FROM test_run AS t, vendor AS v, test_case AS c
+                WHERE t.vendor_id=v.id AND t.case_id=c.id
+                    AND c.name='${test_name}'
+                    AND t.result=${res}
+                    ORDER BY v.name;" \
+                | awk -F, '{ if (t && (t != $1)) {
+                                printf t " " r "); "
+                                r="";}
+                             t=$1
+                             if (r)
+                                r=r ", " $2
+                             else
+                                r="(" $2 ; }
+                            END { if (t) print t r ")"; }'
+        }
+
+        local succeded
+        succeded="$(list_runs 1)"
+        local failed
+        failed="$(list_runs 0)"
+
+        echo "${verdict} - ${test_name}"
+        echo "   ---"
+        if [ -n "${succeded}" ] ; then
+            echo "   Succeeded: ${succeded}"
+        fi
+        if [ -n "${failed}" ] ; then
+            echo "   Failed: ${failed}"
+            if [ "${verdict}" = "not ok" -o "${full_error_report}" = "true" ] ; then
+                # generate diagnostic output, per failed run.
+                __sqlite3_wrapper -csv "
+                SELECT v.name, t.run
+                    FROM test_run AS t, vendor AS v, test_case AS c
+                    WHERE t.vendor_id=v.id AND t.case_id=c.id
+                    AND c.name='${test_name}'
+                    AND t.result=0
+                    ORDER BY t.run DESC;" | \
+                sed 's/,/ /' | \
+                while read -r vendor run; do
+                    echo "   Error messages for ${vendor}, run ${run}:"
+                    __sqlite3_wrapper -csv "
+                    SELECT t.output FROM test_run AS t, test_case AS c
+                        WHERE t.case_id=c.id
+                        AND c.name='${test_name}'
+                        AND t.run='${run}';" | \
+                    sed 's/"/ /g' | \
+                    awk '{print "      L" NR ": \"" $0 "\""}'
+                done
+            fi
+        fi
+        echo "   ..."
+    done
+}
+# --
diff --git a/ci-automation/test.sh b/ci-automation/test.sh
new file mode 100644
index 0000000000..6be101ee16
--- /dev/null
+++ b/ci-automation/test.sh
@@ -0,0 +1,252 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
+#
+# test_run() should be called w/ the positional INPUT parameters below.
+
+# Test scenarios runner stub.
+#   This script will run test scenarios for a single image type.
+#   Tests will be started inside the mantle container.
+#   This script is generic and will use a vendor-specific test runner from
+#    "ci-automation/vendor-testing/<image>.sh.
+#
+# PREREQUISITES:
+#
+#   1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt
+#   2. Scripts repo version tag of OS image version to be built is available and checked out.
+#   3. Mantle container docker image reference is stored in sdk_container/.repo/manifests/mantle-container.
+#   4. Vendor image and torcx docker tarball + manifest to run tests for are available on buildcache
+#         ( images/[ARCH]/[FLATCAR_VERSION]/ )
+#
+# INPUT:
+#
+#   1. Architecture (ARCH) of the TARGET vm images ("arm64", "amd64").
+#   2. Image type to be tested. One of:
+#      ami, azure, azure_pro, digitalocean, gce, gce_pro, packet, qemu, qemu_uefi, vmware
+#
+# OPTIONAL INPUT:
+#
+#   3. List of tests / test patterns. Defaults to "*" (all tests).
+#      All positional arguments after the first 2 (see above) are tests / patterns of tests to run.
+#
+#   MAX_RETRIES. Environment variable. Number of re-runs to overcome transient failures. Defaults to 20.
+#   PARALLEL_TESTS. Environment variable. Number of test cases to run in parallel.
+#                   Default is image / vendor specific and defined in ci-automation/ci-config.env.
+#
+# OUTPUT:
+#
+#   1. 2 merged TAP reports with all test runs / vendors.
+#        - a "summary" report which contains error messages only for tests which never succeeded (per vendor).
+#        - a "detailed" report which also contains error messages of transient failures which succeeded after re-runs.
+#        These reports will be updated after each (re-)run of each vendor, making the test job safe
+#          to abort at any point - the previous runs' results won't be lost.
+#   2. All intermediate kola tap reports, kola debug output, and merged tap reports (from 1.) published
+#        to buildcache at testing/[VERSION]/[ARCH]/[IMAGE]
+#   3. "./ci-cleanup.sh" with commands to clean up temporary build resources,
+#        to be run after this step finishes / when this step is aborted.
+#
+#
+# LOW-LEVEL / VENDOR SPECIFIC scripts API
+#
+# Vendor scripts are provided with their own sub-directory and are expected to CD into there before
+#  creating any artifacts (see vendor script argument 1 below).
+# The torcx manifest is supplied in
+#   ../
+# relative to the vendor sub-directory. The manifest is updated to include a URL pointing to the docker
+#  torcx tarball on the build cache (for the docker.torcx-manifest-pkgs test).
+#
+# Vendor specific scripts are called with the following positional arguments:
+# 1 - Toplevel tests directory
+#     It contains some additional files needed for running the tests (like torcx manifest or file with channel information).
+# 2 - Working directory for the tests.
+#     The vendor script is expected to keep all artifacts it produces in that directory.
+# 3 - Architecture to test.
+# 4 - Version number to test.
+# 5 - Output TAP file.
+# All following arguments specify test cases / test case patterns to run.
+#
+# The vendor tests should source ci-automation/vendor_test.sh script
+# as a first step - it will do some common steps that the vendor
+# script would need to make anyway. For more information, please refer
+# to the vendor_test.sh file.
+
+# Download torcx manifest and modify URLs pointing to the origin
+# server to point to the build cache. This is because the tests for
+# releases are run before artifacts are uploaded to the origin
+# server. This would make kola's docker.torcx-manifest-pkgs test to
+# fail.
+function __prepare_torcx() {
+    local arch="$1"
+    local vernum="$2"
+    local workdir="$3"
+
+    copy_from_buildcache "images/${arch}/${vernum}/torcx/torcx_manifest.json" "${workdir}"
+
+    # Change URLs from:
+    #
+    # https://${channel}.release.flatcar-linux.net/${arch}-usr/${vernum}/torcx/…
+    #
+    # to:
+    #
+    # https://bincache.flatcar-linux.net/images/${arch}/${vernum}/torcx/…
+    #
+    # This is done in two parts - replacing host part and arch part.
+    #
+    # Replace 'https://${channel}.release.flatcar-linux.net/' with
+    # 'https://bincache.flatcar-linux.net/' matching the initial "url"
+    # JSON key too.
+    local host_match='\("url":\s*"https://\)[a-z]\+\.release\([^/]\+/\)'
+    local host_replace='\1bincache\2'
+    # Replace '${arch}-usr/` part with 'images/${arch}/'.
+    local arch_match='\([a-z0-9]\+\)-usr/'
+    local arch_replace='images/\3/'
+    sed \
+        -e "s#${host_match}${arch_match}#${host_replace}${arch_replace}#g" \
+        "${workdir}/torcx_manifest.json" \
+        >"${workdir}/torcx_manifest_new.json"
+
+    mv "${workdir}/torcx_manifest.json" "${workdir}/torcx_manifest.json.original"
+    mv "${workdir}/torcx_manifest_new.json" "${workdir}/torcx_manifest.json"
+}
+# --
+
+function test_run() {
+    # Run a subshell, so the traps, environment changes and global
+    # variables are not spilled into the caller.
+    (
+        set -euo pipefail
+
+        _test_run_impl "${@}"
+    )
+}
+# --
+
+function _test_run_impl() {
+    local arch="$1" ; shift
+    local image="$1"; shift
+
+    # default to all tests
+    if [ $# -le 0 ] ; then
+        set -- '*'
+    fi
+
+    local retries="${MAX_RETRIES:-20}"
+    local skip_copy_to_bincache=${SKIP_COPY_TO_BINCACHE:-0}
+
+    source ci-automation/tapfile_helper_lib.sh
+    source ci-automation/ci_automation_common.sh
+    source sdk_lib/sdk_container_common.sh
+    init_submodules
+
+    source sdk_container/.repo/manifests/version.txt
+    local vernum="${FLATCAR_VERSION}"
+    local docker_vernum
+    docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
+
+    local work_dir="${TEST_WORK_DIR}"
+    local tests_dir="${work_dir}/${image}"
+    mkdir -p "${tests_dir}"
+
+    # Store git version and git channel as files inside ${work_dir}.
+    # This information might not be available inside the docker
+    # container if this directory is not a main git repo, but rather a
+    # git worktree.
+    get_git_version >"${work_dir}/git_version"
+    get_git_channel >"${work_dir}/git_channel"
+
+    local container_name="flatcar-tests-${arch}-${docker_vernum}-${image}"
+    local mantle_ref
+    mantle_ref=$(cat sdk_container/.repo/manifests/mantle-container)
+
+    # Make the torcx artifacts available to test implementation
+    __prepare_torcx "${arch}" "${vernum}" "${work_dir}"
+
+    local tap_merged_summary="results-${image}.tap"
+    local tap_merged_detailed="results-${image}-detailed.tap"
+    local retry=""
+    local success=false
+    local print_give_up=true
+    local failed_tests=()
+    # A job on each worker prunes old mantle images (docker image prune)
+    echo "docker rm -f '${container_name}'" >> ./ci-cleanup.sh
+
+    # Vendor tests may need to know if it is a first run or a rerun
+    touch "${work_dir}/first_run"
+    for retry in $(seq "${retries}"); do
+        local tapfile="results-run-${retry}.tap"
+        local failfile="failed-run-${retry}.txt"
+
+        # Ignore retcode since tests are flaky. We'll re-run failed tests and
+        #  determine success based on test results (tapfile).
+        set +e
+        touch sdk_container/.env
+        docker run --pull always --rm --name="${container_name}" --privileged --net host -v /dev:/dev \
+          -w /work -v "$PWD":/work "${mantle_ref}" \
+         bash -c "set -o noglob && source sdk_container/.env && ci-automation/vendor-testing/${image}.sh \
+                \"${work_dir}\" \
+                \"${tests_dir}\" \
+                \"${arch}\" \
+                \"${vernum}\" \
+                \"${tapfile}\" \
+                $*"
+        set -e
+        rm -f "${work_dir}/first_run"
+
+        docker run --pull always --rm --name="${container_name}" --privileged --net host -v /dev:/dev \
+          -w /work -v "$PWD":/work "${mantle_ref}" \
+            ci-automation/test_update_reruns.sh \
+                "${arch}" "${vernum}" "${image}" "${retry}" \
+                "${tests_dir}/${tapfile}" \
+                "${tests_dir}/${failfile}" \
+                "${tap_merged_summary}" \
+                "${tap_merged_detailed}"
+
+        readarray -t failed_tests <"${tests_dir}/${failfile}"
+        if [ "${#failed_tests[@]}" -eq 0 ] ; then
+            echo "########### All tests succeeded. ###########"
+            success=true
+            print_give_up=false
+            break
+        fi
+
+        if retest_cycle_broken; then
+            echo "########### Test cycle requested to break ###########"
+            echo "Failed tests:"
+            printf '%s\n' "${failed_tests[@]}"
+            echo "-----------"
+            print_give_up=false
+            break
+        fi
+
+        echo "########### Some tests failed and will be re-run (${retry} / ${retries}). ###########"
+        echo "Failed tests:"
+        printf '%s\n' "${failed_tests[@]}"
+        echo "-----------"
+        set -- "${failed_tests[@]}"
+    done
+
+
+    if ${print_give_up}; then
+        echo "########### All re-runs exhausted ($retries). Giving up. ###########"
+    fi
+
+    if [ ${skip_copy_to_bincache} -eq 0 ];then
+        # publish kola output, TAP files to build cache
+        copy_to_buildcache "testing/${vernum}/${arch}/${image}" \
+            "${tests_dir}/_kola_temp"
+        copy_to_buildcache "testing/${vernum}/${arch}/${image}" \
+            "${tests_dir}/"*.tap
+        copy_to_buildcache "testing/${vernum}/${arch}/${image}" \
+            "${tap_merged_summary}"
+        copy_to_buildcache "testing/${vernum}/${arch}/${image}" \
+            "${tap_merged_detailed}"
+    fi
+    if ! $success; then
+        return 1
+    fi
+}
+# --
diff --git a/ci-automation/test_update_reruns.sh b/ci-automation/test_update_reruns.sh
new file mode 100755
index 0000000000..2fcd79e543
--- /dev/null
+++ b/ci-automation/test_update_reruns.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Helper script for test.sh to update the test failures text file.
+# test.sh uses this to determine which tests need to re-run.
+# This script is run within the SDK container.
+
+set -euo pipefail
+
+arch="$1"
+vernum="$2"
+image="$3"
+retry="$4"
+tapfile="$5"
+failfile="$6"
+merged_summary="$7"
+merged_detailed="$8"
+
+source ci-automation/tapfile_helper_lib.sh
+tap_ingest_tapfile "${tapfile}" "${image}" "${retry}"
+tap_failed_tests_for_vendor "${image}" > "${failfile}"
+tap_generate_report "${arch}" "${vernum}"  > "${merged_summary}"
+tap_generate_report "${arch}" "${vernum}"  "true" > "${merged_detailed}"
diff --git a/ci-automation/util/Dockerfile.pigz b/ci-automation/util/Dockerfile.pigz
new file mode 100644
index 0000000000..33bcfedcaa
--- /dev/null
+++ b/ci-automation/util/Dockerfile.pigz
@@ -0,0 +1,4 @@
+FROM alpine
+RUN apk add pigz
+ENTRYPOINT [ "/usr/bin/pigz" ]
+CMD [ "-h" ]
diff --git a/ci-automation/util/fetch_image.sh b/ci-automation/util/fetch_image.sh
new file mode 100755
index 0000000000..51c4ef3a42
--- /dev/null
+++ b/ci-automation/util/fetch_image.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Helper for fetching a CI stage container image.
+
+set -euo pipefail
+
+function fetch_image_usage() {
+    local version="$1"
+    echo "Usage: fetch_image [-a <arch>]  [-v <version>] <stage>."
+    echo "Fetch and docker load a container image of a CI build stage."
+    echo "  <stage>     - CI build stage to fetch:"
+    echo "                sdk - fetch & install the plain SDK docker image. Note that this only works for the"
+    echo "                    'main' branch since maintenance branches don't build an SDK."
+    echo "                packages - fetch the packages (SDK + binary packages) container image."
+    echo "                image - fetch the images (SDK + packages + image) container image."
+    echo " -v <version> - Custom version to fetch instead of branch version '${version}'"
+    echo " -a <arch>    - OS image target architecture - 'arm64' or 'amd64'. Defaults to 'amd64'."
+}
+# --
+
+function fetch_image() {
+    local stage
+    local arch="amd64"
+    local version="${3:-}"
+
+    local script_root="$(dirname "${BASH_SOURCE[0]}")/../.."
+    source "${script_root}/ci-automation/ci_automation_common.sh"
+
+    local vernum="$(source "${script_root}/sdk_container/.repo/manifests/version.txt";
+                    echo "${FLATCAR_VERSION}")"
+    local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
+
+    while [ 0 -lt $# ] ; do
+        case "$1" in
+        -h) usage; exit 0;;
+        -v) docker_vernum="$2"; shift; shift;;
+        -a) arch="$2";          shift; shift;;
+        *)  if [ -n "${stage:-}" ] ; then
+                echo "ERROR: Spurious positional argument(s): '$@'"
+                fetch_image_usage "${vernum}"
+                exit 1
+            fi
+            stage="$1"
+            shift;;
+        esac
+    done
+
+    local image
+    case "${stage}" in
+        sdk)      image="flatcar-sdk-${arch}";;
+        packages) image="flatcar-packages-${arch}";;
+        image)    image="flatcar-images-${arch}";;
+        *) echo "ERROR: unknown build stage '$1'"
+           fetch_image_usage "${docker_vernum}"
+           exit 1;;
+    esac
+
+    echo "Fetching '${image}:${docker_vernum}'. Depending on your connection this may take a while."
+    docker_image_from_buildcache "${image}" "${docker_vernum}"
+
+    echo "Done! Use"
+    echo "   ./run_sdk_container -t -C ${image}:${docker_vernum}"
+    echo "to start."
+}
+# --
+
+if [ "$(basename "$0")" = "fetch_image.sh" ] ; then
+    fetch_image $@
+fi
diff --git a/ci-automation/vendor-testing/aws.sh b/ci-automation/vendor-testing/aws.sh
new file mode 100755
index 0000000000..ac5043064f
--- /dev/null
+++ b/ci-automation/vendor-testing/aws.sh
@@ -0,0 +1,82 @@
+#!/bin/bash
+# Copyright (c) 2022 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -euo pipefail
+
+# Test execution script for the AWS vendor image.
+# This script is supposed to run in the mantle container.
+
+source ci-automation/vendor_test.sh
+
+board="${CIA_ARCH}-usr"
+escaped_vernum="${CIA_VERNUM//+/-}"
+image_name="ci-${escaped_vernum}-${CIA_ARCH}"
+aws_instance_type_var="AWS_${CIA_ARCH}_INSTANCE_TYPE"
+aws_instance_type="${!aws_instance_type_var}"
+more_aws_instance_types_var="AWS_${CIA_ARCH}_MORE_INSTANCE_TYPES"
+set -o noglob # there shouldn't be any instance types with asterisks
+              # in it, but…
+more_aws_instance_types=( ${!more_aws_instance_types_var} )
+set +o noglob
+
+vmdk='flatcar_production_ami_vmdk_image.vmdk'
+tarball="${vmdk}.bz2"
+
+if [[ "${AWS_AMI_ID}" == "" ]]; then
+    if [[ -f "${vmdk}" ]]; then
+        echo "++++ ${CIA_TESTSCRIPT}: using existing ${vmdk} for ${CIA_VERNUM} (${CIA_ARCH}) ++++"
+    else
+        echo "++++ ${CIA_TESTSCRIPT}: downloading ${tarball} for ${CIA_VERNUM} (${CIA_ARCH}) ++++"
+        copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/${tarball}" .
+        lbunzip2 "${tarball}"
+    fi
+
+    aws_bucket="flatcar-kola-ami-import-${AWS_REGION}"
+    aws_s3_path="s3://${aws_bucket}/${escaped_vernum}/${board}/"
+    trap 'ore -d aws delete --region="${AWS_REGION}" --board="${board}" --name="${image_name}" --ami-name="${image_name}" --file="${vmdk}" --bucket "${aws_s3_path}"' EXIT
+    ore aws initialize --region="${AWS_REGION}" --bucket "${aws_bucket}"
+    AWS_AMI_ID=$(ore aws upload --force --region="${AWS_REGION}" --board="${board}" --name="${image_name}" --ami-name="${image_name}" --ami-description="Flatcar Test ${image_name}" --file="${vmdk}" --bucket "${aws_s3_path}" | jq -r .HVM)
+    echo "++++ ${CIA_TESTSCRIPT}: created new AMI ${AWS_AMI_ID} (will be removed after testing) ++++"
+fi
+
+run_kola_tests() {
+    local instance_type="${1}"; shift
+    local instance_tapfile="${1}"; shift
+
+    timeout --signal=SIGQUIT 6h \
+        kola run \
+         --board="${board}" \
+         --basename="${image_name}" \
+         --channel="${CIA_CHANNEL}" \
+         --offering='basic' \
+         --parallel="${AWS_PARALLEL}" \
+         --platform=aws \
+         --aws-ami="${AWS_AMI_ID}" \
+         --aws-region="${AWS_REGION}" \
+         --aws-type="${instance_type}" \
+         --aws-iam-profile="${AWS_IAM_PROFILE}" \
+         --tapfile="${instance_tapfile}" \
+         --torcx-manifest="${CIA_TORCX_MANIFEST}" \
+         "${@}"
+}
+
+query_kola_tests() {
+    shift; # ignore the instance type
+    kola list --platform=aws --filter "${@}"
+}
+
+# these are set in ci-config.env
+export AWS_ACCESS_KEY_ID
+export AWS_SECRET_ACCESS_KEY
+
+run_kola_tests_on_instances \
+    "${aws_instance_type}" \
+    "${CIA_TAPFILE}" \
+    "${CIA_FIRST_RUN}" \
+    "${more_aws_instance_types[@]}" \
+    '--' \
+    'cl.internet' \
+    '--' \
+    "${@}"
diff --git a/ci-automation/vendor-testing/azure.sh b/ci-automation/vendor-testing/azure.sh
new file mode 100755
index 0000000000..17081b3598
--- /dev/null
+++ b/ci-automation/vendor-testing/azure.sh
@@ -0,0 +1,90 @@
+#!/bin/bash
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -euo pipefail
+
+# Test execution script for the azure vendor image.
+# This script is supposed to run in the mantle container.
+
+source ci-automation/vendor_test.sh
+
+# $@ now contains tests / test patterns to run
+
+board="${CIA_ARCH}-usr"
+basename="ci-${CIA_VERNUM//+/-}-${CIA_ARCH}"
+azure_instance_type_var="AZURE_${CIA_ARCH}_MACHINE_SIZE"
+azure_instance_type="${!azure_instance_type_var}"
+azure_vnet_subnet_name="jenkins-vnet-${AZURE_LOCATION}"
+
+azure_profile_config_file=''
+secret_to_file azure_profile_config_file "${AZURE_PROFILE}"
+azure_auth_config_file=''
+secret_to_file azure_auth_config_file "${AZURE_AUTH_CREDENTIALS}"
+
+# Fetch the Azure image if not present
+if [ -f "${AZURE_IMAGE_NAME}" ] ; then
+    echo "++++ ${CIA_TESTSCRIPT}: Using existing ${AZURE_IMAGE_NAME} for testing ${CIA_VERNUM} (${CIA_ARCH}) ++++"
+else
+    echo "++++ ${CIA_TESTSCRIPT}: downloading ${AZURE_IMAGE_NAME} for ${CIA_VERNUM} (${CIA_ARCH}) ++++"
+    copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/${AZURE_IMAGE_NAME}.bz2" .
+    cp --sparse=always <(lbzcat "${AZURE_IMAGE_NAME}.bz2") "${AZURE_IMAGE_NAME}"
+    rm "${AZURE_IMAGE_NAME}.bz2"
+fi
+
+if [[ "${CIA_ARCH}" == "arm64" ]]; then
+  AZURE_USE_GALLERY="--azure-use-gallery"
+fi
+
+
+run_kola_tests() {
+    local instance_type="${1}"; shift
+    local instance_tapfile="${1}"; shift
+    local hyperv_gen="V2"
+    if [ "${instance_type}" = "V1" ]; then
+        hyperv_gen="V1"
+        instance_type="${azure_instance_type}"
+    fi
+
+    # Align timeout with ore azure gc --duration parameter
+    timeout --signal=SIGQUIT 6h \
+      kola run \
+      --board="${board}" \
+      --basename="${basename}" \
+      --parallel="${AZURE_PARALLEL}" \
+      --offering=basic \
+      --platform=azure \
+      --azure-image-file="${AZURE_IMAGE_NAME}" \
+      --azure-location="${AZURE_LOCATION}" \
+      --azure-profile="${azure_profile_config_file}" \
+      --azure-auth="${azure_auth_config_file}" \
+      --torcx-manifest="${CIA_TORCX_MANIFEST}" \
+      --tapfile="${instance_tapfile}" \
+      --azure-size="${instance_type}" \
+      --azure-hyper-v-generation="${hyperv_gen}" \
+      ${AZURE_USE_GALLERY} \
+      ${azure_vnet_subnet_name:+--azure-vnet-subnet-name=${azure_vnet_subnet_name}} \
+      ${AZURE_USE_PRIVATE_IPS:+--azure-use-private-ips=${AZURE_USE_PRIVATE_IPS}} \
+      "${@}"
+}
+
+query_kola_tests() {
+    shift; # ignore the instance type
+    kola list --platform=azure --filter "${@}"
+}
+
+other_instance_types=()
+if [[ "${CIA_ARCH}" = 'amd64' ]]; then
+    other_instance_types+=('V1')
+fi
+
+run_kola_tests_on_instances \
+    "${azure_instance_type}" \
+    "${CIA_TAPFILE}" \
+    "${CIA_FIRST_RUN}" \
+    "${other_instance_types[@]}" \
+    '--' \
+    'cl.internet' \
+    '--' \
+    "${@}"
diff --git a/ci-automation/vendor-testing/digitalocean.sh b/ci-automation/vendor-testing/digitalocean.sh
new file mode 100755
index 0000000000..3ca82ac2e4
--- /dev/null
+++ b/ci-automation/vendor-testing/digitalocean.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+# Copyright (c) 2022 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -euo pipefail
+
+# Test execution script for the Digital Ocean vendor image.
+# This script is supposed to run in the mantle container.
+
+source ci-automation/vendor_test.sh
+
+# We never ran Digital Ocean on arm64, so for now fail it as an
+# unsupported option.
+if [[ "${CIA_ARCH}" == "arm64" ]]; then
+    echo "1..1" > "${CIA_TAPFILE}"
+    echo "not ok - all Digital Ocean tests" >> "${CIA_TAPFILE}"
+    echo "  ---" >> "${CIA_TAPFILE}"
+    echo "  ERROR: ARM64 tests not supported on Digital Ocean." | tee -a "${CIA_TAPFILE}"
+    echo "  ..." >> "${CIA_TAPFILE}"
+    break_retest_cycle
+    exit 1
+fi
+
+image_name="ci-${CIA_VERNUM//+/-}"
+image_url="$(url_from_template "${DIGITALOCEAN_IMAGE_URL_TEMPLATE}" "${CIA_ARCH}" "${CIA_CHANNEL}" 'https' "${CIA_VERNUM}")"
+
+config_file=''
+secret_to_file config_file "${DIGITALOCEAN_TOKEN_JSON}"
+
+ore do create-image \
+    --config-file="${config_file}" \
+    --region="${DIGITALOCEAN_REGION}" \
+    --name="${image_name}" \
+    --url="${image_url}"
+
+trap 'ore do delete-image \
+    --name="${image_name}" \
+    --config-file="${config_file}"' EXIT
+
+set -x
+
+timeout --signal=SIGQUIT 4h\
+    kola run \
+    --do-size="${DIGITALOCEAN_MACHINE_SIZE}" \
+    --do-region="${DIGITALOCEAN_REGION}" \
+    --basename="${image_name}" \
+    --do-config-file="${config_file}" \
+    --do-image="${image_name}" \
+    --parallel="${DIGITALOCEAN_PARALLEL}" \
+    --platform=do \
+    --channel="${CIA_CHANNEL}" \
+    --tapfile="${CIA_TAPFILE}" \
+    --torcx-manifest="${CIA_TORCX_MANIFEST}" \
+    "${@}"
+
+set +x
diff --git a/ci-automation/vendor-testing/equinix_metal.sh b/ci-automation/vendor-testing/equinix_metal.sh
new file mode 100755
index 0000000000..5f4440fd12
--- /dev/null
+++ b/ci-automation/vendor-testing/equinix_metal.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -euo pipefail
+
+# Test execution script for the Equinix Metal vendor image.
+# This script is supposed to run in the mantle container.
+# This script requires the PXE images to be built.
+
+source ci-automation/vendor_test.sh
+
+# Equinix Metal ARM server are not yet hourly available in the default `SV` metro
+equinixmetal_metro_var="EQUINIXMETAL_${CIA_ARCH}_METRO"
+equinixmetal_metro="${!equinixmetal_metro_var}"
+
+EQUINIXMETAL_INSTANCE_TYPE_VAR="EQUINIXMETAL_${CIA_ARCH}_INSTANCE_TYPE"
+EQUINIXMETAL_INSTANCE_TYPE="${!EQUINIXMETAL_INSTANCE_TYPE_VAR}"
+MORE_INSTANCE_TYPES_VAR="EQUINIXMETAL_${CIA_ARCH}_MORE_INSTANCE_TYPES"
+MORE_INSTANCE_TYPES=( ${!MORE_INSTANCE_TYPES_VAR} )
+
+# The maximum is 6h coming from the ore GC duration parameter
+timeout=6h
+
+BASE_URL="http://${BUILDCACHE_SERVER}/images/${CIA_ARCH}/${CIA_VERNUM}"
+
+run_kola_tests() {
+    local instance_type="${1}"; shift
+    local instance_tapfile="${1}"; shift
+
+    timeout --signal=SIGQUIT "${timeout}" \
+        kola run \
+          --board="${CIA_ARCH}-usr" \
+          --basename="ci-${CIA_VERNUM/+/-}-${CIA_ARCH}" \
+          --platform=equinixmetal \
+          --tapfile="${instance_tapfile}" \
+          --parallel="${EQUINIXMETAL_PARALLEL}" \
+          --torcx-manifest="${CIA_TORCX_MANIFEST}" \
+          --equinixmetal-image-url="${BASE_URL}/${EQUINIXMETAL_IMAGE_NAME}" \
+          --equinixmetal-installer-image-kernel-url="${BASE_URL}/${PXE_KERNEL_NAME}" \
+          --equinixmetal-installer-image-cpio-url="${BASE_URL}/${PXE_IMAGE_NAME}" \
+          --equinixmetal-metro="${equinixmetal_metro}" \
+          --equinixmetal-plan="${instance_type}" \
+          --equinixmetal-project="${EQUINIXMETAL_PROJECT}" \
+          --equinixmetal-storage-url="${EQUINIXMETAL_STORAGE_URL}" \
+          --gce-json-key=<(set +x; echo "${GCP_JSON_KEY}" | base64 --decode) \
+          --equinixmetal-api-key="${EQUINIXMETAL_KEY}" \
+          "${@}"
+}
+
+query_kola_tests() {
+    shift; # ignore the instance type
+    kola list --platform=equinixmetal --filter "${@}"
+}
+
+run_kola_tests_on_instances \
+    "${EQUINIXMETAL_INSTANCE_TYPE}" \
+    "${CIA_TAPFILE}" \
+    "${CIA_FIRST_RUN}" \
+    "${MORE_INSTANCE_TYPES[@]}" \
+    '--' \
+    'cl.internet' \
+    '--' \
+    "${@}"
diff --git a/ci-automation/vendor-testing/gce.sh b/ci-automation/vendor-testing/gce.sh
new file mode 100755
index 0000000000..2ab0fd1284
--- /dev/null
+++ b/ci-automation/vendor-testing/gce.sh
@@ -0,0 +1,82 @@
+#!/bin/bash
+# Copyright (c) 2022 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -euo pipefail
+
+# Test execution script for the GCE vendor image.
+# This script is supposed to run in the mantle container.
+
+source ci-automation/vendor_test.sh
+
+# We never run GCE on arm64, so for now fail it as an
+# unsupported option.
+if [[ "${CIA_ARCH}" == "arm64" ]]; then
+    echo "1..1" > "${CIA_TAPFILE}"
+    echo "not ok - all GCE tests" >> "${CIA_TAPFILE}"
+    echo "  ---" >> "${CIA_TAPFILE}"
+    echo "  ERROR: ARM64 tests not supported on GCE." | tee -a "${CIA_TAPFILE}"
+    echo "  ..." >> "${CIA_TAPFILE}"
+    break_retest_cycle
+    exit 1
+fi
+
+GCP_JSON_KEY_PATH=''
+secret_to_file GCP_JSON_KEY_PATH "${GCP_JSON_KEY}"
+
+copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/${GCE_IMAGE_NAME}" .
+gcloud auth activate-service-account --key-file "${GCP_JSON_KEY_PATH}"
+gsutil rm -r "${GCE_GCS_IMAGE_UPLOAD}/${CIA_ARCH}-usr/${CIA_VERNUM}" || true
+gsutil cp "${GCE_IMAGE_NAME}" "${GCE_GCS_IMAGE_UPLOAD}/${CIA_ARCH}-usr/${CIA_VERNUM}/${GCE_IMAGE_NAME}"
+family="ci"
+image_name="${family}-${CIA_VERNUM//[+.]/-}"
+ore gcloud delete-images --json-key="${GCP_JSON_KEY_PATH}" "${image_name}" || true
+ore gcloud create-image \
+    --board="${CIA_ARCH}-usr" \
+    --family="${family}" \
+    --json-key="${GCP_JSON_KEY_PATH}" \
+    --source-root="${GCE_GCS_IMAGE_UPLOAD}" \
+    --source-name="${GCE_IMAGE_NAME}" \
+    --version="${CIA_VERNUM}"
+
+trap 'ore gcloud delete-images \
+    --json-key="${GCP_JSON_KEY_PATH}" \
+    "${image_name}" ; gsutil rm -r "${GCE_GCS_IMAGE_UPLOAD}/${CIA_ARCH}-usr/${CIA_VERNUM}" || true' EXIT
+
+run_kola_tests() {
+    local instance_type="${1}"; shift
+    local instance_tapfile="${1}"; shift
+    local extra_arg=()
+    if [ "${instance_type}" = "gvnic" ]; then
+        extra_arg+=("--gce-gvnic")
+    fi
+    timeout --signal=SIGQUIT 6h \
+    kola run \
+        --basename="${image_name}" \
+        --gce-image="${image_name}" \
+        --gce-json-key="${GCP_JSON_KEY_PATH}" \
+        --gce-machinetype="${GCE_MACHINE_TYPE}" \
+        "${extra_arg[@]}" \
+        --parallel="${GCE_PARALLEL}" \
+        --platform=gce \
+        --channel="${CIA_CHANNEL}" \
+        --tapfile="${instance_tapfile}" \
+        --torcx-manifest="${CIA_TORCX_MANIFEST}" \
+        "${@}"
+}
+
+query_kola_tests() {
+    shift; # ignore the instance type
+    kola list --platform=gce --filter "${@}"
+}
+
+run_kola_tests_on_instances \
+    "default" \
+    "${CIA_TAPFILE}" \
+    "${CIA_FIRST_RUN}" \
+    "gvnic" \
+    '--' \
+    'cl.internet' \
+    '--' \
+    "${@}"
diff --git a/ci-automation/vendor-testing/qemu.sh b/ci-automation/vendor-testing/qemu.sh
new file mode 100755
index 0000000000..99e9d2a9c2
--- /dev/null
+++ b/ci-automation/vendor-testing/qemu.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -euo pipefail
+
+# Test execution script for the qemu vendor image.
+# This script is supposed to run in the mantle container.
+
+source ci-automation/vendor_test.sh
+
+# ARM64 qemu tests only supported on UEFI
+if [ "${CIA_ARCH}" = "arm64" ] && [ "${CIA_TESTSCRIPT}" != "qemu_uefi.sh" ] ; then
+    echo "1..1" > "${CIA_TAPFILE}"
+    echo "not ok - all qemu tests" >> "${CIA_TAPFILE}"
+    echo "  ---" >> "${CIA_TAPFILE}"
+    echo "  ERROR: ARM64 tests only supported on qemu_uefi." | tee -a "${CIA_TAPFILE}"
+    echo "  ..." >> "${CIA_TAPFILE}"
+    break_retest_cycle
+    exit 1
+fi
+
+# Fetch image and BIOS if not present
+if [ -f "${QEMU_IMAGE_NAME}" ] ; then
+    echo "++++ ${CIA_TESTSCRIPT}: Using existing ${QEMU_IMAGE_NAME} for testing ${CIA_VERNUM} (${CIA_ARCH}) ++++"
+else
+    echo "++++ ${CIA_TESTSCRIPT}: downloading ${QEMU_IMAGE_NAME} for ${CIA_VERNUM} (${CIA_ARCH}) ++++"
+    rm -f "${QEMU_IMAGE_NAME}.bz2"
+    copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/${QEMU_IMAGE_NAME}.bz2" .
+    lbunzip2 "${QEMU_IMAGE_NAME}.bz2"
+fi
+
+bios="${QEMU_BIOS}"
+if [ "${CIA_TESTSCRIPT}" = "qemu_uefi.sh" ] ; then
+    bios="${QEMU_UEFI_BIOS}"
+    if [ -f "${bios}" ] ; then
+        echo "++++ ${CIA_TESTSCRIPT}: Using existing ${bios} ++++"
+    else
+        echo "++++ ${CIA_TESTSCRIPT}: downloading ${bios} for ${CIA_VERNUM} (${CIA_ARCH}) ++++"
+        copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/${bios}" .
+    fi
+fi
+
+set -x
+
+kola run \
+    --board="${CIA_ARCH}-usr" \
+    --parallel="${QEMU_PARALLEL}" \
+    --platform=qemu \
+    --qemu-bios="${bios}" \
+    --qemu-image="${QEMU_IMAGE_NAME}" \
+    --tapfile="${CIA_TAPFILE}" \
+    --torcx-manifest="${CIA_TORCX_MANIFEST}" \
+    --qemu-skip-mangle \
+    "${@}"
+
+set +x
diff --git a/ci-automation/vendor-testing/qemu_uefi.sh b/ci-automation/vendor-testing/qemu_uefi.sh
new file mode 120000
index 0000000000..a142d967c8
--- /dev/null
+++ b/ci-automation/vendor-testing/qemu_uefi.sh
@@ -0,0 +1 @@
+qemu.sh
\ No newline at end of file
diff --git a/ci-automation/vendor-testing/qemu_update.sh b/ci-automation/vendor-testing/qemu_update.sh
new file mode 100755
index 0000000000..1ce875f684
--- /dev/null
+++ b/ci-automation/vendor-testing/qemu_update.sh
@@ -0,0 +1,64 @@
+#!/bin/bash
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -euo pipefail
+
+# Test execution script for the update payload using the previous
+# release as starting point, and doing a second update from the current
+# build to itself again.
+# This script is supposed to run in the mantle container.
+
+source ci-automation/vendor_test.sh
+
+if [ "$*" != "" ] && [ "$*" != "*" ] && [ "$*" != "cl.update.payload" ]; then
+    echo "1..1" > "${CIA_TAPFILE}"
+    echo "not ok - all qemu update tests" >> "${CIA_TAPFILE}"
+    echo "  ---" >> "${CIA_TAPFILE}"
+    echo "  ERROR: Only cl.update.payload is supported, got '$*'." | tee -a "${CIA_TAPFILE}"
+    echo "  ..." >> "${CIA_TAPFILE}"
+    break_retest_cycle
+    exit 1
+fi
+
+mkdir -p tmp/
+if [ -f tmp/flatcar_test_update.gz ] ; then
+    echo "++++ ${CIA_TESTSCRIPT}: Using existing ./tmp/flatcar_test_update.gz for testing ${CIA_VERNUM} (${CIA_ARCH}) ++++"
+else
+    echo "++++ ${CIA_TESTSCRIPT}: downloading flatcar_test_update.gz for ${CIA_VERNUM} (${CIA_ARCH}) ++++"
+    copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/flatcar_test_update.gz" tmp/
+fi
+
+if [ -f tmp/flatcar_production_image_previous.bin ] ; then
+    echo "++++ ${CIA_TESTSCRIPT}: Using existing ./tmp/flatcar_production_image_previous.bin for testing update to ${CIA_VERNUM} (${CIA_ARCH}) from previous ${CIA_CHANNEL} ++++"
+else
+    echo "++++ ${CIA_TESTSCRIPT}: downloading flatcar_production_image_previous.bin from previous ${CIA_CHANNEL} ++++"
+    rm -f tmp/flatcar_production_image_previous.bin.bz2
+    curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 "https://${CIA_CHANNEL}.release.flatcar-linux.net/${CIA_ARCH}-usr/current/flatcar_production_image.bin.bz2"
+    mv flatcar_production_image.bin.bz2 tmp/flatcar_production_image_previous.bin.bz2
+    lbunzip2 -k -f tmp/flatcar_production_image_previous.bin.bz2
+fi
+
+bios="${QEMU_BIOS}"
+if [ "${CIA_ARCH}" = "arm64" ]; then
+    bios="${QEMU_UEFI_BIOS}"
+    if [ -f "${bios}" ] ; then
+        echo "++++ qemu_update.sh: Using existing ./${bios} ++++"
+    else
+        echo "++++ qemu_update.sh: downloading ${bios} for ${CIA_VERNUM} (${CIA_ARCH}) ++++"
+        copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/${bios}" .
+    fi
+fi
+
+kola run \
+    --board="${CIA_ARCH}-usr" \
+    --parallel="${QEMU_PARALLEL}" \
+    --platform=qemu \
+    --qemu-bios="${bios}" \
+    --qemu-image=tmp/flatcar_production_image_previous.bin \
+    --tapfile="${CIA_TAPFILE}" \
+    --torcx-manifest="${CIA_TORCX_MANIFEST}" \
+    --update-payload=tmp/flatcar_test_update.gz \
+    --qemu-skip-mangle \
+    cl.update.payload
diff --git a/ci-automation/vendor-testing/vmware.sh b/ci-automation/vendor-testing/vmware.sh
new file mode 100755
index 0000000000..c0c57a6c30
--- /dev/null
+++ b/ci-automation/vendor-testing/vmware.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+# Copyright (c) 2022 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -euo pipefail
+
+# Test execution script for the VMware ESX vendor image.
+# This script is supposed to run in the mantle container.
+
+source ci-automation/vendor_test.sh
+
+# We never ran VMware ESX on arm64, so for now fail it as an
+# unsupported option.
+if [[ "${CIA_ARCH}" == "arm64" ]]; then
+    echo "1..1" > "${CIA_TAPFILE}"
+    echo "not ok - all qemu tests" >> "${CIA_TAPFILE}"
+    echo "  ---" >> "${CIA_TAPFILE}"
+    echo "  ERROR: ARM64 tests not supported on VMware ESX." | tee -a "${CIA_TAPFILE}"
+    echo "  ..." >> "${CIA_TAPFILE}"
+    break_retest_cycle
+    exit 1
+fi
+
+# Fetch image if not present.
+if [ -f "${VMWARE_ESX_IMAGE_NAME}" ] ; then
+    echo "++++ ${CIA_TESTSCRIPT}: Using existing ${work_dir}/${VMWARE_ESX_IMAGE_NAME} for testing ${CIA_VERNUM} (${CIA_ARCH}) ++++"
+else
+    echo "++++ ${CIA_TESTSCRIPT}: downloading ${VMWARE_ESX_IMAGE_NAME} for ${CIA_VERNUM} (${CIA_ARCH}) ++++"
+    copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/${VMWARE_ESX_IMAGE_NAME}" .
+fi
+
+config_file=''
+secret_to_file config_file "${VMWARE_ESX_CREDS}"
+
+# If we are using static IPs, then delete every VM that is running
+# because we'll use all available spots. This is to avoid entering a
+# broken state if there are some left-over VMs from manual usage or a
+# forcefully terminated job.
+#
+# The assumption here is that we can do it without any interference
+# with other CI Vms, because we have acquired a resource lock to those
+# VMs.
+static_ips="$(jq '.["default"]["static_ips"]' "${config_file}")"
+if [[ "${static_ips}" -ne 0 ]]; then
+    ore esx --esx-config-file "${config_file}" remove-vms || :
+fi
+
+kola_test_basename="ci-${CIA_VERNUM//+/-}"
+
+trap 'ore esx --esx-config-file "${config_file}" remove-vms \
+    --pattern "${kola_test_basename}*" || :' EXIT
+
+set -x
+
+sudo timeout --signal=SIGQUIT 2h kola run \
+    --board="${CIA_ARCH}-usr" \
+    --basename="${kola_test_basename}" \
+    --channel="${CIA_CHANNEL}" \
+    --platform=esx \
+    --tapfile="${CIA_TAPFILE}" \
+    --parallel="${VMWARE_ESX_PARALLEL}" \
+    --torcx-manifest="${CIA_TORCX_MANIFEST}" \
+    --esx-config-file "${config_file}" \
+    --esx-ova-path "${VMWARE_ESX_IMAGE_NAME}" \
+    "${@}"
+
+set +x
diff --git a/ci-automation/vendor_test.sh b/ci-automation/vendor_test.sh
new file mode 100644
index 0000000000..f0d657e36d
--- /dev/null
+++ b/ci-automation/vendor_test.sh
@@ -0,0 +1,391 @@
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Vendor test helper script. Sourced by vendor tests. Does some
+# initial setup.
+#
+#
+# The initial setup consist of creating the vendor working directory
+# for the vendor test script, specifying the variables described below
+# and changing the current working directory to the vendor working
+# directory.
+#
+#
+# The vendor test script is expected to keep all artifacts it produces
+# in its current working directory.
+#
+#
+# The script specifies the following variables for the vendor test
+# script to use:
+#
+# CIA_VERNUM:
+#   Image version. In case of developer builds it comes with a suffix,
+#   so it looks like "3217.0.0+nightly-20220422-0155". For release
+#   builds the version will be without suffix, so it looks like
+#   "3217.0.0". Whether the build is a release or a developer one is
+#   reflected in CIA_BUILD_TYPE variable described below.
+#
+# CIA_ARCH:
+#   Architecture to test. Currently it is either "amd64" or "arm64".
+#
+# CIA_TAPFILE:
+#   Where the TAP reports should be written. Usually just passed to
+#   kola throught the --tapfile parameter.
+#
+# CIA_CHANNEL:
+#   A channel. Either "alpha", "beta", "stable" or "lts". Used to find
+#   the last release for the update check.
+#
+# CIA_TESTSCRIPT:
+#   Name of the vendor script. May be useful in some messages.
+#
+# CIA_GIT_VERSION:
+#   The most recent tag for the current commit.
+#
+# CIA_BUILD_TYPE:
+#   It's either "release" or "developer", based on the CIA_VERNUM
+#   variable.
+#
+# CIA_TORCX_MANIFEST:
+#   Path to the Torcx manifest. Usually passed to kola through the
+#   --torcx-manifest parameter.
+#
+# CIA_FIRST_RUN:
+#   1 if this is a first run, 0 if it is a rerun of failed tests.
+#
+#
+# After this script is sourced, the parameters in ${@} specify test
+# cases / test case patterns to run.
+
+
+# "ciavts" stands for Continuous Integration Automation Vendor Test
+# Setup. This prefix is used to easily unset all the variables with
+# this prefix before leaving this file.
+
+ciavts_main_work_dir="${1}"; shift
+ciavts_work_dir="${1}"; shift
+ciavts_arch="${1}"; shift
+ciavts_vernum="${1}"; shift
+ciavts_tapfile="${1}"; shift
+
+# $@ now contains tests / test patterns to run
+
+source ci-automation/ci_automation_common.sh
+
+mkdir -p "${ciavts_work_dir}"
+
+ciavts_testscript=$(basename "${0}")
+ciavts_git_version=$(cat "${ciavts_main_work_dir}/git_version")
+ciavts_channel=$(cat "${ciavts_main_work_dir}/git_channel")
+if [[ "${ciavts_channel}" = 'developer' ]]; then
+    ciavts_channel='alpha'
+fi
+# If vernum is like 3200.0.0+whatever, it's a developer build,
+# otherwise it's a release build.
+ciavts_type='developer'
+if [[ "${ciavts_vernum%%+*}" = "${ciavts_vernum}" ]]; then
+    ciavts_type='release'
+fi
+
+# Make these paths absolute to avoid problems when changing
+# directories.
+ciavts_tapfile="${PWD}/${ciavts_work_dir}/${ciavts_tapfile}"
+ciavts_torcx_manifest="${PWD}/${ciavts_main_work_dir}/torcx_manifest.json"
+
+ciavts_first_run=0
+if [[ -f "${ciavts_main_work_dir}/first_run" ]]; then
+    ciavts_first_run=1
+fi
+
+echo "++++ Running ${ciavts_testscript} inside ${ciavts_work_dir} ++++"
+
+cd "${ciavts_work_dir}"
+
+CIA_VERNUM="${ciavts_vernum}"
+CIA_ARCH="${ciavts_arch}"
+CIA_TAPFILE="${ciavts_tapfile}"
+CIA_CHANNEL="${ciavts_channel}"
+CIA_TESTSCRIPT="${ciavts_testscript}"
+CIA_GIT_VERSION="${ciavts_git_version}"
+CIA_BUILD_TYPE="${ciavts_type}"
+CIA_TORCX_MANIFEST="${ciavts_torcx_manifest}"
+CIA_FIRST_RUN="${ciavts_first_run}"
+
+# Unset all variables with ciavts_ prefix now.
+unset -v "${!ciavts_@}"
+
+# Prefixes all test names in the tap file with a given prefix, so the
+# test name like "cl.basic" will become "extra-test.[${prefix}].cl.basic".
+#
+# Typical use:
+#   prefix_tap_file "${instance_type}" "${tapfile}"
+#
+# Parameters:
+# 1 - prefix
+# 2 - tap file, modified in place
+function prefix_tap_file() {
+    local prefix="${1}"; shift
+    local tap_file="${1}"; shift
+    # drop the dots from prefix
+    local actual_prefix="extra-test.[${prefix}]."
+
+    sed --in-place --expression 's/^\(\s*\(not\)\?\s*ok[^-]*\s*-\s*\)\(\S\)/\1'"${actual_prefix}"'\3/g' "${tap_file}"
+}
+
+# Filters the test names, so it puts only the real names of the
+# prefixed tests into the chosen variable. For example for prefix
+# "foo", it will ignore the test name like "cl.basic", but will print
+# "cl.internet" for a test name like "extra-test.[foo].cl.internet".
+# "*" is treated specially - it will be inserted into the chosen
+# variable if it is passed.
+#
+# Typical use:
+#   filter_prefixed_tests tests_to_run "${instance_type}" "${@}"
+#   if [[ "${#tests_to_run[@]}" -gt 0 ]]; then …; fi
+#
+# Parameters:
+# 1 - name of an array variable where the filtering results will be stored
+# 2 - prefix
+# @ - test names
+function filter_prefixed_tests() {
+    local var_name="${1}"; shift
+    local prefix="${1}"; shift
+    # rest of the parameters are test names
+    local -n results="${var_name}"
+    local name
+    local stripped_name
+    # clear the array, so it will contain results of current filtering
+    # only
+    results=()
+    for name; do
+        stripped_name="${name#extra-test.\[${prefix}\].}"
+        if [[ "${stripped_name}" != "${name}" ]]; then
+            results+=( "${stripped_name}" )
+            continue
+        elif [[ "${name}" = '*' ]]; then
+            results+=( '*' )
+        fi
+    done
+}
+
+# Filters out the extra tests from the passed test names. Ignored test
+# names begin with "extra-test.". The results of the filtering are
+# inserted into the chosen variable.
+#
+# Typical use:
+#   filter_out_prefixed_tests tests_to_run "${@}"
+#   if [[ "${#tests_to_run[@]}" -gt 0 ]]; then …; fi
+#
+# Parameters:
+# 1 - name of an array variable where the filtering results will be stored
+# @ - test names
+function filter_out_prefixed_tests() {
+    local var_name="${1}"; shift
+    local -n results="${var_name}"
+    local name
+    # clear the array, so it will contain results of current filtering
+    # only
+    results=()
+    for name; do
+        if [[ "${name#extra-test.}" = "${name}" ]]; then
+            results+=( "${name}" )
+        fi
+    done
+}
+
+# Merges into the first (main) tap file the contents of other tap
+# files. It is very simple - the function assumes that all the tap
+# files begin with a line like:
+#
+# 1..${number_of_tests}
+#
+# Other lines that are processed should begin like:
+#
+# (not)? ok - ${test_name}
+#
+# Any other lines are copied verbatim.
+#
+# The other tap files should already be preprocessed by
+# prefix_tap_file to avoid duplicated test names.
+#
+# Typical use:
+#   merge_tap_files "${tap_file}" extra-validation-*.tap
+#   rm -f extra-validation-*.tap
+#
+# Parameters:
+# 1 - main tap file
+# @ - other tap files
+function merge_tap_files() {
+    local main_tap_file="${1}"; shift
+    # rest of the parameters are other tap files
+
+    local main_test_count=0
+    if [[ -f "${main_tap_file}" ]]; then
+        main_test_count=$(head --lines=1 "${main_tap_file}" | grep --only-matching '[0-9]\+$')
+    fi
+    local other_test_count
+    local other_tap_file
+    local tmp_tap_file="${main_tap_file}.mtf.tmp"
+    for other_tap_file; do
+        if [[ ! -f "${other_tap_file}" ]]; then
+            continue
+        fi
+        other_test_count=$(head --lines=1 "${other_tap_file}" | grep --only-matching '[0-9]\+$' || echo 0 )
+        ((main_test_count+=other_test_count))
+    done
+    echo "1..${main_test_count}" >"${tmp_tap_file}"
+    if [[ -f "${main_tap_file}" ]]; then
+        tail --lines=+2 "${main_tap_file}" >>"${tmp_tap_file}"
+    fi
+    for other_tap_file; do
+        if [[ ! -f "${other_tap_file}" ]]; then
+            continue
+        fi
+        tail --lines=+2 "${other_tap_file}" >>"${tmp_tap_file}"
+    done
+    mv --force "${tmp_tap_file}" "${main_tap_file}"
+}
+
+# Runs or reruns the tests on the main instance and other
+# instances. Other instances usually run a subset of tests only.
+#
+# For this function to work, the caller needs to define two functions
+# beforehand:
+#
+# run_kola_tests that takes the following parameters:
+# 1 - instance type
+# 2 - tap file
+# @ - tests to run
+#
+# query_kola_tests that takes the following parameters:
+# 1 - instance type
+# @ - tests to run
+# This function should print the names of the tests to run. Every line
+# of the output should have one test name to run. Any other cruft in
+# the line will be ignored.
+#
+# Typical use:
+# function run_kola_tests() {
+#     local instance_type="${1}"; shift
+#     local tap_file="${1}"; shift
+#     kola run … "${@}"
+# }
+#
+# function query_kola_tests() {
+#     local instance_type="${1}"; shift
+#     kola list … "${@}"
+# }
+#
+# args=(
+#     "${main_instance}"
+#     "${CIA_TAPFILE}"
+#     "${CIA_FIRST_RUN}"
+#     "${other_instance_types[@]}"
+#     '--'
+#     'cl.internet'
+#     '--'
+#     "${tests_to_run[@]}"
+# )
+# run_kola_tests_on_instances "${args[@]}"
+#
+# Parameters:
+# 1 - main instance type - there all the tests are being run
+# 2 - main tap file
+# 3 - if this is first run (1 if it is, 0 if it is a rerun)
+# @ - other instance types followed by double dash (--) followed by
+#     test names for other instances to filter from the tests to be
+#     run followed by double dash, followed by tests to be run or
+#     rerun
+function run_kola_tests_on_instances() {
+    local main_instance_type="${1}"; shift
+    local main_tapfile="${1}"; shift
+    local is_first_run="${1}"; shift
+    local other_instance_types=()
+    local other_tests=()
+    local arg
+
+    while [[ "${#}" -gt 0 ]]; do
+        arg="${1}"; shift
+        if [[ "${arg}" = '--' ]]; then
+            break
+        fi
+        other_instance_types+=( "${arg}" )
+    done
+
+    while [[ "${#}" -gt 0 ]]; do
+        arg="${1}"; shift
+        if [[ "${arg}" = '--' ]]; then
+            break
+        fi
+        other_tests+=( "${arg}" )
+    done
+
+    # rest of the parameters are tests to be run or rerun
+
+    local instance_type
+    local queried_tests
+    local instance_tests=()
+    local tests_on_instances_running=0
+    local other_tests_for_fgrep
+    other_tests_for_fgrep="$(printf '%s\n' "${other_tests[@]}")"
+
+    for instance_type in "${other_instance_types[@]}"; do
+        # On first run we usually pass the canonical test names like
+        # cl.basic, cl.internet or *, so we decide which tests should
+        # be run on the other instances based on this list. On the
+        # other hand, the rerun will contain names of the failed tests
+        # only, and those are specific - if a test failed on the main
+        # instance, the name of the test will be like cl.basic; if a
+        # test failed on other instance, the name of the test will be
+        # like extra-test.[…].cl.basic. So in case of reruns, we want
+        # to filter the extra tests first then we decide which tests
+        # should be run.
+        if [[ "${is_first_run}" -eq 1 ]]; then
+            set -o noglob # noglob should not be necessary, as
+                          # query_kola_tests shouldn't return a
+                          # wildcard, but better to be safe than sorry
+            queried_tests="$(query_kola_tests "${instance_type}" "${@}")"
+            instance_tests=( $(grep --only-matching --fixed-strings "${other_tests_for_fgrep}" <<<"${queried_tests}" || :) )
+            set +o noglob
+        else
+            filter_prefixed_tests instance_tests "${instance_type}" "${@}"
+        fi
+        if [[ "${#instance_tests[@]}" -gt 0 ]]; then
+            tests_on_instances_running=1
+            (
+                local instance_tapfile="instance_${instance_type}_validate.tap"
+                set +e
+                set -x
+                local output
+                output=$(run_kola_tests "${instance_type}" "${instance_tapfile}" "${instance_tests[@]}" 2>&1)
+                set +x
+                set -e
+                local escaped_instance_type
+                escaped_instance_type="$(sed -e 's/[\/&]/\\&/g' <<<"${instance_type}")"
+                printf "=== START ${instance_type} ===\n%s\n=== END ${instance_type} ===\n" "$(sed -e "s/^/${escaped_instance_type}: /g" <<<"${output}")"
+                prefix_tap_file "${instance_type}" "${instance_tapfile}"
+            ) &
+        fi
+    done
+
+    local main_tests=()
+
+    filter_out_prefixed_tests main_tests "${@}"
+    if [[ "${#main_tests[@]}" -gt 0 ]]; then
+        # run in a subshell, so the set -x and set +e do not pollute
+        # the outer environment
+        (
+            set +e
+            set -x
+            run_kola_tests "${main_instance_type}" "${main_tapfile}" "${main_tests[@]}"
+            true
+        )
+    fi
+
+    if [[ "${tests_on_instances_running}" -eq 1 ]]; then
+        wait
+        merge_tap_files "${main_tapfile}" 'instance_'*'_validate.tap'
+        rm -f 'instance_'*'_validate.tap'
+    fi
+}
diff --git a/ci-automation/vms.sh b/ci-automation/vms.sh
new file mode 100644
index 0000000000..60765b9bd3
--- /dev/null
+++ b/ci-automation/vms.sh
@@ -0,0 +1,139 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
+#
+# vm_build() should be called w/ the positional INPUT parameters below.
+
+# Vendor images build automation stub.
+#   This script will build one or more vendor images ("vm") using a pre-built packages container.
+#
+# PREREQUISITES:
+#
+#   1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt
+#   2. Scripts repo version tag of OS image version to be built is available and checked out.
+#   3. Flatcar packages container is available via build cache server
+#       from "/containers/[VERSION]/flatcar-images-[ARCH]-[FLATCAR_VERSION].tar.gz"
+#       or present locally. Must contain packages.
+#   4. The generic Flatcar image must be present in build cache server.
+#
+# INPUT:
+#
+#   1. Architecture (ARCH) of the TARGET vm images ("arm64", "amd64").
+#   2. Image formats to be built. Can be multiple, separated by spaces.
+#      Run ./image_to_vm.sh -h in the SDK to get a list of supported images.
+#
+# OPTIONAL INPUT:
+#
+#   1. SIGNER. Environment variable. Name of the owner of the artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored.
+#
+#   2. SIGNING_KEY. Environment variable. The artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored.
+#
+# OUTPUT:
+#
+#   1. Exported VM image(s), pushed to buildcache ( images/[ARCH]/[FLATCAR_VERSION]/ )
+#   2. "./ci-cleanup.sh" with commands to clean up temporary build resources,
+#        to be run after this step finishes / when this step is aborted.
+#   3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache.
+#   4. DIGESTS of the artifacts from point 1, pushed to buildcache. If signer key was passed, armored ASCII files of the generated DIGESTS files too, pushed to buildcache.
+
+function vm_build() {
+    # Run a subshell, so the traps, environment changes and global
+    # variables are not spilled into the caller.
+    (
+        set -euo pipefail
+
+        _vm_build_impl "${@}"
+    )
+}
+# --
+
+function _vm_build_impl() {
+    local arch="$1"
+    shift
+    # $@ now contains image formats to build
+
+    source ci-automation/ci_automation_common.sh
+    source ci-automation/gpg_setup.sh
+    init_submodules
+
+    source sdk_container/.repo/manifests/version.txt
+    local vernum="${FLATCAR_VERSION}"
+    local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
+
+    local packages="flatcar-packages-${arch}"
+    local packages_image="${packages}:${docker_vernum}"
+
+    docker_image_from_buildcache "${packages}" "${docker_vernum}"
+
+    local vms="flatcar-vms-${arch}"
+    local vms_container="${vms}-${docker_vernum}"
+
+    # automatically add PXE to formats if we build for Equinix Metal (packet).
+    local has_packet=0
+    local has_pxe=0
+    for format; do
+        [[ "${format}" = 'packet' ]] || [[ "${format}" = 'equinix_metal' ]] && has_packet=1
+        [[ "${format}" = 'pxe' ]] && has_pxe=1
+    done
+
+    [[ ${has_packet} -eq 1 ]] && [[ ${has_pxe} -eq 0 ]] && set -- 'pxe' "${@}"
+
+    # Convert platform names (also used to find the test scripts) to image formats they entail
+    formats="$*"
+    if echo "$formats" | tr ' ' '\n' | grep -q '^vmware'; then
+      formats=$(echo "$formats" | tr ' ' '\n' | sed '/vmware.*/d')
+      formats+=" vmware vmware_insecure vmware_ova vmware_raw"
+    fi
+    if echo "$formats" | tr ' ' '\n' | grep -q -P '^(ami|aws)'; then
+      formats=$(echo "$formats" | tr ' ' '\n' | sed '/ami.*/d' | sed '/aws/d')
+      formats+=" ami ami_vmdk"
+    fi
+    # Keep compatibility with SDK scripts where "equinix_metal" remains unknown.
+    formats=$(echo "$formats" | tr ' ' '\n' | sed 's/equinix_metal/packet/g')
+
+    local images_in="images-in/"
+    rm -rf "${images_in}"
+    copy_dir_from_buildcache "images/${arch}/${vernum}/" "${images_in}"
+    lbunzip2 "${images_in}/flatcar_production_image.bin.bz2"
+    ./run_sdk_container -x ./ci-cleanup.sh -n "${vms_container}" -C "${packages_image}" \
+            -v "${vernum}" \
+            mkdir -p "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest"
+    ./run_sdk_container -n "${vms_container}" -C "${packages_image}" \
+            -v "${vernum}" \
+            mv "${images_in}" "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest-input"
+
+    for format in ${formats}; do
+        echo " ###################  VENDOR '${format}' ################### "
+        COMPRESSION_FORMAT="bz2"
+        if [[ "${format}" =~ ^(openstack|openstack_mini|digitalocean)$ ]];then
+            COMPRESSION_FORMAT="gz,bz2"
+        fi
+        ./run_sdk_container -n "${vms_container}" -C "${packages_image}" \
+            -v "${vernum}" \
+            ./image_to_vm.sh --format "${format}" --board="${arch}-usr" \
+                --from "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest-input" \
+                --to "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest" \
+                --image_compression_formats="${COMPRESSION_FORMAT}" \
+                --only_store_compressed
+    done
+
+    # copy resulting images + push to buildcache
+    local images_out="images/"
+    rm -rf "${images_out}"
+    ./run_sdk_container -n "${vms_container}" -C "${packages_image}" \
+        -v "${vernum}" \
+        mv "${CONTAINER_IMAGE_ROOT}/${arch}-usr/" "./${images_out}/"
+
+    create_digests "${SIGNER}" "images/latest/"*
+    sign_artifacts "${SIGNER}" "images/latest/"*
+    copy_to_buildcache "images/${arch}/${vernum}/" "images/latest/"*
+}
+# --
diff --git a/retag-for-jenkins b/retag-for-jenkins
new file mode 100755
index 0000000000..469efbc131
--- /dev/null
+++ b/retag-for-jenkins
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if [ $# -lt 1 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
+  echo "Usage: $0 TAG"
+  echo "Rebases the free-standing git tag <TAG> on the current branch"
+  echo "(make sure you don't have any uncommited local changes)"
+  echo "E.g., after a Jenkins build 'alpha-9999.99.99-mytest' was started based on"
+  echo "the scripts branch 'mybranch' and you have new changes on the branch,"
+  echo "check out your branch and run"
+  echo "  $0 alpha-9999.99.99-mytest"
+  echo
+  echo "This is required when testing 'scripts' changes with Jenkins and a leaf job fails,"
+  echo "so that instead of restarting the whole build from the 'packages' job, you can"
+  echo "restart the leaf job after retagging. Note: Just starting a leaf job with your"
+  echo "branch as reference is not valid because it would overwrite the nightly build"
+  echo "artifacts!"
+  echo
+  echo "TODO: Add feature to update coreos-overlay/portage-stable submodule refs"
+  exit 1
+fi
+
+TAG="$1"
+BRANCH=$(git rev-parse --abbrev-ref HEAD)
+git fetch --force --tags origin
+BUILD_PATCH=$(git format-patch --output=/dev/stdout "${TAG}~1..${TAG}")
+git checkout --recurse-submodules "${TAG}"
+git reset --hard "${BRANCH}"
+echo "${BUILD_PATCH}" | git am -3 || {
+  git checkout "${TAG}" -- sdk_container/.repo/manifests/version.txt
+  git add sdk_container/.repo/manifests/version.txt
+  git am --continue
+  # This does not handle submodule conflicts: It should use the one
+  # from the TAG (similar to version.txt) unless an explicit new
+  # reference was specified
+} || { echo "Failed to resolve conflict, continue manually" >&2 ; exit 1 ; }
+git tag -d "${TAG}"
+git tag "${TAG}"
+git push --force origin "${TAG}"
diff --git a/run_sdk_container b/run_sdk_container
new file mode 100755
index 0000000000..41aff89c59
--- /dev/null
+++ b/run_sdk_container
@@ -0,0 +1,150 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -euo pipefail
+
+cd $(dirname "$0")
+source sdk_lib/sdk_container_common.sh
+
+arch="all"
+name=""
+
+os_version="$(get_git_version)"
+sdk_version="$(get_sdk_version_from_versionfile)"
+custom_image=""
+tty=""
+remove=""
+cleanup=""
+
+usage() {
+    echo "  Usage:"
+    echo "  $0 [-t] [-v <version>] [-V sdk version] [-a arch] [-n <name> ] [-x <script>] [-C custom-container] [--rm] [container-command]"
+    echo "       Start an SDK container of a given SDK release version."
+    echo "       This will create the container if it does not exist, otherwise start the existing container."
+    echo "       If the container is already running then it will exec into the container."
+    echo
+    echo "      container-command - command to be run in the container instead of"
+    echo "                     an interactive shell."
+    echo "      -t             Attach docker to a TTY (docker -t)"
+    echo "      -v <version> - Sourcetree (OS image) version to use."
+    echo "                     Defaults to '$os_version' (current git commit)."
+    echo "                     FLATCAR_VERSION[_ID] in '$sdk_container_common_versionfile'"
+    echo "                     will be updated accordingly."
+    echo "      -V <SDK ver> - SDK version to use. Defaults to '${sdk_version}'"
+    echo "                     (FLATCAR_SDK_VERSION from '$sdk_container_common_versionfile')."
+    echo "      -a <amd64|arm64|all> - Target architecture (board support) of the SDK."
+    echo "                     'all' (the default) contains support for both amd64 and arm64."
+    echo "      -n <name>    - Custom name to use for the container."
+    echo "      --rm           Remove container afterwards"
+    echo "      -x <script>  - For each resource generated during build (container etc.)"
+    echo "                     add a cleanup line to <script> which, when run, will free"
+    echo "                     the resource. Useful for CI."
+    echo "      -C           - Use an entirely custom container image instead of the SDK's"
+    echo "                     $sdk_container_common_registry/flatcar-sdk-[ARCH]:[SDK VERSION]"
+    echo "                     Useful for CI."
+    echo "      -h             Print this help."
+    echo
+}
+# --
+
+while [ 0 -lt $# ] ; do
+    case "$1" in
+    -h) usage; exit 0;;
+    --help) usage; exit 0;;
+    -t) tty="-t";                  shift;;
+    -v) os_version="$2";    shift; shift;;
+    -V) sdk_version="$2";   shift; shift;;
+    -a) arch="$2";          shift; shift;;
+    -n) name="$2";          shift; shift;;
+    --rm) remove=true;             shift;;
+    -x) cleanup="$2";       shift; shift;;
+    -C) custom_image="$2";  shift; shift;;
+    *) break;;
+    esac
+done
+
+if [ -n "$custom_image" ] ; then
+    container_image_name="${custom_image}"
+else
+    docker_sdk_vernum="$(vernum_to_docker_image_version "${sdk_version}")"
+    container_image_name="$sdk_container_common_registry/flatcar-sdk-${arch}:${docker_sdk_vernum}"
+fi
+
+create_versionfile "$sdk_version" "$os_version"
+
+if [ -z "$name" ] ; then
+    docker_sdk_vernum="$(vernum_to_docker_image_version "${sdk_version}")"
+    docker_os_vernum="$(vernum_to_docker_image_version "${os_version}")"
+    name="flatcar-sdk-${arch}-${docker_sdk_vernum}_os-${docker_os_vernum}"
+fi
+
+filter="^/"
+if "${is_podman}"; then
+  filter=""
+fi
+stat="$($docker ps --all --no-trunc --filter name="${filter}$name\$" --format '{{.Status}}'\
+        | cut -f1 -d' ')"
+
+# pass SDK related environment variables and gcloud auth
+#  into container
+setup_sdk_env
+setup_gsutil
+
+mkdir -p "__build__/images"
+mkdir -p "sdk_container/.cache/sdks"
+
+hostname="${name:0:63}"
+hostname="${hostname//./_}"
+
+if [ -n "$cleanup" ] ; then
+    echo "$docker container rm -f '${name}'" >> "$cleanup"
+fi
+
+if [ -z "$stat" ] ; then
+    yell "Creating a new container '$name'"
+
+    gpg_volumes=$(gnupg_ssh_gcloud_mount_opts)
+
+    if [ -z "$custom_image" ]; then
+	(
+            source ci-automation/ci_automation_common.sh
+            docker_image_from_registry_or_buildcache "flatcar-sdk-${arch}" "${docker_sdk_vernum}"
+	)
+    fi
+
+    $docker create $tty -i \
+       -v /dev:/dev \
+       -v "$(pwd)/sdk_container:/mnt/host/source/" \
+       -v "$(pwd)/sdk_container/git-override/.git-coreos-overlay:/mnt/host/source/src/third_party/coreos-overlay/.git" \
+       -v "$(pwd)/sdk_container/git-override/.git-portage-stable:/mnt/host/source/src/third_party/portage-stable/.git" \
+       -v "$(pwd)/__build__/images:/mnt/host/source/src/build" \
+       -v "$(pwd):/mnt/host/source/src/scripts" \
+       $gpg_volumes \
+       --privileged \
+       --network host \
+       -e SDK_USER_ID="$(id -u)" \
+       -e SDK_GROUP_ID="$(id -g)" \
+       --name="$name" \
+       --hostname="$hostname" \
+       --entrypoint /bin/bash \
+       "${container_image_name}" -l
+fi
+
+if [ "$stat" != "Up" ] ; then
+    yell "Starting stopped container '$name'"
+    if [ "${remove}" = "true" ]; then
+      remove_command="$docker rm -f $name"
+    else
+      remove_command=":"
+    fi
+    trap "$docker stop -t 0 $name ; ${remove_command}" EXIT
+    $docker start "$name"
+fi
+
+# Workaround: The SDK expects to be able to write to /etc/hosts
+$docker exec "$name" sh -c 'cp /etc/hosts /etc/hosts2; umount /etc/hosts ; mv /etc/hosts2 /etc/hosts'
+
+$docker exec $tty -i "$name" /mnt/host/source/src/scripts/sdk_lib/sdk_entry.sh "$@"
diff --git a/sdk_container/.repo/manifests/mantle-container b/sdk_container/.repo/manifests/mantle-container
new file mode 100644
index 0000000000..5ab79c8290
--- /dev/null
+++ b/sdk_container/.repo/manifests/mantle-container
@@ -0,0 +1 @@
+ghcr.io/flatcar-linux/mantle:git-11159768f92f7a16ffa8681fbe3998aaff60a79b
diff --git a/sdk_container/.repo/manifests/os-release b/sdk_container/.repo/manifests/os-release
new file mode 100644
index 0000000000..3942c3a2d9
--- /dev/null
+++ b/sdk_container/.repo/manifests/os-release
@@ -0,0 +1,6 @@
+NAME="Flatcar Container Linux by Kinvolk"
+ID=flatcar
+ID_LIKE=coreos
+ANSI_COLOR="38;5;75"
+HOME_URL="https://flatcar.org/"
+BUG_REPORT_URL="https://issues.flatcar.org"
diff --git a/sdk_container/.repo/manifests/version.txt b/sdk_container/.repo/manifests/version.txt
new file mode 100644
index 0000000000..bd1fb57296
--- /dev/null
+++ b/sdk_container/.repo/manifests/version.txt
@@ -0,0 +1,4 @@
+FLATCAR_VERSION=3301.0.0+nightly-20220715-2100
+FLATCAR_VERSION_ID=3301.0.0
+FLATCAR_BUILD_ID="nightly-20220715-2100"
+FLATCAR_SDK_VERSION=3301.0.0+nightly-20220715-2100
diff --git a/sdk_container/git-override/.git-coreos-overlay b/sdk_container/git-override/.git-coreos-overlay
new file mode 100644
index 0000000000..507e22fe6e
--- /dev/null
+++ b/sdk_container/git-override/.git-coreos-overlay
@@ -0,0 +1 @@
+gitdir: ../../scripts/.git/modules/sdk_container/src/third_party/coreos-overlay
diff --git a/sdk_container/git-override/.git-portage-stable b/sdk_container/git-override/.git-portage-stable
new file mode 100644
index 0000000000..b3af445704
--- /dev/null
+++ b/sdk_container/git-override/.git-portage-stable
@@ -0,0 +1 @@
+gitdir: ../../scripts/.git/modules/sdk_container/src/third_party/portage-stable
diff --git a/sdk_container/git-override/README.md b/sdk_container/git-override/README.md
new file mode 100644
index 0000000000..fca5ac22ad
--- /dev/null
+++ b/sdk_container/git-override/README.md
@@ -0,0 +1,5 @@
+# GIT overrides for submodules
+
+In the SDK container, the scripts root is at a different relative path to the submodules.
+The overrides in this directory are into `coreos-overlay/.git` and `portage-stable/.git` so the submodules can find their parents.
+
diff --git a/sdk_container/src/third_party/coreos-overlay b/sdk_container/src/third_party/coreos-overlay
new file mode 160000
index 0000000000..159b0048db
--- /dev/null
+++ b/sdk_container/src/third_party/coreos-overlay
@@ -0,0 +1 @@
+Subproject commit 159b0048db19748bc624850e4c0cda0167951544
diff --git a/sdk_container/src/third_party/portage-stable b/sdk_container/src/third_party/portage-stable
new file mode 160000
index 0000000000..f54be57551
--- /dev/null
+++ b/sdk_container/src/third_party/portage-stable
@@ -0,0 +1 @@
+Subproject commit f54be57551aa95b419727d010b053dfa29fb1ad1
diff --git a/sdk_lib/90_env_keep b/sdk_lib/90_env_keep
new file mode 100644
index 0000000000..995b9a1a3d
--- /dev/null
+++ b/sdk_lib/90_env_keep
@@ -0,0 +1,9 @@
+Defaults env_keep += "FLATCAR_BUILD_ID COREOS_OFFICIAL \
+                      EMAIL GIT_AUTHOR_EMAIL GIT_AUTHOR_NAME \
+                      GIT_COMMITTER_EMAIL GIT_COMMITTER_NAME \
+                      GIT_PROXY_COMMAND GIT_SSH RSYNC_PROXY \
+                      GNUPGHOME GPG_AGENT_INFO SSH_AUTH_SOCK \
+                      BOTO_PATH GOOGLE_APPLICATION_CREDENTIALS \
+                      USE FEATURES PORTAGE_USERNAME FORCE_STAGES \
+                      SIGNER \
+                      all_proxy ftp_proxy http_proxy https_proxy no_proxy"
diff --git a/sdk_lib/Dockerfile.lean-arch b/sdk_lib/Dockerfile.lean-arch
new file mode 100644
index 0000000000..ee3bbf68ed
--- /dev/null
+++ b/sdk_lib/Dockerfile.lean-arch
@@ -0,0 +1,29 @@
+ARG VERSION
+
+FROM flatcar-sdk-build:${VERSION} as meta
+ARG RMARCH
+ARG RMCROSS
+
+RUN if [ -n "$RMCROSS" ]; then \
+        sudo crossdev --clean --force "$RMCROSS"; \
+    fi
+
+RUN if [ -n "$RMARCH" ]; then \
+        sudo rm -rf /build/$RMARCH; \
+        sudo rm -f /usr/local/bin/*-$RMARCH; \
+    fi
+
+# Note: .repo/manifests/version.txt will survive this. That's intended.
+RUN sudo rm -rf /mnt/host/source/*
+
+FROM scratch
+
+COPY --from=meta / /
+COPY --from=meta --chown=sdk:sdk /home/sdk /home/sdk
+RUN chown -R sdk:sdk /mnt/host/source
+
+# This is not used when starting the container via ./run_sdk_container
+#  but it's useful for standalone container use.
+RUN mkdir -p /mnt/host/source/src/scripts
+COPY --chown=sdk:sdk sdk_lib/sdk_init_selfcontained.sh /mnt/host/source/src/
+ENTRYPOINT /home/sdk/sdk_entry.sh
diff --git a/sdk_lib/Dockerfile.sdk-build b/sdk_lib/Dockerfile.sdk-build
new file mode 100644
index 0000000000..2e5eca9997
--- /dev/null
+++ b/sdk_lib/Dockerfile.sdk-build
@@ -0,0 +1,17 @@
+ARG VERSION
+FROM flatcar-sdk-import:${VERSION}
+
+ARG BINHOST
+ARG OFFICIAL=0
+
+# mark build as official where appropriate
+RUN echo "export COREOS_OFFICIAL=$OFFICIAL" > /mnt/host/source/.env
+
+RUN /home/sdk/sdk_entry.sh ./setup_board --board="arm64-usr" --binhost="${BINHOST}/arm64-usr"
+RUN /home/sdk/sdk_entry.sh ./setup_board --board="arm64-usr" --regen_configs
+
+RUN /home/sdk/sdk_entry.sh ./setup_board --board="amd64-usr" --binhost="${BINHOST}/amd64-usr"
+RUN /home/sdk/sdk_entry.sh ./setup_board --board="amd64-usr" --regen_configs
+
+RUN rm /mnt/host/source/.env
+RUN rm -rf /home/sdk/toolchain-pkgs
diff --git a/sdk_lib/Dockerfile.sdk-import b/sdk_lib/Dockerfile.sdk-import
new file mode 100644
index 0000000000..6e78369a12
--- /dev/null
+++ b/sdk_lib/Dockerfile.sdk-import
@@ -0,0 +1,58 @@
+ARG VERSION
+FROM flatcar-sdk-tarball:${VERSION}
+
+# Make build scripts believe we're in the SDK chroot (which technically, we are)
+RUN touch /etc/debian_chroot
+RUN chmod 644 /etc/passwd
+RUN chmod 644 /etc/group
+
+# User "root" is not in /etc/passwd / group in the SDK tarball
+RUN echo 'root:x:0:0:root:/root:/bin/bash' >>/etc/passwd
+RUN echo 'root:x:0:' >>/etc/group
+
+RUN if ! grep -q portage /etc/group ; then \
+        echo "portage::250:portage" >>/etc/group; \
+    fi
+RUN if ! grep -q portage /etc/passwd; then \
+        echo "portage:x:250:250:portage:/var/tmp/portage:/bin/false" >>/etc/passwd; \
+    fi
+
+# fix "Unable to unshare: EPERM ..." in containers
+#  (see https://github.com/gentoo/gentoo-docker-images/issues/81)
+RUN echo 'export FEATURES="-ipc-sandbox -network-sandbox -pid-sandbox"' \
+        >> /etc/skel/.bashrc
+
+RUN groupadd sdk
+RUN useradd -g sdk -G portage sdk
+RUN echo "sdk ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/sdk-user
+
+RUN rm -rf /mnt/host/source
+COPY --chown=sdk:sdk sdk_container/ /mnt/host/source
+COPY --chown=sdk:sdk . /mnt/host/source/src/scripts
+RUN chown sdk:sdk /mnt/host/source
+
+COPY sdk_lib/90_env_keep /etc/sudoers.d/90_env_keep
+RUN chmod 0440 /etc/sudoers.d/90_env_keep
+
+USER sdk:sdk
+RUN mkdir -p /mnt/host/source/src/scripts /mnt/host/source/src/build
+RUN ln -s /mnt/host/source /home/sdk/trunk
+
+RUN rm /home/sdk/.bashrc
+RUN cp /etc/skel/.bashrc /home/sdk
+RUN echo "cd /home/sdk/trunk/src/scripts" >> /home/sdk/.bashrc
+RUN echo 'export PATH="$PATH:/usr/local/bin:/usr/local/sbin"' >> /home/sdk/.bashrc
+
+# user and SDK environment variables pass-through into container
+RUN echo "if [ -f /mnt/host/source/.env ]; then source /mnt/host/source/.env; fi" >> /home/sdk/.bashrc
+RUN echo "if [ -f /mnt/host/source/.sdkenv ]; then source /mnt/host/source/.sdkenv; fi" >> /home/sdk/.bashrc
+
+COPY --chown=sdk:sdk sdk_lib/sdk_entry.sh /home/sdk
+RUN chmod 755 /home/sdk/sdk_entry.sh
+
+USER root:root
+# This should be a NOP; if you see packages being rebuilt
+#  it's likely that submodules and SDK tarball are out of sync
+RUN /home/sdk/sdk_entry.sh ./update_chroot --toolchain_boards="amd64-usr arm64-usr"
+
+ENTRYPOINT /home/sdk/sdk_entry.sh
diff --git a/sdk_lib/Dockerfile.sdk-update b/sdk_lib/Dockerfile.sdk-update
new file mode 100644
index 0000000000..409b07c8a4
--- /dev/null
+++ b/sdk_lib/Dockerfile.sdk-update
@@ -0,0 +1,21 @@
+ARG BASE
+
+FROM ${BASE}
+COPY --chown=sdk:sdk sdk_container/ /mnt/host/source
+COPY --chown=sdk:sdk . /mnt/host/source/src/scripts
+
+# Disable all sandboxing for SDK updates since some core packages
+#  (like GO) fail to build from a permission error otherwise.
+RUN cp /home/sdk/.bashrc /home/sdk/.bashrc.bak
+RUN echo 'export FEATURES="-sandbox -usersandbox -ipc-sandbox -network-sandbox -pid-sandbox"' \
+        >> /home/sdk/.bashrc
+
+RUN chown sdk:sdk /mnt/host/source
+RUN /home/sdk/sdk_entry.sh ./update_chroot --toolchain_boards="amd64-usr arm64-usr"
+
+RUN /home/sdk/sdk_entry.sh ./setup_board --board="arm64-usr" --regen_configs
+RUN /home/sdk/sdk_entry.sh ./setup_board --board="amd64-usr" --regen_configs
+
+# Restore original .bashrc to remove sandbox disablement
+RUN mv /home/sdk/.bashrc.bak /home/sdk/.bashrc
+RUN chown sdk:sdk /home/sdk/.bashrc
diff --git a/sdk_lib/sdk_container_common.sh b/sdk_lib/sdk_container_common.sh
new file mode 100644
index 0000000000..e8fd718bff
--- /dev/null
+++ b/sdk_lib/sdk_container_common.sh
@@ -0,0 +1,291 @@
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# This file contains common functions used across SDK container scripts.
+
+#
+# globals
+#
+sdk_container_common_versionfile="sdk_container/.repo/manifests/version.txt"
+sdk_container_common_registry="ghcr.io/flatcar"
+sdk_container_common_env_file="sdk_container/.sdkenv"
+
+# Check for podman and docker; use docker if present, podman alternatively.
+# Podman needs 'sudo' since we need privileged containers for the SDK.
+
+is_podman=false
+if command -v podman >/dev/null; then
+    # podman is present
+    if command -v docker >/dev/null ; then
+        # docker is present, too
+        if docker help | grep -q -i podman; then
+            # "docker" is actually podman.
+            # NOTE that 'docker --version' does not reliably work for podman detection
+            #  since 'podman' uses argv[0] in its version string.
+            #  A symlink docker->podman will result in 'podman' using the 'docker' argv[0].
+            is_podman=true
+        fi
+    else
+        # docker is not present
+        is_podman=true
+    fi
+fi
+
+docker="docker"
+if "${is_podman}"; then
+  docker="sudo podman"
+fi
+
+# Common "echo" function
+
+function yell() {
+    echo -e "\n###### $@ ######"
+}
+# --
+
+# Guess the SDK version from the current git commit.
+#
+function get_git_version() {
+    local tag="$(git tag --points-at HEAD)"
+    if [ -z "$tag" ] ; then
+        git describe --tags
+    else
+        echo "$tag"
+    fi
+}
+# --
+
+function get_sdk_version_from_versionfile() {
+    ( source "$sdk_container_common_versionfile"; echo "$FLATCAR_SDK_VERSION"; )
+}
+# --
+
+function get_version_from_versionfile() {
+    ( source "$sdk_container_common_versionfile"; echo "$FLATCAR_VERSION"; )
+}
+# --
+
+# return true if a given version number is an official build
+#
+function is_official() {
+    local vernum="$1"
+
+    local official="$(echo "$vernum" | sed -n 's/^[0-9]\+\.[0-9]\+\.[0-9]\+$/true/p')"
+
+    test -n "$official"
+}
+# --
+
+# extract the build ID suffix from a version string ("alpha-3244.0.1-nightly2" => "nightly2")
+#
+function build_id_from_version() {
+    local version="$1"
+
+    # support vernums and versions ("alpha-"... is optional)
+    echo "${version}" | sed -n 's/^\([a-z]\+-\)\?[0-9.]\+[-+]\(.*\)$/\2/p'
+}
+# --
+
+# Get channel from a version string ("alpha-3244.0.1-nightly2" => "alpha")
+#
+function channel_from_version() {
+    local version="$1"
+    local channel=""
+
+    channel=$(echo "${version}" | cut -d - -f 1)
+    if [ "${channel}" != "alpha" ] && [ "${channel}" != "beta" ] && [ "${channel}" != "stable" ] && [ "${channel}" != "lts" ]; then
+        channel="developer"
+    fi
+    echo "${channel}"
+}
+# --
+
+function get_git_channel() {
+	channel_from_version "$(get_git_version)"
+}
+# --
+
+# extract the version number (w/o build ID) from a version string ("alpha-3244.0.1-nightly2" => "3244.0.1")
+#
+function vernum_from_version() {
+    local version="$1"
+
+    # support vernums and versions ("alpha-"... is optional)
+    echo "${version}" | sed -n 's/^\([a-z]\+-\)\?\([0-9.]\+\).*/\2/p'
+}
+# --
+
+# Strip prefix from version string if present ("alpha-3233.0.0[-...]" => "3233.0.0[-...]")
+#  and add a "+[build suffix]" if this is a non-official build. The "+" matches the version
+#  string generation in the build scripts.
+function strip_version_prefix() {
+    local version="$1"
+
+    local build_id="$(build_id_from_version "${version}")"
+    local version_id="$(vernum_from_version "${version}")"
+
+    if [ -n "${build_id}" ] ; then
+        echo "${version_id}+${build_id}"
+    else
+        echo "${version_id}"
+    fi
+}
+# --
+
+# Derive docker-safe image version string from vernum.
+#
+function vernum_to_docker_image_version() {
+    local vernum="$1"
+    echo "$vernum" | sed 's/[+]/-/g'
+}
+# --
+
+# Creates the Flatcar build / SDK version file.
+# Must be called from the script root.
+#
+#  In the versionfile, FLATCAR_VERSION is the OS image version _number_ plus a build ID if this is no
+#   official build. The FLATCAR_VERSION_ID is the plain vernum w/o build ID - it's the same as FLATCAR_VERSION
+#   for official builds. The FLATCAR_BUILD_ID is the build ID suffix for non-official builds.
+#   Lastly, the FLATCAR_SDK_VERSION is the full version number (including build ID if no official SDK release)
+#   the OS image is to be built with.
+#
+function create_versionfile() {
+    local sdk_version="$1"
+    local os_version="${2:-$sdk_version}"
+    local build_id="$(build_id_from_version "${os_version}")"
+    local version_id="$(vernum_from_version "${os_version}")"
+
+    sdk_version="$(strip_version_prefix "${sdk_version}")"
+    os_version="$(strip_version_prefix "${os_version}")"
+    yell "Writing versionfile '$sdk_container_common_versionfile' to SDK '$sdk_version', OS '$os_version'."
+
+    cat >"$sdk_container_common_versionfile" <<EOF
+FLATCAR_VERSION=${os_version}
+FLATCAR_VERSION_ID=${version_id}
+FLATCAR_BUILD_ID="${build_id}"
+FLATCAR_SDK_VERSION=${sdk_version}
+EOF
+}
+# --
+
+#
+# Set up SDK environment variables.
+#  Environment vars are put in a file that is sourced by the container's
+#  .bashrc (if present). GNUPGHOME and SSH_AUTH_SOCK are set
+#  to container-specific paths if applicable.
+
+function setup_sdk_env() {
+    local var
+
+    rm -f "$sdk_container_common_env_file"
+
+    # conditionally set up gnupg, ssh socket, and gcloud auth / boto
+    #  depending on availability on the host
+    GNUPGHOME="${GNUPGHOME:-$HOME/.gnupg}"
+    if [ -d "${GNUPGHOME}" ] ; then
+        echo "GNUPGHOME=\"/home/sdk/.gnupg\""  >> "$sdk_container_common_env_file"
+        echo "export GNUPGHOME"  >> "$sdk_container_common_env_file"
+        export GNUPGHOME
+    fi
+
+    if [ -e "${SSH_AUTH_SOCK:-}" ] ; then
+        local sockname="$(basename "${SSH_AUTH_SOCK}")"
+        echo "SSH_AUTH_SOCK=\"/run/sdk/ssh/$sockname\""  >> "$sdk_container_common_env_file"
+        echo "export SSH_AUTH_SOCK"  >> "$sdk_container_common_env_file"
+    fi
+
+    # keep in sync with 90_env_keep
+    for var in FLATCAR_BUILD_ID COREOS_OFFICIAL \
+        EMAIL GIT_AUTHOR_EMAIL GIT_AUTHOR_NAME \
+        GIT_COMMITTER_EMAIL GIT_COMMITTER_NAME \
+        GIT_PROXY_COMMAND GIT_SSH RSYNC_PROXY \
+        GPG_AGENT_INFO FORCE_STAGES \
+        SIGNER \
+        all_proxy ftp_proxy http_proxy https_proxy no_proxy; do
+
+        if [ -n "${!var:-}" ] ; then
+            echo "${var}=\"${!var}\"" >> "$sdk_container_common_env_file"
+            echo "export ${var}" >> "$sdk_container_common_env_file"
+        fi
+    done
+}
+# --
+
+# Set up gcloud legacy creds (via GOOGLE_APPLICATION_CREDENTIALS)
+#  for the SDK container.
+#  This will also create a boto config right next to the
+#  GOOGLE_APPLICATION_CREDENTIALS json file.
+
+function setup_gsutil() {
+    local creds="${GOOGLE_APPLICATION_CREDENTIALS:-$HOME/.config/gcloud/application_default_credentials.json}"
+    if [ ! -e "$creds" ]; then
+        return
+    fi
+
+    local creds_dir="$(dirname "$creds")"
+    local botofile="$creds_dir/boto-flatcar-sdk"
+
+    # TODO t-lo: move generation of boto file to sdk_entry so
+    #               it's only created inside the container.
+
+    # read creds file and create boto file for gsutil
+    local tmp="$(mktemp)"
+    trap "rm -f '$tmp'" EXIT
+
+    local oauth_refresh="$(jq  -r '.refresh_token' "$creds")"
+    local client_id="$(jq  -r '.client_id' "$creds")"
+    local client_secret="$(jq  -r '.client_secret' "$creds")"
+
+    cat >>"$tmp" <<EOF
+[Credentials]
+gs_oauth2_refresh_token = $oauth_refresh
+
+[OAuth2]
+client_id = $client_id
+client_secret = $client_secret
+EOF
+    mv "$tmp" "$botofile"
+
+    echo "BOTO_PATH=\"$botofile\"" >> "$sdk_container_common_env_file"
+    echo "export BOTO_PATH" >> "$sdk_container_common_env_file"
+    echo "GOOGLE_APPLICATION_CREDENTIALS=\"$creds\"" >> "$sdk_container_common_env_file"
+    echo "export GOOGLE_APPLICATION_CREDENTIALS" >> "$sdk_container_common_env_file"
+
+    BOTO_PATH="$botofile"
+    GOOGLE_APPLICATION_CREDENTIALS="$creds"
+    export BOTO_PATH
+    export GOOGLE_APPLICATION_CREDENTIALS
+}
+
+   
+# --
+
+# Generate volume mount command line options for docker
+#  to pass gpg, ssh, and gcloud auth host directories
+#  into the SDK container.
+
+function gnupg_ssh_gcloud_mount_opts() {
+    local sdk_gnupg_home="/home/sdk/.gnupg"
+    local gpgagent_dir="/run/user/$(id -u)/gnupg"
+
+    # pass host GPG home and Agent directories to container
+    if [ -d "$GNUPGHOME" ] ; then
+        echo "-v $GNUPGHOME:$sdk_gnupg_home"
+    fi
+    if [ -d "$gpgagent_dir" ] ; then
+        echo "-v $gpgagent_dir:$gpgagent_dir"
+    fi
+
+    if [ -e "${SSH_AUTH_SOCK:-}" ] ; then
+        local sshsockdir="$(dirname "$SSH_AUTH_SOCK")"
+        echo "-v $sshsockdir:/run/sdk/ssh"
+    fi
+
+    if [ -e "${GOOGLE_APPLICATION_CREDENTIALS:-}" ] ; then
+        local creds_dir="$(dirname "${GOOGLE_APPLICATION_CREDENTIALS}")"
+        if [ -d "$creds_dir" ] ; then
+            echo "-v $creds_dir:$creds_dir"
+        fi
+    fi
+}
diff --git a/sdk_lib/sdk_entry.sh b/sdk_lib/sdk_entry.sh
new file mode 100755
index 0000000000..eca47d7fa8
--- /dev/null
+++ b/sdk_lib/sdk_entry.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+
+if [ -n "${SDK_USER_ID:-}" ] ; then
+    # If the "core" user from /usr/share/baselayout/passwd has the same ID, allow to take it instead
+    usermod --non-unique -u $SDK_USER_ID sdk
+fi
+if [ -n "${SDK_GROUP_ID:-}" ] ; then
+    groupmod --non-unique -g $SDK_GROUP_ID sdk
+fi
+
+chown -R sdk:sdk /home/sdk
+
+# Check if the OS image version we're working on is newer than
+#  the SDK container version and if it is, update the boards
+#  chroot portage conf to point to the correct binhost.
+(
+    source /etc/lsb-release # SDK version in DISTRIB_RELEASE
+    source /mnt/host/source/.repo/manifests/version.txt # OS image version in FLATCAR_VERSION_ID
+    version="${FLATCAR_VERSION_ID}"
+
+    # If this is a nightly build tag we can use pre-built binaries directly from the
+    #  build cache.
+    if [[ "${FLATCAR_BUILD_ID}" =~ ^nightly-.*$ ]] ; then
+        version="${FLATCAR_VERSION_ID}+${FLATCAR_BUILD_ID}"
+    fi
+
+    if [ "${version}" != "${DISTRIB_RELEASE}" ] ; then
+        for target in amd64-usr arm64-usr; do
+            if [ ! -d "/build/$target" ] ; then
+                continue
+            fi
+            if [ -f "/build/$target/etc/target-version.txt" ] ; then
+                source "/build/$target/etc/target-version.txt"
+                if [ "${TARGET_FLATCAR_VERSION}" = "${version}" ] ; then
+                    continue # already updated
+                fi
+            fi
+
+            echo
+            echo "Updating board support in '/build/${target}' to use package cache for version '${version}'"
+            echo "---"
+            sudo su sdk -l -c "/home/sdk/trunk/src/scripts/setup_board --board='$target' --regen_configs_only"
+            echo "TARGET_FLATCAR_VERSION='${version}'" | sudo tee "/build/$target/etc/target-version.txt" >/dev/null
+        done
+    fi
+)
+
+# This is ugly.
+#   We need to sudo su - sdk -c so the SDK user gets a fresh login.
+#    'sdk' is member of multiple groups, and plain docker USER only
+#    allows specifying membership of a single group.
+#    When a command is passed to the container, we run, respectively:
+#    sudo su - sdk -c "<command>".
+#   Then, we need to preserve whitespaces in arguments of commands
+#    passed to the container, e.g.
+#    ./update_chroot --toolchain_boards="amd64-usr arm64-usr".
+#    This is done via a separate ".cmd" file since we have used up
+#    our quotes for su -c "<cmd>" already.
+if [ $# -gt 0 ] ; then
+    cmd="/home/sdk/.cmd"
+    echo -n "exec bash -l -i -c '" >"$cmd"
+    for arg in "$@"; do
+        echo -n "\"$arg\" " >>"$cmd"
+    done
+    echo "'" >>"$cmd"
+    chmod 755 "$cmd"
+    sudo su sdk -c "$cmd"
+    rc=$?
+    rm -f "$cmd"
+    exit $rc
+else
+    exec sudo su -l sdk
+fi
diff --git a/sdk_lib/sdk_init_selfcontained.sh b/sdk_lib/sdk_init_selfcontained.sh
new file mode 100755
index 0000000000..3ddda3c5e6
--- /dev/null
+++ b/sdk_lib/sdk_init_selfcontained.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+echo "This script will initialise your Flatcar SDK container as a self-contained SDK."
+echo "Please note that the preferred way of using the Flatcar SDK container is by cloning"
+echo "   https://github.com/flatcar/scripts"
+echo "and using the ./run_sdk_container script."
+
+echo
+echo "Press [RETURN] to continue, CTRL+C to abort"
+echo
+
+read junk
+unset junk
+
+# --
+
+function clone_version() {
+    local repo="$1"
+    local dest="$2"
+    local version="$3"
+
+    git clone https://github.com/flatcar/$repo "$dest"
+    git -C "${dest}" fetch --all
+    local tag=$(git -C "${dest}" tag -l | grep "${version}")
+    git -C "${dest}" checkout "$tag"
+}
+# --
+
+version="$(source /mnt/host/source/.repo/manifests/version.txt; echo $FLATCAR_VERSION)"
+
+mkdir -p /home/sdk/trunk/src/third_party/
+
+clone_version scripts /home/sdk/trunk/src/scripts "$version"
+clone_version portage-stable /home/sdk/trunk/src/third_party/portage-stable "$version"
+clone_version coreos-overlay /home/sdk/trunk/src/third_party/coreos-overlay "$version"
+
diff --git a/settings.env b/settings.env
new file mode 100644
index 0000000000..a0b0cd537d
--- /dev/null
+++ b/settings.env
@@ -0,0 +1,8 @@
+# Flatcar SDK settings
+
+# Binary package caches, for releases and for development (nightlies etc.)
+SETTING_BINPKG_SERVER_PROD="https://mirror.release.flatcar-linux.net"
+
+# development servers / bin caches.
+SETTING_BINPKG_SERVER_DEV="https://bucket.release.flatcar-linux.net/flatcar-jenkins"
+SETTING_BINPKG_SERVER_DEV_CONTAINERISED="https://bincache.flatcar-linux.net"
diff --git a/update_sdk_container_image b/update_sdk_container_image
new file mode 100755
index 0000000000..79dedd6f60
--- /dev/null
+++ b/update_sdk_container_image
@@ -0,0 +1,100 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This script will update an SDK container image and create a new minor version.
+
+
+set -eu
+
+cd $(dirname "$0")
+source sdk_lib/sdk_container_common.sh
+
+os_version="$(get_version_from_versionfile)"
+base_sdk_version="$(get_sdk_version_from_versionfile)"
+new_sdk_version=""
+
+keep="false"
+cleanup=""
+
+usage() {
+    echo "  $0 - Update SDK container image."
+    echo "       Create a new container image based on the current SDK ($base_sdk_version)"
+    echo "       with current changes from coreos-overlay and portage-stable."
+    echo
+    echo "       Just like build_sdk_container_image the resulting container comes in 3 flavours:"
+    echo "        1.    flatcar-sdk-all - includes both ARM64 and AMD64 support"
+    echo "        2.+3. flatcar-sdk-(amd64|arm64) - only includes support for one target."
+    echo "  Usage:"
+    echo "  $0  [-k] [-x <script>] <new-sdk-version>"
+    echo
+    echo "      <new-sdk-version> is the new SDK version to be built."
+    echo "      -k           - Keep intermediate container image."
+    echo "      -x <script>  - For each resource generated during build (container etc.)"
+    echo "                     add a cleanup line to <script> which, when run, will free"
+    echo "                     the resource. Useful for CI."
+    echo
+}
+# --
+
+
+while [ 0 -lt $# ] ; do
+    case "$1" in
+    -h) usage; exit 0;;
+    -k) keep="true";      shift;;
+    -x) cleanup="$2";     shift; shift;;
+    *)  if [ -z "$new_sdk_version" ] ;  then
+            new_sdk_version="$1"; shift
+        else
+            echo "ERROR: spurious positional parameter '$@'."
+            usage
+            exit 1
+        fi;;
+    esac
+done
+
+if [ -z "$new_sdk_version" ] ; then
+    echo
+    echo "ERROR: missing target SDK version."
+    echo
+    usage
+    exit 1
+fi
+# --
+
+docker_vernum="$(vernum_to_docker_image_version "${new_sdk_version}")"
+sdk_build_image="flatcar-sdk-build:${docker_vernum}"
+
+if [ -n "$cleanup" ] ; then
+    echo "$docker image rm -f '${sdk_build_image}'" >> "$cleanup"
+fi
+
+yell "Creating new SDK container image ${new_sdk_version} from ${base_sdk_version}"
+create_versionfile "${new_sdk_version}" "${os_version}"
+
+$docker build -t "${sdk_build_image}" \
+    --build-arg BASE="$sdk_container_common_registry/flatcar-sdk-all:${base_sdk_version}" \
+    -f sdk_lib/Dockerfile.sdk-update \
+    .
+
+for a in all arm64 amd64; do
+    yell "Creating '$a' arch SDK image"
+    rmarch=""; rmcross=""
+    case $a in
+        arm64) rmarch="amd64-usr"; rmcross="x86_64-cros-linux-gnu";;
+        amd64) rmarch="arm64-usr"; rmcross="aarch64-cros-linux-gnu";;
+    esac
+    $docker build -t "$sdk_container_common_registry/flatcar-sdk-${a}:${docker_vernum}" \
+                 --build-arg VERSION="${docker_vernum}" \
+                 --build-arg RMARCH="${rmarch}" \
+                 --build-arg RMCROSS="${rmcross}" \
+                 -f sdk_lib/Dockerfile.lean-arch \
+                 .
+done
+
+if ! $keep; then
+    yell "Cleaning up intermediate container image"
+    $docker rmi flatcar-sdk-build:"${docker_vernum}"
+fi