Merge pull request #334 from flatcar-linux/krnowak/sign-images

ci-automation: Sign the artifacts

ci-automation/ci_automation_common.sh

@@ -175,7 +175,8 @@ function docker_image_to_buildcache() {
     local tarball="$(basename "$image")-${version}.tar.gz"
 
     $docker save "${image}":"${version}" | $PIGZ -c > "${tarball}"
-    copy_to_buildcache "containers/${version}" "${tarball}"
+    sign_artifacts "${SIGNER:-}" "${tarball}"
+    copy_to_buildcache "containers/${version}" "${tarball}"*
 }
 # --
 
@@ -299,3 +300,50 @@ function secret_to_file() {
     config_ref="/proc/${$}/fd/${fd}"
 }
 # --
+
+# Creates signatures for the passed files and directories. In case of
+# directory, all files inside are signed. Files ending with .asc or
+# .sig or .gpg are ignored, though. This function is a noop if signer
+# is empty.
+#
+# Typical use:
+#   sign_artifacts "${SIGNER}" artifact.tar.gz
+#   copy_to_buildcache "artifacts/directory" artifact.tar.gz*
+#
+# Parameters:
+#
+# 1 - signer whose key is expected to be already imported into the
+#       keyring
+# @ - files and directories to sign
+function sign_artifacts() {
+    local signer="${1}"; shift
+    # rest of the parameters are directories/files to sign
+    local to_sign=()
+    local file
+    local files
+
+    if [[ -z "${signer}" ]]; then
+        return
+    fi
+
+    for file; do
+        files=()
+        if [[ -d "${file}" ]]; then
+            readarray -d '' files < <(find "${file}" ! -type d -print0)
+        elif [[ -e "${file}" ]]; then
+            files+=( "${file}" )
+        fi
+        for file in "${files[@]}"; do
+            if [[ "${file}" =~ \.(asc|gpg|sig)$ ]]; then
+                continue
+            fi
+            to_sign+=( "${file}" )
+        done
+    done
+    for file in "${to_sign[@]}"; do
+        gpg --batch --local-user "${signer}" \
+            --output "${file}.sig" \
+            --detach-sign "${file}"
+    done
+}
+# --

ci-automation/garbage_collect.sh

@@ -24,10 +24,19 @@
 #   in the scripts repo. The newest 50 builds will be retained,
 #   all older builds will be purged (50 is the default, see OPTIONAL INPUT above).
 
-set -eu
-
 function garbage_collect() {
-    local keep="${1:-50}" 
+    # Run a subshell, so the traps, environment changes and global
+    # variables are not spilled into the caller.
+    (
+        set -euo pipefail
+
+        _garbage_collect_impl "${@}"
+    )
+}
+# --
+
+function _garbage_collect_impl() {
+    local keep="${1:-50}"
     local dry_run="${DRY_RUN:-}"
     local purge_versions="${PURGE_VERSIONS:-}"
 
@@ -53,7 +62,7 @@ function garbage_collect() {
 
     local sshcmd="$(gen_sshcmd)"
 
-    echo 
+    echo
     echo "######## The following version(s) will be purged ########"
     if [ "$dry_run" = "y" ] ; then
         echo
@@ -61,13 +70,13 @@ function garbage_collect() {
         echo
     fi
     echo "${purge_versions}" | awk -v keep="${keep}" '{if ($0 == "") next; printf "%5d %s\n", NR + keep - 1, $0}'
-    echo 
-    echo 
+    echo
+    echo
 
     local version
     for version in ${purge_versions}; do
         echo "--------------------------------------------"
-        echo 
+        echo
         echo "#### Processing version '${version}' ####"
         echo
 
@@ -144,3 +153,4 @@ function garbage_collect() {
       --env VMWARE_ESX_CREDS \
       -w /work -v "$PWD":/work "${mantle_ref}" /work/ci-automation/garbage_collect_cloud.sh
 }
+# --

ci-automation/gpg_setup.sh

@@ -0,0 +1,31 @@
+# Common gpg setup code to be sourced by other scripts in this
+# directory. It will set up GnuPG home directory, possibly with a key
+# from SIGNING_KEY environment variable.
+#
+# After this file is sourced, SIGNER is always defined and exported,
+# even if empty. SIGNING_KEY is clobbered.
+
+: ${SIGNING_KEY:=''}
+: ${SIGNER:=''}
+
+if [[ "${HOME}/.gnupg" -ef "${PWD}/.gnupg" ]]; then
+    echo 'Do not source ${BASH_SOURCE} directly in your home directory - it will clobber your GnuPG directory!' >&2
+    exit 1
+fi
+
+export GNUPGHOME="${PWD}/.gnupg"
+rm -rf "${GNUPGHOME}"
+trap 'rm -rf "${GNUPGHOME}"' EXIT
+mkdir --mode=0700 "${GNUPGHOME}"
+# Sometimes this directory is not automatically created thus making
+# further private key imports to fail. Let's create it here as a
+# workaround.
+mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/"
+if [[ -n "${SIGNING_KEY}" ]] && [[ -n "${SIGNER}" ]]; then
+    gpg --import "${SIGNING_KEY}"
+else
+    SIGNER=''
+fi
+export SIGNER
+# Clobber signing key variable, we don't need it any more.
+export SIGNING_KEY=''

ci-automation/image.sh

@@ -23,6 +23,16 @@
 #
 #   1. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64").
 #
+# OPTIONAL INPUT:
+#
+#   1. SIGNER. Environment variable. Name of the owner of the artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored.
+#
+#   2. SIGNING_KEY. Environment variable. The artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored.
+#
 # OUTPUT:
 #
 #   1. Exported container image with OS image, dev container, and related artifacts at
@@ -31,16 +41,27 @@
 #        pushed to buildcache.
 #   2. "./ci-cleanup.sh" with commands to clean up temporary build resources,
 #        to be run after this step finishes / when this step is aborted.
-
-set -eu
+#   3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache.
 
 function image_build() {
+    # Run a subshell, so the traps, environment changes and global
+    # variables are not spilled into the caller.
+    (
+        set -euo pipefail
+
+        _image_build_impl "${@}"
+    )
+}
+# --
+
+function _image_build_impl() {
     local arch="$1"
 
     source sdk_lib/sdk_container_common.sh
     local channel=""
     channel="$(get_git_channel)"
     source ci-automation/ci_automation_common.sh
+    source ci-automation/gpg_setup.sh
     init_submodules
 
     source sdk_container/.repo/manifests/version.txt

ci-automation/packages.sh

@@ -45,6 +45,14 @@
 #       This version will be checked out / pulled from remote in the portage-stable git submodule.
 #       The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed.
 #
+#   5. SIGNER. Environment variable. Name of the owner of the artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored.
+#
+#   6. SIGNING_KEY. Environment variable. The artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored.
+#
 # OUTPUT:
 #
 #   1. Exported container image "flatcar-packages-[ARCH]-[VERSION].tar.gz" with binary packages
@@ -55,17 +63,27 @@
 #        - sdk_container/.repo/manifests/version.txt denotes new FLATCAR OS version
 #   3. "./ci-cleanup.sh" with commands to clean up temporary build resources,
 #        to be run after this step finishes / when this step is aborted.
-
-
-set -eu
+#   4. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache.
 
 function packages_build() {
+    # Run a subshell, so the traps, environment changes and global
+    # variables are not spilled into the caller.
+    (
+        set -euo pipefail
+
+        _packages_build_impl "${@}"
+    )
+}
+# --
+
+function _packages_build_impl() {
     local version="$1"
     local arch="$2"
     local coreos_git="${3:-}"
     local portage_git="${4:-}"
 
     source ci-automation/ci_automation_common.sh
+    source ci-automation/gpg_setup.sh
     init_submodules
 
     check_version_string "${version}"
@@ -157,9 +175,12 @@ function packages_build() {
     docker_commit_to_buildcache "${packages_container}" "${packages_image}" "${docker_vernum}"
 
     # Publish torcx manifest and docker tarball to "images" cache so tests can pull it later.
-    copy_to_buildcache "images/${arch}/${vernum}/torcx" \
-        "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json"
-    copy_to_buildcache "images/${arch}/${vernum}/torcx" \
+    sign_artifacts "${SIGNER}" \
+        "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json" \
         "${torcx_tmp}/torcx/pkgs/${arch}-usr/docker/"*/*.torcx.tgz
+    copy_to_buildcache "images/${arch}/${vernum}/torcx" \
+        "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json"*
+    copy_to_buildcache "images/${arch}/${vernum}/torcx" \
+        "${torcx_tmp}/torcx/pkgs/${arch}-usr/docker/"*/*.torcx.tgz*
 }
 # --

ci-automation/push_pkgs.sh

@@ -25,13 +25,22 @@
 #
 #   1. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64").
 #
+# OPTIONAL INPUT:
+#
+#   1. SIGNER. Environment variable. Name of the owner of the artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored.
+#
+#   2. SIGNING_KEY. Environment variable. The artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored.
+#
 # OUTPUT:
 #
 #   1. Binary packages published to buildcache at "boards/[ARCH]-usr/[VERSION]/pkgs".
 #   2. "./ci-cleanup.sh" with commands to clean up temporary build resources,
 #        to be run after this step finishes / when this step is aborted.
-
-set -eu
+#   3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache.
 
 # This function is run _inside_ the SDK container
 function image_build__copy_to_bincache() {
@@ -40,15 +49,35 @@ function image_build__copy_to_bincache() {
 
     source ci-automation/ci_automation_common.sh
 
+    # change the owner of the files and directories in __build__ back
+    # to ourselves, otherwise we could fail to sign the artifacts as
+    # we lacked write permissions in the directory of the signed
+    # artifact
+    local uid=$(id --user)
+    local gid=$(id --group)
     cd /build/$arch-usr/var/lib/portage/pkgs/
+    sudo chown --recursive "${uid}:${gid}" .
+    sign_artifacts "${SIGNER}" *
     copy_to_buildcache "boards/$arch-usr/$version/pkgs" *
 }
 # --
 
 function push_packages() {
+    # Run a subshell, so the traps, environment changes and global
+    # variables are not spilled into the caller.
+    (
+        set -euo pipefail
+
+        _push_packages_impl "${@}"
+    )
+}
+# --
+
+function _push_packages_impl() {
     local arch="$1"
 
     source ci-automation/ci_automation_common.sh
+    source ci-automation/gpg_setup.sh
     init_submodules
 
     source sdk_container/.repo/manifests/version.txt

ci-automation/sdk_bootstrap.sh

@@ -39,6 +39,14 @@
 #   5. ARCH. Environment variable. Target architecture for the SDK to run on.
 #        Either "amd64" or "arm64"; defaults to "amd64" if not set.
 #
+#   6. SIGNER. Environment variable. Name of the owner of the artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored.
+#
+#   7. SIGNING_KEY. Environment variable. The artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored.
+#
 # OUTPUT:
 #
 #   1. SDK tarball (gentoo catalyst output) of the new SDK, pushed to buildcache.
@@ -47,10 +55,20 @@
 #        - sdk_container/.repo/manifests/version.txt denotes new SDK version
 #   3. "./ci-cleanup.sh" with commands to clean up temporary build resources,
 #        to be run after this step finishes / when this step is aborted.
-
-set -eu
+#   4. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache.
 
 function sdk_bootstrap() {
+    # Run a subshell, so the traps, environment changes and global
+    # variables are not spilled into the caller.
+    (
+        set -euo pipefail
+
+        _sdk_bootstrap_impl "${@}"
+    )
+}
+# --
+
+function _sdk_bootstrap_impl() {
     local seed_version="$1"
     local version="$2"
     local coreos_git="${3-}"
@@ -58,6 +76,7 @@ function sdk_bootstrap() {
     : ${ARCH:="amd64"}
 
     source ci-automation/ci_automation_common.sh
+    source ci-automation/gpg_setup.sh
     init_submodules
 
     check_version_string "${version}"
@@ -115,7 +134,15 @@ function sdk_bootstrap() {
     source sdk_container/.repo/manifests/version.txt
     local dest_tarball="flatcar-sdk-${ARCH}-${FLATCAR_SDK_VERSION}.tar.bz2"
 
+    # change the owner of the files and directories in __build__ back
+    # to ourselves, otherwise we could fail to sign the artifacts as
+    # we lacked write permissions in the directory of the signed
+    # artifact
+    local uid=$(id --user)
+    local gid=$(id --group)
+    sudo chown --recursive "${uid}:${gid}" __build__
     cd "__build__/images/catalyst/builds/flatcar-sdk"
+    sign_artifacts "${SIGNER}" "${dest_tarball}"*
     copy_to_buildcache "sdk/${ARCH}/${FLATCAR_SDK_VERSION}" "${dest_tarball}"*
     cd -
 }

ci-automation/sdk_container.sh

@@ -19,22 +19,41 @@
 #       SDK tarball is available on BUILDCACHE/sdk/[ARCH]/[VERSION]/flatcar-sdk-[ARCH]-[VERSION].tar.bz2
 #
 # OPTIONAL INPUT:
-
+#
 #   2. ARCH. Environment variable. Target architecture for the SDK to run on.
 #        Either "amd64" or "arm64"; defaults to "amd64" if not set.
 #
+#   3. SIGNER. Environment variable. Name of the owner of the artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored.
+#
+#   4. SIGNING_KEY. Environment variable. The artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored.
+#
 # OUTPUT:
 #
 #   1. SDK container image of the new SDK, published to buildcache.
 #   2. "./ci-cleanup.sh" with commands to clean up temporary build resources,
 #        to be run after this step finishes / when this step is aborted.
-
-set -eu
+#   3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache.
 
 function sdk_container_build() {
+    # Run a subshell, so the traps, environment changes and global
+    # variables are not spilled into the caller.
+    (
+        set -euo pipefail
+
+        _sdk_container_build_impl "${@}"
+    )
+}
+# --
+
+function _sdk_container_build_impl() {
     : ${ARCH:="amd64"}
 
     source ci-automation/ci_automation_common.sh
+    source ci-automation/gpg_setup.sh
 
     init_submodules
 

ci-automation/tapfile_helper_lib.sh

@@ -6,7 +6,7 @@
 
 # Helper script for extracting information from TAP files and for merging multiple
 #  TAP files into one report.
-# The script uses a temporary SQLite DB for querzing and for result generation.
+# The script uses a temporary SQLite DB for querying and for result generation.
 #
 # Brief usage overview (scroll down for parameters etc.):
 #   tap_ingest_tapfile - add test results from tap file to the DB 

ci-automation/test.sh

@@ -74,8 +74,6 @@
 # script would need to make anyway. For more information, please refer
 # to the vendor_test.sh file.
 
-set -euo pipefail
-
 # Download torcx package and manifest, add build cache URL to manifest
 #  so the docker.torcx-manifest-pkgs test can use it.
 function __prepare_torcx() {
@@ -102,6 +100,17 @@ function __prepare_torcx() {
 # --
 
 function test_run() {
+    # Run a subshell, so the traps, environment changes and global
+    # variables are not spilled into the caller.
+    (
+        set -euo pipefail
+
+        _test_run_impl "${@}"
+    )
+}
+# --
+
+function _test_run_impl() {
     local arch="$1" ; shift
     local image="$1"; shift
 

ci-automation/vms.sh

@@ -25,20 +25,41 @@
 #   2. Image formats to be built. Can be multiple, separated by spaces.
 #      Run ./image_to_vm.sh -h in the SDK to get a list of supported images.
 #
+# OPTIONAL INPUT:
+#
+#   1. SIGNER. Environment variable. Name of the owner of the artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored.
+#
+#   2. SIGNING_KEY. Environment variable. The artifact signing key.
+#        Defaults to nothing if not set - in such case, artifacts will not be signed.
+#        If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored.
+#
 # OUTPUT:
 #
 #   1. Exported VM image(s), pushed to buildcache ( images/[ARCH]/[FLATCAR_VERSION]/ )
 #   2. "./ci-cleanup.sh" with commands to clean up temporary build resources,
 #        to be run after this step finishes / when this step is aborted.
-
-set -eu
+#   3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache.
 
 function vm_build() {
+    # Run a subshell, so the traps, environment changes and global
+    # variables are not spilled into the caller.
+    (
+        set -euo pipefail
+
+        _vm_build_impl "${@}"
+    )
+}
+# --
+
+function _vm_build_impl() {
     local arch="$1"
     shift
     # $@ now contains image formats to build
 
     source ci-automation/ci_automation_common.sh
+    source ci-automation/gpg_setup.sh
     init_submodules
 
     source sdk_container/.repo/manifests/version.txt
@@ -96,6 +117,7 @@ function vm_build() {
         cp --reflink=auto -R "${CONTAINER_IMAGE_ROOT}/${arch}-usr/" "./${images_out}/"
 
     cd "images/latest"
+    sign_artifacts "${SIGNER}" *
     copy_to_buildcache "images/${arch}/${vernum}/" *
 }
 # --

sdk_lib/90_env_keep

@@ -5,4 +5,5 @@ Defaults env_keep += "FLATCAR_BUILD_ID COREOS_OFFICIAL \
                       GNUPGHOME GPG_AGENT_INFO SSH_AUTH_SOCK \
                       BOTO_PATH GOOGLE_APPLICATION_CREDENTIALS \
                       USE FEATURES PORTAGE_USERNAME FORCE_STAGES \
+                      SIGNER \
                       all_proxy ftp_proxy http_proxy https_proxy no_proxy"

sdk_lib/sdk_container_common.sh

@@ -201,6 +201,7 @@ function setup_sdk_env() {
         GIT_COMMITTER_EMAIL GIT_COMMITTER_NAME \
         GIT_PROXY_COMMAND GIT_SSH RSYNC_PROXY \
         GPG_AGENT_INFO FORCE_STAGES \
+        SIGNER \
         all_proxy ftp_proxy http_proxy https_proxy no_proxy; do
 
         if [ -n "${!var:-}" ] ; then