From 698d0de1291efdc55a859b2a74c289377068515b Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 3 Jun 2022 14:44:24 +0200 Subject: [PATCH 1/5] ci-automation: Trivial fixes Dropped some trailing whitespace, fixed a typo. Trivial. --- ci-automation/garbage_collect.sh | 10 +++++----- ci-automation/tapfile_helper_lib.sh | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci-automation/garbage_collect.sh b/ci-automation/garbage_collect.sh index 72b2f9ded1..4bd2b691c4 100644 --- a/ci-automation/garbage_collect.sh +++ b/ci-automation/garbage_collect.sh @@ -27,7 +27,7 @@ set -eu function garbage_collect() { - local keep="${1:-50}" + local keep="${1:-50}" local dry_run="${DRY_RUN:-}" local purge_versions="${PURGE_VERSIONS:-}" @@ -53,7 +53,7 @@ function garbage_collect() { local sshcmd="$(gen_sshcmd)" - echo + echo echo "######## The following version(s) will be purged ########" if [ "$dry_run" = "y" ] ; then echo @@ -61,13 +61,13 @@ function garbage_collect() { echo fi echo "${purge_versions}" | awk -v keep="${keep}" '{if ($0 == "") next; printf "%5d %s\n", NR + keep - 1, $0}' - echo - echo + echo + echo local version for version in ${purge_versions}; do echo "--------------------------------------------" - echo + echo echo "#### Processing version '${version}' ####" echo diff --git a/ci-automation/tapfile_helper_lib.sh b/ci-automation/tapfile_helper_lib.sh index b47e1895fd..bbb7c6c4c5 100644 --- a/ci-automation/tapfile_helper_lib.sh +++ b/ci-automation/tapfile_helper_lib.sh @@ -6,7 +6,7 @@ # Helper script for extracting information from TAP files and for merging multiple # TAP files into one report. -# The script uses a temporary SQLite DB for querzing and for result generation. +# The script uses a temporary SQLite DB for querying and for result generation. # # Brief usage overview (scroll down for parameters etc.): # tap_ingest_tapfile - add test results from tap file to the DB From 090d7ec176a14b0eabec8004e6083746864197db Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 3 Jun 2022 14:49:04 +0200 Subject: [PATCH 2/5] ci-automation: Run functions in subshells The functions are sourcing other files that define global variables, so they will spill into the callers shell unnecessarily. We will also add some functionality that uses traps in follow-up commits, so it's good to limit the scope of traps too. --- ci-automation/garbage_collect.sh | 14 ++++++++++++-- ci-automation/image.sh | 13 +++++++++++-- ci-automation/packages.sh | 14 +++++++++++--- ci-automation/push_pkgs.sh | 13 +++++++++++-- ci-automation/sdk_bootstrap.sh | 13 +++++++++++-- ci-automation/sdk_container.sh | 13 +++++++++++-- ci-automation/test.sh | 13 +++++++++++-- ci-automation/vms.sh | 13 +++++++++++-- 8 files changed, 89 insertions(+), 17 deletions(-) diff --git a/ci-automation/garbage_collect.sh b/ci-automation/garbage_collect.sh index 4bd2b691c4..cff5d441a5 100644 --- a/ci-automation/garbage_collect.sh +++ b/ci-automation/garbage_collect.sh @@ -24,9 +24,18 @@ # in the scripts repo. The newest 50 builds will be retained, # all older builds will be purged (50 is the default, see OPTIONAL INPUT above). -set -eu - function garbage_collect() { + # Run a subshell, so the traps, environment changes and global + # variables are not spilled into the caller. + ( + set -euo pipefail + + _garbage_collect_impl "${@}" + ) +} +# -- + +function _garbage_collect_impl() { local keep="${1:-50}" local dry_run="${DRY_RUN:-}" local purge_versions="${PURGE_VERSIONS:-}" @@ -144,3 +153,4 @@ function garbage_collect() { --env VMWARE_ESX_CREDS \ -w /work -v "$PWD":/work "${mantle_ref}" /work/ci-automation/garbage_collect_cloud.sh } +# -- diff --git a/ci-automation/image.sh b/ci-automation/image.sh index 87ac8ca65e..5c0234a9a6 100644 --- a/ci-automation/image.sh +++ b/ci-automation/image.sh @@ -32,9 +32,18 @@ # 2. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. -set -eu - function image_build() { + # Run a subshell, so the traps, environment changes and global + # variables are not spilled into the caller. + ( + set -euo pipefail + + _image_build_impl "${@}" + ) +} +# -- + +function _image_build_impl() { local arch="$1" source sdk_lib/sdk_container_common.sh diff --git a/ci-automation/packages.sh b/ci-automation/packages.sh index 40d1d9d19e..a3ae052d21 100644 --- a/ci-automation/packages.sh +++ b/ci-automation/packages.sh @@ -56,10 +56,18 @@ # 3. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. - -set -eu - function packages_build() { + # Run a subshell, so the traps, environment changes and global + # variables are not spilled into the caller. + ( + set -euo pipefail + + _packages_build_impl "${@}" + ) +} +# -- + +function _packages_build_impl() { local version="$1" local arch="$2" local coreos_git="${3:-}" diff --git a/ci-automation/push_pkgs.sh b/ci-automation/push_pkgs.sh index 2f65c886bd..78e25af599 100644 --- a/ci-automation/push_pkgs.sh +++ b/ci-automation/push_pkgs.sh @@ -31,8 +31,6 @@ # 2. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. -set -eu - # This function is run _inside_ the SDK container function image_build__copy_to_bincache() { local arch="$1" @@ -46,6 +44,17 @@ function image_build__copy_to_bincache() { # -- function push_packages() { + # Run a subshell, so the traps, environment changes and global + # variables are not spilled into the caller. + ( + set -euo pipefail + + _push_packages_impl "${@}" + ) +} +# -- + +function _push_packages_impl() { local arch="$1" source ci-automation/ci_automation_common.sh diff --git a/ci-automation/sdk_bootstrap.sh b/ci-automation/sdk_bootstrap.sh index 135f71cf6c..13b0c88020 100644 --- a/ci-automation/sdk_bootstrap.sh +++ b/ci-automation/sdk_bootstrap.sh @@ -48,9 +48,18 @@ # 3. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. -set -eu - function sdk_bootstrap() { + # Run a subshell, so the traps, environment changes and global + # variables are not spilled into the caller. + ( + set -euo pipefail + + _sdk_bootstrap_impl "${@}" + ) +} +# -- + +function _sdk_bootstrap_impl() { local seed_version="$1" local version="$2" local coreos_git="${3-}" diff --git a/ci-automation/sdk_container.sh b/ci-automation/sdk_container.sh index 01cffe632a..6234eb783c 100644 --- a/ci-automation/sdk_container.sh +++ b/ci-automation/sdk_container.sh @@ -29,9 +29,18 @@ # 2. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. -set -eu - function sdk_container_build() { + # Run a subshell, so the traps, environment changes and global + # variables are not spilled into the caller. + ( + set -euo pipefail + + _sdk_container_build_impl "${@}" + ) +} +# -- + +function _sdk_container_build_impl() { : ${ARCH:="amd64"} source ci-automation/ci_automation_common.sh diff --git a/ci-automation/test.sh b/ci-automation/test.sh index 8d7b48facd..18b2ffdacd 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -74,8 +74,6 @@ # script would need to make anyway. For more information, please refer # to the vendor_test.sh file. -set -euo pipefail - # Download torcx package and manifest, add build cache URL to manifest # so the docker.torcx-manifest-pkgs test can use it. function __prepare_torcx() { @@ -102,6 +100,17 @@ function __prepare_torcx() { # -- function test_run() { + # Run a subshell, so the traps, environment changes and global + # variables are not spilled into the caller. + ( + set -euo pipefail + + _test_run_impl "${@}" + ) +} +# -- + +function _test_run_impl() { local arch="$1" ; shift local image="$1"; shift diff --git a/ci-automation/vms.sh b/ci-automation/vms.sh index 7140fcf243..37da63af1a 100644 --- a/ci-automation/vms.sh +++ b/ci-automation/vms.sh @@ -31,9 +31,18 @@ # 2. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. -set -eu - function vm_build() { + # Run a subshell, so the traps, environment changes and global + # variables are not spilled into the caller. + ( + set -euo pipefail + + _vm_build_impl "${@}" + ) +} +# -- + +function _vm_build_impl() { local arch="$1" shift # $@ now contains image formats to build From 0e0eb67ca2dcf4402b81b7ccdd873bbefdc19e52 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 3 Jun 2022 14:52:17 +0200 Subject: [PATCH 3/5] ci-automation: Set up keys for signing Not used for anything yet. This sets up a temporary GPGHOME directory and a trap that will remove it after we are done. --- ci-automation/gpg_setup.sh | 31 +++++++++++++++++++++++++++++++ ci-automation/image.sh | 12 ++++++++++++ ci-automation/packages.sh | 10 ++++++++++ ci-automation/push_pkgs.sh | 12 ++++++++++++ ci-automation/sdk_bootstrap.sh | 10 ++++++++++ ci-automation/sdk_container.sh | 12 +++++++++++- ci-automation/vms.sh | 12 ++++++++++++ 7 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 ci-automation/gpg_setup.sh diff --git a/ci-automation/gpg_setup.sh b/ci-automation/gpg_setup.sh new file mode 100644 index 0000000000..d88eea2334 --- /dev/null +++ b/ci-automation/gpg_setup.sh @@ -0,0 +1,31 @@ +# Common gpg setup code to be sourced by other scripts in this +# directory. It will set up GnuPG home directory, possibly with a key +# from SIGNING_KEY environment variable. +# +# After this file is sourced, SIGNER is always defined and exported, +# even if empty. SIGNING_KEY is clobbered. + +: ${SIGNING_KEY:=''} +: ${SIGNER:=''} + +if [[ "${HOME}/.gnupg" -ef "${PWD}/.gnupg" ]]; then + echo 'Do not source ${BASH_SOURCE} directly in your home directory - it will clobber your GnuPG directory!' >&2 + exit 1 +fi + +export GNUPGHOME="${PWD}/.gnupg" +rm -rf "${GNUPGHOME}" +trap 'rm -rf "${GNUPGHOME}"' EXIT +mkdir --mode=0700 "${GNUPGHOME}" +# Sometimes this directory is not automatically created thus making +# further private key imports to fail. Let's create it here as a +# workaround. +mkdir -p --mode=0700 "${GNUPGHOME}/private-keys-v1.d/" +if [[ -n "${SIGNING_KEY}" ]] && [[ -n "${SIGNER}" ]]; then + gpg --import "${SIGNING_KEY}" +else + SIGNER='' +fi +export SIGNER +# Clobber signing key variable, we don't need it any more. +export SIGNING_KEY='' diff --git a/ci-automation/image.sh b/ci-automation/image.sh index 5c0234a9a6..9fff6b8e10 100644 --- a/ci-automation/image.sh +++ b/ci-automation/image.sh @@ -23,6 +23,16 @@ # # 1. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64"). # +# OPTIONAL INPUT: +# +# 1. SIGNER. Environment variable. Name of the owner of the artifact signing key. +# Defaults to nothing if not set - in such case, artifacts will not be signed. +# If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored. +# +# 2. SIGNING_KEY. Environment variable. The artifact signing key. +# Defaults to nothing if not set - in such case, artifacts will not be signed. +# If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored. +# # OUTPUT: # # 1. Exported container image with OS image, dev container, and related artifacts at @@ -31,6 +41,7 @@ # pushed to buildcache. # 2. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. +# 3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache. function image_build() { # Run a subshell, so the traps, environment changes and global @@ -50,6 +61,7 @@ function _image_build_impl() { local channel="" channel="$(get_git_channel)" source ci-automation/ci_automation_common.sh + source ci-automation/gpg_setup.sh init_submodules source sdk_container/.repo/manifests/version.txt diff --git a/ci-automation/packages.sh b/ci-automation/packages.sh index a3ae052d21..be1775b0c5 100644 --- a/ci-automation/packages.sh +++ b/ci-automation/packages.sh @@ -45,6 +45,14 @@ # This version will be checked out / pulled from remote in the portage-stable git submodule. # The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed. # +# 5. SIGNER. Environment variable. Name of the owner of the artifact signing key. +# Defaults to nothing if not set - in such case, artifacts will not be signed. +# If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored. +# +# 6. SIGNING_KEY. Environment variable. The artifact signing key. +# Defaults to nothing if not set - in such case, artifacts will not be signed. +# If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored. +# # OUTPUT: # # 1. Exported container image "flatcar-packages-[ARCH]-[VERSION].tar.gz" with binary packages @@ -55,6 +63,7 @@ # - sdk_container/.repo/manifests/version.txt denotes new FLATCAR OS version # 3. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. +# 4. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache. function packages_build() { # Run a subshell, so the traps, environment changes and global @@ -74,6 +83,7 @@ function _packages_build_impl() { local portage_git="${4:-}" source ci-automation/ci_automation_common.sh + source ci-automation/gpg_setup.sh init_submodules check_version_string "${version}" diff --git a/ci-automation/push_pkgs.sh b/ci-automation/push_pkgs.sh index 78e25af599..bb4c0e1b9a 100644 --- a/ci-automation/push_pkgs.sh +++ b/ci-automation/push_pkgs.sh @@ -25,11 +25,22 @@ # # 1. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64"). # +# OPTIONAL INPUT: +# +# 1. SIGNER. Environment variable. Name of the owner of the artifact signing key. +# Defaults to nothing if not set - in such case, artifacts will not be signed. +# If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored. +# +# 2. SIGNING_KEY. Environment variable. The artifact signing key. +# Defaults to nothing if not set - in such case, artifacts will not be signed. +# If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored. +# # OUTPUT: # # 1. Binary packages published to buildcache at "boards/[ARCH]-usr/[VERSION]/pkgs". # 2. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. +# 3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache. # This function is run _inside_ the SDK container function image_build__copy_to_bincache() { @@ -58,6 +69,7 @@ function _push_packages_impl() { local arch="$1" source ci-automation/ci_automation_common.sh + source ci-automation/gpg_setup.sh init_submodules source sdk_container/.repo/manifests/version.txt diff --git a/ci-automation/sdk_bootstrap.sh b/ci-automation/sdk_bootstrap.sh index 13b0c88020..c09388cb7e 100644 --- a/ci-automation/sdk_bootstrap.sh +++ b/ci-automation/sdk_bootstrap.sh @@ -39,6 +39,14 @@ # 5. ARCH. Environment variable. Target architecture for the SDK to run on. # Either "amd64" or "arm64"; defaults to "amd64" if not set. # +# 6. SIGNER. Environment variable. Name of the owner of the artifact signing key. +# Defaults to nothing if not set - in such case, artifacts will not be signed. +# If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored. +# +# 7. SIGNING_KEY. Environment variable. The artifact signing key. +# Defaults to nothing if not set - in such case, artifacts will not be signed. +# If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored. +# # OUTPUT: # # 1. SDK tarball (gentoo catalyst output) of the new SDK, pushed to buildcache. @@ -47,6 +55,7 @@ # - sdk_container/.repo/manifests/version.txt denotes new SDK version # 3. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. +# 4. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache. function sdk_bootstrap() { # Run a subshell, so the traps, environment changes and global @@ -67,6 +76,7 @@ function _sdk_bootstrap_impl() { : ${ARCH:="amd64"} source ci-automation/ci_automation_common.sh + source ci-automation/gpg_setup.sh init_submodules check_version_string "${version}" diff --git a/ci-automation/sdk_container.sh b/ci-automation/sdk_container.sh index 6234eb783c..0677743b8a 100644 --- a/ci-automation/sdk_container.sh +++ b/ci-automation/sdk_container.sh @@ -19,15 +19,24 @@ # SDK tarball is available on BUILDCACHE/sdk/[ARCH]/[VERSION]/flatcar-sdk-[ARCH]-[VERSION].tar.bz2 # # OPTIONAL INPUT: - +# # 2. ARCH. Environment variable. Target architecture for the SDK to run on. # Either "amd64" or "arm64"; defaults to "amd64" if not set. # +# 3. SIGNER. Environment variable. Name of the owner of the artifact signing key. +# Defaults to nothing if not set - in such case, artifacts will not be signed. +# If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored. +# +# 4. SIGNING_KEY. Environment variable. The artifact signing key. +# Defaults to nothing if not set - in such case, artifacts will not be signed. +# If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored. +# # OUTPUT: # # 1. SDK container image of the new SDK, published to buildcache. # 2. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. +# 3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache. function sdk_container_build() { # Run a subshell, so the traps, environment changes and global @@ -44,6 +53,7 @@ function _sdk_container_build_impl() { : ${ARCH:="amd64"} source ci-automation/ci_automation_common.sh + source ci-automation/gpg_setup.sh init_submodules diff --git a/ci-automation/vms.sh b/ci-automation/vms.sh index 37da63af1a..cfcc6e3a95 100644 --- a/ci-automation/vms.sh +++ b/ci-automation/vms.sh @@ -25,11 +25,22 @@ # 2. Image formats to be built. Can be multiple, separated by spaces. # Run ./image_to_vm.sh -h in the SDK to get a list of supported images. # +# OPTIONAL INPUT: +# +# 1. SIGNER. Environment variable. Name of the owner of the artifact signing key. +# Defaults to nothing if not set - in such case, artifacts will not be signed. +# If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored. +# +# 2. SIGNING_KEY. Environment variable. The artifact signing key. +# Defaults to nothing if not set - in such case, artifacts will not be signed. +# If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored. +# # OUTPUT: # # 1. Exported VM image(s), pushed to buildcache ( images/[ARCH]/[FLATCAR_VERSION]/ ) # 2. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. +# 3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache. function vm_build() { # Run a subshell, so the traps, environment changes and global @@ -48,6 +59,7 @@ function _vm_build_impl() { # $@ now contains image formats to build source ci-automation/ci_automation_common.sh + source ci-automation/gpg_setup.sh init_submodules source sdk_container/.repo/manifests/version.txt From 89e82185d0f9ff3f7b990fdb701170430bc70921 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 3 Jun 2022 14:53:05 +0200 Subject: [PATCH 4/5] sdk: Forward SIGNER environment variable Some of the signing may happen inside the SDK container, so make sure to forward the SIGNER environment variable, as it will be used by the signing function, when it's introduced. --- sdk_lib/90_env_keep | 1 + sdk_lib/sdk_container_common.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/sdk_lib/90_env_keep b/sdk_lib/90_env_keep index e678b7b1c7..995b9a1a3d 100644 --- a/sdk_lib/90_env_keep +++ b/sdk_lib/90_env_keep @@ -5,4 +5,5 @@ Defaults env_keep += "FLATCAR_BUILD_ID COREOS_OFFICIAL \ GNUPGHOME GPG_AGENT_INFO SSH_AUTH_SOCK \ BOTO_PATH GOOGLE_APPLICATION_CREDENTIALS \ USE FEATURES PORTAGE_USERNAME FORCE_STAGES \ + SIGNER \ all_proxy ftp_proxy http_proxy https_proxy no_proxy" diff --git a/sdk_lib/sdk_container_common.sh b/sdk_lib/sdk_container_common.sh index 1e97ef813c..fc88e98d16 100644 --- a/sdk_lib/sdk_container_common.sh +++ b/sdk_lib/sdk_container_common.sh @@ -201,6 +201,7 @@ function setup_sdk_env() { GIT_COMMITTER_EMAIL GIT_COMMITTER_NAME \ GIT_PROXY_COMMAND GIT_SSH RSYNC_PROXY \ GPG_AGENT_INFO FORCE_STAGES \ + SIGNER \ all_proxy ftp_proxy http_proxy https_proxy no_proxy; do if [ -n "${!var:-}" ] ; then From 527bd2237b50c36831cf77223d027b7fca0fbf32 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 3 Jun 2022 14:54:54 +0200 Subject: [PATCH 5/5] ci-automation: Sign artifacts and upload the signatures It uses the SIGNER environment variable to decide whether the signatures should be created or not. It expect the key of the SIGNER to exist in GPGHOME, and that's what gpg_setup.sh is already doing. In some places we need to recursively change the owner of the directory that contains artifacts to be signed, otherwise we won't be able to create new files with signatures there. This is because some of the artifacts are either created inside the SDK container (so the created files belong to root outside the container) or are created with `sudo`. --- ci-automation/ci_automation_common.sh | 50 ++++++++++++++++++++++++++- ci-automation/packages.sh | 9 +++-- ci-automation/push_pkgs.sh | 8 +++++ ci-automation/sdk_bootstrap.sh | 8 +++++ ci-automation/vms.sh | 1 + 5 files changed, 72 insertions(+), 4 deletions(-) diff --git a/ci-automation/ci_automation_common.sh b/ci-automation/ci_automation_common.sh index ce01d55765..25e47c12df 100644 --- a/ci-automation/ci_automation_common.sh +++ b/ci-automation/ci_automation_common.sh @@ -175,7 +175,8 @@ function docker_image_to_buildcache() { local tarball="$(basename "$image")-${version}.tar.gz" $docker save "${image}":"${version}" | $PIGZ -c > "${tarball}" - copy_to_buildcache "containers/${version}" "${tarball}" + sign_artifacts "${SIGNER:-}" "${tarball}" + copy_to_buildcache "containers/${version}" "${tarball}"* } # -- @@ -299,3 +300,50 @@ function secret_to_file() { config_ref="/proc/${$}/fd/${fd}" } # -- + +# Creates signatures for the passed files and directories. In case of +# directory, all files inside are signed. Files ending with .asc or +# .sig or .gpg are ignored, though. This function is a noop if signer +# is empty. +# +# Typical use: +# sign_artifacts "${SIGNER}" artifact.tar.gz +# copy_to_buildcache "artifacts/directory" artifact.tar.gz* +# +# Parameters: +# +# 1 - signer whose key is expected to be already imported into the +# keyring +# @ - files and directories to sign +function sign_artifacts() { + local signer="${1}"; shift + # rest of the parameters are directories/files to sign + local to_sign=() + local file + local files + + if [[ -z "${signer}" ]]; then + return + fi + + for file; do + files=() + if [[ -d "${file}" ]]; then + readarray -d '' files < <(find "${file}" ! -type d -print0) + elif [[ -e "${file}" ]]; then + files+=( "${file}" ) + fi + for file in "${files[@]}"; do + if [[ "${file}" =~ \.(asc|gpg|sig)$ ]]; then + continue + fi + to_sign+=( "${file}" ) + done + done + for file in "${to_sign[@]}"; do + gpg --batch --local-user "${signer}" \ + --output "${file}.sig" \ + --detach-sign "${file}" + done +} +# -- diff --git a/ci-automation/packages.sh b/ci-automation/packages.sh index be1775b0c5..a6813e280a 100644 --- a/ci-automation/packages.sh +++ b/ci-automation/packages.sh @@ -175,9 +175,12 @@ function _packages_build_impl() { docker_commit_to_buildcache "${packages_container}" "${packages_image}" "${docker_vernum}" # Publish torcx manifest and docker tarball to "images" cache so tests can pull it later. - copy_to_buildcache "images/${arch}/${vernum}/torcx" \ - "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json" - copy_to_buildcache "images/${arch}/${vernum}/torcx" \ + sign_artifacts "${SIGNER}" \ + "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json" \ "${torcx_tmp}/torcx/pkgs/${arch}-usr/docker/"*/*.torcx.tgz + copy_to_buildcache "images/${arch}/${vernum}/torcx" \ + "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json"* + copy_to_buildcache "images/${arch}/${vernum}/torcx" \ + "${torcx_tmp}/torcx/pkgs/${arch}-usr/docker/"*/*.torcx.tgz* } # -- diff --git a/ci-automation/push_pkgs.sh b/ci-automation/push_pkgs.sh index bb4c0e1b9a..4b0c153148 100644 --- a/ci-automation/push_pkgs.sh +++ b/ci-automation/push_pkgs.sh @@ -49,7 +49,15 @@ function image_build__copy_to_bincache() { source ci-automation/ci_automation_common.sh + # change the owner of the files and directories in __build__ back + # to ourselves, otherwise we could fail to sign the artifacts as + # we lacked write permissions in the directory of the signed + # artifact + local uid=$(id --user) + local gid=$(id --group) cd /build/$arch-usr/var/lib/portage/pkgs/ + sudo chown --recursive "${uid}:${gid}" . + sign_artifacts "${SIGNER}" * copy_to_buildcache "boards/$arch-usr/$version/pkgs" * } # -- diff --git a/ci-automation/sdk_bootstrap.sh b/ci-automation/sdk_bootstrap.sh index c09388cb7e..b35f4192ca 100644 --- a/ci-automation/sdk_bootstrap.sh +++ b/ci-automation/sdk_bootstrap.sh @@ -134,7 +134,15 @@ function _sdk_bootstrap_impl() { source sdk_container/.repo/manifests/version.txt local dest_tarball="flatcar-sdk-${ARCH}-${FLATCAR_SDK_VERSION}.tar.bz2" + # change the owner of the files and directories in __build__ back + # to ourselves, otherwise we could fail to sign the artifacts as + # we lacked write permissions in the directory of the signed + # artifact + local uid=$(id --user) + local gid=$(id --group) + sudo chown --recursive "${uid}:${gid}" __build__ cd "__build__/images/catalyst/builds/flatcar-sdk" + sign_artifacts "${SIGNER}" "${dest_tarball}"* copy_to_buildcache "sdk/${ARCH}/${FLATCAR_SDK_VERSION}" "${dest_tarball}"* cd - } diff --git a/ci-automation/vms.sh b/ci-automation/vms.sh index cfcc6e3a95..ca69164485 100644 --- a/ci-automation/vms.sh +++ b/ci-automation/vms.sh @@ -117,6 +117,7 @@ function _vm_build_impl() { cp --reflink=auto -R "${CONTAINER_IMAGE_ROOT}/${arch}-usr/" "./${images_out}/" cd "images/latest" + sign_artifacts "${SIGNER}" * copy_to_buildcache "images/${arch}/${vernum}/" * } # --