From 133cb6b52f0cdd13c7d34c0a1e904ff12859791c Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Mon, 11 Jul 2022 20:04:54 +0200 Subject: [PATCH 1/3] ci-automation: Factor out listing files into a separate function This will come in handy when listing files for creating digests files. --- ci-automation/ci_automation_common.sh | 67 +++++++++++++++++++++------ 1 file changed, 52 insertions(+), 15 deletions(-) diff --git a/ci-automation/ci_automation_common.sh b/ci-automation/ci_automation_common.sh index adc2b85d77..a4f1d1e60e 100644 --- a/ci-automation/ci_automation_common.sh +++ b/ci-automation/ci_automation_common.sh @@ -332,26 +332,13 @@ function sign_artifacts() { # rest of the parameters are directories/files to sign local to_sign=() local file - local files if [[ -z "${signer}" ]]; then return fi - for file; do - files=() - if [[ -d "${file}" ]]; then - readarray -d '' files < <(find "${file}" ! -type d -print0) - elif [[ -e "${file}" ]]; then - files+=( "${file}" ) - fi - for file in "${files[@]}"; do - if [[ "${file}" =~ \.(asc|gpg|sig)$ ]]; then - continue - fi - to_sign+=( "${file}" ) - done - done + list_files to_sign 'asc,gpg,sig' "${@}" + for file in "${to_sign[@]}"; do gpg --batch --local-user "${signer}" \ --output "${file}.sig" \ @@ -359,3 +346,53 @@ function sign_artifacts() { done } # -- + +# Puts a filtered list of files from the passed files and directories +# in the passed variable. The filtering is done by ignoring files that +# end with the passed extensions. The extensions list should not +# contain the leading dot. +# +# Typical use: +# local all_files=() +# local ignored_extensions='sh,py,pl' # ignore the shell, python and perl scripts +# list_files all_files "${ignored_extensions}" "${directories_and_files[@]}" +# +# Parameters: +# +# 1 - name of an array variable where the filtered files will be stored +# 2 - comma-separated list of extensions that will be used for filtering files +# @ - files and directories to scan for files +function list_files() { + local files_variable_name="${1}"; shift + local ignored_extensions="${1}"; shift + # rest of the parameters are files or directories to list + local -n files="${files_variable_name}" + local file + local tmp_files + local pattern='' + + if [[ -n "${ignored_extensions}" ]]; then + pattern='\.('"${ignored_extensions//,/|}"')$' + fi + + files=() + for file; do + tmp_files=() + if [[ -d "${file}" ]]; then + readarray -d '' tmp_files < <(find "${file}" ! -type d -print0) + elif [[ -e "${file}" ]]; then + tmp_files+=( "${file}" ) + fi + if [[ -z "${pattern}" ]]; then + files+=( "${tmp_files[@]}" ) + continue + fi + for file in "${tmp_files[@]}"; do + if [[ "${file}" =~ ${pattern} ]]; then + continue + fi + files+=( "${file}" ) + done + done +} +# -- From d475d367665e70786e5d63be48be2be10961519e Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Mon, 11 Jul 2022 20:06:22 +0200 Subject: [PATCH 2/3] ci-automation: Add a function for generating digests It works in a similar way to sign_artifacts - it takes a signer, a list of files and directories, and generates digests next to the respective files. --- ci-automation/ci_automation_common.sh | 56 +++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/ci-automation/ci_automation_common.sh b/ci-automation/ci_automation_common.sh index a4f1d1e60e..903925d6bd 100644 --- a/ci-automation/ci_automation_common.sh +++ b/ci-automation/ci_automation_common.sh @@ -347,6 +347,62 @@ function sign_artifacts() { } # -- +# Creates digests files and armored ASCII files out of them for the +# passed files and directories. In case of directory, all files inside +# it are processed. No new digests file is created if there is one +# already for the processed file. Same for armored ASCII file. Files +# ending with .asc or .sig or .gpg or .DIGESTS are not processed. The +# armored ASCII files won't be created if the signer is empty. +# +# Typical use: +# create_digests "${SIGNER}" artifact.tar.gz +# sign_artifacts "${SIGNER}" artifact.tar.gz* +# copy_to_buildcache "artifacts/directory" artifact.tar.gz* +# +# Parameters: +# +# 1 - signer whose key is expected to be already imported into the +# keyring +# @ - files and directories to create digests for +function create_digests() { + local signer="${1}"; shift + # rest of the parameters are files or directories to create + # digests for + local to_digest=() + local file + local df + local fbn + local hash_type + local output + local af + + list_files to_digest 'asc,gpg,sig,DIGESTS' "${@}" + + for file in "${to_digest[@]}"; do + df="${file}.DIGESTS" + if [[ ! -e "${df}" ]]; then + touch "${df}" + fbn=$(basename "${file}") + # TODO: modernize - drop md5 and sha1, add b2 + for hash_type in md5 sha1 sha512; do + echo "# ${hash_type} HASH" | tr "a-z" "A-Z" >>"${df}" + output=$("${hash_type}sum" "${file}") + echo "${output%% *} ${fbn}" >>"${df}" + done + fi + if [[ -z "${signer}" ]]; then + continue + fi + af="${df}.asc" + if [[ ! -e "${af}" ]]; then + gpg --batch --local-user "${signer}" \ + --output "${af}" \ + --clearsign "${df}" + fi + done +} +# -- + # Puts a filtered list of files from the passed files and directories # in the passed variable. The filtering is done by ignoring files that # end with the passed extensions. The extensions list should not From 4e6f44e7b8d2b162f201bdd82b2dca691dc42258 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Mon, 11 Jul 2022 20:09:58 +0200 Subject: [PATCH 3/3] ci-automation: Generate digests files for the built artifacts --- ci-automation/ci_automation_common.sh | 3 ++- ci-automation/image.sh | 2 ++ ci-automation/packages.sh | 6 +++++- ci-automation/sdk_bootstrap.sh | 11 +++++++---- ci-automation/sdk_container.sh | 1 + ci-automation/vms.sh | 2 ++ 6 files changed, 19 insertions(+), 6 deletions(-) diff --git a/ci-automation/ci_automation_common.sh b/ci-automation/ci_automation_common.sh index 903925d6bd..e1a96e563a 100644 --- a/ci-automation/ci_automation_common.sh +++ b/ci-automation/ci_automation_common.sh @@ -187,7 +187,8 @@ function docker_image_to_buildcache() { local tarball="$(basename "$image")-${version}.tar.gz" $docker save "${image}":"${version}" | $PIGZ -c > "${tarball}" - sign_artifacts "${SIGNER:-}" "${tarball}" + create_digests "${SIGNER:-}" "${tarball}" + sign_artifacts "${SIGNER:-}" "${tarball}"* copy_to_buildcache "containers/${version}" "${tarball}"* } # -- diff --git a/ci-automation/image.sh b/ci-automation/image.sh index e1d796b093..83fc2a30ba 100644 --- a/ci-automation/image.sh +++ b/ci-automation/image.sh @@ -39,6 +39,7 @@ # 2. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. # 3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache. +# 4. DIGESTS of the artifacts from point 1, pushed to buildcache. If signer key was passed, armored ASCII files of the generated DIGESTS files too, pushed to buildcache. function image_build() { # Run a subshell, so the traps, environment changes and global @@ -105,6 +106,7 @@ function _image_build_impl() { # Delete uncompressed generic image before signing and upload rm "images/latest/flatcar_production_image.bin" "images/latest/flatcar_production_update.bin" + create_digests "${SIGNER}" "images/latest/"* sign_artifacts "${SIGNER}" "images/latest/"* copy_to_buildcache "images/${arch}/${vernum}/" "images/latest/"* diff --git a/ci-automation/packages.sh b/ci-automation/packages.sh index a6813e280a..77ab19aa28 100644 --- a/ci-automation/packages.sh +++ b/ci-automation/packages.sh @@ -64,6 +64,7 @@ # 3. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. # 4. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache. +# 5. DIGESTS of the artifacts from point 1, pushed to buildcache. If signer key was passed, armored ASCII files of the generated DIGESTS files too, pushed to buildcache. function packages_build() { # Run a subshell, so the traps, environment changes and global @@ -175,9 +176,12 @@ function _packages_build_impl() { docker_commit_to_buildcache "${packages_container}" "${packages_image}" "${docker_vernum}" # Publish torcx manifest and docker tarball to "images" cache so tests can pull it later. - sign_artifacts "${SIGNER}" \ + create_digests "${SIGNER}" \ "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json" \ "${torcx_tmp}/torcx/pkgs/${arch}-usr/docker/"*/*.torcx.tgz + sign_artifacts "${SIGNER}" \ + "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json"* \ + "${torcx_tmp}/torcx/pkgs/${arch}-usr/docker/"*/*.torcx.tgz* copy_to_buildcache "images/${arch}/${vernum}/torcx" \ "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json"* copy_to_buildcache "images/${arch}/${vernum}/torcx" \ diff --git a/ci-automation/sdk_bootstrap.sh b/ci-automation/sdk_bootstrap.sh index b35f4192ca..b2e53d0b07 100644 --- a/ci-automation/sdk_bootstrap.sh +++ b/ci-automation/sdk_bootstrap.sh @@ -56,6 +56,7 @@ # 3. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. # 4. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache. +# 5. DIGESTS of the artifacts from point 1, pushed to buildcache. If signer key was passed, armored ASCII files of the generated DIGESTS files too, pushed to buildcache. function sdk_bootstrap() { # Run a subshell, so the traps, environment changes and global @@ -141,9 +142,11 @@ function _sdk_bootstrap_impl() { local uid=$(id --user) local gid=$(id --group) sudo chown --recursive "${uid}:${gid}" __build__ - cd "__build__/images/catalyst/builds/flatcar-sdk" - sign_artifacts "${SIGNER}" "${dest_tarball}"* - copy_to_buildcache "sdk/${ARCH}/${FLATCAR_SDK_VERSION}" "${dest_tarball}"* - cd - + ( + cd "__build__/images/catalyst/builds/flatcar-sdk" + create_digests "${SIGNER}" "${dest_tarball}" + sign_artifacts "${SIGNER}" "${dest_tarball}"* + copy_to_buildcache "sdk/${ARCH}/${FLATCAR_SDK_VERSION}" "${dest_tarball}"* + ) } # -- diff --git a/ci-automation/sdk_container.sh b/ci-automation/sdk_container.sh index 0677743b8a..fa77faa356 100644 --- a/ci-automation/sdk_container.sh +++ b/ci-automation/sdk_container.sh @@ -37,6 +37,7 @@ # 2. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. # 3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache. +# 4. DIGESTS of the artifacts from point 1, pushed to buildcache. If signer key was passed, armored ASCII files of the generated DIGESTS files too, pushed to buildcache. function sdk_container_build() { # Run a subshell, so the traps, environment changes and global diff --git a/ci-automation/vms.sh b/ci-automation/vms.sh index b832ab0084..60765b9bd3 100644 --- a/ci-automation/vms.sh +++ b/ci-automation/vms.sh @@ -42,6 +42,7 @@ # 2. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. # 3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache. +# 4. DIGESTS of the artifacts from point 1, pushed to buildcache. If signer key was passed, armored ASCII files of the generated DIGESTS files too, pushed to buildcache. function vm_build() { # Run a subshell, so the traps, environment changes and global @@ -131,6 +132,7 @@ function _vm_build_impl() { -v "${vernum}" \ mv "${CONTAINER_IMAGE_ROOT}/${arch}-usr/" "./${images_out}/" + create_digests "${SIGNER}" "images/latest/"* sign_artifacts "${SIGNER}" "images/latest/"* copy_to_buildcache "images/${arch}/${vernum}/" "images/latest/"* }