mirrors/flatcar-scripts
1
0
Fork 0
mirror of https://github.com/flatcar/scripts.git synced 2025-09-22 06:01:41 +02:00
Code Issues Packages Projects Releases Wiki Activity

ci-automation: Use zstd to compress container images

Browse Source 
We currently use gzip together with pigz (parallel gzip) for importing
container images, and this is a lengthy operation (takes multiple minutes). By
moving to zstd we gain on all fronts: zstd produces smaller files, and is
faster to decompress/compress then pigz while using less resources.

Signed-off-by: Jeremi Piotrowski <jpiotrowski@microsoft.com>
This commit is contained in:
Jeremi Piotrowski 2023-06-27 11:36:35 +02:00
parent 389c89ec64
commit c19f0edbff
1 changed files with 8 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
ci-automation/ci_automation_common.sh
View File

@ -155,10 +155,10 @@ function docker_image_to_buildcache() {
local version="$2" local version="$2"
# strip potential container registry prefix # strip potential container registry prefix
local tarball="$(basename "$image")-${version}.tar.gz" local tarball="$(basename "$image")-${version}.tar.zst"
local id_file="$(basename "$image")-${version}.id" local id_file="$(basename "$image")-${version}.id"
$docker save "${image}":"${version}" | $PIGZ -c > "${tarball}" $docker save "${image}":"${version}" | zstd -T0 -o "${tarball}"
# Cut the "sha256:" prefix that is present in Docker but not in Podman # Cut the "sha256:" prefix that is present in Docker but not in Podman
$docker image inspect "${image}":"${version}" | jq -r '.[].Id' | sed 's/^sha256://' > "${id_file}" $docker image inspect "${image}":"${version}" | jq -r '.[].Id' | sed 's/^sha256://' > "${id_file}"
create_digests "${SIGNER:-}" "${tarball}" "${id_file}" create_digests "${SIGNER:-}" "${tarball}" "${id_file}"
@ -180,7 +180,8 @@ function docker_commit_to_buildcache() {
function docker_image_from_buildcache() { function docker_image_from_buildcache() {
local name="$1" local name="$1"
local version="$2" local version="$2"
local tgz="${name}-${version}.tar.gz" local compr="${3:-zst}"
local tgz="${name}-${version}.tar.${compr}"
local id_file="${name}-${version}.id" local id_file="${name}-${version}.id"
local id_file_url="https://${BUILDCACHE_SERVER}/containers/${version}/${id_file}" local id_file_url="https://${BUILDCACHE_SERVER}/containers/${version}/${id_file}"
local id_file_url_release="https://mirror.release.flatcar-linux.net/containers/${version}/${id_file}" local id_file_url_release="https://mirror.release.flatcar-linux.net/containers/${version}/${id_file}"
@ -214,7 +215,8 @@ function docker_image_from_buildcache() {
--retry-connrefused --retry-max-time 60 --connect-timeout 20 \ --retry-connrefused --retry-max-time 60 --connect-timeout 20 \
--remote-name "${url_release}" --remote-name "${url_release}"
cat "${tgz}" | $PIGZ -d -c | $docker load # zstd can handle zlib as well :)
zstd -d -c ${tgz} | $docker load
rm "${tgz}" rm "${tgz}"
} }
@ -229,7 +231,8 @@ function docker_image_from_registry_or_buildcache() {
fi fi
echo "Falling back to tar ball download..." >&2 echo "Falling back to tar ball download..." >&2
docker_image_from_buildcache "${image}" "${version}" docker_image_from_buildcache "${image}" "${version}" zst || \
docker_image_from_buildcache "${image}" "${version}" gz
} }
# -- # --