From c6c3cc152a43a719c9c1f8b0a4abd77fe5eee90d Mon Sep 17 00:00:00 2001 From: Kai Lueke Date: Wed, 16 Mar 2022 15:09:01 +0100 Subject: [PATCH 1/3] run_sdk_container: Fall back to tar ball download for SDK image The nightly SDK image is not pushed to a registry but has to be downloaded from the build server as tar ball. Fall back to the tar ball import for a better user experience. To reuse the ci logic it had to support the "docker" env variable. The use of the pigz container is not always needed if the user has pigz available. --- ci-automation/ci-config.env | 6 ++++-- ci-automation/ci_automation_common.sh | 14 ++++++++------ run_sdk_container | 5 ++++- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/ci-automation/ci-config.env b/ci-automation/ci-config.env index 18bf88f4c8..91833b0a87 100644 --- a/ci-automation/ci-config.env +++ b/ci-automation/ci-config.env @@ -17,8 +17,10 @@ CONTAINER_REGISTRY="ghcr.io/flatcar-linux" GC_BUCKET="flatcar-linux" -# No PIGZ on Flatcar -PIGZ="docker run --rm -i ghcr.io/flatcar-linux/pigz --fast" +if ! command -v pigz > /dev/null; then + # No PIGZ on Flatcar + PIGZ="docker run --rm -i ghcr.io/flatcar-linux/pigz --fast" +fi CI_GIT_AUTHOR="flatcar-ci" CI_GIT_EMAIL="infra+ci@flatcar-linux.org" diff --git a/ci-automation/ci_automation_common.sh b/ci-automation/ci_automation_common.sh index 181821ad92..dccd941000 100644 --- a/ci-automation/ci_automation_common.sh +++ b/ci-automation/ci_automation_common.sh @@ -8,6 +8,7 @@ source ci-automation/ci-config.env : ${PIGZ:=pigz} +: ${docker:=docker} function init_submodules() { git submodule init @@ -136,7 +137,7 @@ function image_exists_locally() { local version="$2" local image="${name}:${version}" - local image_exists="$(docker images "${image}" \ + local image_exists="$($docker images "${image}" \ --no-trunc --format '{{.Repository}}:{{.Tag}}')" [ "${image}" = "${image_exists}" ] @@ -157,7 +158,7 @@ function docker_image_fullname() { local image="$1" local version="$2" - docker images --no-trunc --format '{{.Repository}}:{{.Tag}}' \ + $docker images --no-trunc --format '{{.Repository}}:{{.Tag}}' \ | grep -E "^(${CONTAINER_REGISTRY}/)*${image}:${version}$" } # -- @@ -169,7 +170,7 @@ function docker_image_to_buildcache() { # strip potential container registry prefix local tarball="$(basename "$image")-${version}.tar.gz" - docker save "${image}":"${version}" | $PIGZ -c > "${tarball}" + $docker save "${image}":"${version}" | $PIGZ -c > "${tarball}" copy_to_buildcache "containers/${version}" "${tarball}" } # -- @@ -179,7 +180,7 @@ function docker_commit_to_buildcache() { local image_name="$2" local image_version="$3" - docker commit "${container}" "${image_name}:${image_version}" + $docker commit "${container}" "${image_name}:${image_version}" docker_image_to_buildcache "${image_name}" "${image_version}" } # -- @@ -199,7 +200,7 @@ function docker_image_from_buildcache() { --retry-connrefused --retry-max-time 60 --connect-timeout 20 \ --remote-name "${url}" - cat "${tgz}" | $PIGZ -d -c | docker load + cat "${tgz}" | $PIGZ -d -c | $docker load rm "${tgz}" } @@ -213,10 +214,11 @@ function docker_image_from_registry_or_buildcache() { return fi - if docker pull "${CONTAINER_REGISTRY}/${image}:${version}" ; then + if $docker pull "${CONTAINER_REGISTRY}/${image}:${version}" ; then return fi + echo "Falling back to tar ball download..." >&2 docker_image_from_buildcache "${image}" "${version}" } # -- diff --git a/run_sdk_container b/run_sdk_container index 225a2cdf91..d698f5d1f3 100755 --- a/run_sdk_container +++ b/run_sdk_container @@ -4,7 +4,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -set -eu +set -euo pipefail cd $(dirname "$0") source sdk_lib/sdk_container_common.sh @@ -104,6 +104,9 @@ if [ -z "$stat" ] ; then gpg_volumes=$(gnupg_ssh_gcloud_mount_opts) + source ci-automation/ci_automation_common.sh + docker_image_from_registry_or_buildcache "flatcar-sdk-${arch}" "${docker_sdk_vernum}" + $docker create $tty -i \ -v /dev:/dev \ -v "$(pwd)/sdk_container:/mnt/host/source/" \ From 9f42f3bbe1a1ca4ac4168f277dcd8111187c4aac Mon Sep 17 00:00:00 2001 From: Kai Lueke Date: Wed, 16 Mar 2022 15:36:32 +0100 Subject: [PATCH 2/3] run_sdk_container: recognize --help flag When the user passes --help we should not start a container and throw a late error message afterwards but show the help directly. --- run_sdk_container | 1 + 1 file changed, 1 insertion(+) diff --git a/run_sdk_container b/run_sdk_container index d698f5d1f3..aa934ab68c 100755 --- a/run_sdk_container +++ b/run_sdk_container @@ -51,6 +51,7 @@ usage() { while [ 0 -lt $# ] ; do case "$1" in -h) usage; exit 0;; + --help) usage; exit 0;; -t) tty="-t"; shift;; -v) os_version="$2"; shift; shift;; -V) sdk_version="$2"; shift; shift;; From 04421f70f95c584265ea473675eb1eaf7ea08925 Mon Sep 17 00:00:00 2001 From: Kai Lueke Date: Wed, 16 Mar 2022 15:55:14 +0100 Subject: [PATCH 3/3] run_sdk_container: add remove flag At the moment one must remove the leftover containers manually. Add a flag to do so when stopping the container. --- run_sdk_container | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/run_sdk_container b/run_sdk_container index aa934ab68c..8ddf7a9858 100755 --- a/run_sdk_container +++ b/run_sdk_container @@ -16,11 +16,12 @@ os_version="$(get_git_version)" sdk_version="$(get_sdk_version_from_versionfile)" custom_image="" tty="" +remove="" cleanup="" usage() { echo " Usage:" - echo " $0 [-t] [-v ] [-V sdk version] [-a arch] [-n ] [-x