From 9403dacabeab08a2c0efb4233f0e8659bb2647ec Mon Sep 17 00:00:00 2001 From: Kai Lueke Date: Wed, 16 Mar 2022 15:09:01 +0100 Subject: [PATCH 1/3] run_sdk_container: Fall back to tar ball download for SDK image The nightly SDK image is not pushed to a registry but has to be downloaded from the build server as tar ball. Fall back to the tar ball import for a better user experience. To reuse the ci logic it had to support the "docker" env variable. The use of the pigz container is not always needed if the user has pigz available. --- ci-automation/ci-config.env | 6 ++++-- ci-automation/ci_automation_common.sh | 14 ++++++++------ run_sdk_container | 5 ++++- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/ci-automation/ci-config.env b/ci-automation/ci-config.env index 18bf88f4c8..91833b0a87 100644 --- a/ci-automation/ci-config.env +++ b/ci-automation/ci-config.env @@ -17,8 +17,10 @@ CONTAINER_REGISTRY="ghcr.io/flatcar-linux" GC_BUCKET="flatcar-linux" -# No PIGZ on Flatcar -PIGZ="docker run --rm -i ghcr.io/flatcar-linux/pigz --fast" +if ! command -v pigz > /dev/null; then + # No PIGZ on Flatcar + PIGZ="docker run --rm -i ghcr.io/flatcar-linux/pigz --fast" +fi CI_GIT_AUTHOR="flatcar-ci" CI_GIT_EMAIL="infra+ci@flatcar-linux.org" diff --git a/ci-automation/ci_automation_common.sh b/ci-automation/ci_automation_common.sh index e98a44ca45..2cb574ecaf 100644 --- a/ci-automation/ci_automation_common.sh +++ b/ci-automation/ci_automation_common.sh @@ -8,6 +8,7 @@ source ci-automation/ci-config.env : ${PIGZ:=pigz} +: ${docker:=docker} function init_submodules() { git submodule init @@ -134,7 +135,7 @@ function image_exists_locally() { local version="$2" local image="${name}:${version}" - local image_exists="$(docker images "${image}" \ + local image_exists="$($docker images "${image}" \ --no-trunc --format '{{.Repository}}:{{.Tag}}')" [ "${image}" = "${image_exists}" ] @@ -155,7 +156,7 @@ function docker_image_fullname() { local image="$1" local version="$2" - docker images --no-trunc --format '{{.Repository}}:{{.Tag}}' \ + $docker images --no-trunc --format '{{.Repository}}:{{.Tag}}' \ | grep -E "^(${CONTAINER_REGISTRY}/)*${image}:${version}$" } # -- @@ -167,7 +168,7 @@ function docker_image_to_buildcache() { # strip potential container registry prefix local tarball="$(basename "$image")-${version}.tar.gz" - docker save "${image}":"${version}" | $PIGZ -c > "${tarball}" + $docker save "${image}":"${version}" | $PIGZ -c > "${tarball}" copy_to_buildcache "containers/${version}" "${tarball}" } # -- @@ -177,7 +178,7 @@ function docker_commit_to_buildcache() { local image_name="$2" local image_version="$3" - docker commit "${container}" "${image_name}:${image_version}" + $docker commit "${container}" "${image_name}:${image_version}" docker_image_to_buildcache "${image_name}" "${image_version}" } # -- @@ -197,7 +198,7 @@ function docker_image_from_buildcache() { --retry-connrefused --retry-max-time 60 --connect-timeout 20 \ --remote-name "${url}" - cat "${tgz}" | $PIGZ -d -c | docker load + cat "${tgz}" | $PIGZ -d -c | $docker load rm "${tgz}" } @@ -211,10 +212,11 @@ function docker_image_from_registry_or_buildcache() { return fi - if docker pull "${CONTAINER_REGISTRY}/${image}:${version}" ; then + if $docker pull "${CONTAINER_REGISTRY}/${image}:${version}" ; then return fi + echo "Falling back to tar ball download..." >&2 docker_image_from_buildcache "${image}" "${version}" } # -- diff --git a/run_sdk_container b/run_sdk_container index 225a2cdf91..d698f5d1f3 100755 --- a/run_sdk_container +++ b/run_sdk_container @@ -4,7 +4,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -set -eu +set -euo pipefail cd $(dirname "$0") source sdk_lib/sdk_container_common.sh @@ -104,6 +104,9 @@ if [ -z "$stat" ] ; then gpg_volumes=$(gnupg_ssh_gcloud_mount_opts) + source ci-automation/ci_automation_common.sh + docker_image_from_registry_or_buildcache "flatcar-sdk-${arch}" "${docker_sdk_vernum}" + $docker create $tty -i \ -v /dev:/dev \ -v "$(pwd)/sdk_container:/mnt/host/source/" \ From 4948026d618d7878b5e56c22d22199b2c0173153 Mon Sep 17 00:00:00 2001 From: Kai Lueke Date: Wed, 16 Mar 2022 15:36:32 +0100 Subject: [PATCH 2/3] run_sdk_container: recognize --help flag When the user passes --help we should not start a container and throw a late error message afterwards but show the help directly. --- run_sdk_container | 1 + 1 file changed, 1 insertion(+) diff --git a/run_sdk_container b/run_sdk_container index d698f5d1f3..aa934ab68c 100755 --- a/run_sdk_container +++ b/run_sdk_container @@ -51,6 +51,7 @@ usage() { while [ 0 -lt $# ] ; do case "$1" in -h) usage; exit 0;; + --help) usage; exit 0;; -t) tty="-t"; shift;; -v) os_version="$2"; shift; shift;; -V) sdk_version="$2"; shift; shift;; From 86265626605c0742ff96cc0132e9e3ec5601b11b Mon Sep 17 00:00:00 2001 From: Kai Lueke Date: Wed, 16 Mar 2022 15:55:14 +0100 Subject: [PATCH 3/3] run_sdk_container: add remove flag At the moment one must remove the leftover containers manually. Add a flag to do so when stopping the container. --- run_sdk_container | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/run_sdk_container b/run_sdk_container index aa934ab68c..8ddf7a9858 100755 --- a/run_sdk_container +++ b/run_sdk_container @@ -16,11 +16,12 @@ os_version="$(get_git_version)" sdk_version="$(get_sdk_version_from_versionfile)" custom_image="" tty="" +remove="" cleanup="" usage() { echo " Usage:" - echo " $0 [-t] [-v ] [-V sdk version] [-a arch] [-n ] [-x