diff --git a/ci-automation/garbage_collect.sh b/ci-automation/garbage_collect.sh index f58ae8e952..e8dbc92c11 100644 --- a/ci-automation/garbage_collect.sh +++ b/ci-automation/garbage_collect.sh @@ -18,6 +18,8 @@ # Flatcar CI automation garbage collector. # This script removes development (non-official) build artifacts: # - SDK tarballs, build step containers, and vendor images on buildcache +# - SDK containers built via Github actions (e.g. from PRs). +# See https://github.com/flatcar/scripts/blob/main/.github/workflows/update-sdk.yaml # - tags from the scripts repository # # Garbage collection is based on development (non-official) version tags @@ -142,6 +144,12 @@ function _garbage_collect_impl() { fi done + echo + echo "########################################" + echo + echo Running cloud garbace collector + echo + local mantle_ref mantle_ref=$(cat sdk_container/.repo/manifests/mantle-container) docker run --pull always --rm --net host \ @@ -153,5 +161,14 @@ function _garbage_collect_impl() { --env VMWARE_ESX_CREDS \ --env OPENSTACK_CREDS \ -w /work -v "$PWD":/work "${mantle_ref}" /work/ci-automation/garbage_collect_cloud.sh + + echo + echo "#############################################" + echo + echo Running Github CI SDK garbace collector + echo + + source ci-automation/garbage_collect_github_ci_sdk.sh + garbage_collect_github_ci } # -- diff --git a/ci-automation/garbage_collect_github_ci_sdk.sh b/ci-automation/garbage_collect_github_ci_sdk.sh new file mode 100644 index 0000000000..2388e58db8 --- /dev/null +++ b/ci-automation/garbage_collect_github_ci_sdk.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# +# Copyright (c) 2021 The Flatcar Maintainers. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# >>> This file is supposed to be SOURCED from the repository ROOT. <<< +# +# garbage_collect_github_ci() should be called after sourcing. +# +# OPTIONAL INPUT +# - Number of (recent) Github SDK builds to keep. Defaults to 20. +# - DRY_RUN (Env variable). Set to "y" to just list what would be done but not +# actually purge anything. + +# Flatcar Github CI SDK rebuild automation garbage collector. +# This script removes development (non-official) SDK image builds generated via Github CI. +# +# Garbage collection is based on development (non-official) SDK versions listed on +# https://bincache.flatcar-linux.net/containers/ +# and following the pattern [VERSION_NUMBER]*-github-*. The newest 20 builds will be retained, +# all older builds will be purged (20 is the default, see OPTIONAL INPUT above). + +function garbage_collect_github_ci() { + # Run a subshell, so the traps, environment changes and global + # variables are not spilled into the caller. + ( + set -euo pipefail + + _garbage_collect_github_ci_impl "${@}" + ) +} +# -- + +function _garbage_collect_github_ci_impl() { + local keep="${1:-50}" + local dry_run="${DRY_RUN:-}" + + # Example version string + # + # + local versions_detected="$(curl -s https://bincache.flatcar-linux.net/containers/ \ + | grep -E '\' \ + | sed 's:.*\"./\([^/]\+\)/".*:\1:' )" + + # Sort versions by date. Since version numbers can differ and this would impact sort, we + # 1. insert a "/" between "...-github-[pr-XXX]-" and "[date]..." + # 2. sort with delimiter "/" and sorting key 2 (i.e. the date part) + # 3. remove the "/" + local versions_sorted="$(echo "${versions_detected}" \ + | sed 's/\(-github\(-pr-[0-9]*\)*-\)/\1\//' \ + | sort -k 2 -t / \ + | sed 's:/::')" + + echo "######## Full list of version(s) found ########" + echo "${versions_sorted}" | awk '{printf "%5d %s\n", NR, $0}' + + local keep="$((keep + 1))" # for tail -n+... + local purge_versions="$(echo "${versions_sorted}" \ + | tail -n+"${keep}")" + + source ci-automation/ci_automation_common.sh + local sshcmd="$(gen_sshcmd)" + + echo + echo "######## The following version(s) will be purged ########" + if [ "$dry_run" = "y" ] ; then + echo + echo "(NOTE this is just a dry run since DRY_RUN=y)" + echo + fi + echo "${purge_versions}" | awk -v keep="${keep}" '{if ($0 == "") next; printf "%5d %s\n", NR + keep - 1, $0}' + echo + echo + + local version="" + for version in ${purge_versions}; do + echo "--------------------------------------------" + echo + echo "#### Processing version '${version}' ####" + echo + + local rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${version}/" + + echo "## The following files will be removed ##" + $sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \ + "ls -la ${rmpat} || true" + + if [ "$dry_run" != "y" ] ; then + set -x + $sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \ + "rm -rf ${rmpat}" + set +x + else + echo "## (DRY_RUN=y so not doing anything) ##" + fi + done +} +# --