mirror of
https://github.com/flatcar/scripts.git
synced 2026-05-04 11:51:14 +02:00
ci-automation/garbage_collect: Add github CI SDK builds
This change adds a garbage collector script for SDK containers built via Github actions CI automation https://github.com/flatcar/scripts/blob/main/.github/workflows/update-sdk.yaml These SDK container builds do not create a version tag in the scripts repo. The garbage collector therefore retrieves versions from a directory listing of https://bincache.flatcar-linux.net/containers/ . Signed-off-by: Thilo Fromm <thilofromm@microsoft.com>
This commit is contained in:
Thilo Fromm
parent
0e8cde89fe
commit
9b3a67e72b
@ -18,6 +18,8 @@
|
||||
# Flatcar CI automation garbage collector.
|
||||
# This script removes development (non-official) build artifacts:
|
||||
# - SDK tarballs, build step containers, and vendor images on buildcache
|
||||
# - SDK containers built via Github actions (e.g. from PRs).
|
||||
# See https://github.com/flatcar/scripts/blob/main/.github/workflows/update-sdk.yaml
|
||||
# - tags from the scripts repository
|
||||
#
|
||||
# Garbage collection is based on development (non-official) version tags
|
||||
@ -142,6 +144,12 @@ function _garbage_collect_impl() {
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
echo "########################################"
|
||||
echo
|
||||
echo Running cloud garbace collector
|
||||
echo
|
||||
|
||||
local mantle_ref
|
||||
mantle_ref=$(cat sdk_container/.repo/manifests/mantle-container)
|
||||
docker run --pull always --rm --net host \
|
||||
@ -153,5 +161,14 @@ function _garbage_collect_impl() {
|
||||
--env VMWARE_ESX_CREDS \
|
||||
--env OPENSTACK_CREDS \
|
||||
-w /work -v "$PWD":/work "${mantle_ref}" /work/ci-automation/garbage_collect_cloud.sh
|
||||
|
||||
echo
|
||||
echo "#############################################"
|
||||
echo
|
||||
echo Running Github CI SDK garbace collector
|
||||
echo
|
||||
|
||||
source ci-automation/garbage_collect_github_ci_sdk.sh
|
||||
garbage_collect_github_ci
|
||||
}
|
||||
# --
|
||||
|
||||
99
ci-automation/garbage_collect_github_ci_sdk.sh
Normal file
99
ci-automation/garbage_collect_github_ci_sdk.sh
Normal file
@ -0,0 +1,99 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Copyright (c) 2021 The Flatcar Maintainers.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
|
||||
#
|
||||
# garbage_collect_github_ci() should be called after sourcing.
|
||||
#
|
||||
# OPTIONAL INPUT
|
||||
# - Number of (recent) Github SDK builds to keep. Defaults to 20.
|
||||
# - DRY_RUN (Env variable). Set to "y" to just list what would be done but not
|
||||
# actually purge anything.
|
||||
|
||||
# Flatcar Github CI SDK rebuild automation garbage collector.
|
||||
# This script removes development (non-official) SDK image builds generated via Github CI.
|
||||
#
|
||||
# Garbage collection is based on development (non-official) SDK versions listed on
|
||||
# https://bincache.flatcar-linux.net/containers/
|
||||
# and following the pattern [VERSION_NUMBER]*-github-*. The newest 20 builds will be retained,
|
||||
# all older builds will be purged (20 is the default, see OPTIONAL INPUT above).
|
||||
|
||||
function garbage_collect_github_ci() {
|
||||
# Run a subshell, so the traps, environment changes and global
|
||||
# variables are not spilled into the caller.
|
||||
(
|
||||
set -euo pipefail
|
||||
|
||||
_garbage_collect_github_ci_impl "${@}"
|
||||
)
|
||||
}
|
||||
# --
|
||||
|
||||
function _garbage_collect_github_ci_impl() {
|
||||
local keep="${1:-50}"
|
||||
local dry_run="${DRY_RUN:-}"
|
||||
|
||||
# Example version string
|
||||
# <a href="./3598.0.0-nightly-20230508-2100-github-2023_05_09__08_06_54/">
|
||||
# <a href="./3598.0.0-nightly-20230508-2100-github-pr-12345-2023_05_09__08_06_54/">
|
||||
local versions_detected="$(curl -s https://bincache.flatcar-linux.net/containers/ \
|
||||
| grep -E '\<a href="\./[0-9]+\.[0-9]+.[0-9]+.+-github-.*/">' \
|
||||
| sed 's:.*\"./\([^/]\+\)/".*:\1:' )"
|
||||
|
||||
# Sort versions by date. Since version numbers can differ and this would impact sort, we
|
||||
# 1. insert a "/" between "...-github-[pr-XXX]-" and "[date]..."
|
||||
# 2. sort with delimiter "/" and sorting key 2 (i.e. the date part)
|
||||
# 3. remove the "/"
|
||||
local versions_sorted="$(echo "${versions_detected}" \
|
||||
| sed 's/\(-github\(-pr-[0-9]*\)*-\)/\1\//' \
|
||||
| sort -k 2 -t / \
|
||||
| sed 's:/::')"
|
||||
|
||||
echo "######## Full list of version(s) found ########"
|
||||
echo "${versions_sorted}" | awk '{printf "%5d %s\n", NR, $0}'
|
||||
|
||||
local keep="$((keep + 1))" # for tail -n+...
|
||||
local purge_versions="$(echo "${versions_sorted}" \
|
||||
| tail -n+"${keep}")"
|
||||
|
||||
source ci-automation/ci_automation_common.sh
|
||||
local sshcmd="$(gen_sshcmd)"
|
||||
|
||||
echo
|
||||
echo "######## The following version(s) will be purged ########"
|
||||
if [ "$dry_run" = "y" ] ; then
|
||||
echo
|
||||
echo "(NOTE this is just a dry run since DRY_RUN=y)"
|
||||
echo
|
||||
fi
|
||||
echo "${purge_versions}" | awk -v keep="${keep}" '{if ($0 == "") next; printf "%5d %s\n", NR + keep - 1, $0}'
|
||||
echo
|
||||
echo
|
||||
|
||||
local version=""
|
||||
for version in ${purge_versions}; do
|
||||
echo "--------------------------------------------"
|
||||
echo
|
||||
echo "#### Processing version '${version}' ####"
|
||||
echo
|
||||
|
||||
local rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${version}/"
|
||||
|
||||
echo "## The following files will be removed ##"
|
||||
$sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \
|
||||
"ls -la ${rmpat} || true"
|
||||
|
||||
if [ "$dry_run" != "y" ] ; then
|
||||
set -x
|
||||
$sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \
|
||||
"rm -rf ${rmpat}"
|
||||
set +x
|
||||
else
|
||||
echo "## (DRY_RUN=y so not doing anything) ##"
|
||||
fi
|
||||
done
|
||||
}
|
||||
# --
|
||||
Loading…
x
Reference in New Issue
Block a user