From aaf9deac28c9222b235884b560630de4a2d70fda Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Fri, 26 Jan 2024 15:47:54 +0100 Subject: [PATCH] ci-automation/garbage_collect_github_ci_sdk.sh: add min_age parameter This change adds a min_age parameter to the github CI SDK garbage collector. The parameter specifies a minimum age (in days) for artifacts to be garbage collected. NOTE that this can result in more artifacts being kept than specified via the "keep" parameter if artifacts are younger than min_age. The change also has garbage_collect.sh pass the min_age parameter to garbage_collect_github_ci_sdk.sh. Signed-off-by: Thilo Fromm --- ci-automation/garbage_collect.sh | 4 ++- .../garbage_collect_github_ci_sdk.sh | 28 ++++++++++++++++--- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/ci-automation/garbage_collect.sh b/ci-automation/garbage_collect.sh index 15bb5be855..3c7bb5a049 100644 --- a/ci-automation/garbage_collect.sh +++ b/ci-automation/garbage_collect.sh @@ -24,6 +24,8 @@ # - Number of (recent) versions to keep. Defaults to 50. # Explicitly setting this value will reset the minimum age (see below) to 0 days. # - Minimum age of version tag to be purged, in days. Defaults to 14. +# Only artifacts older than this AND exceeding the builds to keep threshold +# will be removed. # - PURGE_VERSIONS (Env variable). Space-separated list of versions to purge # instead of all but the 50 most recent ones. # Setting this will IGNORE minimum age and number of versions to keep. @@ -269,6 +271,6 @@ function _garbage_collect_impl() { echo source ci-automation/garbage_collect_github_ci_sdk.sh - garbage_collect_github_ci + garbage_collect_github_ci 1 "${min_age_days}" } # -- diff --git a/ci-automation/garbage_collect_github_ci_sdk.sh b/ci-automation/garbage_collect_github_ci_sdk.sh index 0d12cbe82a..ac11d66247 100644 --- a/ci-automation/garbage_collect_github_ci_sdk.sh +++ b/ci-automation/garbage_collect_github_ci_sdk.sh @@ -10,6 +10,9 @@ # # OPTIONAL INPUT # - Number of (recent) Github SDK builds to keep. Defaults to 20. +# - Minimum age of version tag to be purged, in days. Defaults to 14. +# Only artifacts older than this AND exceeding the builds to keep threshold +# will be removed. # - DRY_RUN (Env variable). Set to "y" to just list what would be done but not # actually purge anything. @@ -34,8 +37,10 @@ function garbage_collect_github_ci() { function _garbage_collect_github_ci_impl() { local keep="${1:-20}" + local min_age_days="${2:-14}" local dry_run="${DRY_RUN:-}" + local min_age_date="$(date -d "${min_age_days} days ago" +'%Y_%m_%d')" # Example version string # # @@ -49,15 +54,30 @@ function _garbage_collect_github_ci_impl() { # 3. remove the "/" local versions_sorted="$(echo "${versions_detected}" \ | sed 's/\(-github\(-pr-[0-9]*\)*-\)/\1\//' \ - | sort -k 2 -t / \ + | sort -k 2 -t / -r \ | sed 's:/::')" + echo + echo "Number of versions to keep: '${keep}'" + echo "Keep newer than: '${min_age_date}'" + echo + echo "######## Full list of version(s) found ########" echo "${versions_sorted}" | awk '{printf "%5d %s\n", NR, $0}' - local tail_keep="$((keep + 1))" # for tail -n+... local purge_versions - mapfile -t purge_versions < <(tail -n+"${tail_keep}" <<<"${versions_sorted}") + mapfile -t purge_versions < <(echo "${versions_sorted}" \ + | awk -v keep="${keep}" -v min_age="${min_age_date}" '{ + if (keep > 0) { + keep = keep - 1 + next + } + ts = gensub(".*-github-([0-9_]+)__.*","\\1","g",$1) + if (ts > min_age) + next + + print $1 + }') source ci-automation/ci_automation_common.sh local sshcmd="$(gen_sshcmd)" @@ -69,7 +89,7 @@ function _garbage_collect_github_ci_impl() { echo "(NOTE this is just a dry run since DRY_RUN=y)" echo fi - printf '%s\n' "${purge_versions[@]}" | awk -v keep="${keep}" '{if ($0 == "") next; printf "%5d %s\n", NR + keep, $0}' + printf '%s\n' "${purge_versions[@]}" | awk '{if ($0 == "") next; printf "%5d %s\n", NR, $0}' echo echo