From c4af762e26b74ea3b9b8d76e6228f89e51c752ba Mon Sep 17 00:00:00 2001
From: Kai Lueke <kailuke@microsoft.com>
Date: Thu, 28 Apr 2022 15:25:40 +0900
Subject: [PATCH] ci-automation/garbage_collect: clean up kola cloud resources

The garbage collect job should also clean up kola resources if a test
job failed to do so due to forced terminator or misbehavior. The
cleanup is done by "ore" which needs credentials like kola.

Run ore from the mantle container image. Unfortunately Docker does not
support Podman's --env-host option and the env vars had to be passed
explicitly. While --env-file=<(env) would work it contains a lot of
variables that cause the container to behave a bit weird.
---
 ci-automation/garbage_collect.sh       | 11 +++++++++++
 ci-automation/garbage_collect_cloud.sh | 19 +++++++++++++++++++
 2 files changed, 30 insertions(+)
 create mode 100755 ci-automation/garbage_collect_cloud.sh

diff --git a/ci-automation/garbage_collect.sh b/ci-automation/garbage_collect.sh
index 8fb302918c..72b2f9ded1 100644
--- a/ci-automation/garbage_collect.sh
+++ b/ci-automation/garbage_collect.sh
@@ -132,4 +132,15 @@ function garbage_collect() {
             echo "## (DRY_RUN=y so not doing anything) ##"
         fi
     done
+
+    local mantle_ref
+    mantle_ref=$(cat sdk_container/.repo/manifests/mantle-container)
+    docker run --pull always --rm --net host \
+      --env AZURE_AUTH_CREDENTIALS --env AZURE_PROFILE \
+      --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY \
+      --env DIGITALOCEAN_TOKEN_JSON \
+      --env EQUINIXMETAL_KEY --env EQUINIXMETAL_PROJECT \
+      --env GCP_JSON_KEY \
+      --env VMWARE_ESX_CREDS \
+      -w /work -v "$PWD":/work "${mantle_ref}" /work/ci-automation/garbage_collect_cloud.sh
 }
diff --git a/ci-automation/garbage_collect_cloud.sh b/ci-automation/garbage_collect_cloud.sh
new file mode 100755
index 0000000000..33d145a7d5
--- /dev/null
+++ b/ci-automation/garbage_collect_cloud.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+set -euo pipefail
+timeout --signal=SIGQUIT 60m ore aws gc --access-id "${AWS_ACCESS_KEY_ID}" --secret-key "${AWS_SECRET_ACCESS_KEY}"
+timeout --signal=SIGQUIT 60m ore do gc --config-file=<(echo "${DIGITALOCEAN_TOKEN_JSON}" | base64 --decode)
+timeout --signal=SIGQUIT 60m ore gcloud gc --json-key <(echo "${GCP_JSON_KEY}" | base64 --decode)
+# Because the Azure file gets read multiple times it can't be passed like <(cmd) because bash backs this FD
+# by a pipe meaning the data is gone after reading. We can create an FD (the FD number is assigned to
+# variable through exec {NAME}) manually and use a file under /tmp to back it instead, allowing multiple
+# reads.
+echo "${AZURE_PROFILE}" | base64 --decode > /tmp/azure_profile
+exec {azure_profile}</tmp/azure_profile
+rm /tmp/azure_profile
+echo "${AZURE_AUTH_CREDENTIALS}" | base64 --decode > /tmp/azure_auth
+exec {azure_auth}</tmp/azure_auth
+rm /tmp/azure_auth
+timeout --signal=SIGQUIT 60m ore azure gc --duration 6h \
+  --azure-profile="/proc/$$/fd/${azure_profile}" --azure-auth="/proc/$$/fd/${azure_auth}"
+timeout --signal=SIGQUIT 60m ore equinixmetal gc --duration 6h \
+  --project="${EQUINIXMETAL_PROJECT}" --gs-json-key=<(echo "${GCP_JSON_KEY}" | base64 --decode) --api-key="${EQUINIXMETAL_KEY}"