ci-automation/garbage_collect: clean up kola cloud resources

The garbage collect job should also clean up kola resources if a test
job failed to do so due to forced terminator or misbehavior. The
cleanup is done by "ore" which needs credentials like kola.

Run ore from the mantle container image. Unfortunately Docker does not
support Podman's --env-host option and the env vars had to be passed
explicitly. While --env-file=<(env) would work it contains a lot of
variables that cause the container to behave a bit weird.

ci-automation/garbage_collect.sh

@@ -132,4 +132,15 @@ function garbage_collect() {
             echo "## (DRY_RUN=y so not doing anything) ##"
         fi
     done
+
+    local mantle_ref
+    mantle_ref=$(cat sdk_container/.repo/manifests/mantle-container)
+    docker run --pull always --rm --net host \
+      --env AZURE_AUTH_CREDENTIALS --env AZURE_PROFILE \
+      --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY \
+      --env DIGITALOCEAN_TOKEN_JSON \
+      --env EQUINIXMETAL_KEY --env EQUINIXMETAL_PROJECT \
+      --env GCP_JSON_KEY \
+      --env VMWARE_ESX_CREDS \
+      -w /work -v "$PWD":/work "${mantle_ref}" /work/ci-automation/garbage_collect_cloud.sh
 }

ci-automation/garbage_collect_cloud.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -euo pipefail
+timeout --signal=SIGQUIT 60m ore aws gc --access-id "${AWS_ACCESS_KEY_ID}" --secret-key "${AWS_SECRET_ACCESS_KEY}"
+timeout --signal=SIGQUIT 60m ore do gc --config-file=<(echo "${DIGITALOCEAN_TOKEN_JSON}" | base64 --decode)
+timeout --signal=SIGQUIT 60m ore gcloud gc --json-key <(echo "${GCP_JSON_KEY}" | base64 --decode)
+# Because the Azure file gets read multiple times it can't be passed like <(cmd) because bash backs this FD
+# by a pipe meaning the data is gone after reading. We can create an FD (the FD number is assigned to
+# variable through exec {NAME}) manually and use a file under /tmp to back it instead, allowing multiple
+# reads.
+echo "${AZURE_PROFILE}" | base64 --decode > /tmp/azure_profile
+exec {azure_profile}</tmp/azure_profile
+rm /tmp/azure_profile
+echo "${AZURE_AUTH_CREDENTIALS}" | base64 --decode > /tmp/azure_auth
+exec {azure_auth}</tmp/azure_auth
+rm /tmp/azure_auth
+timeout --signal=SIGQUIT 60m ore azure gc --duration 6h \
+  --azure-profile="/proc/$$/fd/${azure_profile}" --azure-auth="/proc/$$/fd/${azure_auth}"
+timeout --signal=SIGQUIT 60m ore equinixmetal gc --duration 6h \
+  --project="${EQUINIXMETAL_PROJECT}" --gs-json-key=<(echo "${GCP_JSON_KEY}" | base64 --decode) --api-key="${EQUINIXMETAL_KEY}"